Compare commits
36
Commits
+19
-5
@@ -6,9 +6,20 @@
|
|||||||
**/build
|
**/build
|
||||||
web/build
|
web/build
|
||||||
|
|
||||||
# Flutter mobile client — built separately on developer machines / Flutter CI.
|
# The Android client — built by its own job, never from this context. The APK
|
||||||
# Including it in the Go build context wastes ~70 files and invalidates the
|
# reaches the image through client/, downloaded as a CI artifact, so nothing
|
||||||
# `COPY . .` layer cache on every Flutter-only change.
|
# here reads android/ sources.
|
||||||
|
#
|
||||||
|
# This block named `flutter_client/` until 2026-09-10 and lost its PATTERN when
|
||||||
|
# that tree was deleted, leaving a comment describing an exclusion that was no
|
||||||
|
# longer happening. android/ never took its place, so 4.1 MB of Gradle project
|
||||||
|
# has been entering the context and busting the `COPY . .` layer on every
|
||||||
|
# Android-only change.
|
||||||
|
android/
|
||||||
|
|
||||||
|
# Local `make build` output — an 18 MB binary the image never uses, since the
|
||||||
|
# builder stage compiles its own.
|
||||||
|
bin/
|
||||||
|
|
||||||
# Docs and IDE noise
|
# Docs and IDE noise
|
||||||
docs/
|
docs/
|
||||||
@@ -26,5 +37,8 @@ docs/
|
|||||||
!.env.example
|
!.env.example
|
||||||
|
|
||||||
# CI workflow files don't need to ship in the image.
|
# CI workflow files don't need to ship in the image.
|
||||||
.forgejo/
|
#
|
||||||
.github/
|
# This said `.forgejo/` and `.github/` — neither of which this repo has. Gitea
|
||||||
|
# Actions reads `.gitea/`, so the one directory that actually exists was the
|
||||||
|
# one not excluded, and every workflow edit invalidated the context.
|
||||||
|
.gitea/
|
||||||
|
|||||||
@@ -80,15 +80,12 @@ jobs:
|
|||||||
|
|
||||||
- name: Upload debug APK
|
- name: Upload debug APK
|
||||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||||
# Mirrored action, never actions/upload-artifact. @v4+ throws
|
# Stock action: it works on this forge since the runner moved to
|
||||||
# GHESNotSupportedError client-side on the hostname (no server setting
|
# gitea/runner 3.x, which edits upload-artifact's client-side GHES refusal
|
||||||
# reaches that check), and @v3 is worse — it reports success while Gitea
|
# out of the action bundle (Scribe snippet #2271). Never @v3 — it reports
|
||||||
# serves artifacts back only through the v4 API, so the upload is stored
|
# success while Gitea serves artifacts back only through the v4 API, and
|
||||||
# and invisible to every retrieval path. @v3 is what left 72 unreachable
|
# it is what left 72 unreachable artifacts on this repo (Scribe 2270).
|
||||||
# artifacts on this repo. Pinned by SHA because the mirror auto-syncs;
|
uses: actions/upload-artifact@v7
|
||||||
# full URL because DEFAULT_ACTIONS_URL sends bare owner/repo to github.com.
|
|
||||||
# See Scribe issues 2255 / 2270.
|
|
||||||
uses: https://git.fabledsword.com/bvandeusen/upload-artifact@cb8afe72b42edc798abfb8fcb556cf660d894245
|
|
||||||
with:
|
with:
|
||||||
name: minstrel-android-debug-${{ github.sha }}
|
name: minstrel-android-debug-${{ github.sha }}
|
||||||
path: android/app/build/outputs/apk/debug/app-debug.apk
|
path: android/app/build/outputs/apk/debug/app-debug.apk
|
||||||
|
|||||||
+291
-93
@@ -2,15 +2,71 @@ name: release
|
|||||||
|
|
||||||
# Builds and pushes the minstrel container image to the Gitea registry.
|
# Builds and pushes the minstrel container image to the Gitea registry.
|
||||||
#
|
#
|
||||||
# push to main → :main and :latest (latest-release APK bundled)
|
# push to dev → :dev (freshly-built dev APK bundled)
|
||||||
# push tag vYYYY.MM.DD → :vYYYY.MM.DD and :latest (freshly-built APK bundled)
|
# push to main → :latest + :<sha> (latest-release APK bundled)
|
||||||
# workflow_dispatch → manual trigger (same rules based on the ref)
|
# push tag vYYYY.MM.DD.HHMM → :latest (fresh APK bundled)
|
||||||
|
# workflow_dispatch → manual trigger (same rules based on the ref)
|
||||||
#
|
#
|
||||||
# Release model: per-day CalVer tags (no trailing patch digit). The day's
|
# That is the whole tag map, and it is family rule 145 + 147 as written.
|
||||||
# tag is intentionally mutable — if a second release happens the same day,
|
#
|
||||||
# move the tag with `git push -f origin vYYYY.MM.DD` and the image tag of
|
# :<sha> on main is the ROLLBACK UNIT — every production commit addressable
|
||||||
# the same name gets overwritten. :latest is updated by every main push
|
# without a release ceremony. It is minted only on main, where rollback is
|
||||||
# AND every tag push, so it always reflects the newest blessed image.
|
# actually worth having: merges are gated (rule 2) so they number in the dozens
|
||||||
|
# per year, while on dev they would be one per push, forever, for a channel
|
||||||
|
# whose entire contract is that it moves.
|
||||||
|
#
|
||||||
|
# There are NO :<version> image tags. This repo published :vYYYY.MM.DD.HHMM
|
||||||
|
# until 2026-09-10 and it was the inverse of the rule on both counts — minting
|
||||||
|
# a version tag nobody pinned while the rollback unit the rule names did not
|
||||||
|
# exist here at all. Git and the build's own self-reported version answer
|
||||||
|
# "which build is this"; a third name for the same thing is upkeep for a model
|
||||||
|
# we do not run. Operator, 2026-09-10: "only things like the APK need that kind
|
||||||
|
# of versioning for their update process."
|
||||||
|
#
|
||||||
|
# There is no :main either. :latest tracks main's tip with no gate between them
|
||||||
|
# (rule 147), so a second name for the same image sends readers looking for a
|
||||||
|
# distinction that does not exist.
|
||||||
|
#
|
||||||
|
# The dev channel exists so testing a build does not require shipping one.
|
||||||
|
# Before it, the only way to get an APK onto a phone was to cut a release,
|
||||||
|
# which made `main` the staging area by default. `:dev` carries its own
|
||||||
|
# freshly-built APK, signed with the SAME key as release builds — a different
|
||||||
|
# key cannot install over the stable app, so anyone crossing channels would
|
||||||
|
# have to uninstall and lose their data.
|
||||||
|
#
|
||||||
|
# :dev is published ALONE, with no per-commit tag. A rolling channel is
|
||||||
|
# rolling by definition; a commit-addressable image for it would be a
|
||||||
|
# rollback target nobody ever pulls, kept forever. Recovery on dev is to fix
|
||||||
|
# forward.
|
||||||
|
#
|
||||||
|
# Note what this repo does NOT need: a cross-repo dispatch to refresh the
|
||||||
|
# channel when its bundled APK is rebuilt. That mechanism exists elsewhere in
|
||||||
|
# the family because the app and the server live in separate repos. Minstrel
|
||||||
|
# is a monorepo — one push builds the APK and the image in the same run from
|
||||||
|
# the same commit, so the channel cannot go stale against its own artifact.
|
||||||
|
# The requirement is satisfied structurally; copying the mechanism would add
|
||||||
|
# a moving part to fix a problem that does not exist here.
|
||||||
|
#
|
||||||
|
# Release model: the tag IS the artifact's version name with a `v` in front.
|
||||||
|
# `v2026.09.10.1432` and `2026.09.10.1432` are the same string, derived from
|
||||||
|
# the tagged commit's UTC timestamp — so there is no mismatch to reconcile
|
||||||
|
# between what the tag says and what the APK reports, and nothing to look up
|
||||||
|
# when minting one.
|
||||||
|
#
|
||||||
|
# TAGS ARE IMMUTABLE. Never move, retarget or delete a published tag. A
|
||||||
|
# same-day second release is not a collision — HHMM makes every tag unique
|
||||||
|
# by construction, so the answer is simply another tag.
|
||||||
|
#
|
||||||
|
# This block used to say the opposite: that the per-day tag was
|
||||||
|
# "intentionally mutable" and that a same-day re-cut should
|
||||||
|
# `git push -f origin vYYYY.MM.DD`. That instruction is what the family
|
||||||
|
# rulebook now forbids outright, and it has incidents behind it — moving a
|
||||||
|
# same-day tag forward once took a published release down with it. Anyone
|
||||||
|
# installing from a tag is holding something the tag no longer points at,
|
||||||
|
# which is a worse failure than an extra row in the tag list.
|
||||||
|
#
|
||||||
|
# :latest is updated by every main push AND every tag push, so it always
|
||||||
|
# reflects the newest blessed image.
|
||||||
#
|
#
|
||||||
# APK pipeline: on tag pushes the android-release job builds + signs the
|
# APK pipeline: on tag pushes the android-release job builds + signs the
|
||||||
# Android APK and uploads it as a workflow artifact. The image-release
|
# Android APK and uploads it as a workflow artifact. The image-release
|
||||||
@@ -24,33 +80,37 @@ name: release
|
|||||||
# :latest (not just tags), a main build with no APK would silently strip
|
# :latest (not just tags), a main build with no APK would silently strip
|
||||||
# the in-app update channel off :latest until the next release. So on
|
# the in-app update channel off :latest until the next release. So on
|
||||||
# non-tag builds image-release pulls the MOST RECENT release's signed APK
|
# non-tag builds image-release pulls the MOST RECENT release's signed APK
|
||||||
# and reconstructs its exact versionName (tag + commit-count, the same
|
# AND the version sidecar published beside it — the recorded values, not
|
||||||
# formula android-release bakes in) for the version sidecar — no rebuild,
|
# recomputed ones — so no rebuild is needed, just a rebundle. Tag builds
|
||||||
# just rebundle. Tag builds keep bundling their own freshly-built APK.
|
# keep bundling their own freshly-built APK.
|
||||||
#
|
#
|
||||||
# Android testing (lint + detekt + unit tests, debug APK upload on main)
|
# Android testing (lint + detekt + unit tests, debug APK upload on main)
|
||||||
# lives in android.yml and runs independently on every push.
|
# lives in android.yml and runs independently on every push.
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main, dev]
|
||||||
tags: ['v*']
|
tags: ['v*']
|
||||||
paths-ignore:
|
paths-ignore:
|
||||||
- 'docs/**'
|
- 'docs/**'
|
||||||
- '**/*.md'
|
- '**/*.md'
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
# Force-moving the per-day tag (or rapidly re-pushing to main) should
|
# A rapid re-push to main should supersede the in-flight build — the
|
||||||
# supersede the in-flight build — the operator explicitly wants the
|
# operator explicitly wants the later commit to win. Tags no longer enter
|
||||||
# later commit to win.
|
# into this: they are immutable and unique, so no tag build can ever be
|
||||||
|
# superseded by another run on the same ref.
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ${{ github.workflow }}-${{ github.ref }}
|
group: ${{ github.workflow }}-${{ github.ref }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
android-release:
|
android-release:
|
||||||
name: Build signed APK (tag releases only)
|
name: Build signed APK (releases and dev)
|
||||||
if: startsWith(github.ref, 'refs/tags/v')
|
# Also builds on `dev`, which is what makes a test channel possible at
|
||||||
|
# all. Without it the only way to get a build onto a phone was to cut a
|
||||||
|
# release, which quietly turns `main` into the staging area.
|
||||||
|
if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev'
|
||||||
runs-on: flutter-ci
|
runs-on: flutter-ci
|
||||||
container:
|
container:
|
||||||
image: git.fabledsword.com/bvandeusen/ci-android:36
|
image: git.fabledsword.com/bvandeusen/ci-android:36
|
||||||
@@ -75,14 +135,18 @@ jobs:
|
|||||||
outputs:
|
outputs:
|
||||||
version_name: ${{ steps.ver.outputs.name }}
|
version_name: ${{ steps.ver.outputs.name }}
|
||||||
version_code: ${{ steps.ver.outputs.code }}
|
version_code: ${{ steps.ver.outputs.code }}
|
||||||
|
channel: ${{ steps.ver.outputs.channel }}
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
# fetch-depth: 0 retrieves full history; default shallow clone
|
# Full history. The version name now reads only the tip commit's
|
||||||
# would return 1 for `git rev-list --count HEAD`, breaking the
|
# timestamp, so a shallow clone would technically serve — but this
|
||||||
# iteration suffix.
|
# job derives a value that ships to devices, and a shallow checkout
|
||||||
|
# changes what git-derived values resolve to WITHOUT failing. The
|
||||||
|
# whole failure class here is a green build carrying a wrong
|
||||||
|
# version, so the cheap guarantee is worth keeping.
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Compute release version
|
- name: Compute release version
|
||||||
@@ -91,12 +155,23 @@ jobs:
|
|||||||
working-directory: ${{ github.workspace }}
|
working-directory: ${{ github.workspace }}
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
TAG="${GITHUB_REF#refs/tags/v}"
|
# The derivation lives in ci/version.sh, not here, so it can be
|
||||||
COMMIT_COUNT=$(git rev-list --count HEAD)
|
# executed by a test on every push. Anything inline in this file is
|
||||||
VERSION_NAME="${TAG}.${COMMIT_COUNT}"
|
# unverifiable until a release is already running.
|
||||||
echo "name=${VERSION_NAME}" >> "$GITHUB_OUTPUT"
|
out="$(ci/version.sh HEAD)"
|
||||||
echo "code=${COMMIT_COUNT}" >> "$GITHUB_OUTPUT"
|
printf '%s\n' "${out}" >> "$GITHUB_OUTPUT"
|
||||||
echo "::notice::APK version: ${VERSION_NAME} (code=${COMMIT_COUNT})"
|
|
||||||
|
# The channel is a property of the LANE, not of the commit, which is
|
||||||
|
# why it is derived here rather than in version.sh. Same commit built
|
||||||
|
# on dev and on main reports the same NAME and differs only here —
|
||||||
|
# that is the whole point of separating the two values.
|
||||||
|
if [ "${GITHUB_REF}" = "refs/heads/dev" ]; then
|
||||||
|
channel=dev
|
||||||
|
else
|
||||||
|
channel=stable
|
||||||
|
fi
|
||||||
|
echo "channel=${channel}" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "::notice::APK $(printf '%s' "${out}" | tr '\n' ' ') channel=${channel}"
|
||||||
|
|
||||||
# Checked BEFORE the expensive work, not after it. "Attach APK to gitea
|
# Checked BEFORE the expensive work, not after it. "Attach APK to gitea
|
||||||
# Release" below resolves the release by tag and fails if it is absent —
|
# Release" below resolves the release by tag and fails if it is absent —
|
||||||
@@ -108,6 +183,7 @@ jobs:
|
|||||||
# the release together, so this passes). A bare `git push origin vX` is the
|
# the release together, so this passes). A bare `git push origin vX` is the
|
||||||
# case this catches.
|
# case this catches.
|
||||||
- name: Release must exist for this tag
|
- name: Release must exist for this tag
|
||||||
|
if: startsWith(github.ref, 'refs/tags/v')
|
||||||
shell: bash
|
shell: bash
|
||||||
working-directory: ${{ github.workspace }}
|
working-directory: ${{ github.workspace }}
|
||||||
env:
|
env:
|
||||||
@@ -156,13 +232,12 @@ jobs:
|
|||||||
-PMINSTREL_VERSION_CODE=${{ steps.ver.outputs.code }}
|
-PMINSTREL_VERSION_CODE=${{ steps.ver.outputs.code }}
|
||||||
|
|
||||||
- name: Upload APK as workflow artifact
|
- name: Upload APK as workflow artifact
|
||||||
# Mirrored action, never actions/upload-artifact — @v4+ refuses on the
|
# Stock action (snippet #2271) — never @v3, which uploads something Gitea
|
||||||
# hostname, @v3 uploads something Gitea will never serve back. This is
|
# will never serve back. This is the producing half of a pair:
|
||||||
# the producing half of a pair: image-release downloads `minstrel-apk`
|
# image-release downloads `minstrel-apk` below. Any upload v4+ pairs with
|
||||||
# below with the matching download-artifact mirror. Both must stay on
|
# any download v4+ on this forge (every combination tested 2026-09-10,
|
||||||
# the v4 protocol — mixing a v3 upload with a v4 download (or the
|
# Scribe spike #3843), so the two pins need not move together.
|
||||||
# reverse) yields an empty listing, not an error. See Scribe 2255 / 2270.
|
uses: actions/upload-artifact@v7
|
||||||
uses: https://git.fabledsword.com/bvandeusen/upload-artifact@cb8afe72b42edc798abfb8fcb556cf660d894245
|
|
||||||
with:
|
with:
|
||||||
name: minstrel-apk
|
name: minstrel-apk
|
||||||
path: android/app/build/outputs/apk/release/app-release.apk
|
path: android/app/build/outputs/apk/release/app-release.apk
|
||||||
@@ -171,9 +246,15 @@ jobs:
|
|||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
|
|
||||||
- name: Attach APK to gitea Release
|
- name: Attach APK to gitea Release
|
||||||
|
# Tag releases only. A dev build has no Release to hang assets on and
|
||||||
|
# does not need one — the :dev image bundles the APK, and the server
|
||||||
|
# serves it from /api/client/apk like any other.
|
||||||
|
if: startsWith(github.ref, 'refs/tags/v')
|
||||||
shell: bash
|
shell: bash
|
||||||
env:
|
env:
|
||||||
CI_TOKEN: ${{ secrets.CI_TOKEN }}
|
CI_TOKEN: ${{ secrets.CI_TOKEN }}
|
||||||
|
VERSION_NAME: ${{ steps.ver.outputs.name }}
|
||||||
|
VERSION_CODE: ${{ steps.ver.outputs.code }}
|
||||||
run: |
|
run: |
|
||||||
set -euxo pipefail
|
set -euxo pipefail
|
||||||
TAG="${GITHUB_REF#refs/tags/}"
|
TAG="${GITHUB_REF#refs/tags/}"
|
||||||
@@ -181,6 +262,20 @@ jobs:
|
|||||||
APK_PATH="app/build/outputs/apk/release/app-release.apk"
|
APK_PATH="app/build/outputs/apk/release/app-release.apk"
|
||||||
ls -lh "${APK_PATH}"
|
ls -lh "${APK_PATH}"
|
||||||
|
|
||||||
|
# Publish the version sidecar as a release asset next to the APK.
|
||||||
|
#
|
||||||
|
# This is what lets a later :latest build stop RECONSTRUCTING the
|
||||||
|
# bundled APK's version and simply read what was recorded. The
|
||||||
|
# ordering key in particular cannot be re-derived after the fact —
|
||||||
|
# it is build-time minutes, so once this job ends the value exists
|
||||||
|
# nowhere else. Reconstruction could only ever recover the name,
|
||||||
|
# and only by duplicating a formula that then has to be kept in
|
||||||
|
# step across two files.
|
||||||
|
SIDECAR_PATH="/tmp/minstrel.apk.version"
|
||||||
|
printf '{"name":"%s","code":%s,"channel":"stable"}\n' \
|
||||||
|
"${VERSION_NAME}" "${VERSION_CODE}" > "${SIDECAR_PATH}"
|
||||||
|
cat "${SIDECAR_PATH}"
|
||||||
|
|
||||||
RELEASE_JSON="$(curl -fsSL \
|
RELEASE_JSON="$(curl -fsSL \
|
||||||
-H "Authorization: token ${CI_TOKEN}" \
|
-H "Authorization: token ${CI_TOKEN}" \
|
||||||
"https://git.fabledsword.com/api/v1/repos/${REPO}/releases/tags/${TAG}")"
|
"https://git.fabledsword.com/api/v1/repos/${REPO}/releases/tags/${TAG}")"
|
||||||
@@ -202,6 +297,20 @@ jobs:
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Same treatment for the sidecar. Named `.apk.version` so the
|
||||||
|
# downloader's `\.apk$` match cannot pick it up by mistake.
|
||||||
|
SIDECAR_HTTP=$(curl -sS -L -o /tmp/upload-sidecar.out -w '%{http_code}' \
|
||||||
|
-H "Authorization: token ${CI_TOKEN}" \
|
||||||
|
-F "attachment=@${SIDECAR_PATH}" \
|
||||||
|
"https://git.fabledsword.com/api/v1/repos/${REPO}/releases/${RELEASE_ID}/assets?name=minstrel-${TAG}.apk.version")
|
||||||
|
echo "sidecar_upload_http=${SIDECAR_HTTP}"
|
||||||
|
cat /tmp/upload-sidecar.out || true
|
||||||
|
echo
|
||||||
|
if [ "${SIDECAR_HTTP}" -lt 200 ] || [ "${SIDECAR_HTTP}" -ge 300 ]; then
|
||||||
|
echo "::error::version sidecar upload returned HTTP ${SIDECAR_HTTP}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
image-release:
|
image-release:
|
||||||
name: Build + push container image
|
name: Build + push container image
|
||||||
# `needs:` waits for android-release. For tag pushes android-release
|
# `needs:` waits for android-release. For tag pushes android-release
|
||||||
@@ -222,11 +331,16 @@ jobs:
|
|||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
# Full history + tags so non-tag :latest builds can resolve the
|
# Full history, and rule 149 names this specifically: any job that
|
||||||
# latest release tag's commit count and reconstruct the bundled
|
# DERIVES the version name needs it, because a shallow clone changes
|
||||||
# APK's exact versionName (see "Bundle latest release APK" below).
|
# what git-derived values resolve to WITHOUT failing — a too-low
|
||||||
|
# value, silently, with every lane green.
|
||||||
|
#
|
||||||
|
# This job was depth-1 while it took the version from GITHUB_REF. It
|
||||||
|
# now runs ci/version.sh itself, because with :<version> image tags
|
||||||
|
# gone the server's self-reported version is the only thing that says
|
||||||
|
# which build an image is.
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
fetch-tags: true
|
|
||||||
|
|
||||||
- name: Detect buildable project
|
- name: Detect buildable project
|
||||||
id: guard
|
id: guard
|
||||||
@@ -244,21 +358,68 @@ jobs:
|
|||||||
if: steps.guard.outputs.ready == 'true'
|
if: steps.guard.outputs.ready == 'true'
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then
|
set -euo pipefail
|
||||||
VERSION="${GITHUB_REF#refs/tags/}"
|
|
||||||
echo "args=-t ${IMAGE}:${VERSION} -t ${IMAGE}:latest" >> "$GITHUB_OUTPUT"
|
# THE VERSION, and it is derived the same way on every ref — the
|
||||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
# branch decides the CHANNEL, never the version (family rule 149).
|
||||||
echo "::notice::Release build: ${VERSION} + latest"
|
#
|
||||||
else
|
# This used to be three different things: the literal string "main"
|
||||||
# Main is the protected, post-PR-merge branch. Treat it as the
|
# on main, "dev" on dev, and the tag name on a tag. None of them
|
||||||
# rolling stable channel — every main push moves :latest.
|
# ordered, and the first two were the same string forever — two dev
|
||||||
# Pinned consumers can target :vYYYY.MM.DD; everyone else
|
# images eight weeks apart were indistinguishable in the UI. That
|
||||||
# gets the newest main.
|
# mattered little while :vYYYY.MM.DD.HHMM existed to identify a
|
||||||
echo "args=-t ${IMAGE}:main -t ${IMAGE}:latest" >> "$GITHUB_OUTPUT"
|
# build; with version image tags gone, this IS how an operator tells
|
||||||
echo "version=main" >> "$GITHUB_OUTPUT"
|
# which build a container is running.
|
||||||
echo "::notice::Main-branch build: :main + :latest"
|
#
|
||||||
|
# `sed -n s///p` rather than `grep`: it exits 0 when nothing matches,
|
||||||
|
# so the empty check below is actually reachable. A grep here would
|
||||||
|
# kill the step at the assignment under the runner's pipefail — the
|
||||||
|
# exact bug that took down the first main build after the version
|
||||||
|
# rework.
|
||||||
|
VERSION="$(ci/version.sh HEAD | sed -n 's/^name=//p')"
|
||||||
|
if [ -z "${VERSION}" ]; then
|
||||||
|
echo "::error::could not derive a build version from ci/version.sh"
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then
|
||||||
|
# A release refreshes the CHANNEL and mints nothing else.
|
||||||
|
#
|
||||||
|
# The tag build exists to produce the signed APK and attach it to
|
||||||
|
# the release; the image it rebuilds is the SAME SOURCE as the main
|
||||||
|
# build minutes earlier, differing only in which APK is baked in.
|
||||||
|
# Rule 145 is explicit about that case: when the same source is
|
||||||
|
# rebuilt with different contents, publish the moving channel tag
|
||||||
|
# and never a commit-addressable one.
|
||||||
|
#
|
||||||
|
# :latest must move here rather than waiting for the next main
|
||||||
|
# push, or the channel would carry the PREVIOUS release's APK
|
||||||
|
# indefinitely — a channel that cannot refresh itself (rule 146).
|
||||||
|
CHANNEL=stable
|
||||||
|
echo "args=-t ${IMAGE}:latest" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "::notice::Release build ${VERSION}: refreshing :latest around the new APK"
|
||||||
|
elif [[ "${GITHUB_REF}" == "refs/heads/dev" ]]; then
|
||||||
|
# The rolling test channel, and :dev ALONE — deliberately no
|
||||||
|
# per-commit tag. A rolling channel is rolling by definition, so a
|
||||||
|
# commit-addressable image here would be a rollback target nobody
|
||||||
|
# has ever pulled, accumulating in the registry forever. Recovery
|
||||||
|
# on dev is to fix forward.
|
||||||
|
CHANNEL=dev
|
||||||
|
echo "args=-t ${IMAGE}:dev" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "::notice::Dev-branch build ${VERSION}: :dev"
|
||||||
|
else
|
||||||
|
# The production line: :latest tracks main's tip (rule 147) and
|
||||||
|
# :<sha> is the rollback unit (rule 145). Full 40-char SHA, matching
|
||||||
|
# the family's other repos, so a rollback target is addressable
|
||||||
|
# straight from the commit anyone is reading.
|
||||||
|
CHANNEL=stable
|
||||||
|
echo "args=-t ${IMAGE}:latest -t ${IMAGE}:${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "::notice::Main-branch build ${VERSION}: :latest + :${GITHUB_SHA}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "channel=${CHANNEL}" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
- name: Registry login
|
- name: Registry login
|
||||||
if: steps.guard.outputs.ready == 'true'
|
if: steps.guard.outputs.ready == 'true'
|
||||||
shell: bash
|
shell: bash
|
||||||
@@ -267,54 +428,57 @@ jobs:
|
|||||||
| docker login git.fabledsword.com -u "${{ github.actor }}" --password-stdin
|
| docker login git.fabledsword.com -u "${{ github.actor }}" --password-stdin
|
||||||
|
|
||||||
- name: Download signed APK artifact
|
- name: Download signed APK artifact
|
||||||
# Tag pushes only — android-release just produced this. Non-tag
|
# Tag and dev pushes — android-release just produced this. Only `main`
|
||||||
# builds take the "Bundle latest release APK" path below instead.
|
# takes the "Bundle latest release APK" path below, because it is the
|
||||||
if: steps.guard.outputs.ready == 'true' && startsWith(github.ref, 'refs/tags/v')
|
# one ref that moves a channel without building an APK of its own.
|
||||||
# Consuming half of the pair — never actions/download-artifact. Same fork,
|
if: >-
|
||||||
# same reason: upstream's client-side GHES check rejects this hostname
|
steps.guard.outputs.ready == 'true' &&
|
||||||
# before it connects. bvandeusen/download-artifact mirrors
|
(startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev')
|
||||||
# code.forgejo.org/forgejo/download-artifact.
|
# Consuming half of the pair: stock download-artifact, which works here for
|
||||||
#
|
# the same reason as the upload (gitea/runner 3.x edits the GHES refusal
|
||||||
# SHA below is that fork's `v6` tag. Match on @actions/artifact, NOT on
|
# out of the bundle; snippet #2271). v8 runs on node24, which every
|
||||||
# the action's own version number — the two actions release on unrelated
|
# CI-runner image carries — the runner uses the image's own node.
|
||||||
# cadences, and download v5 would pair a ^2.3.2 client with this file's
|
uses: actions/download-artifact@v8
|
||||||
# ^4.0.0 uploader. v6 is the tag whose bundled library major (^4.0.0) is
|
|
||||||
# the same one proven against this instance by the upload side.
|
|
||||||
# Deliberately NOT v7: it moves to node24 and upstream requires runner
|
|
||||||
# >= 2.327.1 for it, which act_runner does not claim to satisfy.
|
|
||||||
# Pinned, not tagged — the mirror auto-syncs every 8h.
|
|
||||||
uses: https://git.fabledsword.com/bvandeusen/download-artifact@8d4e9521a5f7e5f8b6351f341f719f9f45a92a3a
|
|
||||||
with:
|
with:
|
||||||
name: minstrel-apk
|
name: minstrel-apk
|
||||||
path: client/
|
path: client/
|
||||||
|
|
||||||
- name: Stage bundled APK + version sidecar
|
- name: Stage bundled APK + version sidecar
|
||||||
if: steps.guard.outputs.ready == 'true' && startsWith(github.ref, 'refs/tags/v')
|
if: >-
|
||||||
|
steps.guard.outputs.ready == 'true' &&
|
||||||
|
(startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/dev')
|
||||||
shell: bash
|
shell: bash
|
||||||
env:
|
env:
|
||||||
# Pulled from android-release.outputs.version_name so the
|
# All three pulled from android-release's outputs so the sidecar the
|
||||||
# sidecar string the server hands clients matches the
|
# server hands clients matches exactly what is baked into the APK
|
||||||
# versionName baked into the APK they're comparing against.
|
# they are comparing against.
|
||||||
APK_VERSION_NAME: ${{ needs.android-release.outputs.version_name }}
|
APK_VERSION_NAME: ${{ needs.android-release.outputs.version_name }}
|
||||||
|
APK_VERSION_CODE: ${{ needs.android-release.outputs.version_code }}
|
||||||
|
APK_CHANNEL: ${{ needs.android-release.outputs.channel }}
|
||||||
run: |
|
run: |
|
||||||
set -euxo pipefail
|
set -euxo pipefail
|
||||||
# The artifact lands as `app-release.apk` (the original Gradle
|
# The artifact lands as `app-release.apk` (the original Gradle
|
||||||
# output name). The Dockerfile COPYs client/* into /app/client/
|
# output name). The Dockerfile COPYs client/* into /app/client/
|
||||||
# and the server reads minstrel.apk + minstrel.apk.version.
|
# and the server reads minstrel.apk + minstrel.apk.version.
|
||||||
mv client/app-release.apk client/minstrel.apk
|
mv client/app-release.apk client/minstrel.apk
|
||||||
echo "${APK_VERSION_NAME}" > client/minstrel.apk.version
|
printf '{"name":"%s","code":%s,"channel":"%s"}\n' \
|
||||||
|
"${APK_VERSION_NAME}" "${APK_VERSION_CODE}" "${APK_CHANNEL}" \
|
||||||
|
> client/minstrel.apk.version
|
||||||
|
cat client/minstrel.apk.version
|
||||||
ls -lh client/
|
ls -lh client/
|
||||||
|
|
||||||
- name: Bundle latest release APK (non-tag :latest builds)
|
- name: Bundle latest release APK (non-tag :latest builds)
|
||||||
# Main pushes don't build an APK, but they DO move :latest — so
|
# Main pushes don't build an APK, but they DO move :latest — so
|
||||||
# without this the in-app update channel would vanish from :latest
|
# without this the in-app update channel would vanish from :latest
|
||||||
# until the next tag. Pull the most-recent release's signed APK and
|
# until the next tag. Pull the most-recent release's signed APK and
|
||||||
# reconstruct its exact versionName (${TAG#v}.$(git rev-list --count
|
# the sidecar published beside it, so what the server reports is what
|
||||||
# TAG) — identical to android-release's formula) so the version
|
# that build actually recorded rather than something re-derived here.
|
||||||
# sidecar the server hands clients matches the installed build.
|
|
||||||
# Degrades to an empty client/ (404 update channel) — never a wrong
|
# Degrades to an empty client/ (404 update channel) — never a wrong
|
||||||
# version — if no release / APK asset / tag-count can be resolved.
|
# version — if no release or APK asset can be resolved. That
|
||||||
if: steps.guard.outputs.ready == 'true' && !startsWith(github.ref, 'refs/tags/v')
|
# degradation only actually works because the greps below carry
|
||||||
|
# `|| true`; under the runner's default pipefail a non-matching grep
|
||||||
|
# kills the step instead of falling through to the empty-case branch.
|
||||||
|
if: steps.guard.outputs.ready == 'true' && github.ref == 'refs/heads/main'
|
||||||
shell: bash
|
shell: bash
|
||||||
env:
|
env:
|
||||||
CI_TOKEN: ${{ secrets.CI_TOKEN }}
|
CI_TOKEN: ${{ secrets.CI_TOKEN }}
|
||||||
@@ -326,19 +490,40 @@ jobs:
|
|||||||
if [ -z "${REL_JSON}" ]; then
|
if [ -z "${REL_JSON}" ]; then
|
||||||
echo "::notice::no published release — image ships without bundled APK"; exit 0
|
echo "::notice::no published release — image ships without bundled APK"; exit 0
|
||||||
fi
|
fi
|
||||||
TAG="$(printf '%s' "${REL_JSON}" | grep -oP '"tag_name":\s*"\K[^"]+' | head -1)"
|
# `|| true` on every one of these, and it is load-bearing rather
|
||||||
APK_URL="$(printf '%s' "${REL_JSON}" | grep -oP '"browser_download_url":\s*"\K[^"]+' | grep -E '\.apk$' | head -1)"
|
# than defensive habit. The runner already invokes this shell as
|
||||||
|
# `bash -e -o pipefail`, so a pipeline whose grep matches NOTHING
|
||||||
|
# exits non-zero even though `head` succeeded — and the step dies at
|
||||||
|
# the assignment, before ever reaching the `if` written to handle the
|
||||||
|
# empty case. Every "degrades gracefully" branch below is unreachable
|
||||||
|
# without this.
|
||||||
|
TAG="$(printf '%s' "${REL_JSON}" | grep -oP '"tag_name":\s*"\K[^"]+' | head -1)" || true
|
||||||
|
APK_URL="$(printf '%s' "${REL_JSON}" | grep -oP '"browser_download_url":\s*"\K[^"]+' | grep -E '\.apk$' | head -1)" || true
|
||||||
if [ -z "${TAG}" ] || [ -z "${APK_URL}" ]; then
|
if [ -z "${TAG}" ] || [ -z "${APK_URL}" ]; then
|
||||||
echo "::notice::latest release '${TAG:-?}' has no APK asset — image ships without bundled APK"; exit 0
|
echo "::notice::latest release '${TAG:-?}' has no APK asset — image ships without bundled APK"; exit 0
|
||||||
fi
|
fi
|
||||||
COUNT="$(git rev-list --count "${TAG}" 2>/dev/null || true)"
|
|
||||||
if [ -z "${COUNT}" ]; then
|
|
||||||
echo "::notice::could not resolve commit count for ${TAG} (tag not fetched?) — skipping APK bundle"; exit 0
|
|
||||||
fi
|
|
||||||
VERSION_NAME="${TAG#v}.${COUNT}"
|
|
||||||
curl -fsSL -H "Authorization: token ${CI_TOKEN}" -o client/minstrel.apk "${APK_URL}"
|
curl -fsSL -H "Authorization: token ${CI_TOKEN}" -o client/minstrel.apk "${APK_URL}"
|
||||||
echo "${VERSION_NAME}" > client/minstrel.apk.version
|
|
||||||
echo "::notice::bundled release APK ${TAG} as version ${VERSION_NAME}"
|
# Take the version the release RECORDED rather than recomputing it.
|
||||||
|
# This used to re-derive the name from the tagged commit, which meant
|
||||||
|
# the formula lived in two files that had to be kept in step, and it
|
||||||
|
# could only ever recover the name — the ordering key is build-time
|
||||||
|
# minutes and does not exist anywhere after that build ends.
|
||||||
|
SIDECAR_URL="$(printf '%s' "${REL_JSON}" | grep -oP '"browser_download_url":\s*"\K[^"]+' | grep -E '\.apk\.version$' | head -1)" || true
|
||||||
|
if [ -n "${SIDECAR_URL}" ]; then
|
||||||
|
curl -fsSL -H "Authorization: token ${CI_TOKEN}" -o client/minstrel.apk.version "${SIDECAR_URL}"
|
||||||
|
cat client/minstrel.apk.version
|
||||||
|
else
|
||||||
|
# Releases published before sidecars were attached. Their name is
|
||||||
|
# still recoverable from the tag, but their ordering key genuinely
|
||||||
|
# is not — so it is reported ABSENT rather than guessed. A wrong
|
||||||
|
# key is an install the platform refuses; an absent one just tells
|
||||||
|
# the client to fall back to comparing names, which is exactly
|
||||||
|
# what those builds already do.
|
||||||
|
echo "::notice::release ${TAG} predates the version sidecar — bundling with name only, no ordering key"
|
||||||
|
printf '{"name":"%s","code":null,"channel":"stable"}\n' "${TAG#v}" > client/minstrel.apk.version
|
||||||
|
fi
|
||||||
|
echo "::notice::bundled release APK from ${TAG}"
|
||||||
ls -lh client/
|
ls -lh client/
|
||||||
|
|
||||||
- name: Build and push
|
- name: Build and push
|
||||||
@@ -346,6 +531,7 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
docker buildx build \
|
docker buildx build \
|
||||||
--build-arg MINSTREL_VERSION="${{ steps.tags.outputs.version }}" \
|
--build-arg MINSTREL_VERSION="${{ steps.tags.outputs.version }}" \
|
||||||
|
--build-arg MINSTREL_CHANNEL="${{ steps.tags.outputs.channel }}" \
|
||||||
--push ${{ steps.tags.outputs.args }} .
|
--push ${{ steps.tags.outputs.args }} .
|
||||||
|
|
||||||
# Verifies a tag release actually ended up complete, and names the specific
|
# Verifies a tag release actually ended up complete, and names the specific
|
||||||
@@ -356,8 +542,8 @@ jobs:
|
|||||||
# `failure` with none executed and image-release showed `skipped`. The run was
|
# `failure` with none executed and image-release showed `skipped`. The run was
|
||||||
# red, but the *release page rendered fine*, and `main`'s own push build had
|
# red, but the *release page rendered fine*, and `main`'s own push build had
|
||||||
# already moved `:latest`, so the code was deployable and nothing looked
|
# already moved `:latest`, so the code was deployable and nothing looked
|
||||||
# obviously wrong. The release was simply missing its APK and its immutable
|
# obviously wrong. The release was simply missing its APK and its image,
|
||||||
# `:vYYYY.MM.DD` image, which is easy to skim past.
|
# which is easy to skim past.
|
||||||
#
|
#
|
||||||
# This job cannot prevent that (the cause was a runner failing to launch, not
|
# This job cannot prevent that (the cause was a runner failing to launch, not
|
||||||
# anything in this file). What it does is turn an incomplete release into an
|
# anything in this file). What it does is turn an incomplete release into an
|
||||||
@@ -408,18 +594,30 @@ jobs:
|
|||||||
# missing when v2026.08.07 had to be re-cut. `always()` on this job means
|
# missing when v2026.08.07 had to be re-cut. `always()` on this job means
|
||||||
# it runs even when image-release failed, so without this the guard would
|
# it runs even when image-release failed, so without this the guard would
|
||||||
# cheerfully verify an incomplete release.
|
# cheerfully verify an incomplete release.
|
||||||
- name: Immutable image tag must exist
|
#
|
||||||
|
# This asserted `:${TAG}` — the :vYYYY.MM.DD.HHMM image — until
|
||||||
|
# 2026-09-10. Version image tags are no longer published (rule 145), so
|
||||||
|
# that assertion would now fail every release for a tag nothing mints.
|
||||||
|
# The rollback target it was really protecting is the :<sha> image, which
|
||||||
|
# main's own build published for this same commit before the tag was cut.
|
||||||
|
#
|
||||||
|
# Checking it here earns its keep twice over: it still catches an image
|
||||||
|
# push that silently did not happen, and it additionally proves the
|
||||||
|
# ORDERING — a tag cut on a commit whose main build never completed has
|
||||||
|
# no rollback target, and that is worth failing on rather than
|
||||||
|
# discovering during an incident.
|
||||||
|
- name: Rollback image must exist for the tagged commit
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
TAG="${GITHUB_REF#refs/tags/}"
|
|
||||||
IMAGE="git.fabledsword.com/bvandeusen/minstrel"
|
IMAGE="git.fabledsword.com/bvandeusen/minstrel"
|
||||||
|
|
||||||
echo "${{ secrets.CI_TOKEN }}" \
|
echo "${{ secrets.CI_TOKEN }}" \
|
||||||
| docker login git.fabledsword.com -u "${{ github.actor }}" --password-stdin
|
| docker login git.fabledsword.com -u "${{ github.actor }}" --password-stdin
|
||||||
|
|
||||||
if ! docker manifest inspect "${IMAGE}:${TAG}" > /dev/null 2>&1; then
|
if ! docker manifest inspect "${IMAGE}:${GITHUB_SHA}" > /dev/null 2>&1; then
|
||||||
echo "::error::image ${IMAGE}:${TAG} was never pushed — the release tag has no immutable image, so there is nothing to pin or roll back to. Re-run this workflow run."
|
echo "::error::image ${IMAGE}:${GITHUB_SHA} does not exist — this commit has no rollback target."
|
||||||
|
echo "::error::That image is published by the MAIN build of this commit, not by the tag build. If main's build never ran or failed, fix that first; a release whose commit cannot be rolled back to is the thing this check exists to refuse."
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
echo "::notice::image verified: ${IMAGE}:${TAG}"
|
echo "::notice::rollback target verified: ${IMAGE}:${GITHUB_SHA}"
|
||||||
|
|||||||
@@ -32,6 +32,12 @@ on:
|
|||||||
- 'cmd/**'
|
- 'cmd/**'
|
||||||
- '.golangci.yml'
|
- '.golangci.yml'
|
||||||
- '.gitea/workflows/test-go.yml'
|
- '.gitea/workflows/test-go.yml'
|
||||||
|
# The release lane's own trigger is `main` + tags, so nothing it
|
||||||
|
# contains is exercised until a release is already running. These two
|
||||||
|
# entries are what let internal/server/release_version_test.go guard
|
||||||
|
# the version derivation on ordinary dev pushes instead.
|
||||||
|
- 'ci/**'
|
||||||
|
- '.gitea/workflows/release.yml'
|
||||||
|
|
||||||
# pull_request trigger intentionally omitted — see test-web.yml for
|
# pull_request trigger intentionally omitted — see test-web.yml for
|
||||||
# the rationale (single-author repo, push covers PR-merge equivalent).
|
# the rationale (single-author repo, push covers PR-merge equivalent).
|
||||||
|
|||||||
@@ -12,6 +12,11 @@
|
|||||||
# Test binary, built with `go test -c`
|
# Test binary, built with `go test -c`
|
||||||
*.test
|
*.test
|
||||||
|
|
||||||
|
# `make build` output. bin/minstrel was tracked until 2026-09-10 — an 18 MB
|
||||||
|
# binary committed by accident, last refreshed by a commit about web test
|
||||||
|
# mocks, and re-dirtied by every local build since.
|
||||||
|
bin/
|
||||||
|
|
||||||
# Bundled Android APK + version sidecar (#397). Populated by CI for
|
# Bundled Android APK + version sidecar (#397). Populated by CI for
|
||||||
# tag releases; never committed. README in client/ explains the flow.
|
# tag releases; never committed. README in client/ explains the flow.
|
||||||
client/minstrel.apk
|
client/minstrel.apk
|
||||||
|
|||||||
+20
-5
@@ -15,17 +15,32 @@ COPY . .
|
|||||||
# Overwrite the committed placeholder with the freshly-built SPA assets.
|
# Overwrite the committed placeholder with the freshly-built SPA assets.
|
||||||
COPY --from=web /web/build ./web/build
|
COPY --from=web /web/build ./web/build
|
||||||
ENV CGO_ENABLED=0
|
ENV CGO_ENABLED=0
|
||||||
# Version stamping: release.yml passes the git tag via MINSTREL_VERSION
|
# Version stamping. release.yml passes the DERIVED version name
|
||||||
# build-arg; local `docker build` falls back to "dev". Surfaced at
|
# (YYYY.MM.DD.HHMM) and the lane's channel; a local `docker build` falls back
|
||||||
# /healthz for operator-side image-version verification.
|
# to "dev"/"local". Both are surfaced at /healthz.
|
||||||
|
#
|
||||||
|
# These are two values on purpose (family rule 149): the same commit built on
|
||||||
|
# dev and on main reports the same NAME and differs only in CHANNEL. Folding
|
||||||
|
# the channel into the version string is what the rule forbids — the version
|
||||||
|
# used to BE the channel word here ("main"/"dev"), which meant two dev images
|
||||||
|
# eight weeks apart were indistinguishable.
|
||||||
ARG MINSTREL_VERSION=dev
|
ARG MINSTREL_VERSION=dev
|
||||||
|
ARG MINSTREL_CHANNEL=local
|
||||||
RUN go build -trimpath \
|
RUN go build -trimpath \
|
||||||
-ldflags="-s -w -X 'git.fabledsword.com/bvandeusen/minstrel/internal/server.ServerVersion=${MINSTREL_VERSION}'" \
|
-ldflags="-s -w \
|
||||||
|
-X 'git.fabledsword.com/bvandeusen/minstrel/internal/server.ServerVersion=${MINSTREL_VERSION}' \
|
||||||
|
-X 'git.fabledsword.com/bvandeusen/minstrel/internal/server.ServerChannel=${MINSTREL_CHANNEL}'" \
|
||||||
-o /out/minstrel ./cmd/minstrel
|
-o /out/minstrel ./cmd/minstrel
|
||||||
|
|
||||||
FROM debian:bookworm-slim
|
FROM debian:bookworm-slim
|
||||||
|
# ffmpeg: duration probes and the exact-tier audio hash (a SHA-256 of the
|
||||||
|
# encoded audio packets, so no decode). libchromaprint-tools: fpcalc, the
|
||||||
|
# acoustic fingerprint that tells the same recording at two bitrates apart
|
||||||
|
# from two different recordings (M400). Both are baked in at build time so a
|
||||||
|
# deployed instance never fetches either (rule 164); fpcalc is shelled out
|
||||||
|
# rather than bound because CGO_ENABLED=0 above rules out cgo.
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends ca-certificates ffmpeg \
|
&& apt-get install -y --no-install-recommends ca-certificates ffmpeg libchromaprint-tools \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN groupadd --system --gid 1000 minstrel \
|
RUN groupadd --system --gid 1000 minstrel \
|
||||||
|
|||||||
@@ -37,8 +37,12 @@ services:
|
|||||||
ports: ['4533:4533']
|
ports: ['4533:4533']
|
||||||
volumes:
|
volumes:
|
||||||
# Your music library. Point ./music at wherever your audio files
|
# Your music library. Point ./music at wherever your audio files
|
||||||
# live. Mounted read-only — Minstrel never writes to your library.
|
# live. Writable, because Minstrel deletes a file when an admin asks
|
||||||
- ./music:/music:ro
|
# it to (for example, quarantine's "Delete file"). It never moves,
|
||||||
|
# renames or retags anything. The container runs as uid 1000, so that
|
||||||
|
# user needs write access to the folders. Mount it :ro to forbid even
|
||||||
|
# deletes: those actions then refuse, say why, and delete nothing.
|
||||||
|
- ./music:/music
|
||||||
# Generated data: playlist cover collages, artist art, caches.
|
# Generated data: playlist cover collages, artist art, caches.
|
||||||
# The path must match MINSTREL_STORAGE_DATA_DIR, which the image
|
# The path must match MINSTREL_STORAGE_DATA_DIR, which the image
|
||||||
# sets to /app/data — keep this mount on /app/data or your cache
|
# sets to /app/data — keep this mount on /app/data or your cache
|
||||||
@@ -47,7 +51,7 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
MINSTREL_DATABASE_URL: postgres://minstrel:minstrel@db:5432/minstrel?sslmode=disable
|
MINSTREL_DATABASE_URL: postgres://minstrel:minstrel@db:5432/minstrel?sslmode=disable
|
||||||
# Colon-separated library roots to scan; must match the container
|
# Colon-separated library roots to scan; must match the container
|
||||||
# path of the read-only music mount above (/music here).
|
# path of the music mount above (/music here).
|
||||||
MINSTREL_LIBRARY_SCAN_PATHS: /music
|
MINSTREL_LIBRARY_SCAN_PATHS: /music
|
||||||
depends_on: [db]
|
depends_on: [db]
|
||||||
|
|
||||||
@@ -112,11 +116,21 @@ Most operational keys have a `MINSTREL_<SECTION>_<FIELD>` env override. Recommen
|
|||||||
|
|
||||||
Image tags (`git.fabledsword.com/bvandeusen/minstrel:<tag>`):
|
Image tags (`git.fabledsword.com/bvandeusen/minstrel:<tag>`):
|
||||||
|
|
||||||
- `:latest` — the newest blessed image. Moves on every `main` push **and** every release. Recommended for most operators.
|
- `:latest` — production. Tracks `main`'s tip and moves on every `main` push and every release. What most operators should run.
|
||||||
- `:vYYYY.MM.DD` — immutable per-day release tags. Pin one of these for a deployment you don't want moving under you. (Per-day CalVer — no trailing patch digit; a same-day re-cut moves the tag forward.)
|
- `:<commit-sha>` — the rollback unit. Every `main` push publishes one, so any production commit is addressable without a release ceremony. Immutable: a given SHA tag is never re-pushed. Pin one if you need a deployment that cannot change under you, and use it to roll back.
|
||||||
- `:main` — the rolling post-merge tip. Same image as `:latest` at push time; choose it if you want to track `main` explicitly rather than the release line.
|
- `:dev` — the rolling test channel, rebuilt on every push to `dev` and carrying its own freshly-built Android APK. Run this to try something before it ships. It moves constantly, has no per-commit tag, and its only recovery path is forward — if a `:dev` image is broken, the fix is the next push, not a rollback.
|
||||||
|
|
||||||
Every `:latest` and every `:vYYYY.MM.DD` bundles the current signed Android APK, so the in-app update channel is always live. Database migrations run automatically at startup; rollbacks require restoring a Postgres dump.
|
That is the whole tag map. **There are no version-numbered image tags**, and no `:main`. Git and the build's own self-reported version answer "which build is this" — the Settings page shows it, and so does `/healthz`. Release *tags* in git are still `vYYYY.MM.DD.HHMM`; they name a changelog entry and the APK attached to it, not an image.
|
||||||
|
|
||||||
|
Rolling back to `:<commit-sha>` pins the **server code** at that commit — not the server-and-app pair. The Android APK is baked in at image build time, so a SHA image carries whichever app was current when that commit was built, which may be older than what `:latest` bundles now. If both halves matter, check what the image bundles rather than trusting the tag's name.
|
||||||
|
|
||||||
|
Every `:latest`, `:<commit-sha>` and `:dev` bundles a signed Android APK, so the in-app update channel is always live. All are signed with the same key, so a phone can move between the stable and dev channels without uninstalling — point it at a `:dev` server and the in-app updater offers that channel's build.
|
||||||
|
|
||||||
|
The app reports which channel it is on alongside its version, and decides whether an update is available using the build's ordering key rather than its displayed name — the same value Android installs by, so an offer it makes is one the platform will accept.
|
||||||
|
|
||||||
|
Database migrations run automatically at startup; rollbacks require restoring a Postgres dump.
|
||||||
|
|
||||||
|
Releases up to 2026-09-10 also published a `:vYYYY.MM.DD[.HHMM]` image tag. Those images still exist and still work — they are simply not extended.
|
||||||
|
|
||||||
## Specs
|
## Specs
|
||||||
|
|
||||||
@@ -150,7 +164,7 @@ Two concurrent dev processes:
|
|||||||
|
|
||||||
- Day-to-day work happens on `dev` (or feature branches merged into `dev`).
|
- Day-to-day work happens on `dev` (or feature branches merged into `dev`).
|
||||||
- `main` is **protected** — changes land via PR from `dev`.
|
- `main` is **protected** — changes land via PR from `dev`.
|
||||||
- Releases are cut by tagging `v*` off `main`; the release workflow builds and pushes the container image to the Gitea registry.
|
- Releases are cut by tagging `v*` off `main`; the release workflow builds the signed APK, attaches it to the release, and refreshes `:latest` around it.
|
||||||
|
|
||||||
Task and milestone tracking: Fable (`Minstrel` project, id 12).
|
Task and milestone tracking: Fable (`Minstrel` project, id 12).
|
||||||
|
|
||||||
|
|||||||
@@ -21,13 +21,24 @@ android {
|
|||||||
applicationId = "com.fabledsword.minstrel"
|
applicationId = "com.fabledsword.minstrel"
|
||||||
minSdk = 26
|
minSdk = 26
|
||||||
targetSdk = 36
|
targetSdk = 36
|
||||||
// versionName / versionCode are released-build values injected by
|
// versionName / versionCode are released-build values injected by CI.
|
||||||
// CI from the git tag + commit count. Local / debug builds fall
|
// Local / debug builds fall back to "dev" so the About card reads
|
||||||
// back to "dev" so the About card reads honestly. Releases ship
|
// honestly.
|
||||||
// versionName="YYYY.MM.DD.<commits>" (e.g. "2026.06.02.142") and
|
//
|
||||||
// versionCode=<commits>, which is monotonic forever and lets the
|
// versionName is "YYYY.MM.DD.HHMM" from the COMMIT's timestamp, so
|
||||||
// shared isVersionNewer comparator distinguish two same-day
|
// every lane building this source reports the same string and the
|
||||||
// re-cuts (the iteration suffix differs).
|
// channel is the only thing that differs between them.
|
||||||
|
//
|
||||||
|
// versionCode is minutes since 2020-01-01 at BUILD time. It is the
|
||||||
|
// value the platform decides installs by, so it must be monotonic by
|
||||||
|
// construction.
|
||||||
|
//
|
||||||
|
// This comment used to say versionCode was a commit count and that it
|
||||||
|
// was "monotonic forever". It was neither — a commit count runs ahead
|
||||||
|
// on `dev`, so a dev build outranked the `main` release meant to
|
||||||
|
// replace it and Android refused the install as a downgrade. Worth
|
||||||
|
// knowing the claim was here, stated as a reassurance, while the bug
|
||||||
|
// it denied was live.
|
||||||
val versionNameOverride =
|
val versionNameOverride =
|
||||||
(project.findProperty("MINSTREL_VERSION_NAME") as String?)?.takeIf { it.isNotBlank() }
|
(project.findProperty("MINSTREL_VERSION_NAME") as String?)?.takeIf { it.isNotBlank() }
|
||||||
val versionCodeOverride =
|
val versionCodeOverride =
|
||||||
|
|||||||
@@ -15,10 +15,14 @@ import androidx.compose.material3.HorizontalDivider
|
|||||||
import androidx.compose.material3.MaterialTheme
|
import androidx.compose.material3.MaterialTheme
|
||||||
import androidx.compose.material3.OutlinedButton
|
import androidx.compose.material3.OutlinedButton
|
||||||
import androidx.compose.material3.Scaffold
|
import androidx.compose.material3.Scaffold
|
||||||
|
import androidx.compose.material3.SnackbarHost
|
||||||
|
import androidx.compose.material3.SnackbarHostState
|
||||||
import androidx.compose.material3.Text
|
import androidx.compose.material3.Text
|
||||||
import androidx.compose.material3.TextButton
|
import androidx.compose.material3.TextButton
|
||||||
import androidx.compose.runtime.Composable
|
import androidx.compose.runtime.Composable
|
||||||
|
import androidx.compose.runtime.LaunchedEffect
|
||||||
import androidx.compose.runtime.getValue
|
import androidx.compose.runtime.getValue
|
||||||
|
import androidx.compose.runtime.remember
|
||||||
import androidx.compose.ui.Alignment
|
import androidx.compose.ui.Alignment
|
||||||
import androidx.compose.ui.Modifier
|
import androidx.compose.ui.Modifier
|
||||||
import androidx.compose.ui.text.style.TextOverflow
|
import androidx.compose.ui.text.style.TextOverflow
|
||||||
@@ -42,6 +46,12 @@ fun AdminQuarantineScreen(
|
|||||||
viewModel: AdminQuarantineViewModel = hiltViewModel(),
|
viewModel: AdminQuarantineViewModel = hiltViewModel(),
|
||||||
) {
|
) {
|
||||||
val state by viewModel.uiState.collectAsStateWithLifecycle()
|
val state by viewModel.uiState.collectAsStateWithLifecycle()
|
||||||
|
val snackbarHostState = remember { SnackbarHostState() }
|
||||||
|
LaunchedEffect(Unit) {
|
||||||
|
viewModel.transientMessages.collect { msg ->
|
||||||
|
snackbarHostState.showSnackbar(msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
Scaffold(
|
Scaffold(
|
||||||
contentWindowInsets = ShellContentWindowInsets,
|
contentWindowInsets = ShellContentWindowInsets,
|
||||||
modifier = Modifier.fillMaxSize(),
|
modifier = Modifier.fillMaxSize(),
|
||||||
@@ -53,6 +63,7 @@ fun AdminQuarantineScreen(
|
|||||||
onBack = { navController.popBackStack() },
|
onBack = { navController.popBackStack() },
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
|
snackbarHost = { SnackbarHost(snackbarHostState) },
|
||||||
) { inner ->
|
) { inner ->
|
||||||
PullToRefreshScaffold(
|
PullToRefreshScaffold(
|
||||||
onRefresh = { viewModel.refresh().join() },
|
onRefresh = { viewModel.refresh().join() },
|
||||||
|
|||||||
+14
-1
@@ -10,10 +10,13 @@ import com.fabledsword.minstrel.events.EventsStream
|
|||||||
import com.fabledsword.minstrel.models.AdminQuarantineItemRef
|
import com.fabledsword.minstrel.models.AdminQuarantineItemRef
|
||||||
import dagger.hilt.android.lifecycle.HiltViewModel
|
import dagger.hilt.android.lifecycle.HiltViewModel
|
||||||
import kotlinx.coroutines.Job
|
import kotlinx.coroutines.Job
|
||||||
|
import kotlinx.coroutines.channels.Channel
|
||||||
|
import kotlinx.coroutines.flow.Flow
|
||||||
import kotlinx.coroutines.flow.MutableStateFlow
|
import kotlinx.coroutines.flow.MutableStateFlow
|
||||||
import kotlinx.coroutines.flow.StateFlow
|
import kotlinx.coroutines.flow.StateFlow
|
||||||
import kotlinx.coroutines.flow.asStateFlow
|
import kotlinx.coroutines.flow.asStateFlow
|
||||||
import kotlinx.coroutines.flow.filter
|
import kotlinx.coroutines.flow.filter
|
||||||
|
import kotlinx.coroutines.flow.receiveAsFlow
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
|
|
||||||
@@ -34,6 +37,15 @@ class AdminQuarantineViewModel @Inject constructor(
|
|||||||
private val internal = MutableStateFlow<AdminQuarantineUiState>(AdminQuarantineUiState.Loading)
|
private val internal = MutableStateFlow<AdminQuarantineUiState>(AdminQuarantineUiState.Loading)
|
||||||
val uiState: StateFlow<AdminQuarantineUiState> = internal.asStateFlow()
|
val uiState: StateFlow<AdminQuarantineUiState> = internal.asStateFlow()
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One-shot messages for the screen's snackbar. A failed action has to say
|
||||||
|
* why: the row quietly reappearing reads as a glitch, and for a Delete
|
||||||
|
* file refused by a read-only library it hides the one thing the
|
||||||
|
* operator can fix (#3918).
|
||||||
|
*/
|
||||||
|
private val transientMessagesChannel = Channel<String>(Channel.BUFFERED)
|
||||||
|
val transientMessages: Flow<String> = transientMessagesChannel.receiveAsFlow()
|
||||||
|
|
||||||
init {
|
init {
|
||||||
refresh()
|
refresh()
|
||||||
viewModelScope.launch {
|
viewModelScope.launch {
|
||||||
@@ -86,8 +98,9 @@ class AdminQuarantineViewModel @Inject constructor(
|
|||||||
try {
|
try {
|
||||||
action(trackId)
|
action(trackId)
|
||||||
} catch (
|
} catch (
|
||||||
@Suppress("TooGenericExceptionCaught", "SwallowedException") e: Throwable,
|
@Suppress("TooGenericExceptionCaught") e: Throwable,
|
||||||
) {
|
) {
|
||||||
|
transientMessagesChannel.trySend(ErrorCopy.fromThrowable(e))
|
||||||
refresh()
|
refresh()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,18 +37,35 @@ object ErrorCopy {
|
|||||||
* as connection failures.
|
* as connection failures.
|
||||||
*/
|
*/
|
||||||
fun fromThrowable(t: Throwable): String = when (t) {
|
fun fromThrowable(t: Throwable): String = when (t) {
|
||||||
is HttpException -> messageFor(codeFromHttp(t))
|
is HttpException -> fromHttp(t)
|
||||||
is IOException -> messageFor("connection_refused")
|
is IOException -> messageFor("connection_refused")
|
||||||
else -> TABLE.getValue("unknown")
|
else -> TABLE.getValue("unknown")
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun codeFromHttp(e: HttpException): String {
|
/**
|
||||||
|
* Codes whose server message carries specifics the operator needs in
|
||||||
|
* order to act — which directory, which uid — that fixed copy cannot say.
|
||||||
|
* For these the message follows the copy (#3918). Kept to a named set on
|
||||||
|
* purpose: most server messages are internal detail. Mirrors web's
|
||||||
|
* errors.ts.
|
||||||
|
*/
|
||||||
|
private val DETAIL_CODES = setOf("library_not_writable", "file_delete_failed")
|
||||||
|
|
||||||
|
private fun fromHttp(e: HttpException): String {
|
||||||
|
val body = bodyFromHttp(e)
|
||||||
|
val copy = messageFor(body.code.ifEmpty { "unknown" })
|
||||||
|
return if (body.code in DETAIL_CODES && body.message.isNotBlank()) {
|
||||||
|
"$copy ${body.message}"
|
||||||
|
} else {
|
||||||
|
copy
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun bodyFromHttp(e: HttpException): Body {
|
||||||
val raw = runCatching { e.response()?.errorBody()?.string() }.getOrNull()
|
val raw = runCatching { e.response()?.errorBody()?.string() }.getOrNull()
|
||||||
?: return "unknown"
|
?: return Body()
|
||||||
val code = runCatching { json.decodeFromString<Envelope>(raw).error?.code }
|
return runCatching { json.decodeFromString<Envelope>(raw).error }
|
||||||
.getOrNull()
|
.getOrNull() ?: Body()
|
||||||
.orEmpty()
|
|
||||||
return code.ifEmpty { "unknown" }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private val TABLE: Map<String, String> = mapOf(
|
private val TABLE: Map<String, String> = mapOf(
|
||||||
@@ -58,6 +75,7 @@ object ErrorCopy {
|
|||||||
"forbidden" to "You don't have permission to do that.",
|
"forbidden" to "You don't have permission to do that.",
|
||||||
"not_authorized" to "You don't have permission to do that.",
|
"not_authorized" to "You don't have permission to do that.",
|
||||||
"invalid_credentials" to "Wrong username or password.",
|
"invalid_credentials" to "Wrong username or password.",
|
||||||
|
"rate_limited" to "Too many attempts. Wait a few minutes and try again.",
|
||||||
"wrong_password" to "Current password is incorrect.",
|
"wrong_password" to "Current password is incorrect.",
|
||||||
"password_too_short" to "Password must be at least 8 characters.",
|
"password_too_short" to "Password must be at least 8 characters.",
|
||||||
"username_invalid" to "That username isn't valid.",
|
"username_invalid" to "That username isn't valid.",
|
||||||
@@ -99,6 +117,8 @@ object ErrorCopy {
|
|||||||
"request_not_pending" to "This request is no longer pending.",
|
"request_not_pending" to "This request is no longer pending.",
|
||||||
"request_not_found" to "That request no longer exists.",
|
"request_not_found" to "That request no longer exists.",
|
||||||
"track_not_found" to "That track no longer exists.",
|
"track_not_found" to "That track no longer exists.",
|
||||||
|
"library_not_writable" to "The music library isn't writable by the server.",
|
||||||
|
"file_delete_failed" to "The file couldn't be deleted.",
|
||||||
"album_not_found" to "That album no longer exists.",
|
"album_not_found" to "That album no longer exists.",
|
||||||
"artist_not_found" to "That artist no longer exists.",
|
"artist_not_found" to "That artist no longer exists.",
|
||||||
"playlist_not_found" to "That playlist no longer exists.",
|
"playlist_not_found" to "That playlist no longer exists.",
|
||||||
|
|||||||
@@ -1,15 +1,26 @@
|
|||||||
package com.fabledsword.minstrel.models
|
package com.fabledsword.minstrel.models
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wire shape returned by `GET /api/client/version`. Mirrors
|
* The server-bundled APK, as reported by `GET /api/client/version`.
|
||||||
* the Flutter client's `UpdateInfo`.
|
|
||||||
*
|
*
|
||||||
* `version` is the server-bundled APK version (may have a leading
|
* Three values that are deliberately kept apart:
|
||||||
* "v" from the git tag); `apkUrl` is server-relative (e.g.
|
*
|
||||||
* `/api/client/apk`); `sizeBytes` is the download size.
|
* - [version] is a LABEL for people — "YYYY.MM.DD.HHMM", derived from the
|
||||||
|
* build's commit, so two channels carrying the same code read the same.
|
||||||
|
* Display this; never decide on it when [code] is present.
|
||||||
|
* - [code] is the ORDERING KEY, and is the same value Android itself
|
||||||
|
* installs by. It answers "may this be installed over that?", which the
|
||||||
|
* name cannot. Null when the server predates the field.
|
||||||
|
* - [channel] is a SIBLING FIELD, never a suffix inside the name. Reported
|
||||||
|
* verbatim rather than validated, so an unexpected value is shown rather
|
||||||
|
* than dropped.
|
||||||
|
*
|
||||||
|
* [apkUrl] is server-relative (e.g. `/api/client/apk`).
|
||||||
*/
|
*/
|
||||||
data class UpdateInfo(
|
data class UpdateInfo(
|
||||||
val version: String,
|
val version: String,
|
||||||
|
val code: Long?,
|
||||||
|
val channel: String?,
|
||||||
val apkUrl: String,
|
val apkUrl: String,
|
||||||
val sizeBytes: Long,
|
val sizeBytes: Long,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -4,12 +4,26 @@ import kotlinx.serialization.SerialName
|
|||||||
import kotlinx.serialization.Serializable
|
import kotlinx.serialization.Serializable
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wire shape for `GET /api/client/version`. Defaults match Flutter:
|
* Wire shape for `GET /api/client/version`.
|
||||||
* apk_url falls back to `/api/client/apk` if the server omits it.
|
*
|
||||||
|
* `apkUrl` falls back to `/api/client/apk` if the server omits it.
|
||||||
|
*
|
||||||
|
* [code] MUST stay nullable, and this is not a style preference. The app's
|
||||||
|
* Json is configured with `coerceInputValues = true`, which replaces a JSON
|
||||||
|
* null with the declared default for a NON-nullable property — so writing
|
||||||
|
* `val code: Long = 0` would turn "this server reports no ordering key" into
|
||||||
|
* "this build's ordering key is 0", silently, with no error anywhere. A
|
||||||
|
* nullable type is what keeps absent distinguishable from zero, and the
|
||||||
|
* distinction is the whole reason the field exists.
|
||||||
|
*
|
||||||
|
* A server predating the ordering key sends neither [code] nor [channel];
|
||||||
|
* both arrive null and the caller falls back to comparing names.
|
||||||
*/
|
*/
|
||||||
@Serializable
|
@Serializable
|
||||||
data class UpdateInfoWire(
|
data class UpdateInfoWire(
|
||||||
val version: String = "",
|
val version: String = "",
|
||||||
|
val code: Long? = null,
|
||||||
|
val channel: String? = null,
|
||||||
@SerialName("apk_url") val apkUrl: String = "/api/client/apk",
|
@SerialName("apk_url") val apkUrl: String = "/api/client/apk",
|
||||||
@SerialName("size_bytes") val sizeBytes: Long = 0,
|
@SerialName("size_bytes") val sizeBytes: Long = 0,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -109,8 +109,11 @@ private fun MiniCover(coverUrl: String, contentDescription: String) {
|
|||||||
* NowPlayingScreen via [onExpandClick].
|
* NowPlayingScreen via [onExpandClick].
|
||||||
*
|
*
|
||||||
* Layout (Column):
|
* Layout (Column):
|
||||||
* - Slim seek slider at the top (4dp track)
|
* - Slim seek slider pinned at the top (4dp track)
|
||||||
* - Row: cover | title/artist column | like | prev | play/pause | next
|
* - Row: cover | title/artist column | like | prev | play/pause | next.
|
||||||
|
* Weighted so it fills the rest of the fixed-height bar and centres its
|
||||||
|
* own content; otherwise the row keeps its intrinsic 48dp and the
|
||||||
|
* leftover height collects at the bottom as dead surface.
|
||||||
*
|
*
|
||||||
* No kebab on the mini bar (operator 2026-06-01): the full kebab
|
* No kebab on the mini bar (operator 2026-06-01): the full kebab
|
||||||
* surface lives on NowPlayingScreen, and dropping it from the mini
|
* surface lives on NowPlayingScreen, and dropping it from the mini
|
||||||
@@ -164,6 +167,12 @@ fun MiniPlayer(
|
|||||||
durationMs = state.durationMs,
|
durationMs = state.durationMs,
|
||||||
)
|
)
|
||||||
MiniRow(
|
MiniRow(
|
||||||
|
// Take whatever the progress fill leaves. Without this the
|
||||||
|
// Column stacks 4dp + the row's intrinsic 48dp from the top
|
||||||
|
// and the remaining 28dp of an 80dp bar sits empty
|
||||||
|
// underneath — the content looked top-aligned rather than
|
||||||
|
// centred, with a dead strip above the gesture bar.
|
||||||
|
modifier = Modifier.weight(1f),
|
||||||
track = track,
|
track = track,
|
||||||
isPlaying = state.isPlaying,
|
isPlaying = state.isPlaying,
|
||||||
isUpnpLoading = state.isUpnpLoading,
|
isUpnpLoading = state.isUpnpLoading,
|
||||||
@@ -205,6 +214,7 @@ private fun MiniProgressFill(positionMs: Long, durationMs: Long) {
|
|||||||
@Composable
|
@Composable
|
||||||
@Suppress("LongParameterList")
|
@Suppress("LongParameterList")
|
||||||
private fun MiniRow(
|
private fun MiniRow(
|
||||||
|
modifier: Modifier,
|
||||||
track: TrackRef,
|
track: TrackRef,
|
||||||
isPlaying: Boolean,
|
isPlaying: Boolean,
|
||||||
isUpnpLoading: Boolean,
|
isUpnpLoading: Boolean,
|
||||||
@@ -216,7 +226,7 @@ private fun MiniRow(
|
|||||||
onToggleLike: () -> Unit,
|
onToggleLike: () -> Unit,
|
||||||
) {
|
) {
|
||||||
Row(
|
Row(
|
||||||
modifier = Modifier
|
modifier = modifier
|
||||||
.fillMaxWidth()
|
.fillMaxWidth()
|
||||||
.padding(horizontal = 12.dp),
|
.padding(horizontal = 12.dp),
|
||||||
verticalAlignment = Alignment.CenterVertically,
|
verticalAlignment = Alignment.CenterVertically,
|
||||||
|
|||||||
+17
-4
@@ -9,7 +9,7 @@ import com.fabledsword.minstrel.update.data.ApkInstaller
|
|||||||
import com.fabledsword.minstrel.update.data.InstallStage
|
import com.fabledsword.minstrel.update.data.InstallStage
|
||||||
import com.fabledsword.minstrel.update.data.UpdateRepository
|
import com.fabledsword.minstrel.update.data.UpdateRepository
|
||||||
import com.fabledsword.minstrel.update.data.isBusy
|
import com.fabledsword.minstrel.update.data.isBusy
|
||||||
import com.fabledsword.minstrel.update.data.isVersionNewer
|
import com.fabledsword.minstrel.update.data.isUpdateAvailable
|
||||||
import com.fabledsword.minstrel.update.data.message
|
import com.fabledsword.minstrel.update.data.message
|
||||||
import com.fabledsword.minstrel.update.data.stage
|
import com.fabledsword.minstrel.update.data.stage
|
||||||
import dagger.hilt.android.lifecycle.HiltViewModel
|
import dagger.hilt.android.lifecycle.HiltViewModel
|
||||||
@@ -37,6 +37,10 @@ sealed interface UpdateCheckResult {
|
|||||||
|
|
||||||
data class AboutUiState(
|
data class AboutUiState(
|
||||||
val installedVersion: String = BuildConfig.VERSION_NAME,
|
val installedVersion: String = BuildConfig.VERSION_NAME,
|
||||||
|
// The value the platform installs by, and therefore the one the update
|
||||||
|
// check must decide on. Held in state rather than read inline so a test
|
||||||
|
// can drive the comparison without a BuildConfig.
|
||||||
|
val installedCode: Long = BuildConfig.VERSION_CODE.toLong(),
|
||||||
val isChecking: Boolean = false,
|
val isChecking: Boolean = false,
|
||||||
val installStage: InstallStage = InstallStage.IDLE,
|
val installStage: InstallStage = InstallStage.IDLE,
|
||||||
val installMessage: String? = null,
|
val installMessage: String? = null,
|
||||||
@@ -45,8 +49,9 @@ data class AboutUiState(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Backs the About card's update controls. "Check for updates" calls
|
* Backs the About card's update controls. "Check for updates" calls
|
||||||
* [UpdateRepository.getLatest], compares versus the build's
|
* [UpdateRepository.getLatest], compares versus this build via
|
||||||
* VERSION_NAME via [isVersionNewer], and reports the terminal state.
|
* [isUpdateAvailable] — on the ordering key where the server reports one,
|
||||||
|
* on the name otherwise — and reports the terminal state.
|
||||||
* When an update is available, [install] downloads the APK via
|
* When an update is available, [install] downloads the APK via
|
||||||
* [ApkInstaller] and installs it — routing the user to the "install
|
* [ApkInstaller] and installs it — routing the user to the "install
|
||||||
* unknown apps" settings page first when that permission hasn't been
|
* unknown apps" settings page first when that permission hasn't been
|
||||||
@@ -66,9 +71,17 @@ class AboutCardViewModel @Inject constructor(
|
|||||||
viewModelScope.launch {
|
viewModelScope.launch {
|
||||||
internal.update { it.copy(isChecking = true, installMessage = null) }
|
internal.update { it.copy(isChecking = true, installMessage = null) }
|
||||||
val installed = internal.value.installedVersion
|
val installed = internal.value.installedVersion
|
||||||
|
val installedCode = internal.value.installedCode
|
||||||
val result = runCatching { repository.getLatest() }
|
val result = runCatching { repository.getLatest() }
|
||||||
.map { latest ->
|
.map { latest ->
|
||||||
if (isVersionNewer(latest.version, installed)) {
|
if (
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = latest.code,
|
||||||
|
serverName = latest.version,
|
||||||
|
installedCode = installedCode,
|
||||||
|
installedName = installed,
|
||||||
|
)
|
||||||
|
) {
|
||||||
UpdateCheckResult.UpdateAvailable(latest)
|
UpdateCheckResult.UpdateAvailable(latest)
|
||||||
} else {
|
} else {
|
||||||
UpdateCheckResult.Latest
|
UpdateCheckResult.Latest
|
||||||
|
|||||||
+10
-2
@@ -19,7 +19,8 @@ private const val POLL_INTERVAL_MS = 24 * 60 * 60 * 1000L
|
|||||||
/**
|
/**
|
||||||
* Drives the shell's soft "update available" banner. Polls
|
* Drives the shell's soft "update available" banner. Polls
|
||||||
* `/api/client/version` at launch + every 24h and, when the bundled
|
* `/api/client/version` at launch + every 24h and, when the bundled
|
||||||
* APK is strictly newer than this build, exposes its [UpdateInfo] so
|
* APK outranks this build — by ordering key where the server reports one,
|
||||||
|
* by name otherwise — exposes its [UpdateInfo] so
|
||||||
* [com.fabledsword.minstrel.update.ui.UpdateBanner] can nudge an
|
* [com.fabledsword.minstrel.update.ui.UpdateBanner] can nudge an
|
||||||
* install. Mirrors Flutter's `ClientUpdateController`.
|
* install. Mirrors Flutter's `ClientUpdateController`.
|
||||||
*
|
*
|
||||||
@@ -58,6 +59,13 @@ class UpdateBannerController @Inject constructor(
|
|||||||
|
|
||||||
private suspend fun runOnce() {
|
private suspend fun runOnce() {
|
||||||
val info = runCatching { repository.getLatest() }.getOrNull() ?: return
|
val info = runCatching { repository.getLatest() }.getOrNull() ?: return
|
||||||
latest.value = info.takeIf { isVersionNewer(it.version, BuildConfig.VERSION_NAME) }
|
latest.value = info.takeIf {
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = it.code,
|
||||||
|
serverName = it.version,
|
||||||
|
installedCode = BuildConfig.VERSION_CODE.toLong(),
|
||||||
|
installedName = BuildConfig.VERSION_NAME,
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,10 +21,39 @@ class UpdateRepository @Inject constructor(retrofit: Retrofit) {
|
|||||||
|
|
||||||
private fun UpdateInfoWire.toDomain(): UpdateInfo = UpdateInfo(
|
private fun UpdateInfoWire.toDomain(): UpdateInfo = UpdateInfo(
|
||||||
version = version,
|
version = version,
|
||||||
|
code = code,
|
||||||
|
channel = channel,
|
||||||
apkUrl = apkUrl,
|
apkUrl = apkUrl,
|
||||||
sizeBytes = sizeBytes,
|
sizeBytes = sizeBytes,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* True when [server] should be offered over the installed build.
|
||||||
|
*
|
||||||
|
* **Decide on the ordering key whenever the server sends one.** That is the
|
||||||
|
* same value Android's package installer compares, so an offer made this way
|
||||||
|
* implies an install the platform will actually accept. The app used to
|
||||||
|
* compare NAMES while the platform installed by `versionCode`, with nothing
|
||||||
|
* keeping the two orderings consistent — so it could offer a build Android
|
||||||
|
* then refused as a downgrade, or stay quiet about one it would have taken.
|
||||||
|
*
|
||||||
|
* Name comparison survives only as the fallback for a server that predates
|
||||||
|
* the field. A null code means "this server cannot tell me" — never "zero" —
|
||||||
|
* because treating absent as zero would rank every such server as infinitely
|
||||||
|
* old and offer its build to everyone, forever.
|
||||||
|
*/
|
||||||
|
fun isUpdateAvailable(
|
||||||
|
serverCode: Long?,
|
||||||
|
serverName: String,
|
||||||
|
installedCode: Long,
|
||||||
|
installedName: String,
|
||||||
|
): Boolean =
|
||||||
|
if (serverCode != null) {
|
||||||
|
serverCode > installedCode
|
||||||
|
} else {
|
||||||
|
isVersionNewer(serverName, installedName)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* True when [server] is strictly newer than [installed]. Mirrors
|
* True when [server] is strictly newer than [installed]. Mirrors
|
||||||
* Flutter's `isVersionNewer` — splits both strings on `.`, parses
|
* Flutter's `isVersionNewer` — splits both strings on `.`, parses
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package com.fabledsword.minstrel.api
|
||||||
|
|
||||||
|
import okhttp3.MediaType.Companion.toMediaType
|
||||||
|
import okhttp3.ResponseBody.Companion.toResponseBody
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
import retrofit2.HttpException
|
||||||
|
import retrofit2.Response
|
||||||
|
import java.io.IOException
|
||||||
|
|
||||||
|
class ErrorCopyTest {
|
||||||
|
private fun httpError(status: Int, body: String): HttpException =
|
||||||
|
HttpException(
|
||||||
|
Response.error<Unit>(status, body.toResponseBody("application/json".toMediaType())),
|
||||||
|
)
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun libraryNotWritableAppendsTheServerDetail() {
|
||||||
|
val detail = "Minstrel runs as uid 1000, gid 1000 and cannot delete from /music/A " +
|
||||||
|
"(read-only file system). The library mount must be writable by that user. " +
|
||||||
|
"Nothing was deleted."
|
||||||
|
val e = httpError(409, """{"error":{"code":"library_not_writable","message":"$detail"}}""")
|
||||||
|
|
||||||
|
assertEquals(
|
||||||
|
"${ErrorCopy.messageFor("library_not_writable")} $detail",
|
||||||
|
ErrorCopy.fromThrowable(e),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun detailCodeWithoutAMessageShowsTheCopyAlone() {
|
||||||
|
val e = httpError(409, """{"error":{"code":"library_not_writable","message":""}}""")
|
||||||
|
|
||||||
|
assertEquals(ErrorCopy.messageFor("library_not_writable"), ErrorCopy.fromThrowable(e))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server messages are usually internal detail; appending them for every
|
||||||
|
// code would leak driver errors into snackbars. This pins the scope.
|
||||||
|
@Test
|
||||||
|
fun otherCodesNeverCarryTheServerMessage() {
|
||||||
|
val e = httpError(404, """{"error":{"code":"track_not_found","message":"pgx: no rows"}}""")
|
||||||
|
|
||||||
|
assertEquals(ErrorCopy.messageFor("track_not_found"), ErrorCopy.fromThrowable(e))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun anUnparseableBodyFallsBackToUnknown() {
|
||||||
|
val e = httpError(500, "not json")
|
||||||
|
|
||||||
|
assertEquals(ErrorCopy.messageFor("unknown"), ErrorCopy.fromThrowable(e))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun transportFailureMapsToConnectionRefused() {
|
||||||
|
assertEquals(
|
||||||
|
ErrorCopy.messageFor("connection_refused"),
|
||||||
|
ErrorCopy.fromThrowable(IOException("refused")),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
+163
@@ -0,0 +1,163 @@
|
|||||||
|
package com.fabledsword.minstrel.update.data
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
import kotlin.test.assertFalse
|
||||||
|
import kotlin.test.assertTrue
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The update channel had no tests at all before this. That is worth saying
|
||||||
|
* out loud, because the thing it decides — whether anyone is ever offered an
|
||||||
|
* update — fails silently in both directions: an update nobody is offered
|
||||||
|
* looks exactly like being up to date, and nobody files a bug about a prompt
|
||||||
|
* they never saw.
|
||||||
|
*/
|
||||||
|
class UpdateVersioningTest {
|
||||||
|
@Test
|
||||||
|
fun `decides on the ordering key when the server reports one`() {
|
||||||
|
assertTrue(
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = 3523847, serverName = "2026.09.10.1432",
|
||||||
|
installedCode = 3519456, installedName = "2026.09.09.1828",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assertFalse(
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = 3519456, serverName = "2026.09.09.1828",
|
||||||
|
installedCode = 3523847, installedName = "2026.09.10.1432",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `an equal ordering key is not an update`() {
|
||||||
|
assertFalse(
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = 3523847, serverName = "2026.09.10.1432",
|
||||||
|
installedCode = 3523847, installedName = "2026.09.10.1432",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The property the whole rework exists for: the offer must agree with what
|
||||||
|
* the platform will actually install. Where the two disagree, the ordering
|
||||||
|
* key wins, because that is the value Android compares.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun `the ordering key wins even when the name disagrees`() {
|
||||||
|
// Name looks older, key is newer — e.g. an older commit rebuilt later.
|
||||||
|
assertTrue(
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = 9_000_000, serverName = "2020.01.01.0000",
|
||||||
|
installedCode = 1, installedName = "2099.12.31.2359",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
// Name looks newer, key is not. Offering this would be offering an
|
||||||
|
// install the platform then refuses as a downgrade.
|
||||||
|
assertFalse(
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = 1, serverName = "2099.12.31.2359",
|
||||||
|
installedCode = 9_000_000, installedName = "2020.01.01.0000",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `falls back to the name when the server reports no ordering key`() {
|
||||||
|
assertTrue(
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = null, serverName = "2026.09.10.1432",
|
||||||
|
installedCode = 3519456, installedName = "2026.09.09.1828",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assertFalse(
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = null, serverName = "2026.09.09.1828",
|
||||||
|
installedCode = 3519456, installedName = "2026.09.10.1432",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A null code must never be read as zero. Zero would rank every
|
||||||
|
* older server as infinitely behind and offer its build to everyone,
|
||||||
|
* forever — so this asserts the fallback runs instead of a comparison
|
||||||
|
* against 0 succeeding by accident.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun `a null ordering key is absent, not zero`() {
|
||||||
|
// installedCode is 0 here: if null coerced to 0, "0 > 0" would be
|
||||||
|
// false and this would wrongly report no update despite a newer name.
|
||||||
|
assertTrue(
|
||||||
|
isUpdateAvailable(
|
||||||
|
serverCode = null, serverName = "2026.09.10.1432",
|
||||||
|
installedCode = 0, installedName = "2026.09.09.1828",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The recorded migration constraint, pinned so it cannot be forgotten:
|
||||||
|
* the old scheme's fourth segment was a commit count (~1895), the new
|
||||||
|
* one is HHMM. Across a day boundary the date decides and all is well.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun `new-scheme name outranks an old-scheme name on a later day`() {
|
||||||
|
assertTrue(isVersionNewer("2026.09.10.1432", "2026.09.09.1895"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ...but on the SAME day the comparison comes down to HHMM against a
|
||||||
|
* commit count, and any build before ~19:00 UTC reads as older. This is
|
||||||
|
* why the first new-scheme release had to be cut on a later calendar day.
|
||||||
|
* Asserting the trap so nobody "fixes" it by accident.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun `same-day new-scheme name can read older than an old-scheme name`() {
|
||||||
|
assertFalse(isVersionNewer("2026.09.09.1828", "2026.09.09.1895"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `name comparison degrades per segment rather than discarding`() {
|
||||||
|
// The string is still compared rather than rejected outright: an
|
||||||
|
// earlier segment decides and the unparseable tail never matters.
|
||||||
|
assertTrue(isVersionNewer("2026.09.10.1432-dev", "2026.09.09.1828"))
|
||||||
|
|
||||||
|
// A shorter name pads with zeros instead of being refused.
|
||||||
|
assertTrue(isVersionNewer("2026.09.10", "2026.09.09.9999"))
|
||||||
|
assertFalse(isVersionNewer("2026.09.10", "2026.09.10.0"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* What "costs that segment's precision" actually means, and it is worth
|
||||||
|
* pinning because it is a real edge rather than a nicety: when the
|
||||||
|
* unparseable segment is the DECIDING one, it reads as 0 and loses. So a
|
||||||
|
* `-dev` suffixed build compares as older than an unsuffixed one from the
|
||||||
|
* same minute.
|
||||||
|
*
|
||||||
|
* That is the correct behaviour for a degrading parser — it is bounded
|
||||||
|
* loss rather than a discarded string — but it is exactly why the channel
|
||||||
|
* belongs in its own field and never in the name.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun `an unparseable deciding segment reads as zero and loses`() {
|
||||||
|
assertFalse(isVersionNewer("2026.09.10.1432-dev", "2026.09.10.1000"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Both sides unparseable (branch-name builds) falls back to string
|
||||||
|
* inequality, so a dev build still surfaces rather than comparing equal
|
||||||
|
* and going silent.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
fun `two unparseable names fall back to string inequality`() {
|
||||||
|
assertTrue(isVersionNewer("main", "dev"))
|
||||||
|
assertFalse(isVersionNewer("dev", "dev"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `a leading v is ignored on either side`() {
|
||||||
|
assertTrue(isVersionNewer("v2026.09.10.1432", "2026.09.09.1828"))
|
||||||
|
assertFalse(isVersionNewer("v2026.09.10.1432", "v2026.09.10.1432"))
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
+20
-47
@@ -62,57 +62,30 @@ None.
|
|||||||
- **Go toolchain pin.** `go.mod` is on `go 1.25.0` because `golang.org/x/crypto v0.51.0` declares 1.25 as its minimum. `ci-go:1.26` satisfies this with headroom. Future `x/crypto` bumps that move the Go floor should be paired with an image-tag bump in this file + the workflows.
|
- **Go toolchain pin.** `go.mod` is on `go 1.25.0` because `golang.org/x/crypto v0.51.0` declares 1.25 as its minimum. `ci-go:1.26` satisfies this with headroom. Future `x/crypto` bumps that move the Go floor should be paired with an image-tag bump in this file + the workflows.
|
||||||
- **In-app update channel — `needs:`, not polling.** `release.yml`'s `image-release` job declares `needs: [android-release]`, so on tag pushes the signed APK is guaranteed present before the image build starts — no polling window, no race. (The old cross-workflow polling against `flutter.yml` is gone with that workflow.) On non-tag `main` pushes `android-release` is skipped and `image-release` instead pulls the most recent release's APK and reconstructs its exact `versionName`, so `:latest` never ships without an update channel. It degrades to an empty `client/` — never a wrong version — if no release, asset, or tag commit-count can be resolved.
|
- **In-app update channel — `needs:`, not polling.** `release.yml`'s `image-release` job declares `needs: [android-release]`, so on tag pushes the signed APK is guaranteed present before the image build starts — no polling window, no race. (The old cross-workflow polling against `flutter.yml` is gone with that workflow.) On non-tag `main` pushes `android-release` is skipped and `image-release` instead pulls the most recent release's APK and reconstructs its exact `versionName`, so `:latest` never ships without an update channel. It degrades to an empty `client/` — never a wrong version — if no release, asset, or tag commit-count can be resolved.
|
||||||
- **Cache server reachability.** `test-web.yml` does NOT use `cache: 'npm'` on `actions/setup-node` — the Gitea Actions cache server isn't reachable from this runner's container network and `setup-node` was burning ~4m41s on ETIMEDOUT before failing open. With the migration to `ci-go:1.26`, `setup-node` is removed entirely (Node is in the image). The cache concern reappears if a future change re-introduces a network-dependent action.
|
- **Cache server reachability.** `test-web.yml` does NOT use `cache: 'npm'` on `actions/setup-node` — the Gitea Actions cache server isn't reachable from this runner's container network and `setup-node` was burning ~4m41s on ETIMEDOUT before failing open. With the migration to `ci-go:1.26`, `setup-node` is removed entirely (Node is in the image). The cache concern reappears if a future change re-introduces a network-dependent action.
|
||||||
- **Artifacts — use the mirrored actions, never `actions/{upload,download}-artifact`.**
|
- **Artifacts — stock `actions/upload-artifact@v7` and `actions/download-artifact@v8`; never `@v3`.**
|
||||||
```yaml
|
```yaml
|
||||||
uses: https://git.fabledsword.com/bvandeusen/upload-artifact@cb8afe72b42edc798abfb8fcb556cf660d894245
|
uses: actions/upload-artifact@v7
|
||||||
uses: https://git.fabledsword.com/bvandeusen/download-artifact@8d4e9521a5f7e5f8b6351f341f719f9f45a92a3a
|
uses: actions/download-artifact@v8
|
||||||
```
|
```
|
||||||
Upstream's `@v4+` cannot work against this instance and no server-side change
|
Stock works on this forge since the runner moved to gitea/runner 3.x, which
|
||||||
will help: `isGhes()` rejects any hostname that isn't `github.com` /
|
edits the actions' client-side `isGhes()` refusal out of their bundles. Proven
|
||||||
`*.ghe.com` / `*.localhost` and throws before it opens a connection, so the
|
on 2026-09-10 for upload v4–v7 and download v4–v8 (Scribe spike #3843). Until
|
||||||
server is never asked what it supports. `@v3` is worse — it reports success,
|
then this repo pinned SHA mirrors of the Forgejo project's forks, because
|
||||||
and Gitea then serves artifacts back only through the v4 API
|
upstream threw on the hostname before it opened a connection (Scribe 2255).
|
||||||
(`content_encoding = application/zip`), so a v3 upload is stored but invisible
|
|
||||||
to every retrieval path. A green job producing nothing retrievable; that is how
|
|
||||||
72 unreachable artifacts accumulated on this repo. Scribe issues 2255 / 2270.
|
|
||||||
|
|
||||||
Both are pull mirrors of the Forgejo project's forks
|
`@v3` is still broken: it reports success, and Gitea serves artifacts back only
|
||||||
(`code.forgejo.org/forgejo/{upload,download}-artifact`, one commit on upstream
|
through the v4 API (`content_encoding = application/zip`), so a v3 upload is
|
||||||
disabling that check), mirrored so CI depends on commits we hold and pinned by
|
stored but invisible to every retrieval path. That is how 72 unreachable
|
||||||
SHA because the mirrors auto-sync every 8h — a moved upstream tag would
|
artifacts accumulated on this repo (Scribe 2270).
|
||||||
otherwise silently change what runs.
|
|
||||||
|
|
||||||
**Match the pins on `@actions/artifact`, not on the actions' own version
|
**Pairing no longer needs managing.** This entry used to pin upload v5 against
|
||||||
numbers.** The two actions release on unrelated cadences, so equal version
|
download v6 so both bundled `@actions/artifact` ^4.0.0, warning that a mismatch
|
||||||
numbers do NOT mean a compatible pair — upload `v5` bundles `@actions/artifact`
|
across `release.yml`'s producer/consumer pair would list empty. Tested, and not
|
||||||
^4.0.0 while download `v5` bundles ^2.3.2. The pins above are upload **v5** and
|
true on this instance: every download major v4–v8 read the artifacts of every
|
||||||
download **v6**, which is the pairing that puts ^4.0.0 on both sides. This
|
upload major v4–v7, by name and by pattern (CI-runner run 6312). The only real
|
||||||
matters because `release.yml` is a producer/consumer pair — `android-release`
|
protocol break is v3 → v4. node24 is no longer a concern either — every
|
||||||
uploads `minstrel-apk`, `image-release` downloads it — and a protocol mismatch
|
CI-runner image carries Node 24 and the runner runs actions with the image's
|
||||||
across it yields an empty listing rather than an error, exactly the silent
|
`node`.
|
||||||
failure this entry exists to prevent.
|
|
||||||
|
|
||||||
| tag | `@actions/artifact` | runtime |
|
|
||||||
|---|---|---|
|
|
||||||
| upload v4 | ^2.1.1 | node20 |
|
|
||||||
| **upload v5** ← pinned | **^4.0.0** | node20 |
|
|
||||||
| download v4 | ^2.1.1 | node20 |
|
|
||||||
| download v5 | ^2.3.2 | node20 |
|
|
||||||
| **download v6** ← pinned | **^4.0.0** | node20 |
|
|
||||||
| download v7 | ^5.0.0 | **node24** |
|
|
||||||
|
|
||||||
The only true protocol break in this history was **v3 → v4** (upstream:
|
|
||||||
"Downloading artifacts that were created from `actions/upload-artifact@v3` and
|
|
||||||
below are not supported"); v4-and-up are one family. Later majors are mostly
|
|
||||||
ergonomics and runtime — upload v4 forbids re-uploading a name and caps a job
|
|
||||||
at 500 artifacts; download v5 made by-ID extraction match by-name.
|
|
||||||
|
|
||||||
**Do not jump the download pin to v7.** That major is a runner requirement, not
|
|
||||||
a feature change: it moves to `runs.using: node24` and upstream states it
|
|
||||||
"requires a minimum Actions Runner version of 2.327.1 … if you are using
|
|
||||||
self-hosted runners, ensure they are updated before upgrading." act_runner is
|
|
||||||
not GitHub's runner and makes no such version claim, so node24 is unverified
|
|
||||||
here. Everything currently pinned is node20.
|
|
||||||
|
|
||||||
Upload steps set `if-no-files-found: error` rather than the default `warn`, so
|
Upload steps set `if-no-files-found: error` rather than the default `warn`, so
|
||||||
an upload that matches nothing fails its own job instead of failing the
|
an upload that matches nothing fails its own job instead of failing the
|
||||||
|
|||||||
Executable
+159
@@ -0,0 +1,159 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Derives the three values a build is stamped with, and the tag that names it.
|
||||||
|
#
|
||||||
|
# name=YYYY.MM.DD.HHMM label for people, from the timestamp of the newest
|
||||||
|
# commit that CHANGED SOMETHING SHIPPED (see SHIPPED)
|
||||||
|
# code=<int> ordering key, minutes since 2020-01-01 at BUILD time
|
||||||
|
# tag=v<name> what a release of this commit must be called
|
||||||
|
#
|
||||||
|
# Usage: ci/version.sh [<commit-ish>] (default HEAD)
|
||||||
|
#
|
||||||
|
# This exists as a script rather than inline workflow YAML for one reason:
|
||||||
|
# release.yml only runs on `main` and on tags, so anything living inside it is
|
||||||
|
# unverifiable until a release is already happening — which is the worst
|
||||||
|
# possible moment to discover the version is wrong, because the failure mode
|
||||||
|
# is silent (an update nobody is offered looks exactly like being current).
|
||||||
|
# As a script it can be executed by a test on every push instead.
|
||||||
|
#
|
||||||
|
# The two clocks are deliberate and are NOT interchangeable:
|
||||||
|
#
|
||||||
|
# The NAME answers "is this the same code?" — so it must read identically on
|
||||||
|
# every lane that builds this commit. Commit time does that; build time
|
||||||
|
# prints two different strings for one thing.
|
||||||
|
#
|
||||||
|
# The CODE answers "may this be installed over that?" — so it must be
|
||||||
|
# monotonic BY CONSTRUCTION. Build time is; commit time is not (rebuild an
|
||||||
|
# older commit and it goes down, which on a phone is a refused install), and
|
||||||
|
# a commit COUNT is worse still, because it runs ahead on `dev` and inverts
|
||||||
|
# against `main`.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
readonly EPOCH_2020=1577836800 # 2020-01-01T00:00:00Z
|
||||||
|
readonly REF="${1:-HEAD}"
|
||||||
|
|
||||||
|
# Both clocks are overridable so a test can pin them. Nothing but tests should
|
||||||
|
# set these — the defaults are the real derivation.
|
||||||
|
# The paths that do NOT ship, in either artifact. Everything else counts.
|
||||||
|
#
|
||||||
|
# A DENYLIST, and the direction is the whole point. As an allowlist, the list
|
||||||
|
# has to be updated by whoever adds a directory and nothing fails if they
|
||||||
|
# don't — so the failure mode is a changed artifact keeping its old version,
|
||||||
|
# silently, on a green run. That is a build lying about what it is. Inverted,
|
||||||
|
# new content counts by default and the only way to wrongly EXCLUDE something
|
||||||
|
# is to name it here deliberately.
|
||||||
|
#
|
||||||
|
# The two error directions are not symmetric, which is why this is not taste:
|
||||||
|
# wrongly excluded → changed artifact, unchanged version. A silent lie.
|
||||||
|
# wrongly included → version moves when nothing shipped. Cosmetic noise in
|
||||||
|
# a string nobody sorts.
|
||||||
|
#
|
||||||
|
# THIS REPO SHIPS TWO ARTIFACTS FROM ONE DERIVATION, and that is why the list
|
||||||
|
# is shorter than it looks like it should be. The server image ships cmd/,
|
||||||
|
# internal/, shared/, web/, config.example.yaml and client/; the APK ships
|
||||||
|
# android/. Neither ships the other's sources — but excluding android/ here
|
||||||
|
# would stop an Android-only commit from moving the APK's OWN version, which
|
||||||
|
# is the dangerous direction. So this is the union: exclude only what ships in
|
||||||
|
# NEITHER, and accept that an Android commit also nudges the server's reported
|
||||||
|
# version. Over-inclusion across the two, which is the harmless direction.
|
||||||
|
#
|
||||||
|
# The family's other repos (roundtable / roundtable-android) each keep a
|
||||||
|
# tighter list because they are separate repos with one artifact apiece. Do
|
||||||
|
# not copy theirs onto this one.
|
||||||
|
readonly SHIPPED=(
|
||||||
|
.
|
||||||
|
':!.gitea' # CI workflows — including this script's own caller
|
||||||
|
':!ci' # CI scripts — including this script
|
||||||
|
':!docs'
|
||||||
|
':!tools' # asset/font generators; their OUTPUT ships, they do not
|
||||||
|
':!deploy' # test-database bootstrap SQL
|
||||||
|
':!bin' # local `make build` output
|
||||||
|
':!*.md'
|
||||||
|
':!Makefile'
|
||||||
|
':!docker-compose.yml'
|
||||||
|
':!.env.example'
|
||||||
|
':!.gitignore'
|
||||||
|
':!.dockerignore'
|
||||||
|
':!renovate.json'
|
||||||
|
':!.golangci.yml'
|
||||||
|
|
||||||
|
# TESTS DO NOT SHIP, so they must not re-version an artifact.
|
||||||
|
#
|
||||||
|
# Named as globs rather than a directory because this repo has no tests/
|
||||||
|
# tree to exclude: Go tests sit inline beside the code they cover, and the
|
||||||
|
# web suite sits beside its modules. `go build` drops *_test.go outright and
|
||||||
|
# the Vite build never imports a .test.ts, so neither reaches an artifact.
|
||||||
|
#
|
||||||
|
# A commit touching a test AND its source still moves the version — the
|
||||||
|
# source path matches on its own. Only a test-ONLY commit is inert, which is
|
||||||
|
# the whole intent.
|
||||||
|
#
|
||||||
|
# Patterns match what exists today and nothing speculative: there are no
|
||||||
|
# .spec.* files, no __tests__/ directories and no androidTest/ tree. If any
|
||||||
|
# appear they will re-version until named here, which is the harmless
|
||||||
|
# direction and the reason this list is a denylist.
|
||||||
|
':!*_test.go' # 158 files, inline beside the code
|
||||||
|
':!*.test.ts' # 114 files
|
||||||
|
':!*.test.js'
|
||||||
|
':!android/app/src/test' # JVM unit tests; no androidTest tree exists
|
||||||
|
':!web/vitest.config.ts' # test-harness config, not build config
|
||||||
|
':!web/vitest.setup.ts'
|
||||||
|
)
|
||||||
|
|
||||||
|
commit_epoch="${MINSTREL_COMMIT_EPOCH:-}"
|
||||||
|
if [ -z "${commit_epoch}" ]; then
|
||||||
|
commit_epoch="$(git log --format=%ct -1 "${REF}" -- "${SHIPPED[@]}")"
|
||||||
|
# Loudly, on purpose. A silent fallback here is the landmine this whole
|
||||||
|
# script exists to avoid: a plausible-looking version that is quietly wrong,
|
||||||
|
# on a green run. Realistically this means a shallow clone (no commit in
|
||||||
|
# range touches the shipped set) rather than a repo of pure CI config.
|
||||||
|
if [ -z "${commit_epoch}" ]; then
|
||||||
|
echo "version.sh: no commit under '${REF}' touches the shipped file set — shallow clone? (needs fetch-depth: 0)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
now_epoch="${MINSTREL_NOW_EPOCH:-$(date -u +%s)}"
|
||||||
|
|
||||||
|
if ! name="$(date -u -d "@${commit_epoch}" +%Y.%m.%d.%H%M 2>/dev/null)"; then
|
||||||
|
echo "version.sh: could not read a commit timestamp from '${commit_epoch}'" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! [ "${now_epoch}" -eq "${now_epoch}" ] 2>/dev/null; then
|
||||||
|
echo "version.sh: build timestamp '${now_epoch}' is not a number" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
code=$(( (now_epoch - EPOCH_2020) / 60 ))
|
||||||
|
|
||||||
|
# Assert the shape here, at the source. A malformed name builds, signs and
|
||||||
|
# publishes perfectly happily; it only surfaces later as an update channel
|
||||||
|
# that has quietly stopped offering anything.
|
||||||
|
if [[ ! "${name}" =~ ^[0-9]{4}\.[0-9]{2}\.[0-9]{2}\.[0-9]{4}$ ]]; then
|
||||||
|
echo "version.sh: name '${name}' is not YYYY.MM.DD.HHMM" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# A non-positive key means the build clock is set before 2020, and every
|
||||||
|
# comparison downstream would be nonsense.
|
||||||
|
if [ "${code}" -le 0 ]; then
|
||||||
|
echo "version.sh: ordering key '${code}' is not positive — build clock wrong?" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Android's versionCode is a signed 32-bit int and the platform refuses an APK
|
||||||
|
# whose code exceeds it. At ~525k minutes a year this is four thousand years
|
||||||
|
# away in normal operation, so the realistic cause is a build machine with a
|
||||||
|
# badly wrong clock — which produces a code that is not merely too large but
|
||||||
|
# also unreachably high, permanently blocking every real build that follows
|
||||||
|
# from ever outranking it. Cheaper to refuse the build than to discover that
|
||||||
|
# from a phone that will not update.
|
||||||
|
readonly VERSION_CODE_CEILING=2147483647
|
||||||
|
if [ "${code}" -gt "${VERSION_CODE_CEILING}" ]; then
|
||||||
|
echo "version.sh: ordering key '${code}' exceeds versionCode's int32 ceiling — build clock wrong?" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# KEY=VALUE, which is also exactly $GITHUB_OUTPUT's format.
|
||||||
|
echo "name=${name}"
|
||||||
|
echo "code=${code}"
|
||||||
|
echo "tag=v${name}"
|
||||||
+24
-1
@@ -122,7 +122,15 @@ func run() error {
|
|||||||
}
|
}
|
||||||
defer pool.Close()
|
defer pool.Close()
|
||||||
|
|
||||||
scanner := library.New(pool, logger, cfg.Library.ScanPaths)
|
// Fingerprinting settings (M400 #3913): one instance, shared by the scanner,
|
||||||
|
// the fingerprint backfill, the duplicate sweep and the admin API, so a save
|
||||||
|
// reaches all of them without a restart. A load failure is logged, not fatal:
|
||||||
|
// the service falls back to the shipped defaults.
|
||||||
|
fpSettings, fpErr := library.NewFingerprintSettingsService(ctx, pool)
|
||||||
|
if fpErr != nil {
|
||||||
|
logger.Warn("fingerprint settings: using defaults", "err", fpErr)
|
||||||
|
}
|
||||||
|
scanner := library.New(pool, logger, cfg.Library.ScanPaths, fpSettings)
|
||||||
|
|
||||||
contact := cfg.Library.ContactEmail
|
contact := cfg.Library.ContactEmail
|
||||||
if contact == "" {
|
if contact == "" {
|
||||||
@@ -214,6 +222,17 @@ func run() error {
|
|||||||
// SQL, no external calls; empty on single-user servers.
|
// SQL, no external calls; empty on single-user servers.
|
||||||
go coplay.NewWorker(pool, logger.With("component", "coplay")).Run(ctx)
|
go coplay.NewWorker(pool, logger.With("component", "coplay")).Run(ctx)
|
||||||
|
|
||||||
|
// Fingerprint backfill (M400 #3908): fingerprints the tracks the scan never
|
||||||
|
// will — everything imported before fingerprinting existed, and rows derived
|
||||||
|
// by an older method. A worker of its own rather than a scan stage; see
|
||||||
|
// internal/library/fingerprint_backfill.go for why.
|
||||||
|
go library.NewFingerprintBackfillWorker(pool, logger.With("component", "fingerprint_backfill"), fpSettings).Run(ctx)
|
||||||
|
|
||||||
|
// Duplicate sweep (M400 #3910): proposes groups of tracks holding one
|
||||||
|
// recording, from the fingerprints above. Sweeps only when fingerprints have
|
||||||
|
// changed since the last sweep.
|
||||||
|
go library.NewDuplicateSweepWorker(pool, logger.With("component", "duplicate_sweep"), fpSettings).Run(ctx)
|
||||||
|
|
||||||
// Start the tag-enrichment worker (#1490). Reconciles the compiled-in
|
// Start the tag-enrichment worker (#1490). Reconciles the compiled-in
|
||||||
// tag providers with tag_provider_settings, bumps the sources version if
|
// tag providers with tag_provider_settings, bumps the sources version if
|
||||||
// the provider set changed (re-opening settled rows), then drains tracks
|
// the provider set changed (re-opening settled rows), then drains tracks
|
||||||
@@ -357,6 +376,10 @@ func run() error {
|
|||||||
srv.PlaylistScheduler = playlistScheduler
|
srv.PlaylistScheduler = playlistScheduler
|
||||||
srv.RecSettings = recSettings
|
srv.RecSettings = recSettings
|
||||||
srv.TagSettings = tagSettings
|
srv.TagSettings = tagSettings
|
||||||
|
srv.FingerprintSettings = fpSettings
|
||||||
|
// The sweeper above holds this same instance, so a save from the admin
|
||||||
|
// card changes what it does on its next tick (#3936).
|
||||||
|
srv.ReacqSettings = reacqSettings
|
||||||
srv.StreamSecret = cfg.StreamSecret
|
srv.StreamSecret = cfg.StreamSecret
|
||||||
httpServer := &http.Server{
|
httpServer := &http.Server{
|
||||||
Addr: cfg.Server.Address,
|
Addr: cfg.Server.Address,
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
package api
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||||
)
|
)
|
||||||
|
|
||||||
// coverageRollupResp is the wire shape for GET /api/admin/library/coverage.
|
// coverageRollupResp is the wire shape for GET /api/admin/library/coverage.
|
||||||
@@ -36,3 +38,46 @@ func (h *handlers) handleGetLibraryCoverage(w http.ResponseWriter, r *http.Reque
|
|||||||
PendingNoMbid: row.PendingNoMbid,
|
PendingNoMbid: row.PendingNoMbid,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fingerprintCoverageResp is the wire shape for GET /api/admin/library/fingerprints.
|
||||||
|
// fingerprinted + rejected + pending = total. Missing tracks are not counted:
|
||||||
|
// there is no file to fingerprint. Enabled travels with the counts because with
|
||||||
|
// fingerprinting off (#3913) pending never shrinks, and a gauge that implies
|
||||||
|
// progress would be promising work nothing is doing.
|
||||||
|
type fingerprintCoverageResp struct {
|
||||||
|
Total int64 `json:"total"`
|
||||||
|
Fingerprinted int64 `json:"fingerprinted"`
|
||||||
|
Rejected int64 `json:"rejected"`
|
||||||
|
Pending int64 `json:"pending"`
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// fingerprintCoverage reads the gauge against the current settings: a print at
|
||||||
|
// another length counts as pending, because the backfill will re-derive it.
|
||||||
|
func (h *handlers) fingerprintCoverage(ctx context.Context) (fingerprintCoverageResp, error) {
|
||||||
|
cfg := h.fingerprintSettings.Get()
|
||||||
|
row, err := library.FingerprintCoverage(ctx, h.pool, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return fingerprintCoverageResp{}, err
|
||||||
|
}
|
||||||
|
return fingerprintCoverageResp{
|
||||||
|
Total: row.Total,
|
||||||
|
Fingerprinted: row.Fingerprinted,
|
||||||
|
Rejected: row.Rejected,
|
||||||
|
Pending: row.Pending,
|
||||||
|
Enabled: cfg.Enabled,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleGetFingerprintCoverage implements GET /api/admin/library/fingerprints:
|
||||||
|
// how far the fingerprint backfill (#3908) has got. The backfill is its own
|
||||||
|
// worker spanning many passes, with no scan run to attach a tally to, so its
|
||||||
|
// progress is read live here. Always 200; zeros on an empty library.
|
||||||
|
func (h *handlers) handleGetFingerprintCoverage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
cov, err := h.fingerprintCoverage(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
writeErrWithLog(w, h.logger, "admin: get fingerprint coverage", apierror.InternalMsg("lookup failed", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, cov)
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,316 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||||
|
)
|
||||||
|
|
||||||
|
// duplicateMemberView is one copy in a proposed duplicate group. LikeCount and
|
||||||
|
// PlayCount span every user: the report is admin-only, and what a copy carries
|
||||||
|
// is the fact the operator weighs when choosing which to keep.
|
||||||
|
type duplicateMemberView struct {
|
||||||
|
TrackID string `json:"track_id"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
ArtistName string `json:"artist_name"`
|
||||||
|
AlbumID string `json:"album_id"`
|
||||||
|
AlbumTitle string `json:"album_title"`
|
||||||
|
FilePath string `json:"file_path"`
|
||||||
|
FileFormat string `json:"file_format"`
|
||||||
|
FileSize int64 `json:"file_size"`
|
||||||
|
DurationSec int32 `json:"duration_sec"`
|
||||||
|
AddedAt string `json:"added_at"`
|
||||||
|
LikeCount int64 `json:"like_count"`
|
||||||
|
PlayCount int64 `json:"play_count"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// duplicateGroupView is one proposal. SurvivorTrackID and SurvivorReason are
|
||||||
|
// the copy the report proposes keeping and the rule that chose it
|
||||||
|
// (library.ProposeSurvivor) — a default the merge (#3911) lets the operator
|
||||||
|
// override.
|
||||||
|
type duplicateGroupView struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Tier string `json:"tier"`
|
||||||
|
WorstBitErrorRate *float32 `json:"worst_bit_error_rate"`
|
||||||
|
DetectedAt string `json:"detected_at"`
|
||||||
|
SurvivorTrackID string `json:"survivor_track_id"`
|
||||||
|
SurvivorReason string `json:"survivor_reason"`
|
||||||
|
Members []duplicateMemberView `json:"members"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// duplicateSweepView is the latest sweep. State is "never" when none has run,
|
||||||
|
// which is what lets the page tell an empty report apart from a sweep that
|
||||||
|
// found nothing.
|
||||||
|
type duplicateSweepView struct {
|
||||||
|
State string `json:"state"`
|
||||||
|
StartedAt *string `json:"started_at"`
|
||||||
|
FinishedAt *string `json:"finished_at"`
|
||||||
|
Candidates *int32 `json:"candidates"`
|
||||||
|
GroupsFound *int32 `json:"groups_found"`
|
||||||
|
OversizeClusters *int32 `json:"oversize_clusters"`
|
||||||
|
ErrorMessage *string `json:"error_message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// adminDuplicatesResponse is the paged report. Total counts groups.
|
||||||
|
type adminDuplicatesResponse struct {
|
||||||
|
Sweep duplicateSweepView `json:"sweep"`
|
||||||
|
Fingerprints fingerprintCoverageResp `json:"fingerprints"`
|
||||||
|
Total int64 `json:"total"`
|
||||||
|
Limit int `json:"limit"`
|
||||||
|
Offset int `json:"offset"`
|
||||||
|
Groups []duplicateGroupView `json:"groups"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleListDuplicates implements GET /api/admin/library/duplicates (#3912).
|
||||||
|
//
|
||||||
|
// Read-only. The sweep's state and the fingerprint backfill's progress travel
|
||||||
|
// with the groups because an empty report means three different things — still
|
||||||
|
// fingerprinting, never swept, or swept and clean — and the page has to say which.
|
||||||
|
func (h *handlers) handleListDuplicates(w http.ResponseWriter, r *http.Request) {
|
||||||
|
limit, offset, err := parsePaging(r.URL.Query())
|
||||||
|
if err != nil {
|
||||||
|
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_paging")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx := r.Context()
|
||||||
|
q := dbq.New(h.pool)
|
||||||
|
|
||||||
|
sweep := duplicateSweepView{State: "never"}
|
||||||
|
last, err := q.GetLatestDuplicateSweep(ctx)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
sweep = duplicateSweepViewOf(last)
|
||||||
|
case !errors.Is(err, pgx.ErrNoRows):
|
||||||
|
h.logger.Error("admin: latest duplicate sweep", "err", err)
|
||||||
|
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cov, err := h.fingerprintCoverage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("admin: fingerprint coverage", "err", err)
|
||||||
|
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
total, err := q.CountPendingDuplicateGroups(ctx)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("admin: count duplicate groups", "err", err)
|
||||||
|
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rows, err := q.ListPendingDuplicateGroupMembers(ctx, dbq.ListPendingDuplicateGroupMembersParams{
|
||||||
|
PageLimit: int32(limit), PageOffset: int32(offset),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("admin: list duplicate groups", "err", err)
|
||||||
|
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, adminDuplicatesResponse{
|
||||||
|
Sweep: sweep,
|
||||||
|
Fingerprints: cov,
|
||||||
|
Total: total,
|
||||||
|
Limit: limit,
|
||||||
|
Offset: offset,
|
||||||
|
Groups: foldDuplicateGroups(rows),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func duplicateSweepViewOf(s dbq.DuplicateSweep) duplicateSweepView {
|
||||||
|
v := duplicateSweepView{
|
||||||
|
State: "running",
|
||||||
|
Candidates: s.Candidates,
|
||||||
|
GroupsFound: s.GroupsFound,
|
||||||
|
OversizeClusters: s.OversizeClusters,
|
||||||
|
ErrorMessage: s.ErrorMessage,
|
||||||
|
}
|
||||||
|
started := formatTimestamp(s.StartedAt)
|
||||||
|
v.StartedAt = &started
|
||||||
|
if s.FinishedAt.Valid {
|
||||||
|
finished := formatTimestamp(s.FinishedAt)
|
||||||
|
v.FinishedAt = &finished
|
||||||
|
v.State = "finished"
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// foldDuplicateGroups folds the one-row-per-member query result into groups and
|
||||||
|
// proposes each group's survivor. It relies on the query ordering members of a
|
||||||
|
// group together, so a run-length fold is enough and the page order holds.
|
||||||
|
func foldDuplicateGroups(rows []dbq.ListPendingDuplicateGroupMembersRow) []duplicateGroupView {
|
||||||
|
groups := make([]duplicateGroupView, 0, 8)
|
||||||
|
var candidates [][]library.SurvivorCandidate
|
||||||
|
for _, row := range rows {
|
||||||
|
id := uuidToString(row.GroupID)
|
||||||
|
if n := len(groups); n == 0 || groups[n-1].ID != id {
|
||||||
|
groups = append(groups, duplicateGroupView{
|
||||||
|
ID: id,
|
||||||
|
Tier: row.Tier,
|
||||||
|
WorstBitErrorRate: row.WorstBitErrorRate,
|
||||||
|
DetectedAt: formatTimestamp(row.DetectedAt),
|
||||||
|
})
|
||||||
|
candidates = append(candidates, nil)
|
||||||
|
}
|
||||||
|
n := len(groups) - 1
|
||||||
|
trackID := uuidToString(row.TrackID)
|
||||||
|
groups[n].Members = append(groups[n].Members, duplicateMemberView{
|
||||||
|
TrackID: trackID,
|
||||||
|
Title: row.Title,
|
||||||
|
ArtistName: row.ArtistName,
|
||||||
|
AlbumID: uuidToString(row.AlbumID),
|
||||||
|
AlbumTitle: row.AlbumTitle,
|
||||||
|
FilePath: row.FilePath,
|
||||||
|
FileFormat: row.FileFormat,
|
||||||
|
FileSize: row.FileSize,
|
||||||
|
DurationSec: row.DurationMs / 1000,
|
||||||
|
AddedAt: formatTimestamp(row.AddedAt),
|
||||||
|
LikeCount: row.LikeCount,
|
||||||
|
PlayCount: row.PlayCount,
|
||||||
|
})
|
||||||
|
candidates[n] = append(candidates[n], library.SurvivorCandidate{
|
||||||
|
TrackID: trackID, FileFormat: row.FileFormat, FileSize: row.FileSize, AddedAt: row.AddedAt.Time,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for i := range groups {
|
||||||
|
groups[i].SurvivorTrackID, groups[i].SurvivorReason = library.ProposeSurvivor(candidates[i])
|
||||||
|
}
|
||||||
|
return groups
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleRunDuplicateSweep implements POST /api/admin/library/duplicates/sweep:
|
||||||
|
// 202 when a sweep starts, 409 sweep_in_progress when one is already running.
|
||||||
|
// The sweep outlives the request, so it runs on a background context, as
|
||||||
|
// handleTriggerScan's scan does.
|
||||||
|
func (h *handlers) handleRunDuplicateSweep(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
// Runs whatever the sweep interval says: the interval paces the automatic
|
||||||
|
// sweep, and an operator pressing the button has already decided.
|
||||||
|
started, err := library.TryStartDuplicateSweep(
|
||||||
|
context.Background(), h.pool, h.logger.With("source", "manual"), h.fingerprintSettings.Get(),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("admin: start duplicate sweep", "err", err)
|
||||||
|
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !started {
|
||||||
|
writeAdminJSONErr(w, http.StatusConflict, "sweep_in_progress")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusAccepted, map[string]bool{"started": true})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleDismissDuplicateGroup implements POST
|
||||||
|
// /api/admin/library/duplicates/{id}/dismiss: "these are not duplicates". The
|
||||||
|
// sweep keeps the dismissal and will not propose that set of tracks again. 404
|
||||||
|
// duplicate_group_not_pending when the group was already resolved or is gone.
|
||||||
|
func (h *handlers) handleDismissDuplicateGroup(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := parseUUID(chi.URLParam(r, "id"))
|
||||||
|
if !ok {
|
||||||
|
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_id")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
n, err := dbq.New(h.pool).DismissDuplicateGroup(r.Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("admin: dismiss duplicate group", "err", err)
|
||||||
|
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n == 0 {
|
||||||
|
writeAdminJSONErr(w, http.StatusNotFound, "duplicate_group_not_pending")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]string{"status": "dismissed"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeDuplicateRequest chooses the copy to keep. An empty survivor_track_id
|
||||||
|
// keeps the report's proposal.
|
||||||
|
type mergeDuplicateRequest struct {
|
||||||
|
SurvivorTrackID string `json:"survivor_track_id"`
|
||||||
|
Unmonitor bool `json:"unmonitor"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeDuplicateResponse reports what the merge removed. RemovedPaths are files
|
||||||
|
// deleted from disk; the operator reads them to know exactly what went.
|
||||||
|
type mergeDuplicateResponse struct {
|
||||||
|
SurvivorTrackID string `json:"survivor_track_id"`
|
||||||
|
RemovedPaths []string `json:"removed_paths"`
|
||||||
|
LidarrUnmonitorFailed *bool `json:"lidarr_unmonitor_failed,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeRequestBodyLimit bounds the request body. It holds one id and a flag.
|
||||||
|
const mergeRequestBodyLimit = 1 << 16
|
||||||
|
|
||||||
|
// handleMergeDuplicateGroup implements POST /api/admin/library/duplicates/{id}/merge
|
||||||
|
// (#3911): keep one copy, move the others' likes, plays and playlist entries onto
|
||||||
|
// it, and delete the others' files and rows.
|
||||||
|
//
|
||||||
|
// Errors:
|
||||||
|
// - 409 library_not_writable / 500 file_delete_failed when a file could not be
|
||||||
|
// removed — nothing was changed (fileRemoveAPIError)
|
||||||
|
// - 404 duplicate_group_not_pending when the group was already resolved
|
||||||
|
// - 400 survivor_not_in_group, invalid_id, invalid_body
|
||||||
|
func (h *handlers) handleMergeDuplicateGroup(w http.ResponseWriter, r *http.Request) {
|
||||||
|
admin, ok := requireUser(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
groupID, ok := parseUUID(chi.URLParam(r, "id"))
|
||||||
|
if !ok {
|
||||||
|
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_id")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var body mergeDuplicateRequest
|
||||||
|
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, mergeRequestBodyLimit)).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var survivorID pgtype.UUID // invalid: keep the proposal
|
||||||
|
if body.SurvivorTrackID != "" {
|
||||||
|
if survivorID, ok = parseUUID(body.SurvivorTrackID); !ok {
|
||||||
|
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_id")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
res, unmonitorFailed, err := h.tracks.MergeDuplicates(r.Context(), groupID, survivorID, admin.ID, body.Unmonitor)
|
||||||
|
if err != nil {
|
||||||
|
if apiErr, ok := fileRemoveAPIError(err); ok {
|
||||||
|
logFileRemoveFailure(h.logger, apiErr, "group_id", uuidToString(groupID))
|
||||||
|
writeErr(w, apiErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, library.ErrDuplicateGroupNotPending):
|
||||||
|
writeAdminJSONErr(w, http.StatusNotFound, "duplicate_group_not_pending")
|
||||||
|
case errors.Is(err, library.ErrSurvivorNotInGroup):
|
||||||
|
writeAdminJSONErr(w, http.StatusBadRequest, "survivor_not_in_group")
|
||||||
|
default:
|
||||||
|
h.logger.Error("admin: merge duplicate group", "group_id", uuidToString(groupID), "err", err)
|
||||||
|
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
resp := mergeDuplicateResponse{
|
||||||
|
SurvivorTrackID: uuidToString(res.Survivor.TrackID),
|
||||||
|
RemovedPaths: make([]string, 0, len(res.Removed)),
|
||||||
|
}
|
||||||
|
for _, c := range res.Removed {
|
||||||
|
resp.RemovedPaths = append(resp.RemovedPaths, c.FilePath)
|
||||||
|
}
|
||||||
|
if body.Unmonitor && unmonitorFailed {
|
||||||
|
failed := true
|
||||||
|
resp.LidarrUnmonitorFailed = &failed
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, resp)
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
)
|
||||||
|
|
||||||
|
func dupUUID(b byte) pgtype.UUID {
|
||||||
|
var u pgtype.UUID
|
||||||
|
u.Bytes[15] = b
|
||||||
|
u.Valid = true
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
|
||||||
|
func dupTS(t time.Time) pgtype.Timestamptz { return pgtype.Timestamptz{Time: t, Valid: true} }
|
||||||
|
|
||||||
|
// Rows arrive one per member, members of a group together. The fold must keep
|
||||||
|
// groups apart, keep the query's order, and propose each group's survivor from
|
||||||
|
// its own members only.
|
||||||
|
func TestFoldDuplicateGroups(t *testing.T) {
|
||||||
|
older := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
newer := older.Add(48 * time.Hour)
|
||||||
|
ber := float32(0.04)
|
||||||
|
rows := []dbq.ListPendingDuplicateGroupMembersRow{
|
||||||
|
// Group 1: identical audio, sizes tie, the older copy should be kept.
|
||||||
|
{GroupID: dupUUID(1), Tier: "exact", DetectedAt: dupTS(newer), TrackID: dupUUID(10),
|
||||||
|
Title: "WWW", FileFormat: "mp3", FileSize: 6_900_000, DurationMs: 215_400, AddedAt: dupTS(newer), PlayCount: 3},
|
||||||
|
{GroupID: dupUUID(1), Tier: "exact", DetectedAt: dupTS(newer), TrackID: dupUUID(11),
|
||||||
|
Title: "WWW", FileFormat: "mp3", FileSize: 6_900_000, DurationMs: 215_400, AddedAt: dupTS(older), LikeCount: 1},
|
||||||
|
// Group 2: the same recording, FLAC against MP3.
|
||||||
|
{GroupID: dupUUID(2), Tier: "acoustic", WorstBitErrorRate: &ber, DetectedAt: dupTS(older), TrackID: dupUUID(20),
|
||||||
|
Title: "Lovesick", FileFormat: "mp3", FileSize: 9_000_000, DurationMs: 198_000, AddedAt: dupTS(older)},
|
||||||
|
{GroupID: dupUUID(2), Tier: "acoustic", WorstBitErrorRate: &ber, DetectedAt: dupTS(older), TrackID: dupUUID(21),
|
||||||
|
Title: "Lovesick", FileFormat: "flac", FileSize: 30_000_000, DurationMs: 198_000, AddedAt: dupTS(newer)},
|
||||||
|
}
|
||||||
|
|
||||||
|
got := foldDuplicateGroups(rows)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("folded %d groups, want 2", len(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
g1, g2 := got[0], got[1]
|
||||||
|
if g1.ID != uuidToString(dupUUID(1)) || len(g1.Members) != 2 || g1.WorstBitErrorRate != nil {
|
||||||
|
t.Fatalf("group 1 = %+v, want the exact pair with no score", g1)
|
||||||
|
}
|
||||||
|
if g1.SurvivorTrackID != uuidToString(dupUUID(11)) || g1.SurvivorReason != "in the library longest" {
|
||||||
|
t.Errorf("group 1 survivor = (%s, %q), want the older copy", g1.SurvivorTrackID, g1.SurvivorReason)
|
||||||
|
}
|
||||||
|
if g1.Members[0].DurationSec != 215 || g1.Members[0].PlayCount != 3 || g1.Members[1].LikeCount != 1 {
|
||||||
|
t.Errorf("group 1 members lost their facts: %+v", g1.Members)
|
||||||
|
}
|
||||||
|
|
||||||
|
if g2.Tier != "acoustic" || g2.WorstBitErrorRate == nil || *g2.WorstBitErrorRate != ber {
|
||||||
|
t.Fatalf("group 2 = %+v, want the acoustic pair with its score", g2)
|
||||||
|
}
|
||||||
|
// Chosen from group 2's own members: a survivor leaking across groups is
|
||||||
|
// exactly what a wrong fold boundary would produce.
|
||||||
|
if g2.SurvivorTrackID != uuidToString(dupUUID(21)) || g2.SurvivorReason != "lossless (flac)" {
|
||||||
|
t.Errorf("group 2 survivor = (%s, %q), want the FLAC copy", g2.SurvivorTrackID, g2.SurvivorReason)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fingerprintSettingsBody is the wire shape for GET and PUT
|
||||||
|
// /api/admin/library/fingerprint-settings (M400 #3913). The threshold travels as
|
||||||
|
// the bit-error rate the matcher uses; the card presents it as a match percentage.
|
||||||
|
type fingerprintSettingsBody struct {
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
ChromaprintLengthSec int32 `json:"chromaprint_length_sec"`
|
||||||
|
AcousticMaxBitErrorRate float64 `json:"acoustic_max_bit_error_rate"`
|
||||||
|
BackfillConcurrency int32 `json:"backfill_concurrency"`
|
||||||
|
SweepIntervalHours int32 `json:"sweep_interval_hours"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func fingerprintSettingsBodyOf(s library.FingerprintSettings) fingerprintSettingsBody {
|
||||||
|
return fingerprintSettingsBody{
|
||||||
|
Enabled: s.Enabled,
|
||||||
|
ChromaprintLengthSec: s.ChromaprintLengthSec,
|
||||||
|
AcousticMaxBitErrorRate: s.AcousticMaxBitErrorRate,
|
||||||
|
BackfillConcurrency: s.BackfillConcurrency,
|
||||||
|
SweepIntervalHours: s.SweepIntervalHours,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleGetFingerprintSettings implements GET /api/admin/library/fingerprint-settings.
|
||||||
|
func (h *handlers) handleGetFingerprintSettings(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
writeJSON(w, http.StatusOK, fingerprintSettingsBodyOf(h.fingerprintSettings.Get()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleUpdateFingerprintSettings implements PUT /api/admin/library/fingerprint-settings.
|
||||||
|
//
|
||||||
|
// A whole-row write. A body that leaves a field out decodes it as zero, which no
|
||||||
|
// field accepts, so a partial save is refused rather than zeroing what it omitted.
|
||||||
|
// The saved settings reach the scanner and both workers at once: they share the
|
||||||
|
// service instance.
|
||||||
|
func (h *handlers) handleUpdateFingerprintSettings(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req fingerprintSettingsBody
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
|
writeErr(w, apierror.BadRequest("invalid_body", "malformed JSON"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
saved, err := h.fingerprintSettings.Set(r.Context(), library.FingerprintSettings{
|
||||||
|
Enabled: req.Enabled,
|
||||||
|
ChromaprintLengthSec: req.ChromaprintLengthSec,
|
||||||
|
AcousticMaxBitErrorRate: req.AcousticMaxBitErrorRate,
|
||||||
|
BackfillConcurrency: req.BackfillConcurrency,
|
||||||
|
SweepIntervalHours: req.SweepIntervalHours,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
// Validation mirrors migration 0061's CHECKs and names the field.
|
||||||
|
if errors.Is(err, library.ErrFingerprintSettingOutOfRange) {
|
||||||
|
writeErr(w, apierror.BadRequest("invalid_setting", err.Error()))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeErrWithLog(w, h.logger, "admin fingerprint settings: update failed", apierror.Internal(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, fingerprintSettingsBodyOf(saved))
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetFingerprintSettings_ServesDefaultsWithoutAService(t *testing.T) {
|
||||||
|
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
h.handleGetFingerprintSettings(rec, httptest.NewRequest(http.MethodGet, "/api/admin/library/fingerprint-settings", nil))
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want 200", rec.Code)
|
||||||
|
}
|
||||||
|
var got fingerprintSettingsBody
|
||||||
|
if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if want := fingerprintSettingsBodyOf(library.DefaultFingerprintSettings); got != want {
|
||||||
|
t.Fatalf("body = %+v, want the defaults %+v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateFingerprintSettings_Rejects(t *testing.T) {
|
||||||
|
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
|
||||||
|
for name, tc := range map[string]struct {
|
||||||
|
body string
|
||||||
|
code string
|
||||||
|
mentions string
|
||||||
|
}{
|
||||||
|
"a value out of range, naming the field": {
|
||||||
|
body: `{"enabled":true,"chromaprint_length_sec":5,"acoustic_max_bit_error_rate":0.15,"backfill_concurrency":2,"sweep_interval_hours":1}`,
|
||||||
|
code: "invalid_setting",
|
||||||
|
mentions: "chromaprint_length_sec",
|
||||||
|
},
|
||||||
|
// A partial body would otherwise zero every field it left out.
|
||||||
|
"a body missing fields": {body: `{"enabled":false}`, code: "invalid_setting"},
|
||||||
|
"malformed JSON": {body: `{"enabled":`, code: "invalid_body"},
|
||||||
|
} {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
h.handleUpdateFingerprintSettings(rec, httptest.NewRequest(
|
||||||
|
http.MethodPut, "/api/admin/library/fingerprint-settings", strings.NewReader(tc.body)))
|
||||||
|
body := rec.Body.String()
|
||||||
|
if rec.Code != http.StatusBadRequest || !strings.Contains(body, `"`+tc.code+`"`) || !strings.Contains(body, tc.mentions) {
|
||||||
|
t.Errorf("%s: status %d body %s; want 400 %s mentioning %q", name, rec.Code, body, tc.code, tc.mentions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -133,6 +133,13 @@ func (h *handlers) handleDeleteQuarantineFile(w http.ResponseWriter, r *http.Req
|
|||||||
}
|
}
|
||||||
action, err := h.lidarrQuarantine.DeleteFile(r.Context(), id, admin.ID)
|
action, err := h.lidarrQuarantine.DeleteFile(r.Context(), id, admin.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
// Written in the enveloped shape, not writeAdminJSONErr's bare code: the
|
||||||
|
// message is the part that tells the operator which directory and uid.
|
||||||
|
if apiErr, ok := fileRemoveAPIError(err); ok {
|
||||||
|
logFileRemoveFailure(h.logger, apiErr, "track_id", uuidToString(id))
|
||||||
|
writeErr(w, apiErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, lidarrquarantine.ErrTrackNotFound):
|
case errors.Is(err, lidarrquarantine.ErrTrackNotFound):
|
||||||
writeAdminJSONErr(w, http.StatusNotFound, "track_not_found")
|
writeAdminJSONErr(w, http.StatusNotFound, "track_not_found")
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ func installQuarantineClientFn(t *testing.T, h *handlers) {
|
|||||||
}
|
}
|
||||||
return lidarr.NewClient(c.BaseURL, c.APIKey)
|
return lidarr.NewClient(c.BaseURL, c.APIKey)
|
||||||
}
|
}
|
||||||
h.lidarrQuarantine = lidarrquarantine.NewService(h.pool, cfg, clientFn)
|
h.lidarrQuarantine = lidarrquarantine.NewService(h.pool, cfg, clientFn, h.dataDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
// flagDirect bypasses the HTTP handler to seed a quarantine row via the
|
// flagDirect bypasses the HTTP handler to seed a quarantine row via the
|
||||||
|
|||||||
@@ -97,14 +97,16 @@ type tuningSnapshot struct {
|
|||||||
func (h *handlers) tuningSnapshot() tuningSnapshot {
|
func (h *handlers) tuningSnapshot() tuningSnapshot {
|
||||||
var out tuningSnapshot
|
var out tuningSnapshot
|
||||||
out.Profiles = map[string]weightsResp{
|
out.Profiles = map[string]weightsResp{
|
||||||
recsettings.ScopeRadio: weightsRespFrom(h.recSettings.Weights(recsettings.ScopeRadio)),
|
recsettings.ScopeRadio: weightsRespFrom(h.recSettings.Weights(recsettings.ScopeRadio)),
|
||||||
recsettings.ScopeDailyMix: weightsRespFrom(h.recSettings.Weights(recsettings.ScopeDailyMix)),
|
recsettings.ScopeDailyMix: weightsRespFrom(h.recSettings.Weights(recsettings.ScopeDailyMix)),
|
||||||
|
recsettings.ScopeSongsLike: weightsRespFrom(h.recSettings.Weights(recsettings.ScopeSongsLike)),
|
||||||
}
|
}
|
||||||
out.Taste = tasteRespFrom(h.recSettings.Taste())
|
out.Taste = tasteRespFrom(h.recSettings.Taste())
|
||||||
out.Discover = discoverRespFrom(h.recSettings.Discover())
|
out.Discover = discoverRespFrom(h.recSettings.Discover())
|
||||||
out.Shipped.Profiles = map[string]weightsResp{
|
out.Shipped.Profiles = map[string]weightsResp{
|
||||||
recsettings.ScopeRadio: weightsRespFrom(recsettings.ShippedRadioWeights()),
|
recsettings.ScopeRadio: weightsRespFrom(recsettings.ShippedRadioWeights()),
|
||||||
recsettings.ScopeDailyMix: weightsRespFrom(recsettings.ShippedDailyMixWeights()),
|
recsettings.ScopeDailyMix: weightsRespFrom(recsettings.ShippedDailyMixWeights()),
|
||||||
|
recsettings.ScopeSongsLike: weightsRespFrom(recsettings.ShippedSongsLikeWeights()),
|
||||||
}
|
}
|
||||||
out.Shipped.Taste = tasteRespFrom(recsettings.ShippedTasteTuning())
|
out.Shipped.Taste = tasteRespFrom(recsettings.ShippedTasteTuning())
|
||||||
out.Shipped.Discover = discoverRespFrom(recsettings.ShippedDiscoverTuning())
|
out.Shipped.Discover = discoverRespFrom(recsettings.ShippedDiscoverTuning())
|
||||||
|
|||||||
@@ -23,15 +23,17 @@ type removeTrackResponse struct {
|
|||||||
|
|
||||||
// handleRemoveTrack implements DELETE /api/admin/tracks/{id}?unmonitor=true|false.
|
// handleRemoveTrack implements DELETE /api/admin/tracks/{id}?unmonitor=true|false.
|
||||||
//
|
//
|
||||||
// Admin-only (gated by auth.RequireAdmin on the /admin route group). Always
|
// Admin-only (gated by auth.RequireAdmin on the /admin route group). Deletes the
|
||||||
// deletes the file + DB row and runs the album/artist cascade tidy-up. When
|
// file, then the DB row, and runs the album/artist cascade tidy-up — and deletes
|
||||||
|
// nothing at all when the file cannot be removed (#3918). When
|
||||||
// unmonitor=true and the track has an mbid, also calls Lidarr.UnmonitorTrack
|
// unmonitor=true and the track has an mbid, also calls Lidarr.UnmonitorTrack
|
||||||
// — failure there is non-fatal (the destructive part already completed) and
|
// — failure there is non-fatal (the destructive part already completed) and
|
||||||
// surfaces as `lidarr_unmonitor_failed: true` in the success envelope.
|
// surfaces as `lidarr_unmonitor_failed: true` in the success envelope.
|
||||||
//
|
//
|
||||||
// Per spec §5, Lidarr-side errors during the unmonitor step do NOT map to
|
// Per spec §5, Lidarr-side errors during the unmonitor step do NOT map to
|
||||||
// wire error codes; the only error codes this handler emits are not_found,
|
// wire error codes. The codes this handler emits are not_found,
|
||||||
// server_error, plus the auth codes the middleware emits upstream.
|
// library_not_writable (409) and file_delete_failed when the file could not be
|
||||||
|
// removed, server_error, plus the auth codes the middleware emits upstream.
|
||||||
func (h *handlers) handleRemoveTrack(w http.ResponseWriter, r *http.Request) {
|
func (h *handlers) handleRemoveTrack(w http.ResponseWriter, r *http.Request) {
|
||||||
idStr := chi.URLParam(r, "id")
|
idStr := chi.URLParam(r, "id")
|
||||||
trackID, ok := parseUUID(idStr)
|
trackID, ok := parseUUID(idStr)
|
||||||
@@ -66,6 +68,11 @@ func (h *handlers) handleRemoveTrack(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeErr(w, &apierror.Error{Status: http.StatusNotFound, Code: "not_found", Message: "track not found"})
|
writeErr(w, &apierror.Error{Status: http.StatusNotFound, Code: "not_found", Message: "track not found"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if apiErr, ok := fileRemoveAPIError(err); ok {
|
||||||
|
logFileRemoveFailure(h.logger, apiErr, "track_id", idStr)
|
||||||
|
writeErr(w, apiErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
h.logger.Error("api: remove track failed", "err", err, "track_id", idStr)
|
h.logger.Error("api: remove track failed", "err", err, "track_id", idStr)
|
||||||
writeErr(w, apierror.InternalMsg("remove failed", err))
|
writeErr(w, apierror.InternalMsg("remove failed", err))
|
||||||
return
|
return
|
||||||
|
|||||||
+61
-26
@@ -7,6 +7,7 @@ package api
|
|||||||
import (
|
import (
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
@@ -24,6 +25,7 @@ import (
|
|||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/playevents"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/playevents"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/playlists"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/playlists"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/reacquisition"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/reacquisition"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/recommendation"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/recsettings"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/recsettings"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/tags"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/tags"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/tracks"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/tracks"
|
||||||
@@ -32,29 +34,36 @@ import (
|
|||||||
// Mount attaches /api/* handlers to r. Public endpoints (login) are outside
|
// Mount attaches /api/* handlers to r. Public endpoints (login) are outside
|
||||||
// RequireUser; everything else is gated by the middleware. The events writer
|
// RequireUser; everything else is gated by the middleware. The events writer
|
||||||
// is shared with the Subsonic mount so /rest/scrobble feeds the same store.
|
// is shared with the Subsonic mount so /rest/scrobble feeds the same store.
|
||||||
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService) {
|
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService) {
|
||||||
rng := rand.New(rand.NewSource(rand.Int63()))
|
rng := rand.New(rand.NewSource(rand.Int63()))
|
||||||
h := &handlers{
|
h := &handlers{
|
||||||
pool: pool, logger: logger, events: events, recCfg: recCfg,
|
pool: pool, logger: logger, events: events, recCfg: recCfg,
|
||||||
recSettings: recSettings,
|
recSettings: recSettings,
|
||||||
rng: rng.Float64,
|
rng: rng.Float64,
|
||||||
lidarrCfg: lidarrCfg,
|
lidarrCfg: lidarrCfg,
|
||||||
lidarrRequests: lidarrReqs,
|
lidarrRequests: lidarrReqs,
|
||||||
lidarrQuarantine: lidarrQuar,
|
lidarrQuarantine: lidarrQuar,
|
||||||
tracks: tracksSvc,
|
tracks: tracksSvc,
|
||||||
playlists: playlistsSvc,
|
playlists: playlistsSvc,
|
||||||
coverart: coverEnricher,
|
coverart: coverEnricher,
|
||||||
coverSettings: coverSettings,
|
coverSettings: coverSettings,
|
||||||
tagSettings: tagSettings,
|
tagSettings: tagSettings,
|
||||||
scanner: scanner,
|
scanner: scanner,
|
||||||
scanCfg: scanCfg,
|
scanCfg: scanCfg,
|
||||||
dataDir: dataDir,
|
dataDir: dataDir,
|
||||||
mailer: sender,
|
mailer: sender,
|
||||||
eventbus: bus,
|
eventbus: bus,
|
||||||
playlistScheduler: playlistScheduler,
|
playlistScheduler: playlistScheduler,
|
||||||
streamSecret: streamSecret,
|
streamSecret: streamSecret,
|
||||||
netSettings: netSettings,
|
netSettings: netSettings,
|
||||||
reacqSettings: reacqSettings,
|
reacqSettings: reacqSettings,
|
||||||
|
fingerprintSettings: fpSettings,
|
||||||
|
librarySize: recommendation.NewLibrarySize(nil),
|
||||||
|
loginGuard: auth.NewLoginGuard(),
|
||||||
|
registerLimit: auth.NewAttemptLimiter(registerPerAddressMax, time.Hour),
|
||||||
|
forgotAddressLimit: auth.NewAttemptLimiter(forgotPerAddressMax, time.Hour),
|
||||||
|
forgotEmailLimit: auth.NewAttemptLimiter(forgotPerEmailMax, time.Hour),
|
||||||
|
resetLimit: auth.NewAttemptLimiter(resetFailuresPerAddressMax, 15*time.Minute),
|
||||||
}
|
}
|
||||||
|
|
||||||
r.Route("/api", func(api chi.Router) {
|
r.Route("/api", func(api chi.Router) {
|
||||||
@@ -213,6 +222,16 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
|||||||
admin.Get("/library/missing", h.handleListMissingTracks)
|
admin.Get("/library/missing", h.handleListMissingTracks)
|
||||||
|
|
||||||
admin.Get("/library/coverage", h.handleGetLibraryCoverage)
|
admin.Get("/library/coverage", h.handleGetLibraryCoverage)
|
||||||
|
admin.Get("/library/fingerprints", h.handleGetFingerprintCoverage)
|
||||||
|
admin.Get("/library/fingerprint-settings", h.handleGetFingerprintSettings)
|
||||||
|
admin.Put("/library/fingerprint-settings", h.handleUpdateFingerprintSettings)
|
||||||
|
// Duplicates report (#3912): proposals from the duplicate sweep, a
|
||||||
|
// trigger to sweep now, dismissal, and the merge (#3911), which deletes
|
||||||
|
// the removed copies' files after moving their history onto the kept one.
|
||||||
|
admin.Get("/library/duplicates", h.handleListDuplicates)
|
||||||
|
admin.Post("/library/duplicates/sweep", h.handleRunDuplicateSweep)
|
||||||
|
admin.Post("/library/duplicates/{id}/dismiss", h.handleDismissDuplicateGroup)
|
||||||
|
admin.Post("/library/duplicates/{id}/merge", h.handleMergeDuplicateGroup)
|
||||||
|
|
||||||
admin.Get("/invites", h.handleListInvites)
|
admin.Get("/invites", h.handleListInvites)
|
||||||
admin.Post("/invites", h.handleCreateInvite)
|
admin.Post("/invites", h.handleCreateInvite)
|
||||||
@@ -268,12 +287,16 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
|||||||
}
|
}
|
||||||
|
|
||||||
type handlers struct {
|
type handlers struct {
|
||||||
pool *pgxpool.Pool
|
pool *pgxpool.Pool
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
events *playevents.Writer
|
events *playevents.Writer
|
||||||
recCfg config.RecommendationConfig
|
recCfg config.RecommendationConfig
|
||||||
recSettings *recsettings.Service
|
recSettings *recsettings.Service
|
||||||
rng func() float64
|
rng func() float64
|
||||||
|
// librarySize memoises the track count that sizes the candidate pool
|
||||||
|
// (#3880). Held here rather than counted per request: the count is a
|
||||||
|
// full table scan, and library size only moves when a scan runs.
|
||||||
|
librarySize *recommendation.LibrarySize
|
||||||
lidarrCfg *lidarrconfig.Service
|
lidarrCfg *lidarrconfig.Service
|
||||||
lidarrRequests *lidarrrequests.Service
|
lidarrRequests *lidarrrequests.Service
|
||||||
lidarrQuarantine *lidarrquarantine.Service
|
lidarrQuarantine *lidarrquarantine.Service
|
||||||
@@ -292,6 +315,18 @@ type handlers struct {
|
|||||||
// missing files (milestone #290) — grace window, backoff, attempt caps.
|
// missing files (milestone #290) — grace window, backoff, attempt caps.
|
||||||
// Cached in the service, so the admin card reads it without a query.
|
// Cached in the service, so the admin card reads it without a query.
|
||||||
reacqSettings *reacquisition.SettingsService
|
reacqSettings *reacquisition.SettingsService
|
||||||
|
// fingerprintSettings is the fingerprinting policy (M400 #3913), the same
|
||||||
|
// instance the scanner and the fingerprint workers read, so a save from the
|
||||||
|
// admin card reaches them without a restart. Nil serves the defaults.
|
||||||
|
fingerprintSettings *library.FingerprintSettingsService
|
||||||
|
// loginGuard throttles failed logins per account and per address, and
|
||||||
|
// the limiters below cap the other unauthenticated auth routes. All are
|
||||||
|
// nil-safe, so tests that build handlers directly run unthrottled.
|
||||||
|
loginGuard *auth.LoginGuard
|
||||||
|
registerLimit *auth.AttemptLimiter
|
||||||
|
forgotAddressLimit *auth.AttemptLimiter
|
||||||
|
forgotEmailLimit *auth.AttemptLimiter
|
||||||
|
resetLimit *auth.AttemptLimiter
|
||||||
// netSettings caches the trusted reverse-proxy depth read by the auth
|
// netSettings caches the trusted reverse-proxy depth read by the auth
|
||||||
// middleware on every request and edited from the admin network card.
|
// middleware on every request and edited from the admin network card.
|
||||||
netSettings *netsettings.Service
|
netSettings *netsettings.Service
|
||||||
|
|||||||
+29
-1
@@ -19,6 +19,20 @@ import (
|
|||||||
// so an abandoned laptop doesn't stay logged in forever.
|
// so an abandoned laptop doesn't stay logged in forever.
|
||||||
const sessionCookieMaxAge = 30 * 24 * time.Hour
|
const sessionCookieMaxAge = 30 * 24 * time.Hour
|
||||||
|
|
||||||
|
// Limits on the unauthenticated auth routes other than login (which uses
|
||||||
|
// auth.LoginGuard). Per address, per window as wired in Mount.
|
||||||
|
const (
|
||||||
|
// registerPerAddressMax: 10 registrations an hour. Every attempt counts,
|
||||||
|
// typos included, which is still far above a household's need.
|
||||||
|
registerPerAddressMax = 10
|
||||||
|
// forgotPerAddressMax / forgotPerEmailMax: reset emails an hour. The
|
||||||
|
// per-email cap is what keeps one inbox from being mailbombed.
|
||||||
|
forgotPerAddressMax = 5
|
||||||
|
forgotPerEmailMax = 3
|
||||||
|
// resetFailuresPerAddressMax: wrong or expired reset tokens per 15 min.
|
||||||
|
resetFailuresPerAddressMax = 20
|
||||||
|
)
|
||||||
|
|
||||||
func (h *handlers) handleLogout(w http.ResponseWriter, r *http.Request) {
|
func (h *handlers) handleLogout(w http.ResponseWriter, r *http.Request) {
|
||||||
// The session token can be on the cookie OR bearer header — RequireUser
|
// The session token can be on the cookie OR bearer header — RequireUser
|
||||||
// accepted either. Re-resolve it here so we can delete the row.
|
// accepted either. Re-resolve it here so we can delete the row.
|
||||||
@@ -70,10 +84,22 @@ func (h *handlers) handleLogin(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Checked before any lookup or bcrypt, so a throttled guess costs the
|
||||||
|
// server nothing and learns nothing.
|
||||||
|
addr := auth.ClientIP(r, h.netSettings.Hops())
|
||||||
|
if blocked, wait := h.loginGuard.Blocked(req.Username, addr); blocked {
|
||||||
|
writeRateLimited(w, wait)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
q := dbq.New(h.pool)
|
q := dbq.New(h.pool)
|
||||||
user, err := q.GetUserByUsername(r.Context(), req.Username)
|
user, err := q.GetUserByUsername(r.Context(), req.Username)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
// Same bcrypt time as a wrong password, so timing doesn't say
|
||||||
|
// which usernames exist.
|
||||||
|
auth.DummyVerify(req.Password)
|
||||||
|
h.loginGuard.Fail(req.Username, addr)
|
||||||
writeErr(w, apierror.Unauthorized("invalid_credentials", "invalid username or password"))
|
writeErr(w, apierror.Unauthorized("invalid_credentials", "invalid username or password"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -82,9 +108,11 @@ func (h *handlers) handleLogin(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !auth.VerifyPassword(user.PasswordHash, req.Password) {
|
if !auth.VerifyPassword(user.PasswordHash, req.Password) {
|
||||||
|
h.loginGuard.Fail(req.Username, addr)
|
||||||
writeErr(w, apierror.Unauthorized("invalid_credentials", "invalid username or password"))
|
writeErr(w, apierror.Unauthorized("invalid_credentials", "invalid username or password"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
h.loginGuard.Succeed(req.Username)
|
||||||
|
|
||||||
token, err := auth.MintSessionToken()
|
token, err := auth.MintSessionToken()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -100,7 +128,7 @@ func (h *handlers) handleLogin(w http.ResponseWriter, r *http.Request) {
|
|||||||
// the active-sessions surface: a session that was born somewhere the
|
// the active-sessions surface: a session that was born somewhere the
|
||||||
// user recognises but is being used from somewhere they don't is the
|
// user recognises but is being used from somewhere they don't is the
|
||||||
// case this whole surface exists to surface.
|
// case this whole surface exists to surface.
|
||||||
Ip: auth.ClientIP(r, h.netSettings.Hops()),
|
Ip: addr,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
h.logger.Error("api: insert session failed", "err", err)
|
h.logger.Error("api: insert session failed", "err", err)
|
||||||
writeErr(w, apierror.InternalMsg("insert failed", err))
|
writeErr(w, apierror.InternalMsg("insert failed", err))
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/jackc/pgx/v5/pgtype"
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
||||||
)
|
)
|
||||||
@@ -47,6 +48,19 @@ func (h *handlers) handleForgotPassword(w http.ResponseWriter, r *http.Request)
|
|||||||
}
|
}
|
||||||
email := strings.ToLower(strings.TrimSpace(req.Email))
|
email := strings.ToLower(strings.TrimSpace(req.Email))
|
||||||
|
|
||||||
|
// Throttled per address (a spray) and per email (a mailbomb of one
|
||||||
|
// inbox). Applied whether or not the email matches, so a 429 says
|
||||||
|
// nothing about which addresses are registered.
|
||||||
|
addr := auth.ClientIP(r, h.netSettings.Hops())
|
||||||
|
blockedAddr, waitAddr := h.forgotAddressLimit.Blocked(addr)
|
||||||
|
blockedEmail, waitEmail := h.forgotEmailLimit.Blocked(email)
|
||||||
|
if blockedAddr || blockedEmail {
|
||||||
|
writeRateLimited(w, max(waitAddr, waitEmail))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.forgotAddressLimit.Record(addr)
|
||||||
|
h.forgotEmailLimit.Record(email)
|
||||||
|
|
||||||
q := dbq.New(h.pool)
|
q := dbq.New(h.pool)
|
||||||
var matched bool
|
var matched bool
|
||||||
var auditTarget pgtype.UUID
|
var auditTarget pgtype.UUID
|
||||||
|
|||||||
@@ -55,6 +55,15 @@ type registerReq struct {
|
|||||||
// - 409 username_taken (PG unique violation on users.username)
|
// - 409 username_taken (PG unique violation on users.username)
|
||||||
// - 500 server_error otherwise
|
// - 500 server_error otherwise
|
||||||
func (h *handlers) handleRegister(w http.ResponseWriter, r *http.Request) {
|
func (h *handlers) handleRegister(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// Every attempt counts, not only failures: a successful registration is
|
||||||
|
// the thing being sprayed when the mode is open.
|
||||||
|
addr := auth.ClientIP(r, h.netSettings.Hops())
|
||||||
|
if blocked, wait := h.registerLimit.Blocked(addr); blocked {
|
||||||
|
writeRateLimited(w, wait)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.registerLimit.Record(addr)
|
||||||
|
|
||||||
var req registerReq
|
var req registerReq
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
writeErr(w, apierror.BadRequest("invalid_body", "invalid JSON body"))
|
writeErr(w, apierror.BadRequest("invalid_body", "invalid JSON body"))
|
||||||
@@ -175,7 +184,7 @@ func (h *handlers) handleRegister(w http.ResponseWriter, r *http.Request) {
|
|||||||
UserID: user.ID,
|
UserID: user.ID,
|
||||||
TokenHash: auth.HashSessionToken(sessionToken),
|
TokenHash: auth.HashSessionToken(sessionToken),
|
||||||
UserAgent: r.UserAgent(),
|
UserAgent: r.UserAgent(),
|
||||||
Ip: auth.ClientIP(r, h.netSettings.Hops()),
|
Ip: addr,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
h.logger.Error("register: insert session failed", "err", err)
|
h.logger.Error("register: insert session failed", "err", err)
|
||||||
writeErr(w, apierror.Internal(err))
|
writeErr(w, apierror.Internal(err))
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -42,6 +43,15 @@ func (h *handlers) handleResetPassword(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Tokens carry 256 bits, so guessing one is hopeless; the limit is on
|
||||||
|
// failed tries per address so that nobody gets to find that out at our
|
||||||
|
// expense.
|
||||||
|
addr := auth.ClientIP(r, h.netSettings.Hops())
|
||||||
|
if blocked, wait := h.resetLimit.Blocked(addr); blocked {
|
||||||
|
writeRateLimited(w, wait)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
q := dbq.New(h.pool)
|
q := dbq.New(h.pool)
|
||||||
|
|
||||||
// Look up the reset record so we know which user to update before
|
// Look up the reset record so we know which user to update before
|
||||||
@@ -50,6 +60,7 @@ func (h *handlers) handleResetPassword(w http.ResponseWriter, r *http.Request) {
|
|||||||
reset, err := q.GetPasswordReset(r.Context(), req.Token)
|
reset, err := q.GetPasswordReset(r.Context(), req.Token)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
h.resetLimit.Record(addr)
|
||||||
writeErr(w, apierror.BadRequest("invalid_token", ""))
|
writeErr(w, apierror.BadRequest("invalid_token", ""))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -69,6 +80,7 @@ func (h *handlers) handleResetPassword(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if rows == 0 {
|
if rows == 0 {
|
||||||
|
h.resetLimit.Record(addr)
|
||||||
writeErr(w, apierror.BadRequest("invalid_token", ""))
|
writeErr(w, apierror.BadRequest("invalid_token", ""))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func testHandlers(t *testing.T) (*handlers, *pgxpool.Pool) {
|
|||||||
}
|
}
|
||||||
lidarrCfg := lidarrconfig.New(pool)
|
lidarrCfg := lidarrconfig.New(pool)
|
||||||
lidarrReqs := lidarrrequests.NewService(pool, lidarrCfg, nil, nil)
|
lidarrReqs := lidarrrequests.NewService(pool, lidarrCfg, nil, nil)
|
||||||
lidarrQuar := lidarrquarantine.NewService(pool, lidarrCfg, nil)
|
lidarrQuar := lidarrquarantine.NewService(pool, lidarrCfg, nil, "")
|
||||||
// tracks.Service has no Lidarr unmonitorer in tests by default; the
|
// tracks.Service has no Lidarr unmonitorer in tests by default; the
|
||||||
// admin-tracks tests below override h.tracks via installTracksLidarrStub
|
// admin-tracks tests below override h.tracks via installTracksLidarrStub
|
||||||
// when they need a stubbed Lidarr.
|
// when they need a stubbed Lidarr.
|
||||||
@@ -179,6 +179,36 @@ func TestHandleLogin_UnknownUserReturns401(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A guesser who reaches the account limit is refused with 429 before any
|
||||||
|
// password check, so even the right password is turned away until the window
|
||||||
|
// passes, and the response says when to come back.
|
||||||
|
func TestHandleLogin_ThrottledAfterRepeatedFailures(t *testing.T) {
|
||||||
|
h, pool := testHandlers(t)
|
||||||
|
h.loginGuard = auth.NewLoginGuard()
|
||||||
|
seedUser(t, pool, "alice", "hunter2", false)
|
||||||
|
|
||||||
|
login := func(password string) *httptest.ResponseRecorder {
|
||||||
|
body := strings.NewReader(`{"username":"test-alice","password":"` + password + `"}`)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/api/auth/login", body)
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.handleLogin(w, req)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
if w := login("wrong"); w.Code != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("attempt %d: status = %d, want 401", i+1, w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w := login("hunter2")
|
||||||
|
if w.Code != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("status = %d, want 429 once the account limit is reached", w.Code)
|
||||||
|
}
|
||||||
|
if w.Header().Get("Retry-After") == "" {
|
||||||
|
t.Error("429 without Retry-After")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandleLogin_MalformedBodyReturns400(t *testing.T) {
|
func TestHandleLogin_MalformedBodyReturns400(t *testing.T) {
|
||||||
h, _ := testHandlers(t)
|
h, _ := testHandlers(t)
|
||||||
req := httptest.NewRequest(http.MethodPost, "/api/auth/login",
|
req := httptest.NewRequest(http.MethodPost, "/api/auth/login",
|
||||||
|
|||||||
@@ -6,19 +6,21 @@ package api
|
|||||||
// /app/client/ at image build time.
|
// /app/client/ at image build time.
|
||||||
//
|
//
|
||||||
// Both endpoints are authenticated — the bandwidth cost of the APK
|
// Both endpoints are authenticated — the bandwidth cost of the APK
|
||||||
// (~30-60 MB) makes anonymous access an abuse vector. The Flutter
|
// (~30-60 MB) makes anonymous access an abuse vector. The client only
|
||||||
// client's polling only fires after login (banner mounts in the post-
|
// polls after login, so this gate is invisible to the actual update flow.
|
||||||
// login shell), so this gate is invisible to the actual update flow.
|
|
||||||
//
|
//
|
||||||
// /api/client/apk additionally rate-limits per user to a single
|
// /api/client/apk additionally rate-limits per user to a single
|
||||||
// download every 60s. Real install flows fire one download per
|
// download every 60s. Real install flows fire one download per
|
||||||
// update; anything tighter is scripted/abusive.
|
// update; anything tighter is scripted/abusive.
|
||||||
//
|
//
|
||||||
// Returns 404 gracefully when the APK isn't present (dev environments,
|
// Returns 404 gracefully when the APK isn't present (dev environments,
|
||||||
// pre-CI-wiring); the Flutter client treats 404 as "no update channel
|
// pre-CI-wiring); the client treats 404 as "no update channel available."
|
||||||
// available."
|
//
|
||||||
|
// (These paragraphs said "the Flutter client" until 2026-09-10. That client
|
||||||
|
// was deleted in v2026.08.18 — the Android app is the only one now.)
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
@@ -84,8 +86,36 @@ func clientAPKAllowDownload(userID string, now time.Time) time.Duration {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// clientVersionSidecar is the JSON written beside the bundled APK by
|
||||||
|
// release.yml. It carries three values that are deliberately separate:
|
||||||
|
//
|
||||||
|
// - Name is a LABEL for people, "YYYY.MM.DD.HHMM" from the commit's
|
||||||
|
// timestamp. Two channels carrying the same code report the same name.
|
||||||
|
// - Code is the ORDERING KEY, minutes since 2020-01-01 at build time, and
|
||||||
|
// is the value Android itself installs by. It answers "may this be
|
||||||
|
// installed over that?" — the name never does.
|
||||||
|
// - Channel is a SIBLING FIELD, never a suffix inside the name.
|
||||||
|
//
|
||||||
|
// JSON rather than a positional line on purpose. The obvious growth path for
|
||||||
|
// the old one-value file was "<name> <code>", which a first-space split
|
||||||
|
// silently mangles the moment a third field appears: the code stops parsing,
|
||||||
|
// and the reader falls back to name comparison WITHOUT erroring.
|
||||||
|
type clientVersionSidecar struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
// Pointer, not int64: absent must stay distinguishable from zero. An
|
||||||
|
// artifact published before codes were recorded genuinely has no code —
|
||||||
|
// zero would claim it is infinitely old rather than unknown.
|
||||||
|
Code *int64 `json:"code"`
|
||||||
|
Channel string `json:"channel"`
|
||||||
|
}
|
||||||
|
|
||||||
type clientVersionResponse struct {
|
type clientVersionResponse struct {
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
|
// omitempty on both: the client must be able to tell "this server does
|
||||||
|
// not report a code" from "this build's code is 0", because those call
|
||||||
|
// for different behaviour on the other end.
|
||||||
|
Code *int64 `json:"code,omitempty"`
|
||||||
|
Channel string `json:"channel,omitempty"`
|
||||||
APKURL string `json:"apk_url"`
|
APKURL string `json:"apk_url"`
|
||||||
SizeBytes int64 `json:"size_bytes"`
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
}
|
}
|
||||||
@@ -117,8 +147,25 @@ func (h *handlers) handleClientVersion(w http.ResponseWriter, _ *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var sidecar clientVersionSidecar
|
||||||
|
if err := json.Unmarshal(versionBytes, &sidecar); err != nil {
|
||||||
|
// Fail LOUDLY rather than serving a blank version. The failure mode
|
||||||
|
// this avoids is the one that never gets reported: if an unreadable
|
||||||
|
// sidecar produced an empty name, every client would compare against
|
||||||
|
// nothing, conclude it was current, and go quiet — "I cannot read
|
||||||
|
// this" and "there is nothing newer" would be the same answer.
|
||||||
|
writeErrWithLog(w, h.logger, "client_version: sidecar is not valid JSON", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if sidecar.Name == "" {
|
||||||
|
http.Error(w, `{"error":{"code":"bad_client_version","message":"version sidecar has no name"}}`, http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
writeJSON(w, http.StatusOK, clientVersionResponse{
|
writeJSON(w, http.StatusOK, clientVersionResponse{
|
||||||
Version: strings.TrimSpace(string(versionBytes)),
|
Version: strings.TrimSpace(sidecar.Name),
|
||||||
|
Code: sidecar.Code,
|
||||||
|
Channel: strings.TrimSpace(sidecar.Channel),
|
||||||
APKURL: "/api/client/apk",
|
APKURL: "/api/client/apk",
|
||||||
SizeBytes: stat.Size(),
|
SizeBytes: stat.Size(),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -77,18 +77,32 @@ func TestClientVersion_404WhenAPKButNoVersion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClientVersion_200WithBothFiles(t *testing.T) {
|
// writeClientAssets stages an APK plus a raw sidecar body, and returns the
|
||||||
|
// APK's size so callers can assert size_bytes without recomputing it.
|
||||||
|
func writeClientAssets(t *testing.T, sidecar string) int64 {
|
||||||
|
t.Helper()
|
||||||
dir := withClientAPKDir(t)
|
dir := withClientAPKDir(t)
|
||||||
body := []byte("fake apk content")
|
body := []byte("fake apk content")
|
||||||
if err := os.WriteFile(filepath.Join(dir, clientAPKFilename), body, 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, clientAPKFilename), body, 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(dir, clientVersionFile), []byte("v2026.05.10\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, clientVersionFile), []byte(sidecar), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
return int64(len(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
func getClientVersion(t *testing.T) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
|
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
|
||||||
rr := httptest.NewRecorder()
|
rr := httptest.NewRecorder()
|
||||||
h.handleClientVersion(rr, httptest.NewRequest(http.MethodGet, "/api/client/version", nil))
|
h.handleClientVersion(rr, httptest.NewRequest(http.MethodGet, "/api/client/version", nil))
|
||||||
|
return rr
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientVersion_200WithBothFiles(t *testing.T) {
|
||||||
|
size := writeClientAssets(t, `{"name":"2026.09.10.1432","code":3523847,"channel":"stable"}`+"\n")
|
||||||
|
rr := getClientVersion(t)
|
||||||
if rr.Code != http.StatusOK {
|
if rr.Code != http.StatusOK {
|
||||||
t.Fatalf("want 200, got %d (body: %s)", rr.Code, rr.Body.String())
|
t.Fatalf("want 200, got %d (body: %s)", rr.Code, rr.Body.String())
|
||||||
}
|
}
|
||||||
@@ -96,14 +110,69 @@ func TestClientVersion_200WithBothFiles(t *testing.T) {
|
|||||||
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if resp.Version != "v2026.05.10" {
|
if resp.Version != "2026.09.10.1432" {
|
||||||
t.Errorf("version: want trimmed v2026.05.10, got %q", resp.Version)
|
t.Errorf("version: want 2026.09.10.1432, got %q", resp.Version)
|
||||||
|
}
|
||||||
|
if resp.Code == nil {
|
||||||
|
t.Fatal("code: want 3523847, got absent — the client decides on this, so absent means it silently falls back to name comparison")
|
||||||
|
}
|
||||||
|
if *resp.Code != 3523847 {
|
||||||
|
t.Errorf("code: want 3523847, got %d", *resp.Code)
|
||||||
|
}
|
||||||
|
if resp.Channel != "stable" {
|
||||||
|
t.Errorf("channel: want stable, got %q", resp.Channel)
|
||||||
}
|
}
|
||||||
if resp.APKURL != "/api/client/apk" {
|
if resp.APKURL != "/api/client/apk" {
|
||||||
t.Errorf("apk_url: want /api/client/apk, got %q", resp.APKURL)
|
t.Errorf("apk_url: want /api/client/apk, got %q", resp.APKURL)
|
||||||
}
|
}
|
||||||
if resp.SizeBytes != int64(len(body)) {
|
if resp.SizeBytes != size {
|
||||||
t.Errorf("size_bytes: want %d, got %d", len(body), resp.SizeBytes)
|
t.Errorf("size_bytes: want %d, got %d", size, resp.SizeBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A release published before ordering keys were recorded has a name and
|
||||||
|
// genuinely no code. That must arrive as ABSENT, not as 0 — zero would claim
|
||||||
|
// the build is infinitely old and offer an update to everyone forever.
|
||||||
|
func TestClientVersion_CodeAbsentIsOmittedNotZero(t *testing.T) {
|
||||||
|
writeClientAssets(t, `{"name":"2026.09.09","code":null,"channel":"stable"}`)
|
||||||
|
rr := getClientVersion(t)
|
||||||
|
if rr.Code != http.StatusOK {
|
||||||
|
t.Fatalf("want 200, got %d (body: %s)", rr.Code, rr.Body.String())
|
||||||
|
}
|
||||||
|
var resp clientVersionResponse
|
||||||
|
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resp.Code != nil {
|
||||||
|
t.Errorf("code: want absent, got %d", *resp.Code)
|
||||||
|
}
|
||||||
|
// The wire must omit the key entirely, so a client can distinguish
|
||||||
|
// "this server reports no code" from "this build's code is 0".
|
||||||
|
var raw map[string]any
|
||||||
|
if err := json.Unmarshal(rr.Body.Bytes(), &raw); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, present := raw["code"]; present {
|
||||||
|
t.Errorf("code key should be omitted entirely, body was %s", rr.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The failure this guards is the one nobody reports: if an unreadable sidecar
|
||||||
|
// produced an empty version, every client would compare against nothing,
|
||||||
|
// decide it was current, and go quiet. "I cannot read this" and "there is
|
||||||
|
// nothing newer" must not be the same answer.
|
||||||
|
func TestClientVersion_MalformedSidecarErrorsRatherThanReportingNothing(t *testing.T) {
|
||||||
|
for _, sidecar := range []string{
|
||||||
|
"2026.09.10.1432", // the OLD plain-text format
|
||||||
|
`{"name":"x",`, // truncated JSON
|
||||||
|
`{"code":123,"channel":"dev"}`, // valid JSON, no name
|
||||||
|
"",
|
||||||
|
} {
|
||||||
|
writeClientAssets(t, sidecar)
|
||||||
|
rr := getClientVersion(t)
|
||||||
|
if rr.Code == http.StatusOK {
|
||||||
|
t.Errorf("sidecar %q: want an error status, got 200 with body %s", sidecar, rr.Body.String())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,10 @@ package api
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||||
)
|
)
|
||||||
@@ -36,6 +39,17 @@ func writeErr(w http.ResponseWriter, err error) {
|
|||||||
}})
|
}})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeRateLimited answers 429 with Retry-After in whole seconds, rounded
|
||||||
|
// up so a client that honours it never arrives a moment early.
|
||||||
|
func writeRateLimited(w http.ResponseWriter, wait time.Duration) {
|
||||||
|
secs := int(math.Ceil(wait.Seconds()))
|
||||||
|
if secs < 1 {
|
||||||
|
secs = 1
|
||||||
|
}
|
||||||
|
w.Header().Set("Retry-After", strconv.Itoa(secs))
|
||||||
|
writeErr(w, apierror.TooManyRequests("rate_limited", "too many attempts; try again later"))
|
||||||
|
}
|
||||||
|
|
||||||
// writeErrWithLog logs the error at Error level and writes the response.
|
// writeErrWithLog logs the error at Error level and writes the response.
|
||||||
// Use for 500-class errors where the operator needs the cause in logs.
|
// Use for 500-class errors where the operator needs the cause in logs.
|
||||||
func writeErrWithLog(w http.ResponseWriter, logger *slog.Logger, msg string, err error) {
|
func writeErrWithLog(w http.ResponseWriter, logger *slog.Logger, msg string, err error) {
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fileRemoveAPIError answers a delete that could not reach the track's file
|
||||||
|
// (#3918). Both delete endpoints use it, so the operator gets the same
|
||||||
|
// explanation from the admin remove-track action and from quarantine's Delete
|
||||||
|
// file.
|
||||||
|
//
|
||||||
|
// The unwritable case is a 409 rather than a 500 because nothing is broken: the
|
||||||
|
// request conflicts with how the library is mounted, and the fix is the
|
||||||
|
// operator's. The message names the directory — removal writes to the parent,
|
||||||
|
// not the file — and the uid/gid the process runs as, which is the half of a
|
||||||
|
// permission problem invisible from the host. Every case says nothing was
|
||||||
|
// deleted, because that is exactly what the operator will be worried about.
|
||||||
|
func fileRemoveAPIError(err error) (*apierror.Error, bool) {
|
||||||
|
var fre *library.FileRemoveError
|
||||||
|
if !errors.As(err, &fre) {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
if fre.NotWritable() {
|
||||||
|
return &apierror.Error{
|
||||||
|
Status: http.StatusConflict,
|
||||||
|
Code: "library_not_writable",
|
||||||
|
Message: fmt.Sprintf(
|
||||||
|
"Minstrel runs as uid %d, gid %d and cannot delete from %s (%s). "+
|
||||||
|
"The library mount must be writable by that user. Nothing was deleted.",
|
||||||
|
fre.UID, fre.GID, fre.Dir(), fre.Reason()),
|
||||||
|
Cause: err,
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
return &apierror.Error{
|
||||||
|
Status: http.StatusInternalServerError,
|
||||||
|
Code: "file_delete_failed",
|
||||||
|
Message: fmt.Sprintf("Could not delete %s (%s). Nothing was deleted.", fre.Path, fre.Reason()),
|
||||||
|
Cause: err,
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// logFileRemoveFailure records a delete that could not reach its file. An
|
||||||
|
// unwritable library is an environment fact the operator can fix, so it is a
|
||||||
|
// Warn; anything else is a real fault.
|
||||||
|
func logFileRemoveFailure(logger *slog.Logger, apiErr *apierror.Error, attrs ...any) {
|
||||||
|
attrs = append(attrs, "code", apiErr.Code, "err", apiErr.Cause)
|
||||||
|
if apiErr.Status == http.StatusConflict {
|
||||||
|
logger.Warn("api: track file could not be deleted", attrs...)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logger.Error("api: track file could not be deleted", attrs...)
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||||
|
)
|
||||||
|
|
||||||
|
const removeTestPath = "/music/Moe Shop/WWW (2020)/01 - WWW.mp3"
|
||||||
|
|
||||||
|
// removeFailure builds the error a delete service returns when the file would
|
||||||
|
// not go, wrapped the way lidarrquarantine.DeleteFile and tracks.RemoveTrack
|
||||||
|
// wrap it — the mapping has to see through that.
|
||||||
|
func removeFailure(errno syscall.Errno) error {
|
||||||
|
return fmt.Errorf("delete file: %w", &library.FileRemoveError{
|
||||||
|
Path: removeTestPath, UID: 1000, GID: 1000,
|
||||||
|
Err: &fs.PathError{Op: "remove", Path: removeTestPath, Err: errno},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFileRemoveAPIError(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
errno syscall.Errno
|
||||||
|
wantStatus int
|
||||||
|
wantCode string
|
||||||
|
wantIn []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "read-only mount", errno: syscall.EROFS,
|
||||||
|
wantStatus: http.StatusConflict, wantCode: "library_not_writable",
|
||||||
|
wantIn: []string{"uid 1000, gid 1000", "/music/Moe Shop/WWW (2020)", "read-only file system", "Nothing was deleted"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "permission denied", errno: syscall.EACCES,
|
||||||
|
wantStatus: http.StatusConflict, wantCode: "library_not_writable",
|
||||||
|
wantIn: []string{"permission denied", "Nothing was deleted"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "operation not permitted", errno: syscall.EPERM,
|
||||||
|
wantStatus: http.StatusConflict, wantCode: "library_not_writable",
|
||||||
|
wantIn: []string{"operation not permitted"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "i/o error", errno: syscall.EIO,
|
||||||
|
wantStatus: http.StatusInternalServerError, wantCode: "file_delete_failed",
|
||||||
|
wantIn: []string{removeTestPath, "input/output error", "Nothing was deleted"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
apiErr, ok := fileRemoveAPIError(removeFailure(tc.errno))
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("a wrapped *library.FileRemoveError was not recognised")
|
||||||
|
}
|
||||||
|
if apiErr.Status != tc.wantStatus || apiErr.Code != tc.wantCode {
|
||||||
|
t.Fatalf("got %d %s, want %d %s", apiErr.Status, apiErr.Code, tc.wantStatus, tc.wantCode)
|
||||||
|
}
|
||||||
|
for _, want := range tc.wantIn {
|
||||||
|
if !strings.Contains(apiErr.Message, want) {
|
||||||
|
t.Errorf("message %q lacks %q", apiErr.Message, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The unwritable answer must name the DIRECTORY. Removal needs write access to
|
||||||
|
// the parent, so a message naming the file would send the operator to fix the
|
||||||
|
// wrong permissions. The directory is a prefix of the file path, which is why a
|
||||||
|
// plain "contains the directory" check could never catch that regression.
|
||||||
|
func TestFileRemoveAPIError_NotWritableNamesTheDirectoryNotTheFile(t *testing.T) {
|
||||||
|
apiErr, _ := fileRemoveAPIError(removeFailure(syscall.EROFS))
|
||||||
|
if strings.Contains(apiErr.Message, "01 - WWW.mp3") {
|
||||||
|
t.Fatalf("message names the file rather than its directory: %q", apiErr.Message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFileRemoveAPIError_IgnoresOtherErrors(t *testing.T) {
|
||||||
|
for name, err := range map[string]error{
|
||||||
|
"nil": nil,
|
||||||
|
"plain error": errors.New("delete track: connection reset"),
|
||||||
|
"path error": &fs.PathError{Op: "remove", Path: removeTestPath, Err: syscall.EROFS},
|
||||||
|
"not found": library.ErrTrackNotFound,
|
||||||
|
} {
|
||||||
|
if _, ok := fileRemoveAPIError(err); ok {
|
||||||
|
t.Errorf("%s: mapped as a file-remove failure", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -465,7 +465,7 @@ func TestRoutesRegisteredInMount(t *testing.T) {
|
|||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
w := playevents.NewWriter(h.pool, slog.New(slog.NewTextHandler(io.Discard, nil)),
|
w := playevents.NewWriter(h.pool, slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||||
30*time.Minute, 0.5, 30000)
|
30*time.Minute, 0.5, 30000)
|
||||||
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.recSettings, h.lidarrCfg, h.lidarrRequests, h.lidarrQuarantine, h.tracks, h.playlists, h.coverart, h.coverSettings, h.tagSettings, h.scanner, h.scanCfg, h.dataDir, nil, eventbus.New(), nil, nil, h.netSettings, nil)
|
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.recSettings, h.lidarrCfg, h.lidarrRequests, h.lidarrQuarantine, h.tracks, h.playlists, h.coverart, h.coverSettings, h.tagSettings, h.scanner, h.scanCfg, h.dataDir, nil, eventbus.New(), nil, nil, h.netSettings, nil, nil)
|
||||||
|
|
||||||
paths := []string{
|
paths := []string{
|
||||||
"/api/artists",
|
"/api/artists",
|
||||||
@@ -484,6 +484,9 @@ func TestRoutesRegisteredInMount(t *testing.T) {
|
|||||||
// wired.
|
// wired.
|
||||||
"/api/admin/library/missing",
|
"/api/admin/library/missing",
|
||||||
"/api/admin/library/reacquisition",
|
"/api/admin/library/reacquisition",
|
||||||
|
"/api/admin/library/fingerprints",
|
||||||
|
"/api/admin/library/fingerprint-settings",
|
||||||
|
"/api/admin/library/duplicates",
|
||||||
}
|
}
|
||||||
for _, p := range paths {
|
for _, p := range paths {
|
||||||
req := httptest.NewRequest(http.MethodGet, p, nil)
|
req := httptest.NewRequest(http.MethodGet, p, nil)
|
||||||
|
|||||||
+24
-2
@@ -87,10 +87,24 @@ func (h *handlers) handleRadio(w http.ResponseWriter, r *http.Request) {
|
|||||||
currentVec.DeviceClass = latestDeviceClass(r.Context(), q, user.ID, h.logger)
|
currentVec.DeviceClass = latestDeviceClass(r.Context(), q, user.ID, h.logger)
|
||||||
|
|
||||||
exclude := parseExcludeParam(r.URL.Query().Get("exclude"))
|
exclude := parseExcludeParam(r.URL.Query().Get("exclude"))
|
||||||
limits := recommendation.DefaultCandidateSourceLimits()
|
// Size the pool to the library (#3880). A fixed ~170 candidates samples a
|
||||||
|
// shrinking fraction of a growing collection, which is what made the
|
||||||
|
// recommendations feel less relevant as the library grew. Degrades to the
|
||||||
|
// base limits if the count is unavailable — never fails the request over a
|
||||||
|
// sizing hint.
|
||||||
|
librarySize := h.librarySize.Get(r.Context(), func(ctx context.Context) (int64, error) {
|
||||||
|
return recommendation.CountLibraryTracks(ctx, q)
|
||||||
|
})
|
||||||
|
limits := recommendation.ScaleForLibrary(
|
||||||
|
recommendation.DefaultCandidateSourceLimits(), librarySize,
|
||||||
|
)
|
||||||
candidates, err := recommendation.LoadCandidatesFromSimilarity(
|
candidates, err := recommendation.LoadCandidatesFromSimilarity(
|
||||||
r.Context(), q, user.ID, seedID,
|
r.Context(), q, user.ID, seedID,
|
||||||
h.recCfg.RecentlyPlayedHours, currentVec, exclude, limits,
|
h.recCfg.RecentlyPlayedHours, currentVec, exclude, limits,
|
||||||
|
// A fresh seed per request (#3889): radio is a new session each time
|
||||||
|
// and SHOULD draw differently. The system mixes are the surfaces that
|
||||||
|
// promise repeatability; this is not one of them.
|
||||||
|
strconv.FormatInt(time.Now().UnixNano(), 36),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Warn("api: radio: similarity-pool failed; falling back to whole-library", "err", err)
|
h.logger.Warn("api: radio: similarity-pool failed; falling back to whole-library", "err", err)
|
||||||
@@ -108,7 +122,15 @@ func (h *handlers) handleRadio(w http.ResponseWriter, r *http.Request) {
|
|||||||
// Scoring weights come from the DB-backed tuning lab (#1250) —
|
// Scoring weights come from the DB-backed tuning lab (#1250) —
|
||||||
// read per request so an admin change takes effect live.
|
// read per request so an admin change takes effect live.
|
||||||
weights := h.recSettings.Weights(recsettings.ScopeRadio)
|
weights := h.recSettings.Weights(recsettings.ScopeRadio)
|
||||||
picks := recommendation.Shuffle(candidates, weights, time.Now().UTC(), h.rng, limit-1)
|
// Diversity caps (#3882). Radio had none while every sibling surface did,
|
||||||
|
// which is how a whole session could come back from one artist. Scaled to
|
||||||
|
// the requested length so a 20-track radio and a 200-track one are capped
|
||||||
|
// alike; Shuffle relaxes them rather than returning a short radio.
|
||||||
|
//
|
||||||
|
// limit-1 because the seed track occupies the first slot and is prepended
|
||||||
|
// below — the caps govern the tracks that FOLLOW it.
|
||||||
|
caps := recommendation.RadioDiversityCaps(limit - 1)
|
||||||
|
picks := recommendation.Shuffle(candidates, weights, time.Now().UTC(), h.rng, limit-1, caps)
|
||||||
|
|
||||||
out := make([]TrackRef, 0, len(picks)+1)
|
out := make([]TrackRef, 0, len(picks)+1)
|
||||||
out = append(out, trackRefFrom(track, album.Title, artist.Name))
|
out = append(out, trackRefFrom(track, album.Title, artist.Name))
|
||||||
|
|||||||
@@ -54,6 +54,10 @@ func Unauthorized(code, message string) *Error {
|
|||||||
return &Error{Status: 401, Code: code, Message: message}
|
return &Error{Status: 401, Code: code, Message: message}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TooManyRequests(code, message string) *Error {
|
||||||
|
return &Error{Status: 429, Code: code, Message: message}
|
||||||
|
}
|
||||||
|
|
||||||
func Internal(cause error) *Error {
|
func Internal(cause error) *Error {
|
||||||
return &Error{Status: 500, Code: "server_error", Message: "internal server error", Cause: cause}
|
return &Error{Status: 500, Code: "server_error", Message: "internal server error", Cause: cause}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,6 +55,11 @@ const (
|
|||||||
// exercised.
|
// exercised.
|
||||||
ActionSessionRevoke Action = "session_revoke"
|
ActionSessionRevoke Action = "session_revoke"
|
||||||
ActionSessionRevokeOthers Action = "session_revoke_others"
|
ActionSessionRevokeOthers Action = "session_revoke_others"
|
||||||
|
|
||||||
|
// Duplicate merge (#3911). Irreversible: a copy's file and row are removed
|
||||||
|
// and its history moved onto the copy kept. The metadata names both, so the
|
||||||
|
// log can answer "where did that file go" long after the report is gone.
|
||||||
|
ActionDuplicateMerge Action = "duplicate_merge"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Write inserts one audit_log row. metadata is marshaled as JSON;
|
// Write inserts one audit_log row. metadata is marshaled as JSON;
|
||||||
|
|||||||
@@ -168,6 +168,7 @@ func TestWrite_AllActionConstantsArePersisted(t *testing.T) {
|
|||||||
audit.ActionTokenRegenerate,
|
audit.ActionTokenRegenerate,
|
||||||
audit.ActionForgotPasswordInit,
|
audit.ActionForgotPasswordInit,
|
||||||
audit.ActionPasswordResetByEmail,
|
audit.ActionPasswordResetByEmail,
|
||||||
|
audit.ActionDuplicateMerge,
|
||||||
}
|
}
|
||||||
for _, a := range actions {
|
for _, a := range actions {
|
||||||
if err := audit.Write(context.Background(), pool, nilUUID, nilUUID, a, nil); err != nil {
|
if err := audit.Write(context.Background(), pool, nilUUID, nilUUID, a, nil); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,205 @@
|
|||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AttemptLimiter caps how many attempts a key may make inside a fixed
|
||||||
|
// window. It is the throttle in front of every password-shaped check:
|
||||||
|
// native login, register, forgot/reset password and Subsonic /rest auth.
|
||||||
|
//
|
||||||
|
// In memory on purpose. Minstrel is a single process, the counts only need
|
||||||
|
// to outlive a guessing run rather than a restart, and a table would put a
|
||||||
|
// write on every failed login. Each key costs one small struct, and expired
|
||||||
|
// keys are swept as the map grows, so a spray across many addresses cannot
|
||||||
|
// hold memory past one window.
|
||||||
|
type AttemptLimiter struct {
|
||||||
|
max int
|
||||||
|
window time.Duration
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
buckets map[string]*attemptBucket
|
||||||
|
nextSweep int
|
||||||
|
}
|
||||||
|
|
||||||
|
type attemptBucket struct {
|
||||||
|
count int
|
||||||
|
start time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// sweepFloor is the map size below which expired keys are left in place;
|
||||||
|
// past it, a sweep runs whenever the map doubles from its last swept size.
|
||||||
|
const sweepFloor = 1024
|
||||||
|
|
||||||
|
// NewAttemptLimiter returns a limiter allowing max attempts per key per
|
||||||
|
// window.
|
||||||
|
func NewAttemptLimiter(max int, window time.Duration) *AttemptLimiter {
|
||||||
|
return &AttemptLimiter{
|
||||||
|
max: max,
|
||||||
|
window: window,
|
||||||
|
now: time.Now,
|
||||||
|
buckets: map[string]*attemptBucket{},
|
||||||
|
nextSweep: sweepFloor,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Blocked reports whether key has used its attempts for the current window,
|
||||||
|
// and if so how long until the window resets. It records nothing, so a check
|
||||||
|
// can run before the expensive work and the outcome be recorded after.
|
||||||
|
func (l *AttemptLimiter) Blocked(key string) (bool, time.Duration) {
|
||||||
|
if l == nil || key == "" {
|
||||||
|
return false, 0
|
||||||
|
}
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
b, ok := l.buckets[key]
|
||||||
|
if !ok {
|
||||||
|
return false, 0
|
||||||
|
}
|
||||||
|
now := l.now()
|
||||||
|
if now.Sub(b.start) >= l.window {
|
||||||
|
delete(l.buckets, key)
|
||||||
|
return false, 0
|
||||||
|
}
|
||||||
|
if b.count < l.max {
|
||||||
|
return false, 0
|
||||||
|
}
|
||||||
|
return true, b.start.Add(l.window).Sub(now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record counts one attempt against key.
|
||||||
|
func (l *AttemptLimiter) Record(key string) {
|
||||||
|
if l == nil || key == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
now := l.now()
|
||||||
|
b, ok := l.buckets[key]
|
||||||
|
if !ok || now.Sub(b.start) >= l.window {
|
||||||
|
l.buckets[key] = &attemptBucket{count: 1, start: now}
|
||||||
|
l.maybeSweep(now)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.count++
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset forgets key, as after a successful login: the user who finally got
|
||||||
|
// their password right should not carry their typos into the next window.
|
||||||
|
func (l *AttemptLimiter) Reset(key string) {
|
||||||
|
if l == nil || key == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
delete(l.buckets, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// maybeSweep drops expired buckets once the map has doubled since the last
|
||||||
|
// sweep. Callers hold l.mu.
|
||||||
|
func (l *AttemptLimiter) maybeSweep(now time.Time) {
|
||||||
|
if len(l.buckets) < l.nextSweep {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for k, b := range l.buckets {
|
||||||
|
if now.Sub(b.start) >= l.window {
|
||||||
|
delete(l.buckets, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
l.nextSweep = max(sweepFloor, 2*len(l.buckets))
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoginGuard pairs a per-account and a per-address limiter, the shape every
|
||||||
|
// password check uses. The account limit stops a slow guess at one user from
|
||||||
|
// many addresses; the address limit stops one address spraying many users.
|
||||||
|
// Only failures are recorded, so a user who signs in correctly is never
|
||||||
|
// counted at all.
|
||||||
|
type LoginGuard struct {
|
||||||
|
account *AttemptLimiter
|
||||||
|
address *AttemptLimiter
|
||||||
|
}
|
||||||
|
|
||||||
|
// Login limits: 10 failures per account and 50 per address per 15 minutes,
|
||||||
|
// the same numbers ThoughtSync settled on. Generous enough that a user
|
||||||
|
// fumbling a password manager never meets them, tight enough that an online
|
||||||
|
// guess against bcrypt gets ~1,000 tries a day per account.
|
||||||
|
const (
|
||||||
|
loginWindow = 15 * time.Minute
|
||||||
|
loginAccountMax = 10
|
||||||
|
loginAddressMax = 50
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewLoginGuard returns a guard with the default login limits.
|
||||||
|
func NewLoginGuard() *LoginGuard {
|
||||||
|
return &LoginGuard{
|
||||||
|
account: NewAttemptLimiter(loginAccountMax, loginWindow),
|
||||||
|
address: NewAttemptLimiter(loginAddressMax, loginWindow),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Blocked reports whether either the account or the address is over its
|
||||||
|
// limit, with the longer of the two waits. Check it BEFORE verifying the
|
||||||
|
// password, so a blocked guess costs no bcrypt.
|
||||||
|
func (g *LoginGuard) Blocked(account, address string) (bool, time.Duration) {
|
||||||
|
if g == nil {
|
||||||
|
return false, 0
|
||||||
|
}
|
||||||
|
aBlocked, aWait := g.account.Blocked(accountKey(account))
|
||||||
|
ipBlocked, ipWait := g.address.Blocked(address)
|
||||||
|
return aBlocked || ipBlocked, max(aWait, ipWait)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fail records a failed attempt against both the account and the address.
|
||||||
|
// An unknown username counts against its name all the same, so the limit
|
||||||
|
// gives away nothing about which accounts exist.
|
||||||
|
func (g *LoginGuard) Fail(account, address string) {
|
||||||
|
if g == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
g.account.Record(accountKey(account))
|
||||||
|
g.address.Record(address)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Succeed clears the account's failures. The address keeps its count: one
|
||||||
|
// address that guessed fifty accounts and got one right is still spraying.
|
||||||
|
func (g *LoginGuard) Succeed(account string) {
|
||||||
|
if g == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
g.account.Reset(accountKey(account))
|
||||||
|
}
|
||||||
|
|
||||||
|
// accountKey folds case so "Admin" and "admin" share one budget. Usernames
|
||||||
|
// are compared exactly by the lookup, but the guesser shouldn't get a fresh
|
||||||
|
// allowance per capitalisation.
|
||||||
|
func accountKey(account string) string {
|
||||||
|
return strings.ToLower(strings.TrimSpace(account))
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
dummyHashOnce sync.Once
|
||||||
|
dummyHash []byte
|
||||||
|
)
|
||||||
|
|
||||||
|
// DummyVerify spends the same bcrypt time a real password check would, for
|
||||||
|
// the path where the username doesn't exist. Without it, an unknown user
|
||||||
|
// answers in microseconds and a known one in ~50ms, and the uniform error
|
||||||
|
// message hides nothing.
|
||||||
|
func DummyVerify(plaintext string) {
|
||||||
|
dummyHashOnce.Do(func() {
|
||||||
|
// What the hash is of doesn't matter — no account carries it. Its
|
||||||
|
// cost does: DefaultCost, the same as every stored password.
|
||||||
|
h, err := bcrypt.GenerateFromPassword([]byte("minstrel-dummy-password"), bcrypt.DefaultCost)
|
||||||
|
if err == nil {
|
||||||
|
dummyHash = h
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if dummyHash != nil {
|
||||||
|
_ = bcrypt.CompareHashAndPassword(dummyHash, []byte(plaintext))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeClock lets a test step through a window without sleeping.
|
||||||
|
type fakeClock struct{ t time.Time }
|
||||||
|
|
||||||
|
func (c *fakeClock) now() time.Time { return c.t }
|
||||||
|
|
||||||
|
func newTestLimiter(max int, window time.Duration) (*AttemptLimiter, *fakeClock) {
|
||||||
|
c := &fakeClock{t: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)}
|
||||||
|
l := NewAttemptLimiter(max, window)
|
||||||
|
l.now = c.now
|
||||||
|
return l, c
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAttemptLimiter_BlocksAtMaxAndReportsWait(t *testing.T) {
|
||||||
|
l, c := newTestLimiter(3, 15*time.Minute)
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if blocked, _ := l.Blocked("k"); blocked {
|
||||||
|
t.Fatalf("blocked after %d attempts, want allowed below max", i)
|
||||||
|
}
|
||||||
|
l.Record("k")
|
||||||
|
}
|
||||||
|
c.t = c.t.Add(5 * time.Minute)
|
||||||
|
blocked, wait := l.Blocked("k")
|
||||||
|
if !blocked {
|
||||||
|
t.Fatal("not blocked after max attempts")
|
||||||
|
}
|
||||||
|
if wait != 10*time.Minute {
|
||||||
|
t.Errorf("wait = %v, want the 10m left in the window", wait)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAttemptLimiter_WindowExpiryClears(t *testing.T) {
|
||||||
|
l, c := newTestLimiter(1, time.Minute)
|
||||||
|
l.Record("k")
|
||||||
|
if blocked, _ := l.Blocked("k"); !blocked {
|
||||||
|
t.Fatal("want blocked inside the window")
|
||||||
|
}
|
||||||
|
c.t = c.t.Add(time.Minute)
|
||||||
|
if blocked, _ := l.Blocked("k"); blocked {
|
||||||
|
t.Fatal("still blocked once the window has passed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAttemptLimiter_KeysAreIndependentAndResetClears(t *testing.T) {
|
||||||
|
l, _ := newTestLimiter(1, time.Minute)
|
||||||
|
l.Record("a")
|
||||||
|
if blocked, _ := l.Blocked("b"); blocked {
|
||||||
|
t.Fatal("one key's attempts blocked another")
|
||||||
|
}
|
||||||
|
l.Reset("a")
|
||||||
|
if blocked, _ := l.Blocked("a"); blocked {
|
||||||
|
t.Fatal("Reset did not clear the key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAttemptLimiter_SweepDropsExpiredKeys(t *testing.T) {
|
||||||
|
l, c := newTestLimiter(5, time.Minute)
|
||||||
|
// One short of the floor: no sweep yet, however stale these become.
|
||||||
|
for i := 0; i < sweepFloor-1; i++ {
|
||||||
|
l.Record("k" + time.Duration(i).String())
|
||||||
|
}
|
||||||
|
c.t = c.t.Add(2 * time.Minute)
|
||||||
|
l.Record("fresh") // reaches the floor and triggers a sweep
|
||||||
|
if n := len(l.buckets); n != 1 {
|
||||||
|
t.Errorf("buckets after sweep = %d, want only the fresh key", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAttemptLimiter_NilAndEmptyKeyAreNoOps(t *testing.T) {
|
||||||
|
var l *AttemptLimiter
|
||||||
|
l.Record("k")
|
||||||
|
if blocked, _ := l.Blocked("k"); blocked {
|
||||||
|
t.Error("nil limiter blocked")
|
||||||
|
}
|
||||||
|
real, _ := newTestLimiter(1, time.Minute)
|
||||||
|
real.Record("")
|
||||||
|
if blocked, _ := real.Blocked(""); blocked {
|
||||||
|
t.Error("empty key was counted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoginGuard_AccountLimitSpansAddressesAndFoldsCase(t *testing.T) {
|
||||||
|
g := NewLoginGuard()
|
||||||
|
for i := 0; i < loginAccountMax; i++ {
|
||||||
|
g.Fail("Alice", "10.0.0."+string(rune('0'+i%10)))
|
||||||
|
}
|
||||||
|
if blocked, _ := g.Blocked("alice", "192.0.2.1"); !blocked {
|
||||||
|
t.Fatal("account limit should hold from a fresh address and any capitalisation")
|
||||||
|
}
|
||||||
|
g.Succeed("ALICE")
|
||||||
|
if blocked, _ := g.Blocked("alice", "192.0.2.1"); blocked {
|
||||||
|
t.Fatal("Succeed should clear the account's failures")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoginGuard_AddressLimitSpansAccounts(t *testing.T) {
|
||||||
|
g := NewLoginGuard()
|
||||||
|
for i := 0; i < loginAddressMax; i++ {
|
||||||
|
g.Fail("user"+time.Duration(i).String(), "203.0.113.9")
|
||||||
|
}
|
||||||
|
if blocked, _ := g.Blocked("someone-new", "203.0.113.9"); !blocked {
|
||||||
|
t.Fatal("address that sprayed many accounts should be blocked")
|
||||||
|
}
|
||||||
|
g.Succeed("someone-new")
|
||||||
|
if blocked, _ := g.Blocked("someone-new", "203.0.113.9"); !blocked {
|
||||||
|
t.Fatal("a success must not clear the address count")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,457 @@
|
|||||||
|
// Code generated by sqlc. DO NOT EDIT.
|
||||||
|
// versions:
|
||||||
|
// sqlc v1.31.1
|
||||||
|
// source: duplicates.sql
|
||||||
|
|
||||||
|
package dbq
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
)
|
||||||
|
|
||||||
|
const addDuplicateGroupMember = `-- name: AddDuplicateGroupMember :exec
|
||||||
|
INSERT INTO duplicate_group_members (group_id, track_id)
|
||||||
|
VALUES ($1, $2)
|
||||||
|
ON CONFLICT DO NOTHING
|
||||||
|
`
|
||||||
|
|
||||||
|
type AddDuplicateGroupMemberParams struct {
|
||||||
|
GroupID pgtype.UUID
|
||||||
|
TrackID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) AddDuplicateGroupMember(ctx context.Context, arg AddDuplicateGroupMemberParams) error {
|
||||||
|
_, err := q.db.Exec(ctx, addDuplicateGroupMember, arg.GroupID, arg.TrackID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
const countPendingDuplicateGroups = `-- name: CountPendingDuplicateGroups :one
|
||||||
|
SELECT count(*)::bigint
|
||||||
|
FROM duplicate_groups g
|
||||||
|
WHERE g.status = 'pending'
|
||||||
|
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2
|
||||||
|
`
|
||||||
|
|
||||||
|
// Proposals awaiting review. A group left with one member — its other tracks
|
||||||
|
// deleted since the sweep — is no proposal at all and is not counted; the next
|
||||||
|
// sweep retires it.
|
||||||
|
func (q *Queries) CountPendingDuplicateGroups(ctx context.Context) (int64, error) {
|
||||||
|
row := q.db.QueryRow(ctx, countPendingDuplicateGroups)
|
||||||
|
var column_1 int64
|
||||||
|
err := row.Scan(&column_1)
|
||||||
|
return column_1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
const deleteStalePendingDuplicateGroups = `-- name: DeleteStalePendingDuplicateGroups :execrows
|
||||||
|
DELETE FROM duplicate_groups g
|
||||||
|
WHERE g.status = 'pending'
|
||||||
|
AND g.last_seen_sweep_id IS DISTINCT FROM $1
|
||||||
|
AND (g.last_seen_sweep_id IS NULL
|
||||||
|
OR (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = g.last_seen_sweep_id)
|
||||||
|
< (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = $1))
|
||||||
|
`
|
||||||
|
|
||||||
|
// A pending proposal this sweep did not find again no longer describes the
|
||||||
|
// library: a member was re-fingerprinted, merged away or went missing. Dismissed
|
||||||
|
// groups are kept regardless — they are the memory of a decision.
|
||||||
|
//
|
||||||
|
// Only proposals last confirmed by an EARLIER sweep go. Should two sweeps ever
|
||||||
|
// overlap (a manual trigger racing the worker), neither may delete what the other
|
||||||
|
// has just found.
|
||||||
|
func (q *Queries) DeleteStalePendingDuplicateGroups(ctx context.Context, sweepID pgtype.UUID) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, deleteStalePendingDuplicateGroups, sweepID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const dismissDuplicateGroup = `-- name: DismissDuplicateGroup :execrows
|
||||||
|
UPDATE duplicate_groups
|
||||||
|
SET status = 'dismissed', resolved_at = now()
|
||||||
|
WHERE id = $1 AND status = 'pending'
|
||||||
|
`
|
||||||
|
|
||||||
|
// "These are not duplicates." Only a pending group can be dismissed; zero rows
|
||||||
|
// means it was already resolved or no longer exists.
|
||||||
|
func (q *Queries) DismissDuplicateGroup(ctx context.Context, id pgtype.UUID) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, dismissDuplicateGroup, id)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const finishDuplicateSweep = `-- name: FinishDuplicateSweep :exec
|
||||||
|
UPDATE duplicate_sweeps
|
||||||
|
SET finished_at = now(),
|
||||||
|
candidates = $1,
|
||||||
|
groups_found = $2,
|
||||||
|
oversize_clusters = $3,
|
||||||
|
error_message = NULLIF($4::text, '')
|
||||||
|
WHERE id = $5
|
||||||
|
`
|
||||||
|
|
||||||
|
type FinishDuplicateSweepParams struct {
|
||||||
|
Candidates *int32
|
||||||
|
GroupsFound *int32
|
||||||
|
OversizeClusters *int32
|
||||||
|
ErrorMessage string
|
||||||
|
ID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) FinishDuplicateSweep(ctx context.Context, arg FinishDuplicateSweepParams) error {
|
||||||
|
_, err := q.db.Exec(ctx, finishDuplicateSweep,
|
||||||
|
arg.Candidates,
|
||||||
|
arg.GroupsFound,
|
||||||
|
arg.OversizeClusters,
|
||||||
|
arg.ErrorMessage,
|
||||||
|
arg.ID,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
const getInFlightDuplicateSweep = `-- name: GetInFlightDuplicateSweep :one
|
||||||
|
SELECT id, started_at
|
||||||
|
FROM duplicate_sweeps
|
||||||
|
WHERE finished_at IS NULL
|
||||||
|
ORDER BY started_at DESC
|
||||||
|
LIMIT 1
|
||||||
|
`
|
||||||
|
|
||||||
|
type GetInFlightDuplicateSweepRow struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
StartedAt pgtype.Timestamptz
|
||||||
|
}
|
||||||
|
|
||||||
|
// The guard against two sweeps at once: "in flight" is finished_at IS NULL.
|
||||||
|
func (q *Queries) GetInFlightDuplicateSweep(ctx context.Context) (GetInFlightDuplicateSweepRow, error) {
|
||||||
|
row := q.db.QueryRow(ctx, getInFlightDuplicateSweep)
|
||||||
|
var i GetInFlightDuplicateSweepRow
|
||||||
|
err := row.Scan(&i.ID, &i.StartedAt)
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
|
||||||
|
const getLatestDuplicateSweep = `-- name: GetLatestDuplicateSweep :one
|
||||||
|
SELECT id, started_at, finished_at, candidates, groups_found, oversize_clusters, error_message
|
||||||
|
FROM duplicate_sweeps
|
||||||
|
ORDER BY started_at DESC
|
||||||
|
LIMIT 1
|
||||||
|
`
|
||||||
|
|
||||||
|
func (q *Queries) GetLatestDuplicateSweep(ctx context.Context) (DuplicateSweep, error) {
|
||||||
|
row := q.db.QueryRow(ctx, getLatestDuplicateSweep)
|
||||||
|
var i DuplicateSweep
|
||||||
|
err := row.Scan(
|
||||||
|
&i.ID,
|
||||||
|
&i.StartedAt,
|
||||||
|
&i.FinishedAt,
|
||||||
|
&i.Candidates,
|
||||||
|
&i.GroupsFound,
|
||||||
|
&i.OversizeClusters,
|
||||||
|
&i.ErrorMessage,
|
||||||
|
)
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
|
||||||
|
const getLatestFingerprintComputedAt = `-- name: GetLatestFingerprintComputedAt :one
|
||||||
|
SELECT max(computed_at)::timestamptz AS latest FROM track_fingerprints
|
||||||
|
`
|
||||||
|
|
||||||
|
// Whether a sweep has anything new to look at: fingerprints written since the
|
||||||
|
// last sweep started.
|
||||||
|
func (q *Queries) GetLatestFingerprintComputedAt(ctx context.Context) (pgtype.Timestamptz, error) {
|
||||||
|
row := q.db.QueryRow(ctx, getLatestFingerprintComputedAt)
|
||||||
|
var latest pgtype.Timestamptz
|
||||||
|
err := row.Scan(&latest)
|
||||||
|
return latest, err
|
||||||
|
}
|
||||||
|
|
||||||
|
const listDismissedDuplicateMemberSets = `-- name: ListDismissedDuplicateMemberSets :many
|
||||||
|
SELECT g.id, array_agg(m.track_id ORDER BY m.track_id)::uuid[] AS track_ids
|
||||||
|
FROM duplicate_groups g
|
||||||
|
JOIN duplicate_group_members m ON m.group_id = g.id
|
||||||
|
WHERE g.status = 'dismissed'
|
||||||
|
GROUP BY g.id
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListDismissedDuplicateMemberSetsRow struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
TrackIds []pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the operator has already said are not duplicates. A new proposal whose
|
||||||
|
// every member sat together in one of these is not proposed again.
|
||||||
|
func (q *Queries) ListDismissedDuplicateMemberSets(ctx context.Context) ([]ListDismissedDuplicateMemberSetsRow, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listDismissedDuplicateMemberSets)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []ListDismissedDuplicateMemberSetsRow
|
||||||
|
for rows.Next() {
|
||||||
|
var i ListDismissedDuplicateMemberSetsRow
|
||||||
|
if err := rows.Scan(&i.ID, &i.TrackIds); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const listDuplicateCandidates = `-- name: ListDuplicateCandidates :many
|
||||||
|
SELECT t.id, t.duration_ms, f.audio_stream_sha256, f.chromaprint
|
||||||
|
FROM tracks t
|
||||||
|
JOIN track_fingerprints f ON f.track_id = t.id
|
||||||
|
WHERE t.missing_since IS NULL
|
||||||
|
AND f.fingerprint_version >= $1
|
||||||
|
AND f.chromaprint IS NOT NULL
|
||||||
|
-- Only chromaprints taken at the current length: prints at two lengths are not
|
||||||
|
-- comparable, and after a length change the backfill is still re-deriving the
|
||||||
|
-- rest (#3913).
|
||||||
|
AND f.chromaprint_length_sec = $2
|
||||||
|
AND (t.duration_ms, t.id) > ($3::integer, $4::uuid)
|
||||||
|
ORDER BY t.duration_ms, t.id
|
||||||
|
LIMIT $5
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListDuplicateCandidatesParams struct {
|
||||||
|
CurrentVersion int16
|
||||||
|
ChromaprintLengthSec int32
|
||||||
|
AfterDurationMs int32
|
||||||
|
AfterID pgtype.UUID
|
||||||
|
PageLimit int32
|
||||||
|
}
|
||||||
|
|
||||||
|
type ListDuplicateCandidatesRow struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
DurationMs int32
|
||||||
|
AudioStreamSha256 []byte
|
||||||
|
Chromaprint []int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// The acoustic tier's input, one page at a time in (duration_ms, id) order so the
|
||||||
|
// sweep holds only a sliding window of durations. Tracks without a chromaprint
|
||||||
|
// cannot be compared acoustically and are left out; any exact duplicates among
|
||||||
|
// them come from ListExactDuplicateHashes.
|
||||||
|
func (q *Queries) ListDuplicateCandidates(ctx context.Context, arg ListDuplicateCandidatesParams) ([]ListDuplicateCandidatesRow, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listDuplicateCandidates,
|
||||||
|
arg.CurrentVersion,
|
||||||
|
arg.ChromaprintLengthSec,
|
||||||
|
arg.AfterDurationMs,
|
||||||
|
arg.AfterID,
|
||||||
|
arg.PageLimit,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []ListDuplicateCandidatesRow
|
||||||
|
for rows.Next() {
|
||||||
|
var i ListDuplicateCandidatesRow
|
||||||
|
if err := rows.Scan(
|
||||||
|
&i.ID,
|
||||||
|
&i.DurationMs,
|
||||||
|
&i.AudioStreamSha256,
|
||||||
|
&i.Chromaprint,
|
||||||
|
); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const listExactDuplicateHashes = `-- name: ListExactDuplicateHashes :many
|
||||||
|
SELECT f.audio_stream_sha256,
|
||||||
|
array_agg(t.id ORDER BY t.id)::uuid[] AS track_ids
|
||||||
|
FROM track_fingerprints f
|
||||||
|
JOIN tracks t ON t.id = f.track_id
|
||||||
|
WHERE t.missing_since IS NULL
|
||||||
|
AND f.fingerprint_version >= $1
|
||||||
|
AND f.audio_stream_sha256 IS NOT NULL
|
||||||
|
GROUP BY f.audio_stream_sha256
|
||||||
|
HAVING count(*) > 1
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListExactDuplicateHashesRow struct {
|
||||||
|
AudioStreamSha256 []byte
|
||||||
|
TrackIds []pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// The exact tier, library-wide in one pass: identical encoded audio shared by
|
||||||
|
// more than one present track.
|
||||||
|
func (q *Queries) ListExactDuplicateHashes(ctx context.Context, currentVersion int16) ([]ListExactDuplicateHashesRow, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listExactDuplicateHashes, currentVersion)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []ListExactDuplicateHashesRow
|
||||||
|
for rows.Next() {
|
||||||
|
var i ListExactDuplicateHashesRow
|
||||||
|
if err := rows.Scan(&i.AudioStreamSha256, &i.TrackIds); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const listPendingDuplicateGroupMembers = `-- name: ListPendingDuplicateGroupMembers :many
|
||||||
|
WITH page AS (
|
||||||
|
SELECT g.id, g.tier, g.worst_bit_error_rate, g.detected_at
|
||||||
|
FROM duplicate_groups g
|
||||||
|
WHERE g.status = 'pending'
|
||||||
|
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2
|
||||||
|
ORDER BY g.detected_at DESC, g.id
|
||||||
|
LIMIT $2 OFFSET $1
|
||||||
|
)
|
||||||
|
SELECT p.id AS group_id,
|
||||||
|
p.tier,
|
||||||
|
p.worst_bit_error_rate,
|
||||||
|
p.detected_at,
|
||||||
|
t.id AS track_id,
|
||||||
|
t.title,
|
||||||
|
artists.name AS artist_name,
|
||||||
|
albums.id AS album_id,
|
||||||
|
albums.title AS album_title,
|
||||||
|
t.file_path,
|
||||||
|
t.file_format,
|
||||||
|
t.file_size,
|
||||||
|
t.duration_ms,
|
||||||
|
t.added_at,
|
||||||
|
(SELECT count(*) FROM general_likes l WHERE l.track_id = t.id)::bigint AS like_count,
|
||||||
|
(SELECT count(*) FROM play_events e WHERE e.track_id = t.id)::bigint AS play_count
|
||||||
|
FROM page p
|
||||||
|
JOIN duplicate_group_members m ON m.group_id = p.id
|
||||||
|
JOIN tracks t ON t.id = m.track_id
|
||||||
|
JOIN albums ON albums.id = t.album_id
|
||||||
|
JOIN artists ON artists.id = t.artist_id
|
||||||
|
ORDER BY p.detected_at DESC, p.id, t.id
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListPendingDuplicateGroupMembersParams struct {
|
||||||
|
PageOffset int32
|
||||||
|
PageLimit int32
|
||||||
|
}
|
||||||
|
|
||||||
|
type ListPendingDuplicateGroupMembersRow struct {
|
||||||
|
GroupID pgtype.UUID
|
||||||
|
Tier string
|
||||||
|
WorstBitErrorRate *float32
|
||||||
|
DetectedAt pgtype.Timestamptz
|
||||||
|
TrackID pgtype.UUID
|
||||||
|
Title string
|
||||||
|
ArtistName string
|
||||||
|
AlbumID pgtype.UUID
|
||||||
|
AlbumTitle string
|
||||||
|
FilePath string
|
||||||
|
FileFormat string
|
||||||
|
FileSize int64
|
||||||
|
DurationMs int32
|
||||||
|
AddedAt pgtype.Timestamptz
|
||||||
|
LikeCount int64
|
||||||
|
PlayCount int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// One page of proposals, newest first, flattened to one row per member so the
|
||||||
|
// handler folds them without a query per group. What each copy carries — likes
|
||||||
|
// and plays from every user — is here because it is what the operator weighs
|
||||||
|
// when deciding which copy to keep.
|
||||||
|
func (q *Queries) ListPendingDuplicateGroupMembers(ctx context.Context, arg ListPendingDuplicateGroupMembersParams) ([]ListPendingDuplicateGroupMembersRow, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listPendingDuplicateGroupMembers, arg.PageOffset, arg.PageLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []ListPendingDuplicateGroupMembersRow
|
||||||
|
for rows.Next() {
|
||||||
|
var i ListPendingDuplicateGroupMembersRow
|
||||||
|
if err := rows.Scan(
|
||||||
|
&i.GroupID,
|
||||||
|
&i.Tier,
|
||||||
|
&i.WorstBitErrorRate,
|
||||||
|
&i.DetectedAt,
|
||||||
|
&i.TrackID,
|
||||||
|
&i.Title,
|
||||||
|
&i.ArtistName,
|
||||||
|
&i.AlbumID,
|
||||||
|
&i.AlbumTitle,
|
||||||
|
&i.FilePath,
|
||||||
|
&i.FileFormat,
|
||||||
|
&i.FileSize,
|
||||||
|
&i.DurationMs,
|
||||||
|
&i.AddedAt,
|
||||||
|
&i.LikeCount,
|
||||||
|
&i.PlayCount,
|
||||||
|
); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const startDuplicateSweep = `-- name: StartDuplicateSweep :one
|
||||||
|
INSERT INTO duplicate_sweeps DEFAULT VALUES RETURNING id, started_at
|
||||||
|
`
|
||||||
|
|
||||||
|
type StartDuplicateSweepRow struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
StartedAt pgtype.Timestamptz
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) StartDuplicateSweep(ctx context.Context) (StartDuplicateSweepRow, error) {
|
||||||
|
row := q.db.QueryRow(ctx, startDuplicateSweep)
|
||||||
|
var i StartDuplicateSweepRow
|
||||||
|
err := row.Scan(&i.ID, &i.StartedAt)
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
|
||||||
|
const upsertDuplicateGroup = `-- name: UpsertDuplicateGroup :one
|
||||||
|
INSERT INTO duplicate_groups (member_key, tier, worst_bit_error_rate, last_seen_sweep_id)
|
||||||
|
VALUES ($1, $2, $3, $4)
|
||||||
|
ON CONFLICT (member_key) DO UPDATE
|
||||||
|
SET tier = EXCLUDED.tier,
|
||||||
|
worst_bit_error_rate = EXCLUDED.worst_bit_error_rate,
|
||||||
|
last_seen_sweep_id = EXCLUDED.last_seen_sweep_id
|
||||||
|
WHERE duplicate_groups.status = 'pending'
|
||||||
|
RETURNING id
|
||||||
|
`
|
||||||
|
|
||||||
|
type UpsertDuplicateGroupParams struct {
|
||||||
|
MemberKey string
|
||||||
|
Tier string
|
||||||
|
WorstBitErrorRate *float32
|
||||||
|
SweepID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// Proposes a group, or refreshes one already pending. A group already dismissed
|
||||||
|
// or merged is left exactly as it is: the WHERE on the update makes the conflict
|
||||||
|
// a no-op, and the caller sees no row.
|
||||||
|
func (q *Queries) UpsertDuplicateGroup(ctx context.Context, arg UpsertDuplicateGroupParams) (pgtype.UUID, error) {
|
||||||
|
row := q.db.QueryRow(ctx, upsertDuplicateGroup,
|
||||||
|
arg.MemberKey,
|
||||||
|
arg.Tier,
|
||||||
|
arg.WorstBitErrorRate,
|
||||||
|
arg.SweepID,
|
||||||
|
)
|
||||||
|
var id pgtype.UUID
|
||||||
|
err := row.Scan(&id)
|
||||||
|
return id, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
// Code generated by sqlc. DO NOT EDIT.
|
||||||
|
// versions:
|
||||||
|
// sqlc v1.31.1
|
||||||
|
// source: fingerprint_settings.sql
|
||||||
|
|
||||||
|
package dbq
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
)
|
||||||
|
|
||||||
|
const getFingerprintSettings = `-- name: GetFingerprintSettings :one
|
||||||
|
SELECT id, enabled, chromaprint_length_sec, acoustic_max_bit_error_rate, backfill_concurrency, sweep_interval_hours, updated_at FROM fingerprint_settings WHERE id = true
|
||||||
|
`
|
||||||
|
|
||||||
|
func (q *Queries) GetFingerprintSettings(ctx context.Context) (FingerprintSetting, error) {
|
||||||
|
row := q.db.QueryRow(ctx, getFingerprintSettings)
|
||||||
|
var i FingerprintSetting
|
||||||
|
err := row.Scan(
|
||||||
|
&i.ID,
|
||||||
|
&i.Enabled,
|
||||||
|
&i.ChromaprintLengthSec,
|
||||||
|
&i.AcousticMaxBitErrorRate,
|
||||||
|
&i.BackfillConcurrency,
|
||||||
|
&i.SweepIntervalHours,
|
||||||
|
&i.UpdatedAt,
|
||||||
|
)
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
|
||||||
|
const updateFingerprintSettings = `-- name: UpdateFingerprintSettings :one
|
||||||
|
UPDATE fingerprint_settings
|
||||||
|
SET enabled = $1,
|
||||||
|
chromaprint_length_sec = $2,
|
||||||
|
acoustic_max_bit_error_rate = $3,
|
||||||
|
backfill_concurrency = $4,
|
||||||
|
sweep_interval_hours = $5,
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = true
|
||||||
|
RETURNING id, enabled, chromaprint_length_sec, acoustic_max_bit_error_rate, backfill_concurrency, sweep_interval_hours, updated_at
|
||||||
|
`
|
||||||
|
|
||||||
|
type UpdateFingerprintSettingsParams struct {
|
||||||
|
Enabled bool
|
||||||
|
ChromaprintLengthSec int32
|
||||||
|
AcousticMaxBitErrorRate float64
|
||||||
|
BackfillConcurrency int32
|
||||||
|
SweepIntervalHours int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whole-row write from the admin card; migration 0061's CHECKs are the backstop
|
||||||
|
// behind the service's own validation.
|
||||||
|
func (q *Queries) UpdateFingerprintSettings(ctx context.Context, arg UpdateFingerprintSettingsParams) (FingerprintSetting, error) {
|
||||||
|
row := q.db.QueryRow(ctx, updateFingerprintSettings,
|
||||||
|
arg.Enabled,
|
||||||
|
arg.ChromaprintLengthSec,
|
||||||
|
arg.AcousticMaxBitErrorRate,
|
||||||
|
arg.BackfillConcurrency,
|
||||||
|
arg.SweepIntervalHours,
|
||||||
|
)
|
||||||
|
var i FingerprintSetting
|
||||||
|
err := row.Scan(
|
||||||
|
&i.ID,
|
||||||
|
&i.Enabled,
|
||||||
|
&i.ChromaprintLengthSec,
|
||||||
|
&i.AcousticMaxBitErrorRate,
|
||||||
|
&i.BackfillConcurrency,
|
||||||
|
&i.SweepIntervalHours,
|
||||||
|
&i.UpdatedAt,
|
||||||
|
)
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
// Code generated by sqlc. DO NOT EDIT.
|
||||||
|
// versions:
|
||||||
|
// sqlc v1.31.1
|
||||||
|
// source: fingerprints.sql
|
||||||
|
|
||||||
|
package dbq
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
)
|
||||||
|
|
||||||
|
const deleteTrackFingerprint = `-- name: DeleteTrackFingerprint :exec
|
||||||
|
DELETE FROM track_fingerprints WHERE track_id = $1
|
||||||
|
`
|
||||||
|
|
||||||
|
// A file changed but could not be fingerprinted, for a reason unrelated to the
|
||||||
|
// file. The stored row describes the OLD bytes, so it goes and the backfill
|
||||||
|
// re-derives it — nothing may keep trusting a stale identity.
|
||||||
|
func (q *Queries) DeleteTrackFingerprint(ctx context.Context, trackID pgtype.UUID) error {
|
||||||
|
_, err := q.db.Exec(ctx, deleteTrackFingerprint, trackID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
const getFingerprintCoverage = `-- name: GetFingerprintCoverage :one
|
||||||
|
SELECT count(*)::bigint AS total,
|
||||||
|
count(*) FILTER (
|
||||||
|
WHERE f.fingerprint_version >= $1
|
||||||
|
AND f.chromaprint_length_sec = $2
|
||||||
|
AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL
|
||||||
|
)::bigint AS fingerprinted,
|
||||||
|
count(*) FILTER (
|
||||||
|
WHERE f.fingerprint_version >= $1
|
||||||
|
AND f.chromaprint_length_sec = $2
|
||||||
|
AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL)
|
||||||
|
)::bigint AS rejected,
|
||||||
|
count(*) FILTER (
|
||||||
|
WHERE f.track_id IS NULL
|
||||||
|
OR f.fingerprint_version < $1
|
||||||
|
OR f.chromaprint_length_sec <> $2
|
||||||
|
)::bigint AS pending
|
||||||
|
FROM tracks t
|
||||||
|
LEFT JOIN track_fingerprints f ON f.track_id = t.id
|
||||||
|
WHERE t.missing_since IS NULL
|
||||||
|
`
|
||||||
|
|
||||||
|
type GetFingerprintCoverageParams struct {
|
||||||
|
CurrentVersion int16
|
||||||
|
ChromaprintLengthSec int32
|
||||||
|
}
|
||||||
|
|
||||||
|
type GetFingerprintCoverageRow struct {
|
||||||
|
Total int64
|
||||||
|
Fingerprinted int64
|
||||||
|
Rejected int64
|
||||||
|
Pending int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// The admin gauge for the backfill. fingerprinted + rejected + pending = total.
|
||||||
|
// "Current" means derived by the current method AT the current length: a row at
|
||||||
|
// another length is pending, because the backfill will re-derive it. rejected is
|
||||||
|
// a current row with a NULL half: a tool ran and refused the file, which is
|
||||||
|
// settled rather than waiting. Missing tracks are excluded, or the gauge could
|
||||||
|
// never reach the end.
|
||||||
|
func (q *Queries) GetFingerprintCoverage(ctx context.Context, arg GetFingerprintCoverageParams) (GetFingerprintCoverageRow, error) {
|
||||||
|
row := q.db.QueryRow(ctx, getFingerprintCoverage, arg.CurrentVersion, arg.ChromaprintLengthSec)
|
||||||
|
var i GetFingerprintCoverageRow
|
||||||
|
err := row.Scan(
|
||||||
|
&i.Total,
|
||||||
|
&i.Fingerprinted,
|
||||||
|
&i.Rejected,
|
||||||
|
&i.Pending,
|
||||||
|
)
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
|
||||||
|
const listTracksNeedingFingerprint = `-- name: ListTracksNeedingFingerprint :many
|
||||||
|
SELECT t.id, t.file_path
|
||||||
|
FROM tracks t
|
||||||
|
LEFT JOIN track_fingerprints f ON f.track_id = t.id
|
||||||
|
WHERE t.missing_since IS NULL
|
||||||
|
-- A row taken at another length is as stale as one from an older method:
|
||||||
|
-- chromaprints at two lengths cannot be compared (#3913).
|
||||||
|
AND (f.track_id IS NULL
|
||||||
|
OR f.fingerprint_version < $1
|
||||||
|
OR f.chromaprint_length_sec <> $2)
|
||||||
|
AND t.id > $3
|
||||||
|
ORDER BY t.id
|
||||||
|
LIMIT $4
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListTracksNeedingFingerprintParams struct {
|
||||||
|
CurrentVersion int16
|
||||||
|
ChromaprintLengthSec int32
|
||||||
|
AfterID pgtype.UUID
|
||||||
|
BatchLimit int32
|
||||||
|
}
|
||||||
|
|
||||||
|
type ListTracksNeedingFingerprintRow struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
FilePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// The backfill's work queue (#3908): tracks with no fingerprint, or one derived
|
||||||
|
// by an older method. Keyset-paged on id so a pass visits each track at most
|
||||||
|
// once. That cursor is load-bearing: an inconclusive attempt writes no row, so
|
||||||
|
// without it a file that keeps timing out would be listed again straight away
|
||||||
|
// and retried in a tight loop. Missing tracks are skipped — there is no file to
|
||||||
|
// read.
|
||||||
|
func (q *Queries) ListTracksNeedingFingerprint(ctx context.Context, arg ListTracksNeedingFingerprintParams) ([]ListTracksNeedingFingerprintRow, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listTracksNeedingFingerprint,
|
||||||
|
arg.CurrentVersion,
|
||||||
|
arg.ChromaprintLengthSec,
|
||||||
|
arg.AfterID,
|
||||||
|
arg.BatchLimit,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []ListTracksNeedingFingerprintRow
|
||||||
|
for rows.Next() {
|
||||||
|
var i ListTracksNeedingFingerprintRow
|
||||||
|
if err := rows.Scan(&i.ID, &i.FilePath); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const upsertTrackFingerprint = `-- name: UpsertTrackFingerprint :exec
|
||||||
|
INSERT INTO track_fingerprints (
|
||||||
|
track_id, audio_stream_sha256, chromaprint, fingerprint_version, chromaprint_length_sec
|
||||||
|
) VALUES (
|
||||||
|
$1, $2, $3,
|
||||||
|
$4, $5
|
||||||
|
)
|
||||||
|
ON CONFLICT (track_id) DO UPDATE SET
|
||||||
|
audio_stream_sha256 = EXCLUDED.audio_stream_sha256,
|
||||||
|
chromaprint = EXCLUDED.chromaprint,
|
||||||
|
fingerprint_version = EXCLUDED.fingerprint_version,
|
||||||
|
chromaprint_length_sec = EXCLUDED.chromaprint_length_sec,
|
||||||
|
computed_at = now()
|
||||||
|
`
|
||||||
|
|
||||||
|
type UpsertTrackFingerprintParams struct {
|
||||||
|
TrackID pgtype.UUID
|
||||||
|
AudioStreamSha256 []byte
|
||||||
|
Chromaprint []int32
|
||||||
|
FingerprintVersion int16
|
||||||
|
ChromaprintLengthSec int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// Written whenever a track's fingerprint is derived: by the scan when a file is
|
||||||
|
// new or its bytes changed, and by the backfill (#3908) for rows derived by an
|
||||||
|
// older method. Replaces the row wholesale — a fingerprint of the old bytes has
|
||||||
|
// no standing once the file has changed.
|
||||||
|
func (q *Queries) UpsertTrackFingerprint(ctx context.Context, arg UpsertTrackFingerprintParams) error {
|
||||||
|
_, err := q.db.Exec(ctx, upsertTrackFingerprint,
|
||||||
|
arg.TrackID,
|
||||||
|
arg.AudioStreamSha256,
|
||||||
|
arg.Chromaprint,
|
||||||
|
arg.FingerprintVersion,
|
||||||
|
arg.ChromaprintLengthSec,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,339 @@
|
|||||||
|
// Code generated by sqlc. DO NOT EDIT.
|
||||||
|
// versions:
|
||||||
|
// sqlc v1.31.1
|
||||||
|
// source: merge.sql
|
||||||
|
|
||||||
|
package dbq
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
)
|
||||||
|
|
||||||
|
const listDuplicateGroupMergeMembers = `-- name: ListDuplicateGroupMergeMembers :many
|
||||||
|
SELECT t.id, t.file_path, t.file_format, t.file_size, t.added_at, t.album_id,
|
||||||
|
t.mbid, albums.mbid AS album_mbid
|
||||||
|
FROM duplicate_group_members m
|
||||||
|
JOIN tracks t ON t.id = m.track_id
|
||||||
|
JOIN albums ON albums.id = t.album_id
|
||||||
|
WHERE m.group_id = $1
|
||||||
|
ORDER BY t.id
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListDuplicateGroupMergeMembersRow struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
FilePath string
|
||||||
|
FileFormat string
|
||||||
|
FileSize int64
|
||||||
|
AddedAt pgtype.Timestamptz
|
||||||
|
AlbumID pgtype.UUID
|
||||||
|
Mbid *string
|
||||||
|
AlbumMbid *string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) ListDuplicateGroupMergeMembers(ctx context.Context, groupID pgtype.UUID) ([]ListDuplicateGroupMergeMembersRow, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listDuplicateGroupMergeMembers, groupID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []ListDuplicateGroupMergeMembersRow
|
||||||
|
for rows.Next() {
|
||||||
|
var i ListDuplicateGroupMergeMembersRow
|
||||||
|
if err := rows.Scan(
|
||||||
|
&i.ID,
|
||||||
|
&i.FilePath,
|
||||||
|
&i.FileFormat,
|
||||||
|
&i.FileSize,
|
||||||
|
&i.AddedAt,
|
||||||
|
&i.AlbumID,
|
||||||
|
&i.Mbid,
|
||||||
|
&i.AlbumMbid,
|
||||||
|
); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const lockDuplicateGroupForMerge = `-- name: LockDuplicateGroupForMerge :one
|
||||||
|
|
||||||
|
SELECT id, tier, status
|
||||||
|
FROM duplicate_groups
|
||||||
|
WHERE id = $1
|
||||||
|
FOR UPDATE
|
||||||
|
`
|
||||||
|
|
||||||
|
type LockDuplicateGroupForMergeRow struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
Tier string
|
||||||
|
Status string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Duplicate merge (Scribe #3911). Every statement here runs inside the one
|
||||||
|
// transaction library.MergeDuplicateGroup opens, after the removed copy's file
|
||||||
|
// is already gone. The loser's own track row is deleted last with DeleteTrack;
|
||||||
|
// what these do is move everything it carries onto the survivor first, so that
|
||||||
|
// delete's CASCADE finds nothing left to destroy.
|
||||||
|
// Locks the group for the rest of the transaction, so two merges of one group
|
||||||
|
// cannot run at once.
|
||||||
|
func (q *Queries) LockDuplicateGroupForMerge(ctx context.Context, id pgtype.UUID) (LockDuplicateGroupForMergeRow, error) {
|
||||||
|
row := q.db.QueryRow(ctx, lockDuplicateGroupForMerge, id)
|
||||||
|
var i LockDuplicateGroupForMergeRow
|
||||||
|
err := row.Scan(&i.ID, &i.Tier, &i.Status)
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
|
||||||
|
const markDuplicateGroupMerged = `-- name: MarkDuplicateGroupMerged :execrows
|
||||||
|
UPDATE duplicate_groups
|
||||||
|
SET status = 'merged', resolved_at = now()
|
||||||
|
WHERE id = $1 AND status = 'pending'
|
||||||
|
`
|
||||||
|
|
||||||
|
func (q *Queries) MarkDuplicateGroupMerged(ctx context.Context, id pgtype.UUID) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, markDuplicateGroupMerged, id)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeCopyGeneralLikes = `-- name: MergeCopyGeneralLikes :many
|
||||||
|
|
||||||
|
INSERT INTO general_likes (user_id, track_id, liked_at)
|
||||||
|
SELECT user_id, $1::uuid, liked_at
|
||||||
|
FROM general_likes
|
||||||
|
WHERE track_id = $2::uuid
|
||||||
|
ON CONFLICT (user_id, track_id) DO UPDATE
|
||||||
|
SET liked_at = LEAST(general_likes.liked_at, EXCLUDED.liked_at)
|
||||||
|
RETURNING user_id
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeCopyGeneralLikesParams struct {
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collision-safe merges: a unique key includes track_id, so the survivor may
|
||||||
|
// already hold a matching row. Copy what it lacks; DeleteTrack's CASCADE then
|
||||||
|
// removes the loser's originals.
|
||||||
|
// One like per user. A user who liked both copies keeps a single like, dated to
|
||||||
|
// the earlier of the two.
|
||||||
|
func (q *Queries) MergeCopyGeneralLikes(ctx context.Context, arg MergeCopyGeneralLikesParams) ([]pgtype.UUID, error) {
|
||||||
|
rows, err := q.db.Query(ctx, mergeCopyGeneralLikes, arg.SurvivorID, arg.LoserID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []pgtype.UUID
|
||||||
|
for rows.Next() {
|
||||||
|
var user_id pgtype.UUID
|
||||||
|
if err := rows.Scan(&user_id); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, user_id)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeCopyTrackSimilarity = `-- name: MergeCopyTrackSimilarity :execrows
|
||||||
|
INSERT INTO track_similarity (track_a_id, track_b_id, score, source, fetched_at)
|
||||||
|
SELECT CASE WHEN track_a_id = $1::uuid THEN $2::uuid ELSE track_a_id END,
|
||||||
|
CASE WHEN track_b_id = $1::uuid THEN $2::uuid ELSE track_b_id END,
|
||||||
|
score, source, fetched_at
|
||||||
|
FROM track_similarity
|
||||||
|
WHERE (track_a_id = $1::uuid OR track_b_id = $1::uuid)
|
||||||
|
AND (CASE WHEN track_a_id = $1::uuid THEN $2::uuid ELSE track_a_id END)
|
||||||
|
<> (CASE WHEN track_b_id = $1::uuid THEN $2::uuid ELSE track_b_id END)
|
||||||
|
ON CONFLICT (track_a_id, track_b_id, source) DO NOTHING
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeCopyTrackSimilarityParams struct {
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rewrites the loser to the survivor on either side of an edge. An edge between
|
||||||
|
// the two copies would become a track similar to itself — the table forbids
|
||||||
|
// that, and it means nothing — so it is dropped. An edge the survivor already
|
||||||
|
// has from the same source is kept as it is.
|
||||||
|
func (q *Queries) MergeCopyTrackSimilarity(ctx context.Context, arg MergeCopyTrackSimilarityParams) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, mergeCopyTrackSimilarity, arg.LoserID, arg.SurvivorID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeCopyTrackTags = `-- name: MergeCopyTrackTags :execrows
|
||||||
|
INSERT INTO track_tags (track_id, tag, weight)
|
||||||
|
SELECT $1::uuid, tag, weight
|
||||||
|
FROM track_tags
|
||||||
|
WHERE track_id = $2::uuid
|
||||||
|
ON CONFLICT (track_id, tag) DO NOTHING
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeCopyTrackTagsParams struct {
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) MergeCopyTrackTags(ctx context.Context, arg MergeCopyTrackTagsParams) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, mergeCopyTrackTags, arg.SurvivorID, arg.LoserID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeInheritTrackMbid = `-- name: MergeInheritTrackMbid :exec
|
||||||
|
UPDATE tracks AS survivor
|
||||||
|
SET mbid = loser.mbid
|
||||||
|
FROM tracks AS loser
|
||||||
|
WHERE survivor.id = $1::uuid
|
||||||
|
AND loser.id = $2::uuid
|
||||||
|
AND survivor.mbid IS NULL
|
||||||
|
AND loser.mbid IS NOT NULL
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeInheritTrackMbidParams struct {
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// A recording MBID is what the similarity pipeline keys on. If only the removed
|
||||||
|
// copy carried one, the survivor takes it rather than going dark to similarity.
|
||||||
|
func (q *Queries) MergeInheritTrackMbid(ctx context.Context, arg MergeInheritTrackMbidParams) error {
|
||||||
|
_, err := q.db.Exec(ctx, mergeInheritTrackMbid, arg.SurvivorID, arg.LoserID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeRepointContextualLikes = `-- name: MergeRepointContextualLikes :execrows
|
||||||
|
UPDATE contextual_likes SET track_id = $1::uuid WHERE track_id = $2::uuid
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeRepointContextualLikesParams struct {
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) MergeRepointContextualLikes(ctx context.Context, arg MergeRepointContextualLikesParams) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, mergeRepointContextualLikes, arg.SurvivorID, arg.LoserID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeRepointLidarrRequests = `-- name: MergeRepointLidarrRequests :execrows
|
||||||
|
UPDATE lidarr_requests SET matched_track_id = $1::uuid
|
||||||
|
WHERE matched_track_id = $2::uuid
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeRepointLidarrRequestsParams struct {
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) MergeRepointLidarrRequests(ctx context.Context, arg MergeRepointLidarrRequestsParams) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, mergeRepointLidarrRequests, arg.SurvivorID, arg.LoserID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeRepointPlayEvents = `-- name: MergeRepointPlayEvents :execrows
|
||||||
|
|
||||||
|
UPDATE play_events SET track_id = $1::uuid WHERE track_id = $2::uuid
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeRepointPlayEventsParams struct {
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plain repoints: no unique key involves track_id, so moving rows cannot collide.
|
||||||
|
func (q *Queries) MergeRepointPlayEvents(ctx context.Context, arg MergeRepointPlayEventsParams) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, mergeRepointPlayEvents, arg.SurvivorID, arg.LoserID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeRepointPlaybackErrors = `-- name: MergeRepointPlaybackErrors :execrows
|
||||||
|
UPDATE playback_errors SET track_id = $1::uuid WHERE track_id = $2::uuid
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeRepointPlaybackErrorsParams struct {
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) MergeRepointPlaybackErrors(ctx context.Context, arg MergeRepointPlaybackErrorsParams) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, mergeRepointPlaybackErrors, arg.SurvivorID, arg.LoserID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeRepointPlaylistTracks = `-- name: MergeRepointPlaylistTracks :many
|
||||||
|
UPDATE playlist_tracks SET track_id = $1::uuid
|
||||||
|
WHERE track_id = $2::uuid
|
||||||
|
RETURNING playlist_id
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeRepointPlaylistTracksParams struct {
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// playlist_tracks is keyed by (playlist_id, position), so repointing keeps every
|
||||||
|
// entry exactly where it was. A playlist that held both copies simply holds the
|
||||||
|
// survivor twice — the user put two entries there, and both stay.
|
||||||
|
func (q *Queries) MergeRepointPlaylistTracks(ctx context.Context, arg MergeRepointPlaylistTracksParams) ([]pgtype.UUID, error) {
|
||||||
|
rows, err := q.db.Query(ctx, mergeRepointPlaylistTracks, arg.SurvivorID, arg.LoserID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []pgtype.UUID
|
||||||
|
for rows.Next() {
|
||||||
|
var playlist_id pgtype.UUID
|
||||||
|
if err := rows.Scan(&playlist_id); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, playlist_id)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergeRepointSkipEvents = `-- name: MergeRepointSkipEvents :execrows
|
||||||
|
UPDATE skip_events SET track_id = $1::uuid WHERE track_id = $2::uuid
|
||||||
|
`
|
||||||
|
|
||||||
|
type MergeRepointSkipEventsParams struct {
|
||||||
|
SurvivorID pgtype.UUID
|
||||||
|
LoserID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) MergeRepointSkipEvents(ctx context.Context, arg MergeRepointSkipEventsParams) (int64, error) {
|
||||||
|
result, err := q.db.Exec(ctx, mergeRepointSkipEvents, arg.SurvivorID, arg.LoserID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected(), nil
|
||||||
|
}
|
||||||
@@ -297,6 +297,42 @@ type DiscoverTuning struct {
|
|||||||
UpdatedAt pgtype.Timestamptz
|
UpdatedAt pgtype.Timestamptz
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type DuplicateGroup struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
MemberKey string
|
||||||
|
Tier string
|
||||||
|
WorstBitErrorRate *float32
|
||||||
|
Status string
|
||||||
|
DetectedAt pgtype.Timestamptz
|
||||||
|
LastSeenSweepID pgtype.UUID
|
||||||
|
ResolvedAt pgtype.Timestamptz
|
||||||
|
}
|
||||||
|
|
||||||
|
type DuplicateGroupMember struct {
|
||||||
|
GroupID pgtype.UUID
|
||||||
|
TrackID pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
type DuplicateSweep struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
StartedAt pgtype.Timestamptz
|
||||||
|
FinishedAt pgtype.Timestamptz
|
||||||
|
Candidates *int32
|
||||||
|
GroupsFound *int32
|
||||||
|
OversizeClusters *int32
|
||||||
|
ErrorMessage *string
|
||||||
|
}
|
||||||
|
|
||||||
|
type FingerprintSetting struct {
|
||||||
|
ID bool
|
||||||
|
Enabled bool
|
||||||
|
ChromaprintLengthSec int32
|
||||||
|
AcousticMaxBitErrorRate float64
|
||||||
|
BackfillConcurrency int32
|
||||||
|
SweepIntervalHours int32
|
||||||
|
UpdatedAt pgtype.Timestamptz
|
||||||
|
}
|
||||||
|
|
||||||
type GeneralLike struct {
|
type GeneralLike struct {
|
||||||
UserID pgtype.UUID
|
UserID pgtype.UUID
|
||||||
TrackID pgtype.UUID
|
TrackID pgtype.UUID
|
||||||
@@ -667,6 +703,15 @@ type Track struct {
|
|||||||
MissingSince pgtype.Timestamptz
|
MissingSince pgtype.Timestamptz
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TrackFingerprint struct {
|
||||||
|
TrackID pgtype.UUID
|
||||||
|
AudioStreamSha256 []byte
|
||||||
|
Chromaprint []int32
|
||||||
|
FingerprintVersion int16
|
||||||
|
ComputedAt pgtype.Timestamptz
|
||||||
|
ChromaprintLengthSec int32
|
||||||
|
}
|
||||||
|
|
||||||
type TrackSimilarity struct {
|
type TrackSimilarity struct {
|
||||||
TrackAID pgtype.UUID
|
TrackAID pgtype.UUID
|
||||||
TrackBID pgtype.UUID
|
TrackBID pgtype.UUID
|
||||||
|
|||||||
@@ -829,7 +829,7 @@ similar_artists AS (
|
|||||||
JOIN seed_artist sa ON asim.artist_a_id = sa.artist_id
|
JOIN seed_artist sa ON asim.artist_a_id = sa.artist_id
|
||||||
WHERE asim.source = 'listenbrainz'
|
WHERE asim.source = 'listenbrainz'
|
||||||
AND t.id NOT IN (SELECT id FROM excluded_ids)
|
AND t.id NOT IN (SELECT id FROM excluded_ids)
|
||||||
ORDER BY asim.score DESC, random()
|
ORDER BY asim.score DESC, md5(t.id::text || $12::text)
|
||||||
LIMIT $6
|
LIMIT $6
|
||||||
),
|
),
|
||||||
tag_overlap AS (
|
tag_overlap AS (
|
||||||
@@ -857,7 +857,7 @@ likes_overlap AS (
|
|||||||
WHERE t.id = gl.track_id
|
WHERE t.id = gl.track_id
|
||||||
AND trim(g_overlap.g) IN (SELECT tag FROM seed_tags)
|
AND trim(g_overlap.g) IN (SELECT tag FROM seed_tags)
|
||||||
)
|
)
|
||||||
ORDER BY random()
|
ORDER BY md5(gl.track_id::text || $12::text)
|
||||||
LIMIT $8
|
LIMIT $8
|
||||||
),
|
),
|
||||||
taste_overlap AS (
|
taste_overlap AS (
|
||||||
@@ -884,7 +884,7 @@ coplay_artists AS (
|
|||||||
WHERE asim.source = 'user_cooccurrence'
|
WHERE asim.source = 'user_cooccurrence'
|
||||||
AND t.id NOT IN (SELECT id FROM excluded_ids)
|
AND t.id NOT IN (SELECT id FROM excluded_ids)
|
||||||
AND t.id <> $2
|
AND t.id <> $2
|
||||||
ORDER BY asim.score DESC, random()
|
ORDER BY asim.score DESC, md5(t.id::text || $12::text)
|
||||||
LIMIT $11
|
LIMIT $11
|
||||||
),
|
),
|
||||||
random_fill AS (
|
random_fill AS (
|
||||||
@@ -900,7 +900,7 @@ random_fill AS (
|
|||||||
UNION SELECT track_id FROM taste_overlap
|
UNION SELECT track_id FROM taste_overlap
|
||||||
UNION SELECT track_id FROM coplay_artists
|
UNION SELECT track_id FROM coplay_artists
|
||||||
)
|
)
|
||||||
ORDER BY random()
|
ORDER BY md5(t.id::text || $12::text)
|
||||||
LIMIT $9
|
LIMIT $9
|
||||||
)
|
)
|
||||||
SELECT
|
SELECT
|
||||||
@@ -938,17 +938,18 @@ GROUP BY t.id, t.title, t.album_id, t.artist_id, t.duration_ms, t.file_path,
|
|||||||
`
|
`
|
||||||
|
|
||||||
type LoadRadioCandidatesV2Params struct {
|
type LoadRadioCandidatesV2Params struct {
|
||||||
UserID pgtype.UUID
|
UserID pgtype.UUID
|
||||||
ID pgtype.UUID
|
ID pgtype.UUID
|
||||||
Column3 interface{}
|
Column3 interface{}
|
||||||
Column4 []pgtype.UUID
|
Column4 []pgtype.UUID
|
||||||
Limit int32
|
Limit int32
|
||||||
Limit_2 int32
|
Limit_2 int32
|
||||||
Limit_3 int32
|
Limit_3 int32
|
||||||
Limit_4 int32
|
Limit_4 int32
|
||||||
Limit_5 int32
|
Limit_5 int32
|
||||||
Limit_6 int32
|
Limit_6 int32
|
||||||
Limit_7 int32
|
Limit_7 int32
|
||||||
|
Column12 string
|
||||||
}
|
}
|
||||||
|
|
||||||
type LoadRadioCandidatesV2Row struct {
|
type LoadRadioCandidatesV2Row struct {
|
||||||
@@ -971,8 +972,22 @@ type LoadRadioCandidatesV2Row struct {
|
|||||||
// enter the pool even when the similarity/random arms miss them; scored
|
// enter the pool even when the similarity/random arms miss them; scored
|
||||||
// in Go via TasteMatch, so sim_score here is 0 pool-inclusion),
|
// in Go via TasteMatch, so sim_score here is 0 pool-inclusion),
|
||||||
// $11 coplay_artists K (#1533 — tracks by artists co-played across the
|
// $11 coplay_artists K (#1533 — tracks by artists co-played across the
|
||||||
// instance with the seed's artist; source='user_cooccurrence').
|
// instance with the seed's artist; source='user_cooccurrence'),
|
||||||
|
// $12 order_seed (text) — see below.
|
||||||
//
|
//
|
||||||
|
// $12 REPLACES `ORDER BY random()` IN FOUR ARMS (#3889). Those arms returned
|
||||||
|
// a stable set only while their LIMIT exceeded the rows eligible for them: at
|
||||||
|
// that point they returned all of them and the order stopped mattering,
|
||||||
|
// because the caller sorts by track id before scoring. Below that threshold
|
||||||
|
// they returned a random SUBSET, and two builds on the same day drew
|
||||||
|
// different ones — so "daily determinism" held by accident, and only for
|
||||||
|
// libraries smaller than the limits.
|
||||||
|
//
|
||||||
|
// md5(id || seed) keeps the intent — an arbitrary spread that changes when
|
||||||
|
// the seed does — while making it reproducible for a given seed. The CALLER
|
||||||
|
// decides what that means: system mixes pass a per-(user, day) string and get
|
||||||
|
// the determinism they promise; radio passes a fresh value per request and
|
||||||
|
// keeps varying, which is what a radio should do.
|
||||||
// Returns same shape as LoadRadioCandidates plus similarity_score column.
|
// Returns same shape as LoadRadioCandidates plus similarity_score column.
|
||||||
func (q *Queries) LoadRadioCandidatesV2(ctx context.Context, arg LoadRadioCandidatesV2Params) ([]LoadRadioCandidatesV2Row, error) {
|
func (q *Queries) LoadRadioCandidatesV2(ctx context.Context, arg LoadRadioCandidatesV2Params) ([]LoadRadioCandidatesV2Row, error) {
|
||||||
rows, err := q.db.Query(ctx, loadRadioCandidatesV2,
|
rows, err := q.db.Query(ctx, loadRadioCandidatesV2,
|
||||||
@@ -987,6 +1002,7 @@ func (q *Queries) LoadRadioCandidatesV2(ctx context.Context, arg LoadRadioCandid
|
|||||||
arg.Limit_5,
|
arg.Limit_5,
|
||||||
arg.Limit_6,
|
arg.Limit_6,
|
||||||
arg.Limit_7,
|
arg.Limit_7,
|
||||||
|
arg.Column12,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -148,39 +148,36 @@ func (q *Queries) DeleteTrack(ctx context.Context, id pgtype.UUID) (DeleteTrackR
|
|||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
|
|
||||||
const findMissingTrackByFingerprint = `-- name: FindMissingTrackByFingerprint :many
|
const findMissingTrackByAudioHash = `-- name: FindMissingTrackByAudioHash :many
|
||||||
SELECT id, file_path FROM tracks
|
SELECT t.id, t.file_path
|
||||||
WHERE missing_since IS NOT NULL
|
FROM tracks t
|
||||||
AND file_size = $1
|
JOIN track_fingerprints f ON f.track_id = t.id
|
||||||
AND duration_ms = $2
|
WHERE t.missing_since IS NOT NULL
|
||||||
|
AND f.audio_stream_sha256 = $1
|
||||||
LIMIT 2
|
LIMIT 2
|
||||||
`
|
`
|
||||||
|
|
||||||
type FindMissingTrackByFingerprintParams struct {
|
type FindMissingTrackByAudioHashRow struct {
|
||||||
FileSize int64
|
|
||||||
DurationMs int32
|
|
||||||
}
|
|
||||||
|
|
||||||
type FindMissingTrackByFingerprintRow struct {
|
|
||||||
ID pgtype.UUID
|
ID pgtype.UUID
|
||||||
FilePath string
|
FilePath string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Move detection fallback for files with no MBID (#2528). Exact byte size AND
|
// Move detection fallback for files with no MBID (#2528, #3914). The audio stream
|
||||||
// exact decoded duration is a strong pair: a plain move or rename preserves
|
// hash identifies the encoded audio itself, so it survives a rename, a move and a
|
||||||
// both, while a re-encode changes at least one — and a re-encode genuinely is a
|
// retag — anything short of a re-encode. It replaced (file_size, duration_ms),
|
||||||
// different file, so failing to match there is correct rather than a gap.
|
// which could pair two unrelated files that happened to share a byte count and a
|
||||||
|
// duration, and missed a file retagged in place, whose size changes.
|
||||||
//
|
//
|
||||||
// Same missing-only constraint and same LIMIT 2 rationale as the MBID variant.
|
// Same missing-only constraint and same LIMIT 2 rationale as the MBID variant.
|
||||||
func (q *Queries) FindMissingTrackByFingerprint(ctx context.Context, arg FindMissingTrackByFingerprintParams) ([]FindMissingTrackByFingerprintRow, error) {
|
func (q *Queries) FindMissingTrackByAudioHash(ctx context.Context, audioStreamSha256 []byte) ([]FindMissingTrackByAudioHashRow, error) {
|
||||||
rows, err := q.db.Query(ctx, findMissingTrackByFingerprint, arg.FileSize, arg.DurationMs)
|
rows, err := q.db.Query(ctx, findMissingTrackByAudioHash, audioStreamSha256)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
var items []FindMissingTrackByFingerprintRow
|
var items []FindMissingTrackByAudioHashRow
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var i FindMissingTrackByFingerprintRow
|
var i FindMissingTrackByAudioHashRow
|
||||||
if err := rows.Scan(&i.ID, &i.FilePath); err != nil {
|
if err := rows.Scan(&i.ID, &i.FilePath); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
-- Drop the rows the narrower constraints are about to forbid, or re-adding
|
||||||
|
-- them fails against existing data (the 0051 down-migration pattern).
|
||||||
|
DELETE FROM recommendation_weight_profiles WHERE profile = 'songs_like';
|
||||||
|
DELETE FROM recommendation_tuning_audit WHERE scope = 'songs_like';
|
||||||
|
|
||||||
|
ALTER TABLE recommendation_tuning_audit
|
||||||
|
DROP CONSTRAINT recommendation_tuning_audit_scope_check;
|
||||||
|
ALTER TABLE recommendation_tuning_audit
|
||||||
|
ADD CONSTRAINT recommendation_tuning_audit_scope_check
|
||||||
|
CHECK (scope IN ('radio', 'daily_mix', 'taste', 'discover'));
|
||||||
|
|
||||||
|
ALTER TABLE recommendation_weight_profiles
|
||||||
|
DROP CONSTRAINT recommendation_weight_profiles_profile_check;
|
||||||
|
ALTER TABLE recommendation_weight_profiles
|
||||||
|
ADD CONSTRAINT recommendation_weight_profiles_profile_check
|
||||||
|
CHECK (profile IN ('radio', 'daily_mix'));
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
-- 0057_songs_like_tuning.up.sql — a THIRD weight profile, for Songs-like
|
||||||
|
-- (Scribe #3881, milestone #398).
|
||||||
|
--
|
||||||
|
-- Songs-like shared the `daily_mix` profile with For-You, and that is the bug.
|
||||||
|
-- The two surfaces want opposite things: For-You is a broad "what will they
|
||||||
|
-- enjoy today", Songs-like answers "what sounds like THIS", and under one set
|
||||||
|
-- of weights the broad answer wins. Operator, 2026-09-10: "I'm expecting to
|
||||||
|
-- get a consistent sound and style from the experience... I was getting a
|
||||||
|
-- seeming wide variety of music from each one when I was hoping to stay in a
|
||||||
|
-- certain neighborhood."
|
||||||
|
--
|
||||||
|
-- Under the shared daily_mix weights, an UNRELATED track the user had liked
|
||||||
|
-- and not played recently scored 1.0 + 2.0 + 1.0 = 4.0 before taste, while a
|
||||||
|
-- PERFECT similarity match they had not liked scored 1.0 + 1.5 = 2.5. Liking
|
||||||
|
-- something outranked sounding like the seed. Splitting the profile is what
|
||||||
|
-- lets similarity dominate here without making For-You narrow.
|
||||||
|
--
|
||||||
|
-- Rows are seeded by the recsettings boot reconcile, not here, so shipped
|
||||||
|
-- defaults live in exactly one place (Go) — same as 0040.
|
||||||
|
|
||||||
|
-- Rule #36: a new value for a CHECK-gated column needs the constraint
|
||||||
|
-- rewritten in the SAME change, or the first row written under the new
|
||||||
|
-- profile fails at runtime rather than at migrate time.
|
||||||
|
ALTER TABLE recommendation_weight_profiles
|
||||||
|
DROP CONSTRAINT recommendation_weight_profiles_profile_check;
|
||||||
|
ALTER TABLE recommendation_weight_profiles
|
||||||
|
ADD CONSTRAINT recommendation_weight_profiles_profile_check
|
||||||
|
CHECK (profile IN ('radio', 'daily_mix', 'songs_like'));
|
||||||
|
|
||||||
|
-- The audit table gates the same name on a separate constraint. Missing this
|
||||||
|
-- one would let the profile be seeded and then fail on the first knob turn —
|
||||||
|
-- green at boot, 500 on first use.
|
||||||
|
ALTER TABLE recommendation_tuning_audit
|
||||||
|
DROP CONSTRAINT recommendation_tuning_audit_scope_check;
|
||||||
|
ALTER TABLE recommendation_tuning_audit
|
||||||
|
ADD CONSTRAINT recommendation_tuning_audit_scope_check
|
||||||
|
CHECK (scope IN ('radio', 'daily_mix', 'taste', 'discover', 'songs_like'));
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
DROP TABLE track_fingerprints;
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
-- 0058_track_fingerprints.up.sql — an acoustic identity per track (Scribe
|
||||||
|
-- milestone #400: #3905, #3906).
|
||||||
|
--
|
||||||
|
-- A table of its own rather than columns on tracks, for the hot path's sake:
|
||||||
|
-- tracks is read with SELECT * by eight queries, among them ListTracksByAlbum,
|
||||||
|
-- SearchTracks and GetTracksByIDs — album pages, search, the Subsonic surface.
|
||||||
|
-- A ~4 KB chromaprint column on tracks would be de-TOASTed on every one of
|
||||||
|
-- those reads to carry a value only the duplicate sweep ever looks at.
|
||||||
|
--
|
||||||
|
-- What a row means, which the backfill depends on:
|
||||||
|
-- no row never fingerprinted
|
||||||
|
-- fingerprint_version < current derived by an older method; re-derive it
|
||||||
|
-- fingerprint_version = current attempted; a NULL value means that tool
|
||||||
|
-- failed on this file, and it is not retried
|
||||||
|
-- until the file changes
|
||||||
|
-- A failure that says nothing about the file — a timeout, a cancelled scan, a
|
||||||
|
-- missing binary — writes no row at all, so the backfill tries again.
|
||||||
|
CREATE TABLE track_fingerprints (
|
||||||
|
-- CASCADE is right here, unlike for the likes and play history M400's
|
||||||
|
-- merge has to carry across: a fingerprint describes one file's bytes and
|
||||||
|
-- means nothing once that file's row is gone.
|
||||||
|
track_id uuid PRIMARY KEY REFERENCES tracks (id) ON DELETE CASCADE,
|
||||||
|
-- SHA-256 of the ENCODED audio packets (ffmpeg -c:a copy -f hash), not of
|
||||||
|
-- decoded samples. internal/library/fingerprint.go says why.
|
||||||
|
audio_stream_sha256 bytea
|
||||||
|
CHECK (audio_stream_sha256 IS NULL OR octet_length(audio_stream_sha256) = 32),
|
||||||
|
-- fpcalc -raw -signed: the same 32 bits per item, stored signed because
|
||||||
|
-- integer is.
|
||||||
|
chromaprint integer[],
|
||||||
|
fingerprint_version smallint NOT NULL,
|
||||||
|
computed_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
-- The exact duplicate tier is an equality match on this column. Partial
|
||||||
|
-- because a NULL is never looked up — it only means the hash was not taken.
|
||||||
|
CREATE INDEX track_fingerprints_audio_stream_sha256
|
||||||
|
ON track_fingerprints (audio_stream_sha256)
|
||||||
|
WHERE audio_stream_sha256 IS NOT NULL;
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
DROP INDEX IF EXISTS tracks_duration_id_idx;
|
||||||
|
DROP TABLE duplicate_group_members;
|
||||||
|
DROP TABLE duplicate_groups;
|
||||||
|
DROP TABLE duplicate_sweeps;
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
-- 0059_duplicate_groups.up.sql — proposed duplicates and the sweeps that find
|
||||||
|
-- them (Scribe milestone #400: #3910).
|
||||||
|
--
|
||||||
|
-- The sweep compares fingerprints (track_fingerprints, 0058) and proposes groups
|
||||||
|
-- of tracks that hold one recording. Nothing here merges anything: a group is a
|
||||||
|
-- proposal the operator reviews, and the merge (#3911) is a separate act.
|
||||||
|
|
||||||
|
-- One row per sweep. Lets the report tell "the sweep has never run" apart from
|
||||||
|
-- "it ran and found nothing", and gives the in-flight guard something to check,
|
||||||
|
-- the same way scan_runs does for the library scan.
|
||||||
|
CREATE TABLE duplicate_sweeps (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
started_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
finished_at timestamptz,
|
||||||
|
candidates integer,
|
||||||
|
groups_found integer,
|
||||||
|
oversize_clusters integer,
|
||||||
|
error_message text
|
||||||
|
);
|
||||||
|
CREATE INDEX duplicate_sweeps_started_at_idx ON duplicate_sweeps (started_at DESC);
|
||||||
|
|
||||||
|
CREATE TABLE duplicate_groups (
|
||||||
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
-- The group's identity: its member track ids, sorted and joined. A sweep
|
||||||
|
-- that finds the same tracks again updates this row rather than proposing
|
||||||
|
-- them twice, and a dismissal stays attached to the set it was made about.
|
||||||
|
member_key text NOT NULL UNIQUE,
|
||||||
|
-- Rule 36: a new value for either CHECK swaps the constraint in the same
|
||||||
|
-- migration.
|
||||||
|
tier text NOT NULL CHECK (tier IN ('exact', 'acoustic')),
|
||||||
|
-- Largest disagreement between any two members; NULL for exact groups,
|
||||||
|
-- which have no score.
|
||||||
|
worst_bit_error_rate real,
|
||||||
|
status text NOT NULL DEFAULT 'pending'
|
||||||
|
CHECK (status IN ('pending', 'dismissed', 'merged')),
|
||||||
|
detected_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
last_seen_sweep_id uuid REFERENCES duplicate_sweeps (id) ON DELETE SET NULL,
|
||||||
|
resolved_at timestamptz
|
||||||
|
);
|
||||||
|
CREATE INDEX duplicate_groups_status_idx ON duplicate_groups (status);
|
||||||
|
|
||||||
|
CREATE TABLE duplicate_group_members (
|
||||||
|
group_id uuid NOT NULL REFERENCES duplicate_groups (id) ON DELETE CASCADE,
|
||||||
|
-- CASCADE is right here: a track that genuinely leaves the library has no
|
||||||
|
-- place in a proposal about its duplicates.
|
||||||
|
track_id uuid NOT NULL REFERENCES tracks (id) ON DELETE CASCADE,
|
||||||
|
PRIMARY KEY (group_id, track_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX duplicate_group_members_track_idx ON duplicate_group_members (track_id);
|
||||||
|
|
||||||
|
-- The sweep streams candidates in (duration_ms, id) order, keyset-paged, so it
|
||||||
|
-- only ever holds a few seconds' worth of durations in memory. Without this each
|
||||||
|
-- page would sort the whole library again.
|
||||||
|
CREATE INDEX tracks_duration_id_idx ON tracks (duration_ms, id);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
DROP INDEX IF EXISTS play_events_track_idx;
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
-- 0060_play_events_track_index.up.sql — play_events by track (Scribe #3912, #3911).
|
||||||
|
--
|
||||||
|
-- play_events is indexed by (user_id, started_at) and (user_id, track_id), both
|
||||||
|
-- led by user. Nothing reached it by track alone until the duplicates report,
|
||||||
|
-- which shows each copy's play count — a scan of the whole table per copy — and
|
||||||
|
-- the merge (#3911), which repoints a duplicate's play history onto the copy
|
||||||
|
-- being kept. Both ask "every play of this track", whoever played it.
|
||||||
|
CREATE INDEX play_events_track_idx ON play_events (track_id);
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE track_fingerprints DROP COLUMN chromaprint_length_sec;
|
||||||
|
DROP TABLE fingerprint_settings;
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
-- 0061_fingerprint_settings.up.sql — fingerprinting's knobs, in admin Settings
|
||||||
|
-- (Scribe #3913, milestone #400). Rule 25: anything an operator might tune is a
|
||||||
|
-- database row, changed without a restart. Singleton in the style of
|
||||||
|
-- reacquisition_settings (0056).
|
||||||
|
CREATE TABLE fingerprint_settings (
|
||||||
|
id boolean PRIMARY KEY DEFAULT true,
|
||||||
|
|
||||||
|
-- Fingerprinting new files, the backfill, and the duplicate sweep. Off stops
|
||||||
|
-- the decode work entirely — the reason to turn it off is a slow NAS, and
|
||||||
|
-- that is the operator's call. On by default: a library that cannot tell its
|
||||||
|
-- duplicates apart is what milestone #400 exists to end.
|
||||||
|
enabled boolean NOT NULL DEFAULT true,
|
||||||
|
|
||||||
|
-- Seconds of audio fpcalc fingerprints. Chromaprints taken at different
|
||||||
|
-- lengths cannot be compared, which is why track_fingerprints records the
|
||||||
|
-- length each row was taken at (below): change this and every chromaprint is
|
||||||
|
-- re-derived, and until then only rows at the new length are compared.
|
||||||
|
chromaprint_length_sec integer NOT NULL DEFAULT 120,
|
||||||
|
|
||||||
|
-- The most disagreement two aligned fingerprints may show and still be
|
||||||
|
-- proposed as one recording. Unrelated audio sits near 0.5, so the ceiling
|
||||||
|
-- stays well clear of it.
|
||||||
|
acoustic_max_bit_error_rate double precision NOT NULL DEFAULT 0.15,
|
||||||
|
|
||||||
|
-- Files the backfill decodes at once. Decoding competes with playback
|
||||||
|
-- transcoding for CPU and with streaming for the mount.
|
||||||
|
backfill_concurrency integer NOT NULL DEFAULT 2,
|
||||||
|
|
||||||
|
-- The least time between duplicate sweeps. A sweep still runs only when
|
||||||
|
-- fingerprints have changed since the last one.
|
||||||
|
sweep_interval_hours integer NOT NULL DEFAULT 1,
|
||||||
|
|
||||||
|
-- When the settings were last saved. A new threshold or length can change
|
||||||
|
-- what a sweep finds, so a save makes a sweep due.
|
||||||
|
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
|
||||||
|
CONSTRAINT fingerprint_settings_singleton CHECK (id = true),
|
||||||
|
CONSTRAINT fingerprint_settings_length_range
|
||||||
|
CHECK (chromaprint_length_sec >= 30 AND chromaprint_length_sec <= 600),
|
||||||
|
CONSTRAINT fingerprint_settings_threshold_range
|
||||||
|
CHECK (acoustic_max_bit_error_rate >= 0.01 AND acoustic_max_bit_error_rate <= 0.35),
|
||||||
|
CONSTRAINT fingerprint_settings_concurrency_range
|
||||||
|
CHECK (backfill_concurrency >= 1 AND backfill_concurrency <= 8),
|
||||||
|
CONSTRAINT fingerprint_settings_sweep_interval_range
|
||||||
|
CHECK (sweep_interval_hours >= 1 AND sweep_interval_hours <= 168)
|
||||||
|
);
|
||||||
|
INSERT INTO fingerprint_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING;
|
||||||
|
|
||||||
|
-- Every row written so far was taken at fpcalc's default length.
|
||||||
|
ALTER TABLE track_fingerprints ADD COLUMN chromaprint_length_sec integer NOT NULL DEFAULT 120;
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
-- name: StartDuplicateSweep :one
|
||||||
|
INSERT INTO duplicate_sweeps DEFAULT VALUES RETURNING id, started_at;
|
||||||
|
|
||||||
|
-- name: FinishDuplicateSweep :exec
|
||||||
|
UPDATE duplicate_sweeps
|
||||||
|
SET finished_at = now(),
|
||||||
|
candidates = sqlc.arg(candidates),
|
||||||
|
groups_found = sqlc.arg(groups_found),
|
||||||
|
oversize_clusters = sqlc.arg(oversize_clusters),
|
||||||
|
error_message = NULLIF(sqlc.arg(error_message)::text, '')
|
||||||
|
WHERE id = sqlc.arg(id);
|
||||||
|
|
||||||
|
-- name: GetInFlightDuplicateSweep :one
|
||||||
|
-- The guard against two sweeps at once: "in flight" is finished_at IS NULL.
|
||||||
|
SELECT id, started_at
|
||||||
|
FROM duplicate_sweeps
|
||||||
|
WHERE finished_at IS NULL
|
||||||
|
ORDER BY started_at DESC
|
||||||
|
LIMIT 1;
|
||||||
|
|
||||||
|
-- name: GetLatestDuplicateSweep :one
|
||||||
|
SELECT id, started_at, finished_at, candidates, groups_found, oversize_clusters, error_message
|
||||||
|
FROM duplicate_sweeps
|
||||||
|
ORDER BY started_at DESC
|
||||||
|
LIMIT 1;
|
||||||
|
|
||||||
|
-- name: GetLatestFingerprintComputedAt :one
|
||||||
|
-- Whether a sweep has anything new to look at: fingerprints written since the
|
||||||
|
-- last sweep started.
|
||||||
|
SELECT max(computed_at)::timestamptz AS latest FROM track_fingerprints;
|
||||||
|
|
||||||
|
-- name: ListExactDuplicateHashes :many
|
||||||
|
-- The exact tier, library-wide in one pass: identical encoded audio shared by
|
||||||
|
-- more than one present track.
|
||||||
|
SELECT f.audio_stream_sha256,
|
||||||
|
array_agg(t.id ORDER BY t.id)::uuid[] AS track_ids
|
||||||
|
FROM track_fingerprints f
|
||||||
|
JOIN tracks t ON t.id = f.track_id
|
||||||
|
WHERE t.missing_since IS NULL
|
||||||
|
AND f.fingerprint_version >= sqlc.arg(current_version)
|
||||||
|
AND f.audio_stream_sha256 IS NOT NULL
|
||||||
|
GROUP BY f.audio_stream_sha256
|
||||||
|
HAVING count(*) > 1;
|
||||||
|
|
||||||
|
-- name: ListDuplicateCandidates :many
|
||||||
|
-- The acoustic tier's input, one page at a time in (duration_ms, id) order so the
|
||||||
|
-- sweep holds only a sliding window of durations. Tracks without a chromaprint
|
||||||
|
-- cannot be compared acoustically and are left out; any exact duplicates among
|
||||||
|
-- them come from ListExactDuplicateHashes.
|
||||||
|
SELECT t.id, t.duration_ms, f.audio_stream_sha256, f.chromaprint
|
||||||
|
FROM tracks t
|
||||||
|
JOIN track_fingerprints f ON f.track_id = t.id
|
||||||
|
WHERE t.missing_since IS NULL
|
||||||
|
AND f.fingerprint_version >= sqlc.arg(current_version)
|
||||||
|
AND f.chromaprint IS NOT NULL
|
||||||
|
-- Only chromaprints taken at the current length: prints at two lengths are not
|
||||||
|
-- comparable, and after a length change the backfill is still re-deriving the
|
||||||
|
-- rest (#3913).
|
||||||
|
AND f.chromaprint_length_sec = sqlc.arg(chromaprint_length_sec)
|
||||||
|
AND (t.duration_ms, t.id) > (sqlc.arg(after_duration_ms)::integer, sqlc.arg(after_id)::uuid)
|
||||||
|
ORDER BY t.duration_ms, t.id
|
||||||
|
LIMIT sqlc.arg(page_limit);
|
||||||
|
|
||||||
|
-- name: ListDismissedDuplicateMemberSets :many
|
||||||
|
-- What the operator has already said are not duplicates. A new proposal whose
|
||||||
|
-- every member sat together in one of these is not proposed again.
|
||||||
|
SELECT g.id, array_agg(m.track_id ORDER BY m.track_id)::uuid[] AS track_ids
|
||||||
|
FROM duplicate_groups g
|
||||||
|
JOIN duplicate_group_members m ON m.group_id = g.id
|
||||||
|
WHERE g.status = 'dismissed'
|
||||||
|
GROUP BY g.id;
|
||||||
|
|
||||||
|
-- name: UpsertDuplicateGroup :one
|
||||||
|
-- Proposes a group, or refreshes one already pending. A group already dismissed
|
||||||
|
-- or merged is left exactly as it is: the WHERE on the update makes the conflict
|
||||||
|
-- a no-op, and the caller sees no row.
|
||||||
|
INSERT INTO duplicate_groups (member_key, tier, worst_bit_error_rate, last_seen_sweep_id)
|
||||||
|
VALUES (sqlc.arg(member_key), sqlc.arg(tier), sqlc.narg(worst_bit_error_rate), sqlc.arg(sweep_id))
|
||||||
|
ON CONFLICT (member_key) DO UPDATE
|
||||||
|
SET tier = EXCLUDED.tier,
|
||||||
|
worst_bit_error_rate = EXCLUDED.worst_bit_error_rate,
|
||||||
|
last_seen_sweep_id = EXCLUDED.last_seen_sweep_id
|
||||||
|
WHERE duplicate_groups.status = 'pending'
|
||||||
|
RETURNING id;
|
||||||
|
|
||||||
|
-- name: AddDuplicateGroupMember :exec
|
||||||
|
INSERT INTO duplicate_group_members (group_id, track_id)
|
||||||
|
VALUES (sqlc.arg(group_id), sqlc.arg(track_id))
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
-- name: DeleteStalePendingDuplicateGroups :execrows
|
||||||
|
-- A pending proposal this sweep did not find again no longer describes the
|
||||||
|
-- library: a member was re-fingerprinted, merged away or went missing. Dismissed
|
||||||
|
-- groups are kept regardless — they are the memory of a decision.
|
||||||
|
--
|
||||||
|
-- Only proposals last confirmed by an EARLIER sweep go. Should two sweeps ever
|
||||||
|
-- overlap (a manual trigger racing the worker), neither may delete what the other
|
||||||
|
-- has just found.
|
||||||
|
DELETE FROM duplicate_groups g
|
||||||
|
WHERE g.status = 'pending'
|
||||||
|
AND g.last_seen_sweep_id IS DISTINCT FROM sqlc.arg(sweep_id)
|
||||||
|
AND (g.last_seen_sweep_id IS NULL
|
||||||
|
OR (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = g.last_seen_sweep_id)
|
||||||
|
< (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = sqlc.arg(sweep_id)));
|
||||||
|
|
||||||
|
-- name: CountPendingDuplicateGroups :one
|
||||||
|
-- Proposals awaiting review. A group left with one member — its other tracks
|
||||||
|
-- deleted since the sweep — is no proposal at all and is not counted; the next
|
||||||
|
-- sweep retires it.
|
||||||
|
SELECT count(*)::bigint
|
||||||
|
FROM duplicate_groups g
|
||||||
|
WHERE g.status = 'pending'
|
||||||
|
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2;
|
||||||
|
|
||||||
|
-- name: ListPendingDuplicateGroupMembers :many
|
||||||
|
-- One page of proposals, newest first, flattened to one row per member so the
|
||||||
|
-- handler folds them without a query per group. What each copy carries — likes
|
||||||
|
-- and plays from every user — is here because it is what the operator weighs
|
||||||
|
-- when deciding which copy to keep.
|
||||||
|
WITH page AS (
|
||||||
|
SELECT g.id, g.tier, g.worst_bit_error_rate, g.detected_at
|
||||||
|
FROM duplicate_groups g
|
||||||
|
WHERE g.status = 'pending'
|
||||||
|
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2
|
||||||
|
ORDER BY g.detected_at DESC, g.id
|
||||||
|
LIMIT sqlc.arg(page_limit) OFFSET sqlc.arg(page_offset)
|
||||||
|
)
|
||||||
|
SELECT p.id AS group_id,
|
||||||
|
p.tier,
|
||||||
|
p.worst_bit_error_rate,
|
||||||
|
p.detected_at,
|
||||||
|
t.id AS track_id,
|
||||||
|
t.title,
|
||||||
|
artists.name AS artist_name,
|
||||||
|
albums.id AS album_id,
|
||||||
|
albums.title AS album_title,
|
||||||
|
t.file_path,
|
||||||
|
t.file_format,
|
||||||
|
t.file_size,
|
||||||
|
t.duration_ms,
|
||||||
|
t.added_at,
|
||||||
|
(SELECT count(*) FROM general_likes l WHERE l.track_id = t.id)::bigint AS like_count,
|
||||||
|
(SELECT count(*) FROM play_events e WHERE e.track_id = t.id)::bigint AS play_count
|
||||||
|
FROM page p
|
||||||
|
JOIN duplicate_group_members m ON m.group_id = p.id
|
||||||
|
JOIN tracks t ON t.id = m.track_id
|
||||||
|
JOIN albums ON albums.id = t.album_id
|
||||||
|
JOIN artists ON artists.id = t.artist_id
|
||||||
|
ORDER BY p.detected_at DESC, p.id, t.id;
|
||||||
|
|
||||||
|
-- name: DismissDuplicateGroup :execrows
|
||||||
|
-- "These are not duplicates." Only a pending group can be dismissed; zero rows
|
||||||
|
-- means it was already resolved or no longer exists.
|
||||||
|
UPDATE duplicate_groups
|
||||||
|
SET status = 'dismissed', resolved_at = now()
|
||||||
|
WHERE id = sqlc.arg(id) AND status = 'pending';
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
-- name: GetFingerprintSettings :one
|
||||||
|
SELECT * FROM fingerprint_settings WHERE id = true;
|
||||||
|
|
||||||
|
-- name: UpdateFingerprintSettings :one
|
||||||
|
-- Whole-row write from the admin card; migration 0061's CHECKs are the backstop
|
||||||
|
-- behind the service's own validation.
|
||||||
|
UPDATE fingerprint_settings
|
||||||
|
SET enabled = sqlc.arg(enabled),
|
||||||
|
chromaprint_length_sec = sqlc.arg(chromaprint_length_sec),
|
||||||
|
acoustic_max_bit_error_rate = sqlc.arg(acoustic_max_bit_error_rate),
|
||||||
|
backfill_concurrency = sqlc.arg(backfill_concurrency),
|
||||||
|
sweep_interval_hours = sqlc.arg(sweep_interval_hours),
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = true
|
||||||
|
RETURNING *;
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
-- name: UpsertTrackFingerprint :exec
|
||||||
|
-- Written whenever a track's fingerprint is derived: by the scan when a file is
|
||||||
|
-- new or its bytes changed, and by the backfill (#3908) for rows derived by an
|
||||||
|
-- older method. Replaces the row wholesale — a fingerprint of the old bytes has
|
||||||
|
-- no standing once the file has changed.
|
||||||
|
INSERT INTO track_fingerprints (
|
||||||
|
track_id, audio_stream_sha256, chromaprint, fingerprint_version, chromaprint_length_sec
|
||||||
|
) VALUES (
|
||||||
|
sqlc.arg(track_id), sqlc.narg(audio_stream_sha256), sqlc.narg(chromaprint),
|
||||||
|
sqlc.arg(fingerprint_version), sqlc.arg(chromaprint_length_sec)
|
||||||
|
)
|
||||||
|
ON CONFLICT (track_id) DO UPDATE SET
|
||||||
|
audio_stream_sha256 = EXCLUDED.audio_stream_sha256,
|
||||||
|
chromaprint = EXCLUDED.chromaprint,
|
||||||
|
fingerprint_version = EXCLUDED.fingerprint_version,
|
||||||
|
chromaprint_length_sec = EXCLUDED.chromaprint_length_sec,
|
||||||
|
computed_at = now();
|
||||||
|
|
||||||
|
-- name: DeleteTrackFingerprint :exec
|
||||||
|
-- A file changed but could not be fingerprinted, for a reason unrelated to the
|
||||||
|
-- file. The stored row describes the OLD bytes, so it goes and the backfill
|
||||||
|
-- re-derives it — nothing may keep trusting a stale identity.
|
||||||
|
DELETE FROM track_fingerprints WHERE track_id = $1;
|
||||||
|
|
||||||
|
-- name: ListTracksNeedingFingerprint :many
|
||||||
|
-- The backfill's work queue (#3908): tracks with no fingerprint, or one derived
|
||||||
|
-- by an older method. Keyset-paged on id so a pass visits each track at most
|
||||||
|
-- once. That cursor is load-bearing: an inconclusive attempt writes no row, so
|
||||||
|
-- without it a file that keeps timing out would be listed again straight away
|
||||||
|
-- and retried in a tight loop. Missing tracks are skipped — there is no file to
|
||||||
|
-- read.
|
||||||
|
SELECT t.id, t.file_path
|
||||||
|
FROM tracks t
|
||||||
|
LEFT JOIN track_fingerprints f ON f.track_id = t.id
|
||||||
|
WHERE t.missing_since IS NULL
|
||||||
|
-- A row taken at another length is as stale as one from an older method:
|
||||||
|
-- chromaprints at two lengths cannot be compared (#3913).
|
||||||
|
AND (f.track_id IS NULL
|
||||||
|
OR f.fingerprint_version < sqlc.arg(current_version)
|
||||||
|
OR f.chromaprint_length_sec <> sqlc.arg(chromaprint_length_sec))
|
||||||
|
AND t.id > sqlc.arg(after_id)
|
||||||
|
ORDER BY t.id
|
||||||
|
LIMIT sqlc.arg(batch_limit);
|
||||||
|
|
||||||
|
-- name: GetFingerprintCoverage :one
|
||||||
|
-- The admin gauge for the backfill. fingerprinted + rejected + pending = total.
|
||||||
|
-- "Current" means derived by the current method AT the current length: a row at
|
||||||
|
-- another length is pending, because the backfill will re-derive it. rejected is
|
||||||
|
-- a current row with a NULL half: a tool ran and refused the file, which is
|
||||||
|
-- settled rather than waiting. Missing tracks are excluded, or the gauge could
|
||||||
|
-- never reach the end.
|
||||||
|
SELECT count(*)::bigint AS total,
|
||||||
|
count(*) FILTER (
|
||||||
|
WHERE f.fingerprint_version >= sqlc.arg(current_version)
|
||||||
|
AND f.chromaprint_length_sec = sqlc.arg(chromaprint_length_sec)
|
||||||
|
AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL
|
||||||
|
)::bigint AS fingerprinted,
|
||||||
|
count(*) FILTER (
|
||||||
|
WHERE f.fingerprint_version >= sqlc.arg(current_version)
|
||||||
|
AND f.chromaprint_length_sec = sqlc.arg(chromaprint_length_sec)
|
||||||
|
AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL)
|
||||||
|
)::bigint AS rejected,
|
||||||
|
count(*) FILTER (
|
||||||
|
WHERE f.track_id IS NULL
|
||||||
|
OR f.fingerprint_version < sqlc.arg(current_version)
|
||||||
|
OR f.chromaprint_length_sec <> sqlc.arg(chromaprint_length_sec)
|
||||||
|
)::bigint AS pending
|
||||||
|
FROM tracks t
|
||||||
|
LEFT JOIN track_fingerprints f ON f.track_id = t.id
|
||||||
|
WHERE t.missing_since IS NULL;
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
-- Duplicate merge (Scribe #3911). Every statement here runs inside the one
|
||||||
|
-- transaction library.MergeDuplicateGroup opens, after the removed copy's file
|
||||||
|
-- is already gone. The loser's own track row is deleted last with DeleteTrack;
|
||||||
|
-- what these do is move everything it carries onto the survivor first, so that
|
||||||
|
-- delete's CASCADE finds nothing left to destroy.
|
||||||
|
|
||||||
|
-- name: LockDuplicateGroupForMerge :one
|
||||||
|
-- Locks the group for the rest of the transaction, so two merges of one group
|
||||||
|
-- cannot run at once.
|
||||||
|
SELECT id, tier, status
|
||||||
|
FROM duplicate_groups
|
||||||
|
WHERE id = sqlc.arg(id)
|
||||||
|
FOR UPDATE;
|
||||||
|
|
||||||
|
-- name: ListDuplicateGroupMergeMembers :many
|
||||||
|
SELECT t.id, t.file_path, t.file_format, t.file_size, t.added_at, t.album_id,
|
||||||
|
t.mbid, albums.mbid AS album_mbid
|
||||||
|
FROM duplicate_group_members m
|
||||||
|
JOIN tracks t ON t.id = m.track_id
|
||||||
|
JOIN albums ON albums.id = t.album_id
|
||||||
|
WHERE m.group_id = sqlc.arg(group_id)
|
||||||
|
ORDER BY t.id;
|
||||||
|
|
||||||
|
-- Plain repoints: no unique key involves track_id, so moving rows cannot collide.
|
||||||
|
|
||||||
|
-- name: MergeRepointPlayEvents :execrows
|
||||||
|
UPDATE play_events SET track_id = sqlc.arg(survivor_id)::uuid WHERE track_id = sqlc.arg(loser_id)::uuid;
|
||||||
|
|
||||||
|
-- name: MergeRepointSkipEvents :execrows
|
||||||
|
UPDATE skip_events SET track_id = sqlc.arg(survivor_id)::uuid WHERE track_id = sqlc.arg(loser_id)::uuid;
|
||||||
|
|
||||||
|
-- name: MergeRepointContextualLikes :execrows
|
||||||
|
UPDATE contextual_likes SET track_id = sqlc.arg(survivor_id)::uuid WHERE track_id = sqlc.arg(loser_id)::uuid;
|
||||||
|
|
||||||
|
-- name: MergeRepointPlaybackErrors :execrows
|
||||||
|
UPDATE playback_errors SET track_id = sqlc.arg(survivor_id)::uuid WHERE track_id = sqlc.arg(loser_id)::uuid;
|
||||||
|
|
||||||
|
-- name: MergeRepointLidarrRequests :execrows
|
||||||
|
UPDATE lidarr_requests SET matched_track_id = sqlc.arg(survivor_id)::uuid
|
||||||
|
WHERE matched_track_id = sqlc.arg(loser_id)::uuid;
|
||||||
|
|
||||||
|
-- name: MergeRepointPlaylistTracks :many
|
||||||
|
-- playlist_tracks is keyed by (playlist_id, position), so repointing keeps every
|
||||||
|
-- entry exactly where it was. A playlist that held both copies simply holds the
|
||||||
|
-- survivor twice — the user put two entries there, and both stay.
|
||||||
|
UPDATE playlist_tracks SET track_id = sqlc.arg(survivor_id)::uuid
|
||||||
|
WHERE track_id = sqlc.arg(loser_id)::uuid
|
||||||
|
RETURNING playlist_id;
|
||||||
|
|
||||||
|
-- Collision-safe merges: a unique key includes track_id, so the survivor may
|
||||||
|
-- already hold a matching row. Copy what it lacks; DeleteTrack's CASCADE then
|
||||||
|
-- removes the loser's originals.
|
||||||
|
|
||||||
|
-- name: MergeCopyGeneralLikes :many
|
||||||
|
-- One like per user. A user who liked both copies keeps a single like, dated to
|
||||||
|
-- the earlier of the two.
|
||||||
|
INSERT INTO general_likes (user_id, track_id, liked_at)
|
||||||
|
SELECT user_id, sqlc.arg(survivor_id)::uuid, liked_at
|
||||||
|
FROM general_likes
|
||||||
|
WHERE track_id = sqlc.arg(loser_id)::uuid
|
||||||
|
ON CONFLICT (user_id, track_id) DO UPDATE
|
||||||
|
SET liked_at = LEAST(general_likes.liked_at, EXCLUDED.liked_at)
|
||||||
|
RETURNING user_id;
|
||||||
|
|
||||||
|
-- name: MergeCopyTrackTags :execrows
|
||||||
|
INSERT INTO track_tags (track_id, tag, weight)
|
||||||
|
SELECT sqlc.arg(survivor_id)::uuid, tag, weight
|
||||||
|
FROM track_tags
|
||||||
|
WHERE track_id = sqlc.arg(loser_id)::uuid
|
||||||
|
ON CONFLICT (track_id, tag) DO NOTHING;
|
||||||
|
|
||||||
|
-- name: MergeCopyTrackSimilarity :execrows
|
||||||
|
-- Rewrites the loser to the survivor on either side of an edge. An edge between
|
||||||
|
-- the two copies would become a track similar to itself — the table forbids
|
||||||
|
-- that, and it means nothing — so it is dropped. An edge the survivor already
|
||||||
|
-- has from the same source is kept as it is.
|
||||||
|
INSERT INTO track_similarity (track_a_id, track_b_id, score, source, fetched_at)
|
||||||
|
SELECT CASE WHEN track_a_id = sqlc.arg(loser_id)::uuid THEN sqlc.arg(survivor_id)::uuid ELSE track_a_id END,
|
||||||
|
CASE WHEN track_b_id = sqlc.arg(loser_id)::uuid THEN sqlc.arg(survivor_id)::uuid ELSE track_b_id END,
|
||||||
|
score, source, fetched_at
|
||||||
|
FROM track_similarity
|
||||||
|
WHERE (track_a_id = sqlc.arg(loser_id)::uuid OR track_b_id = sqlc.arg(loser_id)::uuid)
|
||||||
|
AND (CASE WHEN track_a_id = sqlc.arg(loser_id)::uuid THEN sqlc.arg(survivor_id)::uuid ELSE track_a_id END)
|
||||||
|
<> (CASE WHEN track_b_id = sqlc.arg(loser_id)::uuid THEN sqlc.arg(survivor_id)::uuid ELSE track_b_id END)
|
||||||
|
ON CONFLICT (track_a_id, track_b_id, source) DO NOTHING;
|
||||||
|
|
||||||
|
-- name: MergeInheritTrackMbid :exec
|
||||||
|
-- A recording MBID is what the similarity pipeline keys on. If only the removed
|
||||||
|
-- copy carried one, the survivor takes it rather than going dark to similarity.
|
||||||
|
UPDATE tracks AS survivor
|
||||||
|
SET mbid = loser.mbid
|
||||||
|
FROM tracks AS loser
|
||||||
|
WHERE survivor.id = sqlc.arg(survivor_id)::uuid
|
||||||
|
AND loser.id = sqlc.arg(loser_id)::uuid
|
||||||
|
AND survivor.mbid IS NULL
|
||||||
|
AND loser.mbid IS NOT NULL;
|
||||||
|
|
||||||
|
-- name: MarkDuplicateGroupMerged :execrows
|
||||||
|
UPDATE duplicate_groups
|
||||||
|
SET status = 'merged', resolved_at = now()
|
||||||
|
WHERE id = sqlc.arg(id) AND status = 'pending';
|
||||||
@@ -45,7 +45,22 @@ WHERE t.id <> $2
|
|||||||
-- enter the pool even when the similarity/random arms miss them; scored
|
-- enter the pool even when the similarity/random arms miss them; scored
|
||||||
-- in Go via TasteMatch, so sim_score here is 0 pool-inclusion),
|
-- in Go via TasteMatch, so sim_score here is 0 pool-inclusion),
|
||||||
-- $11 coplay_artists K (#1533 — tracks by artists co-played across the
|
-- $11 coplay_artists K (#1533 — tracks by artists co-played across the
|
||||||
-- instance with the seed's artist; source='user_cooccurrence').
|
-- instance with the seed's artist; source='user_cooccurrence'),
|
||||||
|
-- $12 order_seed (text) — see below.
|
||||||
|
--
|
||||||
|
-- $12 REPLACES `ORDER BY random()` IN FOUR ARMS (#3889). Those arms returned
|
||||||
|
-- a stable set only while their LIMIT exceeded the rows eligible for them: at
|
||||||
|
-- that point they returned all of them and the order stopped mattering,
|
||||||
|
-- because the caller sorts by track id before scoring. Below that threshold
|
||||||
|
-- they returned a random SUBSET, and two builds on the same day drew
|
||||||
|
-- different ones — so "daily determinism" held by accident, and only for
|
||||||
|
-- libraries smaller than the limits.
|
||||||
|
--
|
||||||
|
-- md5(id || seed) keeps the intent — an arbitrary spread that changes when
|
||||||
|
-- the seed does — while making it reproducible for a given seed. The CALLER
|
||||||
|
-- decides what that means: system mixes pass a per-(user, day) string and get
|
||||||
|
-- the determinism they promise; radio passes a fresh value per request and
|
||||||
|
-- keeps varying, which is what a radio should do.
|
||||||
-- Returns same shape as LoadRadioCandidates plus similarity_score column.
|
-- Returns same shape as LoadRadioCandidates plus similarity_score column.
|
||||||
|
|
||||||
WITH
|
WITH
|
||||||
@@ -87,7 +102,7 @@ similar_artists AS (
|
|||||||
JOIN seed_artist sa ON asim.artist_a_id = sa.artist_id
|
JOIN seed_artist sa ON asim.artist_a_id = sa.artist_id
|
||||||
WHERE asim.source = 'listenbrainz'
|
WHERE asim.source = 'listenbrainz'
|
||||||
AND t.id NOT IN (SELECT id FROM excluded_ids)
|
AND t.id NOT IN (SELECT id FROM excluded_ids)
|
||||||
ORDER BY asim.score DESC, random()
|
ORDER BY asim.score DESC, md5(t.id::text || $12::text)
|
||||||
LIMIT $6
|
LIMIT $6
|
||||||
),
|
),
|
||||||
tag_overlap AS (
|
tag_overlap AS (
|
||||||
@@ -115,7 +130,7 @@ likes_overlap AS (
|
|||||||
WHERE t.id = gl.track_id
|
WHERE t.id = gl.track_id
|
||||||
AND trim(g_overlap.g) IN (SELECT tag FROM seed_tags)
|
AND trim(g_overlap.g) IN (SELECT tag FROM seed_tags)
|
||||||
)
|
)
|
||||||
ORDER BY random()
|
ORDER BY md5(gl.track_id::text || $12::text)
|
||||||
LIMIT $8
|
LIMIT $8
|
||||||
),
|
),
|
||||||
taste_overlap AS (
|
taste_overlap AS (
|
||||||
@@ -142,7 +157,7 @@ coplay_artists AS (
|
|||||||
WHERE asim.source = 'user_cooccurrence'
|
WHERE asim.source = 'user_cooccurrence'
|
||||||
AND t.id NOT IN (SELECT id FROM excluded_ids)
|
AND t.id NOT IN (SELECT id FROM excluded_ids)
|
||||||
AND t.id <> $2
|
AND t.id <> $2
|
||||||
ORDER BY asim.score DESC, random()
|
ORDER BY asim.score DESC, md5(t.id::text || $12::text)
|
||||||
LIMIT $11
|
LIMIT $11
|
||||||
),
|
),
|
||||||
random_fill AS (
|
random_fill AS (
|
||||||
@@ -158,7 +173,7 @@ random_fill AS (
|
|||||||
UNION SELECT track_id FROM taste_overlap
|
UNION SELECT track_id FROM taste_overlap
|
||||||
UNION SELECT track_id FROM coplay_artists
|
UNION SELECT track_id FROM coplay_artists
|
||||||
)
|
)
|
||||||
ORDER BY random()
|
ORDER BY md5(t.id::text || $12::text)
|
||||||
LIMIT $9
|
LIMIT $9
|
||||||
)
|
)
|
||||||
SELECT
|
SELECT
|
||||||
|
|||||||
@@ -155,17 +155,19 @@ SELECT id, file_path FROM tracks
|
|||||||
AND mbid = sqlc.arg(mbid)::text
|
AND mbid = sqlc.arg(mbid)::text
|
||||||
LIMIT 2;
|
LIMIT 2;
|
||||||
|
|
||||||
-- name: FindMissingTrackByFingerprint :many
|
-- name: FindMissingTrackByAudioHash :many
|
||||||
-- Move detection fallback for files with no MBID (#2528). Exact byte size AND
|
-- Move detection fallback for files with no MBID (#2528, #3914). The audio stream
|
||||||
-- exact decoded duration is a strong pair: a plain move or rename preserves
|
-- hash identifies the encoded audio itself, so it survives a rename, a move and a
|
||||||
-- both, while a re-encode changes at least one — and a re-encode genuinely is a
|
-- retag — anything short of a re-encode. It replaced (file_size, duration_ms),
|
||||||
-- different file, so failing to match there is correct rather than a gap.
|
-- which could pair two unrelated files that happened to share a byte count and a
|
||||||
|
-- duration, and missed a file retagged in place, whose size changes.
|
||||||
--
|
--
|
||||||
-- Same missing-only constraint and same LIMIT 2 rationale as the MBID variant.
|
-- Same missing-only constraint and same LIMIT 2 rationale as the MBID variant.
|
||||||
SELECT id, file_path FROM tracks
|
SELECT t.id, t.file_path
|
||||||
WHERE missing_since IS NOT NULL
|
FROM tracks t
|
||||||
AND file_size = sqlc.arg(file_size)
|
JOIN track_fingerprints f ON f.track_id = t.id
|
||||||
AND duration_ms = sqlc.arg(duration_ms)
|
WHERE t.missing_since IS NOT NULL
|
||||||
|
AND f.audio_stream_sha256 = sqlc.arg(audio_stream_sha256)
|
||||||
LIMIT 2;
|
LIMIT 2;
|
||||||
|
|
||||||
-- name: AdoptTrackPath :execrows
|
-- name: AdoptTrackPath :execrows
|
||||||
|
|||||||
@@ -87,6 +87,10 @@ var dataTables = []string{
|
|||||||
// pristine Discover knobs rather than whatever a previous test tuned.
|
// pristine Discover knobs rather than whatever a previous test tuned.
|
||||||
"discover_tuning",
|
"discover_tuning",
|
||||||
"recommendation_tuning_audit",
|
"recommendation_tuning_audit",
|
||||||
|
"duplicate_group_members", // M400
|
||||||
|
"duplicate_groups",
|
||||||
|
"duplicate_sweeps",
|
||||||
|
"track_fingerprints", // M400
|
||||||
"tracks",
|
"tracks",
|
||||||
"albums",
|
"albums",
|
||||||
"artists",
|
"artists",
|
||||||
@@ -126,4 +130,15 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) {
|
|||||||
); err != nil {
|
); err != nil {
|
||||||
t.Fatalf("dbtest.ResetDB reset tag-sources version: %v", err)
|
t.Fatalf("dbtest.ResetDB reset tag-sources version: %v", err)
|
||||||
}
|
}
|
||||||
|
// Fingerprinting settings (M400 #3913), a singleton like the counters above.
|
||||||
|
// Every column goes back to its migration default rather than to literals
|
||||||
|
// written here, so a test can pin the Go defaults to the migration's.
|
||||||
|
if _, err := pool.Exec(ctx, `
|
||||||
|
UPDATE fingerprint_settings
|
||||||
|
SET enabled = DEFAULT, chromaprint_length_sec = DEFAULT,
|
||||||
|
acoustic_max_bit_error_rate = DEFAULT, backfill_concurrency = DEFAULT,
|
||||||
|
sweep_interval_hours = DEFAULT, updated_at = DEFAULT`,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("dbtest.ResetDB reset fingerprint settings: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+161
-32
@@ -5,12 +5,16 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
"github.com/jackc/pgx/v5/pgtype"
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
|
||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
|
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
|
||||||
)
|
)
|
||||||
@@ -19,54 +23,179 @@ import (
|
|||||||
// that has no row in tracks.
|
// that has no row in tracks.
|
||||||
var ErrTrackNotFound = errors.New("library: track not found")
|
var ErrTrackNotFound = errors.New("library: track not found")
|
||||||
|
|
||||||
// DeleteTrackFile removes a track file from disk and its row from the
|
// removeFile is os.Remove behind a variable so a test can make removal fail the
|
||||||
// tracks table. Album and artist rows are left untouched.
|
// way a read-only mount or a wrongly-owned directory does. A chmod-based test
|
||||||
|
// cannot stand in for that: root ignores permission bits, so in a CI container
|
||||||
|
// running as root it would pass without ever exercising the failure.
|
||||||
|
var removeFile = os.Remove
|
||||||
|
|
||||||
|
// FileRemoveError reports that a track's file exists but could not be removed.
|
||||||
|
// When DeleteTrackFile returns one, NOTHING was deleted: the row, its likes, its
|
||||||
|
// play history and its playlist memberships are all intact.
|
||||||
|
type FileRemoveError struct {
|
||||||
|
Path string
|
||||||
|
// UID and GID are the identity the server process runs as — the half of a
|
||||||
|
// permission problem the operator cannot see from the host side.
|
||||||
|
UID, GID int
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *FileRemoveError) Error() string { return fmt.Sprintf("remove track file: %v", e.Err) }
|
||||||
|
|
||||||
|
func (e *FileRemoveError) Unwrap() error { return e.Err }
|
||||||
|
|
||||||
|
// Dir is the directory removal needs write access to. Unlinking a file writes to
|
||||||
|
// its PARENT, so a world-writable file inside a read-only directory still cannot
|
||||||
|
// be removed — naming the file's own permissions would send the operator to the
|
||||||
|
// wrong place.
|
||||||
|
func (e *FileRemoveError) Dir() string { return filepath.Dir(e.Path) }
|
||||||
|
|
||||||
|
// NotWritable reports whether the library is unwritable for this process — a
|
||||||
|
// read-only mount or a permission denial — rather than an I/O fault. It is the
|
||||||
|
// case the operator can fix, so callers answer it differently.
|
||||||
|
func (e *FileRemoveError) NotWritable() bool {
|
||||||
|
return errors.Is(e.Err, fs.ErrPermission) || errors.Is(e.Err, syscall.EROFS)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reason is the underlying cause without the path os.Remove already wrapped
|
||||||
|
// around it, for messages that name the directory themselves.
|
||||||
|
func (e *FileRemoveError) Reason() string {
|
||||||
|
var pathErr *fs.PathError
|
||||||
|
if errors.As(e.Err, &pathErr) {
|
||||||
|
return pathErr.Err.Error()
|
||||||
|
}
|
||||||
|
return e.Err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeletedTrack reports what a delete tidied away beyond the track itself.
|
||||||
|
type DeletedTrack struct {
|
||||||
|
// AlbumID is set when the track was its album's last, so the album went too.
|
||||||
|
AlbumID *pgtype.UUID
|
||||||
|
// ArtistID is set when that album was its artist's last, so the artist went too.
|
||||||
|
ArtistID *pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteTrackFile removes a track's file from disk and then its row, tidying
|
||||||
|
// away an album or artist the delete leaves empty. It is the ONLY path that
|
||||||
|
// deletes a track file: the admin remove-track endpoint and quarantine's Delete
|
||||||
|
// file both come through here (#3918).
|
||||||
//
|
//
|
||||||
// Steps:
|
// Order is the whole contract. The file goes first, and if it cannot go — a
|
||||||
// 1. Look up the track to get its file_path.
|
// read-only mount, a permission denial, an I/O error — nothing else happens and
|
||||||
// 2. Remove the file from disk. fs.ErrNotExist is OK — already gone.
|
// a *FileRemoveError comes back. Proceeding past that failure is how #3918 lost
|
||||||
// 3. Delete the tracks row.
|
// history: tracks CASCADEs to play_events, general_likes, contextual_likes,
|
||||||
|
// playlist_tracks, track_tags and playback_errors, so the row and everything
|
||||||
|
// hanging off it were destroyed while the file survived, and the next scan
|
||||||
|
// re-imported it as a brand-new track with none of it.
|
||||||
//
|
//
|
||||||
// Order matters: file first, then DB. If the file delete fails (permission,
|
// A file that is already gone (fs.ErrNotExist) is not a failure; the row is
|
||||||
// I/O error), we leave the DB row alone so the admin can retry.
|
// removed as asked.
|
||||||
//
|
//
|
||||||
// The reverse failure mode — file gone, DB row still present — IS reconciled
|
// This is NOT the missing-file path. That lifecycle is deliberately
|
||||||
// now, and not by this function: the scan's reconcile pass stamps
|
// non-destructive: reconcile stamps missing_since (#2523), selection paths
|
||||||
// tracks.missing_since (#2523), every selection path filters on it, and a file
|
// filter on it, and a returning file is un-marked or adopted (#2528). This is the
|
||||||
// that returns is un-marked or adopted at its new path (#2528). That is the
|
// explicit, irreversible "remove this recording", never the way to tidy up a row
|
||||||
// normal life of a vanished file and it is deliberately non-destructive: the
|
// whose file merely went away.
|
||||||
// row, its play history and its likes survive, because a missing file is a
|
|
||||||
// track Minstrel still knows about (#2527).
|
|
||||||
//
|
//
|
||||||
// So this function is NOT the missing-file path. It is the explicit admin
|
// dataDir, when set, also clears the cached art of an artist the delete removed.
|
||||||
// action "remove this recording from disk and from the library", and it is
|
// logger may be nil.
|
||||||
// irreversible: tracks CASCADEs to play_events, general_likes_tracks,
|
func DeleteTrackFile(
|
||||||
// contextual_likes, track_tags and playback_errors. Reach for it when the
|
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, dataDir string, trackID pgtype.UUID,
|
||||||
// operator means to destroy the record, never to tidy up a row whose file
|
) (DeletedTrack, error) {
|
||||||
// merely went away.
|
if logger == nil {
|
||||||
func DeleteTrackFile(ctx context.Context, pool *pgxpool.Pool, trackID pgtype.UUID) error {
|
logger = slog.Default()
|
||||||
|
}
|
||||||
q := dbq.New(pool)
|
q := dbq.New(pool)
|
||||||
track, err := q.GetTrackByID(ctx, trackID)
|
track, err := q.GetTrackByID(ctx, trackID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
return ErrTrackNotFound
|
return DeletedTrack{}, ErrTrackNotFound
|
||||||
}
|
}
|
||||||
return fmt.Errorf("get track: %w", err)
|
return DeletedTrack{}, fmt.Errorf("get track: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.Remove(track.FilePath); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
if err := removeTrackFileOnDisk(track.FilePath); err != nil {
|
||||||
return fmt.Errorf("remove file: %w", err)
|
return DeletedTrack{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := pool.Exec(ctx, "DELETE FROM tracks WHERE id = $1", trackID); err != nil {
|
// The row and any album or artist it empties go together, so a failure
|
||||||
return fmt.Errorf("delete row: %w", err)
|
// partway cannot leave a deleted track with a ghost album behind it.
|
||||||
|
tx, err := pool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return DeletedTrack{}, fmt.Errorf("begin tx: %w", err)
|
||||||
}
|
}
|
||||||
// Log the change after the delete succeeds. Best-effort: a Warn-level
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
// failure here would leave the cache index orphaned on offline clients
|
tq := dbq.New(tx)
|
||||||
// until the next scan touches the surrounding album.
|
|
||||||
|
deleted, err := tq.DeleteTrack(ctx, trackID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
// Removed by someone else between the lookup and here.
|
||||||
|
return DeletedTrack{}, ErrTrackNotFound
|
||||||
|
}
|
||||||
|
return DeletedTrack{}, fmt.Errorf("delete track: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := tidyEmptiedAlbum(ctx, tq, deleted.AlbumID)
|
||||||
|
if err != nil {
|
||||||
|
return DeletedTrack{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return DeletedTrack{}, fmt.Errorf("commit: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both of these run after the delete has committed, so neither may fail
|
||||||
|
// it: the recording is gone either way. An unlogged change leaves the track
|
||||||
|
// in offline clients' caches until the next scan touches its album; a
|
||||||
|
// leftover art directory is only disk.
|
||||||
if err := syncpkg.LogChange(ctx, pool, syncpkg.EntityTrack,
|
if err := syncpkg.LogChange(ctx, pool, syncpkg.EntityTrack,
|
||||||
syncpkg.FormatUUID(trackID), syncpkg.OpDelete); err != nil {
|
syncpkg.FormatUUID(trackID), syncpkg.OpDelete); err != nil {
|
||||||
return fmt.Errorf("log change: %w", err)
|
logger.Warn("track delete: LogChange failed", "track_id", syncpkg.FormatUUID(trackID), "err", err)
|
||||||
|
}
|
||||||
|
if out.ArtistID != nil && dataDir != "" {
|
||||||
|
if err := coverart.CleanupArtistArt(dataDir, *out.ArtistID); err != nil {
|
||||||
|
logger.Warn("track delete: artist-art cleanup failed",
|
||||||
|
"artist_id", syncpkg.FormatUUID(*out.ArtistID), "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeTrackFileOnDisk is the one rule for removing a track's file, shared by
|
||||||
|
// DeleteTrackFile and the duplicate merge. A file already gone is fine; anything
|
||||||
|
// else comes back as a *FileRemoveError, and the caller must then change nothing
|
||||||
|
// in the database (#3918).
|
||||||
|
func removeTrackFileOnDisk(path string) error {
|
||||||
|
if err := removeFile(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return &FileRemoveError{Path: path, UID: os.Getuid(), GID: os.Getgid(), Err: err}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tidyEmptiedAlbum removes an album a track delete left with no tracks, and its
|
||||||
|
// artist if that album was the artist's last. It runs on the caller's
|
||||||
|
// transaction, so the tidy-up commits or rolls back with the delete itself.
|
||||||
|
func tidyEmptiedAlbum(ctx context.Context, tq *dbq.Queries, albumID pgtype.UUID) (DeletedTrack, error) {
|
||||||
|
var out DeletedTrack
|
||||||
|
album, err := tq.DeleteAlbumIfEmpty(ctx, albumID)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
id := album.ID
|
||||||
|
out.AlbumID = &id
|
||||||
|
artistID, aerr := tq.DeleteArtistIfEmpty(ctx, album.ArtistID)
|
||||||
|
switch {
|
||||||
|
case aerr == nil:
|
||||||
|
out.ArtistID = &artistID
|
||||||
|
case errors.Is(aerr, pgx.ErrNoRows):
|
||||||
|
// The artist still has other albums or stray tracks.
|
||||||
|
default:
|
||||||
|
return DeletedTrack{}, fmt.Errorf("delete artist if empty: %w", aerr)
|
||||||
|
}
|
||||||
|
case errors.Is(err, pgx.ErrNoRows):
|
||||||
|
// The album still has other tracks.
|
||||||
|
default:
|
||||||
|
return DeletedTrack{}, fmt.Errorf("delete album if empty: %w", err)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,9 +4,11 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
|
"io/fs"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5/pgtype"
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
@@ -64,6 +66,15 @@ func seedTrack(t *testing.T, pool *pgxpool.Pool, filePath string) (dbq.Track, db
|
|||||||
return track, album, artist
|
return track, album, artist
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stubRemoveFile makes file removal fail (or succeed) on demand for one test.
|
||||||
|
// See removeFile for why this is a seam rather than a chmod.
|
||||||
|
func stubRemoveFile(t *testing.T, fn func(string) error) {
|
||||||
|
t.Helper()
|
||||||
|
orig := removeFile
|
||||||
|
removeFile = fn
|
||||||
|
t.Cleanup(func() { removeFile = orig })
|
||||||
|
}
|
||||||
|
|
||||||
func TestDeleteTrackFile_HappyPath(t *testing.T) {
|
func TestDeleteTrackFile_HappyPath(t *testing.T) {
|
||||||
pool := newPool(t)
|
pool := newPool(t)
|
||||||
q := dbq.New(pool)
|
q := dbq.New(pool)
|
||||||
@@ -73,9 +84,18 @@ func TestDeleteTrackFile_HappyPath(t *testing.T) {
|
|||||||
if err := os.WriteFile(path, []byte("payload"), 0o644); err != nil {
|
if err := os.WriteFile(path, []byte("payload"), 0o644); err != nil {
|
||||||
t.Fatalf("write file: %v", err)
|
t.Fatalf("write file: %v", err)
|
||||||
}
|
}
|
||||||
track, album, _ := seedTrack(t, pool, path)
|
track, album, artist := seedTrack(t, pool, path)
|
||||||
|
// A sibling keeps the album non-empty, so this case pins that the tidy-up
|
||||||
|
// only removes an album the delete actually emptied.
|
||||||
|
if _, err := q.UpsertTrack(context.Background(), dbq.UpsertTrackParams{
|
||||||
|
Title: "Sibling", AlbumID: album.ID, ArtistID: artist.ID,
|
||||||
|
DurationMs: 1000, FilePath: filepath.Join(dir, "sibling.mp3"), FileSize: 100, FileFormat: "mp3",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("sibling: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
if err := DeleteTrackFile(context.Background(), pool, track.ID); err != nil {
|
got, err := DeleteTrackFile(context.Background(), pool, nil, "", track.ID)
|
||||||
|
if err != nil {
|
||||||
t.Fatalf("DeleteTrackFile: %v", err)
|
t.Fatalf("DeleteTrackFile: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,9 +105,99 @@ func TestDeleteTrackFile_HappyPath(t *testing.T) {
|
|||||||
if _, err := q.GetTrackByID(context.Background(), track.ID); err == nil {
|
if _, err := q.GetTrackByID(context.Background(), track.ID); err == nil {
|
||||||
t.Errorf("track row still exists")
|
t.Errorf("track row still exists")
|
||||||
}
|
}
|
||||||
// Album row preserved (other tracks may reference it).
|
|
||||||
if _, err := q.GetAlbumByID(context.Background(), album.ID); err != nil {
|
if _, err := q.GetAlbumByID(context.Background(), album.ID); err != nil {
|
||||||
t.Errorf("album row vanished: %v", err)
|
t.Errorf("album with a remaining track vanished: %v", err)
|
||||||
|
}
|
||||||
|
if got.AlbumID != nil || got.ArtistID != nil {
|
||||||
|
t.Errorf("reported tidy-up %+v for an album that still has a track", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeleteTrackFile_EmptiedAlbumAndArtistGoToo(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
q := dbq.New(pool)
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "lone.mp3")
|
||||||
|
if err := os.WriteFile(path, []byte("payload"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write file: %v", err)
|
||||||
|
}
|
||||||
|
track, album, artist := seedTrack(t, pool, path)
|
||||||
|
|
||||||
|
got, err := DeleteTrackFile(context.Background(), pool, nil, "", track.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DeleteTrackFile: %v", err)
|
||||||
|
}
|
||||||
|
if got.AlbumID == nil || *got.AlbumID != album.ID {
|
||||||
|
t.Errorf("AlbumID = %v, want %v", got.AlbumID, album.ID)
|
||||||
|
}
|
||||||
|
if got.ArtistID == nil || *got.ArtistID != artist.ID {
|
||||||
|
t.Errorf("ArtistID = %v, want %v", got.ArtistID, artist.ID)
|
||||||
|
}
|
||||||
|
if _, err := q.GetAlbumByID(context.Background(), album.ID); err == nil {
|
||||||
|
t.Errorf("emptied album row still exists")
|
||||||
|
}
|
||||||
|
if _, err := q.GetArtistByID(context.Background(), artist.ID); err == nil {
|
||||||
|
t.Errorf("emptied artist row still exists")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The #3918 proof. A file that cannot be removed must leave EVERYTHING in place:
|
||||||
|
// the row is what carries likes, plays and playlist memberships, and the file
|
||||||
|
// surviving means the next scan would re-import it as a stranger.
|
||||||
|
func TestDeleteTrackFile_UnremovableFileDeletesNothing(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
errno syscall.Errno
|
||||||
|
notWritable bool
|
||||||
|
}{
|
||||||
|
{"read-only mount", syscall.EROFS, true},
|
||||||
|
{"permission denied", syscall.EACCES, true},
|
||||||
|
{"operation not permitted", syscall.EPERM, true},
|
||||||
|
{"i/o error", syscall.EIO, false},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
q := dbq.New(pool)
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "track.mp3")
|
||||||
|
if err := os.WriteFile(path, []byte("payload"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write file: %v", err)
|
||||||
|
}
|
||||||
|
track, album, _ := seedTrack(t, pool, path)
|
||||||
|
stubRemoveFile(t, func(name string) error {
|
||||||
|
return &fs.PathError{Op: "remove", Path: name, Err: tc.errno}
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := DeleteTrackFile(context.Background(), pool, nil, "", track.ID)
|
||||||
|
|
||||||
|
var fre *FileRemoveError
|
||||||
|
if !errors.As(err, &fre) {
|
||||||
|
t.Fatalf("err = %v, want a *FileRemoveError", err)
|
||||||
|
}
|
||||||
|
if fre.NotWritable() != tc.notWritable {
|
||||||
|
t.Errorf("NotWritable = %v, want %v", fre.NotWritable(), tc.notWritable)
|
||||||
|
}
|
||||||
|
if fre.Dir() != dir {
|
||||||
|
t.Errorf("Dir = %q, want the parent directory %q", fre.Dir(), dir)
|
||||||
|
}
|
||||||
|
if fre.Reason() != tc.errno.Error() {
|
||||||
|
t.Errorf("Reason = %q, want %q", fre.Reason(), tc.errno.Error())
|
||||||
|
}
|
||||||
|
if fre.UID != os.Getuid() || fre.GID != os.Getgid() {
|
||||||
|
t.Errorf("identity = %d:%d, want this process's %d:%d", fre.UID, fre.GID, os.Getuid(), os.Getgid())
|
||||||
|
}
|
||||||
|
if _, err := q.GetTrackByID(context.Background(), track.ID); err != nil {
|
||||||
|
t.Errorf("track row was deleted although its file was not: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := q.GetAlbumByID(context.Background(), album.ID); err != nil {
|
||||||
|
t.Errorf("album row was deleted although the track's file was not: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
t.Errorf("file gone although removal was refused: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -97,7 +207,7 @@ func TestDeleteTrackFile_FileAlreadyGoneSucceeds(t *testing.T) {
|
|||||||
|
|
||||||
track, _, _ := seedTrack(t, pool, "/no/such/file/anywhere.mp3")
|
track, _, _ := seedTrack(t, pool, "/no/such/file/anywhere.mp3")
|
||||||
|
|
||||||
if err := DeleteTrackFile(context.Background(), pool, track.ID); err != nil {
|
if _, err := DeleteTrackFile(context.Background(), pool, nil, "", track.ID); err != nil {
|
||||||
t.Fatalf("DeleteTrackFile with missing file: %v", err)
|
t.Fatalf("DeleteTrackFile with missing file: %v", err)
|
||||||
}
|
}
|
||||||
if _, err := q.GetTrackByID(context.Background(), track.ID); err == nil {
|
if _, err := q.GetTrackByID(context.Background(), track.ID); err == nil {
|
||||||
@@ -112,7 +222,7 @@ func TestDeleteTrackFile_NotFoundReturnsErr(t *testing.T) {
|
|||||||
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
||||||
bogus.Valid = true
|
bogus.Valid = true
|
||||||
|
|
||||||
err := DeleteTrackFile(context.Background(), pool, bogus)
|
_, err := DeleteTrackFile(context.Background(), pool, nil, "", bogus)
|
||||||
if !errors.Is(err, ErrTrackNotFound) {
|
if !errors.Is(err, ErrTrackNotFound) {
|
||||||
t.Errorf("err = %v, want ErrTrackNotFound", err)
|
t.Errorf("err = %v, want ErrTrackNotFound", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,430 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"math/bits"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Duplicate matching (M400 #3909).
|
||||||
|
//
|
||||||
|
// Pure functions over fingerprints: no database, no files. This is the part that
|
||||||
|
// decides whether two tracks in the operator's library are proposed as one
|
||||||
|
// recording, so every rule in it has to be falsifiable in a unit test.
|
||||||
|
//
|
||||||
|
// Two tiers, answering different questions:
|
||||||
|
//
|
||||||
|
// exact equal audio_stream_sha256 — the same encoded audio bytes. No score,
|
||||||
|
// no threshold, no false positives (the #3885 pair).
|
||||||
|
// acoustic chromaprint fingerprints that agree closely once aligned — the
|
||||||
|
// same recording at another bitrate or in another codec.
|
||||||
|
//
|
||||||
|
// The acoustic comparison follows the approach of AcoustID's pg_acoustid
|
||||||
|
// (acoustid_compare.c): vote on the relative offset between two fingerprints
|
||||||
|
// using items that agree in their high bits, then measure disagreement at the
|
||||||
|
// winning offset. Reimplemented from that description; no code was copied.
|
||||||
|
// The alignment window and match-bit width below are taken from it.
|
||||||
|
|
||||||
|
// maxAlignOffsetItems bounds how far apart two fingerprints may be shifted and
|
||||||
|
// still be compared: ±120 items, about 15 seconds at chromaprint's ~8 items per
|
||||||
|
// second. Covers a leading silence trimmed differently or a short intro; the
|
||||||
|
// same bound pg_acoustid uses (ACOUSTID_MAX_ALIGN_OFFSET).
|
||||||
|
const maxAlignOffsetItems = 120
|
||||||
|
|
||||||
|
// alignMatchBits is how many high bits two items must share to vote for an
|
||||||
|
// offset. Matching whole 32-bit items would miss the same recording at another
|
||||||
|
// bitrate, whose low bits are noisier; 14 is pg_acoustid's MATCH_BITS.
|
||||||
|
const alignMatchBits = 14
|
||||||
|
|
||||||
|
// minOverlapItems is the least overlap worth a verdict: about 10 seconds. A few
|
||||||
|
// items agreeing perfectly is not evidence that two recordings are one.
|
||||||
|
const minOverlapItems = 80
|
||||||
|
|
||||||
|
// minDistinctFraction rejects low-information fingerprints before they can
|
||||||
|
// match. Near-silence, a sustained tone or a click track produces the same few
|
||||||
|
// items over and over, and two such tracks agree closely without being the
|
||||||
|
// same recording. Real music is overwhelmingly distinct item to item, so this
|
||||||
|
// floor only catches the pathological case. A judgment value, not a measured
|
||||||
|
// one — revisit if the sweep reports real tracks refused for it.
|
||||||
|
const minDistinctFraction = 0.3
|
||||||
|
|
||||||
|
// defaultAcousticMaxBitErrorRate is the most disagreement two aligned
|
||||||
|
// fingerprints may show and still be proposed as one recording. Unrelated audio
|
||||||
|
// sits near 0.5; the same recording re-encoded lands well under 0.1.
|
||||||
|
//
|
||||||
|
// Deliberately conservative. The operator's stated worry is the opposite of a
|
||||||
|
// missed duplicate: "the same song can appear in different albums, usually it's
|
||||||
|
// a different recording", and an instrumental shares its vocal version's
|
||||||
|
// harmony, which chroma features capture. A false merge is the failure that
|
||||||
|
// matters, and the report is reviewed anyway. This is an unmeasured default:
|
||||||
|
// calibrate it against real pairs once the backfill (#3908) has populated the
|
||||||
|
// library, then expose it in Settings (#3913).
|
||||||
|
const defaultAcousticMaxBitErrorRate = 0.15
|
||||||
|
|
||||||
|
// durationToleranceMs is how far apart two tracks' durations may be and still be
|
||||||
|
// compared. Encoders pad and trim a little; different edits differ by more.
|
||||||
|
const durationToleranceMs = 3000
|
||||||
|
|
||||||
|
// maxAcousticGroupSize caps an acoustic group. A cluster bigger than this is far
|
||||||
|
// more likely a shared jingle, a skit or a low-information pattern than eight
|
||||||
|
// copies of one recording, and proposing it would bury the real duplicates.
|
||||||
|
// Exact-tier groups are not capped: identical bytes are identical however many.
|
||||||
|
const maxAcousticGroupSize = 8
|
||||||
|
|
||||||
|
// acousticScore is the result of comparing two fingerprints.
|
||||||
|
type acousticScore struct {
|
||||||
|
// Offset is how many items b is shifted against a: b[i+Offset] aligns with
|
||||||
|
// a[i].
|
||||||
|
Offset int
|
||||||
|
// Overlap is how many aligned items were compared.
|
||||||
|
Overlap int
|
||||||
|
// BitErrorRate is the fraction of differing bits over the overlap, 0..1.
|
||||||
|
BitErrorRate float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// preparedPrint is a fingerprint with the parts every comparison needs worked
|
||||||
|
// out once. The sweep compares each track with every other track within a few
|
||||||
|
// seconds of its duration, so rebuilding the alignment index for each pair would
|
||||||
|
// dominate its cost.
|
||||||
|
type preparedPrint struct {
|
||||||
|
items []int32
|
||||||
|
index map[uint32][]int
|
||||||
|
informative bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// preparePrint indexes a fingerprint's items by their high bits and records
|
||||||
|
// whether it varies enough to be compared at all.
|
||||||
|
func preparePrint(fp []int32) *preparedPrint {
|
||||||
|
// Each bucket keeps only a few positions: a value repeating many times is
|
||||||
|
// uninformative, and letting it vote once per repeat would make every
|
||||||
|
// pairing O(n²).
|
||||||
|
const keepPerBucket = 4
|
||||||
|
p := &preparedPrint{items: fp, index: make(map[uint32][]int, len(fp))}
|
||||||
|
seen := make(map[int32]struct{}, len(fp))
|
||||||
|
for i, v := range fp {
|
||||||
|
seen[v] = struct{}{}
|
||||||
|
key := alignKey(v)
|
||||||
|
if pos := p.index[key]; len(pos) < keepPerBucket {
|
||||||
|
p.index[key] = append(pos, i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.informative = len(fp) > 0 && float64(len(seen)) >= minDistinctFraction*float64(len(fp))
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func alignKey(v int32) uint32 { return uint32(v) >> (32 - alignMatchBits) }
|
||||||
|
|
||||||
|
// compareChromaprint aligns two raw fingerprints and measures how much they
|
||||||
|
// disagree. ok is false when no verdict is possible: no offset gathered any
|
||||||
|
// votes, the overlap at the best offset is too short, or either side carries
|
||||||
|
// too little information to mean anything.
|
||||||
|
func compareChromaprint(a, b []int32) (acousticScore, bool) {
|
||||||
|
return comparePrepared(preparePrint(a), preparePrint(b))
|
||||||
|
}
|
||||||
|
|
||||||
|
// comparePrepared is compareChromaprint over fingerprints already prepared.
|
||||||
|
func comparePrepared(a, b *preparedPrint) (acousticScore, bool) {
|
||||||
|
if len(a.items) < minOverlapItems || len(b.items) < minOverlapItems {
|
||||||
|
return acousticScore{}, false
|
||||||
|
}
|
||||||
|
if !a.informative || !b.informative {
|
||||||
|
return acousticScore{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
offset, ok := bestOffset(a, b)
|
||||||
|
if !ok {
|
||||||
|
return acousticScore{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// a[i] aligns with b[i+offset]; walk the indices valid on both sides.
|
||||||
|
start := max(0, -offset)
|
||||||
|
end := min(len(a.items), len(b.items)-offset)
|
||||||
|
overlap := end - start
|
||||||
|
if overlap < minOverlapItems {
|
||||||
|
return acousticScore{}, false
|
||||||
|
}
|
||||||
|
errBits := 0
|
||||||
|
for i := start; i < end; i++ {
|
||||||
|
errBits += bits.OnesCount32(uint32(a.items[i]) ^ uint32(b.items[i+offset]))
|
||||||
|
}
|
||||||
|
return acousticScore{
|
||||||
|
Offset: offset,
|
||||||
|
Overlap: overlap,
|
||||||
|
BitErrorRate: float64(errBits) / float64(32*overlap),
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// bestOffset returns the relative shift most items agree on.
|
||||||
|
func bestOffset(a, b *preparedPrint) (int, bool) {
|
||||||
|
votes := make([]int, 2*maxAlignOffsetItems+1)
|
||||||
|
for j, v := range b.items {
|
||||||
|
for _, i := range a.index[alignKey(v)] {
|
||||||
|
off := j - i
|
||||||
|
if off >= -maxAlignOffsetItems && off <= maxAlignOffsetItems {
|
||||||
|
votes[off+maxAlignOffsetItems]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
best, bestVotes := 0, 0
|
||||||
|
for k, n := range votes {
|
||||||
|
// Strictly greater keeps the smallest shift on a tie, which is the more
|
||||||
|
// likely truth and keeps the result deterministic.
|
||||||
|
if n > bestVotes || (n == bestVotes && n > 0 && abs(k-maxAlignOffsetItems) < abs(best)) {
|
||||||
|
best, bestVotes = k-maxAlignOffsetItems, n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best, bestVotes > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// fingerprintCandidate is one track as the grouping sees it.
|
||||||
|
type fingerprintCandidate struct {
|
||||||
|
ID string
|
||||||
|
DurationMs int32
|
||||||
|
StreamSHA256 []byte
|
||||||
|
Chromaprint []int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// duplicateTier names what a group's evidence is.
|
||||||
|
type duplicateTier string
|
||||||
|
|
||||||
|
const (
|
||||||
|
tierExact duplicateTier = "exact"
|
||||||
|
tierAcoustic duplicateTier = "acoustic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// duplicateGroup is a set of tracks proposed as one recording. Members are
|
||||||
|
// sorted by ID.
|
||||||
|
type duplicateGroup struct {
|
||||||
|
Tier duplicateTier
|
||||||
|
Members []string
|
||||||
|
// WorstBitErrorRate is the largest disagreement between any two members of
|
||||||
|
// an acoustic group — the weakest evidence the group rests on. Zero for
|
||||||
|
// exact groups.
|
||||||
|
WorstBitErrorRate float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupingResult is what one grouping pass found.
|
||||||
|
type groupingResult struct {
|
||||||
|
Groups []duplicateGroup
|
||||||
|
// OversizeClusters counts acoustic clusters discarded for exceeding
|
||||||
|
// maxAcousticGroupSize. Reported rather than silent: a sudden rise means the
|
||||||
|
// cap or the information floor needs attention.
|
||||||
|
OversizeClusters int
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupUnit is one thing the acoustic pass compares: a single track, or an exact
|
||||||
|
// group standing in for all its byte-identical copies.
|
||||||
|
type groupUnit struct {
|
||||||
|
ids []string // every member, sorted
|
||||||
|
durationMs int32
|
||||||
|
sortKey string // the representative's id: ties on duration break on it
|
||||||
|
print *preparedPrint
|
||||||
|
exact bool // more than one member with identical audio
|
||||||
|
assigned bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// streamGrouper is the acoustic pass over units arriving in (durationMs, sortKey)
|
||||||
|
// order. It holds only the units within durationToleranceMs of the oldest one
|
||||||
|
// not yet settled, so memory is bounded by the densest few seconds of the
|
||||||
|
// library rather than by its size — the whole library's fingerprints would be
|
||||||
|
// hundreds of megabytes.
|
||||||
|
//
|
||||||
|
// A seed can be settled as soon as a unit arrives beyond its window: everything
|
||||||
|
// it could group with has already arrived, and no later seed can reach back to
|
||||||
|
// it because seeds only look forward. That is what makes the streamed result
|
||||||
|
// identical to running the same pass over the whole sorted list.
|
||||||
|
//
|
||||||
|
// Grouping is COMPLETE-LINKAGE: a unit joins a group only if it matches every
|
||||||
|
// unit already in it, within the duration tolerance and the bit-error limit.
|
||||||
|
// Single-linkage would let a chain of near-misses — A close to B, B close to C —
|
||||||
|
// drag A and C, which are not close, into one proposed merge. Complete linkage
|
||||||
|
// also means any member can be chosen as the survivor (#3911).
|
||||||
|
//
|
||||||
|
// When an acoustic group absorbs an exact group, the result is tier acoustic: a
|
||||||
|
// group is only as certain as its weakest link.
|
||||||
|
type streamGrouper struct {
|
||||||
|
maxBitErrorRate float64
|
||||||
|
window []*groupUnit
|
||||||
|
res groupingResult
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStreamGrouper(maxBitErrorRate float64) *streamGrouper {
|
||||||
|
return &streamGrouper{maxBitErrorRate: maxBitErrorRate}
|
||||||
|
}
|
||||||
|
|
||||||
|
// push adds the next unit. Units must arrive in non-decreasing
|
||||||
|
// (durationMs, sortKey) order.
|
||||||
|
func (g *streamGrouper) push(u *groupUnit) {
|
||||||
|
g.window = append(g.window, u)
|
||||||
|
for len(g.window) > 1 && u.durationMs-g.window[0].durationMs > durationToleranceMs {
|
||||||
|
g.settleOldest()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// finish settles every unit still waiting and returns what was found. Groups
|
||||||
|
// are in no particular order; callers sort with sortGroups.
|
||||||
|
func (g *streamGrouper) finish() groupingResult {
|
||||||
|
for len(g.window) > 0 {
|
||||||
|
g.settleOldest()
|
||||||
|
}
|
||||||
|
return g.res
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *streamGrouper) settleOldest() {
|
||||||
|
seed := g.window[0]
|
||||||
|
g.window[0] = nil // release it: the window's backing array outlives the slide
|
||||||
|
g.window = g.window[1:]
|
||||||
|
if seed.assigned {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
group := []*groupUnit{seed}
|
||||||
|
worst := 0.0
|
||||||
|
for _, cand := range g.window {
|
||||||
|
if cand.durationMs-seed.durationMs > durationToleranceMs {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if cand.assigned {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
joined, worstWithCand := true, worst
|
||||||
|
for _, member := range group {
|
||||||
|
if abs32(cand.durationMs-member.durationMs) > durationToleranceMs {
|
||||||
|
joined = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
score, ok := comparePrepared(member.print, cand.print)
|
||||||
|
if !ok || score.BitErrorRate > g.maxBitErrorRate {
|
||||||
|
joined = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
worstWithCand = math.Max(worstWithCand, score.BitErrorRate)
|
||||||
|
}
|
||||||
|
if joined {
|
||||||
|
group = append(group, cand)
|
||||||
|
worst = worstWithCand
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(group) == 1 {
|
||||||
|
if seed.exact {
|
||||||
|
g.res.Groups = append(g.res.Groups, duplicateGroup{Tier: tierExact, Members: seed.ids})
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Count units, not tracks: an absorbed exact group is one piece of acoustic
|
||||||
|
// evidence however many identical files it holds.
|
||||||
|
if len(group) > maxAcousticGroupSize {
|
||||||
|
g.res.OversizeClusters++
|
||||||
|
for _, member := range group {
|
||||||
|
member.assigned = true
|
||||||
|
// The acoustic evidence is untrustworthy; identical bytes are not.
|
||||||
|
// An exact group caught inside an oversize cluster is still proposed.
|
||||||
|
if member.exact {
|
||||||
|
g.res.Groups = append(g.res.Groups, duplicateGroup{Tier: tierExact, Members: member.ids})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var members []string
|
||||||
|
for _, member := range group {
|
||||||
|
member.assigned = true
|
||||||
|
members = append(members, member.ids...)
|
||||||
|
}
|
||||||
|
sort.Strings(members)
|
||||||
|
g.res.Groups = append(g.res.Groups, duplicateGroup{
|
||||||
|
Tier: tierAcoustic, Members: members, WorstBitErrorRate: worst,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupDuplicates proposes duplicate groups among candidates held in memory. It
|
||||||
|
// runs the same streamGrouper the sweep uses, so there is one grouping rule.
|
||||||
|
//
|
||||||
|
// Exact groups come first: tracks sharing an audio stream hash. Each becomes a
|
||||||
|
// single unit for the acoustic pass, represented by its member with the lowest
|
||||||
|
// (duration, id) that has a chromaprint. That is the member the sweep's
|
||||||
|
// duration-ordered stream meets first, which keeps the two identical. An exact
|
||||||
|
// group with no chromaprint at all cannot be compared acoustically and stands
|
||||||
|
// on its own.
|
||||||
|
//
|
||||||
|
// The output does not depend on input order.
|
||||||
|
func groupDuplicates(cands []fingerprintCandidate, maxBitErrorRate float64) groupingResult {
|
||||||
|
byHash := map[string][]fingerprintCandidate{}
|
||||||
|
var units []*groupUnit
|
||||||
|
var printless []duplicateGroup
|
||||||
|
for _, c := range cands {
|
||||||
|
if len(c.StreamSHA256) > 0 {
|
||||||
|
byHash[string(c.StreamSHA256)] = append(byHash[string(c.StreamSHA256)], c)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(c.Chromaprint) > 0 {
|
||||||
|
units = append(units, &groupUnit{
|
||||||
|
ids: []string{c.ID}, durationMs: c.DurationMs, sortKey: c.ID, print: preparePrint(c.Chromaprint),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, group := range byHash {
|
||||||
|
ids := make([]string, len(group))
|
||||||
|
for i, m := range group {
|
||||||
|
ids[i] = m.ID
|
||||||
|
}
|
||||||
|
sort.Strings(ids)
|
||||||
|
|
||||||
|
var rep *fingerprintCandidate
|
||||||
|
for i := range group {
|
||||||
|
m := &group[i]
|
||||||
|
if len(m.Chromaprint) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if rep == nil || m.DurationMs < rep.DurationMs || (m.DurationMs == rep.DurationMs && m.ID < rep.ID) {
|
||||||
|
rep = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if rep == nil {
|
||||||
|
if len(group) > 1 {
|
||||||
|
printless = append(printless, duplicateGroup{Tier: tierExact, Members: ids})
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
units = append(units, &groupUnit{
|
||||||
|
ids: ids, durationMs: rep.DurationMs, sortKey: rep.ID,
|
||||||
|
print: preparePrint(rep.Chromaprint), exact: len(group) > 1,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(units, func(i, j int) bool {
|
||||||
|
if units[i].durationMs != units[j].durationMs {
|
||||||
|
return units[i].durationMs < units[j].durationMs
|
||||||
|
}
|
||||||
|
return units[i].sortKey < units[j].sortKey
|
||||||
|
})
|
||||||
|
g := newStreamGrouper(maxBitErrorRate)
|
||||||
|
for _, u := range units {
|
||||||
|
g.push(u)
|
||||||
|
}
|
||||||
|
res := g.finish()
|
||||||
|
res.Groups = append(res.Groups, printless...)
|
||||||
|
sortGroups(res.Groups)
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
// sortGroups orders groups by their first member. Groups are disjoint, so that
|
||||||
|
// is a total order.
|
||||||
|
func sortGroups(groups []duplicateGroup) {
|
||||||
|
sort.Slice(groups, func(i, j int) bool { return groups[i].Members[0] < groups[j].Members[0] })
|
||||||
|
}
|
||||||
|
|
||||||
|
func abs(n int) int {
|
||||||
|
if n < 0 {
|
||||||
|
return -n
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func abs32(n int32) int32 {
|
||||||
|
if n < 0 {
|
||||||
|
return -n
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
@@ -0,0 +1,241 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"math/rand/v2"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// printLen is a realistic fingerprint length: fpcalc's 120s at ~8 items/second.
|
||||||
|
const printLen = 960
|
||||||
|
|
||||||
|
// randomPrint is a deterministic stand-in for one recording's fingerprint.
|
||||||
|
func randomPrint(seed uint64, n int) []int32 {
|
||||||
|
r := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
|
||||||
|
fp := make([]int32, n)
|
||||||
|
for i := range fp {
|
||||||
|
fp[i] = int32(r.Uint32())
|
||||||
|
}
|
||||||
|
return fp
|
||||||
|
}
|
||||||
|
|
||||||
|
// withBitNoise flips exactly round(fraction × all bits) distinct bits — the
|
||||||
|
// same recording through a different encoder, at a known bit-error rate.
|
||||||
|
func withBitNoise(fp []int32, fraction float64, seed uint64) []int32 {
|
||||||
|
out := append([]int32(nil), fp...)
|
||||||
|
r := rand.New(rand.NewPCG(seed, seed^0x243f6a8885a308d3))
|
||||||
|
total := 32 * len(fp)
|
||||||
|
for _, pos := range r.Perm(total)[:int(math.Round(fraction*float64(total)))] {
|
||||||
|
out[pos/32] ^= int32(uint32(1) << (pos % 32))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func constantPrint(v int32, n int) []int32 {
|
||||||
|
fp := make([]int32, n)
|
||||||
|
for i := range fp {
|
||||||
|
fp[i] = v
|
||||||
|
}
|
||||||
|
return fp
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompareChromaprint(t *testing.T) {
|
||||||
|
base := randomPrint(1, printLen)
|
||||||
|
|
||||||
|
t.Run("identical", func(t *testing.T) {
|
||||||
|
got, ok := compareChromaprint(base, base)
|
||||||
|
if !ok || got.BitErrorRate != 0 || got.Offset != 0 || got.Overlap != printLen {
|
||||||
|
t.Fatalf("got %+v ok=%v, want an exact alignment", got, ok)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("re-encoded: known bit noise is measured exactly", func(t *testing.T) {
|
||||||
|
got, ok := compareChromaprint(base, withBitNoise(base, 0.03, 2))
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("a re-encode was not comparable")
|
||||||
|
}
|
||||||
|
if want := math.Round(0.03*32*printLen) / (32 * printLen); got.BitErrorRate != want {
|
||||||
|
t.Fatalf("BitErrorRate = %v, want %v", got.BitErrorRate, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// b starts 40 items later in the same audio: b[j] = a[j+40], so a[i] aligns
|
||||||
|
// with b[i-40].
|
||||||
|
t.Run("offset inside the window is recovered", func(t *testing.T) {
|
||||||
|
got, ok := compareChromaprint(base, base[40:])
|
||||||
|
if !ok || got.Offset != -40 || got.BitErrorRate != 0 || got.Overlap != printLen-40 {
|
||||||
|
t.Fatalf("got %+v ok=%v, want offset -40 with no error", got, ok)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("offset beyond the window never matches", func(t *testing.T) {
|
||||||
|
got, ok := compareChromaprint(base, base[200:])
|
||||||
|
if ok && got.BitErrorRate <= defaultAcousticMaxBitErrorRate {
|
||||||
|
t.Fatalf("a 200-item shift matched: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unrelated recordings sit near 0.5", func(t *testing.T) {
|
||||||
|
got, ok := compareChromaprint(base, randomPrint(99, printLen))
|
||||||
|
if ok && got.BitErrorRate < 0.4 {
|
||||||
|
t.Fatalf("unrelated fingerprints scored %v", got.BitErrorRate)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("too short an overlap gives no verdict", func(t *testing.T) {
|
||||||
|
if got, ok := compareChromaprint(base, base[:minOverlapItems-1]); ok {
|
||||||
|
t.Fatalf("a %d-item fingerprint was compared: %+v", minOverlapItems-1, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Two near-silent tracks agree perfectly without being one recording. The
|
||||||
|
// information floor is the only thing standing between them and a merge.
|
||||||
|
t.Run("low-information fingerprints give no verdict", func(t *testing.T) {
|
||||||
|
silence := constantPrint(0x1234, printLen)
|
||||||
|
if got, ok := compareChromaprint(silence, silence); ok {
|
||||||
|
t.Fatalf("silence compared as a match: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("the threshold separates close from not close", func(t *testing.T) {
|
||||||
|
near, _ := compareChromaprint(base, withBitNoise(base, 0.10, 3))
|
||||||
|
far, _ := compareChromaprint(base, withBitNoise(base, 0.20, 4))
|
||||||
|
if near.BitErrorRate > defaultAcousticMaxBitErrorRate {
|
||||||
|
t.Errorf("10%% noise (%v) is over the threshold", near.BitErrorRate)
|
||||||
|
}
|
||||||
|
if far.BitErrorRate <= defaultAcousticMaxBitErrorRate {
|
||||||
|
t.Errorf("20%% noise (%v) is under the threshold", far.BitErrorRate)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGroupDuplicates_ExactTier(t *testing.T) {
|
||||||
|
hash := []byte("sha256-of-www-instrumental-bytes")
|
||||||
|
res := groupDuplicates([]fingerprintCandidate{
|
||||||
|
{ID: "www-01", DurationMs: 215000, StreamSHA256: hash},
|
||||||
|
{ID: "www-02", DurationMs: 215000, StreamSHA256: hash},
|
||||||
|
{ID: "lovesick", DurationMs: 198000, StreamSHA256: []byte("another")},
|
||||||
|
}, defaultAcousticMaxBitErrorRate)
|
||||||
|
want := []duplicateGroup{{Tier: tierExact, Members: []string{"www-01", "www-02"}}}
|
||||||
|
if !reflect.DeepEqual(res.Groups, want) {
|
||||||
|
t.Fatalf("groups = %+v, want %+v", res.Groups, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGroupDuplicates_AcousticPair(t *testing.T) {
|
||||||
|
p := randomPrint(10, printLen)
|
||||||
|
res := groupDuplicates([]fingerprintCandidate{
|
||||||
|
{ID: "album", DurationMs: 240000, Chromaprint: p},
|
||||||
|
{ID: "compilation", DurationMs: 241000, Chromaprint: withBitNoise(p, 0.05, 11)},
|
||||||
|
}, defaultAcousticMaxBitErrorRate)
|
||||||
|
if len(res.Groups) != 1 || res.Groups[0].Tier != tierAcoustic ||
|
||||||
|
!reflect.DeepEqual(res.Groups[0].Members, []string{"album", "compilation"}) {
|
||||||
|
t.Fatalf("groups = %+v, want one acoustic pair", res.Groups)
|
||||||
|
}
|
||||||
|
if got := res.Groups[0].WorstBitErrorRate; math.Abs(got-0.05) > 0.001 {
|
||||||
|
t.Fatalf("WorstBitErrorRate = %v, want about 0.05", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A is close to B and B is close to C, but A and C are not close. Under
|
||||||
|
// single linkage all three would be proposed as one recording; complete linkage
|
||||||
|
// must keep C out.
|
||||||
|
func TestGroupDuplicates_NoChaining(t *testing.T) {
|
||||||
|
a := randomPrint(20, printLen)
|
||||||
|
b := withBitNoise(a, 0.10, 21)
|
||||||
|
c := withBitNoise(b, 0.10, 22)
|
||||||
|
if s, _ := compareChromaprint(a, c); s.BitErrorRate <= defaultAcousticMaxBitErrorRate {
|
||||||
|
t.Fatalf("fixture broken: A and C are close (%v), so this cannot test chaining", s.BitErrorRate)
|
||||||
|
}
|
||||||
|
res := groupDuplicates([]fingerprintCandidate{
|
||||||
|
{ID: "a", DurationMs: 200000, Chromaprint: a},
|
||||||
|
{ID: "b", DurationMs: 200000, Chromaprint: b},
|
||||||
|
{ID: "c", DurationMs: 200000, Chromaprint: c},
|
||||||
|
}, defaultAcousticMaxBitErrorRate)
|
||||||
|
if len(res.Groups) != 1 || !reflect.DeepEqual(res.Groups[0].Members, []string{"a", "b"}) {
|
||||||
|
t.Fatalf("groups = %+v, want only {a, b}", res.Groups)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGroupDuplicates_DurationTolerance(t *testing.T) {
|
||||||
|
p := randomPrint(30, printLen)
|
||||||
|
res := groupDuplicates([]fingerprintCandidate{
|
||||||
|
{ID: "edit", DurationMs: 200000, Chromaprint: p},
|
||||||
|
{ID: "extended", DurationMs: 200000 + durationToleranceMs + 1, Chromaprint: p},
|
||||||
|
}, defaultAcousticMaxBitErrorRate)
|
||||||
|
if len(res.Groups) != 0 {
|
||||||
|
t.Fatalf("tracks %dms apart were grouped: %+v", durationToleranceMs+1, res.Groups)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nine tracks that all match are far likelier a shared jingle than nine copies
|
||||||
|
// of one recording. The cluster must be reported, not proposed.
|
||||||
|
func TestGroupDuplicates_OversizeClusterIsDiscarded(t *testing.T) {
|
||||||
|
p := randomPrint(40, printLen)
|
||||||
|
var cands []fingerprintCandidate
|
||||||
|
for i := range maxAcousticGroupSize + 1 {
|
||||||
|
cands = append(cands, fingerprintCandidate{
|
||||||
|
ID: string(rune('a' + i)), DurationMs: 30000, Chromaprint: withBitNoise(p, 0.01, uint64(100+i)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
res := groupDuplicates(cands, defaultAcousticMaxBitErrorRate)
|
||||||
|
if len(res.Groups) != 0 || res.OversizeClusters != 1 {
|
||||||
|
t.Fatalf("groups = %+v, oversize = %d; want none proposed and 1 oversize", res.Groups, res.OversizeClusters)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two byte-identical copies plus a re-encode of the same recording are one
|
||||||
|
// group, and it is only as certain as its weakest link.
|
||||||
|
func TestGroupDuplicates_ExactGroupAbsorbedIntoAcoustic(t *testing.T) {
|
||||||
|
p := randomPrint(50, printLen)
|
||||||
|
hash := []byte("same-bytes")
|
||||||
|
res := groupDuplicates([]fingerprintCandidate{
|
||||||
|
{ID: "x1", DurationMs: 180000, StreamSHA256: hash, Chromaprint: p},
|
||||||
|
{ID: "x2", DurationMs: 180000, StreamSHA256: hash, Chromaprint: p},
|
||||||
|
{ID: "y", DurationMs: 180000, StreamSHA256: []byte("other-bytes"), Chromaprint: withBitNoise(p, 0.03, 51)},
|
||||||
|
}, defaultAcousticMaxBitErrorRate)
|
||||||
|
want := []string{"x1", "x2", "y"}
|
||||||
|
if len(res.Groups) != 1 || res.Groups[0].Tier != tierAcoustic || !reflect.DeepEqual(res.Groups[0].Members, want) {
|
||||||
|
t.Fatalf("groups = %+v, want one acoustic group %v", res.Groups, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGroupDuplicates_UnrelatedTracksNeverGroup(t *testing.T) {
|
||||||
|
var cands []fingerprintCandidate
|
||||||
|
for i := range 6 {
|
||||||
|
cands = append(cands, fingerprintCandidate{
|
||||||
|
ID: string(rune('a' + i)), DurationMs: 210000, Chromaprint: randomPrint(uint64(60+i), printLen),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if res := groupDuplicates(cands, defaultAcousticMaxBitErrorRate); len(res.Groups) != 0 {
|
||||||
|
t.Fatalf("unrelated recordings were grouped: %+v", res.Groups)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGroupDuplicates_OrderIndependent(t *testing.T) {
|
||||||
|
p := randomPrint(70, printLen)
|
||||||
|
q := randomPrint(71, printLen)
|
||||||
|
hash := []byte("identical")
|
||||||
|
cands := []fingerprintCandidate{
|
||||||
|
{ID: "p1", DurationMs: 200000, Chromaprint: p},
|
||||||
|
{ID: "p2", DurationMs: 201000, Chromaprint: withBitNoise(p, 0.04, 72)},
|
||||||
|
{ID: "q1", DurationMs: 150000, Chromaprint: q},
|
||||||
|
{ID: "q2", DurationMs: 150500, Chromaprint: withBitNoise(q, 0.02, 73)},
|
||||||
|
{ID: "h1", DurationMs: 90000, StreamSHA256: hash},
|
||||||
|
{ID: "h2", DurationMs: 90000, StreamSHA256: hash},
|
||||||
|
{ID: "lone", DurationMs: 200000, Chromaprint: randomPrint(74, printLen)},
|
||||||
|
}
|
||||||
|
want := groupDuplicates(cands, defaultAcousticMaxBitErrorRate)
|
||||||
|
if len(want.Groups) != 3 {
|
||||||
|
t.Fatalf("fixture broken: %d groups, want 3 (p, q, h)", len(want.Groups))
|
||||||
|
}
|
||||||
|
r := rand.New(rand.NewPCG(75, 76))
|
||||||
|
for range 20 {
|
||||||
|
shuffled := append([]fingerprintCandidate(nil), cands...)
|
||||||
|
r.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] })
|
||||||
|
if got := groupDuplicates(shuffled, defaultAcousticMaxBitErrorRate); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("input order changed the result:\n got %+v\n want %+v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,309 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Duplicate merge (M400 #3911).
|
||||||
|
|
||||||
|
// ErrDuplicateGroupNotPending means the group was already merged or dismissed,
|
||||||
|
// no longer exists, or no longer has two members to merge.
|
||||||
|
var ErrDuplicateGroupNotPending = errors.New("library: duplicate group is not pending")
|
||||||
|
|
||||||
|
// ErrSurvivorNotInGroup means the copy chosen to keep is not a member of the group.
|
||||||
|
var ErrSurvivorNotInGroup = errors.New("library: survivor is not a member of the group")
|
||||||
|
|
||||||
|
// MergedCopy is one copy a merge kept or removed.
|
||||||
|
type MergedCopy struct {
|
||||||
|
TrackID pgtype.UUID
|
||||||
|
FilePath string
|
||||||
|
TrackMbid *string
|
||||||
|
AlbumMbid *string
|
||||||
|
}
|
||||||
|
|
||||||
|
// MergeResult says what a merge did.
|
||||||
|
type MergeResult struct {
|
||||||
|
Tier string
|
||||||
|
Survivor MergedCopy
|
||||||
|
Removed []MergedCopy
|
||||||
|
|
||||||
|
// What moved onto the survivor — reported so the operator, and the audit
|
||||||
|
// log, can see that the history was kept rather than take it on trust.
|
||||||
|
PlayEvents int64
|
||||||
|
SkipEvents int64
|
||||||
|
Likes int // users whose like now sits on the survivor
|
||||||
|
PlaylistEntries int
|
||||||
|
|
||||||
|
DeletedAlbumIDs []pgtype.UUID
|
||||||
|
DeletedArtistIDs []pgtype.UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// MergeDuplicateGroup keeps one copy of a duplicate group and removes the rest,
|
||||||
|
// carrying everything the removed copies held onto the one kept.
|
||||||
|
//
|
||||||
|
// survivorID chooses the copy to keep; an invalid (zero) id takes the proposal
|
||||||
|
// from ProposeSurvivor.
|
||||||
|
//
|
||||||
|
// The danger this is built around: every table referencing tracks does so ON
|
||||||
|
// DELETE CASCADE, so deleting a duplicate's row outright silently destroys its
|
||||||
|
// likes, plays, playlist entries and tags. The merge moves all of that onto the
|
||||||
|
// survivor first, and only then deletes the now-empty row.
|
||||||
|
//
|
||||||
|
// It deletes the removed copies' FILES too, and first, before any row changes
|
||||||
|
// (#3918, note #3926). A merge that left the file behind would be undone by the
|
||||||
|
// next scan, which re-imports it as a new track with no history. If a file cannot
|
||||||
|
// be removed, the *FileRemoveError comes back and nothing in the database changes.
|
||||||
|
// With several copies to remove, one file may already be gone when a later one
|
||||||
|
// fails; that copy's row keeps all its history and is marked missing by the next
|
||||||
|
// scan, and retrying the merge picks up where it stopped.
|
||||||
|
//
|
||||||
|
// Everything else happens in one transaction, which holds a lock on the group so
|
||||||
|
// two merges of it cannot run at once. Sync changes for clients' caches are logged
|
||||||
|
// inside it, the way the playlists service logs its own.
|
||||||
|
func MergeDuplicateGroup(
|
||||||
|
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, dataDir string,
|
||||||
|
groupID, survivorID pgtype.UUID,
|
||||||
|
) (MergeResult, error) {
|
||||||
|
if logger == nil {
|
||||||
|
logger = slog.Default()
|
||||||
|
}
|
||||||
|
tx, err := pool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("begin merge: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
|
tq := dbq.New(tx)
|
||||||
|
|
||||||
|
group, err := tq.LockDuplicateGroupForMerge(ctx, groupID)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return MergeResult{}, ErrDuplicateGroupNotPending
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("lock duplicate group: %w", err)
|
||||||
|
}
|
||||||
|
if group.Status != "pending" {
|
||||||
|
return MergeResult{}, ErrDuplicateGroupNotPending
|
||||||
|
}
|
||||||
|
|
||||||
|
members, err := tq.ListDuplicateGroupMergeMembers(ctx, groupID)
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("list group members: %w", err)
|
||||||
|
}
|
||||||
|
if len(members) < 2 {
|
||||||
|
return MergeResult{}, ErrDuplicateGroupNotPending
|
||||||
|
}
|
||||||
|
survivor, losers, err := splitSurvivor(members, survivorID)
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, l := range losers {
|
||||||
|
if err := removeTrackFileOnDisk(l.FilePath); err != nil {
|
||||||
|
return MergeResult{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
res := MergeResult{Tier: group.Tier, Survivor: mergedCopyOf(survivor)}
|
||||||
|
likers := map[string]struct{}{}
|
||||||
|
changes := mergeChanges{}
|
||||||
|
survivorKey := syncpkg.FormatUUID(survivor.ID)
|
||||||
|
|
||||||
|
for _, l := range losers {
|
||||||
|
ids := struct{ survivor, loser pgtype.UUID }{survivor.ID, l.ID}
|
||||||
|
loserKey := syncpkg.FormatUUID(l.ID)
|
||||||
|
|
||||||
|
n, err := tq.MergeRepointPlayEvents(ctx, dbq.MergeRepointPlayEventsParams{SurvivorID: ids.survivor, LoserID: ids.loser})
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("move play events: %w", err)
|
||||||
|
}
|
||||||
|
res.PlayEvents += n
|
||||||
|
n, err = tq.MergeRepointSkipEvents(ctx, dbq.MergeRepointSkipEventsParams{SurvivorID: ids.survivor, LoserID: ids.loser})
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("move skip events: %w", err)
|
||||||
|
}
|
||||||
|
res.SkipEvents += n
|
||||||
|
if _, err := tq.MergeRepointContextualLikes(ctx, dbq.MergeRepointContextualLikesParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("move contextual likes: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := tq.MergeRepointPlaybackErrors(ctx, dbq.MergeRepointPlaybackErrorsParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("move playback errors: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := tq.MergeRepointLidarrRequests(ctx, dbq.MergeRepointLidarrRequestsParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("move lidarr request matches: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
playlists, err := tq.MergeRepointPlaylistTracks(ctx, dbq.MergeRepointPlaylistTracksParams{SurvivorID: ids.survivor, LoserID: ids.loser})
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("move playlist entries: %w", err)
|
||||||
|
}
|
||||||
|
res.PlaylistEntries += len(playlists)
|
||||||
|
for _, pl := range playlists {
|
||||||
|
plKey := syncpkg.FormatUUID(pl)
|
||||||
|
changes.playlistDelete(syncpkg.EncodePlaylistTrackID(plKey, loserKey))
|
||||||
|
changes.playlistUpsert(syncpkg.EncodePlaylistTrackID(plKey, survivorKey))
|
||||||
|
}
|
||||||
|
|
||||||
|
users, err := tq.MergeCopyGeneralLikes(ctx, dbq.MergeCopyGeneralLikesParams{SurvivorID: ids.survivor, LoserID: ids.loser})
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("move likes: %w", err)
|
||||||
|
}
|
||||||
|
for _, u := range users {
|
||||||
|
userKey := syncpkg.FormatUUID(u)
|
||||||
|
likers[userKey] = struct{}{}
|
||||||
|
changes.likeDelete(syncpkg.EncodeLikeID(userKey, loserKey))
|
||||||
|
changes.likeUpsert(syncpkg.EncodeLikeID(userKey, survivorKey))
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := tq.MergeCopyTrackTags(ctx, dbq.MergeCopyTrackTagsParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("merge tags: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := tq.MergeCopyTrackSimilarity(ctx, dbq.MergeCopyTrackSimilarityParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("merge similarity: %w", err)
|
||||||
|
}
|
||||||
|
if err := tq.MergeInheritTrackMbid(ctx, dbq.MergeInheritTrackMbidParams{SurvivorID: ids.survivor, LoserID: ids.loser}); err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("inherit recording mbid: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Everything the loser carried now sits on the survivor, so the CASCADE
|
||||||
|
// this delete sets off has nothing left to destroy.
|
||||||
|
deleted, err := tq.DeleteTrack(ctx, l.ID)
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("delete merged copy: %w", err)
|
||||||
|
}
|
||||||
|
tidied, err := tidyEmptiedAlbum(ctx, tq, deleted.AlbumID)
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, err
|
||||||
|
}
|
||||||
|
if tidied.AlbumID != nil {
|
||||||
|
res.DeletedAlbumIDs = append(res.DeletedAlbumIDs, *tidied.AlbumID)
|
||||||
|
}
|
||||||
|
if tidied.ArtistID != nil {
|
||||||
|
res.DeletedArtistIDs = append(res.DeletedArtistIDs, *tidied.ArtistID)
|
||||||
|
}
|
||||||
|
res.Removed = append(res.Removed, mergedCopyOf(l))
|
||||||
|
changes.trackDelete(loserKey)
|
||||||
|
}
|
||||||
|
res.Likes = len(likers)
|
||||||
|
|
||||||
|
marked, err := tq.MarkDuplicateGroupMerged(ctx, groupID)
|
||||||
|
if err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("mark group merged: %w", err)
|
||||||
|
}
|
||||||
|
if marked != 1 {
|
||||||
|
return MergeResult{}, ErrDuplicateGroupNotPending
|
||||||
|
}
|
||||||
|
if err := changes.log(ctx, tx); err != nil {
|
||||||
|
return MergeResult{}, err
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return MergeResult{}, fmt.Errorf("commit merge: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// After commit, like DeleteTrackFile: a leftover art directory is only disk.
|
||||||
|
if dataDir != "" {
|
||||||
|
for _, artistID := range res.DeletedArtistIDs {
|
||||||
|
if err := coverart.CleanupArtistArt(dataDir, artistID); err != nil {
|
||||||
|
logger.Warn("duplicate merge: artist-art cleanup failed",
|
||||||
|
"artist_id", syncpkg.FormatUUID(artistID), "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitSurvivor separates the copy to keep from the copies to remove. An
|
||||||
|
// invalid survivorID takes ProposeSurvivor's choice.
|
||||||
|
func splitSurvivor(
|
||||||
|
members []dbq.ListDuplicateGroupMergeMembersRow, survivorID pgtype.UUID,
|
||||||
|
) (dbq.ListDuplicateGroupMergeMembersRow, []dbq.ListDuplicateGroupMergeMembersRow, error) {
|
||||||
|
want := ""
|
||||||
|
if survivorID.Valid {
|
||||||
|
want = syncpkg.FormatUUID(survivorID)
|
||||||
|
} else {
|
||||||
|
cands := make([]SurvivorCandidate, len(members))
|
||||||
|
for i, m := range members {
|
||||||
|
cands[i] = SurvivorCandidate{
|
||||||
|
TrackID: syncpkg.FormatUUID(m.ID), FileFormat: m.FileFormat, FileSize: m.FileSize, AddedAt: m.AddedAt.Time,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
want, _ = ProposeSurvivor(cands)
|
||||||
|
}
|
||||||
|
|
||||||
|
var survivor dbq.ListDuplicateGroupMergeMembersRow
|
||||||
|
found := false
|
||||||
|
var losers []dbq.ListDuplicateGroupMergeMembersRow
|
||||||
|
for _, m := range members {
|
||||||
|
if syncpkg.FormatUUID(m.ID) == want {
|
||||||
|
survivor, found = m, true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
losers = append(losers, m)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return dbq.ListDuplicateGroupMergeMembersRow{}, nil, ErrSurvivorNotInGroup
|
||||||
|
}
|
||||||
|
return survivor, losers, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergedCopyOf(m dbq.ListDuplicateGroupMergeMembersRow) MergedCopy {
|
||||||
|
return MergedCopy{TrackID: m.ID, FilePath: m.FilePath, TrackMbid: m.Mbid, AlbumMbid: m.AlbumMbid}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeChanges collects the sync-log entries a merge owes clients' caches, each
|
||||||
|
// once: a user who liked two removed copies still gets one upsert for the
|
||||||
|
// survivor.
|
||||||
|
type mergeChanges struct {
|
||||||
|
tracks, likeDeletes, likeUpserts, playlistDeletes, playlistUpserts map[string]struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func addTo(set *map[string]struct{}, id string) {
|
||||||
|
if *set == nil {
|
||||||
|
*set = map[string]struct{}{}
|
||||||
|
}
|
||||||
|
(*set)[id] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *mergeChanges) trackDelete(id string) { addTo(&c.tracks, id) }
|
||||||
|
func (c *mergeChanges) likeDelete(id string) { addTo(&c.likeDeletes, id) }
|
||||||
|
func (c *mergeChanges) likeUpsert(id string) { addTo(&c.likeUpserts, id) }
|
||||||
|
func (c *mergeChanges) playlistDelete(id string) { addTo(&c.playlistDeletes, id) }
|
||||||
|
func (c *mergeChanges) playlistUpsert(id string) { addTo(&c.playlistUpserts, id) }
|
||||||
|
|
||||||
|
func (c *mergeChanges) log(ctx context.Context, tx pgx.Tx) error {
|
||||||
|
for _, entry := range []struct {
|
||||||
|
kind syncpkg.EntityType
|
||||||
|
ids map[string]struct{}
|
||||||
|
op syncpkg.Op
|
||||||
|
}{
|
||||||
|
{syncpkg.EntityTrack, c.tracks, syncpkg.OpDelete},
|
||||||
|
{syncpkg.EntityLikeTrack, c.likeDeletes, syncpkg.OpDelete},
|
||||||
|
{syncpkg.EntityLikeTrack, c.likeUpserts, syncpkg.OpUpsert},
|
||||||
|
{syncpkg.EntityPlaylistTrack, c.playlistDeletes, syncpkg.OpDelete},
|
||||||
|
{syncpkg.EntityPlaylistTrack, c.playlistUpserts, syncpkg.OpUpsert},
|
||||||
|
} {
|
||||||
|
if len(entry.ids) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ids := make([]string, 0, len(entry.ids))
|
||||||
|
for id := range entry.ids {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
sort.Strings(ids)
|
||||||
|
if err := syncpkg.LogChanges(ctx, tx, entry.kind, ids, entry.op); err != nil {
|
||||||
|
return fmt.Errorf("log merge changes: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,281 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mergeFixture is a library with one duplicate pair carrying history on both
|
||||||
|
// copies, and a neighbour track for similarity edges.
|
||||||
|
type mergeFixture struct {
|
||||||
|
pool *pgxpool.Pool
|
||||||
|
keep, remove, other dbq.Track
|
||||||
|
keepPath, removePath string
|
||||||
|
groupID pgtype.UUID
|
||||||
|
alice, bob dbq.User
|
||||||
|
playlistID pgtype.UUID
|
||||||
|
removePlaylistPos int32
|
||||||
|
aliceEarlierLikeOnRem time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMergeFixture(t *testing.T) mergeFixture {
|
||||||
|
t.Helper()
|
||||||
|
pool := newPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
q := dbq.New(pool)
|
||||||
|
dir := t.TempDir()
|
||||||
|
f := mergeFixture{pool: pool}
|
||||||
|
|
||||||
|
f.keepPath = filepath.Join(dir, "keep.flac")
|
||||||
|
f.removePath = filepath.Join(dir, "remove.mp3")
|
||||||
|
for _, p := range []string{f.keepPath, f.removePath} {
|
||||||
|
if err := os.WriteFile(p, []byte("audio"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write %s: %v", p, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var album dbq.Album
|
||||||
|
var artist dbq.Artist
|
||||||
|
f.keep, album, artist = seedTrack(t, pool, f.keepPath)
|
||||||
|
upsert := func(title, path string) dbq.Track {
|
||||||
|
t.Helper()
|
||||||
|
tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
|
||||||
|
Title: title, AlbumID: album.ID, ArtistID: artist.ID,
|
||||||
|
DurationMs: 215000, FilePath: path, FileSize: 100, FileFormat: "mp3",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("track %s: %v", title, err)
|
||||||
|
}
|
||||||
|
return tr
|
||||||
|
}
|
||||||
|
f.remove = upsert("WWW (copy)", f.removePath)
|
||||||
|
f.other = upsert("Neighbour", filepath.Join(dir, "other.mp3"))
|
||||||
|
|
||||||
|
mustExec := func(sql string, args ...any) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := pool.Exec(ctx, sql, args...); err != nil {
|
||||||
|
t.Fatalf("exec %q: %v", sql, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Only the copy being removed carries a recording MBID.
|
||||||
|
mustExec(`UPDATE tracks SET mbid = 'rec-www' WHERE id = $1`, f.remove.ID)
|
||||||
|
|
||||||
|
user := func(name string) dbq.User {
|
||||||
|
t.Helper()
|
||||||
|
u, err := q.CreateUser(ctx, dbq.CreateUserParams{
|
||||||
|
Username: dbtest.TestUserPrefix + name, PasswordHash: "x", ApiToken: name + "-merge-token",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("user %s: %v", name, err)
|
||||||
|
}
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
f.alice, f.bob = user("merge-alice"), user("merge-bob")
|
||||||
|
|
||||||
|
// Alice liked both copies, the removed one first; Bob liked only the removed one.
|
||||||
|
f.aliceEarlierLikeOnRem = time.Now().Add(-72 * time.Hour).UTC().Truncate(time.Microsecond)
|
||||||
|
mustExec(`INSERT INTO general_likes (user_id, track_id, liked_at) VALUES ($1, $2, $3), ($1, $4, now()), ($5, $2, now())`,
|
||||||
|
f.alice.ID, f.remove.ID, f.aliceEarlierLikeOnRem, f.keep.ID, f.bob.ID)
|
||||||
|
|
||||||
|
now := pgtype.Timestamptz{Time: time.Now(), Valid: true}
|
||||||
|
session, err := q.InsertPlaySession(ctx, dbq.InsertPlaySessionParams{UserID: f.alice.ID, StartedAt: now})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("session: %v", err)
|
||||||
|
}
|
||||||
|
for _, track := range []dbq.Track{f.remove, f.remove, f.keep} {
|
||||||
|
if _, err := q.InsertPlayEvent(ctx, dbq.InsertPlayEventParams{
|
||||||
|
UserID: f.alice.ID, TrackID: track.ID, SessionID: session.ID, StartedAt: now,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("play event: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := q.InsertSkipEvent(ctx, dbq.InsertSkipEventParams{
|
||||||
|
UserID: f.alice.ID, TrackID: f.remove.ID, SessionID: session.ID, SkippedAt: now, PositionMs: 1000,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("skip event: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
pl, err := q.CreatePlaylist(ctx, dbq.CreatePlaylistParams{UserID: f.alice.ID, Name: "merge-mix"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("playlist: %v", err)
|
||||||
|
}
|
||||||
|
f.playlistID = pl.ID
|
||||||
|
entry, err := q.AppendPlaylistTrack(ctx, dbq.AppendPlaylistTrackParams{PlaylistID: pl.ID, TrackID: f.remove.ID})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("playlist entry: %v", err)
|
||||||
|
}
|
||||||
|
f.removePlaylistPos = entry.Position
|
||||||
|
|
||||||
|
mustExec(`INSERT INTO track_tags (track_id, tag, weight) VALUES ($1, 'j-pop', 1), ($1, 'house', 0.5), ($2, 'house', 0.9)`,
|
||||||
|
f.remove.ID, f.keep.ID)
|
||||||
|
mustExec(`INSERT INTO track_similarity (track_a_id, track_b_id, score, source) VALUES
|
||||||
|
($1, $3, 0.8, 'listenbrainz'),
|
||||||
|
($2, $3, 0.7, 'listenbrainz'),
|
||||||
|
($1, $2, 0.99, 'listenbrainz'),
|
||||||
|
($3, $1, 0.6, 'musicbrainz_tag')`, f.remove.ID, f.keep.ID, f.other.ID)
|
||||||
|
|
||||||
|
if err := pool.QueryRow(ctx,
|
||||||
|
`INSERT INTO duplicate_groups (member_key, tier) VALUES ('merge-fixture', 'exact') RETURNING id`,
|
||||||
|
).Scan(&f.groupID); err != nil {
|
||||||
|
t.Fatalf("group: %v", err)
|
||||||
|
}
|
||||||
|
mustExec(`INSERT INTO duplicate_group_members (group_id, track_id) VALUES ($1, $2), ($1, $3)`,
|
||||||
|
f.groupID, f.keep.ID, f.remove.ID)
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f mergeFixture) count(t *testing.T, sql string, args ...any) int {
|
||||||
|
t.Helper()
|
||||||
|
var n int
|
||||||
|
if err := f.pool.QueryRow(context.Background(), sql, args...).Scan(&n); err != nil {
|
||||||
|
t.Fatalf("count %q: %v", sql, err)
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// The #3911 proof: after a merge, every piece of history the removed copy held
|
||||||
|
// is on the copy kept, nothing is doubled, and the removed copy — row and file —
|
||||||
|
// is gone.
|
||||||
|
func TestMergeDuplicateGroup_Integration(t *testing.T) {
|
||||||
|
f := newMergeFixture(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
res, err := MergeDuplicateGroup(ctx, f.pool, nil, "", f.groupID, f.keep.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("merge: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Removed) != 1 || res.Removed[0].FilePath != f.removePath || res.Survivor.TrackID != f.keep.ID {
|
||||||
|
t.Fatalf("result = %+v, want the removed copy reported and the kept one as survivor", res)
|
||||||
|
}
|
||||||
|
if res.PlayEvents != 2 || res.SkipEvents != 1 || res.Likes != 2 || res.PlaylistEntries != 1 {
|
||||||
|
t.Errorf("moved = plays %d skips %d likes %d playlist %d, want 2, 1, 2, 1",
|
||||||
|
res.PlayEvents, res.SkipEvents, res.Likes, res.PlaylistEntries)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(f.removePath); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Errorf("removed copy's file still on disk: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(f.keepPath); err != nil {
|
||||||
|
t.Errorf("kept copy's file is gone: %v", err)
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM tracks WHERE id = $1`, f.remove.ID); n != 0 {
|
||||||
|
t.Errorf("removed copy's row still exists")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Likes: one per user, Alice's dated to her earlier like.
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM general_likes WHERE track_id = $1`, f.keep.ID); n != 2 {
|
||||||
|
t.Errorf("likes on the kept copy = %d, want 2 (Alice once, Bob)", n)
|
||||||
|
}
|
||||||
|
var aliceLiked time.Time
|
||||||
|
if err := f.pool.QueryRow(ctx, `SELECT liked_at FROM general_likes WHERE user_id = $1 AND track_id = $2`,
|
||||||
|
f.alice.ID, f.keep.ID).Scan(&aliceLiked); err != nil {
|
||||||
|
t.Fatalf("alice's like: %v", err)
|
||||||
|
}
|
||||||
|
if !aliceLiked.Equal(f.aliceEarlierLikeOnRem) {
|
||||||
|
t.Errorf("alice's like dated %v, want her earlier like %v", aliceLiked, f.aliceEarlierLikeOnRem)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plays and skips move exactly: none lost, none invented.
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM play_events WHERE track_id = $1`, f.keep.ID); n != 3 {
|
||||||
|
t.Errorf("plays on the kept copy = %d, want 3", n)
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM skip_events WHERE track_id = $1`, f.keep.ID); n != 1 {
|
||||||
|
t.Errorf("skips on the kept copy = %d, want 1", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The playlist entry stays where it was and now plays the kept copy.
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM playlist_tracks WHERE playlist_id = $1 AND position = $2 AND track_id = $3`,
|
||||||
|
f.playlistID, f.removePlaylistPos, f.keep.ID); n != 1 {
|
||||||
|
t.Errorf("playlist entry at position %d does not point at the kept copy", f.removePlaylistPos)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tags are a union; the kept copy's own weight wins where both had the tag.
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM track_tags WHERE track_id = $1`, f.keep.ID); n != 2 {
|
||||||
|
t.Errorf("tags on the kept copy = %d, want 2 (house, j-pop)", n)
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM track_tags WHERE track_id = $1 AND tag = 'house' AND weight = 0.9`, f.keep.ID); n != 1 {
|
||||||
|
t.Errorf("the kept copy's own house weight was overwritten")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Similarity: rewritten onto the kept copy, no duplicate edge, no self-edge,
|
||||||
|
// nothing left pointing at the removed copy.
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM track_similarity WHERE track_a_id = $1 AND track_b_id = $2 AND source = 'listenbrainz'`,
|
||||||
|
f.keep.ID, f.other.ID); n != 1 {
|
||||||
|
t.Errorf("listenbrainz edge keep→other = %d rows, want exactly 1", n)
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM track_similarity WHERE track_a_id = $1 AND track_b_id = $2 AND source = 'musicbrainz_tag'`,
|
||||||
|
f.other.ID, f.keep.ID); n != 1 {
|
||||||
|
t.Errorf("musicbrainz_tag edge other→keep was not carried over")
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM track_similarity WHERE track_a_id = track_b_id`); n != 0 {
|
||||||
|
t.Errorf("a self-edge was written")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The removed copy's recording MBID is inherited; the group is closed.
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM tracks WHERE id = $1 AND mbid = 'rec-www'`, f.keep.ID); n != 1 {
|
||||||
|
t.Errorf("the kept copy did not inherit the recording MBID")
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM duplicate_groups WHERE id = $1 AND status = 'merged' AND resolved_at IS NOT NULL`, f.groupID); n != 1 {
|
||||||
|
t.Errorf("group was not marked merged")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second merge of the same group is refused rather than repeated.
|
||||||
|
if _, err := MergeDuplicateGroup(ctx, f.pool, nil, "", f.groupID, f.keep.ID); !errors.Is(err, ErrDuplicateGroupNotPending) {
|
||||||
|
t.Errorf("second merge err = %v, want ErrDuplicateGroupNotPending", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// When the removed copy's file cannot go, nothing may change: its likes, plays
|
||||||
|
// and row stay exactly where they were, and the group stays pending.
|
||||||
|
func TestMergeDuplicateGroup_UnremovableFileChangesNothing(t *testing.T) {
|
||||||
|
f := newMergeFixture(t)
|
||||||
|
stubRemoveFile(t, func(name string) error {
|
||||||
|
return &fs.PathError{Op: "remove", Path: name, Err: syscall.EROFS}
|
||||||
|
})
|
||||||
|
|
||||||
|
_, err := MergeDuplicateGroup(context.Background(), f.pool, nil, "", f.groupID, f.keep.ID)
|
||||||
|
var fre *FileRemoveError
|
||||||
|
if !errors.As(err, &fre) || !fre.NotWritable() {
|
||||||
|
t.Fatalf("err = %v, want a not-writable *FileRemoveError", err)
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM tracks WHERE id = $1`, f.remove.ID); n != 1 {
|
||||||
|
t.Errorf("the copy's row was deleted although its file was not")
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM general_likes WHERE track_id = $1`, f.remove.ID); n != 2 {
|
||||||
|
t.Errorf("likes on the copy = %d, want both still there", n)
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM play_events WHERE track_id = $1`, f.remove.ID); n != 2 {
|
||||||
|
t.Errorf("plays on the copy = %d, want both still there", n)
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM duplicate_groups WHERE id = $1 AND status = 'pending'`, f.groupID); n != 1 {
|
||||||
|
t.Errorf("group left pending = false, want it still pending")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeDuplicateGroup_SurvivorMustBeAMember(t *testing.T) {
|
||||||
|
f := newMergeFixture(t)
|
||||||
|
var stranger pgtype.UUID
|
||||||
|
stranger.Bytes[15], stranger.Valid = 0xEE, true
|
||||||
|
|
||||||
|
_, err := MergeDuplicateGroup(context.Background(), f.pool, nil, "", f.groupID, stranger)
|
||||||
|
if !errors.Is(err, ErrSurvivorNotInGroup) {
|
||||||
|
t.Fatalf("err = %v, want ErrSurvivorNotInGroup", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(f.removePath); err != nil {
|
||||||
|
t.Errorf("a refused merge removed a file: %v", err)
|
||||||
|
}
|
||||||
|
if n := f.count(t, `SELECT count(*) FROM duplicate_groups WHERE id = $1 AND status = 'pending'`, f.groupID); n != 1 {
|
||||||
|
t.Errorf("a refused merge changed the group")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SurvivorCandidate is what choosing which copy to keep needs to know about one
|
||||||
|
// member of a duplicate group.
|
||||||
|
type SurvivorCandidate struct {
|
||||||
|
TrackID string
|
||||||
|
FileFormat string
|
||||||
|
FileSize int64
|
||||||
|
AddedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// losslessFormats are the scanned extensions that are lossless by definition.
|
||||||
|
// m4a is left out on purpose: it holds either ALAC or AAC, and the scanner
|
||||||
|
// records only the extension, so calling it lossless would sometimes prefer an
|
||||||
|
// AAC copy over a FLAC one.
|
||||||
|
var losslessFormats = map[string]bool{"flac": true, "wav": true}
|
||||||
|
|
||||||
|
// ProposeSurvivor picks which copy of a duplicate group to keep, and gives the
|
||||||
|
// reason in words the operator reads beside it. It is a default, not a verdict:
|
||||||
|
// the report shows it and the merge (#3911) lets the operator choose another.
|
||||||
|
//
|
||||||
|
// In order:
|
||||||
|
// 1. lossless over lossy — the one difference no later step can recover
|
||||||
|
// 2. the larger file — for one recording at one duration that is the higher
|
||||||
|
// bitrate. The scanner does not record bitrate (tracks.bitrate is never
|
||||||
|
// filled), so file size is the signal that actually exists
|
||||||
|
// 3. the copy in the library longest — the one most likely to carry the play
|
||||||
|
// history and likes, so the merge moves the least
|
||||||
|
// 4. the lowest track id, so the choice is stable between page loads
|
||||||
|
func ProposeSurvivor(cands []SurvivorCandidate) (trackID, reason string) {
|
||||||
|
if len(cands) == 0 {
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
ranked := append([]SurvivorCandidate(nil), cands...)
|
||||||
|
sort.SliceStable(ranked, func(i, j int) bool { return survivorBefore(ranked[i], ranked[j]) })
|
||||||
|
best := ranked[0]
|
||||||
|
if len(ranked) == 1 {
|
||||||
|
return best.TrackID, "the only copy"
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reason names the first rule that separated the best copy from the
|
||||||
|
// runner-up — the rule that actually decided, not every rule it passed.
|
||||||
|
next := ranked[1]
|
||||||
|
switch {
|
||||||
|
case isLossless(best) != isLossless(next):
|
||||||
|
return best.TrackID, "lossless (" + strings.ToLower(best.FileFormat) + ")"
|
||||||
|
case best.FileSize != next.FileSize:
|
||||||
|
return best.TrackID, "largest file"
|
||||||
|
case !best.AddedAt.Equal(next.AddedAt):
|
||||||
|
return best.TrackID, "in the library longest"
|
||||||
|
default:
|
||||||
|
return best.TrackID, "copies are otherwise identical"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func survivorBefore(a, b SurvivorCandidate) bool {
|
||||||
|
if isLossless(a) != isLossless(b) {
|
||||||
|
return isLossless(a)
|
||||||
|
}
|
||||||
|
if a.FileSize != b.FileSize {
|
||||||
|
return a.FileSize > b.FileSize
|
||||||
|
}
|
||||||
|
if !a.AddedAt.Equal(b.AddedAt) {
|
||||||
|
return a.AddedAt.Before(b.AddedAt)
|
||||||
|
}
|
||||||
|
return a.TrackID < b.TrackID
|
||||||
|
}
|
||||||
|
|
||||||
|
func isLossless(c SurvivorCandidate) bool {
|
||||||
|
return losslessFormats[strings.ToLower(c.FileFormat)]
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProposeSurvivor(t *testing.T) {
|
||||||
|
older := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
newer := older.Add(24 * time.Hour)
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
cands []SurvivorCandidate
|
||||||
|
wantID string
|
||||||
|
wantReason string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
// Lossless wins even against a much larger lossy file, and even
|
||||||
|
// when the lossy copy has been in the library longer.
|
||||||
|
name: "lossless beats larger and older",
|
||||||
|
cands: []SurvivorCandidate{
|
||||||
|
{TrackID: "mp3", FileFormat: "mp3", FileSize: 90_000_000, AddedAt: older},
|
||||||
|
{TrackID: "flac", FileFormat: "FLAC", FileSize: 30_000_000, AddedAt: newer},
|
||||||
|
},
|
||||||
|
wantID: "flac", wantReason: "lossless (flac)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// m4a may be AAC; it must not outrank an mp3 just for being m4a.
|
||||||
|
name: "m4a is not treated as lossless",
|
||||||
|
cands: []SurvivorCandidate{
|
||||||
|
{TrackID: "m4a", FileFormat: "m4a", FileSize: 5_000_000, AddedAt: older},
|
||||||
|
{TrackID: "mp3", FileFormat: "mp3", FileSize: 9_000_000, AddedAt: newer},
|
||||||
|
},
|
||||||
|
wantID: "mp3", wantReason: "largest file",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "larger file wins among lossy copies",
|
||||||
|
cands: []SurvivorCandidate{
|
||||||
|
{TrackID: "128k", FileFormat: "mp3", FileSize: 3_400_000, AddedAt: older},
|
||||||
|
{TrackID: "320k", FileFormat: "mp3", FileSize: 8_600_000, AddedAt: newer},
|
||||||
|
},
|
||||||
|
wantID: "320k", wantReason: "largest file",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The #3885 pair: identical audio, sizes equal but for the tags.
|
||||||
|
name: "the longest-standing copy wins when size ties",
|
||||||
|
cands: []SurvivorCandidate{
|
||||||
|
{TrackID: "www-02", FileFormat: "mp3", FileSize: 6_900_000, AddedAt: newer},
|
||||||
|
{TrackID: "www-01", FileFormat: "mp3", FileSize: 6_900_000, AddedAt: older},
|
||||||
|
},
|
||||||
|
wantID: "www-01", wantReason: "in the library longest",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a full tie falls back to the lowest id, stably",
|
||||||
|
cands: []SurvivorCandidate{
|
||||||
|
{TrackID: "b", FileFormat: "mp3", FileSize: 1, AddedAt: older},
|
||||||
|
{TrackID: "a", FileFormat: "mp3", FileSize: 1, AddedAt: older},
|
||||||
|
},
|
||||||
|
wantID: "a", wantReason: "copies are otherwise identical",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "one copy",
|
||||||
|
cands: []SurvivorCandidate{{TrackID: "only", FileFormat: "mp3", FileSize: 1, AddedAt: older}},
|
||||||
|
wantID: "only", wantReason: "the only copy",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
id, reason := ProposeSurvivor(tc.cands)
|
||||||
|
if id != tc.wantID || reason != tc.wantReason {
|
||||||
|
t.Fatalf("ProposeSurvivor = (%q, %q), want (%q, %q)", id, reason, tc.wantID, tc.wantReason)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The reason must name the rule that decided. Across three copies that is the
|
||||||
|
// comparison between first and second place, not the first rule any pair
|
||||||
|
// differs on: here the lossy copy differs from the others by format, but the
|
||||||
|
// two FLACs are separated by size.
|
||||||
|
func TestProposeSurvivor_ReasonIsTheDecidingRule(t *testing.T) {
|
||||||
|
at := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
id, reason := ProposeSurvivor([]SurvivorCandidate{
|
||||||
|
{TrackID: "mp3", FileFormat: "mp3", FileSize: 99_000_000, AddedAt: at},
|
||||||
|
{TrackID: "flac-small", FileFormat: "flac", FileSize: 20_000_000, AddedAt: at},
|
||||||
|
{TrackID: "flac-big", FileFormat: "flac", FileSize: 40_000_000, AddedAt: at},
|
||||||
|
})
|
||||||
|
if id != "flac-big" || reason != "largest file" {
|
||||||
|
t.Fatalf("got (%q, %q), want (flac-big, largest file)", id, reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,398 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Duplicate sweep (M400 #3910).
|
||||||
|
//
|
||||||
|
// Reads fingerprints, runs them through the matcher and records what it proposes
|
||||||
|
// in duplicate_groups. It never merges or deletes anything the operator has not
|
||||||
|
// asked for: a group is a proposal, reviewed in the admin report (#3912).
|
||||||
|
//
|
||||||
|
// Blocked on duration, deliberately not on title: the #3885 pair are titled
|
||||||
|
// "WWW" and "WWW (instrumental)", so a title block would have missed the case
|
||||||
|
// that started the milestone. Candidates stream in (duration_ms, id) order and
|
||||||
|
// the grouper holds only a few seconds of durations at a time.
|
||||||
|
|
||||||
|
// duplicateCandidatePage is how many candidates one query returns. Each row
|
||||||
|
// carries a ~4 KB fingerprint, so a page is about 2 MB.
|
||||||
|
const duplicateCandidatePage = 500
|
||||||
|
|
||||||
|
// duplicateSweepTick is how often the worker checks whether a sweep is due. The
|
||||||
|
// operator's sweep interval (#3913) is the least time between sweeps; the tick
|
||||||
|
// only bounds how late past it one starts. With nothing due, a tick is two cheap
|
||||||
|
// aggregate queries.
|
||||||
|
const duplicateSweepTick = time.Hour
|
||||||
|
|
||||||
|
// sweepIntervalSlack absorbs the moment between a tick and the sweep it starts
|
||||||
|
// stamping started_at. Without it a one-hour interval checked on a one-hour tick
|
||||||
|
// would find the last sweep a moment under an hour old, and skip every other tick.
|
||||||
|
const sweepIntervalSlack = 5 * time.Minute
|
||||||
|
|
||||||
|
// staleDuplicateSweepThreshold is the age past which an in-flight sweep is
|
||||||
|
// assumed dead — a crash mid-sweep leaves finished_at NULL for ever — and another
|
||||||
|
// may start. Twice the library scan's threshold, because a sweep compares
|
||||||
|
// fingerprints across the whole library and can legitimately run long on a big
|
||||||
|
// one.
|
||||||
|
const staleDuplicateSweepThreshold = 2 * time.Hour
|
||||||
|
|
||||||
|
// duplicateSweepFinishTimeout bounds recording that a sweep ended. It runs on a
|
||||||
|
// context detached from the sweep's own, so a sweep cancelled at shutdown still
|
||||||
|
// closes its row rather than leaving it in flight until the reaper.
|
||||||
|
const duplicateSweepFinishTimeout = 10 * time.Second
|
||||||
|
|
||||||
|
// DuplicateSweepResult tallies one sweep.
|
||||||
|
type DuplicateSweepResult struct {
|
||||||
|
Candidates int // tracks with a chromaprint that were streamed
|
||||||
|
Groups int // groups the matcher found
|
||||||
|
Proposed int // written as pending, new or refreshed
|
||||||
|
Suppressed int // not proposed: already dismissed or resolved by the operator
|
||||||
|
Retired int // pending proposals this sweep did not find again, removed
|
||||||
|
Oversize int // acoustic clusters too large to propose
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunDuplicateSweep runs one sweep and records it in duplicate_sweeps. cfg is a
|
||||||
|
// snapshot: one sweep applies one threshold and one length throughout.
|
||||||
|
func RunDuplicateSweep(
|
||||||
|
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings,
|
||||||
|
) (DuplicateSweepResult, error) {
|
||||||
|
return runDuplicateSweep(ctx, pool, logger, cfg, duplicateCandidatePage)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDuplicateSweep(
|
||||||
|
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings, pageSize int32,
|
||||||
|
) (DuplicateSweepResult, error) {
|
||||||
|
q := dbq.New(pool)
|
||||||
|
sweep, err := q.StartDuplicateSweep(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return DuplicateSweepResult{}, fmt.Errorf("start duplicate sweep: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
res, runErr := sweepDuplicates(ctx, q, sweep.ID, cfg, pageSize)
|
||||||
|
|
||||||
|
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), duplicateSweepFinishTimeout)
|
||||||
|
defer cancel()
|
||||||
|
errMsg := ""
|
||||||
|
if runErr != nil {
|
||||||
|
errMsg = runErr.Error()
|
||||||
|
}
|
||||||
|
candidates, groups, oversize := int32(res.Candidates), int32(res.Groups), int32(res.Oversize)
|
||||||
|
if ferr := q.FinishDuplicateSweep(finishCtx, dbq.FinishDuplicateSweepParams{
|
||||||
|
ID: sweep.ID, Candidates: &candidates, GroupsFound: &groups, OversizeClusters: &oversize,
|
||||||
|
ErrorMessage: errMsg,
|
||||||
|
}); ferr != nil {
|
||||||
|
logger.Error("duplicate sweep: recording the end of the sweep failed", "err", ferr)
|
||||||
|
if runErr == nil {
|
||||||
|
runErr = fmt.Errorf("finish duplicate sweep: %w", ferr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("duplicate sweep complete",
|
||||||
|
"candidates", res.Candidates, "groups", res.Groups, "proposed", res.Proposed,
|
||||||
|
"suppressed", res.Suppressed, "retired", res.Retired, "oversize", res.Oversize, "err", runErr)
|
||||||
|
return res, runErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func sweepDuplicates(
|
||||||
|
ctx context.Context, q *dbq.Queries, sweepID pgtype.UUID, cfg FingerprintSettings, pageSize int32,
|
||||||
|
) (DuplicateSweepResult, error) {
|
||||||
|
var res DuplicateSweepResult
|
||||||
|
|
||||||
|
// Exact tier, library-wide, in one query.
|
||||||
|
exactRows, err := q.ListExactDuplicateHashes(ctx, fingerprintVersion)
|
||||||
|
if err != nil {
|
||||||
|
return res, fmt.Errorf("list exact duplicates: %w", err)
|
||||||
|
}
|
||||||
|
exactMembers := make([][]string, len(exactRows))
|
||||||
|
exactOf := map[string]int{}
|
||||||
|
for i, row := range exactRows {
|
||||||
|
exactMembers[i] = formatUUIDs(row.TrackIds)
|
||||||
|
for _, id := range exactMembers[i] {
|
||||||
|
exactOf[id] = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
exactSeen := make([]bool, len(exactRows))
|
||||||
|
|
||||||
|
// Acoustic tier, streamed in duration order. The first member of an exact
|
||||||
|
// group the stream meets stands in for the whole group; the rest are skipped.
|
||||||
|
// Only prints at the current length are streamed: a print at another length
|
||||||
|
// cannot be compared, and is waiting on the backfill to be re-derived.
|
||||||
|
grouper := newStreamGrouper(cfg.AcousticMaxBitErrorRate)
|
||||||
|
params := dbq.ListDuplicateCandidatesParams{
|
||||||
|
CurrentVersion: fingerprintVersion,
|
||||||
|
ChromaprintLengthSec: cfg.ChromaprintLengthSec,
|
||||||
|
// Durations are never negative, and the all-zero uuid sorts first: every
|
||||||
|
// row is after this cursor. Valid must be true, or "> NULL" matches nothing.
|
||||||
|
AfterDurationMs: -1,
|
||||||
|
AfterID: pgtype.UUID{Valid: true},
|
||||||
|
PageLimit: pageSize,
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
rows, err := q.ListDuplicateCandidates(ctx, params)
|
||||||
|
if err != nil {
|
||||||
|
return res, fmt.Errorf("list duplicate candidates: %w", err)
|
||||||
|
}
|
||||||
|
for _, row := range rows {
|
||||||
|
res.Candidates++
|
||||||
|
id := syncpkg.FormatUUID(row.ID)
|
||||||
|
unit := &groupUnit{ids: []string{id}, durationMs: row.DurationMs, sortKey: id}
|
||||||
|
if gi, ok := exactOf[id]; ok {
|
||||||
|
if exactSeen[gi] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
exactSeen[gi] = true
|
||||||
|
unit.ids, unit.exact = exactMembers[gi], true
|
||||||
|
}
|
||||||
|
unit.print = preparePrint(row.Chromaprint)
|
||||||
|
grouper.push(unit)
|
||||||
|
}
|
||||||
|
if int32(len(rows)) < pageSize {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
last := rows[len(rows)-1]
|
||||||
|
params.AfterDurationMs, params.AfterID = last.DurationMs, last.ID
|
||||||
|
}
|
||||||
|
|
||||||
|
found := grouper.finish()
|
||||||
|
// Exact groups none of whose members has a chromaprint never reached the
|
||||||
|
// stream. Identical bytes need no acoustic evidence.
|
||||||
|
for gi, seen := range exactSeen {
|
||||||
|
if !seen {
|
||||||
|
found.Groups = append(found.Groups, duplicateGroup{Tier: tierExact, Members: exactMembers[gi]})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sortGroups(found.Groups)
|
||||||
|
res.Groups, res.Oversize = len(found.Groups), found.OversizeClusters
|
||||||
|
|
||||||
|
dismissed, err := q.ListDismissedDuplicateMemberSets(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return res, fmt.Errorf("list dismissed duplicate groups: %w", err)
|
||||||
|
}
|
||||||
|
dismissedSets := make([]map[string]struct{}, 0, len(dismissed))
|
||||||
|
for _, d := range dismissed {
|
||||||
|
set := map[string]struct{}{}
|
||||||
|
for _, id := range formatUUIDs(d.TrackIds) {
|
||||||
|
set[id] = struct{}{}
|
||||||
|
}
|
||||||
|
dismissedSets = append(dismissedSets, set)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, group := range found.Groups {
|
||||||
|
if coveredByDismissal(group.Members, dismissedSets) {
|
||||||
|
res.Suppressed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
up := dbq.UpsertDuplicateGroupParams{
|
||||||
|
MemberKey: strings.Join(group.Members, ","),
|
||||||
|
Tier: string(group.Tier),
|
||||||
|
SweepID: sweepID,
|
||||||
|
}
|
||||||
|
if group.Tier == tierAcoustic {
|
||||||
|
worst := float32(group.WorstBitErrorRate)
|
||||||
|
up.WorstBitErrorRate = &worst
|
||||||
|
}
|
||||||
|
groupID, err := q.UpsertDuplicateGroup(ctx, up)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
// This exact member set was already dismissed or merged.
|
||||||
|
res.Suppressed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return res, fmt.Errorf("upsert duplicate group: %w", err)
|
||||||
|
}
|
||||||
|
for _, id := range group.Members {
|
||||||
|
var trackID pgtype.UUID
|
||||||
|
if err := trackID.Scan(id); err != nil {
|
||||||
|
return res, fmt.Errorf("parse track id %q: %w", id, err)
|
||||||
|
}
|
||||||
|
if err := q.AddDuplicateGroupMember(ctx, dbq.AddDuplicateGroupMemberParams{
|
||||||
|
GroupID: groupID, TrackID: trackID,
|
||||||
|
}); err != nil {
|
||||||
|
return res, fmt.Errorf("add duplicate group member: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
res.Proposed++
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only after a complete sweep: a sweep that failed partway has no basis for
|
||||||
|
// concluding that anything it did not reach has gone.
|
||||||
|
retired, err := q.DeleteStalePendingDuplicateGroups(ctx, sweepID)
|
||||||
|
if err != nil {
|
||||||
|
return res, fmt.Errorf("retire stale duplicate groups: %w", err)
|
||||||
|
}
|
||||||
|
res.Retired = int(retired)
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// coveredByDismissal reports whether every member of a proposal sat together in
|
||||||
|
// one group the operator dismissed. A subset counts: dismissing {A, B, C} said
|
||||||
|
// none of them are copies of each other, so proposing {A, B} again would be
|
||||||
|
// asking the same question twice. A superset does not count: a new copy joining
|
||||||
|
// is new evidence, and worth asking about.
|
||||||
|
func coveredByDismissal(members []string, dismissed []map[string]struct{}) bool {
|
||||||
|
for _, set := range dismissed {
|
||||||
|
covered := true
|
||||||
|
for _, id := range members {
|
||||||
|
if _, ok := set[id]; !ok {
|
||||||
|
covered = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if covered {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatUUIDs(ids []pgtype.UUID) []string {
|
||||||
|
out := make([]string, len(ids))
|
||||||
|
for i, id := range ids {
|
||||||
|
out[i] = syncpkg.FormatUUID(id)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// TryStartDuplicateSweep starts a sweep in the background unless one is already
|
||||||
|
// running, reaping a sweep that has been in flight past
|
||||||
|
// staleDuplicateSweepThreshold. Mirrors TryStartScan. The sweep runs on ctx, so
|
||||||
|
// a caller answering an HTTP request must pass a context that outlives it.
|
||||||
|
func TryStartDuplicateSweep(
|
||||||
|
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings,
|
||||||
|
) (bool, error) {
|
||||||
|
q := dbq.New(pool)
|
||||||
|
row, err := q.GetInFlightDuplicateSweep(ctx)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
age := time.Since(row.StartedAt.Time)
|
||||||
|
if age <= staleDuplicateSweepThreshold {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
logger.Warn("reaping stale duplicate sweep", "id", syncpkg.FormatUUID(row.ID), "age", age)
|
||||||
|
if ferr := q.FinishDuplicateSweep(ctx, dbq.FinishDuplicateSweepParams{
|
||||||
|
ID: row.ID, ErrorMessage: "reaped (stale)",
|
||||||
|
}); ferr != nil {
|
||||||
|
return false, fmt.Errorf("reap stale duplicate sweep: %w", ferr)
|
||||||
|
}
|
||||||
|
case !errors.Is(err, pgx.ErrNoRows):
|
||||||
|
return false, fmt.Errorf("duplicate sweep in-flight check: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
if _, err := RunDuplicateSweep(ctx, pool, logger, cfg); err != nil {
|
||||||
|
logger.Warn("duplicate sweep failed", "err", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DuplicateSweepWorker sweeps whenever its input has changed, at most once per
|
||||||
|
// the operator's sweep interval.
|
||||||
|
type DuplicateSweepWorker struct {
|
||||||
|
pool *pgxpool.Pool
|
||||||
|
logger *slog.Logger
|
||||||
|
settings *FingerprintSettingsService
|
||||||
|
tick time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDuplicateSweepWorker builds a worker with the production cadence. settings
|
||||||
|
// is shared with the admin API; nil runs on defaults.
|
||||||
|
func NewDuplicateSweepWorker(
|
||||||
|
pool *pgxpool.Pool, logger *slog.Logger, settings *FingerprintSettingsService,
|
||||||
|
) *DuplicateSweepWorker {
|
||||||
|
return &DuplicateSweepWorker{pool: pool, logger: logger, settings: settings, tick: duplicateSweepTick}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run blocks until ctx is cancelled, checking once at start and then each tick.
|
||||||
|
func (w *DuplicateSweepWorker) Run(ctx context.Context) {
|
||||||
|
w.tickOnce(ctx)
|
||||||
|
t := time.NewTicker(w.tick)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
w.tickOnce(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tickOnce contains one check so nothing it does can stop the next tick (rule 157).
|
||||||
|
func (w *DuplicateSweepWorker) tickOnce(ctx context.Context) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
w.logger.Error("duplicate sweep: tick panicked", "panic", r)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
cfg := w.settings.Get()
|
||||||
|
due, err := duplicateSweepDue(ctx, dbq.New(w.pool), cfg, time.Now())
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() == nil {
|
||||||
|
w.logger.Warn("duplicate sweep: due check failed", "err", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !due {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := TryStartDuplicateSweep(ctx, w.pool, w.logger, cfg); err != nil {
|
||||||
|
w.logger.Warn("duplicate sweep: start failed", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// duplicateSweepDue reads what sweepIsDue decides on.
|
||||||
|
func duplicateSweepDue(ctx context.Context, q *dbq.Queries, cfg FingerprintSettings, now time.Time) (bool, error) {
|
||||||
|
latest, err := q.GetLatestFingerprintComputedAt(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("latest fingerprint: %w", err)
|
||||||
|
}
|
||||||
|
var lastStart pgtype.Timestamptz
|
||||||
|
last, err := q.GetLatestDuplicateSweep(ctx)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
lastStart = last.StartedAt
|
||||||
|
case !errors.Is(err, pgx.ErrNoRows):
|
||||||
|
return false, fmt.Errorf("latest duplicate sweep: %w", err)
|
||||||
|
}
|
||||||
|
return sweepIsDue(latest, lastStart, cfg, now), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sweepIsDue reports whether a sweep should start: something it reads has
|
||||||
|
// changed since the last sweep started, and the operator's interval has passed.
|
||||||
|
//
|
||||||
|
// Two things can change its answer. Fingerprints are its input, so any written
|
||||||
|
// after the last sweep began count; while the backfill runs that is true every
|
||||||
|
// tick, which is what the interval is for. And a settings save counts, because a
|
||||||
|
// new threshold or length changes what the same fingerprints group into.
|
||||||
|
func sweepIsDue(latestFingerprint, lastSweepStart pgtype.Timestamptz, cfg FingerprintSettings, now time.Time) bool {
|
||||||
|
if !latestFingerprint.Valid {
|
||||||
|
return false // nothing fingerprinted yet
|
||||||
|
}
|
||||||
|
if !lastSweepStart.Valid {
|
||||||
|
return true // never swept
|
||||||
|
}
|
||||||
|
interval := time.Duration(cfg.SweepIntervalHours) * time.Hour
|
||||||
|
if now.Sub(lastSweepStart.Time) < interval-sweepIntervalSlack {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return latestFingerprint.Time.After(lastSweepStart.Time) || cfg.UpdatedAt.After(lastSweepStart.Time)
|
||||||
|
}
|
||||||
@@ -0,0 +1,250 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCoveredByDismissal(t *testing.T) {
|
||||||
|
dismissed := []map[string]struct{}{{"a": {}, "b": {}, "c": {}}}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
members []string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"the same set", []string{"a", "b", "c"}, true},
|
||||||
|
{"a subset of it", []string{"a", "b"}, true},
|
||||||
|
// A new copy joining is new evidence: ask again.
|
||||||
|
{"a superset of it", []string{"a", "b", "c", "d"}, false},
|
||||||
|
{"overlapping only in part", []string{"a", "d"}, false},
|
||||||
|
{"unrelated", []string{"x", "y"}, false},
|
||||||
|
} {
|
||||||
|
if got := coveredByDismissal(tc.members, dismissed); got != tc.want {
|
||||||
|
t.Errorf("%s: coveredByDismissal = %v, want %v", tc.name, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDuplicateSweep_Integration pins what the sweep proposes, what it leaves out,
|
||||||
|
// and how re-sweeping treats a dismissal and a proposal that no longer holds.
|
||||||
|
func TestDuplicateSweep_Integration(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
q := dbq.New(pool)
|
||||||
|
dir := t.TempDir()
|
||||||
|
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||||
|
|
||||||
|
// seedTrack's own track has no fingerprint row: it must be absent from the
|
||||||
|
// report, not grouped with every other track lacking one.
|
||||||
|
_, album, artist := seedTrack(t, pool, filepath.Join(dir, "unfingerprinted.mp3"))
|
||||||
|
hash := func(b byte) []byte { return bytes.Repeat([]byte{b}, 32) }
|
||||||
|
add := func(name string, durationMs int32, sum []byte, print []int32) string {
|
||||||
|
t.Helper()
|
||||||
|
tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
|
||||||
|
Title: name, AlbumID: album.ID, ArtistID: artist.ID, DurationMs: durationMs,
|
||||||
|
FilePath: filepath.Join(dir, name+".mp3"), FileSize: 100, FileFormat: "mp3",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("track %s: %v", name, err)
|
||||||
|
}
|
||||||
|
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
|
||||||
|
TrackID: tr.ID, AudioStreamSha256: sum, Chromaprint: print, FingerprintVersion: fingerprintVersion,
|
||||||
|
ChromaprintLengthSec: defaultChromaprintLengthSec,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("fingerprint %s: %v", name, err)
|
||||||
|
}
|
||||||
|
return syncpkg.FormatUUID(tr.ID)
|
||||||
|
}
|
||||||
|
key := func(ids ...string) string {
|
||||||
|
sorted := append([]string(nil), ids...)
|
||||||
|
sort.Strings(sorted)
|
||||||
|
return strings.Join(sorted, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
recording := randomPrint(200, printLen)
|
||||||
|
onAlbum := add("recording-album", 240000, hash(1), recording)
|
||||||
|
onCompilation := add("recording-compilation", 241000, hash(2), withBitNoise(recording, 0.05, 201))
|
||||||
|
www1 := add("www-01", 215000, hash(9), randomPrint(210, printLen))
|
||||||
|
www2 := add("www-02", 215000, hash(9), randomPrint(210, printLen))
|
||||||
|
// Near-identical duration to the recording, different audio.
|
||||||
|
add("different-song", 240500, hash(3), randomPrint(220, printLen))
|
||||||
|
// Identical to the album copy, but its file is gone: nothing to compare.
|
||||||
|
missing := add("missing-copy", 240000, hash(4), recording)
|
||||||
|
if _, err := pool.Exec(ctx, "UPDATE tracks SET missing_since = now() WHERE file_path LIKE '%missing-copy.mp3'"); err != nil {
|
||||||
|
t.Fatalf("mark missing: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
type stored struct {
|
||||||
|
tier, status string
|
||||||
|
}
|
||||||
|
groups := func() map[string]stored {
|
||||||
|
t.Helper()
|
||||||
|
rows, err := pool.Query(ctx, `SELECT member_key, tier, status FROM duplicate_groups`)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read groups: %v", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := map[string]stored{}
|
||||||
|
for rows.Next() {
|
||||||
|
var k string
|
||||||
|
var s stored
|
||||||
|
if err := rows.Scan(&k, &s.tier, &s.status); err != nil {
|
||||||
|
t.Fatalf("scan group: %v", err)
|
||||||
|
}
|
||||||
|
out[k] = s
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. A page size of one forces the keyset cursor across every candidate.
|
||||||
|
res, err := runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first sweep: %v", err)
|
||||||
|
}
|
||||||
|
// Five tracks carry a chromaprint and a present file.
|
||||||
|
if res.Candidates != 5 || res.Groups != 2 || res.Proposed != 2 {
|
||||||
|
t.Fatalf("first sweep = %+v, want 5 candidates, 2 groups, 2 proposed", res)
|
||||||
|
}
|
||||||
|
acousticKey, exactKey := key(onAlbum, onCompilation), key(www1, www2)
|
||||||
|
got := groups()
|
||||||
|
want := map[string]stored{
|
||||||
|
acousticKey: {"acoustic", "pending"},
|
||||||
|
exactKey: {"exact", "pending"},
|
||||||
|
}
|
||||||
|
if len(got) != len(want) || got[acousticKey] != want[acousticKey] || got[exactKey] != want[exactKey] {
|
||||||
|
t.Fatalf("groups = %+v, want %+v", got, want)
|
||||||
|
}
|
||||||
|
for k := range got {
|
||||||
|
if strings.Contains(k, missing) {
|
||||||
|
t.Fatalf("a missing track was proposed: %s", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. A dismissed group is not proposed again, and the pending one is
|
||||||
|
// refreshed in place rather than duplicated.
|
||||||
|
if _, err := pool.Exec(ctx, "UPDATE duplicate_groups SET status = 'dismissed' WHERE member_key = $1", acousticKey); err != nil {
|
||||||
|
t.Fatalf("dismiss: %v", err)
|
||||||
|
}
|
||||||
|
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second sweep: %v", err)
|
||||||
|
}
|
||||||
|
if res.Proposed != 1 || res.Suppressed != 1 {
|
||||||
|
t.Fatalf("second sweep = %+v, want 1 proposed, 1 suppressed", res)
|
||||||
|
}
|
||||||
|
got = groups()
|
||||||
|
if len(got) != 2 || got[acousticKey].status != "dismissed" || got[exactKey].status != "pending" {
|
||||||
|
t.Fatalf("after dismissal groups = %+v, want the dismissal kept and one pending group", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. A proposal that no longer holds is retired; the dismissal survives it.
|
||||||
|
if _, err := pool.Exec(ctx,
|
||||||
|
"DELETE FROM track_fingerprints f USING tracks t WHERE f.track_id = t.id AND t.file_path LIKE '%www-02.mp3'"); err != nil {
|
||||||
|
t.Fatalf("drop fingerprint: %v", err)
|
||||||
|
}
|
||||||
|
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("third sweep: %v", err)
|
||||||
|
}
|
||||||
|
if res.Retired != 1 {
|
||||||
|
t.Fatalf("third sweep = %+v, want 1 retired", res)
|
||||||
|
}
|
||||||
|
got = groups()
|
||||||
|
if len(got) != 1 || got[acousticKey].status != "dismissed" {
|
||||||
|
t.Fatalf("after retiring groups = %+v, want only the dismissal", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. The sweep record reflects the last run.
|
||||||
|
last, err := q.GetLatestDuplicateSweep(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("latest sweep: %v", err)
|
||||||
|
}
|
||||||
|
if !last.FinishedAt.Valid || last.ErrorMessage != nil {
|
||||||
|
t.Fatalf("latest sweep = %+v, want finished without error", last)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Prints taken at another length are never compared (#3913). Every print
|
||||||
|
// here was taken at the default length, so a sweep at 60s has nothing to read,
|
||||||
|
// rather than scoring 120s prints against each other as if they were 60s ones.
|
||||||
|
atOtherLength := DefaultFingerprintSettings
|
||||||
|
atOtherLength.ChromaprintLengthSec = 60
|
||||||
|
res, err = runDuplicateSweep(ctx, pool, logger, atOtherLength, duplicateCandidatePage)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("sweep at another length: %v", err)
|
||||||
|
}
|
||||||
|
if res.Candidates != 0 || res.Groups != 0 {
|
||||||
|
t.Fatalf("sweep at another length = %+v, want no candidates and no groups", res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. The sweep applies the threshold it is given. The recording's two copies
|
||||||
|
// disagree on about 5% of their bits: grouped at the default, not at 1%.
|
||||||
|
if _, err := pool.Exec(ctx, "DELETE FROM duplicate_groups"); err != nil {
|
||||||
|
t.Fatalf("clear groups: %v", err)
|
||||||
|
}
|
||||||
|
strict := DefaultFingerprintSettings
|
||||||
|
strict.AcousticMaxBitErrorRate = 0.01
|
||||||
|
res, err = runDuplicateSweep(ctx, pool, logger, strict, duplicateCandidatePage)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("strict sweep: %v", err)
|
||||||
|
}
|
||||||
|
if res.Groups != 0 {
|
||||||
|
t.Fatalf("sweep at a 1%% threshold = %+v, want the copies 5%% apart left ungrouped", res)
|
||||||
|
}
|
||||||
|
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("default sweep: %v", err)
|
||||||
|
}
|
||||||
|
if got := groups(); res.Groups != 1 || got[acousticKey] != (stored{"acoustic", "pending"}) {
|
||||||
|
t.Fatalf("sweep at the default threshold = %+v, groups %+v; want the recording's copies proposed", res, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSweepIsDue(t *testing.T) {
|
||||||
|
now := time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC)
|
||||||
|
at := func(ago time.Duration) pgtype.Timestamptz {
|
||||||
|
return pgtype.Timestamptz{Time: now.Add(-ago), Valid: true}
|
||||||
|
}
|
||||||
|
never := pgtype.Timestamptz{}
|
||||||
|
hourly := DefaultFingerprintSettings
|
||||||
|
daily := DefaultFingerprintSettings
|
||||||
|
daily.SweepIntervalHours = 24
|
||||||
|
savedAgo := func(ago time.Duration) FingerprintSettings {
|
||||||
|
s := DefaultFingerprintSettings
|
||||||
|
s.UpdatedAt = now.Add(-ago)
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
latestPrint, lastSweep pgtype.Timestamptz
|
||||||
|
cfg FingerprintSettings
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"nothing fingerprinted", never, never, hourly, false},
|
||||||
|
{"never swept", at(time.Minute), never, hourly, true},
|
||||||
|
{"new fingerprints since the last sweep", at(10 * time.Minute), at(2 * time.Hour), hourly, true},
|
||||||
|
{"nothing new since the last sweep", at(3 * time.Hour), at(2 * time.Hour), hourly, false},
|
||||||
|
// The sweep started a moment after the previous tick, so one tick later
|
||||||
|
// it is a moment under an hour old. Without the slack this is false.
|
||||||
|
{"one tick after an hourly sweep", at(time.Minute), at(time.Hour - 2*time.Second), hourly, true},
|
||||||
|
{"new fingerprints inside the interval", at(time.Minute), at(3 * time.Hour), daily, false},
|
||||||
|
{"new fingerprints past the interval", at(time.Minute), at(25 * time.Hour), daily, true},
|
||||||
|
{"settings saved since the last sweep", at(3 * time.Hour), at(2 * time.Hour), savedAgo(time.Hour), true},
|
||||||
|
{"settings saved before the last sweep", at(3 * time.Hour), at(2 * time.Hour), savedAgo(4 * time.Hour), false},
|
||||||
|
} {
|
||||||
|
if got := sweepIsDue(tc.latestPrint, tc.lastSweep, tc.cfg, now); got != tc.want {
|
||||||
|
t.Errorf("%s: sweepIsDue = %v, want %v", tc.name, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,339 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Acoustic identity (M400).
|
||||||
|
//
|
||||||
|
// Two values per track, because they answer different questions:
|
||||||
|
//
|
||||||
|
// audio_stream_sha256 a SHA-256 of the ENCODED audio packets. Equal means the
|
||||||
|
// same audio bytes, whatever the tags or container around
|
||||||
|
// them say. No threshold and no false positives — this is
|
||||||
|
// what catches two copies of one MP3 that differ only in
|
||||||
|
// their ID3 (#3885).
|
||||||
|
//
|
||||||
|
// chromaprint fpcalc's raw fingerprint. Close means the same
|
||||||
|
// recording, even at another bitrate or in another codec
|
||||||
|
// — the case an exact hash cannot see.
|
||||||
|
//
|
||||||
|
// Both shell out, in the shape probeDurationMs already set: a deadline on every
|
||||||
|
// call, and a failure that leaves the value unset rather than failing the file.
|
||||||
|
// A track with no fingerprint is never a duplicate candidate; it is still a
|
||||||
|
// track.
|
||||||
|
|
||||||
|
// fingerprintTimeout bounds one ffmpeg hash or fpcalc call. Longer than
|
||||||
|
// probeTimeout because both read the audio rather than a header: the hash reads
|
||||||
|
// every packet and fpcalc decodes up to its -length. 60s leaves room for a large
|
||||||
|
// lossless file on a slow network mount; a call needing more is a stall, not a
|
||||||
|
// big file.
|
||||||
|
const fingerprintTimeout = 60 * time.Second
|
||||||
|
|
||||||
|
// fingerprintWaitDelay bounds how long Output may keep waiting on the tool's
|
||||||
|
// pipes after the deadline has killed it. Without it, a child that left a
|
||||||
|
// descendant holding stdout open would block the scan past its own timeout.
|
||||||
|
const fingerprintWaitDelay = 5 * time.Second
|
||||||
|
|
||||||
|
// fingerprintVersion stamps how a track_fingerprints row was derived. Bump it
|
||||||
|
// whenever the derivation changes — the hash arguments or fpcalc's flags — and
|
||||||
|
// the backfill re-derives every row below it. Fingerprints taken by two methods
|
||||||
|
// are not comparable, and nothing else would reveal that the library held a mix.
|
||||||
|
//
|
||||||
|
// The length is deliberately not part of it: it is an operator setting (#3913),
|
||||||
|
// so each row records the length it was taken at and readers compare only rows
|
||||||
|
// at the current one. See fingerprint_settings.go.
|
||||||
|
const fingerprintVersion int16 = 1
|
||||||
|
|
||||||
|
// errFingerprintTimeout marks a tool that ran out of time. Distinct from a
|
||||||
|
// failed exit because a stall is a fact about the mount, not about the file.
|
||||||
|
var errFingerprintTimeout = errors.New("fingerprint tool timed out")
|
||||||
|
|
||||||
|
// defaultChromaprintLengthSec is the shipped value of the length setting (#3913):
|
||||||
|
// how many seconds of audio fpcalc fingerprints. 120 is fpcalc's own default.
|
||||||
|
const defaultChromaprintLengthSec = 120
|
||||||
|
|
||||||
|
// errChromaprintSkipped marks a chromaprint not taken because fingerprinting is
|
||||||
|
// switched off. Inconclusive rather than a verdict: nothing was learned about
|
||||||
|
// the file.
|
||||||
|
var errChromaprintSkipped = errors.New("chromaprint skipped: fingerprinting is off")
|
||||||
|
|
||||||
|
// fpcalcStderrTail caps how much of a failing tool's stderr reaches the log.
|
||||||
|
const fpcalcStderrTail = 512
|
||||||
|
|
||||||
|
// streamHashArgs hashes the encoded audio packets, never decoded samples.
|
||||||
|
//
|
||||||
|
// -c:a copy is the point, not an optimisation. A decoded hash of a lossy file
|
||||||
|
// depends on the decoder's float maths and sample conversion, which can move
|
||||||
|
// between ffmpeg releases — so an image upgrade could silently change every
|
||||||
|
// stored hash, and yesterday's duplicate would stop matching today's copy.
|
||||||
|
// Packet bytes do not move. It is also far cheaper: demux only, no decode.
|
||||||
|
//
|
||||||
|
// -map 0:a keeps embedded cover art (an attached-picture video stream) out of
|
||||||
|
// the hash, so two copies of one recording carrying different art still match.
|
||||||
|
func streamHashArgs(path string) []string {
|
||||||
|
return []string{
|
||||||
|
"-v", "error",
|
||||||
|
"-i", path,
|
||||||
|
"-map", "0:a",
|
||||||
|
"-c:a", "copy",
|
||||||
|
"-f", "hash", "-hash", "sha256",
|
||||||
|
"-",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fpcalcArgs asks for the raw fingerprint as SIGNED integers.
|
||||||
|
//
|
||||||
|
// -raw because the matcher compares items bit by bit, which the compressed form
|
||||||
|
// cannot do without being unpacked first. -signed because the column is Postgres
|
||||||
|
// integer[], which is signed: fpcalc's default prints uint32, and half of those
|
||||||
|
// values do not fit. Signed output is the same 32 bits with no reinterpretation
|
||||||
|
// step left to get wrong.
|
||||||
|
func fpcalcArgs(path string, lengthSec int) []string {
|
||||||
|
return []string{
|
||||||
|
"-raw", "-signed",
|
||||||
|
"-length", strconv.Itoa(lengthSec),
|
||||||
|
path,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fingerprintResult is one attempt at both halves of a track's identity. They
|
||||||
|
// fail independently: a file ffmpeg can demux may still defeat fpcalc.
|
||||||
|
type fingerprintResult struct {
|
||||||
|
streamSHA256 []byte
|
||||||
|
chromaprint []int32
|
||||||
|
hashErr error
|
||||||
|
printErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
// fingerprintOptions is what the settings decide for one attempt. Captured once
|
||||||
|
// per file, so the length a chromaprint was taken at is the length stored with it
|
||||||
|
// even if the setting changes mid-attempt.
|
||||||
|
type fingerprintOptions struct {
|
||||||
|
lengthSec int32
|
||||||
|
// chromaprint false takes the stream hash alone: a demux, no decode.
|
||||||
|
chromaprint bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// computeFingerprint derives the halves opts asks for, for the file at path.
|
||||||
|
func computeFingerprint(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||||
|
var r fingerprintResult
|
||||||
|
r.streamSHA256, r.hashErr = computeAudioStreamSHA256(ctx, path)
|
||||||
|
if !opts.chromaprint {
|
||||||
|
r.printErr = errChromaprintSkipped
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
r.chromaprint, r.printErr = computeChromaprint(ctx, path, opts.lengthSec)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// inconclusive reports whether either half failed for a reason that says
|
||||||
|
// nothing about the file. Such a result must never be stored: stamped at the
|
||||||
|
// current version it would read as "tried, and this file cannot be
|
||||||
|
// fingerprinted", and the backfill would never try it again.
|
||||||
|
func (r fingerprintResult) inconclusive() bool {
|
||||||
|
return isInconclusive(r.hashErr) || isInconclusive(r.printErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isInconclusive names the failures that are not a verdict on the file: a
|
||||||
|
// stall, a cancelled scan, a tool that is not installed, and a chromaprint
|
||||||
|
// skipped because fingerprinting is off. A missing tool matters outside the
|
||||||
|
// image — a dev binary run without fpcalc on PATH must not stamp every track in
|
||||||
|
// the library as unfingerprintable.
|
||||||
|
func isInconclusive(err error) bool {
|
||||||
|
return errors.Is(err, errFingerprintTimeout) ||
|
||||||
|
errors.Is(err, errChromaprintSkipped) ||
|
||||||
|
errors.Is(err, context.Canceled) ||
|
||||||
|
errors.Is(err, context.DeadlineExceeded) ||
|
||||||
|
errors.Is(err, exec.ErrNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fingerprintFile runs the scanner's fingerprinter. A Scanner built without New
|
||||||
|
// gets the real tools rather than a nil-func panic halfway through a scan.
|
||||||
|
func (s *Scanner) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||||
|
if s.fingerprint == nil {
|
||||||
|
return computeFingerprint(ctx, path, opts)
|
||||||
|
}
|
||||||
|
return s.fingerprint(ctx, path, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fingerprintOutcome is what storeFingerprint did with one attempt.
|
||||||
|
type fingerprintOutcome int
|
||||||
|
|
||||||
|
const (
|
||||||
|
outcomeFingerprinted fingerprintOutcome = iota // both halves stored
|
||||||
|
outcomeRejected // stored with a NULL half: a verdict
|
||||||
|
outcomeInconclusive // nothing stored; worth trying again
|
||||||
|
outcomeStoreFailed // the write itself failed
|
||||||
|
)
|
||||||
|
|
||||||
|
// storeFingerprint records one attempt, for the scan (new or changed bytes) and
|
||||||
|
// the backfill (#3908) alike, so there is one rule for what gets written. It
|
||||||
|
// never fails its caller: a missing fingerprint only keeps a track out of
|
||||||
|
// duplicate detection, which is not worth dropping a scan or a pass over.
|
||||||
|
//
|
||||||
|
// lengthSec is the length fp's chromaprint was taken at, stored with it (#3913).
|
||||||
|
func storeFingerprint(
|
||||||
|
ctx context.Context, q *dbq.Queries, logger *slog.Logger,
|
||||||
|
trackID pgtype.UUID, path string, fp fingerprintResult, lengthSec int32,
|
||||||
|
) fingerprintOutcome {
|
||||||
|
if fp.hashErr != nil {
|
||||||
|
logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr)
|
||||||
|
}
|
||||||
|
if fp.printErr != nil {
|
||||||
|
logger.Warn("fingerprint: chromaprint failed", "path", path, "err", fp.printErr)
|
||||||
|
}
|
||||||
|
if fp.inconclusive() {
|
||||||
|
// Any row this track holds describes bytes we could not confirm — the
|
||||||
|
// previous bytes for the scan, an older derivation for the backfill.
|
||||||
|
// Drop it rather than stamp a failure that says nothing about the file.
|
||||||
|
if err := q.DeleteTrackFingerprint(ctx, trackID); err != nil {
|
||||||
|
logger.Warn("fingerprint: clearing stale fingerprint failed", "path", path, "err", err)
|
||||||
|
}
|
||||||
|
return outcomeInconclusive
|
||||||
|
}
|
||||||
|
// A NULL half here is a verdict — the tool ran and rejected this file — and
|
||||||
|
// is stamped at the current version so the backfill does not retry it on
|
||||||
|
// every pass. It is retried when the file changes.
|
||||||
|
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
|
||||||
|
TrackID: trackID,
|
||||||
|
AudioStreamSha256: fp.streamSHA256,
|
||||||
|
Chromaprint: fp.chromaprint,
|
||||||
|
FingerprintVersion: fingerprintVersion,
|
||||||
|
ChromaprintLengthSec: lengthSec,
|
||||||
|
}); err != nil {
|
||||||
|
logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err)
|
||||||
|
return outcomeStoreFailed
|
||||||
|
}
|
||||||
|
if fp.hashErr != nil || fp.printErr != nil {
|
||||||
|
return outcomeRejected
|
||||||
|
}
|
||||||
|
return outcomeFingerprinted
|
||||||
|
}
|
||||||
|
|
||||||
|
// computeAudioStreamSHA256 returns the SHA-256 of the file's encoded audio.
|
||||||
|
func computeAudioStreamSHA256(ctx context.Context, path string) ([]byte, error) {
|
||||||
|
out, err := runFingerprintTool(ctx, "ffmpeg", streamHashArgs(path))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return parseStreamHash(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// computeChromaprint returns the raw acoustic fingerprint of the first
|
||||||
|
// lengthSec seconds of the file.
|
||||||
|
func computeChromaprint(ctx context.Context, path string, lengthSec int32) ([]int32, error) {
|
||||||
|
out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, int(lengthSec)))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return parseFpcalcRaw(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// runFingerprintTool runs one tool under fingerprintTimeout.
|
||||||
|
//
|
||||||
|
// Any non-zero exit is an error, and that deliberately includes fpcalc's exit 3:
|
||||||
|
// "reading failed, but here is a fingerprint of what I got". A partial
|
||||||
|
// fingerprint of a damaged file is not that file's identity. Stored, it would
|
||||||
|
// score against a healthy copy over whatever prefix survived, and could group
|
||||||
|
// or fail to group either way. Absent is better than wrong.
|
||||||
|
func runFingerprintTool(ctx context.Context, name string, args []string) ([]byte, error) {
|
||||||
|
runCtx, cancel := context.WithTimeout(ctx, fingerprintTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(runCtx, name, args...)
|
||||||
|
cmd.WaitDelay = fingerprintWaitDelay
|
||||||
|
out, err := cmd.Output()
|
||||||
|
if err == nil {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
// The caller gave up (a cancelled scan). Report that rather than the
|
||||||
|
// signal-killed exit it caused, so it is never mistaken for a verdict on
|
||||||
|
// the file.
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", name, ctx.Err())
|
||||||
|
}
|
||||||
|
// Named separately so a stall reads as a stall, not as a crash.
|
||||||
|
if errors.Is(runCtx.Err(), context.DeadlineExceeded) {
|
||||||
|
return nil, fmt.Errorf("%s: no result within %s: %w", name, fingerprintTimeout, errFingerprintTimeout)
|
||||||
|
}
|
||||||
|
var exitErr *exec.ExitError
|
||||||
|
if errors.As(err, &exitErr) {
|
||||||
|
return nil, fmt.Errorf("%s exited %d: %s", name, exitErr.ExitCode(), stderrTail(exitErr.Stderr))
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%s: %w", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// stderrTail keeps the END of a failing tool's stderr. ffmpeg and fpcalc print
|
||||||
|
// the actual reason last, after any banner or per-frame warnings, so a cap that
|
||||||
|
// kept the head would log the noise and drop the cause.
|
||||||
|
func stderrTail(stderr []byte) []byte {
|
||||||
|
stderr = bytes.TrimSpace(stderr)
|
||||||
|
if len(stderr) > fpcalcStderrTail {
|
||||||
|
stderr = stderr[len(stderr)-fpcalcStderrTail:]
|
||||||
|
}
|
||||||
|
return stderr
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseStreamHash reads the ffmpeg hash muxer's "SHA256=<hex>" line.
|
||||||
|
func parseStreamHash(out []byte) ([]byte, error) {
|
||||||
|
for _, line := range strings.Split(string(out), "\n") {
|
||||||
|
hexed, ok := strings.CutPrefix(strings.TrimSpace(line), "SHA256=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sum, err := hex.DecodeString(hexed)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("stream hash %q: %w", hexed, err)
|
||||||
|
}
|
||||||
|
if len(sum) != sha256.Size {
|
||||||
|
return nil, fmt.Errorf("stream hash is %d bytes, want %d", len(sum), sha256.Size)
|
||||||
|
}
|
||||||
|
return sum, nil
|
||||||
|
}
|
||||||
|
return nil, errors.New("ffmpeg printed no SHA256= line")
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseFpcalcRaw reads fpcalc's text output:
|
||||||
|
//
|
||||||
|
// DURATION=<seconds>
|
||||||
|
// FINGERPRINT=<int32>,<int32>,...
|
||||||
|
func parseFpcalcRaw(out []byte) ([]int32, error) {
|
||||||
|
for _, line := range strings.Split(string(out), "\n") {
|
||||||
|
list, ok := strings.CutPrefix(strings.TrimSpace(line), "FINGERPRINT=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if list == "" {
|
||||||
|
return nil, errors.New("fpcalc returned an empty fingerprint")
|
||||||
|
}
|
||||||
|
items := strings.Split(list, ",")
|
||||||
|
fp := make([]int32, len(items))
|
||||||
|
for i, item := range items {
|
||||||
|
// ParseInt at 32 bits, not ParseUint: a value past int32 means the
|
||||||
|
// output was unsigned — -signed went missing from the invocation —
|
||||||
|
// and nothing downstream would reinterpret it. Refuse it here.
|
||||||
|
v, err := strconv.ParseInt(item, 10, 32)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("fingerprint item %d %q: %w", i, item, err)
|
||||||
|
}
|
||||||
|
fp[i] = int32(v)
|
||||||
|
}
|
||||||
|
return fp, nil
|
||||||
|
}
|
||||||
|
return nil, errors.New("fpcalc printed no FINGERPRINT= line")
|
||||||
|
}
|
||||||
@@ -0,0 +1,224 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Fingerprint backfill (M400 #3908).
|
||||||
|
//
|
||||||
|
// The scan fingerprints only bytes it has not seen (see scanFile), so every track
|
||||||
|
// imported before fingerprinting existed — and every row derived by an older
|
||||||
|
// fingerprintVersion — needs a pass of its own. That pass is this worker.
|
||||||
|
//
|
||||||
|
// Its own worker rather than a stage in RunScan, for two reasons, both about
|
||||||
|
// time:
|
||||||
|
// - RunScan runs at boot and then every safetyNetScanInterval (12h), and an
|
||||||
|
// in-flight scan older than StuckScanThreshold (1h) is reaped and a second
|
||||||
|
// one started beside it. A stage would have to stop well inside the hour — a
|
||||||
|
// few hundred decodes — so a 50k-track library would take about a month.
|
||||||
|
// - A long stage holds the scan run in flight, and a manual rescan answers 409
|
||||||
|
// for as long as it runs.
|
||||||
|
//
|
||||||
|
// Progress is read live (FingerprintCoverage, the admin gauge) rather than from a
|
||||||
|
// scan_runs tally: the work spans many passes with no single run to attach to.
|
||||||
|
|
||||||
|
// fingerprintBackfillTick is how often the worker looks for work. Once the
|
||||||
|
// library has caught up, a tick is one indexed query; mostly the hour bounds how
|
||||||
|
// long a file that timed out on a slow mount waits before it is tried again.
|
||||||
|
const fingerprintBackfillTick = time.Hour
|
||||||
|
|
||||||
|
// fingerprintBackfillBatch is how many tracks one query hands the worker. Small,
|
||||||
|
// so tracks the scan adds mid-pass are not stuck behind one enormous page.
|
||||||
|
const fingerprintBackfillBatch = 50
|
||||||
|
|
||||||
|
// fingerprintBackfillConcurrency is the shipped value of the concurrency setting
|
||||||
|
// (#3913): how many files are decoded at once. Two is deliberately low: fpcalc
|
||||||
|
// and the stream hash compete with playback transcoding for CPU and with
|
||||||
|
// streaming for the mount, and a backfill that makes playback stutter is worse
|
||||||
|
// than one that takes longer.
|
||||||
|
const fingerprintBackfillConcurrency = 2
|
||||||
|
|
||||||
|
// BackfillFingerprintsResult tallies one pass.
|
||||||
|
type BackfillFingerprintsResult struct {
|
||||||
|
Processed int
|
||||||
|
Fingerprinted int // both halves stored
|
||||||
|
Rejected int // stored with a NULL half: a tool refused the file (settled)
|
||||||
|
Inconclusive int // nothing stored; tried again on a later pass
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *BackfillFingerprintsResult) add(o fingerprintOutcome) {
|
||||||
|
r.Processed++
|
||||||
|
switch o {
|
||||||
|
case outcomeFingerprinted:
|
||||||
|
r.Fingerprinted++
|
||||||
|
case outcomeRejected:
|
||||||
|
r.Rejected++
|
||||||
|
default:
|
||||||
|
r.Inconclusive++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FingerprintBackfillWorker fingerprints the tracks the scan never will.
|
||||||
|
type FingerprintBackfillWorker struct {
|
||||||
|
pool *pgxpool.Pool
|
||||||
|
logger *slog.Logger
|
||||||
|
settings *FingerprintSettingsService
|
||||||
|
tick time.Duration
|
||||||
|
batch int32
|
||||||
|
// fingerprint is a field for the same reason as Scanner.fingerprint: an
|
||||||
|
// integration test pins which tracks a pass touches, not what the tools print.
|
||||||
|
fingerprint func(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFingerprintBackfillWorker builds a worker with the production cadence.
|
||||||
|
// settings is shared with the scanner and the admin API; nil runs on defaults.
|
||||||
|
func NewFingerprintBackfillWorker(
|
||||||
|
pool *pgxpool.Pool, logger *slog.Logger, settings *FingerprintSettingsService,
|
||||||
|
) *FingerprintBackfillWorker {
|
||||||
|
return &FingerprintBackfillWorker{
|
||||||
|
pool: pool,
|
||||||
|
logger: logger,
|
||||||
|
settings: settings,
|
||||||
|
tick: fingerprintBackfillTick,
|
||||||
|
batch: fingerprintBackfillBatch,
|
||||||
|
fingerprint: computeFingerprint,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run blocks until ctx is cancelled: one pass at start, so a fresh deploy does
|
||||||
|
// not sit idle for an hour, then one per tick.
|
||||||
|
func (w *FingerprintBackfillWorker) Run(ctx context.Context) {
|
||||||
|
w.runOnce(ctx)
|
||||||
|
t := time.NewTicker(w.tick)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
w.runOnce(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runOnce contains a pass so that nothing it does — an error, a panic — can stop
|
||||||
|
// the next tick from firing (rule 157).
|
||||||
|
func (w *FingerprintBackfillWorker) runOnce(ctx context.Context) {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
w.logger.Error("fingerprint backfill: pass panicked", "panic", r)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
res, err := w.pass(ctx)
|
||||||
|
if err != nil && ctx.Err() == nil {
|
||||||
|
w.logger.Warn("fingerprint backfill: pass failed", "err", err, "processed", res.Processed)
|
||||||
|
}
|
||||||
|
if res.Processed > 0 {
|
||||||
|
w.logger.Info("fingerprint backfill: pass complete",
|
||||||
|
"processed", res.Processed, "fingerprinted", res.Fingerprinted,
|
||||||
|
"rejected", res.Rejected, "inconclusive", res.Inconclusive)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pass walks every track needing a fingerprint once, keyset-paged on id. The
|
||||||
|
// cursor is what lets a pass end: an inconclusive attempt writes no row, so a
|
||||||
|
// file that keeps timing out would otherwise be listed again immediately and
|
||||||
|
// retried forever within the pass.
|
||||||
|
//
|
||||||
|
// Settings are read before every batch, so a save takes effect within a batch
|
||||||
|
// rather than an hour (#3913): switching fingerprinting off ends the pass, a new
|
||||||
|
// concurrency applies to the next batch, and a new length restarts the walk from
|
||||||
|
// the top at that length, because every row written at the old one went stale
|
||||||
|
// the moment it changed.
|
||||||
|
func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerprintsResult, error) {
|
||||||
|
q := dbq.New(w.pool)
|
||||||
|
var (
|
||||||
|
res BackfillFingerprintsResult
|
||||||
|
mu sync.Mutex
|
||||||
|
)
|
||||||
|
// The all-zero uuid sorts before every real id. Valid must be true: a NULL
|
||||||
|
// cursor would make "id > NULL" match nothing and every pass a silent no-op.
|
||||||
|
start := pgtype.UUID{Valid: true}
|
||||||
|
after := start
|
||||||
|
lengthSec := w.settings.Get().ChromaprintLengthSec
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
cfg := w.settings.Get()
|
||||||
|
if !cfg.Enabled {
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
if cfg.ChromaprintLengthSec != lengthSec {
|
||||||
|
lengthSec, after = cfg.ChromaprintLengthSec, start
|
||||||
|
}
|
||||||
|
rows, err := q.ListTracksNeedingFingerprint(ctx, dbq.ListTracksNeedingFingerprintParams{
|
||||||
|
CurrentVersion: fingerprintVersion,
|
||||||
|
ChromaprintLengthSec: lengthSec,
|
||||||
|
AfterID: after,
|
||||||
|
BatchLimit: w.batch,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return res, fmt.Errorf("list tracks needing fingerprint: %w", err)
|
||||||
|
}
|
||||||
|
if len(rows) == 0 {
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
opts := fingerprintOptions{lengthSec: lengthSec, chromaprint: true}
|
||||||
|
// Validation keeps concurrency at one or more; the floor guards a zero
|
||||||
|
// that would block the first send for ever.
|
||||||
|
sem := make(chan struct{}, max(1, int(cfg.BackfillConcurrency)))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for _, row := range rows {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
sem <- struct{}{}
|
||||||
|
wg.Add(1)
|
||||||
|
go func(trackID pgtype.UUID, path string) {
|
||||||
|
defer wg.Done()
|
||||||
|
defer func() { <-sem }()
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
w.logger.Error("fingerprint backfill: track panicked", "path", path, "panic", r)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
outcome := storeFingerprint(ctx, q, w.logger, trackID, path, w.fingerprintFile(ctx, path, opts), lengthSec)
|
||||||
|
mu.Lock()
|
||||||
|
res.add(outcome)
|
||||||
|
mu.Unlock()
|
||||||
|
}(row.ID, row.FilePath)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
after = rows[len(rows)-1].ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *FingerprintBackfillWorker) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||||
|
if w.fingerprint == nil {
|
||||||
|
return computeFingerprint(ctx, path, opts)
|
||||||
|
}
|
||||||
|
return w.fingerprint(ctx, path, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FingerprintCoverage reports how much of the library carries a current
|
||||||
|
// fingerprint, for the admin gauge. It lives here, beside the backfill, so the
|
||||||
|
// version and length it counts against are the ones the backfill writes.
|
||||||
|
func FingerprintCoverage(
|
||||||
|
ctx context.Context, pool *pgxpool.Pool, cfg FingerprintSettings,
|
||||||
|
) (dbq.GetFingerprintCoverageRow, error) {
|
||||||
|
return dbq.New(pool).GetFingerprintCoverage(ctx, dbq.GetFingerprintCoverageParams{
|
||||||
|
CurrentVersion: fingerprintVersion,
|
||||||
|
ChromaprintLengthSec: cfg.ChromaprintLengthSec,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"maps"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestFingerprintBackfill_Integration pins which tracks a pass touches, that a
|
||||||
|
// pass ends, and that the coverage gauge counts what the pass wrote.
|
||||||
|
func TestFingerprintBackfill_Integration(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
q := dbq.New(pool)
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
_, album, artist := seedTrack(t, pool, filepath.Join(dir, "unfingerprinted.mp3"))
|
||||||
|
addTrack := func(name string) dbq.Track {
|
||||||
|
t.Helper()
|
||||||
|
tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
|
||||||
|
Title: name, AlbumID: album.ID, ArtistID: artist.ID,
|
||||||
|
DurationMs: 1000, FilePath: filepath.Join(dir, name+".mp3"), FileSize: 100, FileFormat: "mp3",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("track %s: %v", name, err)
|
||||||
|
}
|
||||||
|
return tr
|
||||||
|
}
|
||||||
|
current := addTrack("current")
|
||||||
|
stale := addTrack("stale")
|
||||||
|
missing := addTrack("missing")
|
||||||
|
|
||||||
|
sum := bytes.Repeat([]byte{0xCD}, 32)
|
||||||
|
for _, seed := range []struct {
|
||||||
|
track dbq.Track
|
||||||
|
version int16
|
||||||
|
}{
|
||||||
|
{current, fingerprintVersion},
|
||||||
|
{stale, fingerprintVersion - 1},
|
||||||
|
} {
|
||||||
|
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
|
||||||
|
TrackID: seed.track.ID, AudioStreamSha256: sum, Chromaprint: []int32{1},
|
||||||
|
FingerprintVersion: seed.version, ChromaprintLengthSec: defaultChromaprintLengthSec,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("seed fingerprint: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := pool.Exec(ctx, "UPDATE tracks SET missing_since = now() WHERE id = $1", missing.ID); err != nil {
|
||||||
|
t.Fatalf("mark missing: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var mu sync.Mutex
|
||||||
|
calls := map[string]int{}
|
||||||
|
settings, err := NewFingerprintSettingsService(ctx, pool)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("fingerprint settings: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if _, err := settings.Set(context.Background(), DefaultFingerprintSettings); err != nil {
|
||||||
|
t.Errorf("restore fingerprint settings: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
w := NewFingerprintBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), settings)
|
||||||
|
// A batch of one forces the keyset cursor across several queries in a pass.
|
||||||
|
w.batch = 1
|
||||||
|
lengths := map[int32]int{}
|
||||||
|
w.fingerprint = func(_ context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||||
|
name := filepath.Base(path)
|
||||||
|
mu.Lock()
|
||||||
|
calls[name]++
|
||||||
|
lengths[opts.lengthSec]++
|
||||||
|
mu.Unlock()
|
||||||
|
switch name {
|
||||||
|
case "stall.mp3":
|
||||||
|
return fingerprintResult{streamSHA256: sum, printErr: fmt.Errorf("fpcalc: %w", errFingerprintTimeout)}
|
||||||
|
case "rejected.mp3":
|
||||||
|
return fingerprintResult{hashErr: errors.New("ffmpeg exited 1"), printErr: errors.New("fpcalc exited 2")}
|
||||||
|
default:
|
||||||
|
return fingerprintResult{streamSHA256: sum, chromaprint: []int32{7, -7}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
callCount := func(name string) int {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return calls[name]
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Only the track with no row and the stale one are fingerprinted — never
|
||||||
|
// the current one, never the missing one.
|
||||||
|
res, err := w.pass(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first pass: %v", err)
|
||||||
|
}
|
||||||
|
if res.Processed != 2 || res.Fingerprinted != 2 {
|
||||||
|
t.Fatalf("first pass = %+v, want 2 processed, 2 fingerprinted", res)
|
||||||
|
}
|
||||||
|
for name, want := range map[string]int{
|
||||||
|
"unfingerprinted.mp3": 1, "stale.mp3": 1, "current.mp3": 0, "missing.mp3": 0,
|
||||||
|
} {
|
||||||
|
if got := callCount(name); got != want {
|
||||||
|
t.Errorf("%s fingerprinted %d times, want %d", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. A pass after a complete one is a no-op. A backfill that redoes its work
|
||||||
|
// every hour is the expensive way this could be wrong.
|
||||||
|
res, err = w.pass(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second pass: %v", err)
|
||||||
|
}
|
||||||
|
if res.Processed != 0 {
|
||||||
|
t.Fatalf("second pass processed %d tracks, want 0", res.Processed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. An inconclusive file is tried exactly once and the pass ENDS. Without the
|
||||||
|
// keyset cursor it would be re-listed immediately and this call would never
|
||||||
|
// return.
|
||||||
|
addTrack("stall")
|
||||||
|
addTrack("rejected")
|
||||||
|
res, err = w.pass(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("third pass: %v", err)
|
||||||
|
}
|
||||||
|
if res.Processed != 2 || res.Inconclusive != 1 || res.Rejected != 1 {
|
||||||
|
t.Fatalf("third pass = %+v, want 2 processed, 1 inconclusive, 1 rejected", res)
|
||||||
|
}
|
||||||
|
if got := callCount("stall.mp3"); got != 1 {
|
||||||
|
t.Fatalf("stalling file tried %d times in one pass, want exactly 1", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. The gauge counts what the passes wrote, and its buckets add up.
|
||||||
|
cov, err := FingerprintCoverage(ctx, pool, settings.Get())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("coverage: %v", err)
|
||||||
|
}
|
||||||
|
// Five present tracks: unfingerprinted, current, stale, stall, rejected.
|
||||||
|
// The missing track is not counted.
|
||||||
|
if cov.Total != 5 || cov.Fingerprinted != 3 || cov.Rejected != 1 || cov.Pending != 1 {
|
||||||
|
t.Errorf("coverage = %+v, want total 5, fingerprinted 3, rejected 1, pending 1", cov)
|
||||||
|
}
|
||||||
|
if cov.Fingerprinted+cov.Rejected+cov.Pending != cov.Total {
|
||||||
|
t.Errorf("coverage buckets %+v do not sum to the total", cov)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Changing the length (#3913) makes every stored row stale at once — the
|
||||||
|
// gauge shows the whole library pending before the backfill has touched a
|
||||||
|
// file — and the next pass re-derives each at the new length. The failure
|
||||||
|
// this prevents is invisible from the UI: prints at two lengths that silently
|
||||||
|
// never match.
|
||||||
|
shorter := DefaultFingerprintSettings
|
||||||
|
shorter.ChromaprintLengthSec = 60
|
||||||
|
if _, err := settings.Set(ctx, shorter); err != nil {
|
||||||
|
t.Fatalf("change length: %v", err)
|
||||||
|
}
|
||||||
|
cov, err = FingerprintCoverage(ctx, pool, settings.Get())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("coverage after length change: %v", err)
|
||||||
|
}
|
||||||
|
if cov.Total != 5 || cov.Pending != 5 {
|
||||||
|
t.Fatalf("coverage after length change = %+v, want all 5 tracks pending", cov)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
clear(lengths)
|
||||||
|
mu.Unlock()
|
||||||
|
res, err = w.pass(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new-length pass: %v", err)
|
||||||
|
}
|
||||||
|
if res.Processed != 5 {
|
||||||
|
t.Fatalf("new-length pass = %+v, want all 5 present tracks re-derived", res)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
asked := maps.Clone(lengths)
|
||||||
|
mu.Unlock()
|
||||||
|
if len(asked) != 1 || asked[60] != 5 {
|
||||||
|
t.Fatalf("new-length pass asked for lengths %v, want 60s for all 5", asked)
|
||||||
|
}
|
||||||
|
var atOldLength int
|
||||||
|
if err := pool.QueryRow(ctx,
|
||||||
|
"SELECT count(*) FROM track_fingerprints WHERE chromaprint_length_sec <> 60").Scan(&atOldLength); err != nil {
|
||||||
|
t.Fatalf("count old-length rows: %v", err)
|
||||||
|
}
|
||||||
|
if atOldLength != 0 {
|
||||||
|
t.Fatalf("%d fingerprints are still at the old length after a complete pass", atOldLength)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. Switched off, the backfill does nothing, even with work waiting.
|
||||||
|
off := DefaultFingerprintSettings
|
||||||
|
off.Enabled = false
|
||||||
|
if _, err := settings.Set(ctx, off); err != nil {
|
||||||
|
t.Fatalf("switch fingerprinting off: %v", err)
|
||||||
|
}
|
||||||
|
addTrack("later")
|
||||||
|
res, err = w.pass(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("pass with fingerprinting off: %v", err)
|
||||||
|
}
|
||||||
|
if res.Processed != 0 || callCount("later.mp3") != 0 {
|
||||||
|
t.Fatalf("pass with fingerprinting off = %+v (later.mp3 tried %d times), want nothing done",
|
||||||
|
res, callCount("later.mp3"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackfillFingerprintsResult_Add(t *testing.T) {
|
||||||
|
var r BackfillFingerprintsResult
|
||||||
|
for _, o := range []fingerprintOutcome{
|
||||||
|
outcomeFingerprinted, outcomeFingerprinted, outcomeRejected, outcomeInconclusive, outcomeStoreFailed,
|
||||||
|
} {
|
||||||
|
r.add(o)
|
||||||
|
}
|
||||||
|
// A failed write stored nothing, so like an inconclusive attempt it is
|
||||||
|
// tried again next pass — and counts as such.
|
||||||
|
want := BackfillFingerprintsResult{Processed: 5, Fingerprinted: 2, Rejected: 1, Inconclusive: 2}
|
||||||
|
if r != want {
|
||||||
|
t.Errorf("tally = %+v, want %+v", r, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration pins WHEN the scan
|
||||||
|
// fingerprints. The cost of getting it wrong is asymmetric and invisible: a
|
||||||
|
// scan that re-fingerprints unchanged files still produces correct rows, just
|
||||||
|
// by decoding the entire library on every tag-repair pass.
|
||||||
|
//
|
||||||
|
// The fingerprinter is stubbed. CI has no real audio, and the tools' output is
|
||||||
|
// covered by the parser tests; this covers the scan's decisions.
|
||||||
|
func TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("skipping scanner integration in -short mode")
|
||||||
|
}
|
||||||
|
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
|
||||||
|
if dsn == "" {
|
||||||
|
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||||
|
|
||||||
|
if err := db.Migrate(dsn, logger); err != nil {
|
||||||
|
t.Fatalf("migrate: %v", err)
|
||||||
|
}
|
||||||
|
pool, err := pgxpool.New(ctx, dsn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("pool: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(pool.Close)
|
||||||
|
if _, err := pool.Exec(ctx, "TRUNCATE tracks, albums, artists RESTART IDENTITY CASCADE"); err != nil {
|
||||||
|
t.Fatalf("truncate: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
a := filepath.Join(root, "artist/album/01.mp3")
|
||||||
|
b := filepath.Join(root, "artist/album/02.mp3")
|
||||||
|
writeTestMP3(t, a, map[string]string{"TIT2": "One", "TPE1": "Artist", "TALB": "Album", "TRCK": "1"})
|
||||||
|
writeTestMP3(t, b, map[string]string{"TIT2": "Two", "TPE1": "Artist", "TALB": "Album", "TRCK": "2"})
|
||||||
|
|
||||||
|
sum := bytes.Repeat([]byte{0xAB}, 32)
|
||||||
|
chroma := []int32{7, -7, 2147483647}
|
||||||
|
result := fingerprintResult{streamSHA256: sum, chromaprint: chroma}
|
||||||
|
calls := map[string]int{}
|
||||||
|
|
||||||
|
settings, err := NewFingerprintSettingsService(ctx, pool)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("fingerprint settings: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := settings.Set(ctx, DefaultFingerprintSettings); err != nil {
|
||||||
|
t.Fatalf("reset fingerprint settings: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if _, err := settings.Set(context.Background(), DefaultFingerprintSettings); err != nil {
|
||||||
|
t.Errorf("restore fingerprint settings: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
var lastOpts fingerprintOptions
|
||||||
|
scanner := New(pool, logger, []string{root}, settings)
|
||||||
|
scanner.fingerprint = func(_ context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||||
|
calls[path]++
|
||||||
|
lastOpts = opts
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
scan := func(step string) Stats {
|
||||||
|
t.Helper()
|
||||||
|
st, err := scanner.Scan(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: scan: %v", step, err)
|
||||||
|
}
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
type row struct {
|
||||||
|
sha []byte
|
||||||
|
chroma []int32
|
||||||
|
version int16
|
||||||
|
length int32
|
||||||
|
}
|
||||||
|
stored := func(path string) (row, bool) {
|
||||||
|
t.Helper()
|
||||||
|
var r row
|
||||||
|
err := pool.QueryRow(ctx, `
|
||||||
|
SELECT f.audio_stream_sha256, f.chromaprint, f.fingerprint_version, f.chromaprint_length_sec
|
||||||
|
FROM track_fingerprints f JOIN tracks t ON t.id = f.track_id
|
||||||
|
WHERE t.file_path = $1`, path).Scan(&r.sha, &r.chroma, &r.version, &r.length)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return row{}, false
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fingerprint for %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return r, true
|
||||||
|
}
|
||||||
|
// A later step moves mtime forward past the row's updated_at, which is
|
||||||
|
// what the scan reads as "these bytes changed".
|
||||||
|
touch := func(path string, ahead time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
when := time.Now().Add(ahead)
|
||||||
|
if err := os.Chtimes(path, when, when); err != nil {
|
||||||
|
t.Fatalf("chtimes %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. New files are fingerprinted, and stored at the current version.
|
||||||
|
scan("first scan")
|
||||||
|
if calls[a] != 1 || calls[b] != 1 {
|
||||||
|
t.Fatalf("first scan fingerprint calls = %v, want one per file", calls)
|
||||||
|
}
|
||||||
|
got, ok := stored(a)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("first scan stored no fingerprint")
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got.sha, sum) || !slices.Equal(got.chroma, chroma) || got.version != fingerprintVersion ||
|
||||||
|
got.length != defaultChromaprintLengthSec {
|
||||||
|
t.Fatalf("stored %+v, want sha %x chromaprint %v version %d length %d",
|
||||||
|
got, sum, chroma, fingerprintVersion, defaultChromaprintLengthSec)
|
||||||
|
}
|
||||||
|
if !lastOpts.chromaprint || lastOpts.lengthSec != defaultChromaprintLengthSec {
|
||||||
|
t.Fatalf("first scan asked for %+v, want a chromaprint at the default length", lastOpts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. A tag-repair pass re-reads every unchanged file and must not
|
||||||
|
// fingerprint any of them again.
|
||||||
|
//
|
||||||
|
// The Updated count is what makes this able to fail. Without it, a scan
|
||||||
|
// that simply SKIPPED both files would also leave the call counts at one,
|
||||||
|
// and the assertion would pass without the re-read path ever running.
|
||||||
|
if _, err := pool.Exec(ctx, "UPDATE tracks SET duration_ms = 1000, tag_read_version = 0"); err != nil {
|
||||||
|
t.Fatalf("force tag re-read: %v", err)
|
||||||
|
}
|
||||||
|
if st := scan("tag-repair scan"); st.Updated != 2 || st.Skipped != 0 {
|
||||||
|
t.Fatalf("tag-repair scan stats = %+v, want both files re-read (Updated=2 Skipped=0)", st)
|
||||||
|
}
|
||||||
|
if calls[a] != 1 || calls[b] != 1 {
|
||||||
|
t.Fatalf("tag-repair scan re-fingerprinted unchanged files: calls = %v", calls)
|
||||||
|
}
|
||||||
|
if _, ok := stored(a); !ok {
|
||||||
|
t.Fatal("tag-repair scan dropped a stored fingerprint")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Bytes that changed are fingerprinted again, and only those.
|
||||||
|
touch(a, time.Hour)
|
||||||
|
scan("changed-file scan")
|
||||||
|
if calls[a] != 2 || calls[b] != 1 {
|
||||||
|
t.Fatalf("changed-file scan calls = %v, want a=2 b=1", calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. A changed file whose attempt is inconclusive loses its old row: that
|
||||||
|
// row describes the previous bytes, and a stall says nothing about the new
|
||||||
|
// ones.
|
||||||
|
result = fingerprintResult{streamSHA256: sum, printErr: fmt.Errorf("fpcalc: %w", errFingerprintTimeout)}
|
||||||
|
touch(a, 2*time.Hour)
|
||||||
|
scan("inconclusive scan")
|
||||||
|
if _, ok := stored(a); ok {
|
||||||
|
t.Fatal("inconclusive attempt left the previous bytes' fingerprint in place")
|
||||||
|
}
|
||||||
|
if _, ok := stored(b); !ok {
|
||||||
|
t.Fatal("inconclusive attempt on one file removed another file's fingerprint")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. A file the tools reject gets a row at the current version with both
|
||||||
|
// halves NULL — a verdict, so the backfill does not retry it every boot.
|
||||||
|
result = fingerprintResult{
|
||||||
|
hashErr: errors.New("ffmpeg exited 1"),
|
||||||
|
printErr: errors.New("fpcalc exited 2"),
|
||||||
|
}
|
||||||
|
touch(a, 3*time.Hour)
|
||||||
|
scan("rejected scan")
|
||||||
|
got, ok = stored(a)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("a file the tools rejected got no row, so the backfill would retry it forever")
|
||||||
|
}
|
||||||
|
if got.sha != nil || got.chroma != nil || got.version != fingerprintVersion {
|
||||||
|
t.Fatalf("rejected file stored %+v, want both halves NULL at version %d", got, fingerprintVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. With fingerprinting off (#3913) the scan decodes nothing. It still asks
|
||||||
|
// for the stream hash — a demux, and what recognises a moved file — but stores
|
||||||
|
// no row, and a changed file's old row goes: it describes bytes that are gone.
|
||||||
|
result = fingerprintResult{streamSHA256: sum, chromaprint: chroma}
|
||||||
|
off := DefaultFingerprintSettings
|
||||||
|
off.Enabled = false
|
||||||
|
if _, err := settings.Set(ctx, off); err != nil {
|
||||||
|
t.Fatalf("switch fingerprinting off: %v", err)
|
||||||
|
}
|
||||||
|
touch(a, 4*time.Hour)
|
||||||
|
scan("fingerprinting-off scan")
|
||||||
|
if calls[a] != 5 || lastOpts.chromaprint {
|
||||||
|
t.Fatalf("fingerprinting-off scan: calls = %v, last options %+v; want a fifth call asking for no chromaprint",
|
||||||
|
calls, lastOpts)
|
||||||
|
}
|
||||||
|
if _, ok := stored(a); ok {
|
||||||
|
t.Fatal("with fingerprinting off, a changed file kept the previous bytes' fingerprint")
|
||||||
|
}
|
||||||
|
if _, ok := stored(b); !ok {
|
||||||
|
t.Fatal("with fingerprinting off, an unchanged file lost its fingerprint")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 7. The length setting reaches the scan, and is stored with the row.
|
||||||
|
longer := DefaultFingerprintSettings
|
||||||
|
longer.ChromaprintLengthSec = 90
|
||||||
|
if _, err := settings.Set(ctx, longer); err != nil {
|
||||||
|
t.Fatalf("change length: %v", err)
|
||||||
|
}
|
||||||
|
touch(a, 5*time.Hour)
|
||||||
|
scan("new-length scan")
|
||||||
|
got, ok = stored(a)
|
||||||
|
if !ok || got.length != 90 || lastOpts.lengthSec != 90 || !lastOpts.chromaprint {
|
||||||
|
t.Fatalf("new-length scan stored %+v (present %v) after asking for %+v; want a chromaprint at 90s", got, ok, lastOpts)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
|
||||||
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Fingerprinting settings (M400 #3913).
|
||||||
|
//
|
||||||
|
// Rule 25: what an operator might tune lives in the database and changes without
|
||||||
|
// a restart. One service instance is shared by the scanner, the backfill, the
|
||||||
|
// duplicate sweep and the admin API, so a save reaches all of them at once.
|
||||||
|
//
|
||||||
|
// The length is the dangerous knob: chromaprints taken at two lengths cannot be
|
||||||
|
// compared. Rather than bumping fingerprintVersion when it changes, every row
|
||||||
|
// records the length it was taken at (migration 0061) and every reader filters
|
||||||
|
// on the current one. A row at another length is stale to the backfill, pending
|
||||||
|
// to the gauge and invisible to the sweep, which is the effect a version bump
|
||||||
|
// would have, with one difference worth having: setting the length back makes
|
||||||
|
// rows not yet re-derived current again, instead of redoing the library twice.
|
||||||
|
|
||||||
|
// Bounds for each setting, mirrored by migration 0061's CHECKs.
|
||||||
|
const (
|
||||||
|
// Below 30s a print has too few items to align at the matcher's larger
|
||||||
|
// offsets and still overlap by minOverlapItems. Above 600s each print passes
|
||||||
|
// 20 KB and a candidate page of them grows past what one sweep should hold.
|
||||||
|
minChromaprintLengthSec = 30
|
||||||
|
maxChromaprintLengthSec = 600
|
||||||
|
// Unrelated audio sits near 0.5; a ceiling of 0.35 keeps a loosened
|
||||||
|
// threshold well clear of proposing noise.
|
||||||
|
minAcousticMaxBitErrorRate = 0.01
|
||||||
|
maxAcousticMaxBitErrorRate = 0.35
|
||||||
|
minBackfillConcurrency = 1
|
||||||
|
maxBackfillConcurrency = 8
|
||||||
|
minSweepIntervalHours = 1
|
||||||
|
maxSweepIntervalHours = 168
|
||||||
|
|
||||||
|
defaultSweepIntervalHours = 1
|
||||||
|
)
|
||||||
|
|
||||||
|
// FingerprintSettings mirrors the fingerprint_settings row.
|
||||||
|
type FingerprintSettings struct {
|
||||||
|
// Enabled off stops every decode: the scan takes only the stream hash (a
|
||||||
|
// demux, and what recognises a moved file) and stores nothing, and the
|
||||||
|
// backfill idles.
|
||||||
|
Enabled bool
|
||||||
|
ChromaprintLengthSec int32
|
||||||
|
AcousticMaxBitErrorRate float64
|
||||||
|
BackfillConcurrency int32
|
||||||
|
SweepIntervalHours int32
|
||||||
|
// UpdatedAt is when the settings were last saved. Set by the database;
|
||||||
|
// ignored by Set.
|
||||||
|
UpdatedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultFingerprintSettings mirrors migration 0061's column defaults, so a
|
||||||
|
// database that cannot be read still fingerprints the way a fresh install does.
|
||||||
|
var DefaultFingerprintSettings = FingerprintSettings{
|
||||||
|
Enabled: true,
|
||||||
|
ChromaprintLengthSec: defaultChromaprintLengthSec,
|
||||||
|
AcousticMaxBitErrorRate: defaultAcousticMaxBitErrorRate,
|
||||||
|
BackfillConcurrency: fingerprintBackfillConcurrency,
|
||||||
|
SweepIntervalHours: defaultSweepIntervalHours,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrFingerprintSettingOutOfRange is returned by Set for a value migration
|
||||||
|
// 0061's CHECKs would reject, so the API answers 400 naming the field rather
|
||||||
|
// than surfacing a constraint violation.
|
||||||
|
var ErrFingerprintSettingOutOfRange = errors.New("fingerprint setting out of range")
|
||||||
|
|
||||||
|
// FingerprintSettingsService caches the settings and owns their persistence.
|
||||||
|
// Cached because the scanner reads them for every file it fingerprints.
|
||||||
|
type FingerprintSettingsService struct {
|
||||||
|
pool *pgxpool.Pool
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
cur FingerprintSettings
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFingerprintSettingsService loads once and caches. It always returns a
|
||||||
|
// usable service, holding the defaults when the load fails; the error says so.
|
||||||
|
func NewFingerprintSettingsService(ctx context.Context, pool *pgxpool.Pool) (*FingerprintSettingsService, error) {
|
||||||
|
s := &FingerprintSettingsService{pool: pool, cur: DefaultFingerprintSettings}
|
||||||
|
row, err := dbq.New(pool).GetFingerprintSettings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return s, fmt.Errorf("fingerprint settings: load: %w", err)
|
||||||
|
}
|
||||||
|
s.cur = fingerprintSettingsFromRow(row)
|
||||||
|
return s, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns the cached settings. A nil service answers with the defaults, so
|
||||||
|
// a Scanner or worker built without one fingerprints as a fresh install would.
|
||||||
|
func (s *FingerprintSettingsService) Get() FingerprintSettings {
|
||||||
|
if s == nil {
|
||||||
|
return DefaultFingerprintSettings
|
||||||
|
}
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
return s.cur
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set validates, persists and re-caches.
|
||||||
|
func (s *FingerprintSettingsService) Set(ctx context.Context, in FingerprintSettings) (FingerprintSettings, error) {
|
||||||
|
if err := validateFingerprintSettings(in); err != nil {
|
||||||
|
return FingerprintSettings{}, err
|
||||||
|
}
|
||||||
|
if s == nil {
|
||||||
|
return FingerprintSettings{}, errors.New("fingerprint settings: no settings service")
|
||||||
|
}
|
||||||
|
row, err := dbq.New(s.pool).UpdateFingerprintSettings(ctx, dbq.UpdateFingerprintSettingsParams{
|
||||||
|
Enabled: in.Enabled,
|
||||||
|
ChromaprintLengthSec: in.ChromaprintLengthSec,
|
||||||
|
AcousticMaxBitErrorRate: in.AcousticMaxBitErrorRate,
|
||||||
|
BackfillConcurrency: in.BackfillConcurrency,
|
||||||
|
SweepIntervalHours: in.SweepIntervalHours,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return FingerprintSettings{}, fmt.Errorf("fingerprint settings: save: %w", err)
|
||||||
|
}
|
||||||
|
out := fingerprintSettingsFromRow(row)
|
||||||
|
s.mu.Lock()
|
||||||
|
s.cur = out
|
||||||
|
s.mu.Unlock()
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateFingerprintSettings(in FingerprintSettings) error {
|
||||||
|
switch {
|
||||||
|
case in.ChromaprintLengthSec < minChromaprintLengthSec || in.ChromaprintLengthSec > maxChromaprintLengthSec:
|
||||||
|
return fmt.Errorf("%w: chromaprint_length_sec must be %d-%d",
|
||||||
|
ErrFingerprintSettingOutOfRange, minChromaprintLengthSec, maxChromaprintLengthSec)
|
||||||
|
// Written as a negated range so NaN, which fails every comparison, is refused.
|
||||||
|
case !(in.AcousticMaxBitErrorRate >= minAcousticMaxBitErrorRate && in.AcousticMaxBitErrorRate <= maxAcousticMaxBitErrorRate):
|
||||||
|
return fmt.Errorf("%w: acoustic_max_bit_error_rate must be %.2f-%.2f",
|
||||||
|
ErrFingerprintSettingOutOfRange, minAcousticMaxBitErrorRate, maxAcousticMaxBitErrorRate)
|
||||||
|
case in.BackfillConcurrency < minBackfillConcurrency || in.BackfillConcurrency > maxBackfillConcurrency:
|
||||||
|
return fmt.Errorf("%w: backfill_concurrency must be %d-%d",
|
||||||
|
ErrFingerprintSettingOutOfRange, minBackfillConcurrency, maxBackfillConcurrency)
|
||||||
|
case in.SweepIntervalHours < minSweepIntervalHours || in.SweepIntervalHours > maxSweepIntervalHours:
|
||||||
|
return fmt.Errorf("%w: sweep_interval_hours must be %d-%d",
|
||||||
|
ErrFingerprintSettingOutOfRange, minSweepIntervalHours, maxSweepIntervalHours)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fingerprintSettingsFromRow(row dbq.FingerprintSetting) FingerprintSettings {
|
||||||
|
return FingerprintSettings{
|
||||||
|
Enabled: row.Enabled,
|
||||||
|
ChromaprintLengthSec: row.ChromaprintLengthSec,
|
||||||
|
AcousticMaxBitErrorRate: row.AcousticMaxBitErrorRate,
|
||||||
|
BackfillConcurrency: row.BackfillConcurrency,
|
||||||
|
SweepIntervalHours: row.SweepIntervalHours,
|
||||||
|
UpdatedAt: row.UpdatedAt.Time,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateFingerprintSettings(t *testing.T) {
|
||||||
|
with := func(edit func(*FingerprintSettings)) FingerprintSettings {
|
||||||
|
s := DefaultFingerprintSettings
|
||||||
|
edit(&s)
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
valid := map[string]FingerprintSettings{
|
||||||
|
"defaults": DefaultFingerprintSettings,
|
||||||
|
"shortest length": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = minChromaprintLengthSec }),
|
||||||
|
"longest length": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = maxChromaprintLengthSec }),
|
||||||
|
"strictest match": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = minAcousticMaxBitErrorRate }),
|
||||||
|
"loosest match": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = maxAcousticMaxBitErrorRate }),
|
||||||
|
"fewest at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = minBackfillConcurrency }),
|
||||||
|
"most at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = maxBackfillConcurrency }),
|
||||||
|
"shortest interval": with(func(s *FingerprintSettings) { s.SweepIntervalHours = minSweepIntervalHours }),
|
||||||
|
"longest interval": with(func(s *FingerprintSettings) { s.SweepIntervalHours = maxSweepIntervalHours }),
|
||||||
|
"switched off": with(func(s *FingerprintSettings) { s.Enabled = false }),
|
||||||
|
}
|
||||||
|
for name, s := range valid {
|
||||||
|
if err := validateFingerprintSettings(s); err != nil {
|
||||||
|
t.Errorf("%s: rejected: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
invalid := map[string]FingerprintSettings{
|
||||||
|
"length too short": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = minChromaprintLengthSec - 1 }),
|
||||||
|
"length too long": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = maxChromaprintLengthSec + 1 }),
|
||||||
|
"match too strict": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = minAcousticMaxBitErrorRate - 0.001 }),
|
||||||
|
"match too loose": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = maxAcousticMaxBitErrorRate + 0.001 }),
|
||||||
|
"match not a number": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = math.NaN() }),
|
||||||
|
"none at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = 0 }),
|
||||||
|
"too many at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = maxBackfillConcurrency + 1 }),
|
||||||
|
"no interval": with(func(s *FingerprintSettings) { s.SweepIntervalHours = 0 }),
|
||||||
|
"interval too long": with(func(s *FingerprintSettings) { s.SweepIntervalHours = maxSweepIntervalHours + 1 }),
|
||||||
|
}
|
||||||
|
for name, s := range invalid {
|
||||||
|
if err := validateFingerprintSettings(s); !errors.Is(err, ErrFingerprintSettingOutOfRange) {
|
||||||
|
t.Errorf("%s: err = %v, want ErrFingerprintSettingOutOfRange", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFingerprintSettingsService_NilServesDefaults(t *testing.T) {
|
||||||
|
var s *FingerprintSettingsService
|
||||||
|
if got := s.Get(); got != DefaultFingerprintSettings {
|
||||||
|
t.Fatalf("nil service Get = %+v, want the defaults", got)
|
||||||
|
}
|
||||||
|
// Validation still runs first, so a bad value is named rather than hidden
|
||||||
|
// behind the missing service.
|
||||||
|
bad := DefaultFingerprintSettings
|
||||||
|
bad.BackfillConcurrency = 0
|
||||||
|
if _, err := s.Set(context.Background(), bad); !errors.Is(err, ErrFingerprintSettingOutOfRange) {
|
||||||
|
t.Fatalf("nil service Set of a bad value: err = %v, want ErrFingerprintSettingOutOfRange", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFingerprintSettingsService_Integration(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
reload := func(step string) FingerprintSettings {
|
||||||
|
t.Helper()
|
||||||
|
fresh, err := NewFingerprintSettingsService(ctx, pool)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: load: %v", step, err)
|
||||||
|
}
|
||||||
|
return fresh.Get()
|
||||||
|
}
|
||||||
|
withoutTime := func(s FingerprintSettings) FingerprintSettings {
|
||||||
|
s.UpdatedAt = time.Time{}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
svc, err := NewFingerprintSettingsService(ctx, pool)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load: %v", err)
|
||||||
|
}
|
||||||
|
// ResetDB puts every column back to its migration default, so this pins the
|
||||||
|
// Go defaults to migration 0061's. Were they to drift, a database that could
|
||||||
|
// not be read would fingerprint differently from one that could.
|
||||||
|
loaded := svc.Get()
|
||||||
|
if loaded.UpdatedAt.IsZero() {
|
||||||
|
t.Fatal("loaded settings carry no updated_at")
|
||||||
|
}
|
||||||
|
if withoutTime(loaded) != DefaultFingerprintSettings {
|
||||||
|
t.Fatalf("stored defaults = %+v, want the Go defaults %+v", withoutTime(loaded), DefaultFingerprintSettings)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every bound the service accepts, the table accepts too. A CHECK tighter
|
||||||
|
// than validate would turn a value the card allows into a 500.
|
||||||
|
lowest := FingerprintSettings{
|
||||||
|
Enabled: false,
|
||||||
|
ChromaprintLengthSec: minChromaprintLengthSec,
|
||||||
|
AcousticMaxBitErrorRate: minAcousticMaxBitErrorRate,
|
||||||
|
BackfillConcurrency: minBackfillConcurrency,
|
||||||
|
SweepIntervalHours: minSweepIntervalHours,
|
||||||
|
}
|
||||||
|
highest := FingerprintSettings{
|
||||||
|
Enabled: true,
|
||||||
|
ChromaprintLengthSec: maxChromaprintLengthSec,
|
||||||
|
AcousticMaxBitErrorRate: maxAcousticMaxBitErrorRate,
|
||||||
|
BackfillConcurrency: maxBackfillConcurrency,
|
||||||
|
SweepIntervalHours: maxSweepIntervalHours,
|
||||||
|
}
|
||||||
|
for _, step := range []struct {
|
||||||
|
name string
|
||||||
|
want FingerprintSettings
|
||||||
|
}{{"lowest", lowest}, {"highest", highest}} {
|
||||||
|
saved, err := svc.Set(ctx, step.want)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: save: %v", step.name, err)
|
||||||
|
}
|
||||||
|
// A save must move updated_at forward: it is what makes a sweep due.
|
||||||
|
if !saved.UpdatedAt.After(loaded.UpdatedAt) {
|
||||||
|
t.Fatalf("%s: updated_at %v did not move past %v", step.name, saved.UpdatedAt, loaded.UpdatedAt)
|
||||||
|
}
|
||||||
|
if withoutTime(saved) != step.want || withoutTime(svc.Get()) != step.want {
|
||||||
|
t.Fatalf("%s: saved %+v, cached %+v, want %+v", step.name, saved, svc.Get(), step.want)
|
||||||
|
}
|
||||||
|
if got := withoutTime(reload(step.name)); got != step.want {
|
||||||
|
t.Fatalf("%s: table holds %+v, want %+v", step.name, got, step.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An out-of-range save changes nothing, in the cache or the table.
|
||||||
|
before := svc.Get()
|
||||||
|
bad := highest
|
||||||
|
bad.ChromaprintLengthSec = maxChromaprintLengthSec + 1
|
||||||
|
if _, err := svc.Set(ctx, bad); !errors.Is(err, ErrFingerprintSettingOutOfRange) {
|
||||||
|
t.Fatalf("out-of-range save: err = %v, want ErrFingerprintSettingOutOfRange", err)
|
||||||
|
}
|
||||||
|
if svc.Get() != before {
|
||||||
|
t.Fatalf("out-of-range save changed the cache to %+v", svc.Get())
|
||||||
|
}
|
||||||
|
if got := reload("after out-of-range save"); got != before {
|
||||||
|
t.Fatalf("out-of-range save changed the table to %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
package library
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseFpcalcRaw(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
out string
|
||||||
|
want []int32
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "signed output with negatives",
|
||||||
|
out: "DURATION=213\nFINGERPRINT=-1453821711,17,0,2147483647,-2147483648\n",
|
||||||
|
want: []int32{-1453821711, 17, 0, 2147483647, -2147483648},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fingerprint line need not come second",
|
||||||
|
out: "FINGERPRINT=5,6\nDURATION=1\n",
|
||||||
|
want: []int32{5, 6},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// fpcalc's default is uint32. This value only appears when -signed
|
||||||
|
// is missing, and storing it would need a reinterpretation nothing
|
||||||
|
// performs.
|
||||||
|
name: "unsigned output is refused",
|
||||||
|
out: "DURATION=213\nFINGERPRINT=2841145585,17\n",
|
||||||
|
wantErr: "item 0",
|
||||||
|
},
|
||||||
|
{name: "empty fingerprint", out: "DURATION=0\nFINGERPRINT=\n", wantErr: "empty fingerprint"},
|
||||||
|
{name: "no fingerprint line", out: "DURATION=213\n", wantErr: "no FINGERPRINT= line"},
|
||||||
|
{name: "non-numeric item", out: "FINGERPRINT=1,x,3\n", wantErr: "item 1"},
|
||||||
|
{name: "trailing comma", out: "FINGERPRINT=1,2,\n", wantErr: "item 2"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := parseFpcalcRaw([]byte(tc.out))
|
||||||
|
if tc.wantErr != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
|
||||||
|
t.Fatalf("err = %v, want one containing %q", err, tc.wantErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected err: %v", err)
|
||||||
|
}
|
||||||
|
if !slices.Equal(got, tc.want) {
|
||||||
|
t.Fatalf("got %v, want %v", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseStreamHash(t *testing.T) {
|
||||||
|
// The real value ffmpeg printed for both files of the #3885 pair.
|
||||||
|
const www = "24e2daa3b4a534ff1a8d1a76f67810205869daf89f728d83a16625da4d28a18e"
|
||||||
|
|
||||||
|
got, err := parseStreamHash([]byte("SHA256=" + www + "\n"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected err: %v", err)
|
||||||
|
}
|
||||||
|
if len(got) != 32 || got[0] != 0x24 || got[31] != 0x8e {
|
||||||
|
t.Fatalf("decoded %x, want %s", got, www)
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, out := range map[string]string{
|
||||||
|
"no hash line": "",
|
||||||
|
"other hash": "MD5=" + www[:32] + "\n",
|
||||||
|
"not hex": "SHA256=" + strings.Repeat("zz", 32) + "\n",
|
||||||
|
"short digest": "SHA256=" + www[:62] + "\n",
|
||||||
|
"odd hex chars": "SHA256=" + www[:63] + "\n",
|
||||||
|
} {
|
||||||
|
if _, err := parseStreamHash([]byte(out)); err == nil {
|
||||||
|
t.Errorf("%s: parsed %q without error", name, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// followedBy reports whether flag appears in args immediately followed by value.
|
||||||
|
func followedBy(args []string, flag, value string) bool {
|
||||||
|
for i := 0; i+1 < len(args); i++ {
|
||||||
|
if args[i] == flag && args[i+1] == value {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// The exact tier's stored hashes must stay comparable across ffmpeg upgrades,
|
||||||
|
// which only holds while the packets are copied rather than decoded. A decoded
|
||||||
|
// hash still matches within one ffmpeg build, so nothing else would notice the
|
||||||
|
// change until an image upgrade silently broke every stored value.
|
||||||
|
func TestStreamHashArgs_HashPacketsNotSamples(t *testing.T) {
|
||||||
|
args := streamHashArgs("/music/a.mp3")
|
||||||
|
for _, pair := range [][2]string{
|
||||||
|
{"-c:a", "copy"}, // no decode
|
||||||
|
{"-map", "0:a"}, // audio only: cover art stays out of the hash
|
||||||
|
{"-f", "hash"}, // the hash muxer, not a file
|
||||||
|
{"-hash", "sha256"},
|
||||||
|
{"-i", "/music/a.mp3"},
|
||||||
|
} {
|
||||||
|
if !followedBy(args, pair[0], pair[1]) {
|
||||||
|
t.Errorf("streamHashArgs lacks %s %s: %v", pair[0], pair[1], args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFpcalcArgs_RequestSignedRawOutput(t *testing.T) {
|
||||||
|
args := fpcalcArgs("/music/a.flac", 90)
|
||||||
|
for _, flag := range []string{"-raw", "-signed"} {
|
||||||
|
if !slices.Contains(args, flag) {
|
||||||
|
t.Errorf("fpcalcArgs lacks %s: %v", flag, args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !followedBy(args, "-length", "90") {
|
||||||
|
t.Errorf("fpcalcArgs does not pass the requested length: %v", args)
|
||||||
|
}
|
||||||
|
// fpcalc takes the file as its trailing positional argument.
|
||||||
|
if args[len(args)-1] != "/music/a.flac" {
|
||||||
|
t.Errorf("path is not last: %v", args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStderrTail_KeepsTheCauseNotTheBanner(t *testing.T) {
|
||||||
|
banner := bytes.Repeat([]byte("warning: skipping frame\n"), fpcalcStderrTail)
|
||||||
|
got := stderrTail(append(banner, []byte("ERROR: could not decode\n")...))
|
||||||
|
if len(got) != fpcalcStderrTail {
|
||||||
|
t.Fatalf("tail is %d bytes, want the %d-byte cap", len(got), fpcalcStderrTail)
|
||||||
|
}
|
||||||
|
if !bytes.HasSuffix(got, []byte("ERROR: could not decode")) {
|
||||||
|
t.Fatalf("tail dropped the final line: ...%q", got[len(got)-40:])
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := stderrTail([]byte(" short \n")); string(got) != "short" {
|
||||||
|
t.Fatalf("short stderr = %q, want it trimmed and whole", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A stored failure is permanent until the file changes, so the classification
|
||||||
|
// decides whether a track is ever retried. Every inconclusive case here would,
|
||||||
|
// if misfiled as a verdict, silently exclude that track from duplicate
|
||||||
|
// detection for good.
|
||||||
|
func TestIsInconclusive(t *testing.T) {
|
||||||
|
notInstalled := fmt.Errorf("fpcalc: %w", &exec.Error{Name: "fpcalc", Err: exec.ErrNotFound})
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"timeout", fmt.Errorf("fpcalc: no result: %w", errFingerprintTimeout), true},
|
||||||
|
{"scan cancelled", fmt.Errorf("ffmpeg: %w", context.Canceled), true},
|
||||||
|
{"caller deadline", fmt.Errorf("ffmpeg: %w", context.DeadlineExceeded), true},
|
||||||
|
{"tool not installed", notInstalled, true},
|
||||||
|
{"tool rejected the file", errors.New("fpcalc exited 2: could not decode"), false},
|
||||||
|
{"unparseable output", errors.New("fpcalc printed no FINGERPRINT= line"), false},
|
||||||
|
{"success", nil, false},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := isInconclusive(tc.err); got != tc.want {
|
||||||
|
t.Errorf("%s: isInconclusive = %v, want %v", tc.name, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Either half being inconclusive taints the whole result: storing the half that
|
||||||
|
// succeeded would stamp the row at the current version with the other half
|
||||||
|
// NULL, and that NULL would then read as a verdict.
|
||||||
|
func TestFingerprintResult_InconclusiveIfEitherHalfIs(t *testing.T) {
|
||||||
|
stall := fmt.Errorf("fpcalc: %w", errFingerprintTimeout)
|
||||||
|
rejected := errors.New("fpcalc exited 2")
|
||||||
|
for name, tc := range map[string]struct {
|
||||||
|
r fingerprintResult
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
"both succeeded": {fingerprintResult{streamSHA256: []byte{1}, chromaprint: []int32{1}}, false},
|
||||||
|
"hash ok, print stalled": {fingerprintResult{streamSHA256: []byte{1}, printErr: stall}, true},
|
||||||
|
"hash stalled, print ok": {fingerprintResult{hashErr: stall, chromaprint: []int32{1}}, true},
|
||||||
|
"hash ok, print rejected": {fingerprintResult{streamSHA256: []byte{1}, printErr: rejected}, false},
|
||||||
|
"both rejected by the file": {fingerprintResult{hashErr: rejected, printErr: rejected}, false},
|
||||||
|
} {
|
||||||
|
if got := tc.r.inconclusive(); got != tc.want {
|
||||||
|
t.Errorf("%s: inconclusive = %v, want %v", name, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+13
-15
@@ -41,7 +41,7 @@ import (
|
|||||||
// match/ambiguity logic can be tested against a fake.
|
// match/ambiguity logic can be tested against a fake.
|
||||||
type trackAdopter interface {
|
type trackAdopter interface {
|
||||||
FindMissingTrackByMbid(ctx context.Context, mbid string) ([]dbq.FindMissingTrackByMbidRow, error)
|
FindMissingTrackByMbid(ctx context.Context, mbid string) ([]dbq.FindMissingTrackByMbidRow, error)
|
||||||
FindMissingTrackByFingerprint(ctx context.Context, arg dbq.FindMissingTrackByFingerprintParams) ([]dbq.FindMissingTrackByFingerprintRow, error)
|
FindMissingTrackByAudioHash(ctx context.Context, audioStreamSha256 []byte) ([]dbq.FindMissingTrackByAudioHashRow, error)
|
||||||
AdoptTrackPath(ctx context.Context, arg dbq.AdoptTrackPathParams) (int64, error)
|
AdoptTrackPath(ctx context.Context, arg dbq.AdoptTrackPathParams) (int64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,10 +53,10 @@ type trackAdopter interface {
|
|||||||
// pre-#2528 behaviour.
|
// pre-#2528 behaviour.
|
||||||
func (s *Scanner) adoptMovedTrack(
|
func (s *Scanner) adoptMovedTrack(
|
||||||
ctx context.Context, q trackAdopter, newPath string,
|
ctx context.Context, q trackAdopter, newPath string,
|
||||||
fileSize int64, durationMs int32, recordingMBID string,
|
audioHash []byte, recordingMBID string,
|
||||||
) bool {
|
) bool {
|
||||||
// MBID first. It identifies the recording rather than the bytes, so it
|
// MBID first. It identifies the recording rather than the bytes, so it
|
||||||
// survives a re-encode that the fingerprint cannot.
|
// survives a re-encode that the audio hash cannot.
|
||||||
if recordingMBID != "" {
|
if recordingMBID != "" {
|
||||||
rows, err := q.FindMissingTrackByMbid(ctx, recordingMBID)
|
rows, err := q.FindMissingTrackByMbid(ctx, recordingMBID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -67,19 +67,17 @@ func (s *Scanner) adoptMovedTrack(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fingerprint fallback for untagged files. Both components must be real:
|
// Audio-hash fallback for untagged files (#3914): the encoded audio itself,
|
||||||
// duration_ms is 0 when ffprobe failed, and matching 0 against 0 would pair
|
// which a rename, a move or a retag leaves unchanged. Absent when the hash
|
||||||
// up unrelated broken files.
|
// could not be taken, and then nothing is matched — an empty hash must never
|
||||||
if fileSize > 0 && durationMs > 0 {
|
// be looked up, or every unhashable file would pair with every other.
|
||||||
rows, err := q.FindMissingTrackByFingerprint(ctx, dbq.FindMissingTrackByFingerprintParams{
|
if len(audioHash) > 0 {
|
||||||
FileSize: fileSize,
|
rows, err := q.FindMissingTrackByAudioHash(ctx, audioHash)
|
||||||
DurationMs: durationMs,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Warn("library scan: move lookup by fingerprint failed",
|
s.logger.Warn("library scan: move lookup by audio hash failed",
|
||||||
"path", newPath, "err", err)
|
"path", newPath, "err", err)
|
||||||
} else if c, ok := s.uniqueMatch(rowsFromFingerprint(rows), newPath, "fingerprint"); ok {
|
} else if c, ok := s.uniqueMatch(rowsFromAudioHash(rows), newPath, "audio_hash"); ok {
|
||||||
return s.adopt(ctx, q, c, newPath, "fingerprint")
|
return s.adopt(ctx, q, c, newPath, "audio_hash")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,7 +98,7 @@ func rowsFromMbid(rows []dbq.FindMissingTrackByMbidRow) []candidate {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func rowsFromFingerprint(rows []dbq.FindMissingTrackByFingerprintRow) []candidate {
|
func rowsFromAudioHash(rows []dbq.FindMissingTrackByAudioHashRow) []candidate {
|
||||||
out := make([]candidate, 0, len(rows))
|
out := make([]candidate, 0, len(rows))
|
||||||
for _, r := range rows {
|
for _, r := range rows {
|
||||||
out = append(out, candidate{id: r.ID, filePath: r.FilePath})
|
out = append(out, candidate{id: r.ID, filePath: r.FilePath})
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package library
|
package library
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -10,18 +11,21 @@ import (
|
|||||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// fakeAdopter answers the audio-hash lookup only for the hash it holds, the way
|
||||||
|
// the real query does. A fake that returned its rows for any hash could not tell
|
||||||
|
// adoption by identity apart from adoption by coincidence.
|
||||||
type fakeAdopter struct {
|
type fakeAdopter struct {
|
||||||
byMbid []dbq.FindMissingTrackByMbidRow
|
byMbid []dbq.FindMissingTrackByMbidRow
|
||||||
byFingerprint []dbq.FindMissingTrackByFingerprintRow
|
hash []byte
|
||||||
|
byHash []dbq.FindMissingTrackByAudioHashRow
|
||||||
|
mbidErr error
|
||||||
|
hashErr error
|
||||||
|
adoptErr error
|
||||||
|
adoptRows int64
|
||||||
|
|
||||||
mbidErr error
|
mbidQueried []string
|
||||||
fingerprintErr error
|
hashQueried [][]byte
|
||||||
adoptErr error
|
adopted []dbq.AdoptTrackPathParams
|
||||||
adoptRows int64
|
|
||||||
|
|
||||||
mbidQueried []string
|
|
||||||
fingerprintQueried []dbq.FindMissingTrackByFingerprintParams
|
|
||||||
adopted []dbq.AdoptTrackPathParams
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeAdopter) FindMissingTrackByMbid(_ context.Context, mbid string) ([]dbq.FindMissingTrackByMbidRow, error) {
|
func (f *fakeAdopter) FindMissingTrackByMbid(_ context.Context, mbid string) ([]dbq.FindMissingTrackByMbidRow, error) {
|
||||||
@@ -29,11 +33,17 @@ func (f *fakeAdopter) FindMissingTrackByMbid(_ context.Context, mbid string) ([]
|
|||||||
return f.byMbid, f.mbidErr
|
return f.byMbid, f.mbidErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeAdopter) FindMissingTrackByFingerprint(
|
func (f *fakeAdopter) FindMissingTrackByAudioHash(
|
||||||
_ context.Context, arg dbq.FindMissingTrackByFingerprintParams,
|
_ context.Context, audioStreamSha256 []byte,
|
||||||
) ([]dbq.FindMissingTrackByFingerprintRow, error) {
|
) ([]dbq.FindMissingTrackByAudioHashRow, error) {
|
||||||
f.fingerprintQueried = append(f.fingerprintQueried, arg)
|
f.hashQueried = append(f.hashQueried, audioStreamSha256)
|
||||||
return f.byFingerprint, f.fingerprintErr
|
if f.hashErr != nil {
|
||||||
|
return nil, f.hashErr
|
||||||
|
}
|
||||||
|
if !bytes.Equal(audioStreamSha256, f.hash) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return f.byHash, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeAdopter) AdoptTrackPath(_ context.Context, arg dbq.AdoptTrackPathParams) (int64, error) {
|
func (f *fakeAdopter) AdoptTrackPath(_ context.Context, arg dbq.AdoptTrackPathParams) (int64, error) {
|
||||||
@@ -51,10 +61,12 @@ func mbidRow(n byte, path string) dbq.FindMissingTrackByMbidRow {
|
|||||||
return dbq.FindMissingTrackByMbidRow{ID: testUUID(n), FilePath: path}
|
return dbq.FindMissingTrackByMbidRow{ID: testUUID(n), FilePath: path}
|
||||||
}
|
}
|
||||||
|
|
||||||
func fpRow(n byte, path string) dbq.FindMissingTrackByFingerprintRow {
|
func hashRow(n byte, path string) dbq.FindMissingTrackByAudioHashRow {
|
||||||
return dbq.FindMissingTrackByFingerprintRow{ID: testUUID(n), FilePath: path}
|
return dbq.FindMissingTrackByAudioHashRow{ID: testUUID(n), FilePath: path}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func audioHash(b byte) []byte { return bytes.Repeat([]byte{b}, 32) }
|
||||||
|
|
||||||
const (
|
const (
|
||||||
oldPath = "/music/Linkin Park/Minutes to Midnight/02 - Bleed It Out.mp3"
|
oldPath = "/music/Linkin Park/Minutes to Midnight/02 - Bleed It Out.mp3"
|
||||||
newPath = "/music/Linkin Park/Minutes to Midnight/04 - Bleed It Out.mp3"
|
newPath = "/music/Linkin Park/Minutes to Midnight/04 - Bleed It Out.mp3"
|
||||||
@@ -64,47 +76,53 @@ func TestAdoptMovedTrack_MatchesByMbid(t *testing.T) {
|
|||||||
s := testScanner(t)
|
s := testScanner(t)
|
||||||
q := &fakeAdopter{byMbid: []dbq.FindMissingTrackByMbidRow{mbidRow(7, oldPath)}, adoptRows: 1}
|
q := &fakeAdopter{byMbid: []dbq.FindMissingTrackByMbidRow{mbidRow(7, oldPath)}, adoptRows: 1}
|
||||||
|
|
||||||
if !s.adoptMovedTrack(context.Background(), q, newPath, 5_000_000, 200_000, "rec-mbid") {
|
if !s.adoptMovedTrack(context.Background(), q, newPath, audioHash(1), "rec-mbid") {
|
||||||
t.Fatal("expected the moved track to be adopted")
|
t.Fatal("expected the moved track to be adopted")
|
||||||
}
|
}
|
||||||
if len(q.adopted) != 1 {
|
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(7) || q.adopted[0].FilePath != newPath {
|
||||||
t.Fatalf("adopted %d rows, want 1", len(q.adopted))
|
t.Fatalf("adopted = %+v, want row 7 at %q", q.adopted, newPath)
|
||||||
}
|
|
||||||
if q.adopted[0].ID != testUUID(7) {
|
|
||||||
t.Errorf("adopted the wrong row: %v", q.adopted[0].ID)
|
|
||||||
}
|
|
||||||
if q.adopted[0].FilePath != newPath {
|
|
||||||
t.Errorf("adopted FilePath = %q, want %q", q.adopted[0].FilePath, newPath)
|
|
||||||
}
|
}
|
||||||
// MBID matched, so the weaker signal should not have been consulted.
|
// MBID matched, so the weaker signal should not have been consulted.
|
||||||
if len(q.fingerprintQueried) != 0 {
|
if len(q.hashQueried) != 0 {
|
||||||
t.Errorf("queried the fingerprint despite an MBID match")
|
t.Errorf("queried the audio hash despite an MBID match")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAdoptMovedTrack_FallsBackToFingerprint(t *testing.T) {
|
func TestAdoptMovedTrack_FallsBackToAudioHash(t *testing.T) {
|
||||||
s := testScanner(t)
|
s := testScanner(t)
|
||||||
q := &fakeAdopter{byFingerprint: []dbq.FindMissingTrackByFingerprintRow{fpRow(3, oldPath)}, adoptRows: 1}
|
q := &fakeAdopter{hash: audioHash(3), byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(3, oldPath)}, adoptRows: 1}
|
||||||
|
|
||||||
// No MBID: an untagged file, which is exactly what the fallback is for.
|
// No MBID: an untagged file, which is exactly what the fallback is for.
|
||||||
if !s.adoptMovedTrack(context.Background(), q, newPath, 4_200_000, 187_000, "") {
|
if !s.adoptMovedTrack(context.Background(), q, newPath, audioHash(3), "") {
|
||||||
t.Fatal("expected adoption via fingerprint")
|
t.Fatal("expected adoption via the audio hash")
|
||||||
}
|
}
|
||||||
if len(q.mbidQueried) != 0 {
|
if len(q.mbidQueried) != 0 {
|
||||||
t.Errorf("queried by MBID with no MBID available")
|
t.Errorf("queried by MBID with no MBID available")
|
||||||
}
|
}
|
||||||
if len(q.fingerprintQueried) != 1 {
|
if len(q.hashQueried) != 1 || !bytes.Equal(q.hashQueried[0], audioHash(3)) {
|
||||||
t.Fatalf("fingerprint queried %d times, want 1", len(q.fingerprintQueried))
|
t.Fatalf("hash queried = %x, want exactly the file's hash", q.hashQueried)
|
||||||
}
|
|
||||||
got := q.fingerprintQueried[0]
|
|
||||||
if got.FileSize != 4_200_000 || got.DurationMs != 187_000 {
|
|
||||||
t.Errorf("fingerprint = %+v, want size 4200000 duration 187000", got)
|
|
||||||
}
|
}
|
||||||
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(3) {
|
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(3) {
|
||||||
t.Errorf("adopted = %+v, want row 3", q.adopted)
|
t.Errorf("adopted = %+v, want row 3", q.adopted)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The case the old (file_size, duration_ms) pair got wrong: an unrelated file
|
||||||
|
// that happens to share a size and a duration with a missing track. Size and
|
||||||
|
// duration are no longer inputs at all; only the audio itself can match, and a
|
||||||
|
// different recording has a different hash.
|
||||||
|
func TestAdoptMovedTrack_DifferentAudioIsNotAdopted(t *testing.T) {
|
||||||
|
s := testScanner(t)
|
||||||
|
q := &fakeAdopter{hash: audioHash(0xAA), byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(1, oldPath)}, adoptRows: 1}
|
||||||
|
|
||||||
|
if s.adoptMovedTrack(context.Background(), q, newPath, audioHash(0xBB), "") {
|
||||||
|
t.Fatal("adopted a missing track whose audio differs")
|
||||||
|
}
|
||||||
|
if len(q.adopted) != 0 {
|
||||||
|
t.Errorf("adopted = %+v, want none", q.adopted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Two missing rows carrying the same recording MBID means real duplicates.
|
// Two missing rows carrying the same recording MBID means real duplicates.
|
||||||
// Adopting one arbitrarily would attach this file's future history to a coin
|
// Adopting one arbitrarily would attach this file's future history to a coin
|
||||||
// flip, so it must insert fresh instead.
|
// flip, so it must insert fresh instead.
|
||||||
@@ -115,7 +133,7 @@ func TestAdoptMovedTrack_RefusesAmbiguousMbidMatch(t *testing.T) {
|
|||||||
mbidRow(2, "/music/b.mp3"),
|
mbidRow(2, "/music/b.mp3"),
|
||||||
}, adoptRows: 1}
|
}, adoptRows: 1}
|
||||||
|
|
||||||
if s.adoptMovedTrack(context.Background(), q, newPath, 0, 0, "rec-mbid") {
|
if s.adoptMovedTrack(context.Background(), q, newPath, nil, "rec-mbid") {
|
||||||
t.Fatal("expected refusal on an ambiguous MBID match")
|
t.Fatal("expected refusal on an ambiguous MBID match")
|
||||||
}
|
}
|
||||||
if len(q.adopted) != 0 {
|
if len(q.adopted) != 0 {
|
||||||
@@ -123,66 +141,57 @@ func TestAdoptMovedTrack_RefusesAmbiguousMbidMatch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// An ambiguous MBID may still be resolvable by the fingerprint, which is a
|
// An ambiguous MBID may still be resolvable by the audio hash, which is a
|
||||||
// narrower signal — so falling through is allowed to succeed.
|
// narrower signal — so falling through is allowed to succeed.
|
||||||
func TestAdoptMovedTrack_AmbiguousMbidFallsThroughToFingerprint(t *testing.T) {
|
func TestAdoptMovedTrack_AmbiguousMbidFallsThroughToAudioHash(t *testing.T) {
|
||||||
s := testScanner(t)
|
s := testScanner(t)
|
||||||
q := &fakeAdopter{
|
q := &fakeAdopter{
|
||||||
byMbid: []dbq.FindMissingTrackByMbidRow{
|
byMbid: []dbq.FindMissingTrackByMbidRow{
|
||||||
mbidRow(1, "/music/a.mp3"),
|
mbidRow(1, "/music/a.mp3"),
|
||||||
mbidRow(2, "/music/b.mp3"),
|
mbidRow(2, "/music/b.mp3"),
|
||||||
},
|
},
|
||||||
byFingerprint: []dbq.FindMissingTrackByFingerprintRow{fpRow(2, "/music/b.mp3")},
|
hash: audioHash(2),
|
||||||
adoptRows: 1,
|
byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(2, "/music/b.mp3")},
|
||||||
|
adoptRows: 1,
|
||||||
}
|
}
|
||||||
|
|
||||||
if !s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "rec-mbid") {
|
if !s.adoptMovedTrack(context.Background(), q, newPath, audioHash(2), "rec-mbid") {
|
||||||
t.Fatal("expected the fingerprint to disambiguate")
|
t.Fatal("expected the audio hash to disambiguate")
|
||||||
}
|
}
|
||||||
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(2) {
|
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(2) {
|
||||||
t.Errorf("adopted = %+v, want row 2", q.adopted)
|
t.Errorf("adopted = %+v, want row 2", q.adopted)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAdoptMovedTrack_RefusesAmbiguousFingerprintMatch(t *testing.T) {
|
// Two missing tracks with identical audio are duplicates of each other; a new
|
||||||
|
// file matching both cannot be assigned to either.
|
||||||
|
func TestAdoptMovedTrack_RefusesAmbiguousAudioHashMatch(t *testing.T) {
|
||||||
s := testScanner(t)
|
s := testScanner(t)
|
||||||
q := &fakeAdopter{byFingerprint: []dbq.FindMissingTrackByFingerprintRow{
|
q := &fakeAdopter{hash: audioHash(5), byHash: []dbq.FindMissingTrackByAudioHashRow{
|
||||||
fpRow(1, "/music/a.mp3"),
|
hashRow(1, "/music/a.mp3"),
|
||||||
fpRow(2, "/music/b.mp3"),
|
hashRow(2, "/music/b.mp3"),
|
||||||
}, adoptRows: 1}
|
}, adoptRows: 1}
|
||||||
|
|
||||||
if s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "") {
|
if s.adoptMovedTrack(context.Background(), q, newPath, audioHash(5), "") {
|
||||||
t.Fatal("expected refusal on an ambiguous fingerprint match")
|
t.Fatal("expected refusal on an ambiguous audio-hash match")
|
||||||
}
|
}
|
||||||
if len(q.adopted) != 0 {
|
if len(q.adopted) != 0 {
|
||||||
t.Errorf("adopted despite ambiguity: %+v", q.adopted)
|
t.Errorf("adopted despite ambiguity: %+v", q.adopted)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// duration_ms is 0 when ffprobe failed. Matching 0 against 0 would pair up
|
// No hash means it could not be taken. An empty hash must never be looked up:
|
||||||
// unrelated broken files, so the fingerprint must not be attempted.
|
// every unhashable file would pair with every other.
|
||||||
func TestAdoptMovedTrack_SkipsFingerprintWithoutRealValues(t *testing.T) {
|
func TestAdoptMovedTrack_SkipsAudioHashWhenAbsent(t *testing.T) {
|
||||||
tests := []struct {
|
for name, hash := range map[string][]byte{"nil": nil, "empty": {}} {
|
||||||
name string
|
t.Run(name, func(t *testing.T) {
|
||||||
size int64
|
|
||||||
duration int32
|
|
||||||
}{
|
|
||||||
{"no duration", 1000, 0},
|
|
||||||
{"no size", 0, 2000},
|
|
||||||
{"neither", 0, 0},
|
|
||||||
}
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
s := testScanner(t)
|
s := testScanner(t)
|
||||||
q := &fakeAdopter{
|
q := &fakeAdopter{hash: hash, byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(1, oldPath)}, adoptRows: 1}
|
||||||
byFingerprint: []dbq.FindMissingTrackByFingerprintRow{fpRow(1, oldPath)},
|
if s.adoptMovedTrack(context.Background(), q, newPath, hash, "") {
|
||||||
adoptRows: 1,
|
t.Error("adopted without an audio hash")
|
||||||
}
|
}
|
||||||
if s.adoptMovedTrack(context.Background(), q, newPath, tc.size, tc.duration, "") {
|
if len(q.hashQueried) != 0 {
|
||||||
t.Error("adopted on an unusable fingerprint")
|
t.Error("looked up an absent audio hash")
|
||||||
}
|
|
||||||
if len(q.fingerprintQueried) != 0 {
|
|
||||||
t.Error("queried the fingerprint with unusable values")
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -192,7 +201,7 @@ func TestAdoptMovedTrack_NoCandidates(t *testing.T) {
|
|||||||
s := testScanner(t)
|
s := testScanner(t)
|
||||||
q := &fakeAdopter{adoptRows: 1}
|
q := &fakeAdopter{adoptRows: 1}
|
||||||
|
|
||||||
if s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "rec-mbid") {
|
if s.adoptMovedTrack(context.Background(), q, newPath, audioHash(9), "rec-mbid") {
|
||||||
t.Fatal("expected no adoption when nothing matches")
|
t.Fatal("expected no adoption when nothing matches")
|
||||||
}
|
}
|
||||||
if len(q.adopted) != 0 {
|
if len(q.adopted) != 0 {
|
||||||
@@ -209,7 +218,7 @@ func TestAdoptMovedTrack_LostRaceReportsNotAdopted(t *testing.T) {
|
|||||||
adoptRows: 0,
|
adoptRows: 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "rec-mbid") {
|
if s.adoptMovedTrack(context.Background(), q, newPath, audioHash(5), "rec-mbid") {
|
||||||
t.Fatal("expected not-adopted when the update matched no rows")
|
t.Fatal("expected not-adopted when the update matched no rows")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -223,7 +232,7 @@ func TestAdoptMovedTrack_ToleratesQueryErrors(t *testing.T) {
|
|||||||
q *fakeAdopter
|
q *fakeAdopter
|
||||||
}{
|
}{
|
||||||
{"mbid lookup fails", &fakeAdopter{mbidErr: sentinel}},
|
{"mbid lookup fails", &fakeAdopter{mbidErr: sentinel}},
|
||||||
{"fingerprint lookup fails", &fakeAdopter{fingerprintErr: sentinel}},
|
{"audio hash lookup fails", &fakeAdopter{hashErr: sentinel}},
|
||||||
{"adopt fails", &fakeAdopter{
|
{"adopt fails", &fakeAdopter{
|
||||||
byMbid: []dbq.FindMissingTrackByMbidRow{mbidRow(1, oldPath)},
|
byMbid: []dbq.FindMissingTrackByMbidRow{mbidRow(1, oldPath)},
|
||||||
adoptErr: sentinel,
|
adoptErr: sentinel,
|
||||||
@@ -232,24 +241,25 @@ func TestAdoptMovedTrack_ToleratesQueryErrors(t *testing.T) {
|
|||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
s := testScanner(t)
|
s := testScanner(t)
|
||||||
if s.adoptMovedTrack(context.Background(), tc.q, newPath, 1000, 2000, "rec-mbid") {
|
if s.adoptMovedTrack(context.Background(), tc.q, newPath, audioHash(1), "rec-mbid") {
|
||||||
t.Error("reported adoption despite a query error")
|
t.Error("reported adoption despite a query error")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A failed MBID lookup must not stop the fingerprint from being tried.
|
// A failed MBID lookup must not stop the audio hash from being tried.
|
||||||
func TestAdoptMovedTrack_MbidErrorStillTriesFingerprint(t *testing.T) {
|
func TestAdoptMovedTrack_MbidErrorStillTriesAudioHash(t *testing.T) {
|
||||||
s := testScanner(t)
|
s := testScanner(t)
|
||||||
q := &fakeAdopter{
|
q := &fakeAdopter{
|
||||||
mbidErr: errors.New("db hiccup"),
|
mbidErr: errors.New("db hiccup"),
|
||||||
byFingerprint: []dbq.FindMissingTrackByFingerprintRow{fpRow(9, oldPath)},
|
hash: audioHash(9),
|
||||||
adoptRows: 1,
|
byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(9, oldPath)},
|
||||||
|
adoptRows: 1,
|
||||||
}
|
}
|
||||||
|
|
||||||
if !s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "rec-mbid") {
|
if !s.adoptMovedTrack(context.Background(), q, newPath, audioHash(9), "rec-mbid") {
|
||||||
t.Fatal("expected the fingerprint to be tried after an MBID lookup error")
|
t.Fatal("expected the audio hash to be tried after an MBID lookup error")
|
||||||
}
|
}
|
||||||
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(9) {
|
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(9) {
|
||||||
t.Errorf("adopted = %+v, want row 9", q.adopted)
|
t.Errorf("adopted = %+v, want row 9", q.adopted)
|
||||||
@@ -280,9 +290,9 @@ func TestRowConverters(t *testing.T) {
|
|||||||
if len(got) != 2 || got[0].id != testUUID(1) || got[1].filePath != "/b" {
|
if len(got) != 2 || got[0].id != testUUID(1) || got[1].filePath != "/b" {
|
||||||
t.Errorf("rowsFromMbid = %+v", got)
|
t.Errorf("rowsFromMbid = %+v", got)
|
||||||
}
|
}
|
||||||
got = rowsFromFingerprint([]dbq.FindMissingTrackByFingerprintRow{fpRow(3, "/c")})
|
got = rowsFromAudioHash([]dbq.FindMissingTrackByAudioHashRow{hashRow(3, "/c")})
|
||||||
if len(got) != 1 || got[0].id != testUUID(3) || got[0].filePath != "/c" {
|
if len(got) != 1 || got[0].id != testUUID(3) || got[0].filePath != "/c" {
|
||||||
t.Errorf("rowsFromFingerprint = %+v", got)
|
t.Errorf("rowsFromAudioHash = %+v", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -75,10 +75,18 @@ type Scanner struct {
|
|||||||
pool *pgxpool.Pool
|
pool *pgxpool.Pool
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
paths []string
|
paths []string
|
||||||
|
// fingerprint derives a file's acoustic identity (M400). A field so an
|
||||||
|
// integration test can substitute a deterministic one: CI has no real audio
|
||||||
|
// to fingerprint, and what the test pins is WHEN the scan fingerprints, not
|
||||||
|
// what the tools print. Call it through fingerprintFile.
|
||||||
|
fingerprint func(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult
|
||||||
|
// settings is the operator's fingerprinting policy (#3913), shared with the
|
||||||
|
// workers and the admin API. Nil fingerprints with the defaults.
|
||||||
|
settings *FingerprintSettingsService
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string) *Scanner {
|
func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string, settings *FingerprintSettingsService) *Scanner {
|
||||||
return &Scanner{pool: pool, logger: logger, paths: paths}
|
return &Scanner{pool: pool, logger: logger, paths: paths, fingerprint: computeFingerprint, settings: settings}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Scan walks every configured root and upserts any audio file whose mtime is
|
// Scan walks every configured root and upserts any audio file whose mtime is
|
||||||
@@ -295,17 +303,44 @@ func (s *Scanner) scanFile(
|
|||||||
durationMs = probed
|
durationMs = probed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fingerprint only bytes this row has not seen: a new path, or a file whose
|
||||||
|
// mtime moved past the row's. An unchanged file re-read for a tag repair
|
||||||
|
// keeps its stored fingerprint, for the same reason it keeps its duration
|
||||||
|
// above — a tagReadVersion bump must stay bound by tag reads, not become a
|
||||||
|
// decode of the whole library.
|
||||||
|
//
|
||||||
|
// An unchanged file with NO fingerprint yet is the backfill's job (#3908),
|
||||||
|
// deliberately not the scan's. Folding it into the skip check would make
|
||||||
|
// the first scan after an upgrade re-decode every track and push a sync
|
||||||
|
// change to every client for each one.
|
||||||
|
//
|
||||||
|
// Computed before move adoption so adoption can match on the audio hash
|
||||||
|
// (#3914); stored after the upsert, once the row id is known.
|
||||||
|
//
|
||||||
|
// With fingerprinting switched off (#3913) the scan decodes nothing, but it
|
||||||
|
// still takes the stream hash: a demux rather than a decode, and the only
|
||||||
|
// thing that recognises a moved file with no MBID.
|
||||||
|
var fp fingerprintResult
|
||||||
|
fpCfg := s.settings.Get()
|
||||||
|
fingerprinted := !unchanged
|
||||||
|
if fingerprinted {
|
||||||
|
fp = s.fingerprintFile(ctx, path, fingerprintOptions{
|
||||||
|
lengthSec: fpCfg.ChromaprintLengthSec,
|
||||||
|
chromaprint: fpCfg.Enabled,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// A path we've never seen might not be a new track — it might be one that
|
// A path we've never seen might not be a new track — it might be one that
|
||||||
// moved or was renamed (#2528). Adopting re-points the existing row at this
|
// moved or was renamed (#2528). Adopting re-points the existing row at this
|
||||||
// path and clears its missing mark, so the UpsertTrack below conflicts on
|
// path and clears its missing mark, so the UpsertTrack below conflicts on
|
||||||
// file_path and updates THAT row: same track id, likes and play history
|
// file_path and updates THAT row: same track id, likes and play history
|
||||||
// intact. Without this, renumbering an album forks every track on it.
|
// intact. Without this, renumbering an album forks every track on it.
|
||||||
//
|
//
|
||||||
// Runs here rather than earlier because the fingerprint needs the probed
|
// Runs after fingerprinting because, for a file with no MBID, adoption matches
|
||||||
// duration, and only for genuinely unknown paths — a known path is already
|
// on its audio hash (#3914), and only for genuinely unknown paths — a known
|
||||||
// the row we're going to update.
|
// path is already the row we're going to update.
|
||||||
if !knownTrack {
|
if !knownTrack {
|
||||||
if s.adoptMovedTrack(ctx, q, path, info.Size(), durationMs, recordingMBID) {
|
if s.adoptMovedTrack(ctx, q, path, fp.streamSHA256, recordingMBID) {
|
||||||
// Count it as an update: the row existed, and reporting it as Added
|
// Count it as an update: the row existed, and reporting it as Added
|
||||||
// would overstate library growth on every reorganisation.
|
// would overstate library growth on every reorganisation.
|
||||||
knownTrack = true
|
knownTrack = true
|
||||||
@@ -359,6 +394,15 @@ func (s *Scanner) scanFile(
|
|||||||
// touches this track will re-emit the change.
|
// touches this track will re-emit the change.
|
||||||
s.logger.Warn("library scan: LogChange track upsert failed", "track_id", track.ID, "err", err)
|
s.logger.Warn("library scan: LogChange track upsert failed", "track_id", track.ID, "err", err)
|
||||||
}
|
}
|
||||||
|
if fingerprinted {
|
||||||
|
if fpCfg.Enabled {
|
||||||
|
storeFingerprint(ctx, q, s.logger, track.ID, path, fp, fpCfg.ChromaprintLengthSec)
|
||||||
|
} else if err := q.DeleteTrackFingerprint(ctx, track.ID); err != nil {
|
||||||
|
// Off, nothing is stored — but a row describing the previous bytes
|
||||||
|
// must not outlive them, or the sweep would compare audio that is gone.
|
||||||
|
s.logger.Warn("fingerprint: clearing stale fingerprint failed", "path", path, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if knownTrack {
|
if knownTrack {
|
||||||
stats.Updated++
|
stats.Updated++
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ func TestScanner_Integration(t *testing.T) {
|
|||||||
"TIT2": "Solo", "TPE1": "The Artist Y", "TALB": "Y Album", "TRCK": "1",
|
"TIT2": "Solo", "TPE1": "The Artist Y", "TALB": "Y Album", "TRCK": "1",
|
||||||
})
|
})
|
||||||
|
|
||||||
scanner := New(pool, logger, []string{root})
|
scanner := New(pool, logger, []string{root}, nil)
|
||||||
stats, err := scanner.Scan(ctx, nil)
|
stats, err := scanner.Scan(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("first scan: %v", err)
|
t.Fatalf("first scan: %v", err)
|
||||||
@@ -211,9 +211,9 @@ func writeTestMP3(t *testing.T, path string, frames map[string]string) {
|
|||||||
// playlist memberships travel with it — rather than forking into a marked ghost
|
// playlist memberships travel with it — rather than forking into a marked ghost
|
||||||
// plus a fresh zero-history row.
|
// plus a fresh zero-history row.
|
||||||
//
|
//
|
||||||
// Uses the MBID path. The synthetic MP3s here carry no real audio, so ffprobe
|
// Uses the MBID path. The synthetic MP3s here carry no real audio, so their audio
|
||||||
// yields duration 0 and the size+duration fingerprint is deliberately unusable —
|
// stream hash cannot be relied on — which is why the recording MBID is the signal
|
||||||
// which is why the recording MBID is the signal under test.
|
// under test. The audio-hash path is covered by moved_test.go.
|
||||||
//
|
//
|
||||||
// Eight tracks with one rename keeps the marked fraction at 12.5%, under
|
// Eight tracks with one rename keeps the marked fraction at 12.5%, under
|
||||||
// missingMarkMaxFraction. That is load-bearing: if the rename exceeded the cap,
|
// missingMarkMaxFraction. That is load-bearing: if the rename exceeded the cap,
|
||||||
@@ -259,7 +259,7 @@ func TestScanner_AdoptsMovedFile_Integration(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := New(pool, logger, []string{root})
|
scanner := New(pool, logger, []string{root}, nil)
|
||||||
if _, err := scanner.Scan(ctx, nil); err != nil {
|
if _, err := scanner.Scan(ctx, nil); err != nil {
|
||||||
t.Fatalf("first scan: %v", err)
|
t.Fatalf("first scan: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,6 +35,9 @@ var (
|
|||||||
// config changes in lidarrconfig take effect immediately. clientFn returns
|
// config changes in lidarrconfig take effect immediately. clientFn returns
|
||||||
// nil when Lidarr is disabled.
|
// nil when Lidarr is disabled.
|
||||||
type Service struct {
|
type Service struct {
|
||||||
|
// dataDir lets a Delete file that empties an artist clear that artist's
|
||||||
|
// cached art, the same as the admin remove-track path.
|
||||||
|
dataDir string
|
||||||
pool *pgxpool.Pool
|
pool *pgxpool.Pool
|
||||||
lidarrCfg *lidarrconfig.Service
|
lidarrCfg *lidarrconfig.Service
|
||||||
clientFn func() *lidarr.Client
|
clientFn func() *lidarr.Client
|
||||||
@@ -42,11 +45,11 @@ type Service struct {
|
|||||||
|
|
||||||
// NewService constructs a Service. Pass nil for clientFn to disable the
|
// NewService constructs a Service. Pass nil for clientFn to disable the
|
||||||
// Lidarr-using methods (DeleteViaLidarr will return ErrLidarrDisabled).
|
// Lidarr-using methods (DeleteViaLidarr will return ErrLidarrDisabled).
|
||||||
func NewService(pool *pgxpool.Pool, cfg *lidarrconfig.Service, clientFn func() *lidarr.Client) *Service {
|
func NewService(pool *pgxpool.Pool, cfg *lidarrconfig.Service, clientFn func() *lidarr.Client, dataDir string) *Service {
|
||||||
if clientFn == nil {
|
if clientFn == nil {
|
||||||
clientFn = func() *lidarr.Client { return nil }
|
clientFn = func() *lidarr.Client { return nil }
|
||||||
}
|
}
|
||||||
return &Service{pool: pool, lidarrCfg: cfg, clientFn: clientFn}
|
return &Service{pool: pool, lidarrCfg: cfg, clientFn: clientFn, dataDir: dataDir}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Flag inserts or updates a quarantine row for the caller. Re-flagging
|
// Flag inserts or updates a quarantine row for the caller. Re-flagging
|
||||||
@@ -245,10 +248,13 @@ func (s *Service) snapshot(ctx context.Context, q *dbq.Queries, track dbq.Track)
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteFile removes the track file from disk and the tracks row, then
|
// DeleteFile removes the track file from disk and the tracks row (tidying away
|
||||||
// (via FK ON DELETE CASCADE) clears all per-user quarantine rows for that
|
// an album or artist that leaves empty), then — via FK ON DELETE CASCADE —
|
||||||
// track and writes an audit row. If the file deletion fails, the per-user
|
// clears every per-user quarantine row for that track and writes an audit row.
|
||||||
// rows stay so admin can retry. No partial state.
|
//
|
||||||
|
// If the file cannot be removed, nothing is deleted and the per-user rows stay
|
||||||
|
// so the admin can retry: the error wraps a *library.FileRemoveError, which the
|
||||||
|
// handler turns into an answer naming the cause (#3918). No partial state.
|
||||||
func (s *Service) DeleteFile(ctx context.Context, trackID, adminID pgtype.UUID) (dbq.LidarrQuarantineAction, error) {
|
func (s *Service) DeleteFile(ctx context.Context, trackID, adminID pgtype.UUID) (dbq.LidarrQuarantineAction, error) {
|
||||||
q := dbq.New(s.pool)
|
q := dbq.New(s.pool)
|
||||||
track, err := q.GetTrackByID(ctx, trackID)
|
track, err := q.GetTrackByID(ctx, trackID)
|
||||||
@@ -267,7 +273,7 @@ func (s *Service) DeleteFile(ctx context.Context, trackID, adminID pgtype.UUID)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return dbq.LidarrQuarantineAction{}, fmt.Errorf("count: %w", err)
|
return dbq.LidarrQuarantineAction{}, fmt.Errorf("count: %w", err)
|
||||||
}
|
}
|
||||||
if err := library.DeleteTrackFile(ctx, s.pool, trackID); err != nil {
|
if _, err := library.DeleteTrackFile(ctx, s.pool, nil, s.dataDir, trackID); err != nil {
|
||||||
return dbq.LidarrQuarantineAction{}, fmt.Errorf("delete file: %w", err)
|
return dbq.LidarrQuarantineAction{}, fmt.Errorf("delete file: %w", err)
|
||||||
}
|
}
|
||||||
// tracks row is gone; ON DELETE CASCADE on lidarr_quarantine.track_id
|
// tracks row is gone; ON DELETE CASCADE on lidarr_quarantine.track_id
|
||||||
|
|||||||
@@ -87,7 +87,7 @@ func TestFlag_HappyPath(t *testing.T) {
|
|||||||
user := seedUser(t, pool, "alice")
|
user := seedUser(t, pool, "alice")
|
||||||
track, _, _ := seedTrack(t, pool, "Bad Track", "abc")
|
track, _, _ := seedTrack(t, pool, "Bad Track", "abc")
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
row, err := svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", "crackly")
|
row, err := svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", "crackly")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Flag: %v", err)
|
t.Fatalf("Flag: %v", err)
|
||||||
@@ -105,7 +105,7 @@ func TestFlag_UpsertOnSecondFlag(t *testing.T) {
|
|||||||
user := seedUser(t, pool, "alice")
|
user := seedUser(t, pool, "alice")
|
||||||
track, _, _ := seedTrack(t, pool, "T", "x")
|
track, _, _ := seedTrack(t, pool, "T", "x")
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
if _, err := svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", "first"); err != nil {
|
if _, err := svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", "first"); err != nil {
|
||||||
t.Fatalf("first flag: %v", err)
|
t.Fatalf("first flag: %v", err)
|
||||||
}
|
}
|
||||||
@@ -129,7 +129,7 @@ func TestFlag_NonexistentTrackReturnsErrTrackNotFound(t *testing.T) {
|
|||||||
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
||||||
bogus.Valid = true
|
bogus.Valid = true
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
_, err := svc.Flag(context.Background(), user.ID, bogus, "bad_rip", "")
|
_, err := svc.Flag(context.Background(), user.ID, bogus, "bad_rip", "")
|
||||||
if !errors.Is(err, ErrTrackNotFound) {
|
if !errors.Is(err, ErrTrackNotFound) {
|
||||||
t.Errorf("err = %v, want ErrTrackNotFound", err)
|
t.Errorf("err = %v, want ErrTrackNotFound", err)
|
||||||
@@ -141,7 +141,7 @@ func TestFlag_BadReasonRejected(t *testing.T) {
|
|||||||
user := seedUser(t, pool, "alice")
|
user := seedUser(t, pool, "alice")
|
||||||
track, _, _ := seedTrack(t, pool, "T", "x")
|
track, _, _ := seedTrack(t, pool, "T", "x")
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
_, err := svc.Flag(context.Background(), user.ID, track.ID, "garbage", "")
|
_, err := svc.Flag(context.Background(), user.ID, track.ID, "garbage", "")
|
||||||
if !errors.Is(err, ErrBadReason) {
|
if !errors.Is(err, ErrBadReason) {
|
||||||
t.Errorf("err = %v, want ErrBadReason", err)
|
t.Errorf("err = %v, want ErrBadReason", err)
|
||||||
@@ -153,7 +153,7 @@ func TestUnflag_DeletesRow(t *testing.T) {
|
|||||||
user := seedUser(t, pool, "alice")
|
user := seedUser(t, pool, "alice")
|
||||||
track, _, _ := seedTrack(t, pool, "T", "x")
|
track, _, _ := seedTrack(t, pool, "T", "x")
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
if _, err := svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", ""); err != nil {
|
if _, err := svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", ""); err != nil {
|
||||||
t.Fatalf("Flag: %v", err)
|
t.Fatalf("Flag: %v", err)
|
||||||
}
|
}
|
||||||
@@ -171,7 +171,7 @@ func TestListMine_OrderedNewestFirst(t *testing.T) {
|
|||||||
t1, _, _ := seedTrack(t, pool, "T1", "x")
|
t1, _, _ := seedTrack(t, pool, "T1", "x")
|
||||||
t2, _, _ := seedTrack(t, pool, "T2", "y")
|
t2, _, _ := seedTrack(t, pool, "T2", "y")
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
if _, err := svc.Flag(context.Background(), user.ID, t1.ID, "bad_rip", ""); err != nil {
|
if _, err := svc.Flag(context.Background(), user.ID, t1.ID, "bad_rip", ""); err != nil {
|
||||||
t.Fatalf("Flag t1: %v", err)
|
t.Fatalf("Flag t1: %v", err)
|
||||||
}
|
}
|
||||||
@@ -199,7 +199,7 @@ func TestListAdminQueue_AggregatesByTrackWithReasonCounts(t *testing.T) {
|
|||||||
carol := seedUser(t, pool, "carol")
|
carol := seedUser(t, pool, "carol")
|
||||||
track, _, _ := seedTrack(t, pool, "Hot Mess", "abc")
|
track, _, _ := seedTrack(t, pool, "Hot Mess", "abc")
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
if _, err := svc.Flag(context.Background(), alice.ID, track.ID, "bad_rip", ""); err != nil {
|
if _, err := svc.Flag(context.Background(), alice.ID, track.ID, "bad_rip", ""); err != nil {
|
||||||
t.Fatalf("alice flag: %v", err)
|
t.Fatalf("alice flag: %v", err)
|
||||||
}
|
}
|
||||||
@@ -235,7 +235,7 @@ func TestResolve_ClearsRowsAndWritesAudit(t *testing.T) {
|
|||||||
bob := seedUser(t, pool, "bob")
|
bob := seedUser(t, pool, "bob")
|
||||||
track, _, _ := seedTrack(t, pool, "T", "x")
|
track, _, _ := seedTrack(t, pool, "T", "x")
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
if _, err := svc.Flag(context.Background(), alice.ID, track.ID, "bad_rip", ""); err != nil {
|
if _, err := svc.Flag(context.Background(), alice.ID, track.ID, "bad_rip", ""); err != nil {
|
||||||
t.Fatalf("alice flag: %v", err)
|
t.Fatalf("alice flag: %v", err)
|
||||||
}
|
}
|
||||||
@@ -267,7 +267,7 @@ func TestResolve_NoExistingRowsStillWritesAudit(t *testing.T) {
|
|||||||
user := seedUser(t, pool, "alice")
|
user := seedUser(t, pool, "alice")
|
||||||
track, _, _ := seedTrack(t, pool, "T", "x")
|
track, _, _ := seedTrack(t, pool, "T", "x")
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
// No flags applied — resolve a track with zero quarantine rows.
|
// No flags applied — resolve a track with zero quarantine rows.
|
||||||
audit, err := svc.Resolve(context.Background(), track.ID, user.ID)
|
audit, err := svc.Resolve(context.Background(), track.ID, user.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -291,7 +291,7 @@ func TestResolve_TrackNotFound(t *testing.T) {
|
|||||||
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
||||||
bogus.Valid = true
|
bogus.Valid = true
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
_, err := svc.Resolve(context.Background(), bogus, user.ID)
|
_, err := svc.Resolve(context.Background(), bogus, user.ID)
|
||||||
if !errors.Is(err, ErrTrackNotFound) {
|
if !errors.Is(err, ErrTrackNotFound) {
|
||||||
t.Errorf("err = %v, want ErrTrackNotFound", err)
|
t.Errorf("err = %v, want ErrTrackNotFound", err)
|
||||||
@@ -315,7 +315,7 @@ func TestDeleteFile_RemovesFileAndAuditsAffected(t *testing.T) {
|
|||||||
DurationMs: 1000, FilePath: path, FileSize: 1, FileFormat: "mp3",
|
DurationMs: 1000, FilePath: path, FileSize: 1, FileFormat: "mp3",
|
||||||
})
|
})
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
if _, err := svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", ""); err != nil {
|
if _, err := svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", ""); err != nil {
|
||||||
t.Fatalf("Flag: %v", err)
|
t.Fatalf("Flag: %v", err)
|
||||||
}
|
}
|
||||||
@@ -366,7 +366,7 @@ func TestDeleteViaLidarr_FullCascade(t *testing.T) {
|
|||||||
t.Fatalf("save config: %v", err)
|
t.Fatalf("save config: %v", err)
|
||||||
}
|
}
|
||||||
clientFn := func() *lidarr.Client { return lidarr.NewClient(stub.URL, "k") }
|
clientFn := func() *lidarr.Client { return lidarr.NewClient(stub.URL, "k") }
|
||||||
svc := NewService(pool, cfg, clientFn)
|
svc := NewService(pool, cfg, clientFn, "")
|
||||||
|
|
||||||
q := dbq.New(pool)
|
q := dbq.New(pool)
|
||||||
artist, _ := q.UpsertArtist(context.Background(), dbq.UpsertArtistParams{Name: "A", SortName: "A"})
|
artist, _ := q.UpsertArtist(context.Background(), dbq.UpsertArtistParams{Name: "A", SortName: "A"})
|
||||||
@@ -424,7 +424,7 @@ func TestDeleteViaLidarr_LidarrDisabled(t *testing.T) {
|
|||||||
user := seedUser(t, pool, "alice")
|
user := seedUser(t, pool, "alice")
|
||||||
track, _, _ := seedTrack(t, pool, "T", "x")
|
track, _, _ := seedTrack(t, pool, "T", "x")
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
_, _, err := svc.DeleteViaLidarr(context.Background(), track.ID, user.ID)
|
_, _, err := svc.DeleteViaLidarr(context.Background(), track.ID, user.ID)
|
||||||
if !errors.Is(err, ErrLidarrDisabled) {
|
if !errors.Is(err, ErrLidarrDisabled) {
|
||||||
t.Errorf("err = %v, want ErrLidarrDisabled", err)
|
t.Errorf("err = %v, want ErrLidarrDisabled", err)
|
||||||
@@ -456,7 +456,7 @@ func TestDeleteViaLidarr_AlbumMBIDMissing(t *testing.T) {
|
|||||||
cfg := lidarrconfig.New(pool)
|
cfg := lidarrconfig.New(pool)
|
||||||
_ = cfg.Save(context.Background(), lidarrconfig.Config{Enabled: true, BaseURL: stub.URL, APIKey: "k"})
|
_ = cfg.Save(context.Background(), lidarrconfig.Config{Enabled: true, BaseURL: stub.URL, APIKey: "k"})
|
||||||
clientFn := func() *lidarr.Client { return lidarr.NewClient(stub.URL, "k") }
|
clientFn := func() *lidarr.Client { return lidarr.NewClient(stub.URL, "k") }
|
||||||
svc := NewService(pool, cfg, clientFn)
|
svc := NewService(pool, cfg, clientFn, "")
|
||||||
_, _ = svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", "")
|
_, _ = svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", "")
|
||||||
|
|
||||||
_, _, err := svc.DeleteViaLidarr(context.Background(), track.ID, user.ID)
|
_, _, err := svc.DeleteViaLidarr(context.Background(), track.ID, user.ID)
|
||||||
@@ -481,7 +481,7 @@ func TestDeleteViaLidarr_LidarrAlbumNotFound(t *testing.T) {
|
|||||||
cfg := lidarrconfig.New(pool)
|
cfg := lidarrconfig.New(pool)
|
||||||
_ = cfg.Save(context.Background(), lidarrconfig.Config{Enabled: true, BaseURL: stub.URL, APIKey: "k"})
|
_ = cfg.Save(context.Background(), lidarrconfig.Config{Enabled: true, BaseURL: stub.URL, APIKey: "k"})
|
||||||
clientFn := func() *lidarr.Client { return lidarr.NewClient(stub.URL, "k") }
|
clientFn := func() *lidarr.Client { return lidarr.NewClient(stub.URL, "k") }
|
||||||
svc := NewService(pool, cfg, clientFn)
|
svc := NewService(pool, cfg, clientFn, "")
|
||||||
|
|
||||||
track, _, _ := seedTrack(t, pool, "T", "x")
|
track, _, _ := seedTrack(t, pool, "T", "x")
|
||||||
_, _ = svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", "")
|
_, _ = svc.Flag(context.Background(), user.ID, track.ID, "bad_rip", "")
|
||||||
@@ -499,7 +499,7 @@ func TestDeleteFile_TrackNotFound(t *testing.T) {
|
|||||||
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
||||||
bogus.Valid = true
|
bogus.Valid = true
|
||||||
|
|
||||||
svc := NewService(pool, lidarrconfig.New(pool), nil)
|
svc := NewService(pool, lidarrconfig.New(pool), nil, "")
|
||||||
_, err := svc.DeleteFile(context.Background(), bogus, user.ID)
|
_, err := svc.DeleteFile(context.Background(), bogus, user.ID)
|
||||||
if !errors.Is(err, ErrTrackNotFound) {
|
if !errors.Is(err, ErrTrackNotFound) {
|
||||||
t.Errorf("err = %v, want ErrTrackNotFound", err)
|
t.Errorf("err = %v, want ErrTrackNotFound", err)
|
||||||
@@ -521,7 +521,7 @@ func TestDeleteViaLidarr_TrackNotFound(t *testing.T) {
|
|||||||
t.Fatalf("save config: %v", err)
|
t.Fatalf("save config: %v", err)
|
||||||
}
|
}
|
||||||
clientFn := func() *lidarr.Client { return lidarr.NewClient(stub.URL, "k") }
|
clientFn := func() *lidarr.Client { return lidarr.NewClient(stub.URL, "k") }
|
||||||
svc := NewService(pool, cfg, clientFn)
|
svc := NewService(pool, cfg, clientFn, "")
|
||||||
|
|
||||||
var bogus pgtype.UUID
|
var bogus pgtype.UUID
|
||||||
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
bogus.Bytes = [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
|
||||||
|
|||||||
+114
-15
@@ -219,7 +219,30 @@ var (
|
|||||||
// uniform with radio pending trend data.
|
// uniform with radio pending trend data.
|
||||||
ContextTimeWeight: 1.0,
|
ContextTimeWeight: 1.0,
|
||||||
}
|
}
|
||||||
|
// Songs-like's own profile (#3881). Pre-push literal only; shipped
|
||||||
|
// defaults live in recsettings.ShippedSongsLikeWeights and must stay in
|
||||||
|
// sync with it, exactly as systemMixWeights does above.
|
||||||
|
//
|
||||||
|
// SimilarityWeight dominates here and every seed-INDEPENDENT term is
|
||||||
|
// demoted, which is the whole difference between this surface and For-You.
|
||||||
|
// See ShippedSongsLikeWeights for the property the numbers encode.
|
||||||
|
songsLikeWeights = recommendation.ScoringWeights{
|
||||||
|
BaseWeight: 1.0,
|
||||||
|
LikeBoost: 0.5,
|
||||||
|
RecencyWeight: 0.25,
|
||||||
|
SkipPenalty: 2.0,
|
||||||
|
JitterMagnitude: 0.05,
|
||||||
|
ContextWeight: 0.5,
|
||||||
|
SimilarityWeight: 4.0,
|
||||||
|
TasteWeight: 0.25,
|
||||||
|
ContextTimeWeight: 0.5,
|
||||||
|
}
|
||||||
systemTasteConfig = taste.DefaultConfig()
|
systemTasteConfig = taste.DefaultConfig()
|
||||||
|
|
||||||
|
// Sizes the candidate pool to the library (#3880). Cached because the
|
||||||
|
// count is a full table scan and the daily build runs it once per user;
|
||||||
|
// within the TTL every user in a build shares one count.
|
||||||
|
systemLibrarySize = recommendation.NewLibrarySize(nil)
|
||||||
)
|
)
|
||||||
|
|
||||||
// SetSystemMixWeights installs the current daily_mix scoring weights.
|
// SetSystemMixWeights installs the current daily_mix scoring weights.
|
||||||
@@ -230,6 +253,20 @@ func SetSystemMixWeights(w recommendation.ScoringWeights) {
|
|||||||
systemMixWeights = w
|
systemMixWeights = w
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetSongsLikeWeights installs the songs_like scoring profile (#3881).
|
||||||
|
// Same push model as SetSystemMixWeights — recsettings calls it on boot and
|
||||||
|
// after every knob turn, so a tuning change takes effect on the next daily
|
||||||
|
// build with no restart.
|
||||||
|
//
|
||||||
|
// Separate from systemMixWeights because Songs-like and For-You want opposite
|
||||||
|
// things: For-You roams, Songs-like must not. Sharing one profile is what made
|
||||||
|
// "Songs like X" wander.
|
||||||
|
func SetSongsLikeWeights(w recommendation.ScoringWeights) {
|
||||||
|
systemTuningMu.Lock()
|
||||||
|
defer systemTuningMu.Unlock()
|
||||||
|
songsLikeWeights = w
|
||||||
|
}
|
||||||
|
|
||||||
// SetTasteConfig installs the taste-profile build configuration
|
// SetTasteConfig installs the taste-profile build configuration
|
||||||
// (half-life + engagement curve, #1250). Same push model as
|
// (half-life + engagement curve, #1250). Same push model as
|
||||||
// SetSystemMixWeights.
|
// SetSystemMixWeights.
|
||||||
@@ -239,6 +276,23 @@ func SetTasteConfig(c taste.Config) {
|
|||||||
systemTasteConfig = c
|
systemTasteConfig = c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// dailyOrderSeed is the value the randomised candidate arms order by (#3889).
|
||||||
|
//
|
||||||
|
// Per (user, day) so a same-day rebuild draws the SAME set — which is what
|
||||||
|
// TestBuildSystemPlaylists_DailyNonceDeterminism asserts and what those arms
|
||||||
|
// only ever achieved by accident before, when their limits happened to exceed
|
||||||
|
// the eligible rows. It changes on the day boundary, so the mixes still move
|
||||||
|
// daily.
|
||||||
|
func dailyOrderSeed(userID pgtype.UUID, dateStr string) string {
|
||||||
|
return uuidStringPL(userID) + ":" + dateStr
|
||||||
|
}
|
||||||
|
|
||||||
|
func currentSongsLikeWeights() recommendation.ScoringWeights {
|
||||||
|
systemTuningMu.RLock()
|
||||||
|
defer systemTuningMu.RUnlock()
|
||||||
|
return songsLikeWeights
|
||||||
|
}
|
||||||
|
|
||||||
func currentSystemMixWeights() recommendation.ScoringWeights {
|
func currentSystemMixWeights() recommendation.ScoringWeights {
|
||||||
systemTuningMu.RLock()
|
systemTuningMu.RLock()
|
||||||
defer systemTuningMu.RUnlock()
|
defer systemTuningMu.RUnlock()
|
||||||
@@ -392,7 +446,13 @@ func pickWeightedTail(tailPool []recommendation.Candidate, dateStr string, tailN
|
|||||||
// tieBreakHash). The scoring RNG is seeded by userIDHash so jitter is
|
// tieBreakHash). The scoring RNG is seeded by userIDHash so jitter is
|
||||||
// deterministic per (user, day) but rotates across days. Pure — no
|
// deterministic per (user, day) but rotates across days. Pure — no
|
||||||
// truncation, no cap.
|
// truncation, no cap.
|
||||||
func scoreAndSortCandidates(cands []recommendation.Candidate, userID pgtype.UUID, dateStr string, now time.Time) []recommendation.Candidate {
|
// weights is a parameter rather than a read of currentSystemMixWeights()
|
||||||
|
// because this sort IS the selection: the caller caps and truncates in the
|
||||||
|
// order this returns, so whatever profile ranks here decides which tracks
|
||||||
|
// reach the playlist. Scoring with daily_mix here and re-scoring with
|
||||||
|
// songs_like afterwards would have let the new profile relabel tracks it had
|
||||||
|
// no part in choosing — inert where it matters (#3881).
|
||||||
|
func scoreAndSortCandidates(cands []recommendation.Candidate, userID pgtype.UUID, dateStr string, now time.Time, weights recommendation.ScoringWeights) []recommendation.Candidate {
|
||||||
rng := rand.New(rand.NewSource(int64(userIDHash(userID, dateStr))))
|
rng := rand.New(rand.NewSource(int64(userIDHash(userID, dateStr))))
|
||||||
type scored struct {
|
type scored struct {
|
||||||
c recommendation.Candidate
|
c recommendation.Candidate
|
||||||
@@ -411,7 +471,6 @@ func scoreAndSortCandidates(cands []recommendation.Candidate, userID pgtype.UUID
|
|||||||
sort.SliceStable(ordered, func(i, j int) bool {
|
sort.SliceStable(ordered, func(i, j int) bool {
|
||||||
return uuidLessPL(ordered[i].Track.ID, ordered[j].Track.ID)
|
return uuidLessPL(ordered[i].Track.ID, ordered[j].Track.ID)
|
||||||
})
|
})
|
||||||
weights := currentSystemMixWeights()
|
|
||||||
pairs := make([]scored, len(ordered))
|
pairs := make([]scored, len(ordered))
|
||||||
for i, c := range ordered {
|
for i, c := range ordered {
|
||||||
pairs[i] = scored{c: c, score: recommendation.Score(c.Inputs, weights, now, rng.Float64)}
|
pairs[i] = scored{c: c, score: recommendation.Score(c.Inputs, weights, now, rng.Float64)}
|
||||||
@@ -581,6 +640,7 @@ func produceForYou(
|
|||||||
zeroVec,
|
zeroVec,
|
||||||
seeds,
|
seeds,
|
||||||
systemForYouSourceLimits(),
|
systemForYouSourceLimits(),
|
||||||
|
dailyOrderSeed(userID, dateStr),
|
||||||
)
|
)
|
||||||
if cerr != nil {
|
if cerr != nil {
|
||||||
logger.Warn("system playlist: for-you candidates load failed for seed; continuing",
|
logger.Warn("system playlist: for-you candidates load failed for seed; continuing",
|
||||||
@@ -632,6 +692,13 @@ func produceSeedMixes(
|
|||||||
seedPool := pickSeedArtistsFromRows(seedRowsLocal)
|
seedPool := pickSeedArtistsFromRows(seedRowsLocal)
|
||||||
seeds := pickSeedArtistsForDay(seedPool, userID, dateStr)
|
seeds := pickSeedArtistsForDay(seedPool, userID, dateStr)
|
||||||
|
|
||||||
|
// Once per build rather than once per seed artist — six mixes would
|
||||||
|
// otherwise mean six full-table counts for a number that cannot have
|
||||||
|
// changed between them.
|
||||||
|
librarySize := systemLibrarySize.Get(ctx, func(c context.Context) (int64, error) {
|
||||||
|
return recommendation.CountLibraryTracks(c, q)
|
||||||
|
})
|
||||||
|
|
||||||
out := make([]builtPlaylist, 0, len(seeds))
|
out := make([]builtPlaylist, 0, len(seeds))
|
||||||
for _, artistID := range seeds {
|
for _, artistID := range seeds {
|
||||||
artistRow, aerr := q.GetArtistByID(ctx, artistID)
|
artistRow, aerr := q.GetArtistByID(ctx, artistID)
|
||||||
@@ -647,23 +714,47 @@ func produceSeedMixes(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
zeroVec := recommendation.SessionVector{Seed: true}
|
zeroVec := recommendation.SessionVector{Seed: true}
|
||||||
|
// Songs-like's own pool shape, not the default (#3881). Same total
|
||||||
|
// size; the composition shifts toward arms that actually measure
|
||||||
|
// distance from the seed. The default gave ~29% of candidates a
|
||||||
|
// sim_score of literally 0.
|
||||||
|
//
|
||||||
|
// Then scaled to the library (#3880): a bigger collection should put
|
||||||
|
// more genuinely-similar candidates in reach, not the same ~170
|
||||||
|
// regardless. The weights still rank sim_score-0 arms last, so the
|
||||||
|
// growth reaches coherence rather than working against it.
|
||||||
cands, cerr := recommendation.LoadCandidatesFromSimilarity(
|
cands, cerr := recommendation.LoadCandidatesFromSimilarity(
|
||||||
ctx, q, userID, seedTrack, 1, zeroVec, []pgtype.UUID{seedTrack},
|
ctx, q, userID, seedTrack, 1, zeroVec, []pgtype.UUID{seedTrack},
|
||||||
recommendation.DefaultCandidateSourceLimits(),
|
recommendation.ScaleForLibrary(
|
||||||
|
recommendation.SongsLikeCandidateSourceLimits(), librarySize,
|
||||||
|
),
|
||||||
|
dailyOrderSeed(userID, dateStr),
|
||||||
)
|
)
|
||||||
if cerr != nil {
|
if cerr != nil {
|
||||||
logger.Warn("system playlist: seed candidates load failed; skipping",
|
logger.Warn("system playlist: seed candidates load failed; skipping",
|
||||||
"artist_id", uuidStringPL(artistID), "err", cerr)
|
"artist_id", uuidStringPL(artistID), "err", cerr)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// "Songs like X" excludes X's own songs.
|
// The seed artist's own songs are ELIGIBLE here, deliberately.
|
||||||
filtered := make([]recommendation.Candidate, 0, len(cands))
|
//
|
||||||
for _, c := range cands {
|
// This used to filter them out — "Songs like X excludes X's own
|
||||||
if !pgtypeUUIDEqual(c.Track.ArtistID, artistID) {
|
// songs" — which reads as obviously right and is not. The seed is a
|
||||||
filtered = append(filtered, c)
|
// TRACK, and the tracks most likely to sound like it are usually the
|
||||||
}
|
// rest of that artist's catalogue; excluding them threw away the
|
||||||
}
|
// nearest neighbours of the very thing the mix is built around, and
|
||||||
tracks := pickTopN(filtered, userID, dateStr, now, systemMixLength)
|
// then reached further out to replace them. On a surface whose whole
|
||||||
|
// job is staying in one neighbourhood, that is backwards.
|
||||||
|
//
|
||||||
|
// Operator, 2026-09-10: "it should also be able to include music from
|
||||||
|
// the same artist."
|
||||||
|
//
|
||||||
|
// Domination is bounded by the cap rather than by exclusion, which is
|
||||||
|
// the distinction that makes this safe: capCandidatesByAlbumAndArtist
|
||||||
|
// inside pickTopN allows at most discoverMaxTracksPerArtist (3) of a
|
||||||
|
// 25-track mix — 12%, a presence rather than a takeover. The seed
|
||||||
|
// track itself still cannot appear; it is passed as an exclusion to
|
||||||
|
// LoadCandidatesFromSimilarity above.
|
||||||
|
tracks := pickTopN(cands, userID, dateStr, now, systemMixLength)
|
||||||
if len(tracks) == 0 {
|
if len(tracks) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -838,13 +929,20 @@ func BuildSystemPlaylists(ctx context.Context, pool *pgxpool.Pool, logger *slog.
|
|||||||
// truncates to n. Used by Songs-like-X (and as the fallback inside
|
// truncates to n. Used by Songs-like-X (and as the fallback inside
|
||||||
// pickHeadAndTail for small pools).
|
// pickHeadAndTail for small pools).
|
||||||
func pickTopN(cands []recommendation.Candidate, userID pgtype.UUID, dateStr string, now time.Time, n int) []rankedCandidate {
|
func pickTopN(cands []recommendation.Candidate, userID pgtype.UUID, dateStr string, now time.Time, n int) []rankedCandidate {
|
||||||
sorted := scoreAndSortCandidates(cands, userID, dateStr, now)
|
// songs_like, not daily_mix (#3881). produceSeedMixes is this function's
|
||||||
|
// only caller, so the switch moves exactly one surface — For-You ranks
|
||||||
|
// through pickHeadAndTail and keeps the broader daily_mix profile.
|
||||||
|
//
|
||||||
|
// The SAME profile does the selection sort and the final score. Passing
|
||||||
|
// one and using the other is the subtle version of this bug: the playlist
|
||||||
|
// would still be chosen by daily_mix and merely wear songs_like numbers.
|
||||||
|
weights := currentSongsLikeWeights()
|
||||||
|
sorted := scoreAndSortCandidates(cands, userID, dateStr, now, weights)
|
||||||
capped := capCandidatesByAlbumAndArtist(sorted)
|
capped := capCandidatesByAlbumAndArtist(sorted)
|
||||||
if len(capped) > n {
|
if len(capped) > n {
|
||||||
capped = capped[:n]
|
capped = capped[:n]
|
||||||
}
|
}
|
||||||
rng := rand.New(rand.NewSource(int64(userIDHash(userID, dateStr))))
|
rng := rand.New(rand.NewSource(int64(userIDHash(userID, dateStr))))
|
||||||
weights := currentSystemMixWeights()
|
|
||||||
out := make([]rankedCandidate, len(capped))
|
out := make([]rankedCandidate, len(capped))
|
||||||
for i, c := range capped {
|
for i, c := range capped {
|
||||||
out[i] = rankedCandidate{
|
out[i] = rankedCandidate{
|
||||||
@@ -873,10 +971,11 @@ func pickHeadAndTail(
|
|||||||
cands []recommendation.Candidate, seedOf map[pgtype.UUID]int, numSeeds int,
|
cands []recommendation.Candidate, seedOf map[pgtype.UUID]int, numSeeds int,
|
||||||
userID pgtype.UUID, dateStr string, now time.Time, headN, tailN int,
|
userID pgtype.UUID, dateStr string, now time.Time, headN, tailN int,
|
||||||
) []rankedCandidate {
|
) []rankedCandidate {
|
||||||
sorted := scoreAndSortCandidates(cands, userID, dateStr, now)
|
// daily_mix — For-You is the broad surface and keeps the roaming profile.
|
||||||
|
weights := currentSystemMixWeights()
|
||||||
|
sorted := scoreAndSortCandidates(cands, userID, dateStr, now, weights)
|
||||||
capped := capCandidatesByAlbumAndArtist(sorted)
|
capped := capCandidatesByAlbumAndArtist(sorted)
|
||||||
rng := rand.New(rand.NewSource(int64(userIDHash(userID, dateStr))))
|
rng := rand.New(rand.NewSource(int64(userIDHash(userID, dateStr))))
|
||||||
weights := currentSystemMixWeights()
|
|
||||||
|
|
||||||
total := headN + tailN
|
total := headN + tailN
|
||||||
if len(capped) <= total {
|
if len(capped) <= total {
|
||||||
|
|||||||
@@ -2,8 +2,10 @@ package playlists_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -287,6 +289,141 @@ func TestBuildSystemPlaylists_Concurrency(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// seedSharedArtistLibrary seeds artists that genuinely OWN several tracks.
|
||||||
|
//
|
||||||
|
// seedActiveLibrary cannot be used for this: its helper documents that
|
||||||
|
// "artist and album are not deduplicated across calls (mbid-less upsert)",
|
||||||
|
// so every track gets its own artist row and a seed artist always has
|
||||||
|
// exactly one track — the seed itself, which is excluded. A same-artist
|
||||||
|
// assertion against that fixture can never pass no matter what the code
|
||||||
|
// does, which is how the first version of this test failed.
|
||||||
|
//
|
||||||
|
// Albums are not deduplicated either, for the same mbid-less reason, so each
|
||||||
|
// track ends up under its own album row however the titles are written. That
|
||||||
|
// is convenient here rather than a problem: it means the per-ALBUM cap (2)
|
||||||
|
// never binds, and the per-ARTIST cap (3) is unambiguously the thing under
|
||||||
|
// test. Do not "fix" the album titles into something shared without checking
|
||||||
|
// which cap you are then measuring.
|
||||||
|
func seedSharedArtistLibrary(
|
||||||
|
t *testing.T, pool *pgxpool.Pool, name string, numArtists, tracksPerArtist int,
|
||||||
|
) (dbq.User, []pgtype.UUID) {
|
||||||
|
t.Helper()
|
||||||
|
q := dbq.New(pool)
|
||||||
|
ctx := context.Background()
|
||||||
|
u := seedUser(t, pool, name)
|
||||||
|
now := time.Now().UTC()
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
artistIDs := make([]pgtype.UUID, 0, numArtists)
|
||||||
|
for a := 0; a < numArtists; a++ {
|
||||||
|
artistName := name + "-shared-" + string(rune('A'+a))
|
||||||
|
ar, err := q.UpsertArtist(ctx, dbq.UpsertArtistParams{Name: artistName, SortName: artistName})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("seed artist: %v", err)
|
||||||
|
}
|
||||||
|
artistIDs = append(artistIDs, ar.ID)
|
||||||
|
|
||||||
|
for k := 0; k < tracksPerArtist; k++ {
|
||||||
|
albumTitle := fmt.Sprintf("%s - Album %d", artistName, k/2)
|
||||||
|
al, err := q.UpsertAlbum(ctx, dbq.UpsertAlbumParams{
|
||||||
|
Title: albumTitle, SortTitle: albumTitle, ArtistID: ar.ID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("seed album: %v", err)
|
||||||
|
}
|
||||||
|
tk, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
|
||||||
|
Title: fmt.Sprintf("%s-t%d", artistName, k), AlbumID: al.ID, ArtistID: ar.ID,
|
||||||
|
DurationMs: 1000,
|
||||||
|
FilePath: filepath.Join(dir, fmt.Sprintf("%s-%d-%d.mp3", name, a, k)),
|
||||||
|
FileSize: 100, FileFormat: "mp3",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("seed track: %v", err)
|
||||||
|
}
|
||||||
|
// Well clear of the recently-played exclusion window, which uses
|
||||||
|
// the DATABASE clock rather than the build's `now`.
|
||||||
|
for pl := 0; pl < 3; pl++ {
|
||||||
|
seedPlayEvent(t, pool, u.ID, tk.ID,
|
||||||
|
now.Add(-time.Duration(24+a*10+k+pl)*time.Hour), false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return u, artistIDs
|
||||||
|
}
|
||||||
|
|
||||||
|
// The seed artist's own tracks are ELIGIBLE for its "Songs like" mix, and are
|
||||||
|
// bounded by the diversity cap rather than excluded outright (#3881).
|
||||||
|
//
|
||||||
|
// produceSeedMixes used to filter them out — "Songs like X excludes X's own
|
||||||
|
// songs" — which threw away the nearest neighbours of the seed TRACK and then
|
||||||
|
// reached further out to replace them. Operator, 2026-09-10: "it should also
|
||||||
|
// be able to include music from the same artist."
|
||||||
|
//
|
||||||
|
// Asserted end-to-end rather than by reading the source, because the guard has
|
||||||
|
// to survive the filter coming back in a different shape — and because an
|
||||||
|
// absence check would now match the comment explaining why the filter is gone.
|
||||||
|
func TestBuildSystemPlaylists_SongsLikeIncludesItsSeedArtist(t *testing.T) {
|
||||||
|
pool := newPool(t)
|
||||||
|
logger := discardLogger()
|
||||||
|
u, _ := seedSharedArtistLibrary(t, pool, "seedartist", 4, 6)
|
||||||
|
ctx := context.Background()
|
||||||
|
now := time.Date(2026, 5, 4, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
if err := playlists.BuildSystemPlaylists(ctx, pool, logger, u.ID, now, t.TempDir()); err != nil {
|
||||||
|
t.Fatalf("build: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := pool.Query(ctx, `
|
||||||
|
SELECT count(*) FILTER (WHERE t.artist_id = p.seed_artist_id) AS own,
|
||||||
|
count(*) AS total
|
||||||
|
FROM playlists p
|
||||||
|
JOIN playlist_tracks pt ON pt.playlist_id = p.id
|
||||||
|
JOIN tracks t ON t.id = pt.track_id
|
||||||
|
WHERE p.user_id = $1 AND p.system_variant = 'songs_like_artist'
|
||||||
|
GROUP BY p.id
|
||||||
|
`, u.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("query: %v", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
// discoverMaxTracksPerArtist, which this package_test cannot reference.
|
||||||
|
// Duplicated deliberately: if the cap moves, this failing is the point.
|
||||||
|
const maxPerArtist = 3
|
||||||
|
|
||||||
|
mixes, withOwn := 0, 0
|
||||||
|
for rows.Next() {
|
||||||
|
var own, total int
|
||||||
|
if err := rows.Scan(&own, &total); err != nil {
|
||||||
|
t.Fatalf("scan: %v", err)
|
||||||
|
}
|
||||||
|
mixes++
|
||||||
|
if own > 0 {
|
||||||
|
withOwn++
|
||||||
|
}
|
||||||
|
// The bound is what makes inclusion safe. Without it, "include the
|
||||||
|
// seed artist" becomes "the mix is mostly the seed artist", which is
|
||||||
|
// the radio failure (#3882) arriving on a different surface.
|
||||||
|
if own > maxPerArtist {
|
||||||
|
t.Errorf("a songs_like mix carries %d tracks by its own seed artist "+
|
||||||
|
"out of %d; the per-artist cap (%d) is not being applied",
|
||||||
|
own, total, maxPerArtist)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
t.Fatalf("rows: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if mixes == 0 {
|
||||||
|
t.Fatal("no songs_like_artist mixes were built, so this test asserts nothing")
|
||||||
|
}
|
||||||
|
if withOwn == 0 {
|
||||||
|
t.Errorf("none of the %d songs_like mixes contains a single track by its own "+
|
||||||
|
"seed artist — the tracks most likely to sound like the seed are being "+
|
||||||
|
"excluded from the surface whose job is sounding like the seed", mixes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildSystemPlaylists_DailyNonceDeterminism(t *testing.T) {
|
func TestBuildSystemPlaylists_DailyNonceDeterminism(t *testing.T) {
|
||||||
pool := newPool(t)
|
pool := newPool(t)
|
||||||
logger := discardLogger()
|
logger := discardLogger()
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ func buildYouMightLike(
|
|||||||
cands, err := recommendation.LoadCandidatesFromSimilarity(
|
cands, err := recommendation.LoadCandidatesFromSimilarity(
|
||||||
ctx, q, userID, seed, 1, zeroVec,
|
ctx, q, userID, seed, 1, zeroVec,
|
||||||
[]pgtype.UUID{seed}, ymlLimits,
|
[]pgtype.UUID{seed}, ymlLimits,
|
||||||
|
dailyOrderSeed(userID, dateStr),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warn("you-might-like: candidate load failed; skipping",
|
logger.Warn("you-might-like: candidate load failed; skipping",
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user