M400: acoustic duplicate detection, history-preserving merge, and fingerprinting settings #134

Merged
bvandeusen merged 8 commits from dev into main 2026-09-11 20:56:56 -04:00
18 changed files with 1179 additions and 1 deletions
Showing only changes of commit ff493a8c7d - Show all commits
+230
View File
@@ -0,0 +1,230 @@
package api
import (
"context"
"errors"
"net/http"
"github.com/go-chi/chi/v5"
"github.com/jackc/pgx/v5"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
// duplicateMemberView is one copy in a proposed duplicate group. LikeCount and
// PlayCount span every user: the report is admin-only, and what a copy carries
// is the fact the operator weighs when choosing which to keep.
type duplicateMemberView struct {
TrackID string `json:"track_id"`
Title string `json:"title"`
ArtistName string `json:"artist_name"`
AlbumID string `json:"album_id"`
AlbumTitle string `json:"album_title"`
FilePath string `json:"file_path"`
FileFormat string `json:"file_format"`
FileSize int64 `json:"file_size"`
DurationSec int32 `json:"duration_sec"`
AddedAt string `json:"added_at"`
LikeCount int64 `json:"like_count"`
PlayCount int64 `json:"play_count"`
}
// duplicateGroupView is one proposal. SurvivorTrackID and SurvivorReason are
// the copy the report proposes keeping and the rule that chose it
// (library.ProposeSurvivor) — a default the merge (#3911) lets the operator
// override.
type duplicateGroupView struct {
ID string `json:"id"`
Tier string `json:"tier"`
WorstBitErrorRate *float32 `json:"worst_bit_error_rate"`
DetectedAt string `json:"detected_at"`
SurvivorTrackID string `json:"survivor_track_id"`
SurvivorReason string `json:"survivor_reason"`
Members []duplicateMemberView `json:"members"`
}
// duplicateSweepView is the latest sweep. State is "never" when none has run,
// which is what lets the page tell an empty report apart from a sweep that
// found nothing.
type duplicateSweepView struct {
State string `json:"state"`
StartedAt *string `json:"started_at"`
FinishedAt *string `json:"finished_at"`
Candidates *int32 `json:"candidates"`
GroupsFound *int32 `json:"groups_found"`
OversizeClusters *int32 `json:"oversize_clusters"`
ErrorMessage *string `json:"error_message"`
}
// adminDuplicatesResponse is the paged report. Total counts groups.
type adminDuplicatesResponse struct {
Sweep duplicateSweepView `json:"sweep"`
Fingerprints fingerprintCoverageResp `json:"fingerprints"`
Total int64 `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
Groups []duplicateGroupView `json:"groups"`
}
// handleListDuplicates implements GET /api/admin/library/duplicates (#3912).
//
// Read-only. The sweep's state and the fingerprint backfill's progress travel
// with the groups because an empty report means three different things — still
// fingerprinting, never swept, or swept and clean — and the page has to say which.
func (h *handlers) handleListDuplicates(w http.ResponseWriter, r *http.Request) {
limit, offset, err := parsePaging(r.URL.Query())
if err != nil {
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_paging")
return
}
ctx := r.Context()
q := dbq.New(h.pool)
sweep := duplicateSweepView{State: "never"}
last, err := q.GetLatestDuplicateSweep(ctx)
switch {
case err == nil:
sweep = duplicateSweepViewOf(last)
case !errors.Is(err, pgx.ErrNoRows):
h.logger.Error("admin: latest duplicate sweep", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
cov, err := library.FingerprintCoverage(ctx, h.pool)
if err != nil {
h.logger.Error("admin: fingerprint coverage", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
total, err := q.CountPendingDuplicateGroups(ctx)
if err != nil {
h.logger.Error("admin: count duplicate groups", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
rows, err := q.ListPendingDuplicateGroupMembers(ctx, dbq.ListPendingDuplicateGroupMembersParams{
PageLimit: int32(limit), PageOffset: int32(offset),
})
if err != nil {
h.logger.Error("admin: list duplicate groups", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
writeJSON(w, http.StatusOK, adminDuplicatesResponse{
Sweep: sweep,
Fingerprints: fingerprintCoverageResp{
Total: cov.Total, Fingerprinted: cov.Fingerprinted, Rejected: cov.Rejected, Pending: cov.Pending,
},
Total: total,
Limit: limit,
Offset: offset,
Groups: foldDuplicateGroups(rows),
})
}
func duplicateSweepViewOf(s dbq.DuplicateSweep) duplicateSweepView {
v := duplicateSweepView{
State: "running",
Candidates: s.Candidates,
GroupsFound: s.GroupsFound,
OversizeClusters: s.OversizeClusters,
ErrorMessage: s.ErrorMessage,
}
started := formatTimestamp(s.StartedAt)
v.StartedAt = &started
if s.FinishedAt.Valid {
finished := formatTimestamp(s.FinishedAt)
v.FinishedAt = &finished
v.State = "finished"
}
return v
}
// foldDuplicateGroups folds the one-row-per-member query result into groups and
// proposes each group's survivor. It relies on the query ordering members of a
// group together, so a run-length fold is enough and the page order holds.
func foldDuplicateGroups(rows []dbq.ListPendingDuplicateGroupMembersRow) []duplicateGroupView {
groups := make([]duplicateGroupView, 0, 8)
var candidates [][]library.SurvivorCandidate
for _, row := range rows {
id := uuidToString(row.GroupID)
if n := len(groups); n == 0 || groups[n-1].ID != id {
groups = append(groups, duplicateGroupView{
ID: id,
Tier: row.Tier,
WorstBitErrorRate: row.WorstBitErrorRate,
DetectedAt: formatTimestamp(row.DetectedAt),
})
candidates = append(candidates, nil)
}
n := len(groups) - 1
trackID := uuidToString(row.TrackID)
groups[n].Members = append(groups[n].Members, duplicateMemberView{
TrackID: trackID,
Title: row.Title,
ArtistName: row.ArtistName,
AlbumID: uuidToString(row.AlbumID),
AlbumTitle: row.AlbumTitle,
FilePath: row.FilePath,
FileFormat: row.FileFormat,
FileSize: row.FileSize,
DurationSec: row.DurationMs / 1000,
AddedAt: formatTimestamp(row.AddedAt),
LikeCount: row.LikeCount,
PlayCount: row.PlayCount,
})
candidates[n] = append(candidates[n], library.SurvivorCandidate{
TrackID: trackID, FileFormat: row.FileFormat, FileSize: row.FileSize, AddedAt: row.AddedAt.Time,
})
}
for i := range groups {
groups[i].SurvivorTrackID, groups[i].SurvivorReason = library.ProposeSurvivor(candidates[i])
}
return groups
}
// handleRunDuplicateSweep implements POST /api/admin/library/duplicates/sweep:
// 202 when a sweep starts, 409 sweep_in_progress when one is already running.
// The sweep outlives the request, so it runs on a background context, as
// handleTriggerScan's scan does.
func (h *handlers) handleRunDuplicateSweep(w http.ResponseWriter, _ *http.Request) {
started, err := library.TryStartDuplicateSweep(
context.Background(), h.pool, h.logger.With("source", "manual"),
)
if err != nil {
h.logger.Error("admin: start duplicate sweep", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
if !started {
writeAdminJSONErr(w, http.StatusConflict, "sweep_in_progress")
return
}
writeJSON(w, http.StatusAccepted, map[string]bool{"started": true})
}
// handleDismissDuplicateGroup implements POST
// /api/admin/library/duplicates/{id}/dismiss: "these are not duplicates". The
// sweep keeps the dismissal and will not propose that set of tracks again. 404
// duplicate_group_not_pending when the group was already resolved or is gone.
func (h *handlers) handleDismissDuplicateGroup(w http.ResponseWriter, r *http.Request) {
id, ok := parseUUID(chi.URLParam(r, "id"))
if !ok {
writeAdminJSONErr(w, http.StatusBadRequest, "invalid_id")
return
}
n, err := dbq.New(h.pool).DismissDuplicateGroup(r.Context(), id)
if err != nil {
h.logger.Error("admin: dismiss duplicate group", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
return
}
if n == 0 {
writeAdminJSONErr(w, http.StatusNotFound, "duplicate_group_not_pending")
return
}
writeJSON(w, http.StatusOK, map[string]string{"status": "dismissed"})
}
+65
View File
@@ -0,0 +1,65 @@
package api
import (
"testing"
"time"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
func dupUUID(b byte) pgtype.UUID {
var u pgtype.UUID
u.Bytes[15] = b
u.Valid = true
return u
}
func dupTS(t time.Time) pgtype.Timestamptz { return pgtype.Timestamptz{Time: t, Valid: true} }
// Rows arrive one per member, members of a group together. The fold must keep
// groups apart, keep the query's order, and propose each group's survivor from
// its own members only.
func TestFoldDuplicateGroups(t *testing.T) {
older := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
newer := older.Add(48 * time.Hour)
ber := float32(0.04)
rows := []dbq.ListPendingDuplicateGroupMembersRow{
// Group 1: identical audio, sizes tie, the older copy should be kept.
{GroupID: dupUUID(1), Tier: "exact", DetectedAt: dupTS(newer), TrackID: dupUUID(10),
Title: "WWW", FileFormat: "mp3", FileSize: 6_900_000, DurationMs: 215_400, AddedAt: dupTS(newer), PlayCount: 3},
{GroupID: dupUUID(1), Tier: "exact", DetectedAt: dupTS(newer), TrackID: dupUUID(11),
Title: "WWW", FileFormat: "mp3", FileSize: 6_900_000, DurationMs: 215_400, AddedAt: dupTS(older), LikeCount: 1},
// Group 2: the same recording, FLAC against MP3.
{GroupID: dupUUID(2), Tier: "acoustic", WorstBitErrorRate: &ber, DetectedAt: dupTS(older), TrackID: dupUUID(20),
Title: "Lovesick", FileFormat: "mp3", FileSize: 9_000_000, DurationMs: 198_000, AddedAt: dupTS(older)},
{GroupID: dupUUID(2), Tier: "acoustic", WorstBitErrorRate: &ber, DetectedAt: dupTS(older), TrackID: dupUUID(21),
Title: "Lovesick", FileFormat: "flac", FileSize: 30_000_000, DurationMs: 198_000, AddedAt: dupTS(newer)},
}
got := foldDuplicateGroups(rows)
if len(got) != 2 {
t.Fatalf("folded %d groups, want 2", len(got))
}
g1, g2 := got[0], got[1]
if g1.ID != uuidToString(dupUUID(1)) || len(g1.Members) != 2 || g1.WorstBitErrorRate != nil {
t.Fatalf("group 1 = %+v, want the exact pair with no score", g1)
}
if g1.SurvivorTrackID != uuidToString(dupUUID(11)) || g1.SurvivorReason != "in the library longest" {
t.Errorf("group 1 survivor = (%s, %q), want the older copy", g1.SurvivorTrackID, g1.SurvivorReason)
}
if g1.Members[0].DurationSec != 215 || g1.Members[0].PlayCount != 3 || g1.Members[1].LikeCount != 1 {
t.Errorf("group 1 members lost their facts: %+v", g1.Members)
}
if g2.Tier != "acoustic" || g2.WorstBitErrorRate == nil || *g2.WorstBitErrorRate != ber {
t.Fatalf("group 2 = %+v, want the acoustic pair with its score", g2)
}
// Chosen from group 2's own members: a survivor leaking across groups is
// exactly what a wrong fold boundary would produce.
if g2.SurvivorTrackID != uuidToString(dupUUID(21)) || g2.SurvivorReason != "lossless (flac)" {
t.Errorf("group 2 survivor = (%s, %q), want the FLAC copy", g2.SurvivorTrackID, g2.SurvivorReason)
}
}
+5
View File
@@ -216,6 +216,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
admin.Get("/library/coverage", h.handleGetLibraryCoverage)
admin.Get("/library/fingerprints", h.handleGetFingerprintCoverage)
// Duplicates report (#3912): proposals from the duplicate sweep, a
// trigger to sweep now, and dismissal. Nothing here merges or deletes.
admin.Get("/library/duplicates", h.handleListDuplicates)
admin.Post("/library/duplicates/sweep", h.handleRunDuplicateSweep)
admin.Post("/library/duplicates/{id}/dismiss", h.handleDismissDuplicateGroup)
admin.Get("/invites", h.handleListInvites)
admin.Post("/invites", h.handleCreateInvite)
+131
View File
@@ -27,6 +27,23 @@ func (q *Queries) AddDuplicateGroupMember(ctx context.Context, arg AddDuplicateG
return err
}
const countPendingDuplicateGroups = `-- name: CountPendingDuplicateGroups :one
SELECT count(*)::bigint
FROM duplicate_groups g
WHERE g.status = 'pending'
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2
`
// Proposals awaiting review. A group left with one member — its other tracks
// deleted since the sweep — is no proposal at all and is not counted; the next
// sweep retires it.
func (q *Queries) CountPendingDuplicateGroups(ctx context.Context) (int64, error) {
row := q.db.QueryRow(ctx, countPendingDuplicateGroups)
var column_1 int64
err := row.Scan(&column_1)
return column_1, err
}
const deleteStalePendingDuplicateGroups = `-- name: DeleteStalePendingDuplicateGroups :execrows
DELETE FROM duplicate_groups g
WHERE g.status = 'pending'
@@ -51,6 +68,22 @@ func (q *Queries) DeleteStalePendingDuplicateGroups(ctx context.Context, sweepID
return result.RowsAffected(), nil
}
const dismissDuplicateGroup = `-- name: DismissDuplicateGroup :execrows
UPDATE duplicate_groups
SET status = 'dismissed', resolved_at = now()
WHERE id = $1 AND status = 'pending'
`
// "These are not duplicates." Only a pending group can be dismissed; zero rows
// means it was already resolved or no longer exists.
func (q *Queries) DismissDuplicateGroup(ctx context.Context, id pgtype.UUID) (int64, error) {
result, err := q.db.Exec(ctx, dismissDuplicateGroup, id)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const finishDuplicateSweep = `-- name: FinishDuplicateSweep :exec
UPDATE duplicate_sweeps
SET finished_at = now(),
@@ -270,6 +303,104 @@ func (q *Queries) ListExactDuplicateHashes(ctx context.Context, currentVersion i
return items, nil
}
const listPendingDuplicateGroupMembers = `-- name: ListPendingDuplicateGroupMembers :many
WITH page AS (
SELECT g.id, g.tier, g.worst_bit_error_rate, g.detected_at
FROM duplicate_groups g
WHERE g.status = 'pending'
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2
ORDER BY g.detected_at DESC, g.id
LIMIT $2 OFFSET $1
)
SELECT p.id AS group_id,
p.tier,
p.worst_bit_error_rate,
p.detected_at,
t.id AS track_id,
t.title,
artists.name AS artist_name,
albums.id AS album_id,
albums.title AS album_title,
t.file_path,
t.file_format,
t.file_size,
t.duration_ms,
t.added_at,
(SELECT count(*) FROM general_likes l WHERE l.track_id = t.id)::bigint AS like_count,
(SELECT count(*) FROM play_events e WHERE e.track_id = t.id)::bigint AS play_count
FROM page p
JOIN duplicate_group_members m ON m.group_id = p.id
JOIN tracks t ON t.id = m.track_id
JOIN albums ON albums.id = t.album_id
JOIN artists ON artists.id = t.artist_id
ORDER BY p.detected_at DESC, p.id, t.id
`
type ListPendingDuplicateGroupMembersParams struct {
PageOffset int32
PageLimit int32
}
type ListPendingDuplicateGroupMembersRow struct {
GroupID pgtype.UUID
Tier string
WorstBitErrorRate *float32
DetectedAt pgtype.Timestamptz
TrackID pgtype.UUID
Title string
ArtistName string
AlbumID pgtype.UUID
AlbumTitle string
FilePath string
FileFormat string
FileSize int64
DurationMs int32
AddedAt pgtype.Timestamptz
LikeCount int64
PlayCount int64
}
// One page of proposals, newest first, flattened to one row per member so the
// handler folds them without a query per group. What each copy carries — likes
// and plays from every user — is here because it is what the operator weighs
// when deciding which copy to keep.
func (q *Queries) ListPendingDuplicateGroupMembers(ctx context.Context, arg ListPendingDuplicateGroupMembersParams) ([]ListPendingDuplicateGroupMembersRow, error) {
rows, err := q.db.Query(ctx, listPendingDuplicateGroupMembers, arg.PageOffset, arg.PageLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListPendingDuplicateGroupMembersRow
for rows.Next() {
var i ListPendingDuplicateGroupMembersRow
if err := rows.Scan(
&i.GroupID,
&i.Tier,
&i.WorstBitErrorRate,
&i.DetectedAt,
&i.TrackID,
&i.Title,
&i.ArtistName,
&i.AlbumID,
&i.AlbumTitle,
&i.FilePath,
&i.FileFormat,
&i.FileSize,
&i.DurationMs,
&i.AddedAt,
&i.LikeCount,
&i.PlayCount,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const startDuplicateSweep = `-- name: StartDuplicateSweep :one
INSERT INTO duplicate_sweeps DEFAULT VALUES RETURNING id, started_at
`
@@ -0,0 +1 @@
DROP INDEX IF EXISTS play_events_track_idx;
@@ -0,0 +1,8 @@
-- 0060_play_events_track_index.up.sql — play_events by track (Scribe #3912, #3911).
--
-- play_events is indexed by (user_id, started_at) and (user_id, track_id), both
-- led by user. Nothing reached it by track alone until the duplicates report,
-- which shows each copy's play count — a scan of the whole table per copy — and
-- the merge (#3911), which repoints a duplicate's play history onto the copy
-- being kept. Both ask "every play of this track", whoever played it.
CREATE INDEX play_events_track_idx ON play_events (track_id);
+52
View File
@@ -98,3 +98,55 @@ DELETE FROM duplicate_groups g
AND (g.last_seen_sweep_id IS NULL
OR (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = g.last_seen_sweep_id)
< (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = sqlc.arg(sweep_id)));
-- name: CountPendingDuplicateGroups :one
-- Proposals awaiting review. A group left with one member — its other tracks
-- deleted since the sweep — is no proposal at all and is not counted; the next
-- sweep retires it.
SELECT count(*)::bigint
FROM duplicate_groups g
WHERE g.status = 'pending'
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2;
-- name: ListPendingDuplicateGroupMembers :many
-- One page of proposals, newest first, flattened to one row per member so the
-- handler folds them without a query per group. What each copy carries — likes
-- and plays from every user — is here because it is what the operator weighs
-- when deciding which copy to keep.
WITH page AS (
SELECT g.id, g.tier, g.worst_bit_error_rate, g.detected_at
FROM duplicate_groups g
WHERE g.status = 'pending'
AND (SELECT count(*) FROM duplicate_group_members m WHERE m.group_id = g.id) >= 2
ORDER BY g.detected_at DESC, g.id
LIMIT sqlc.arg(page_limit) OFFSET sqlc.arg(page_offset)
)
SELECT p.id AS group_id,
p.tier,
p.worst_bit_error_rate,
p.detected_at,
t.id AS track_id,
t.title,
artists.name AS artist_name,
albums.id AS album_id,
albums.title AS album_title,
t.file_path,
t.file_format,
t.file_size,
t.duration_ms,
t.added_at,
(SELECT count(*) FROM general_likes l WHERE l.track_id = t.id)::bigint AS like_count,
(SELECT count(*) FROM play_events e WHERE e.track_id = t.id)::bigint AS play_count
FROM page p
JOIN duplicate_group_members m ON m.group_id = p.id
JOIN tracks t ON t.id = m.track_id
JOIN albums ON albums.id = t.album_id
JOIN artists ON artists.id = t.artist_id
ORDER BY p.detected_at DESC, p.id, t.id;
-- name: DismissDuplicateGroup :execrows
-- "These are not duplicates." Only a pending group can be dismissed; zero rows
-- means it was already resolved or no longer exists.
UPDATE duplicate_groups
SET status = 'dismissed', resolved_at = now()
WHERE id = sqlc.arg(id) AND status = 'pending';
+77
View File
@@ -0,0 +1,77 @@
package library
import (
"sort"
"strings"
"time"
)
// SurvivorCandidate is what choosing which copy to keep needs to know about one
// member of a duplicate group.
type SurvivorCandidate struct {
TrackID string
FileFormat string
FileSize int64
AddedAt time.Time
}
// losslessFormats are the scanned extensions that are lossless by definition.
// m4a is left out on purpose: it holds either ALAC or AAC, and the scanner
// records only the extension, so calling it lossless would sometimes prefer an
// AAC copy over a FLAC one.
var losslessFormats = map[string]bool{"flac": true, "wav": true}
// ProposeSurvivor picks which copy of a duplicate group to keep, and gives the
// reason in words the operator reads beside it. It is a default, not a verdict:
// the report shows it and the merge (#3911) lets the operator choose another.
//
// In order:
// 1. lossless over lossy — the one difference no later step can recover
// 2. the larger file — for one recording at one duration that is the higher
// bitrate. The scanner does not record bitrate (tracks.bitrate is never
// filled), so file size is the signal that actually exists
// 3. the copy in the library longest — the one most likely to carry the play
// history and likes, so the merge moves the least
// 4. the lowest track id, so the choice is stable between page loads
func ProposeSurvivor(cands []SurvivorCandidate) (trackID, reason string) {
if len(cands) == 0 {
return "", ""
}
ranked := append([]SurvivorCandidate(nil), cands...)
sort.SliceStable(ranked, func(i, j int) bool { return survivorBefore(ranked[i], ranked[j]) })
best := ranked[0]
if len(ranked) == 1 {
return best.TrackID, "the only copy"
}
// The reason names the first rule that separated the best copy from the
// runner-up — the rule that actually decided, not every rule it passed.
next := ranked[1]
switch {
case isLossless(best) != isLossless(next):
return best.TrackID, "lossless (" + strings.ToLower(best.FileFormat) + ")"
case best.FileSize != next.FileSize:
return best.TrackID, "largest file"
case !best.AddedAt.Equal(next.AddedAt):
return best.TrackID, "in the library longest"
default:
return best.TrackID, "copies are otherwise identical"
}
}
func survivorBefore(a, b SurvivorCandidate) bool {
if isLossless(a) != isLossless(b) {
return isLossless(a)
}
if a.FileSize != b.FileSize {
return a.FileSize > b.FileSize
}
if !a.AddedAt.Equal(b.AddedAt) {
return a.AddedAt.Before(b.AddedAt)
}
return a.TrackID < b.TrackID
}
func isLossless(c SurvivorCandidate) bool {
return losslessFormats[strings.ToLower(c.FileFormat)]
}
@@ -0,0 +1,91 @@
package library
import (
"testing"
"time"
)
func TestProposeSurvivor(t *testing.T) {
older := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
newer := older.Add(24 * time.Hour)
cases := []struct {
name string
cands []SurvivorCandidate
wantID string
wantReason string
}{
{
// Lossless wins even against a much larger lossy file, and even
// when the lossy copy has been in the library longer.
name: "lossless beats larger and older",
cands: []SurvivorCandidate{
{TrackID: "mp3", FileFormat: "mp3", FileSize: 90_000_000, AddedAt: older},
{TrackID: "flac", FileFormat: "FLAC", FileSize: 30_000_000, AddedAt: newer},
},
wantID: "flac", wantReason: "lossless (flac)",
},
{
// m4a may be AAC; it must not outrank an mp3 just for being m4a.
name: "m4a is not treated as lossless",
cands: []SurvivorCandidate{
{TrackID: "m4a", FileFormat: "m4a", FileSize: 5_000_000, AddedAt: older},
{TrackID: "mp3", FileFormat: "mp3", FileSize: 9_000_000, AddedAt: newer},
},
wantID: "mp3", wantReason: "largest file",
},
{
name: "larger file wins among lossy copies",
cands: []SurvivorCandidate{
{TrackID: "128k", FileFormat: "mp3", FileSize: 3_400_000, AddedAt: older},
{TrackID: "320k", FileFormat: "mp3", FileSize: 8_600_000, AddedAt: newer},
},
wantID: "320k", wantReason: "largest file",
},
{
// The #3885 pair: identical audio, sizes equal but for the tags.
name: "the longest-standing copy wins when size ties",
cands: []SurvivorCandidate{
{TrackID: "www-02", FileFormat: "mp3", FileSize: 6_900_000, AddedAt: newer},
{TrackID: "www-01", FileFormat: "mp3", FileSize: 6_900_000, AddedAt: older},
},
wantID: "www-01", wantReason: "in the library longest",
},
{
name: "a full tie falls back to the lowest id, stably",
cands: []SurvivorCandidate{
{TrackID: "b", FileFormat: "mp3", FileSize: 1, AddedAt: older},
{TrackID: "a", FileFormat: "mp3", FileSize: 1, AddedAt: older},
},
wantID: "a", wantReason: "copies are otherwise identical",
},
{
name: "one copy",
cands: []SurvivorCandidate{{TrackID: "only", FileFormat: "mp3", FileSize: 1, AddedAt: older}},
wantID: "only", wantReason: "the only copy",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
id, reason := ProposeSurvivor(tc.cands)
if id != tc.wantID || reason != tc.wantReason {
t.Fatalf("ProposeSurvivor = (%q, %q), want (%q, %q)", id, reason, tc.wantID, tc.wantReason)
}
})
}
}
// The reason must name the rule that decided. Across three copies that is the
// comparison between first and second place, not the first rule any pair
// differs on: here the lossy copy differs from the others by format, but the
// two FLACs are separated by size.
func TestProposeSurvivor_ReasonIsTheDecidingRule(t *testing.T) {
at := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
id, reason := ProposeSurvivor([]SurvivorCandidate{
{TrackID: "mp3", FileFormat: "mp3", FileSize: 99_000_000, AddedAt: at},
{TrackID: "flac-small", FileFormat: "flac", FileSize: 20_000_000, AddedAt: at},
{TrackID: "flac-big", FileFormat: "flac", FileSize: 40_000_000, AddedAt: at},
})
if id != "flac-big" || reason != "largest file" {
t.Fatalf("got (%q, %q), want (flac-big, largest file)", id, reason)
}
}
+30
View File
@@ -0,0 +1,30 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { dismissDuplicateGroup, listDuplicates, runDuplicateSweep } from './admin';
vi.mock('./client', () => ({
api: { get: vi.fn(), post: vi.fn() }
}));
import { api } from './client';
describe('admin duplicates API', () => {
beforeEach(() => vi.clearAllMocks());
it('listDuplicates GETs the paged report', async () => {
(api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce({ groups: [] });
await listDuplicates(25, 25);
expect(api.get).toHaveBeenCalledWith('/api/admin/library/duplicates?limit=25&offset=25');
});
it('runDuplicateSweep POSTs the trigger', async () => {
(api.post as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce({ started: true });
await runDuplicateSweep();
expect(api.post).toHaveBeenCalledWith('/api/admin/library/duplicates/sweep', {});
});
it('dismissDuplicateGroup POSTs to the group', async () => {
(api.post as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(undefined);
await dismissDuplicateGroup('g/1');
expect(api.post).toHaveBeenCalledWith('/api/admin/library/duplicates/g%2F1/dismiss', {});
});
});
+32
View File
@@ -4,6 +4,7 @@ import { qk } from './queries';
import type {
ActionResult,
AdminMissingResponse,
AdminDuplicatesResponse,
AdminPlaybackError,
AdminQuarantineRow,
LidarrConfig,
@@ -695,6 +696,37 @@ export async function updateNetworkSettings(hops: number): Promise<NetworkSettin
});
}
// Duplicates report (#3912) -------------------------------------------------
export async function listDuplicates(
offset: number = 0,
limit: number = 25
): Promise<AdminDuplicatesResponse> {
return api.get<AdminDuplicatesResponse>(
`/api/admin/library/duplicates?limit=${limit}&offset=${offset}`
);
}
// Takes a plain offset, like createMissingFilesQuery, so a $derived caller
// re-creates the query on paging. Polls while a sweep might be running: the
// page is where the operator waits for one to finish.
export function createDuplicatesQuery(offset: number = 0, limit: number = 25) {
return createQuery({
queryKey: qk.adminDuplicates(offset),
queryFn: () => listDuplicates(offset, limit),
staleTime: 30_000,
refetchInterval: 15_000
});
}
export async function runDuplicateSweep(): Promise<{ started: boolean }> {
return api.post<{ started: boolean }>('/api/admin/library/duplicates/sweep', {});
}
export async function dismissDuplicateGroup(id: string): Promise<void> {
await api.post(`/api/admin/library/duplicates/${encodeURIComponent(id)}/dismiss`, {});
}
// Missing files (#2527) -----------------------------------------------------
export async function listMissingFiles(
+2
View File
@@ -56,6 +56,8 @@ export const qk = {
['adminDiagnostics', f] as const,
adminMissingFiles: (offset?: number) =>
['adminMissingFiles', { offset: offset ?? 0 }] as const,
adminDuplicates: (offset?: number) =>
['adminDuplicates', { offset: offset ?? 0 }] as const,
adminDiagnosticDevices: (userId?: string) =>
['adminDiagnosticDevices', { userId: userId ?? 'all' }] as const,
smtpConfig: () => ['smtpConfig'] as const,
+50
View File
@@ -418,3 +418,53 @@ export type AdminMissingResponse = {
offset: number;
groups: AdminMissingGroup[];
};
// Duplicates report (#3912) -------------------------------------------------
// One copy in a proposed duplicate group. like_count and play_count cover every
// user: they are what the operator weighs when choosing which copy to keep.
export type AdminDuplicateMember = {
track_id: string;
title: string;
artist_name: string;
album_id: string;
album_title: string;
file_path: string;
file_format: string;
file_size: number;
duration_sec: number;
added_at: string;
like_count: number;
play_count: number;
};
// exact: identical encoded audio. acoustic: the same recording, differently
// encoded; worst_bit_error_rate is the weakest link between any two members.
export type AdminDuplicateGroup = {
id: string;
tier: 'exact' | 'acoustic';
worst_bit_error_rate: number | null;
detected_at: string;
survivor_track_id: string;
survivor_reason: string;
members: AdminDuplicateMember[];
};
export type AdminDuplicateSweep = {
state: 'never' | 'running' | 'finished';
started_at: string | null;
finished_at: string | null;
candidates: number | null;
groups_found: number | null;
oversize_clusters: number | null;
error_message: string | null;
};
export type AdminDuplicatesResponse = {
sweep: AdminDuplicateSweep;
fingerprints: { total: number; fingerprinted: number; rejected: number; pending: number };
total: number;
limit: number;
offset: number;
groups: AdminDuplicateGroup[];
};
+1
View File
@@ -9,6 +9,7 @@
{ href: '/admin/requests', label: 'Requests' },
{ href: '/admin/quarantine', label: 'Quarantine' },
{ href: '/admin/missing-files', label: 'Missing files' },
{ href: '/admin/duplicates', label: 'Duplicates' },
{ href: '/admin/playback-errors', label: 'Playback errors' },
{ href: '/admin/diagnostics', label: 'Diagnostics' },
{ href: '/admin/tuning', label: 'Tuning' },
+4 -1
View File
@@ -52,7 +52,7 @@ describe('AdminTabs', () => {
);
});
test('renders all nine tabs in order', () => {
test('renders all ten tabs in order', () => {
state.pageUrl = new URL('http://localhost/admin');
render(AdminTabs);
const links = screen.getAllByRole('link');
@@ -64,6 +64,9 @@ describe('AdminTabs', () => {
// Missing files sits with Quarantine and Playback errors: the three
// surfaces that show tracks needing an operator's attention.
'Missing files',
// Duplicates follows Missing files: both are library-health reports on
// what the library holds, rather than a queue of user reports.
'Duplicates',
'Playback errors',
'Diagnostics',
'Tuning',
+2
View File
@@ -42,6 +42,8 @@
"track_not_found": "That track no longer exists.",
"library_not_writable": "The music library isn't writable by the server.",
"file_delete_failed": "The file couldn't be deleted.",
"sweep_in_progress": "A duplicate sweep is already running.",
"duplicate_group_not_pending": "That group has already been resolved.",
"album_not_found": "That album no longer exists.",
"artist_not_found": "That artist no longer exists.",
"playlist_not_found": "That playlist no longer exists.",
@@ -0,0 +1,251 @@
<script lang="ts">
import { pageTitle } from '$lib/branding';
import { Copy } from 'lucide-svelte';
import {
createDuplicatesQuery,
runDuplicateSweep,
dismissDuplicateGroup
} from '$lib/api/admin';
import { errMessage } from '$lib/api/errors';
import { pushToast } from '$lib/stores/toast.svelte';
import { relativeTime } from '$lib/utils/relativeTime';
import type { AdminDuplicateGroup, AdminDuplicateMember } from '$lib/api/types';
// Tracks the duplicate sweep believes hold one recording (#3912). A group is a
// proposal: nothing here deletes or merges. Dismissing one says "these are not
// duplicates", and the sweep will not propose that set again.
const PAGE_SIZE = 25;
let offset = $state(0);
let sweeping = $state(false);
let dismissing = $state<string | null>(null);
const queryStore = $derived(createDuplicatesQuery(offset, PAGE_SIZE));
const query = $derived($queryStore);
const data = $derived(query.data);
const groups = $derived((data?.groups ?? []) as AdminDuplicateGroup[]);
const total = $derived(data?.total ?? 0);
const hasMore = $derived(offset + groups.length < total);
const sweep = $derived(data?.sweep);
const prints = $derived(data?.fingerprints);
async function onRunSweep() {
sweeping = true;
try {
await runDuplicateSweep();
pushToast('Duplicate sweep started.');
query.refetch();
} catch (e: unknown) {
pushToast(errMessage(e), 'error');
} finally {
sweeping = false;
}
}
async function onDismiss(group: AdminDuplicateGroup) {
dismissing = group.id;
try {
await dismissDuplicateGroup(group.id);
pushToast('Marked as not duplicates.');
query.refetch();
} catch (e: unknown) {
pushToast(errMessage(e), 'error');
} finally {
dismissing = null;
}
}
// "Identical audio" and "same recording" are different claims, and an
// operator deciding whether to merge needs to know which one they are
// looking at before anything else.
function tierLabel(g: AdminDuplicateGroup): string {
if (g.tier === 'exact') return 'Identical audio';
const match = Math.round((1 - (g.worst_bit_error_rate ?? 0)) * 100);
return `Same recording · ${match}% match`;
}
function sizeLabel(bytes: number): string {
return `${(bytes / 1_000_000).toFixed(1)} MB`;
}
function durationLabel(sec: number): string {
const m = Math.floor(sec / 60);
const s = sec % 60;
return `${m}:${s.toString().padStart(2, '0')}`;
}
function countLabel(n: number, one: string, many: string): string {
return n === 1 ? `1 ${one}` : `${n} ${many}`;
}
// What the copy carries is the fact that decides which to keep.
function historyLabel(m: AdminDuplicateMember): string {
if (m.like_count === 0 && m.play_count === 0) return 'no likes or plays';
return `${countLabel(m.like_count, 'like', 'likes')} · ${countLabel(m.play_count, 'play', 'plays')}`;
}
</script>
<svelte:head><title>{pageTitle('Admin · Duplicates')}</title></svelte:head>
<div class="space-y-6">
<header class="space-y-1">
<div class="flex items-center justify-between gap-4">
<div class="flex items-center gap-2">
<h2 class="font-display text-2xl font-medium text-text-primary">Duplicates</h2>
{#if total > 0}
<span
class="inline-flex items-center rounded-full bg-accent-tint px-2 py-0.5 text-xs text-accent"
data-testid="duplicates-count-pill"
>
{total}
</span>
{/if}
</div>
<button
type="button"
onclick={onRunSweep}
disabled={sweeping || sweep?.state === 'running'}
class="flex h-8 items-center gap-1 rounded-md bg-action-primary px-4 text-sm text-action-fg hover:opacity-90 disabled:opacity-50"
>
{sweeping ? 'Starting…' : 'Sweep now'}
</button>
</div>
<p class="text-text-secondary">
Tracks that hold the same recording more than once. Review each group; nothing is
merged or removed from here.
</p>
</header>
{#if sweep}
<p class="text-sm text-text-secondary" data-testid="sweep-status">
{#if sweep.state === 'never'}
The duplicate sweep hasn't run yet.
{:else if sweep.state === 'running'}
Sweeping now — started {relativeTime(sweep.started_at ?? '')}.
{:else}
Last swept {relativeTime(sweep.finished_at ?? '')}, comparing
{(sweep.candidates ?? 0).toLocaleString()} tracks.
{#if sweep.error_message}
<span class="text-oxblood">It stopped early: {sweep.error_message}</span>
{/if}
{/if}
{#if prints && prints.pending > 0}
{prints.pending.toLocaleString()} tracks are still waiting for a fingerprint and join
the comparison once they have one.
{/if}
</p>
{/if}
{#if query.isPending}
<p class="text-text-secondary">Loading duplicates…</p>
{:else if query.isError}
<p class="text-error">Couldn't load the duplicates report.</p>
{:else if groups.length === 0}
<!-- Three states all show zero groups and mean different things. Telling
them apart is what stops an empty page reading as a broken feature. -->
<div class="rounded-lg border border-border bg-surface p-6 text-center" data-testid="empty-state">
<Copy size={28} strokeWidth={1} class="mx-auto text-text-muted" />
{#if prints && prints.total > 0 && prints.fingerprinted === 0}
<p class="mt-3 text-text-primary">Nothing to compare yet.</p>
<p class="mt-1 text-sm text-text-secondary">
The library is still being fingerprinted — {prints.pending.toLocaleString()} tracks to go.
Duplicates appear here as the sweep finds them.
</p>
{:else if sweep?.state === 'never'}
<p class="mt-3 text-text-primary">The sweep hasn't run yet.</p>
<p class="mt-1 text-sm text-text-secondary">
It runs on its own whenever new fingerprints arrive, or now if you start it.
</p>
{:else if sweep?.state === 'running'}
<p class="mt-3 text-text-primary">Sweeping…</p>
<p class="mt-1 text-sm text-text-secondary">Anything it finds will appear here.</p>
{:else}
<p class="mt-3 text-text-primary">No duplicates found.</p>
<p class="mt-1 text-sm text-text-secondary">
A group appears when two tracks hold identical audio, or the same recording in a
different encoding. Groups you dismiss don't come back.
</p>
{/if}
</div>
{:else}
<ul class="space-y-4">
{#each groups as group (group.id)}
<li class="overflow-hidden rounded-lg border border-border bg-surface" data-testid="duplicate-group">
<div class="flex items-center justify-between gap-4 border-b border-border px-4 py-3">
<div class="min-w-0">
<h3 class="text-sm text-text-primary" data-testid="duplicate-tier">{tierLabel(group)}</h3>
<p class="text-xs text-text-secondary">Found {relativeTime(group.detected_at)}</p>
</div>
<button
type="button"
onclick={() => onDismiss(group)}
disabled={dismissing === group.id}
class="shrink-0 rounded-md border border-border px-3 py-1.5 text-sm text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50"
>
Not duplicates
</button>
</div>
<ul class="divide-y divide-border">
{#each group.members as m (m.track_id)}
{@const keep = m.track_id === group.survivor_track_id}
<li class="flex items-start gap-3 px-4 py-3" data-testid="duplicate-member">
<div class="min-w-0 flex-1 space-y-0.5">
<div class="flex items-center gap-2">
<span class="truncate text-sm text-text-primary">{m.title}</span>
{#if keep}
<!-- The proposed copy to keep, with the rule that chose it.
A default the merge will let the operator override. -->
<span
class="shrink-0 rounded-full bg-accent-tint px-2 py-0.5 text-xs text-accent"
data-testid="survivor-badge"
title="Proposed to keep: {group.survivor_reason}"
>
Keep · {group.survivor_reason}
</span>
{/if}
</div>
<div class="truncate text-xs text-text-secondary">
{m.artist_name} · {m.album_title}
</div>
<div class="truncate font-mono text-xs text-text-muted" title={m.file_path}>
{m.file_path}
</div>
</div>
<div class="shrink-0 space-y-0.5 text-right text-xs text-text-muted">
<div>{m.file_format.toUpperCase()} · {sizeLabel(m.file_size)} · {durationLabel(m.duration_sec)}</div>
<div data-testid="member-history">{historyLabel(m)}</div>
</div>
</li>
{/each}
</ul>
</li>
{/each}
</ul>
{#if hasMore || offset > 0}
<nav class="flex items-center justify-between" aria-label="Duplicates pages">
<button
type="button"
class="rounded border border-border px-3 py-1.5 text-sm text-text-primary hover:bg-surface-hover disabled:cursor-not-allowed disabled:opacity-40"
disabled={offset === 0}
onclick={() => (offset = Math.max(0, offset - PAGE_SIZE))}
>
Previous
</button>
<span class="text-xs text-text-secondary">
{offset + 1}{offset + groups.length} of {total}
</span>
<button
type="button"
class="rounded border border-border px-3 py-1.5 text-sm text-text-primary hover:bg-surface-hover disabled:cursor-not-allowed disabled:opacity-40"
disabled={!hasMore}
onclick={() => (offset += PAGE_SIZE)}
>
Next
</button>
</nav>
{/if}
{/if}
</div>
@@ -0,0 +1,147 @@
import { afterEach, describe, expect, test, vi } from 'vitest';
import { fireEvent, render, screen } from '@testing-library/svelte';
import { mockQuery } from '../../../test-utils/query';
import type { AdminDuplicatesResponse } from '$lib/api/types';
vi.mock('$lib/api/admin', () => ({
createDuplicatesQuery: vi.fn(),
runDuplicateSweep: vi.fn().mockResolvedValue({ started: true }),
dismissDuplicateGroup: vi.fn().mockResolvedValue(undefined)
}));
import AdminDuplicatesPage from './+page.svelte';
import { createDuplicatesQuery, dismissDuplicateGroup, runDuplicateSweep } from '$lib/api/admin';
const HOUR = 3_600_000;
const ago = (ms: number) => new Date(Date.now() - ms).toISOString();
const finishedSweep = {
state: 'finished' as const,
started_at: ago(2 * HOUR),
finished_at: ago(HOUR),
candidates: 1200,
groups_found: 1,
oversize_clusters: 0,
error_message: null
};
const allFingerprinted = { total: 1200, fingerprinted: 1200, rejected: 0, pending: 0 };
function member(id: string, extra: Partial<AdminDuplicatesResponse['groups'][number]['members'][number]> = {}) {
return {
track_id: id,
title: 'WWW',
artist_name: 'Moe Shop',
album_id: 'al-1',
album_title: 'WWW',
file_path: `/music/Moe Shop/WWW (2020)/${id}.mp3`,
file_format: 'mp3',
file_size: 6_900_000,
duration_sec: 215,
added_at: ago(500 * HOUR),
like_count: 0,
play_count: 0,
...extra
};
}
function response(over: Partial<AdminDuplicatesResponse> = {}): AdminDuplicatesResponse {
return {
sweep: finishedSweep,
fingerprints: allFingerprinted,
total: 1,
limit: 25,
offset: 0,
groups: [
{
id: 'g-1',
tier: 'exact',
worst_bit_error_rate: null,
detected_at: ago(HOUR),
survivor_track_id: 'www-01',
survivor_reason: 'in the library longest',
members: [member('www-01', { like_count: 2, play_count: 14 }), member('www-02')]
}
],
...over
};
}
function renderWith(data: AdminDuplicatesResponse | undefined) {
vi.mocked(createDuplicatesQuery).mockReturnValue(
mockQuery({ data }) as ReturnType<typeof createDuplicatesQuery>
);
return render(AdminDuplicatesPage);
}
function text(el: HTMLElement): string {
return (el.textContent ?? '').replace(/\s+/g, ' ').trim();
}
afterEach(() => vi.clearAllMocks());
describe('admin duplicates', () => {
test('shows each group with its members, what they carry, and the copy to keep', () => {
renderWith(response());
expect(screen.getAllByTestId('duplicate-group')).toHaveLength(1);
expect(screen.getAllByTestId('duplicate-member')).toHaveLength(2);
expect(text(screen.getByTestId('duplicate-tier'))).toBe('Identical audio');
expect(text(screen.getByTestId('survivor-badge'))).toBe('Keep · in the library longest');
const history = screen.getAllByTestId('member-history').map(text);
expect(history).toEqual(['2 likes · 14 plays', 'no likes or plays']);
});
// The badge must sit on the proposed survivor, not merely appear somewhere.
test('the keep badge is on the survivor row', () => {
renderWith(response());
const rows = screen.getAllByTestId('duplicate-member');
expect(rows[0].querySelector('[data-testid="survivor-badge"]')).not.toBeNull();
expect(rows[1].querySelector('[data-testid="survivor-badge"]')).toBeNull();
});
test('an acoustic group reads as a match percentage', () => {
const r = response();
r.groups[0] = { ...r.groups[0], tier: 'acoustic', worst_bit_error_rate: 0.04 };
renderWith(r);
expect(text(screen.getByTestId('duplicate-tier'))).toBe('Same recording · 96% match');
});
// Three states all show zero groups. Each must say which it is.
test('empty while still fingerprinting says so', () => {
renderWith(
response({
groups: [],
total: 0,
fingerprints: { total: 1200, fingerprinted: 0, rejected: 0, pending: 1200 }
})
);
expect(text(screen.getByTestId('empty-state'))).toContain('still being fingerprinted');
});
test('empty before any sweep says the sweep has not run', () => {
renderWith(
response({
groups: [],
total: 0,
sweep: { ...finishedSweep, state: 'never', started_at: null, finished_at: null, candidates: null }
})
);
expect(text(screen.getByTestId('empty-state'))).toContain("hasn't run yet");
});
test('empty after a sweep says no duplicates were found', () => {
renderWith(response({ groups: [], total: 0 }));
expect(text(screen.getByTestId('empty-state'))).toContain('No duplicates found');
});
test('Not duplicates dismisses that group', async () => {
renderWith(response());
await fireEvent.click(screen.getByRole('button', { name: 'Not duplicates' }));
expect(dismissDuplicateGroup).toHaveBeenCalledWith('g-1');
});
test('Sweep now starts a sweep', async () => {
renderWith(response());
await fireEvent.click(screen.getByRole('button', { name: 'Sweep now' }));
expect(runDuplicateSweep).toHaveBeenCalledTimes(1);
});
});