feat(admin): the duplicates report — review proposed duplicate groups (M400 #3912)

A new admin tab, Duplicates, beside Missing files: the proposals from the duplicate sweep, with a Sweep now trigger and a Not duplicates dismissal. Nothing on it merges or deletes; the merge is #3911. Each group shows: - whether it is identical audio or the same recording, with a match percentage from the weakest link between members - every copy's format, size, duration, path, and the likes and plays it carries (every user's; this is admin-only, and it is what decides which copy to keep) - the copy proposed to keep, and the rule that chose it The survivor rule is library.ProposeSurvivor, a pure function the merge will reuse: lossless over lossy, then the larger file, then the copy in the library longest, then lowest id. Bitrate is not in it because the scanner never fills tracks.bitrate, and for one recording at one duration a larger file is the higher bitrate. m4a is not counted as lossless: it may be AAC. The reason names the rule that separated first place from second, not every rule the winner passed. An empty report has three causes, and the page says which: still fingerprinting, the sweep has never run, or it ran and found nothing. The sweep's state and the backfill's progress come back with the groups for that reason. Groups left with fewer than two members since the sweep are not shown. GET /api/admin/library/duplicates, POST .../sweep (202, or 409 sweep_in_progress), POST .../{id}/dismiss (404 duplicate_group_not_pending when already resolved). Migration 0060 indexes play_events by track_id. Its only indexes led with user_id, so each copy's play count, and the merge's repointing of play history, would scan the whole table. Web only, like Missing files: Android has no library-health admin screens. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
2026-09-11 17:11:11 -04:00
parent 6379b6c31d
commit ff493a8c7d
18 changed files with 1179 additions and 1 deletions
@@ -0,0 +1,230 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/jackc/pgx/v5"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/library"
+)
+
+// duplicateMemberView is one copy in a proposed duplicate group. LikeCount and
+// PlayCount span every user: the report is admin-only, and what a copy carries
+// is the fact the operator weighs when choosing which to keep.
+type duplicateMemberView struct {
+	TrackID     string `json:"track_id"`
+	Title       string `json:"title"`
+	ArtistName  string `json:"artist_name"`
+	AlbumID     string `json:"album_id"`
+	AlbumTitle  string `json:"album_title"`
+	FilePath    string `json:"file_path"`
+	FileFormat  string `json:"file_format"`
+	FileSize    int64  `json:"file_size"`
+	DurationSec int32  `json:"duration_sec"`
+	AddedAt     string `json:"added_at"`
+	LikeCount   int64  `json:"like_count"`
+	PlayCount   int64  `json:"play_count"`
+}
+
+// duplicateGroupView is one proposal. SurvivorTrackID and SurvivorReason are
+// the copy the report proposes keeping and the rule that chose it
+// (library.ProposeSurvivor) — a default the merge (#3911) lets the operator
+// override.
+type duplicateGroupView struct {
+	ID                string                `json:"id"`
+	Tier              string                `json:"tier"`
+	WorstBitErrorRate *float32              `json:"worst_bit_error_rate"`
+	DetectedAt        string                `json:"detected_at"`
+	SurvivorTrackID   string                `json:"survivor_track_id"`
+	SurvivorReason    string                `json:"survivor_reason"`
+	Members           []duplicateMemberView `json:"members"`
+}
+
+// duplicateSweepView is the latest sweep. State is "never" when none has run,
+// which is what lets the page tell an empty report apart from a sweep that
+// found nothing.
+type duplicateSweepView struct {
+	State            string  `json:"state"`
+	StartedAt        *string `json:"started_at"`
+	FinishedAt       *string `json:"finished_at"`
+	Candidates       *int32  `json:"candidates"`
+	GroupsFound      *int32  `json:"groups_found"`
+	OversizeClusters *int32  `json:"oversize_clusters"`
+	ErrorMessage     *string `json:"error_message"`
+}
+
+// adminDuplicatesResponse is the paged report. Total counts groups.
+type adminDuplicatesResponse struct {
+	Sweep        duplicateSweepView      `json:"sweep"`
+	Fingerprints fingerprintCoverageResp `json:"fingerprints"`
+	Total        int64                   `json:"total"`
+	Limit        int                     `json:"limit"`
+	Offset       int                     `json:"offset"`
+	Groups       []duplicateGroupView    `json:"groups"`
+}
+
+// handleListDuplicates implements GET /api/admin/library/duplicates (#3912).
+//
+// Read-only. The sweep's state and the fingerprint backfill's progress travel
+// with the groups because an empty report means three different things — still
+// fingerprinting, never swept, or swept and clean — and the page has to say which.
+func (h *handlers) handleListDuplicates(w http.ResponseWriter, r *http.Request) {
+	limit, offset, err := parsePaging(r.URL.Query())
+	if err != nil {
+		writeAdminJSONErr(w, http.StatusBadRequest, "invalid_paging")
+		return
+	}
+	ctx := r.Context()
+	q := dbq.New(h.pool)
+
+	sweep := duplicateSweepView{State: "never"}
+	last, err := q.GetLatestDuplicateSweep(ctx)
+	switch {
+	case err == nil:
+		sweep = duplicateSweepViewOf(last)
+	case !errors.Is(err, pgx.ErrNoRows):
+		h.logger.Error("admin: latest duplicate sweep", "err", err)
+		writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
+		return
+	}
+
+	cov, err := library.FingerprintCoverage(ctx, h.pool)
+	if err != nil {
+		h.logger.Error("admin: fingerprint coverage", "err", err)
+		writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
+		return
+	}
+	total, err := q.CountPendingDuplicateGroups(ctx)
+	if err != nil {
+		h.logger.Error("admin: count duplicate groups", "err", err)
+		writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
+		return
+	}
+	rows, err := q.ListPendingDuplicateGroupMembers(ctx, dbq.ListPendingDuplicateGroupMembersParams{
+		PageLimit: int32(limit), PageOffset: int32(offset),
+	})
+	if err != nil {
+		h.logger.Error("admin: list duplicate groups", "err", err)
+		writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
+		return
+	}
+
+	writeJSON(w, http.StatusOK, adminDuplicatesResponse{
+		Sweep: sweep,
+		Fingerprints: fingerprintCoverageResp{
+			Total: cov.Total, Fingerprinted: cov.Fingerprinted, Rejected: cov.Rejected, Pending: cov.Pending,
+		},
+		Total:  total,
+		Limit:  limit,
+		Offset: offset,
+		Groups: foldDuplicateGroups(rows),
+	})
+}
+
+func duplicateSweepViewOf(s dbq.DuplicateSweep) duplicateSweepView {
+	v := duplicateSweepView{
+		State:            "running",
+		Candidates:       s.Candidates,
+		GroupsFound:      s.GroupsFound,
+		OversizeClusters: s.OversizeClusters,
+		ErrorMessage:     s.ErrorMessage,
+	}
+	started := formatTimestamp(s.StartedAt)
+	v.StartedAt = &started
+	if s.FinishedAt.Valid {
+		finished := formatTimestamp(s.FinishedAt)
+		v.FinishedAt = &finished
+		v.State = "finished"
+	}
+	return v
+}
+
+// foldDuplicateGroups folds the one-row-per-member query result into groups and
+// proposes each group's survivor. It relies on the query ordering members of a
+// group together, so a run-length fold is enough and the page order holds.
+func foldDuplicateGroups(rows []dbq.ListPendingDuplicateGroupMembersRow) []duplicateGroupView {
+	groups := make([]duplicateGroupView, 0, 8)
+	var candidates [][]library.SurvivorCandidate
+	for _, row := range rows {
+		id := uuidToString(row.GroupID)
+		if n := len(groups); n == 0 || groups[n-1].ID != id {
+			groups = append(groups, duplicateGroupView{
+				ID:                id,
+				Tier:              row.Tier,
+				WorstBitErrorRate: row.WorstBitErrorRate,
+				DetectedAt:        formatTimestamp(row.DetectedAt),
+			})
+			candidates = append(candidates, nil)
+		}
+		n := len(groups) - 1
+		trackID := uuidToString(row.TrackID)
+		groups[n].Members = append(groups[n].Members, duplicateMemberView{
+			TrackID:     trackID,
+			Title:       row.Title,
+			ArtistName:  row.ArtistName,
+			AlbumID:     uuidToString(row.AlbumID),
+			AlbumTitle:  row.AlbumTitle,
+			FilePath:    row.FilePath,
+			FileFormat:  row.FileFormat,
+			FileSize:    row.FileSize,
+			DurationSec: row.DurationMs / 1000,
+			AddedAt:     formatTimestamp(row.AddedAt),
+			LikeCount:   row.LikeCount,
+			PlayCount:   row.PlayCount,
+		})
+		candidates[n] = append(candidates[n], library.SurvivorCandidate{
+			TrackID: trackID, FileFormat: row.FileFormat, FileSize: row.FileSize, AddedAt: row.AddedAt.Time,
+		})
+	}
+	for i := range groups {
+		groups[i].SurvivorTrackID, groups[i].SurvivorReason = library.ProposeSurvivor(candidates[i])
+	}
+	return groups
+}
+
+// handleRunDuplicateSweep implements POST /api/admin/library/duplicates/sweep:
+// 202 when a sweep starts, 409 sweep_in_progress when one is already running.
+// The sweep outlives the request, so it runs on a background context, as
+// handleTriggerScan's scan does.
+func (h *handlers) handleRunDuplicateSweep(w http.ResponseWriter, _ *http.Request) {
+	started, err := library.TryStartDuplicateSweep(
+		context.Background(), h.pool, h.logger.With("source", "manual"),
+	)
+	if err != nil {
+		h.logger.Error("admin: start duplicate sweep", "err", err)
+		writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
+		return
+	}
+	if !started {
+		writeAdminJSONErr(w, http.StatusConflict, "sweep_in_progress")
+		return
+	}
+	writeJSON(w, http.StatusAccepted, map[string]bool{"started": true})
+}
+
+// handleDismissDuplicateGroup implements POST
+// /api/admin/library/duplicates/{id}/dismiss: "these are not duplicates". The
+// sweep keeps the dismissal and will not propose that set of tracks again. 404
+// duplicate_group_not_pending when the group was already resolved or is gone.
+func (h *handlers) handleDismissDuplicateGroup(w http.ResponseWriter, r *http.Request) {
+	id, ok := parseUUID(chi.URLParam(r, "id"))
+	if !ok {
+		writeAdminJSONErr(w, http.StatusBadRequest, "invalid_id")
+		return
+	}
+	n, err := dbq.New(h.pool).DismissDuplicateGroup(r.Context(), id)
+	if err != nil {
+		h.logger.Error("admin: dismiss duplicate group", "err", err)
+		writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
+		return
+	}
+	if n == 0 {
+		writeAdminJSONErr(w, http.StatusNotFound, "duplicate_group_not_pending")
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]string{"status": "dismissed"})
+}
@@ -0,0 +1,65 @@
+package api
+
+import (
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgtype"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+)
+
+func dupUUID(b byte) pgtype.UUID {
+	var u pgtype.UUID
+	u.Bytes[15] = b
+	u.Valid = true
+	return u
+}
+
+func dupTS(t time.Time) pgtype.Timestamptz { return pgtype.Timestamptz{Time: t, Valid: true} }
+
+// Rows arrive one per member, members of a group together. The fold must keep
+// groups apart, keep the query's order, and propose each group's survivor from
+// its own members only.
+func TestFoldDuplicateGroups(t *testing.T) {
+	older := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
+	newer := older.Add(48 * time.Hour)
+	ber := float32(0.04)
+	rows := []dbq.ListPendingDuplicateGroupMembersRow{
+		// Group 1: identical audio, sizes tie, the older copy should be kept.
+		{GroupID: dupUUID(1), Tier: "exact", DetectedAt: dupTS(newer), TrackID: dupUUID(10),
+			Title: "WWW", FileFormat: "mp3", FileSize: 6_900_000, DurationMs: 215_400, AddedAt: dupTS(newer), PlayCount: 3},
+		{GroupID: dupUUID(1), Tier: "exact", DetectedAt: dupTS(newer), TrackID: dupUUID(11),
+			Title: "WWW", FileFormat: "mp3", FileSize: 6_900_000, DurationMs: 215_400, AddedAt: dupTS(older), LikeCount: 1},
+		// Group 2: the same recording, FLAC against MP3.
+		{GroupID: dupUUID(2), Tier: "acoustic", WorstBitErrorRate: &ber, DetectedAt: dupTS(older), TrackID: dupUUID(20),
+			Title: "Lovesick", FileFormat: "mp3", FileSize: 9_000_000, DurationMs: 198_000, AddedAt: dupTS(older)},
+		{GroupID: dupUUID(2), Tier: "acoustic", WorstBitErrorRate: &ber, DetectedAt: dupTS(older), TrackID: dupUUID(21),
+			Title: "Lovesick", FileFormat: "flac", FileSize: 30_000_000, DurationMs: 198_000, AddedAt: dupTS(newer)},
+	}
+
+	got := foldDuplicateGroups(rows)
+	if len(got) != 2 {
+		t.Fatalf("folded %d groups, want 2", len(got))
+	}
+
+	g1, g2 := got[0], got[1]
+	if g1.ID != uuidToString(dupUUID(1)) || len(g1.Members) != 2 || g1.WorstBitErrorRate != nil {
+		t.Fatalf("group 1 = %+v, want the exact pair with no score", g1)
+	}
+	if g1.SurvivorTrackID != uuidToString(dupUUID(11)) || g1.SurvivorReason != "in the library longest" {
+		t.Errorf("group 1 survivor = (%s, %q), want the older copy", g1.SurvivorTrackID, g1.SurvivorReason)
+	}
+	if g1.Members[0].DurationSec != 215 || g1.Members[0].PlayCount != 3 || g1.Members[1].LikeCount != 1 {
+		t.Errorf("group 1 members lost their facts: %+v", g1.Members)
+	}
+
+	if g2.Tier != "acoustic" || g2.WorstBitErrorRate == nil || *g2.WorstBitErrorRate != ber {
+		t.Fatalf("group 2 = %+v, want the acoustic pair with its score", g2)
+	}
+	// Chosen from group 2's own members: a survivor leaking across groups is
+	// exactly what a wrong fold boundary would produce.
+	if g2.SurvivorTrackID != uuidToString(dupUUID(21)) || g2.SurvivorReason != "lossless (flac)" {
+		t.Errorf("group 2 survivor = (%s, %q), want the FLAC copy", g2.SurvivorTrackID, g2.SurvivorReason)
+	}
+}
@@ -216,6 +216,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev

 				admin.Get("/library/coverage", h.handleGetLibraryCoverage)
 				admin.Get("/library/fingerprints", h.handleGetFingerprintCoverage)
+				// Duplicates report (#3912): proposals from the duplicate sweep, a
+				// trigger to sweep now, and dismissal. Nothing here merges or deletes.
+				admin.Get("/library/duplicates", h.handleListDuplicates)
+				admin.Post("/library/duplicates/sweep", h.handleRunDuplicateSweep)
+				admin.Post("/library/duplicates/{id}/dismiss", h.handleDismissDuplicateGroup)

 				admin.Get("/invites", h.handleListInvites)
 				admin.Post("/invites", h.handleCreateInvite)