4481874482
Forward-fix from running the prior two commits in production. Backfill
on a real library exposed two further issues:
(1) gofmt -s flagged the column-aligned stubMeta methods in
mbids_test.go. Switched to the gofmt-canonical single-space form.
(2) Many albums failed to heal with SQLSTATE 23505 (unique constraint
albums_mbid_unique). Cause: the operator's library has duplicate
album rows in the DB — same MusicBrainz release, split into
multiple rows by the pre-MBID scanner because of subtle title or
artist disagreements between files. When backfill now correctly
extracts the MBID, two rows want to claim the same one and the
partial unique index rejects the second.
The conflict is correct DB behavior — we shouldn't have two rows
with the same MBID — but it's not a write failure to alarm the
operator about. Detect 23505 specifically:
- downgrade the log line from Warn to Info
- track these in a new BackfillMBIDsResult.Duplicates counter
(separate from Skipped, which retains its "no MBID in tag"
meaning)
- leave the duplicate row's mbid NULL; merging duplicates is a
separate (future) operator workflow
Same handling threaded through the scanner heal path so a regular
rescan doesn't generate the noise either.
JSON tally + admin UI gain a "Duplicates" line so the operator
can see how many duplicate rows their library carries.
Follow-up scope (separate task): a duplicate-album merge UX that
reparents the duplicate's tracks to the canonical row and deletes
the orphaned album row.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
153 lines
4.7 KiB
Go
153 lines
4.7 KiB
Go
package library
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
|
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
|
)
|
|
|
|
// LibraryStageTallies wires the scanner.Stats shape into the scan_runs jsonb
|
|
// column. Field names match the DB-side jsonb schema referenced by the
|
|
// admin SPA. Defined here (not on Stats directly) so the wire format
|
|
// stays stable even if the scanner adds internal counters.
|
|
type LibraryStageTallies struct {
|
|
Scanned int `json:"scanned"`
|
|
Added int `json:"added"`
|
|
Updated int `json:"updated"`
|
|
Skipped int `json:"skipped"`
|
|
Errored int `json:"errored"`
|
|
}
|
|
|
|
// MBIDBackfillStageTallies wires BackfillMBIDsResult into the scan_runs jsonb column.
|
|
type MBIDBackfillStageTallies struct {
|
|
Processed int `json:"processed"`
|
|
Healed int `json:"healed"`
|
|
Skipped int `json:"skipped"`
|
|
Duplicates int `json:"duplicates"`
|
|
}
|
|
|
|
// CoverEnrichStageTallies wires EnrichBatch tallies into the scan_runs jsonb column.
|
|
type CoverEnrichStageTallies struct {
|
|
Processed int `json:"processed"`
|
|
Succeeded int `json:"succeeded"`
|
|
Failed int `json:"failed"`
|
|
}
|
|
|
|
// RunScanConfig assembles the knobs for the orchestrator. BackfillCap and
|
|
// EnrichCap mirror the existing boot-time caps (5000 / coverArtBackfillCap)
|
|
// so manual triggers don't accidentally drain the whole library.
|
|
type RunScanConfig struct {
|
|
BackfillCap int
|
|
EnrichCap int
|
|
}
|
|
|
|
// RunScan is the unified scan-chain orchestrator. Creates a scan_runs row,
|
|
// runs file walk → MBID backfill → cover enrich in sequence, persists per-
|
|
// stage tallies as each completes, sets finished_at on the row.
|
|
//
|
|
// Errors at any stage are logged + recorded in error_message but do NOT
|
|
// halt subsequent stages — a scanner failure shouldn't prevent the cover
|
|
// enricher from picking up rows that earlier scans imported.
|
|
//
|
|
// Returns the scan_runs row id so the manual-trigger endpoint can return it.
|
|
func RunScan(
|
|
ctx context.Context,
|
|
pool *pgxpool.Pool,
|
|
scanner *Scanner,
|
|
enricher *coverart.Enricher,
|
|
logger *slog.Logger,
|
|
cfg RunScanConfig,
|
|
) (pgtype.UUID, error) {
|
|
q := dbq.New(pool)
|
|
|
|
row, err := q.InsertScanRun(ctx)
|
|
if err != nil {
|
|
return pgtype.UUID{}, fmt.Errorf("insert scan_run: %w", err)
|
|
}
|
|
logger.Info("scan run started", "id", row.ID)
|
|
|
|
var firstErr error
|
|
captureErr := func(stage string, stageErr error) {
|
|
logger.Warn("scan run stage failed", "stage", stage, "id", row.ID, "err", stageErr)
|
|
if firstErr == nil {
|
|
firstErr = fmt.Errorf("%s: %w", stage, stageErr)
|
|
}
|
|
}
|
|
|
|
// Stage 1: library file-walk scan.
|
|
if scanner != nil {
|
|
stats, serr := scanner.Scan(ctx)
|
|
if serr != nil {
|
|
captureErr("library", serr)
|
|
}
|
|
blob, _ := json.Marshal(LibraryStageTallies{
|
|
Scanned: stats.Scanned, Added: stats.Added, Updated: stats.Updated,
|
|
Skipped: stats.Skipped, Errored: stats.Errored,
|
|
})
|
|
if uerr := q.UpdateScanRunLibrary(ctx, dbq.UpdateScanRunLibraryParams{
|
|
ID: row.ID, Library: blob,
|
|
}); uerr != nil {
|
|
captureErr("library_persist", uerr)
|
|
}
|
|
}
|
|
|
|
// Stage 2: MBID backfill.
|
|
if cfg.BackfillCap != 0 {
|
|
bfRes, bferr := BackfillMBIDs(ctx, pool, logger, cfg.BackfillCap)
|
|
if bferr != nil {
|
|
captureErr("mbid_backfill", bferr)
|
|
}
|
|
blob, _ := json.Marshal(MBIDBackfillStageTallies{
|
|
Processed: bfRes.Processed, Healed: bfRes.Healed,
|
|
Skipped: bfRes.Skipped, Duplicates: bfRes.Duplicates,
|
|
})
|
|
if uerr := q.UpdateScanRunMbidBackfill(ctx, dbq.UpdateScanRunMbidBackfillParams{
|
|
ID: row.ID, MbidBackfill: blob,
|
|
}); uerr != nil {
|
|
captureErr("mbid_backfill_persist", uerr)
|
|
}
|
|
}
|
|
|
|
// Stage 3: cover enrichment.
|
|
if enricher != nil && cfg.EnrichCap != 0 {
|
|
processed, succeeded, failed, ceerr := enricher.EnrichBatch(ctx, cfg.EnrichCap)
|
|
if ceerr != nil {
|
|
captureErr("cover_enrich", ceerr)
|
|
}
|
|
blob, _ := json.Marshal(CoverEnrichStageTallies{
|
|
Processed: processed, Succeeded: succeeded, Failed: failed,
|
|
})
|
|
if uerr := q.UpdateScanRunCoverEnrich(ctx, dbq.UpdateScanRunCoverEnrichParams{
|
|
ID: row.ID, CoverEnrich: blob,
|
|
}); uerr != nil {
|
|
captureErr("cover_enrich_persist", uerr)
|
|
}
|
|
}
|
|
|
|
// Finalize: set finished_at + error message (or empty string if all stages succeeded).
|
|
errMsg := ""
|
|
if firstErr != nil {
|
|
errMsg = firstErr.Error()
|
|
}
|
|
if ferr := q.FinishScanRun(ctx, dbq.FinishScanRunParams{
|
|
ID: row.ID, Column2: errMsg,
|
|
}); ferr != nil {
|
|
// Even the finalize failed; log loudly, return the original error if any.
|
|
logger.Error("scan run finalize failed", "id", row.ID, "err", ferr)
|
|
if firstErr != nil {
|
|
return row.ID, firstErr
|
|
}
|
|
return row.ID, ferr
|
|
}
|
|
|
|
logger.Info("scan run complete", "id", row.ID, "error", errMsg)
|
|
return row.ID, firstErr
|
|
}
|