feat(server,web/m7-cover-sources): scan-orchestrator 4th stage
Adds an artist-art-enrich stage after cover-enrich in RunScan. The stage drains EnrichArtistBatch with a configurable cap and the operator's data_dir, persists processed/succeeded/failed tallies to the scan_runs.artist_art_enrich jsonb column added in migration 0018. The admin /admin Library Scan card grid expands from 3 to 4 columns with the new "Artist art" card mirroring the cover-enrichment one (Processed / Succeeded / Failed). The TS ScanStatus type gains an optional artist_art_enrich field. The Go scanStatusResp gains the matching ArtistArtEnrich json.RawMessage field. cmd/minstrel/main.go (and admin scan-trigger handler) wire the new RunScanConfig fields: ArtistEnrichCap reuses cfg.Library.CoverArtBackfillCap for now (can be split if separate budgets are useful), DataDir is cfg.Storage.DataDir. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+12
-6
@@ -100,8 +100,10 @@ func run() error {
|
|||||||
if _, err := library.RunScan(ctx, pool, scanner, coverEnricher,
|
if _, err := library.RunScan(ctx, pool, scanner, coverEnricher,
|
||||||
logger.With("component", "scan_run"),
|
logger.With("component", "scan_run"),
|
||||||
library.RunScanConfig{
|
library.RunScanConfig{
|
||||||
BackfillCap: 5000,
|
BackfillCap: 5000,
|
||||||
EnrichCap: cfg.Library.CoverArtBackfillCap,
|
EnrichCap: cfg.Library.CoverArtBackfillCap,
|
||||||
|
ArtistEnrichCap: cfg.Library.CoverArtBackfillCap,
|
||||||
|
DataDir: cfg.Storage.DataDir,
|
||||||
},
|
},
|
||||||
); err != nil {
|
); err != nil {
|
||||||
logger.Warn("startup scan run failed", "err", err)
|
logger.Warn("startup scan run failed", "err", err)
|
||||||
@@ -115,8 +117,10 @@ func run() error {
|
|||||||
if _, err := library.RunScan(ctx, pool, nil, coverEnricher,
|
if _, err := library.RunScan(ctx, pool, nil, coverEnricher,
|
||||||
logger.With("component", "scan_run"),
|
logger.With("component", "scan_run"),
|
||||||
library.RunScanConfig{
|
library.RunScanConfig{
|
||||||
BackfillCap: 5000,
|
BackfillCap: 5000,
|
||||||
EnrichCap: cfg.Library.CoverArtBackfillCap,
|
EnrichCap: cfg.Library.CoverArtBackfillCap,
|
||||||
|
ArtistEnrichCap: cfg.Library.CoverArtBackfillCap,
|
||||||
|
DataDir: cfg.Storage.DataDir,
|
||||||
},
|
},
|
||||||
); err != nil {
|
); err != nil {
|
||||||
logger.Warn("boot-only scan run failed", "err", err)
|
logger.Warn("boot-only scan run failed", "err", err)
|
||||||
@@ -154,8 +158,10 @@ func run() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
scanCfg := library.RunScanConfig{
|
scanCfg := library.RunScanConfig{
|
||||||
BackfillCap: 5000,
|
BackfillCap: 5000,
|
||||||
EnrichCap: cfg.Library.CoverArtBackfillCap,
|
EnrichCap: cfg.Library.CoverArtBackfillCap,
|
||||||
|
ArtistEnrichCap: cfg.Library.CoverArtBackfillCap,
|
||||||
|
DataDir: cfg.Storage.DataDir,
|
||||||
}
|
}
|
||||||
srv := server.New(logger, pool, scanner, subsonic.Config{
|
srv := server.New(logger, pool, scanner, subsonic.Config{
|
||||||
AllowPlaintextPassword: cfg.Subsonic.AllowPlaintextPassword,
|
AllowPlaintextPassword: cfg.Subsonic.AllowPlaintextPassword,
|
||||||
|
|||||||
@@ -17,14 +17,15 @@ import (
|
|||||||
// rather than base64 byte arrays — the DB column is bytea-shaped jsonb in
|
// rather than base64 byte arrays — the DB column is bytea-shaped jsonb in
|
||||||
// pgx but the data is already valid JSON.
|
// pgx but the data is already valid JSON.
|
||||||
type scanStatusResp struct {
|
type scanStatusResp struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
StartedAt string `json:"started_at"`
|
StartedAt string `json:"started_at"`
|
||||||
FinishedAt *string `json:"finished_at"`
|
FinishedAt *string `json:"finished_at"`
|
||||||
Library json.RawMessage `json:"library,omitempty"`
|
Library json.RawMessage `json:"library,omitempty"`
|
||||||
MbidBackfill json.RawMessage `json:"mbid_backfill,omitempty"`
|
MbidBackfill json.RawMessage `json:"mbid_backfill,omitempty"`
|
||||||
CoverEnrich json.RawMessage `json:"cover_enrich,omitempty"`
|
CoverEnrich json.RawMessage `json:"cover_enrich,omitempty"`
|
||||||
ErrorMessage *string `json:"error_message,omitempty"`
|
ArtistArtEnrich json.RawMessage `json:"artist_art_enrich,omitempty"`
|
||||||
InFlight bool `json:"in_flight"`
|
ErrorMessage *string `json:"error_message,omitempty"`
|
||||||
|
InFlight bool `json:"in_flight"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleGetScanStatus implements GET /api/admin/scan/status.
|
// handleGetScanStatus implements GET /api/admin/scan/status.
|
||||||
@@ -62,6 +63,9 @@ func (h *handlers) handleGetScanStatus(w http.ResponseWriter, r *http.Request) {
|
|||||||
if len(row.CoverEnrich) > 0 {
|
if len(row.CoverEnrich) > 0 {
|
||||||
resp.CoverEnrich = row.CoverEnrich
|
resp.CoverEnrich = row.CoverEnrich
|
||||||
}
|
}
|
||||||
|
if len(row.ArtistArtEnrich) > 0 {
|
||||||
|
resp.ArtistArtEnrich = row.ArtistArtEnrich
|
||||||
|
}
|
||||||
writeJSON(w, http.StatusOK, resp)
|
writeJSON(w, http.StatusOK, resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -53,20 +53,21 @@ func (q *Queries) GetInFlightScanRun(ctx context.Context) (GetInFlightScanRunRow
|
|||||||
}
|
}
|
||||||
|
|
||||||
const getLatestScanRun = `-- name: GetLatestScanRun :one
|
const getLatestScanRun = `-- name: GetLatestScanRun :one
|
||||||
SELECT id, started_at, finished_at, library, mbid_backfill, cover_enrich, error_message
|
SELECT id, started_at, finished_at, library, mbid_backfill, cover_enrich, artist_art_enrich, error_message
|
||||||
FROM scan_runs
|
FROM scan_runs
|
||||||
ORDER BY started_at DESC
|
ORDER BY started_at DESC
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
`
|
`
|
||||||
|
|
||||||
type GetLatestScanRunRow struct {
|
type GetLatestScanRunRow struct {
|
||||||
ID pgtype.UUID
|
ID pgtype.UUID
|
||||||
StartedAt pgtype.Timestamptz
|
StartedAt pgtype.Timestamptz
|
||||||
FinishedAt pgtype.Timestamptz
|
FinishedAt pgtype.Timestamptz
|
||||||
Library []byte
|
Library []byte
|
||||||
MbidBackfill []byte
|
MbidBackfill []byte
|
||||||
CoverEnrich []byte
|
CoverEnrich []byte
|
||||||
ErrorMessage *string
|
ArtistArtEnrich []byte
|
||||||
|
ErrorMessage *string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Powers GET /api/admin/scan/status. Returns NoRows when no scan has
|
// Powers GET /api/admin/scan/status. Returns NoRows when no scan has
|
||||||
@@ -81,6 +82,7 @@ func (q *Queries) GetLatestScanRun(ctx context.Context) (GetLatestScanRunRow, er
|
|||||||
&i.Library,
|
&i.Library,
|
||||||
&i.MbidBackfill,
|
&i.MbidBackfill,
|
||||||
&i.CoverEnrich,
|
&i.CoverEnrich,
|
||||||
|
&i.ArtistArtEnrich,
|
||||||
&i.ErrorMessage,
|
&i.ErrorMessage,
|
||||||
)
|
)
|
||||||
return i, err
|
return i, err
|
||||||
@@ -106,6 +108,23 @@ func (q *Queries) InsertScanRun(ctx context.Context) (InsertScanRunRow, error) {
|
|||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const updateScanRunArtistArtEnrich = `-- name: UpdateScanRunArtistArtEnrich :exec
|
||||||
|
UPDATE scan_runs
|
||||||
|
SET artist_art_enrich = $2
|
||||||
|
WHERE id = $1
|
||||||
|
`
|
||||||
|
|
||||||
|
type UpdateScanRunArtistArtEnrichParams struct {
|
||||||
|
ID pgtype.UUID
|
||||||
|
ArtistArtEnrich []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// M7 cover-sources: persists the artist-art enrich stage tally JSON.
|
||||||
|
func (q *Queries) UpdateScanRunArtistArtEnrich(ctx context.Context, arg UpdateScanRunArtistArtEnrichParams) error {
|
||||||
|
_, err := q.db.Exec(ctx, updateScanRunArtistArtEnrich, arg.ID, arg.ArtistArtEnrich)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
const updateScanRunCoverEnrich = `-- name: UpdateScanRunCoverEnrich :exec
|
const updateScanRunCoverEnrich = `-- name: UpdateScanRunCoverEnrich :exec
|
||||||
UPDATE scan_runs SET cover_enrich = $2 WHERE id = $1
|
UPDATE scan_runs SET cover_enrich = $2 WHERE id = $1
|
||||||
`
|
`
|
||||||
|
|||||||
@@ -27,11 +27,17 @@ UPDATE scan_runs
|
|||||||
-- name: GetLatestScanRun :one
|
-- name: GetLatestScanRun :one
|
||||||
-- Powers GET /api/admin/scan/status. Returns NoRows when no scan has
|
-- Powers GET /api/admin/scan/status. Returns NoRows when no scan has
|
||||||
-- ever run; the API layer maps that to a 200 with empty payload.
|
-- ever run; the API layer maps that to a 200 with empty payload.
|
||||||
SELECT id, started_at, finished_at, library, mbid_backfill, cover_enrich, error_message
|
SELECT id, started_at, finished_at, library, mbid_backfill, cover_enrich, artist_art_enrich, error_message
|
||||||
FROM scan_runs
|
FROM scan_runs
|
||||||
ORDER BY started_at DESC
|
ORDER BY started_at DESC
|
||||||
LIMIT 1;
|
LIMIT 1;
|
||||||
|
|
||||||
|
-- name: UpdateScanRunArtistArtEnrich :exec
|
||||||
|
-- M7 cover-sources: persists the artist-art enrich stage tally JSON.
|
||||||
|
UPDATE scan_runs
|
||||||
|
SET artist_art_enrich = $2
|
||||||
|
WHERE id = $1;
|
||||||
|
|
||||||
-- name: GetInFlightScanRun :one
|
-- name: GetInFlightScanRun :one
|
||||||
-- Used by POST /api/admin/scan/run to 409 when a scan is already running.
|
-- Used by POST /api/admin/scan/run to 409 when a scan is already running.
|
||||||
-- "In flight" = finished_at IS NULL.
|
-- "In flight" = finished_at IS NULL.
|
||||||
|
|||||||
@@ -40,12 +40,22 @@ type CoverEnrichStageTallies struct {
|
|||||||
Failed int `json:"failed"`
|
Failed int `json:"failed"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ArtistArtEnrichStageTallies wires EnrichArtistBatch tallies into
|
||||||
|
// the scan_runs jsonb column.
|
||||||
|
type ArtistArtEnrichStageTallies struct {
|
||||||
|
Processed int `json:"processed"`
|
||||||
|
Succeeded int `json:"succeeded"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
}
|
||||||
|
|
||||||
// RunScanConfig assembles the knobs for the orchestrator. BackfillCap and
|
// RunScanConfig assembles the knobs for the orchestrator. BackfillCap and
|
||||||
// EnrichCap mirror the existing boot-time caps (5000 / coverArtBackfillCap)
|
// EnrichCap mirror the existing boot-time caps (5000 / coverArtBackfillCap)
|
||||||
// so manual triggers don't accidentally drain the whole library.
|
// so manual triggers don't accidentally drain the whole library.
|
||||||
type RunScanConfig struct {
|
type RunScanConfig struct {
|
||||||
BackfillCap int
|
BackfillCap int
|
||||||
EnrichCap int
|
EnrichCap int
|
||||||
|
ArtistEnrichCap int // M7 cover-sources: cap for the artist-art enrich stage
|
||||||
|
DataDir string // M7 cover-sources: where to write artist-art files
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunScan is the unified scan-chain orchestrator. Creates a scan_runs row,
|
// RunScan is the unified scan-chain orchestrator. Creates a scan_runs row,
|
||||||
@@ -131,6 +141,22 @@ func RunScan(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stage 4: artist art enrichment.
|
||||||
|
if enricher != nil && cfg.ArtistEnrichCap != 0 && cfg.DataDir != "" {
|
||||||
|
processed, succeeded, failed, ceerr := enricher.EnrichArtistBatch(ctx, cfg.ArtistEnrichCap, cfg.DataDir)
|
||||||
|
if ceerr != nil {
|
||||||
|
captureErr("artist_art_enrich", ceerr)
|
||||||
|
}
|
||||||
|
blob, _ := json.Marshal(ArtistArtEnrichStageTallies{
|
||||||
|
Processed: processed, Succeeded: succeeded, Failed: failed,
|
||||||
|
})
|
||||||
|
if uerr := q.UpdateScanRunArtistArtEnrich(ctx, dbq.UpdateScanRunArtistArtEnrichParams{
|
||||||
|
ID: row.ID, ArtistArtEnrich: blob,
|
||||||
|
}); uerr != nil {
|
||||||
|
captureErr("artist_art_enrich_persist", uerr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Finalize: set finished_at + error message (or empty string if all stages succeeded).
|
// Finalize: set finished_at + error message (or empty string if all stages succeeded).
|
||||||
errMsg := ""
|
errMsg := ""
|
||||||
if firstErr != nil {
|
if firstErr != nil {
|
||||||
|
|||||||
@@ -205,6 +205,12 @@ export type ScanStageCoverEnrich = {
|
|||||||
failed: number;
|
failed: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type ScanStageArtistArtEnrich = {
|
||||||
|
processed: number;
|
||||||
|
succeeded: number;
|
||||||
|
failed: number;
|
||||||
|
};
|
||||||
|
|
||||||
export type ScanStatus = {
|
export type ScanStatus = {
|
||||||
id: string;
|
id: string;
|
||||||
started_at: string;
|
started_at: string;
|
||||||
@@ -212,6 +218,7 @@ export type ScanStatus = {
|
|||||||
library?: ScanStageLibrary;
|
library?: ScanStageLibrary;
|
||||||
mbid_backfill?: ScanStageMbidBackfill;
|
mbid_backfill?: ScanStageMbidBackfill;
|
||||||
cover_enrich?: ScanStageCoverEnrich;
|
cover_enrich?: ScanStageCoverEnrich;
|
||||||
|
artist_art_enrich?: ScanStageArtistArtEnrich;
|
||||||
error_message?: string;
|
error_message?: string;
|
||||||
in_flight: boolean;
|
in_flight: boolean;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -343,7 +343,7 @@
|
|||||||
{#if !scan || !scan.id}
|
{#if !scan || !scan.id}
|
||||||
<p class="mt-3 text-sm text-text-muted">No scan has run yet.</p>
|
<p class="mt-3 text-sm text-text-muted">No scan has run yet.</p>
|
||||||
{:else}
|
{:else}
|
||||||
<div class="mt-3 grid gap-3 sm:grid-cols-3">
|
<div class="mt-3 grid gap-3 sm:grid-cols-4">
|
||||||
<!-- Library stage -->
|
<!-- Library stage -->
|
||||||
<div class="rounded border border-border bg-bg p-3">
|
<div class="rounded border border-border bg-bg p-3">
|
||||||
<div class="text-xs font-medium uppercase tracking-wide text-text-muted">Library walk</div>
|
<div class="text-xs font-medium uppercase tracking-wide text-text-muted">Library walk</div>
|
||||||
@@ -388,6 +388,20 @@
|
|||||||
<p class="mt-2 text-sm text-text-muted">{scanInFlight ? 'Pending…' : 'Skipped.'}</p>
|
<p class="mt-2 text-sm text-text-muted">{scanInFlight ? 'Pending…' : 'Skipped.'}</p>
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Artist art enrichment stage -->
|
||||||
|
<div class="rounded border border-border bg-bg p-3">
|
||||||
|
<div class="text-xs font-medium uppercase tracking-wide text-text-muted">Artist art</div>
|
||||||
|
{#if scan.artist_art_enrich}
|
||||||
|
<dl class="mt-2 space-y-0.5 text-sm">
|
||||||
|
<div class="flex justify-between"><dt class="text-text-secondary">Processed</dt><dd>{scan.artist_art_enrich.processed}</dd></div>
|
||||||
|
<div class="flex justify-between"><dt class="text-text-secondary">Succeeded</dt><dd>{scan.artist_art_enrich.succeeded}</dd></div>
|
||||||
|
<div class="flex justify-between"><dt class="text-text-secondary">Failed</dt><dd>{scan.artist_art_enrich.failed}</dd></div>
|
||||||
|
</dl>
|
||||||
|
{:else}
|
||||||
|
<p class="mt-2 text-sm text-text-muted">{scanInFlight ? 'Pending…' : 'Skipped.'}</p>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="mt-3 text-xs text-text-muted">
|
<div class="mt-3 text-xs text-text-muted">
|
||||||
|
|||||||
Reference in New Issue
Block a user