diff --git a/cmd/minstrel/main.go b/cmd/minstrel/main.go index 028f6a82..b8add60c 100644 --- a/cmd/minstrel/main.go +++ b/cmd/minstrel/main.go @@ -24,6 +24,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/gc" "git.fabledsword.com/bvandeusen/minstrel/internal/importer" "git.fabledsword.com/bvandeusen/minstrel/internal/library" + "git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle" "git.fabledsword.com/bvandeusen/minstrel/internal/logging" "git.fabledsword.com/bvandeusen/minstrel/internal/mailer" "git.fabledsword.com/bvandeusen/minstrel/internal/notifications" @@ -341,6 +342,11 @@ func run() error { } go importer.New(pool, scanner, cat, printer, cfg.Library.ScanPaths, logger.With("component", "import")).Run(ctx) + // Download lifecycle (M502 #5523): every ten minutes, a download that + // failed or that nobody shares is retried, then given up and its album + // searched again; an imported one seeds, then is removed. + go lifecycle.New(pool, logger.With("component", "lifecycle")).Run(ctx) + // Duplicate resolver (M498, M502): classifies the sweep's groups and, with // the operator's auto-resolve on, merges the copies that are safe to merge. // Started after SetNotifier so its summaries reach admins. See diff --git a/internal/acquire/memory.go b/internal/acquire/memory.go index bf8ab8f7..9f408de8 100644 --- a/internal/acquire/memory.go +++ b/internal/acquire/memory.go @@ -21,6 +21,9 @@ const ( BlockFiles = "files" // BlockImport: the import found its files were not the album (#5522). BlockImport = "import" + // BlockStalled: given up on because it would not finish (#5523); held + // back until the block's expires_at. + BlockStalled = "stalled" ) // Grab origins, matching grabs_origin_check. @@ -92,11 +95,22 @@ func (m *Memory) blocked(ctx context.Context) map[string]string { // BlockSentence is what a blocked candidate says. func BlockSentence(reason, detail string) string { - if reason == BlockFiles { + switch reason { + case BlockFiles: if detail != "" { return "Set aside for what its files hold: " + detail } return "Set aside for what its files hold." + case BlockImport: + if detail != "" { + return "Refused on import: " + detail + } + return "Refused on import: its files were not the album." + case BlockStalled: + if detail != "" { + return "Given up on for now: " + detail + } + return "Given up on for now: it would not finish." } return "You set this release aside." } diff --git a/internal/acquire/settings.go b/internal/acquire/settings.go index a7a9c9ea..01d4b92a 100644 --- a/internal/acquire/settings.go +++ b/internal/acquire/settings.go @@ -12,8 +12,11 @@ package acquire import ( "context" + "errors" + "fmt" "time" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" @@ -154,6 +157,19 @@ func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) { return downloader.Mapped(d, downloader.PathMap{Local: row.LocalPath, Remote: row.RemotePath}), nil } +// DownloadClientFor builds the download client a grab went to, by the code +// of its acquisition_services row. +func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (downloader.Driver, error) { + row, err := dbq.New(pool).GetAcquisitionService(ctx, code) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("the download client %q is no longer set up", code) + } + return nil, err + } + return DownloadClient(row) +} + // Sources builds every enabled search source. func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) { rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch) diff --git a/internal/api/admin_import.go b/internal/api/admin_import.go index 71675013..3535ab84 100644 --- a/internal/api/admin_import.go +++ b/internal/api/admin_import.go @@ -10,6 +10,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" "git.fabledsword.com/bvandeusen/minstrel/internal/importer" + "git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle" ) // Downloads and their imports (M502 #5522): the Wanted view's Activity tab, @@ -40,6 +41,24 @@ type downloadView struct { Files []string `json:"files"` Skipped int `json:"skipped"` RehomedFrom string `json:"rehomed_from,omitempty"` + // EndDetail says why an ended download ended (#5523). + EndDetail string `json:"end_detail"` + // Seeding: imported and still sharing; Ratio its share ratio as last + // read; SeedReleasedAt when it was removed once it had shared enough. + Seeding bool `json:"seeding"` + Ratio float32 `json:"ratio"` + SeedReleasedAt *string `json:"seed_released_at"` + // Condition is what is wrong with it now, if anything. + Condition *conditionView `json:"condition"` +} + +// conditionView is an open row of the intervention ledger: what was seen, +// since when, and how many times it was retried. +type conditionView struct { + Trigger string `json:"trigger"` + Detail string `json:"detail"` + Retries int32 `json:"retries"` + Since string `json:"since"` } func downloadFrom(g dbq.ListGrabsRow) downloadView { @@ -48,6 +67,18 @@ func downloadFrom(g dbq.ListGrabsRow) downloadView { Origin: g.Origin, GrabbedAt: formatTimestamp(g.GrabbedAt), EndReason: g.EndReason, Progress: g.Progress, ImportState: g.ImportState, ImportReason: g.ImportReason, ImportedFiles: g.ImportedFiles, Destination: g.Destination, Files: []string{}, + EndDetail: g.EndDetail, Seeding: g.Seeding, Ratio: g.Ratio, + } + if g.SeedReleasedAt.Valid { + at := formatTimestamp(g.SeedReleasedAt) + v.SeedReleasedAt = &at + } + if g.Condition != nil { + v.Condition = &conditionView{Trigger: *g.Condition, Detail: deref(g.ConditionDetail), + Since: formatTimestamp(g.ConditionSince)} + if g.ConditionRetries != nil { + v.Condition.Retries = *g.ConditionRetries + } } if g.WantID.Valid { id := uuidToString(g.WantID) @@ -82,6 +113,14 @@ type downloadsPage struct { Items []downloadView `json:"items"` Limit int `json:"limit"` Offset int `json:"offset"` + // ClientsDown are the download clients not answering now (#5523). + ClientsDown []clientDownView `json:"clients_down"` +} + +type clientDownView struct { + Client string `json:"client"` + Detail string `json:"detail"` + Since string `json:"since"` } // handleListDownloads implements GET /api/admin/downloads?offset=: running @@ -97,13 +136,47 @@ func (h *handlers) handleListDownloads(w http.ResponseWriter, r *http.Request) { writeErrWithLog(w, h.logger, "admin downloads: list", apierror.Internal(err)) return } - out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset} + down, err := dbq.New(h.pool).ListClientsDown(r.Context()) + if err != nil { + writeErrWithLog(w, h.logger, "admin downloads: clients down", apierror.Internal(err)) + return + } + out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset, + ClientsDown: make([]clientDownView, 0, len(down))} for _, g := range rows { out.Items = append(out.Items, downloadFrom(g)) } + for _, c := range down { + out.ClientsDown = append(out.ClientsDown, clientDownView{Client: c.Client, Detail: c.Detail, + Since: formatTimestamp(c.FirstSeen)}) + } writeJSON(w, http.StatusOK, out) } +// handleAbandonDownload implements POST /api/admin/downloads/{id}/abandon: a +// running download is given up, its release set aside, the download removed +// and the album searched again. +func (h *handlers) handleAbandonDownload(w http.ResponseWriter, r *http.Request) { + admin, ok := requireUser(w, r) + if !ok { + return + } + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + if _, err := h.lifecycle.Abandon(r.Context(), id, admin.ID); err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), errors.Is(err, lifecycle.ErrNotAbandonable): + writeErr(w, apierror.Conflict("not_running", "that download is no longer running")) + default: + writeErrWithLog(w, h.logger, "admin downloads: abandon", apierror.Internal(err)) + } + return + } + w.WriteHeader(http.StatusNoContent) +} + // handleImportAnyway implements POST /api/admin/downloads/{id}/import: a // download the check held for a person (or one that failed) is imported on // the next pass whatever the check says. @@ -160,13 +233,16 @@ func (h *handlers) writeImportErr(w http.ResponseWriter, what string, err error) } type releaseBlockView struct { - InfoHash string `json:"info_hash"` - Name string `json:"name"` - Reason string `json:"reason"` - Detail string `json:"detail"` - BlockedAt string `json:"blocked_at"` - WantTitle string `json:"want_title"` - WantArtist string `json:"want_artist"` + InfoHash string `json:"info_hash"` + Name string `json:"name"` + Reason string `json:"reason"` + Detail string `json:"detail"` + BlockedAt string `json:"blocked_at"` + // ExpiresAt is when a release given up on may be taken again; null is + // for good (#5523). + ExpiresAt *string `json:"expires_at"` + WantTitle string `json:"want_title"` + WantArtist string `json:"want_artist"` } type releaseBlocksPage struct { @@ -193,6 +269,10 @@ func (h *handlers) handleListReleaseBlocks(w http.ResponseWriter, r *http.Reques for _, b := range rows { v := releaseBlockView{InfoHash: b.InfoHash, Name: b.Name, Reason: b.Reason, Detail: b.Detail, BlockedAt: formatTimestamp(b.BlockedAt)} + if b.ExpiresAt.Valid { + at := formatTimestamp(b.ExpiresAt) + v.ExpiresAt = &at + } if b.WantTitle != nil { v.WantTitle = *b.WantTitle } diff --git a/internal/api/admin_search.go b/internal/api/admin_search.go index 84a17101..35540523 100644 --- a/internal/api/admin_search.go +++ b/internal/api/admin_search.go @@ -37,6 +37,12 @@ type searchSettingsView struct { // is the first of ScanPaths, the library's folders. ImportRoot string `json:"import_root"` ScanPaths []string `json:"scan_paths"` + // The download lifecycle (#5523): seed an imported download to this + // ratio or for this many days; give up on one nobody shares after this + // many hours. 0 is: remove once imported, no time limit, never. + SeedRatio float32 `json:"seed_ratio"` + SeedDays int32 `json:"seed_days"` + StallHours int32 `json:"stall_hours"` } func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView { @@ -44,6 +50,7 @@ func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours, AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass, ImportRoot: s.ImportRoot, ScanPaths: roots, + SeedRatio: s.SeedRatio, SeedDays: s.SeedDays, StallHours: s.StallHours, } if v.ScanPaths == nil { v.ScanPaths = []string{} @@ -76,6 +83,9 @@ type putSearchSettingsBody struct { SearchIntervalHours int32 `json:"search_interval_hours"` AllowedQualities []string `json:"allowed_qualities"` ImportRoot string `json:"import_root"` + SeedRatio float32 `json:"seed_ratio"` + SeedDays int32 `json:"seed_days"` + StallHours int32 `json:"stall_hours"` } // handlePutSearchSettings implements PUT /api/admin/acquisition/settings. @@ -102,9 +112,15 @@ func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Reques writeErr(w, apierror.BadRequest("bad_import_root", "file new music into one of the library's folders")) return } + if body.SeedRatio < 0 || body.SeedRatio > 100 || body.SeedDays < 0 || body.SeedDays > 365 || + body.StallHours < 0 || body.StallHours > 720 { + writeErr(w, apierror.BadRequest("bad_lifecycle", + "seed ratio must be 0 to 100, seed days 0 to 365 and stall hours 0 to 720")) + return + } s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{ AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities, - ImportRoot: body.ImportRoot, + ImportRoot: body.ImportRoot, SeedRatio: body.SeedRatio, SeedDays: body.SeedDays, StallHours: body.StallHours, }) if err != nil { writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err)) diff --git a/internal/api/admin_wanted.go b/internal/api/admin_wanted.go index d5cc061f..8dec9100 100644 --- a/internal/api/admin_wanted.go +++ b/internal/api/admin_wanted.go @@ -62,6 +62,7 @@ type wantGrabView struct { Progress float32 `json:"progress"` ImportState string `json:"import_state"` ImportReason string `json:"import_reason"` + EndDetail string `json:"end_detail"` ImportedAt *string `json:"imported_at"` ImportedFiles int32 `json:"imported_files"` } @@ -164,7 +165,8 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { v.Grab = &wantGrabView{ID: uuidToString(row.GrabID), Name: *row.GrabName, GrabbedAt: formatTimestamp(row.GrabbedAt), Live: !row.GrabEndedAt.Valid, Origin: deref(row.GrabOrigin), Quality: deref(row.GrabQuality), EndReason: deref(row.GrabEndReason), - ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason)} + ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason), + EndDetail: deref(row.GrabEndDetail)} if row.GrabProgress != nil { v.Grab.Progress = *row.GrabProgress } diff --git a/internal/api/api.go b/internal/api/api.go index fc4d6133..1b7a0225 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -23,6 +23,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/library" "git.fabledsword.com/bvandeusen/minstrel/internal/lidarrconfig" "git.fabledsword.com/bvandeusen/minstrel/internal/lidarrquarantine" + "git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle" "git.fabledsword.com/bvandeusen/minstrel/internal/mailer" "git.fabledsword.com/bvandeusen/minstrel/internal/netsettings" "git.fabledsword.com/bvandeusen/minstrel/internal/notifications" @@ -80,6 +81,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev wanted: wanted.New(pool, cat, logger), searcher: wanted.NewSearcher(pool, cat, logger), importer: importer.New(pool, scanner, cat, nil, scannerRoots(scanner), logger), + lifecycle: lifecycle.New(pool, logger), catalogueSearches: newSearchCache(), catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)), leveled: newLeveledRenderer(dataDir, loudSettings, logger), @@ -264,6 +266,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev admin.Post("/downloads/{id}/import", h.handleImportAnyway) admin.Post("/downloads/{id}/retry", h.handleRetryImport) admin.Post("/downloads/{id}/refuse", h.handleRefuseDownload) + admin.Post("/downloads/{id}/abandon", h.handleAbandonDownload) admin.Get("/follows", h.handleListFollows) admin.Get("/follows/{mbid}", h.handleGetFollow) admin.Put("/follows/{mbid}", h.handlePutFollow) @@ -437,6 +440,9 @@ type handlers struct { // importer refuses a download at a person's word (#5522). The import // loop itself runs in main; this one only acts on a request. importer *importer.Importer + // lifecycle gives up on a download at a person's word (#5523); its loop + // runs in main. + lifecycle *lifecycle.Service // catalogueSearches reuses recent MusicBrainz answers; nil asks every // time, which is what handler tests get. catalogueSearches *searchCache diff --git a/internal/db/dbq/imports.sql.go b/internal/db/dbq/imports.sql.go index df96ed4d..1d1bb0c6 100644 --- a/internal/db/dbq/imports.sql.go +++ b/internal/db/dbq/imports.sql.go @@ -16,7 +16,7 @@ UPDATE grabs SET import_state = 'importing', import_attempts = import_attempts + 1, import_updated_at = now(), progress = 1 WHERE id = $1 AND ended_at IS NULL AND import_state IN ('', 'downloading', 'waiting') -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` // Takes a finished download for import. The update is the lock: a second @@ -52,6 +52,9 @@ func (q *Queries) ClaimGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -60,7 +63,7 @@ const forceGrabImport = `-- name: ForceGrabImport :one UPDATE grabs SET import_state = '', import_forced = true, import_reason = '', import_updated_at = now() WHERE id = $1 AND ended_at IS NULL AND import_state IN ('review', 'failed') -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` // "Import anyway": the next pass imports the download whatever the check said. @@ -95,12 +98,15 @@ func (q *Queries) ForceGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } const getGrab = `-- name: GetGrab :one -SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE id = $1 +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at FROM grabs WHERE id = $1 ` func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) { @@ -134,6 +140,9 @@ func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) { &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -202,9 +211,16 @@ const listGrabs = `-- name: ListGrabs :many SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files, - g.destination, w.title AS want_title, w.artist_name AS want_artist + g.destination, g.seeding, g.ratio, g.seed_released_at, + w.title AS want_title, w.artist_name AS want_artist, + di.trigger AS condition, di.detail AS condition_detail, di.remediations AS condition_retries, + di.first_seen AS condition_since FROM grabs g LEFT JOIN wants w ON w.id = g.want_id + LEFT JOIN download_interventions di + ON di.id = (SELECT x.id FROM download_interventions x + WHERE x.grab_id = g.id AND x.outcome = 'open' + ORDER BY x.last_seen DESC LIMIT 1) ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC, g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id LIMIT $2 OFFSET $1 @@ -216,32 +232,40 @@ type ListGrabsParams struct { } type ListGrabsRow struct { - ID pgtype.UUID - WantID pgtype.UUID - Name string - Indexer string - Client string - Quality string - Origin string - Forced bool - GrabbedAt pgtype.Timestamptz - EndedAt pgtype.Timestamptz - EndReason string - EndDetail string - Progress float32 - ImportState string - ImportReason string - ImportDetail []byte - ImportUpdatedAt pgtype.Timestamptz - ImportedAt pgtype.Timestamptz - ImportedFiles int32 - Destination string - WantTitle *string - WantArtist *string + ID pgtype.UUID + WantID pgtype.UUID + Name string + Indexer string + Client string + Quality string + Origin string + Forced bool + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string + Progress float32 + ImportState string + ImportReason string + ImportDetail []byte + ImportUpdatedAt pgtype.Timestamptz + ImportedAt pgtype.Timestamptz + ImportedFiles int32 + Destination string + Seeding bool + Ratio float32 + SeedReleasedAt pgtype.Timestamptz + WantTitle *string + WantArtist *string + Condition *string + ConditionDetail *string + ConditionRetries *int32 + ConditionSince pgtype.Timestamptz } // The Wanted view's Activity tab: downloads waiting on a person first, then -// the rest still running, then the ended ones newest first. +// the rest still running, then the ended ones newest first. Each carries +// what is wrong with it now, if anything (#5523). func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGrabsRow, error) { rows, err := q.db.Query(ctx, listGrabs, arg.Skip, arg.MaxRows) if err != nil { @@ -272,8 +296,15 @@ func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGra &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, &i.WantTitle, &i.WantArtist, + &i.Condition, + &i.ConditionDetail, + &i.ConditionRetries, + &i.ConditionSince, ); err != nil { return nil, err } @@ -287,7 +318,7 @@ func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGra const listLiveGrabs = `-- name: ListLiveGrabs :many -SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title, +SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, g.seeding, g.ratio, g.seed_released_at, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title, w.reason AS want_reason, w.album_id AS want_album_id FROM grabs g LEFT JOIN wants w ON w.id = g.want_id @@ -323,6 +354,9 @@ type ListLiveGrabsRow struct { ImportedAt pgtype.Timestamptz ImportedFiles int32 Destination string + Seeding bool + Ratio float32 + SeedReleasedAt pgtype.Timestamptz ReleaseGroupMbid *string ArtistMbid *string ArtistName *string @@ -370,6 +404,9 @@ func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error) &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, &i.ReleaseGroupMbid, &i.ArtistMbid, &i.ArtistName, @@ -388,10 +425,11 @@ func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error) } const listReleaseBlocksPage = `-- name: ListReleaseBlocksPage :many -SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title, +SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, b.expires_at, w.title AS want_title, w.artist_name AS want_artist FROM release_blocks b LEFT JOIN wants w ON w.id = b.want_id + WHERE b.expires_at IS NULL OR b.expires_at > now() ORDER BY b.blocked_at DESC, b.info_hash LIMIT $2 OFFSET $1 ` @@ -407,6 +445,7 @@ type ListReleaseBlocksPageRow struct { Reason string Detail string BlockedAt pgtype.Timestamptz + ExpiresAt pgtype.Timestamptz WantTitle *string WantArtist *string } @@ -426,6 +465,7 @@ func (q *Queries) ListReleaseBlocksPage(ctx context.Context, arg ListReleaseBloc &i.Reason, &i.Detail, &i.BlockedAt, + &i.ExpiresAt, &i.WantTitle, &i.WantArtist, ); err != nil { @@ -522,13 +562,16 @@ UPDATE grabs import_updated_at = now(), import_forced = false, imported_at = CASE WHEN $1::text = 'imported' THEN now() ELSE imported_at END, + -- The files were copied, so an imported download goes on sharing + -- until the lifecycle removes it (#5523). + seeding = $1::text = 'imported', imported_files = $4, destination = $5, ended_at = CASE WHEN $6::text <> '' THEN now() ELSE ended_at END, end_reason = $6, end_detail = CASE WHEN $6::text <> '' THEN $2::text ELSE end_detail END WHERE id = $7 -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` type RecordImportParams struct { @@ -582,6 +625,9 @@ func (q *Queries) RecordImport(ctx context.Context, arg RecordImportParams) (Gra &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -591,7 +637,7 @@ UPDATE grabs g SET want_id = $1 WHERE g.id = $2 AND g.ended_at IS NULL AND NOT EXISTS (SELECT 1 FROM grabs o WHERE o.want_id = $1 AND o.ended_at IS NULL) -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` type RehomeGrabParams struct { @@ -632,6 +678,9 @@ func (q *Queries) RehomeGrab(ctx context.Context, arg RehomeGrabParams) (Grab, e &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -640,7 +689,7 @@ const retryGrabImport = `-- name: RetryGrabImport :one UPDATE grabs SET import_state = '', import_reason = '', import_updated_at = now() WHERE id = $1 AND ended_at IS NULL AND import_state = 'failed' -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) { @@ -674,6 +723,9 @@ func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } diff --git a/internal/db/dbq/lifecycle.sql.go b/internal/db/dbq/lifecycle.sql.go new file mode 100644 index 00000000..7e02d817 --- /dev/null +++ b/internal/db/dbq/lifecycle.sql.go @@ -0,0 +1,389 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: lifecycle.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const countRemediation = `-- name: CountRemediation :one +UPDATE download_interventions + SET remediations = remediations + 1, + action = $1, + last_seen = now(), + outcome = CASE WHEN $1::text IN ('given_up', 'removed') THEN 'resolved' ELSE outcome END, + settled_at = CASE WHEN $1::text IN ('given_up', 'removed') THEN now() ELSE settled_at END + WHERE client = $2 AND info_hash = $3 + AND trigger = $4 AND outcome = 'open' +RETURNING remediations +` + +type CountRemediationParams struct { + Action string + Client string + InfoHash string + Trigger string +} + +// Minstrel acted on an open condition. A retry climbs the ladder; giving up +// or removing ends the condition, since what it was about is gone. Written +// before the act, so a retry that happened is never uncounted. +func (q *Queries) CountRemediation(ctx context.Context, arg CountRemediationParams) (int32, error) { + row := q.db.QueryRow(ctx, countRemediation, + arg.Action, + arg.Client, + arg.InfoHash, + arg.Trigger, + ) + var remediations int32 + err := row.Scan(&remediations) + return remediations, err +} + +const deleteExpiredBlocks = `-- name: DeleteExpiredBlocks :execrows +DELETE FROM release_blocks WHERE expires_at IS NOT NULL AND expires_at <= now() +` + +// A release given up on is held back for a while; after that the search may +// take it again. +func (q *Queries) DeleteExpiredBlocks(ctx context.Context) (int64, error) { + result, err := q.db.Exec(ctx, deleteExpiredBlocks) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const giveUpGrab = `-- name: GiveUpGrab :one +UPDATE grabs + SET ended_at = now(), end_reason = $1, end_detail = $2, + import_updated_at = now() + WHERE id = $3 AND ended_at IS NULL AND import_state = ANY($4::text[]) +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at +` + +type GiveUpGrabParams struct { + EndReason string + EndDetail string + ID pgtype.UUID + States []string +} + +// Ends a running grab, given up on. Only from the states named: the update +// is the lock against an import that claimed the download meanwhile. +func (q *Queries) GiveUpGrab(ctx context.Context, arg GiveUpGrabParams) (Grab, error) { + row := q.db.QueryRow(ctx, giveUpGrab, + arg.EndReason, + arg.EndDetail, + arg.ID, + arg.States, + ) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, + ) + return i, err +} + +const listClientsDown = `-- name: ListClientsDown :many +SELECT client, detail, first_seen FROM download_interventions + WHERE trigger = 'client_unreachable' AND outcome = 'open' + ORDER BY client +` + +type ListClientsDownRow struct { + Client string + Detail string + FirstSeen pgtype.Timestamptz +} + +// Download clients not answering now, for the Activity tab. +func (q *Queries) ListClientsDown(ctx context.Context) ([]ListClientsDownRow, error) { + rows, err := q.db.Query(ctx, listClientsDown) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListClientsDownRow + for rows.Next() { + var i ListClientsDownRow + if err := rows.Scan(&i.Client, &i.Detail, &i.FirstSeen); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listLeftovers = `-- name: ListLeftovers :many +SELECT id, client, info_hash, grab_id, name, trigger, action, outcome, detail, observations, remediations, first_seen, last_seen, settled_at FROM download_interventions + WHERE trigger = 'not_removed' AND outcome = 'open' AND info_hash <> '' + ORDER BY first_seen +` + +// Downloads given up on that the client would not remove; each pass asks +// again. +func (q *Queries) ListLeftovers(ctx context.Context) ([]DownloadIntervention, error) { + rows, err := q.db.Query(ctx, listLeftovers) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadIntervention + for rows.Next() { + var i DownloadIntervention + if err := rows.Scan( + &i.ID, + &i.Client, + &i.InfoHash, + &i.GrabID, + &i.Name, + &i.Trigger, + &i.Action, + &i.Outcome, + &i.Detail, + &i.Observations, + &i.Remediations, + &i.FirstSeen, + &i.LastSeen, + &i.SettledAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listSeedingGrabs = `-- name: ListSeedingGrabs :many +SELECT id, client, info_hash, name, imported_at FROM grabs + WHERE seeding AND seed_released_at IS NULL + ORDER BY imported_at, id +` + +type ListSeedingGrabsRow struct { + ID pgtype.UUID + Client string + InfoHash string + Name string + ImportedAt pgtype.Timestamptz +} + +// Imported downloads still sharing, oldest import first. +func (q *Queries) ListSeedingGrabs(ctx context.Context) ([]ListSeedingGrabsRow, error) { + rows, err := q.db.Query(ctx, listSeedingGrabs) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListSeedingGrabsRow + for rows.Next() { + var i ListSeedingGrabsRow + if err := rows.Scan( + &i.ID, + &i.Client, + &i.InfoHash, + &i.Name, + &i.ImportedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const liveGrabOnHash = `-- name: LiveGrabOnHash :one +SELECT EXISTS (SELECT 1 FROM grabs WHERE info_hash = $1 AND ended_at IS NULL)::bool +` + +func (q *Queries) LiveGrabOnHash(ctx context.Context, infoHash string) (bool, error) { + row := q.db.QueryRow(ctx, liveGrabOnHash, infoHash) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + +const raiseIntervention = `-- name: RaiseIntervention :one + +INSERT INTO download_interventions (client, info_hash, grab_id, name, trigger, detail) +VALUES ($1, $2, $3, $4, + $5, $6) +ON CONFLICT (client, info_hash, trigger) WHERE outcome = 'open' DO UPDATE SET + observations = download_interventions.observations + 1, + last_seen = now(), + detail = EXCLUDED.detail, + grab_id = COALESCE(EXCLUDED.grab_id, download_interventions.grab_id), + name = CASE WHEN EXCLUDED.name <> '' THEN EXCLUDED.name ELSE download_interventions.name END +RETURNING id, client, info_hash, grab_id, name, trigger, action, outcome, detail, observations, remediations, first_seen, last_seen, settled_at +` + +type RaiseInterventionParams struct { + Client string + InfoHash string + GrabID pgtype.UUID + Name string + Trigger string + Detail string +} + +// The download lifecycle (M502 #5523): the intervention ledger, giving a +// download up, and seeding after import. +// A condition seen, or seen again: a second pass lands on the open row, so +// a download stuck for a week is one row that says how long. The name only +// improves, and the detail is the latest word. +func (q *Queries) RaiseIntervention(ctx context.Context, arg RaiseInterventionParams) (DownloadIntervention, error) { + row := q.db.QueryRow(ctx, raiseIntervention, + arg.Client, + arg.InfoHash, + arg.GrabID, + arg.Name, + arg.Trigger, + arg.Detail, + ) + var i DownloadIntervention + err := row.Scan( + &i.ID, + &i.Client, + &i.InfoHash, + &i.GrabID, + &i.Name, + &i.Trigger, + &i.Action, + &i.Outcome, + &i.Detail, + &i.Observations, + &i.Remediations, + &i.FirstSeen, + &i.LastSeen, + &i.SettledAt, + ) + return i, err +} + +const releaseSeed = `-- name: ReleaseSeed :exec +UPDATE grabs SET seed_released_at = now() WHERE id = $1 AND seed_released_at IS NULL +` + +// The seeding download was removed, or found gone. +func (q *Queries) ReleaseSeed(ctx context.Context, id pgtype.UUID) error { + _, err := q.db.Exec(ctx, releaseSeed, id) + return err +} + +const setGrabRatio = `-- name: SetGrabRatio :exec +UPDATE grabs SET ratio = $1 WHERE id = $2 +` + +type SetGrabRatioParams struct { + Ratio float32 + ID pgtype.UUID +} + +func (q *Queries) SetGrabRatio(ctx context.Context, arg SetGrabRatioParams) error { + _, err := q.db.Exec(ctx, setGrabRatio, arg.Ratio, arg.ID) + return err +} + +const settleEndedGrabs = `-- name: SettleEndedGrabs :execrows +UPDATE download_interventions di + SET outcome = 'resolved', settled_at = now(), last_seen = now() + FROM grabs g + WHERE g.id = di.grab_id AND g.ended_at IS NOT NULL + AND di.outcome = 'open' AND di.trigger <> 'not_removed' +` + +// What was wrong with a download whose grab has ended (imported, refused, +// given up) is over, except a download left in the client to remove. +func (q *Queries) SettleEndedGrabs(ctx context.Context) (int64, error) { + result, err := q.db.Exec(ctx, settleEndedGrabs) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const settleIntervention = `-- name: SettleIntervention :execrows +UPDATE download_interventions + SET outcome = 'resolved', settled_at = now(), last_seen = now() + WHERE client = $1 AND info_hash = $2 + AND trigger = $3 AND outcome = 'open' +` + +type SettleInterventionParams struct { + Client string + InfoHash string + Trigger string +} + +// The condition cleared. +func (q *Queries) SettleIntervention(ctx context.Context, arg SettleInterventionParams) (int64, error) { + result, err := q.db.Exec(ctx, settleIntervention, arg.Client, arg.InfoHash, arg.Trigger) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const settleInterventionsFor = `-- name: SettleInterventionsFor :execrows +UPDATE download_interventions + SET outcome = 'resolved', settled_at = now(), last_seen = now() + WHERE client = $1 AND info_hash = $2 AND outcome = 'open' +` + +type SettleInterventionsForParams struct { + Client string + InfoHash string +} + +// Everything open about one download is over: it is gone from the client. +func (q *Queries) SettleInterventionsFor(ctx context.Context, arg SettleInterventionsForParams) (int64, error) { + result, err := q.db.Exec(ctx, settleInterventionsFor, arg.Client, arg.InfoHash) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index c42a4152..d656ee35 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -226,6 +226,9 @@ type AcquisitionSetting struct { LastPass []byte UpdatedAt pgtype.Timestamptz ImportRoot string + SeedRatio float32 + SeedDays int32 + StallHours int32 } type Album struct { @@ -355,6 +358,23 @@ type DiscoverTuning struct { UpdatedAt pgtype.Timestamptz } +type DownloadIntervention struct { + ID pgtype.UUID + Client string + InfoHash string + GrabID pgtype.UUID + Name string + Trigger string + Action string + Outcome string + Detail string + Observations int32 + Remediations int32 + FirstSeen pgtype.Timestamptz + LastSeen pgtype.Timestamptz + SettledAt pgtype.Timestamptz +} + type DuplicateGroup struct { ID pgtype.UUID MemberKey string @@ -459,6 +479,9 @@ type Grab struct { ImportedAt pgtype.Timestamptz ImportedFiles int32 Destination string + Seeding bool + Ratio float32 + SeedReleasedAt pgtype.Timestamptz } type LibraryChange struct { @@ -701,6 +724,7 @@ type ReleaseBlock struct { WantID pgtype.UUID BlockedBy pgtype.UUID BlockedAt pgtype.Timestamptz + ExpiresAt pgtype.Timestamptz } type ReleaseTypeProfile struct { diff --git a/internal/db/dbq/search.sql.go b/internal/db/dbq/search.sql.go index 9ecde58f..aa30dc94 100644 --- a/internal/db/dbq/search.sql.go +++ b/internal/db/dbq/search.sql.go @@ -12,11 +12,13 @@ import ( ) const blockRelease = `-- name: BlockRelease :exec -INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by) +INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by, expires_at) VALUES ($1, $2, $3, $4, - $5, $6) + $5, $6, $7) ON CONFLICT (info_hash) DO UPDATE - SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now() + SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(), + expires_at = EXCLUDED.expires_at, blocked_by = EXCLUDED.blocked_by + WHERE release_blocks.expires_at IS NOT NULL OR EXCLUDED.expires_at IS NULL ` type BlockReleaseParams struct { @@ -26,8 +28,12 @@ type BlockReleaseParams struct { Detail string WantID pgtype.UUID BlockedBy pgtype.UUID + ExpiresAt pgtype.Timestamptz } +// A release set aside. expires_at NULL is for good. A block for good is never +// shortened by a timed one: only a timed block, or another for good, is +// replaced. func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) error { _, err := q.db.Exec(ctx, blockRelease, arg.InfoHash, @@ -36,6 +42,7 @@ func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) erro arg.Detail, arg.WantID, arg.BlockedBy, + arg.ExpiresAt, ) return err } @@ -89,7 +96,7 @@ func (q *Queries) EndGrab(ctx context.Context, arg EndGrabParams) error { const getAcquisitionSettings = `-- name: GetAcquisitionSettings :one -SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root FROM acquisition_settings WHERE id = true +SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root, seed_ratio, seed_days, stall_hours FROM acquisition_settings WHERE id = true ` // Searching for wants and grabbing releases (M502 #5521). @@ -105,12 +112,15 @@ func (q *Queries) GetAcquisitionSettings(ctx context.Context) (AcquisitionSettin &i.LastPass, &i.UpdatedAt, &i.ImportRoot, + &i.SeedRatio, + &i.SeedDays, + &i.StallHours, ) return i, err } const getLiveGrab = `-- name: GetLiveGrab :one -SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE want_id = $1 AND ended_at IS NULL +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at FROM grabs WHERE want_id = $1 AND ended_at IS NULL ` func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, error) { @@ -144,6 +154,9 @@ func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, er &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -230,7 +243,7 @@ INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` type InsertGrabParams struct { @@ -292,6 +305,9 @@ func (q *Queries) InsertGrab(ctx context.Context, arg InsertGrabParams) (Grab, e &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -350,9 +366,13 @@ func (q *Queries) InsertWantCandidate(ctx context.Context, arg InsertWantCandida } const listReleaseBlocks = `-- name: ListReleaseBlocks :many -SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at FROM release_blocks ORDER BY blocked_at DESC +SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at, expires_at FROM release_blocks + WHERE expires_at IS NULL OR expires_at > now() + ORDER BY blocked_at DESC ` +// Every release set aside and still held back: a stalled one lifts by itself +// at expires_at (#5523). func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error) { rows, err := q.db.Query(ctx, listReleaseBlocks) if err != nil { @@ -370,6 +390,7 @@ func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error) &i.WantID, &i.BlockedBy, &i.BlockedAt, + &i.ExpiresAt, ); err != nil { return nil, err } @@ -582,9 +603,12 @@ UPDATE acquisition_settings search_interval_hours = $2, allowed_qualities = $3, import_root = $4, + seed_ratio = $5, + seed_days = $6, + stall_hours = $7, updated_at = now() WHERE id = true -RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root +RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root, seed_ratio, seed_days, stall_hours ` type UpdateAcquisitionSettingsParams struct { @@ -592,6 +616,9 @@ type UpdateAcquisitionSettingsParams struct { SearchIntervalHours int32 AllowedQualities []string ImportRoot string + SeedRatio float32 + SeedDays int32 + StallHours int32 } func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcquisitionSettingsParams) (AcquisitionSetting, error) { @@ -600,6 +627,9 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui arg.SearchIntervalHours, arg.AllowedQualities, arg.ImportRoot, + arg.SeedRatio, + arg.SeedDays, + arg.StallHours, ) var i AcquisitionSetting err := row.Scan( @@ -611,6 +641,9 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui &i.LastPass, &i.UpdatedAt, &i.ImportRoot, + &i.SeedRatio, + &i.SeedDays, + &i.StallHours, ) return i, err } diff --git a/internal/db/dbq/wanted.sql.go b/internal/db/dbq/wanted.sql.go index 3f467f48..7e7df749 100644 --- a/internal/db/dbq/wanted.sql.go +++ b/internal/db/dbq/wanted.sql.go @@ -30,7 +30,8 @@ SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::big (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity, (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention, - (SELECT COUNT(*) FROM release_blocks)::bigint AS blocked + (SELECT COUNT(*) FROM release_blocks b + WHERE b.expires_at IS NULL OR b.expires_at > now())::bigint AS blocked FROM wants ` @@ -537,7 +538,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason, gr.progress AS grab_progress, gr.import_state AS grab_import_state, - gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at, + gr.import_reason AS grab_import_reason, gr.end_detail AS grab_end_detail, gr.imported_at AS grab_imported_at, gr.imported_files AS grab_imported_files FROM wants w LEFT JOIN users u ON u.id = w.created_by @@ -590,6 +591,7 @@ type ListWantsRow struct { GrabProgress *float32 GrabImportState *string GrabImportReason *string + GrabEndDetail *string GrabImportedAt pgtype.Timestamptz GrabImportedFiles *int32 } @@ -636,6 +638,7 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan &i.GrabProgress, &i.GrabImportState, &i.GrabImportReason, + &i.GrabEndDetail, &i.GrabImportedAt, &i.GrabImportedFiles, ); err != nil { diff --git a/internal/db/migrations/0084_download_lifecycle.down.sql b/internal/db/migrations/0084_download_lifecycle.down.sql new file mode 100644 index 00000000..9e00a4d5 --- /dev/null +++ b/internal/db/migrations/0084_download_lifecycle.down.sql @@ -0,0 +1,23 @@ +ALTER TABLE acquisition_settings + DROP COLUMN seed_ratio, + DROP COLUMN seed_days, + DROP COLUMN stall_hours; + +ALTER TABLE release_blocks DROP COLUMN expires_at; +DELETE FROM release_blocks WHERE reason = 'stalled'; +ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check; +ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check + CHECK (reason IN ('operator', 'files', 'import')); + +DROP INDEX grabs_seeding_idx; +ALTER TABLE grabs + DROP COLUMN seeding, + DROP COLUMN ratio, + DROP COLUMN seed_released_at; + +UPDATE grabs SET end_reason = 'superseded' WHERE end_reason IN ('stalled', 'failed', 'abandoned', 'gone'); +ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check; +ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check + CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held')); + +DROP TABLE download_interventions; diff --git a/internal/db/migrations/0084_download_lifecycle.up.sql b/internal/db/migrations/0084_download_lifecycle.up.sql new file mode 100644 index 00000000..067bf26f --- /dev/null +++ b/internal/db/migrations/0084_download_lifecycle.up.sql @@ -0,0 +1,90 @@ +-- Download lifecycle (M502 #5523), ported from Fabled Librarian's System 104: +-- a download that stalls or fails is given up and its album searched again; +-- an imported download seeds, then is removed. +-- +-- download_interventions what was seen wrong with a download or a client, +-- what Minstrel did about it, and whether it +-- cleared. One open row per (client, download, +-- condition), updated in place. +-- grabs (columns) a grab also ends by being given up (stalled, +-- failed), set aside by a person, or leaving the +-- client; an imported one records its seeding. +-- release_blocks a release given up on is held back for a while, +-- not for ever: expires_at. +-- acquisition_settings how long to seed, and how long a download may sit +-- with nobody sharing it. +CREATE TABLE download_interventions ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + -- The acquisition_services code of the download client. + client text NOT NULL, + -- The torrent, or '' for a condition of the client itself. + info_hash text NOT NULL DEFAULT '', + grab_id uuid REFERENCES grabs (id) ON DELETE SET NULL, + name text NOT NULL DEFAULT '', + -- no_peers nobody is sharing an unfinished download. + -- no_metadata still fetching the torrent's file list. + -- client_error the client gave up on it, in its own words. + -- client_unreachable the client does not answer, or its engine is down. + -- gone the client no longer lists a running download. + -- not_removed given up on, and the client would not remove it. + trigger text NOT NULL CONSTRAINT download_interventions_trigger_check CHECK (trigger IN ( + 'no_peers', 'no_metadata', 'client_error', 'client_unreachable', 'gone', 'not_removed')), + -- escalated seen and shown; nothing done yet. + -- retried asked the trackers again and re-read the files. + -- given_up the download was removed and its album searched again. + -- removed a download left behind was removed. + action text NOT NULL DEFAULT 'escalated' + CONSTRAINT download_interventions_action_check CHECK (action IN ('escalated', 'retried', 'given_up', 'removed')), + outcome text NOT NULL DEFAULT 'open' + CONSTRAINT download_interventions_outcome_check CHECK (outcome IN ('open', 'resolved')), + detail text NOT NULL DEFAULT '', + -- How many passes have seen it, and how many times Minstrel acted on it. + -- The second is the retry ladder's rung, kept here so a restart does not + -- hand a stuck download fresh retries. + observations integer NOT NULL DEFAULT 1, + remediations integer NOT NULL DEFAULT 0, + first_seen timestamptz NOT NULL DEFAULT now(), + last_seen timestamptz NOT NULL DEFAULT now(), + settled_at timestamptz, + CONSTRAINT download_interventions_settled_together CHECK ((outcome = 'open') = (settled_at IS NULL)) +); + +-- One open row per condition: a pass that sees it again lands on the same row. +CREATE UNIQUE INDEX download_interventions_one_open + ON download_interventions (client, info_hash, trigger) WHERE outcome = 'open'; +CREATE INDEX download_interventions_grab_idx ON download_interventions (grab_id) WHERE grab_id IS NOT NULL; + +ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check; +ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check + CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held', 'stalled', 'failed', 'abandoned', 'gone')); + +-- An imported download goes on sharing (the files were copied) until its +-- ratio reaches the setting or its time is up; seed_released_at is when it +-- was removed, or found gone. +ALTER TABLE grabs + ADD COLUMN seeding boolean NOT NULL DEFAULT false, + ADD COLUMN ratio real NOT NULL DEFAULT 0, + ADD COLUMN seed_released_at timestamptz; + +CREATE INDEX grabs_seeding_idx ON grabs (imported_at) WHERE seeding AND seed_released_at IS NULL; + +-- stalled: given up on because it would not finish; held back until +-- expires_at. NULL is for good. +ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check; +ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check + CHECK (reason IN ('operator', 'files', 'import', 'stalled')); +ALTER TABLE release_blocks ADD COLUMN expires_at timestamptz; + +-- FL's rulings: seed to ratio 2.0 or for 14 days at most; a download nobody +-- has shared for 6 hours is retried, then given up. +-- seed_ratio 0 the download is removed once imported. +-- seed_days 0 no time limit: only the ratio ends it. +-- stall_hours 0 never give up on a download for want of peers (a failed +-- one is still given up). +ALTER TABLE acquisition_settings + ADD COLUMN seed_ratio real NOT NULL DEFAULT 2.0 + CONSTRAINT acquisition_settings_seed_ratio_check CHECK (seed_ratio BETWEEN 0 AND 100), + ADD COLUMN seed_days integer NOT NULL DEFAULT 14 + CONSTRAINT acquisition_settings_seed_days_check CHECK (seed_days BETWEEN 0 AND 365), + ADD COLUMN stall_hours integer NOT NULL DEFAULT 6 + CONSTRAINT acquisition_settings_stall_hours_check CHECK (stall_hours BETWEEN 0 AND 720); diff --git a/internal/db/queries/imports.sql b/internal/db/queries/imports.sql index 28bf0ef0..ffc67643 100644 --- a/internal/db/queries/imports.sql +++ b/internal/db/queries/imports.sql @@ -51,6 +51,9 @@ UPDATE grabs import_updated_at = now(), import_forced = false, imported_at = CASE WHEN sqlc.arg(import_state)::text = 'imported' THEN now() ELSE imported_at END, + -- The files were copied, so an imported download goes on sharing + -- until the lifecycle removes it (#5523). + seeding = sqlc.arg(import_state)::text = 'imported', imported_files = sqlc.arg(imported_files), destination = sqlc.arg(destination), ended_at = CASE WHEN sqlc.arg(end_reason)::text <> '' THEN now() ELSE ended_at END, @@ -110,21 +113,30 @@ SELECT DISTINCT COALESCE(t.disc_number, 1)::int AS disc, t.track_number::int AS -- name: ListGrabs :many -- The Wanted view's Activity tab: downloads waiting on a person first, then --- the rest still running, then the ended ones newest first. +-- the rest still running, then the ended ones newest first. Each carries +-- what is wrong with it now, if anything (#5523). SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files, - g.destination, w.title AS want_title, w.artist_name AS want_artist + g.destination, g.seeding, g.ratio, g.seed_released_at, + w.title AS want_title, w.artist_name AS want_artist, + di.trigger AS condition, di.detail AS condition_detail, di.remediations AS condition_retries, + di.first_seen AS condition_since FROM grabs g LEFT JOIN wants w ON w.id = g.want_id + LEFT JOIN download_interventions di + ON di.id = (SELECT x.id FROM download_interventions x + WHERE x.grab_id = g.id AND x.outcome = 'open' + ORDER BY x.last_seen DESC LIMIT 1) ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC, g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); -- name: ListReleaseBlocksPage :many -SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title, +SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, b.expires_at, w.title AS want_title, w.artist_name AS want_artist FROM release_blocks b LEFT JOIN wants w ON w.id = b.want_id + WHERE b.expires_at IS NULL OR b.expires_at > now() ORDER BY b.blocked_at DESC, b.info_hash LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); diff --git a/internal/db/queries/lifecycle.sql b/internal/db/queries/lifecycle.sql new file mode 100644 index 00000000..ca634f2a --- /dev/null +++ b/internal/db/queries/lifecycle.sql @@ -0,0 +1,96 @@ +-- The download lifecycle (M502 #5523): the intervention ledger, giving a +-- download up, and seeding after import. + +-- name: RaiseIntervention :one +-- A condition seen, or seen again: a second pass lands on the open row, so +-- a download stuck for a week is one row that says how long. The name only +-- improves, and the detail is the latest word. +INSERT INTO download_interventions (client, info_hash, grab_id, name, trigger, detail) +VALUES (sqlc.arg(client), sqlc.arg(info_hash), sqlc.narg(grab_id), sqlc.arg(name), + sqlc.arg(trigger), sqlc.arg(detail)) +ON CONFLICT (client, info_hash, trigger) WHERE outcome = 'open' DO UPDATE SET + observations = download_interventions.observations + 1, + last_seen = now(), + detail = EXCLUDED.detail, + grab_id = COALESCE(EXCLUDED.grab_id, download_interventions.grab_id), + name = CASE WHEN EXCLUDED.name <> '' THEN EXCLUDED.name ELSE download_interventions.name END +RETURNING *; + +-- name: SettleIntervention :execrows +-- The condition cleared. +UPDATE download_interventions + SET outcome = 'resolved', settled_at = now(), last_seen = now() + WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash) + AND trigger = sqlc.arg(trigger) AND outcome = 'open'; + +-- name: SettleInterventionsFor :execrows +-- Everything open about one download is over: it is gone from the client. +UPDATE download_interventions + SET outcome = 'resolved', settled_at = now(), last_seen = now() + WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash) AND outcome = 'open'; + +-- name: SettleEndedGrabs :execrows +-- What was wrong with a download whose grab has ended (imported, refused, +-- given up) is over, except a download left in the client to remove. +UPDATE download_interventions di + SET outcome = 'resolved', settled_at = now(), last_seen = now() + FROM grabs g + WHERE g.id = di.grab_id AND g.ended_at IS NOT NULL + AND di.outcome = 'open' AND di.trigger <> 'not_removed'; + +-- name: CountRemediation :one +-- Minstrel acted on an open condition. A retry climbs the ladder; giving up +-- or removing ends the condition, since what it was about is gone. Written +-- before the act, so a retry that happened is never uncounted. +UPDATE download_interventions + SET remediations = remediations + 1, + action = sqlc.arg(action), + last_seen = now(), + outcome = CASE WHEN sqlc.arg(action)::text IN ('given_up', 'removed') THEN 'resolved' ELSE outcome END, + settled_at = CASE WHEN sqlc.arg(action)::text IN ('given_up', 'removed') THEN now() ELSE settled_at END + WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash) + AND trigger = sqlc.arg(trigger) AND outcome = 'open' +RETURNING remediations; + +-- name: ListLeftovers :many +-- Downloads given up on that the client would not remove; each pass asks +-- again. +SELECT * FROM download_interventions + WHERE trigger = 'not_removed' AND outcome = 'open' AND info_hash <> '' + ORDER BY first_seen; + +-- name: ListClientsDown :many +-- Download clients not answering now, for the Activity tab. +SELECT client, detail, first_seen FROM download_interventions + WHERE trigger = 'client_unreachable' AND outcome = 'open' + ORDER BY client; + +-- name: LiveGrabOnHash :one +SELECT EXISTS (SELECT 1 FROM grabs WHERE info_hash = sqlc.arg(info_hash) AND ended_at IS NULL)::bool; + +-- name: GiveUpGrab :one +-- Ends a running grab, given up on. Only from the states named: the update +-- is the lock against an import that claimed the download meanwhile. +UPDATE grabs + SET ended_at = now(), end_reason = sqlc.arg(end_reason), end_detail = sqlc.arg(end_detail), + import_updated_at = now() + WHERE id = sqlc.arg(id) AND ended_at IS NULL AND import_state = ANY(sqlc.arg(states)::text[]) +RETURNING *; + +-- name: ListSeedingGrabs :many +-- Imported downloads still sharing, oldest import first. +SELECT id, client, info_hash, name, imported_at FROM grabs + WHERE seeding AND seed_released_at IS NULL + ORDER BY imported_at, id; + +-- name: SetGrabRatio :exec +UPDATE grabs SET ratio = sqlc.arg(ratio) WHERE id = sqlc.arg(id); + +-- name: ReleaseSeed :exec +-- The seeding download was removed, or found gone. +UPDATE grabs SET seed_released_at = now() WHERE id = $1 AND seed_released_at IS NULL; + +-- name: DeleteExpiredBlocks :execrows +-- A release given up on is held back for a while; after that the search may +-- take it again. +DELETE FROM release_blocks WHERE expires_at IS NOT NULL AND expires_at <= now(); diff --git a/internal/db/queries/search.sql b/internal/db/queries/search.sql index 9dc88be8..30677414 100644 --- a/internal/db/queries/search.sql +++ b/internal/db/queries/search.sql @@ -9,6 +9,9 @@ UPDATE acquisition_settings search_interval_hours = sqlc.arg(search_interval_hours), allowed_qualities = sqlc.arg(allowed_qualities), import_root = sqlc.arg(import_root), + seed_ratio = sqlc.arg(seed_ratio), + seed_days = sqlc.arg(seed_days), + stall_hours = sqlc.arg(stall_hours), updated_at = now() WHERE id = true RETURNING *; @@ -109,14 +112,23 @@ UPDATE grabs WHERE id = sqlc.arg(id) AND ended_at IS NULL; -- name: ListReleaseBlocks :many -SELECT * FROM release_blocks ORDER BY blocked_at DESC; +-- Every release set aside and still held back: a stalled one lifts by itself +-- at expires_at (#5523). +SELECT * FROM release_blocks + WHERE expires_at IS NULL OR expires_at > now() + ORDER BY blocked_at DESC; -- name: BlockRelease :exec -INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by) +-- A release set aside. expires_at NULL is for good. A block for good is never +-- shortened by a timed one: only a timed block, or another for good, is +-- replaced. +INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by, expires_at) VALUES (sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(reason), sqlc.arg(detail), - sqlc.narg(want_id), sqlc.narg(blocked_by)) + sqlc.narg(want_id), sqlc.narg(blocked_by), sqlc.narg(expires_at)) ON CONFLICT (info_hash) DO UPDATE - SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(); + SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(), + expires_at = EXCLUDED.expires_at, blocked_by = EXCLUDED.blocked_by + WHERE release_blocks.expires_at IS NOT NULL OR EXCLUDED.expires_at IS NULL; -- name: UnblockRelease :execrows DELETE FROM release_blocks WHERE info_hash = $1; diff --git a/internal/db/queries/wanted.sql b/internal/db/queries/wanted.sql index d52d47a6..5da52d33 100644 --- a/internal/db/queries/wanted.sql +++ b/internal/db/queries/wanted.sql @@ -194,7 +194,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason, gr.progress AS grab_progress, gr.import_state AS grab_import_state, - gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at, + gr.import_reason AS grab_import_reason, gr.end_detail AS grab_end_detail, gr.imported_at AS grab_imported_at, gr.imported_files AS grab_imported_files FROM wants w LEFT JOIN users u ON u.id = w.created_by @@ -219,5 +219,6 @@ SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::big (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity, (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention, - (SELECT COUNT(*) FROM release_blocks)::bigint AS blocked + (SELECT COUNT(*) FROM release_blocks b + WHERE b.expires_at IS NULL OR b.expires_at > now())::bigint AS blocked FROM wants; diff --git a/internal/dbtest/reset.go b/internal/dbtest/reset.go index 8b138e24..5da0e417 100644 --- a/internal/dbtest/reset.go +++ b/internal/dbtest/reset.go @@ -109,6 +109,7 @@ var dataTables = []string{ "mb_release_groups", // M502 #5519 "mb_artists", // M502 #5519 "requests", // M502 #5520 (was lidarr_requests) + "download_interventions", // M502 #5523 "release_blocks", // M502 #5521 "grabs", // M502 #5521 "want_candidates", // M502 #5521 @@ -187,6 +188,7 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) { if _, err := pool.Exec(ctx, ` UPDATE acquisition_settings SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, import_root = DEFAULT, + seed_ratio = DEFAULT, seed_days = DEFAULT, stall_hours = DEFAULT, last_pass_at = NULL, last_pass = DEFAULT, updated_at = DEFAULT`, ); err != nil { t.Fatalf("dbtest.ResetDB reset acquisition settings: %v", err) diff --git a/internal/importer/importer.go b/internal/importer/importer.go index 5811f03d..1c070fcd 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" @@ -67,8 +66,8 @@ type Importer struct { // Roots are the library's folders (the scan paths). New music is filed // under the one the settings name, or the first. Roots []string - // ClientFor builds the download client a grab went to. Defaults to the - // acquisition_services row of that code. + // ClientFor builds the download client a grab went to. Defaults to + // acquire.DownloadClientFor. ClientFor func(ctx context.Context, code string) (downloader.Driver, error) logger *slog.Logger wake chan struct{} @@ -83,19 +82,10 @@ func New(pool *pgxpool.Pool, scanner Scanner, cat Catalogue, printer Printer, ro } im := &Importer{pool: pool, Scanner: scanner, Catalogue: cat, Printer: printer, Roots: roots, logger: logger, wake: make(chan struct{}, 1)} - im.ClientFor = im.clientFor - return im -} - -func (im *Importer) clientFor(ctx context.Context, code string) (downloader.Driver, error) { - row, err := dbq.New(im.pool).GetAcquisitionService(ctx, code) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return nil, fmt.Errorf("the download client %q is no longer set up", code) - } - return nil, err + im.ClientFor = func(ctx context.Context, code string) (downloader.Driver, error) { + return acquire.DownloadClientFor(ctx, pool, code) } - return acquire.DownloadClient(row) + return im } // Wake asks for a pass now: after a grab, or a person's decision. @@ -201,7 +191,7 @@ func (im *Importer) passClient(ctx context.Context, q *dbq.Queries, code string, } t, ok := byHash[indexer.NormalizeHash(g.InfoHash)] if !ok { - continue // gone from the client: the download lifecycle's (#5523) + continue // gone from the client: the lifecycle settles it (#5523) } if !g.WantID.Valid { wait(g, "Its album is no longer wanted.") diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go new file mode 100644 index 00000000..79ad280a --- /dev/null +++ b/internal/lifecycle/lifecycle.go @@ -0,0 +1,659 @@ +// Package lifecycle looks after a download from the grab to its removal +// (M502 #5523), ported from Fabled Librarian's internal/health and its seed +// release (System 104). +// +// - A download the client has failed is given up at once. +// - One nobody has shared, or still fetching its file list, after the +// stall setting is retried: the trackers asked again and the files +// re-read, three times a pass apart. Then it is given up. +// - Giving up removes the download with its files, holds the release back +// for a week, ends the grab and searches for the album again. There is +// no limit per album: what stops it churning is the hold on each release. +// - A running download the client no longer lists, for half an hour, ends +// and its album is searched again. The release is not held back: nothing +// was wrong with it. +// - An imported download seeds (its files were copied) until its share +// ratio reaches the setting or its days are up, then is removed. +// +// Every condition and every act goes in the intervention ledger, one open +// row per (client, download, condition), so the Activity tab can say what +// is wrong with a download and what was done. +package lifecycle + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// Conditions, matching download_interventions_trigger_check. +const ( + NoPeers = "no_peers" + NoMetadata = "no_metadata" + ClientError = "client_error" + ClientUnreachable = "client_unreachable" + Gone = "gone" + NotRemoved = "not_removed" +) + +// Acts, matching download_interventions_action_check. +const ( + actRetried = "retried" + actGivenUp = "given_up" + actRemoved = "removed" +) + +// End reasons a grab is given with, matching grabs_end_reason_check. +const ( + EndStalled = "stalled" + EndFailed = "failed" + EndAbandoned = "abandoned" + EndGone = "gone" +) + +const ( + // interval: three retries a pass apart is half an hour of asking before + // a stalled download is given up. + interval = 10 * time.Minute + passTimeout = 5 * time.Minute + // Retries is how many times one stuck download is nudged first. + Retries = 3 + // StallBlock is how long a release given up on is held back. + StallBlock = 7 * 24 * time.Hour + // goneAfter: a client restarting can answer without a download for a + // while; half an hour missing is gone. + goneAfter = 30 * time.Minute +) + +// running are the import states of a grab still downloading: the lifecycle +// acts only on these, so an import under way is never pulled from under. +var running = []string{"", "downloading", "waiting"} + +// Settings are the limits, from acquisition_settings. +type Settings struct { + // SeedRatio is the share ratio an imported download seeds to; 0 removes + // it once imported. + SeedRatio float64 + // SeedTime is the longest it seeds; 0 is no limit. + SeedTime time.Duration + // Patience is how long a download may sit with nobody sharing it before + // it is retried, then given up; 0 never gives up for that. + Patience time.Duration +} + +// Defaults are FL's rulings. +var Defaults = Settings{SeedRatio: 2, SeedTime: 14 * 24 * time.Hour, Patience: 6 * time.Hour} + +// Service runs the lifecycle passes. +type Service struct { + pool *pgxpool.Pool + // ClientFor builds a download client by its code. Defaults to + // acquire.DownloadClientFor. + ClientFor func(ctx context.Context, code string) (downloader.Driver, error) + // Now is the clock; nil is time.Now. + Now func() time.Time + logger *slog.Logger + mu sync.Mutex // one pass at a time +} + +// New returns a Service. +func New(pool *pgxpool.Pool, logger *slog.Logger) *Service { + if logger == nil { + logger = slog.Default() + } + return &Service{pool: pool, logger: logger, + ClientFor: func(ctx context.Context, code string) (downloader.Driver, error) { + return acquire.DownloadClientFor(ctx, pool, code) + }} +} + +func (s *Service) now() time.Time { + if s.Now != nil { + return s.Now() + } + return time.Now() +} + +// Run passes a minute after start, then every ten minutes, until ctx ends. +// Each pass is bounded, so a client that never answers costs one pass. +func (s *Service) Run(ctx context.Context) { + first := time.NewTimer(time.Minute) + defer first.Stop() + select { + case <-ctx.Done(): + return + case <-first.C: + } + t := time.NewTicker(interval) + defer t.Stop() + for { + rep := s.Pass(ctx) + if rep.Retried+rep.GivenUp+rep.Gone+rep.Released+rep.Removed > 0 || rep.Error != "" { + s.logger.Info("lifecycle: pass", "retried", rep.Retried, "given_up", rep.GivenUp, "gone", rep.Gone, + "released", rep.Released, "removed", rep.Removed, "raised", rep.Raised, "settled", rep.Settled, + "error", rep.Error) + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +// Report is one pass. +type Report struct { + // Raised and Settled count ledger writes; the rest count acts. + Raised, Settled int + Retried, GivenUp, Gone, Released, Removed int + Error string +} + +// Pass looks at every running download, every seeding one and every one +// left behind, once. +func (s *Service) Pass(ctx context.Context) Report { + s.mu.Lock() + defer s.mu.Unlock() + ctx, cancel := context.WithTimeout(ctx, passTimeout) + defer cancel() + var rep Report + q := dbq.New(s.pool) + set := s.settings(ctx, q) + + if _, err := q.DeleteExpiredBlocks(ctx); err != nil { + s.logger.Warn("lifecycle: lift expired holds", "err", err) + } + if n, err := q.SettleEndedGrabs(ctx); err == nil { + rep.Settled += int(n) + } + live, err := q.ListLiveGrabs(ctx) + if err != nil { + rep.Error = err.Error() + return rep + } + seeds, err := q.ListSeedingGrabs(ctx) + if err != nil { + rep.Error = err.Error() + return rep + } + left, err := q.ListLeftovers(ctx) + if err != nil { + rep.Error = err.Error() + return rep + } + + work := map[string]*clientWork{} + var order []string + of := func(code string) *clientWork { + if w, ok := work[code]; ok { + return w + } + order = append(order, code) + work[code] = &clientWork{} + return work[code] + } + for _, g := range live { + if g.InfoHash != "" { + of(g.Client).live = append(of(g.Client).live, g) + } + } + for _, g := range seeds { + if g.InfoHash != "" { + of(g.Client).seeds = append(of(g.Client).seeds, g) + } + } + for _, l := range left { + of(l.Client).left = append(of(l.Client).left, l) + } + for _, code := range order { + if ctx.Err() != nil { + break + } + s.passClient(ctx, q, code, work[code], set, &rep) + } + return rep +} + +type clientWork struct { + live []dbq.ListLiveGrabsRow + seeds []dbq.ListSeedingGrabsRow + left []dbq.DownloadIntervention +} + +func (s *Service) settings(ctx context.Context, q *dbq.Queries) Settings { + row, err := q.GetAcquisitionSettings(ctx) + if err != nil { + // A blip keeps the defaults rather than stopping the pass. + return Defaults + } + return Settings{ + SeedRatio: float64(row.SeedRatio), + SeedTime: time.Duration(row.SeedDays) * 24 * time.Hour, + Patience: time.Duration(row.StallHours) * time.Hour, + } +} + +func (s *Service) passClient(ctx context.Context, q *dbq.Queries, code string, w *clientWork, set Settings, rep *Report) { + d, err := s.ClientFor(ctx, code) + if err != nil { + s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, firstLine(err.Error()), rep) + return + } + var hashes []string + for _, g := range w.live { + hashes = append(hashes, g.InfoHash) + } + for _, g := range w.seeds { + hashes = append(hashes, g.InfoHash) + } + for _, l := range w.left { + hashes = append(hashes, l.InfoHash) + } + ts, err := d.Torrents(ctx, hashes...) + if err != nil { + // Nothing about any one download is observed through a client that + // cannot answer. + s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, firstLine(err.Error()), rep) + return + } + for _, t := range ts { + if t.EngineDown != "" { + s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, t.EngineDown, rep) + return + } + } + s.settle(ctx, q, code, "", ClientUnreachable, rep) + + byHash := make(map[string]downloader.Torrent, len(ts)) + for _, t := range ts { + byHash[indexer.NormalizeHash(t.Hash)] = t + } + // ABSENCE IS ONLY A MEASUREMENT FROM A CLIENT THAT LISTS SOMETHING. An + // engine switched off, or a store that would not read, answers with an + // empty list; then nothing is concluded from what is missing. + listed := len(ts) > 0 + if !listed { + all, err := d.Torrents(ctx) + listed = err == nil && len(all) > 0 + } + + for _, l := range w.left { + t, ok := byHash[indexer.NormalizeHash(l.InfoHash)] + s.removeAgain(ctx, q, d, l, t, ok, listed, rep) + } + for _, g := range w.live { + t, ok := byHash[indexer.NormalizeHash(g.InfoHash)] + s.judge(ctx, q, d, g, t, ok, listed, set, rep) + } + for _, g := range w.seeds { + t, ok := byHash[indexer.NormalizeHash(g.InfoHash)] + s.seed(ctx, q, d, g, t, ok, set, rep) + } +} + +// judge decides what, if anything, is wrong with one running download, and +// answers each condition either way: raised when true, settled when not. +func (s *Service) judge(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow, + t downloader.Torrent, ok, listed bool, set Settings, rep *Report) { + if !isRunning(g.ImportState) { + return // the importer's, or a person's + } + if !ok { + if !listed { + return + } + row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, Gone, + "The download client no longer lists it.", rep) + if raised && s.now().Sub(row.FirstSeen.Time) >= goneAfter { + s.endGone(ctx, q, g, rep) + } + return + } + s.settle(ctx, q, g.Client, g.InfoHash, Gone, rep) + + if t.Done() { + // Finished: whatever was wrong while it downloaded is over, and the + // import is the importer's. + for _, c := range []string{ClientError, NoMetadata, NoPeers} { + s.settle(ctx, q, g.Client, g.InfoHash, c, rep) + } + return + } + + // The client's own verdict, given up on at once: it has already made + // the decision a retry would ask about (FL #4749). + if t.Failed() { + why := t.FailReason + if why == "" { + why = fmt.Sprintf("the download client reports it %q", t.State) + } + if _, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, ClientError, why, rep); raised { + s.giveUp(ctx, q, d, g, ClientError, EndFailed, why, rep) + } + return + } + s.settle(ctx, q, g.Client, g.InfoHash, ClientError, rep) + + // Below here the stall setting applies, measured from the grab. With it + // off nothing is raised and what was raised clears. + age := s.now().Sub(g.GrabbedAt.Time) + patient := set.Patience > 0 && age > set.Patience + + if patient && t.Fetching() { + row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NoMetadata, + fmt.Sprintf("Still fetching its file list after %s.", span(age)), rep) + if raised { + s.ladder(ctx, q, d, g, NoMetadata, row.Remediations, age, rep) + } + } else { + s.settle(ctx, q, g.Client, g.InfoHash, NoMetadata, rep) + } + // Not while fetching: a torrent with no file list has no swarm to be + // alone in. + if patient && !t.Fetching() && t.Alone() { + row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NoPeers, + fmt.Sprintf("Nobody has shared it in %s; %.0f%% downloaded.", span(age), t.Progress*100), rep) + if raised { + s.ladder(ctx, q, d, g, NoPeers, row.Remediations, age, rep) + } + } else { + s.settle(ctx, q, g.Client, g.InfoHash, NoPeers, rep) + } +} + +// ladder retries a stuck download until the retries are spent, then gives +// it up. The rung is the ledger row's, so a restart hands out no more. +func (s *Service) ladder(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow, + trigger string, done int32, age time.Duration, rep *Report) { + if int(done) < Retries { + // Counted before it happens: a retry that happened and was not + // counted is a bound that never binds. + n, err := q.CountRemediation(ctx, dbq.CountRemediationParams{ + Client: g.Client, InfoHash: g.InfoHash, Trigger: trigger, Action: actRetried}) + if err != nil { + return + } + // Exchange schedules its own announces and has no recheck: for it a + // retry is the time between passes, and the ladder still climbs. + if err := d.Reannounce(ctx, g.InfoHash); err != nil && !errors.Is(err, downloader.ErrUnsupported) { + s.logger.Warn("lifecycle: reannounce", "grab", g.Name, "err", err) + } + if err := d.Recheck(ctx, g.InfoHash); err != nil && !errors.Is(err, downloader.ErrUnsupported) { + s.logger.Warn("lifecycle: recheck", "grab", g.Name, "err", err) + } + s.logger.Info("lifecycle: retried a stalled download", "grab", g.Name, "trigger", trigger, "attempt", n) + rep.Retried++ + return + } + why := fmt.Sprintf("taken %s ago and never finished: %s", span(age), reasonFor(trigger)) + s.giveUp(ctx, q, d, g, trigger, EndStalled, why, rep) +} + +// giveUp ends the grab, holds the release back for StallBlock, removes the +// download with its files and searches for the album again. +func (s *Service) giveUp(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow, + trigger, end, why string, rep *Report) { + until := s.now().Add(StallBlock) + detail := capital(why) + fmt.Sprintf(". Held back until %s; the album is searched for again.", until.Format("2 Jan")) + // The update is the lock: an import that claimed the download meanwhile + // keeps it. + if _, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: end, EndDetail: detail, States: running}); err != nil { + return + } + // Held back before it is removed: if the pass dies between the two, the + // release is still not taken again. + if err := acquire.NewMemory(s.pool).Block(ctx, dbq.BlockReleaseParams{ + InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockStalled, Detail: capital(why) + ".", + WantID: g.WantID, ExpiresAt: pgtype.Timestamptz{Time: until, Valid: true}, + }); err != nil { + s.logger.Warn("lifecycle: hold the release back", "grab", g.Name, "err", err) + } + removed := s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, rep) + if g.WantID.Valid { + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err) + } + } + if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{ + Client: g.Client, InfoHash: g.InfoHash, Trigger: trigger, Action: actGivenUp}); err != nil { + s.logger.Warn("lifecycle: stamp the ledger", "grab", g.Name, "err", err) + } + if removed { + if n, err := q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{ + Client: g.Client, InfoHash: g.InfoHash}); err == nil { + rep.Settled += int(n) + } + } + s.logger.Info("lifecycle: gave up on a download", "grab", g.Name, "trigger", trigger, "why", why) + rep.GivenUp++ +} + +// endGone ends a grab whose download left the client, and searches again. +// The release is not held back: nothing was found wrong with it. +func (s *Service) endGone(ctx context.Context, q *dbq.Queries, g dbq.ListLiveGrabsRow, rep *Report) { + detail := "The download client no longer lists it: removed outside Minstrel. The album is searched for again." + if _, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: EndGone, EndDetail: detail, States: running}); err != nil { + return + } + if g.WantID.Valid { + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err) + } + } + if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{ + Client: g.Client, InfoHash: g.InfoHash, Trigger: Gone, Action: actGivenUp}); err != nil { + s.logger.Warn("lifecycle: stamp the ledger", "grab", g.Name, "err", err) + } + if n, err := q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{ + Client: g.Client, InfoHash: g.InfoHash}); err == nil { + rep.Settled += int(n) + } + s.logger.Info("lifecycle: a download left its client", "grab", g.Name) + rep.Gone++ +} + +// seed removes an imported download once it has shared enough. +// +// Absence is not a verdict: a restarting client answers with a partial list. +// A seeding download missing from it is left until its time is up, then +// recorded as released; there is nothing to delete. +func (s *Service) seed(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListSeedingGrabsRow, + t downloader.Torrent, ok bool, set Settings, rep *Report) { + overdue := set.SeedTime > 0 && g.ImportedAt.Valid && s.now().Sub(g.ImportedAt.Time) >= set.SeedTime + if !ok { + if overdue { + s.release(ctx, q, g.ID, rep) + } + return + } + if err := q.SetGrabRatio(ctx, dbq.SetGrabRatioParams{ID: g.ID, Ratio: float32(t.Ratio)}); err != nil { + s.logger.Warn("lifecycle: record the ratio", "grab", g.Name, "err", err) + } + if t.Ratio < set.SeedRatio && !overdue { + return + } + s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, rep) + s.logger.Info("lifecycle: seeded enough, removed the download", "grab", g.Name, "ratio", t.Ratio, "overdue", overdue) + s.release(ctx, q, g.ID, rep) +} + +func (s *Service) release(ctx context.Context, q *dbq.Queries, id pgtype.UUID, rep *Report) { + if err := q.ReleaseSeed(ctx, id); err != nil { + s.logger.Warn("lifecycle: record the seeding ended", "err", err) + return + } + rep.Released++ +} + +// remove deletes a download and its files. A client that refuses leaves an +// open not_removed row, which every pass tries again. +func (s *Service) remove(ctx context.Context, q *dbq.Queries, d downloader.Driver, code, hash string, + grab pgtype.UUID, name string, rep *Report) bool { + if err := d.Delete(ctx, true, hash); err != nil { + s.logger.Warn("lifecycle: remove the download", "grab", name, "err", err) + s.raise(ctx, q, code, hash, grab, name, NotRemoved, notRemoved(err), rep) + return false + } + return true +} + +// removeAgain asks the client once more to remove a download left behind, +// unless it has been taken again since. +func (s *Service) removeAgain(ctx context.Context, q *dbq.Queries, d downloader.Driver, l dbq.DownloadIntervention, + t downloader.Torrent, ok, listed bool, rep *Report) { + if !ok { + if listed { + s.settle(ctx, q, l.Client, l.InfoHash, NotRemoved, rep) + } + return + } + live, err := q.LiveGrabOnHash(ctx, l.InfoHash) + if err != nil { + return + } + if live { + s.settle(ctx, q, l.Client, l.InfoHash, NotRemoved, rep) + return + } + if err := d.Delete(ctx, true, l.InfoHash); err != nil { + s.raise(ctx, q, l.Client, l.InfoHash, l.GrabID, t.Name, NotRemoved, notRemoved(err), rep) + return + } + if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{ + Client: l.Client, InfoHash: l.InfoHash, Trigger: NotRemoved, Action: actRemoved}); err != nil { + s.logger.Warn("lifecycle: stamp the ledger", "hash", l.InfoHash, "err", err) + } + s.logger.Info("lifecycle: removed a download left behind", "name", l.Name) + rep.Removed++ +} + +// raise records a condition or agrees with the open row that says it. The +// bool is false when the write failed: then nothing is acted on, since +// without the row's count there is no bound. +func (s *Service) raise(ctx context.Context, q *dbq.Queries, code, hash string, grab pgtype.UUID, name, trigger, + detail string, rep *Report) (dbq.DownloadIntervention, bool) { + row, err := q.RaiseIntervention(ctx, dbq.RaiseInterventionParams{ + Client: code, InfoHash: hash, GrabID: grab, Name: name, Trigger: trigger, Detail: detail}) + if err != nil { + s.logger.Warn("lifecycle: record", "client", code, "hash", hash, "trigger", trigger, "err", err) + return row, false + } + rep.Raised++ + return row, true +} + +func (s *Service) settle(ctx context.Context, q *dbq.Queries, code, hash, trigger string, rep *Report) { + n, err := q.SettleIntervention(ctx, dbq.SettleInterventionParams{Client: code, InfoHash: hash, Trigger: trigger}) + if err != nil { + s.logger.Warn("lifecycle: settle", "client", code, "hash", hash, "trigger", trigger, "err", err) + return + } + rep.Settled += int(n) +} + +// ErrNotAbandonable: the download is not running; an import, or a person's +// decision on one, has it. +var ErrNotAbandonable = errors.New("lifecycle: that download is not running") + +// Abandon is a person giving up on a running download: the release is set +// aside for good (their word about it), the download removed with its files, +// and the album searched for again. +func (s *Service) Abandon(ctx context.Context, grabID, by pgtype.UUID) (dbq.Grab, error) { + q := dbq.New(s.pool) + g, err := q.GetGrab(ctx, grabID) + if err != nil { + return dbq.Grab{}, err + } + if g.EndedAt.Valid || !isRunning(g.ImportState) { + return dbq.Grab{}, ErrNotAbandonable + } + const why = "Given up by you before it finished. The album is searched for again." + ended, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: EndAbandoned, EndDetail: why, States: running}) + if err != nil { + return dbq.Grab{}, ErrNotAbandonable + } + if g.InfoHash != "" { + if err := acquire.NewMemory(s.pool).Block(ctx, dbq.BlockReleaseParams{ + InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockOperator, Detail: "Given up by you before it finished.", + WantID: g.WantID, BlockedBy: by, + }); err != nil { + s.logger.Warn("lifecycle: set the release aside", "grab", g.Name, "err", err) + } + var rep Report + if d, err := s.ClientFor(ctx, g.Client); err != nil { + s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NotRemoved, notRemoved(err), &rep) + } else if s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, &rep) { + _, _ = q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{Client: g.Client, InfoHash: g.InfoHash}) + } + } + if g.WantID.Valid { + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err) + } + } + return ended, nil +} + +func isRunning(state string) bool { + for _, r := range running { + if state == r { + return true + } + } + return false +} + +// reasonFor is a condition in a person's words. +func reasonFor(trigger string) string { + switch trigger { + case NoPeers: + return "nobody was sharing it" + case NoMetadata: + return "its file list never arrived" + case ClientError: + return "the download client gave up on it" + } + return "it stalled" +} + +func notRemoved(err error) string { + return "Given up on, but the download client would not remove it: " + firstLine(err.Error()) + + ". Minstrel asks again every ten minutes." +} + +// span is a duration as a person says it. +func span(d time.Duration) string { + switch { + case d < 2*time.Hour: + return fmt.Sprintf("%d minutes", int(d.Minutes())) + case d < 48*time.Hour: + return fmt.Sprintf("%d hours", int(d.Hours())) + } + return fmt.Sprintf("%d days", int(d.Hours()/24)) +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return strings.TrimSpace(s[:i]) + } + return strings.TrimSpace(s) +} diff --git a/internal/lifecycle/lifecycle_test.go b/internal/lifecycle/lifecycle_test.go new file mode 100644 index 00000000..f5cd4c60 --- /dev/null +++ b/internal/lifecycle/lifecycle_test.go @@ -0,0 +1,441 @@ +package lifecycle + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/wanted" +) + +func newPool(t *testing.T) *pgxpool.Pool { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in -short mode") + } + dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("pool: %v", err) + } + t.Cleanup(pool.Close) + dbtest.ResetDB(t, pool) + return pool +} + +const ( + hash = "0123456789abcdef0123456789abcdef01234567" + other = "fedcba9876543210fedcba9876543210fedcba98" +) + +// fakeClient lists the torrents it holds and records what it is asked. +type fakeClient struct { + downloader.Driver + held map[string]downloader.Torrent + err error + deleteErr error + deleted []string + reannounce int + recheck int +} + +func (c *fakeClient) Torrents(_ context.Context, hashes ...string) ([]downloader.Torrent, error) { + if c.err != nil { + return nil, c.err + } + var out []downloader.Torrent + if len(hashes) == 0 { + for _, t := range c.held { + out = append(out, t) + } + return out, nil + } + for _, h := range hashes { + if t, ok := c.held[h]; ok { + out = append(out, t) + } + } + return out, nil +} + +func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error { + if c.deleteErr != nil { + return c.deleteErr + } + c.deleted = append(c.deleted, hashes...) + for _, h := range hashes { + delete(c.held, h) + } + return nil +} + +func (c *fakeClient) Reannounce(context.Context, ...string) error { c.reannounce++; return nil } +func (c *fakeClient) Recheck(context.Context, ...string) error { c.recheck++; return nil } + +type stack struct { + q *dbq.Queries + s *Service + client *fakeClient + want dbq.Want + grab dbq.Grab + clock time.Time +} + +// newStack wants RAM and has a grab of it running in a fake client holding +// t under hash. +func newStack(t *testing.T, tor downloader.Torrent) *stack { + t.Helper() + ctx := context.Background() + pool := newPool(t) + q := dbq.New(pool) + w, err := wanted.WantWith(ctx, q, wanted.WantParams{GroupMBID: "rg-ram", ArtistMBID: "mb-dp", ArtistName: "Daft Punk", + Title: "Random Access Memories", PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: wanted.ReasonRequest}) + if err != nil { + t.Fatal(err) + } + g, err := acquire.NewMemory(pool).Grabbed(ctx, dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash, + Name: "Daft Punk - RAM [FLAC]", Key: hash, Client: "qbittorrent", SavePath: "/downloads", Origin: acquire.OriginSchedule}, + pgtype.UUID{}, "") + if err != nil { + t.Fatal(err) + } + if err := q.MarkWantSearched(ctx, w.ID); err != nil { + t.Fatal(err) + } + tor.Hash = hash + st := &stack{q: q, client: &fakeClient{held: map[string]downloader.Torrent{hash: tor}}, want: w, grab: g, + clock: time.Now()} + st.s = New(pool, slog.New(slog.NewTextHandler(io.Discard, nil))) + st.s.ClientFor = func(context.Context, string) (downloader.Driver, error) { return st.client, nil } + st.s.Now = func() time.Time { return st.clock } + return st +} + +func (st *stack) grabNow(t *testing.T) dbq.Grab { + t.Helper() + g, err := st.q.GetGrab(context.Background(), st.grab.ID) + if err != nil { + t.Fatal(err) + } + return g +} + +func (st *stack) searchedAgain(t *testing.T) bool { + t.Helper() + w, err := st.q.GetWantByReleaseGroup(context.Background(), "rg-ram") + if err != nil { + t.Fatal(err) + } + return !w.SearchedAt.Valid +} + +func (st *stack) ledger(t *testing.T) []dbq.DownloadIntervention { + t.Helper() + rows, err := st.s.pool.Query(context.Background(), `SELECT * FROM download_interventions ORDER BY first_seen, id`) + if err != nil { + t.Fatal(err) + } + out, err := pgx.CollectRows(rows, pgx.RowToStructByPos[dbq.DownloadIntervention]) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestLifecycle_AFailedDownloadIsGivenUpAtOnce_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "error", + FailReason: "No space left on device"}) + rep := st.s.Pass(context.Background()) + if rep.GivenUp != 1 { + t.Fatalf("pass = %+v", rep) + } + g := st.grabNow(t) + if !g.EndedAt.Valid || g.EndReason != EndFailed { + t.Fatalf("grab ended=%v reason=%q", g.EndedAt.Valid, g.EndReason) + } + if len(st.client.deleted) != 1 || !st.searchedAgain(t) { + t.Errorf("deleted %v, searched again %v", st.client.deleted, st.searchedAgain(t)) + } + blocks, err := st.q.ListReleaseBlocks(context.Background()) + if err != nil || len(blocks) != 1 || blocks[0].Reason != acquire.BlockStalled || !blocks[0].ExpiresAt.Valid { + t.Fatalf("blocks = %+v %v", blocks, err) + } + if d := blocks[0].ExpiresAt.Time.Sub(st.clock); d < StallBlock-time.Minute || d > StallBlock+time.Minute { + t.Errorf("held back for %s", d) + } + rows := st.ledger(t) + if len(rows) != 1 || rows[0].Trigger != ClientError || rows[0].Action != actGivenUp || rows[0].Outcome != "resolved" || + rows[0].Detail != "No space left on device" { + t.Errorf("ledger = %+v", rows) + } +} + +func TestLifecycle_ADownloadNobodySharesIsRetriedThenGivenUp_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"}) + ctx := context.Background() + + // Within the stall setting: nothing is wrong yet. + if rep := st.s.Pass(ctx); rep.Retried+rep.GivenUp != 0 || len(st.ledger(t)) != 0 { + t.Fatalf("a young download was judged: %+v", rep) + } + + st.clock = st.clock.Add(7 * time.Hour) + for i := 1; i <= Retries; i++ { + if rep := st.s.Pass(ctx); rep.Retried != 1 || rep.GivenUp != 0 { + t.Fatalf("pass %d = %+v", i, rep) + } + } + if st.client.reannounce != Retries || st.client.recheck != Retries { + t.Errorf("reannounced %d, rechecked %d", st.client.reannounce, st.client.recheck) + } + if st.grabNow(t).EndedAt.Valid { + t.Fatal("given up before the retries ran out") + } + if rep := st.s.Pass(ctx); rep.GivenUp != 1 { + t.Fatalf("last pass = %+v", rep) + } + g := st.grabNow(t) + if g.EndReason != EndStalled || len(st.client.deleted) != 1 || !st.searchedAgain(t) { + t.Errorf("grab %q: %s; deleted %v", g.EndReason, g.EndDetail, st.client.deleted) + } + rows := st.ledger(t) + if len(rows) != 1 || rows[0].Trigger != NoPeers || rows[0].Remediations != Retries+1 || rows[0].Outcome != "resolved" { + t.Errorf("ledger = %+v", rows) + } +} + +func TestLifecycle_AStalledDownloadThatRecoversIsSettled_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"}) + ctx := context.Background() + st.clock = st.clock.Add(7 * time.Hour) + st.s.Pass(ctx) + t2 := st.client.held[hash] + t2.NumSeeds = 3 + st.client.held[hash] = t2 + st.s.Pass(ctx) + rows := st.ledger(t) + if len(rows) != 1 || rows[0].Outcome != "resolved" || rows[0].Action != actRetried { + t.Fatalf("ledger = %+v", rows) + } + if st.grabNow(t).EndedAt.Valid { + t.Error("a download that found peers was given up") + } +} + +func TestLifecycle_TheStallSettingOffNeverGivesUp_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"}) + ctx := context.Background() + if _, err := st.s.pool.Exec(ctx, `UPDATE acquisition_settings SET stall_hours = 0`); err != nil { + t.Fatal(err) + } + st.clock = st.clock.Add(30 * 24 * time.Hour) + for range Retries + 2 { + if rep := st.s.Pass(ctx); rep.Retried+rep.GivenUp != 0 { + t.Fatalf("pass = %+v", rep) + } + } +} + +func TestLifecycle_ADownloadGoneFromTheClientEndsAfterHalfAnHour_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "downloading", NumSeeds: 4}) + ctx := context.Background() + delete(st.client.held, hash) + st.client.held[other] = downloader.Torrent{Hash: other, Name: "Someone else's", Progress: 1, State: "uploading"} + + if rep := st.s.Pass(ctx); rep.Gone != 0 { + t.Fatalf("ended at first sight: %+v", rep) + } + st.clock = st.clock.Add(31 * time.Minute) + if rep := st.s.Pass(ctx); rep.Gone != 1 { + t.Fatalf("pass = %+v", rep) + } + g := st.grabNow(t) + if g.EndReason != EndGone || !st.searchedAgain(t) { + t.Errorf("grab %q, searched again %v", g.EndReason, st.searchedAgain(t)) + } + if blocks, _ := st.q.ListReleaseBlocks(ctx); len(blocks) != 0 { + t.Errorf("a release nothing was wrong with was held back: %+v", blocks) + } +} + +func TestLifecycle_AnEmptyListIsNotAMeasurement_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "downloading"}) + ctx := context.Background() + delete(st.client.held, hash) + st.clock = st.clock.Add(24 * time.Hour) + for range 3 { + if rep := st.s.Pass(ctx); rep.Gone != 0 { + t.Fatalf("pass = %+v", rep) + } + } + if len(st.ledger(t)) != 0 || st.grabNow(t).EndedAt.Valid { + t.Error("a client listing nothing was taken to have lost the download") + } +} + +func TestLifecycle_AnUnreachableClientIsRecordedAndSettled_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"}) + ctx := context.Background() + st.clock = st.clock.Add(7 * time.Hour) + st.client.err = errors.New("dial tcp: connection refused") + st.s.Pass(ctx) + down, err := st.q.ListClientsDown(ctx) + if err != nil || len(down) != 1 || down[0].Client != "qbittorrent" { + t.Fatalf("clients down = %+v %v", down, err) + } + if st.client.reannounce != 0 { + t.Error("a download was judged through a client that did not answer") + } + st.client.err = nil + st.s.Pass(ctx) + if down, _ := st.q.ListClientsDown(ctx); len(down) != 0 { + t.Errorf("still down: %+v", down) + } +} + +func TestLifecycle_AnImportedDownloadSeedsThenIsRemoved_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", Ratio: 0.8}) + ctx := context.Background() + if _, err := st.q.RecordImport(ctx, dbq.RecordImportParams{ID: st.grab.ID, ImportState: "imported", + ImportReason: "Imported.", ImportDetail: []byte("{}"), ImportedFiles: 3, EndReason: "imported"}); err != nil { + t.Fatal(err) + } + if !st.grabNow(t).Seeding { + t.Fatal("an import did not leave its download seeding") + } + if rep := st.s.Pass(ctx); rep.Released != 0 || len(st.client.deleted) != 0 { + t.Fatalf("removed before the ratio: %+v", rep) + } + if g := st.grabNow(t); g.Ratio < 0.79 || g.Ratio > 0.81 { + t.Errorf("ratio recorded %v", g.Ratio) + } + tor := st.client.held[hash] + tor.Ratio = 2.1 + st.client.held[hash] = tor + if rep := st.s.Pass(ctx); rep.Released != 1 || len(st.client.deleted) != 1 { + t.Fatalf("pass = %+v, deleted %v", rep, st.client.deleted) + } + if !st.grabNow(t).SeedReleasedAt.Valid { + t.Error("the release was not recorded") + } +} + +func TestLifecycle_AnImportedDownloadIsRemovedWhenItsDaysAreUp_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", Ratio: 0.1}) + ctx := context.Background() + if _, err := st.q.RecordImport(ctx, dbq.RecordImportParams{ID: st.grab.ID, ImportState: "imported", + ImportReason: "Imported.", ImportDetail: []byte("{}"), ImportedFiles: 3, EndReason: "imported"}); err != nil { + t.Fatal(err) + } + st.clock = st.clock.Add(15 * 24 * time.Hour) + if rep := st.s.Pass(ctx); rep.Released != 1 || len(st.client.deleted) != 1 { + t.Fatalf("pass = %+v", rep) + } +} + +func TestLifecycle_ADownloadTheClientWouldNotRemoveIsAskedAgain_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "error"}) + ctx := context.Background() + st.client.deleteErr = errors.New("HTTP 500") + if rep := st.s.Pass(ctx); rep.GivenUp != 1 { + t.Fatalf("pass = %+v", rep) + } + if !st.grabNow(t).EndedAt.Valid { + t.Fatal("a refused removal held the grab") + } + open := 0 + for _, r := range st.ledger(t) { + if r.Trigger == NotRemoved && r.Outcome == "open" { + open++ + } + } + if open != 1 { + t.Fatalf("ledger = %+v", st.ledger(t)) + } + st.client.deleteErr = nil + if rep := st.s.Pass(ctx); rep.Removed != 1 || len(st.client.deleted) != 1 { + t.Fatalf("pass = %+v", rep) + } + for _, r := range st.ledger(t) { + if r.Outcome == "open" { + t.Errorf("still open: %+v", r) + } + } +} + +func TestLifecycle_APersonGivingUpSetsTheReleaseAsideForGood_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "downloading"}) + ctx := context.Background() + if _, err := st.s.Abandon(ctx, st.grab.ID, pgtype.UUID{}); err != nil { + t.Fatal(err) + } + g := st.grabNow(t) + if g.EndReason != EndAbandoned || len(st.client.deleted) != 1 || !st.searchedAgain(t) { + t.Errorf("grab %q, deleted %v", g.EndReason, st.client.deleted) + } + blocks, _ := st.q.ListReleaseBlocks(ctx) + if len(blocks) != 1 || blocks[0].Reason != acquire.BlockOperator || blocks[0].ExpiresAt.Valid { + t.Errorf("blocks = %+v", blocks) + } + if _, err := st.s.Abandon(ctx, st.grab.ID, pgtype.UUID{}); !errors.Is(err, ErrNotAbandonable) { + t.Errorf("abandoned twice: %v", err) + } +} + +func TestLifecycle_AHoldExpiresButABlockForGoodIsNeverShortened_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "x", Progress: 0.4, State: "downloading"}) + ctx := context.Background() + mem := acquire.NewMemory(st.s.pool) + past := pgtype.Timestamptz{Time: time.Now().Add(-time.Hour), Valid: true} + if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: other, Name: "old", Reason: acquire.BlockStalled, ExpiresAt: past}); err != nil { + t.Fatal(err) + } + if blocks, _ := st.q.ListReleaseBlocks(ctx); len(blocks) != 0 { + t.Errorf("an expired hold still holds: %+v", blocks) + } + if n, err := st.q.DeleteExpiredBlocks(ctx); err != nil || n != 1 { + t.Errorf("lifted %d %v", n, err) + } + + if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: hash, Name: "x", Reason: acquire.BlockOperator}); err != nil { + t.Fatal(err) + } + soon := pgtype.Timestamptz{Time: time.Now().Add(time.Hour), Valid: true} + if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: hash, Name: "x", Reason: acquire.BlockStalled, ExpiresAt: soon}); err != nil { + t.Fatal(err) + } + blocks, _ := st.q.ListReleaseBlocks(ctx) + if len(blocks) != 1 || blocks[0].Reason != acquire.BlockOperator || blocks[0].ExpiresAt.Valid { + t.Errorf("a block for good was shortened: %+v", blocks) + } +} + +func TestSpan(t *testing.T) { + for _, c := range []struct { + d time.Duration + want string + }{{45 * time.Minute, "45 minutes"}, {7 * time.Hour, "7 hours"}, {3 * 24 * time.Hour, "3 days"}} { + if got := span(c.d); got != c.want { + t.Errorf("span(%s) = %q, want %q", c.d, got, c.want) + } + } +} diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index c0cc5728..738e16e6 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -630,6 +630,8 @@ export type WantGrab = { progress: number; import_state: ImportState; import_reason: string; + // Why an ended download ended (#5523). + end_detail: string; imported_at: string | null; imported_files: number; }; @@ -710,6 +712,12 @@ export type SearchSettings = { // scan_paths), and the library's folders (#5522). import_root: string; scan_paths: string[]; + // The download lifecycle (#5523): an imported download seeds to this + // ratio or for this many days (0: removed once imported; no time limit), + // and one nobody shares is given up after this many hours (0: never). + seed_ratio: number; + seed_days: number; + stall_hours: number; }; export type WantCounts = { @@ -761,17 +769,42 @@ export type Download = { files: string[]; skipped: number; rehomed_from?: string; + // Why an ended download ended; for an imported one, its seeding (#5523). + end_detail: string; + seeding: boolean; + ratio: number; + seed_released_at: string | null; + // What is wrong with it now, if anything. + condition: DownloadCondition | null; }; -export type DownloadsPage = { items: Download[]; limit: number; offset: number }; +// An open condition on a download, from the lifecycle's ledger (#5523). +export type DownloadCondition = { + trigger: 'no_peers' | 'no_metadata' | 'client_error' | 'client_unreachable' | 'gone' | 'not_removed'; + detail: string; + retries: number; + since: string; +}; + +// A download client not answering now. +export type ClientDown = { client: string; detail: string; since: string }; + +export type DownloadsPage = { + items: Download[]; + limit: number; + offset: number; + clients_down: ClientDown[]; +}; // A release set aside: by a person, by its files, or by an import's check. export type ReleaseBlock = { info_hash: string; name: string; - reason: 'operator' | 'files' | 'import'; + reason: 'operator' | 'files' | 'import' | 'stalled'; detail: string; blocked_at: string; + // When a release given up on may be taken again; null is for good. + expires_at: string | null; want_title: string; want_artist: string; }; diff --git a/web/src/lib/api/wanted.ts b/web/src/lib/api/wanted.ts index f09aa9aa..147e5d28 100644 --- a/web/src/lib/api/wanted.ts +++ b/web/src/lib/api/wanted.ts @@ -160,6 +160,9 @@ export async function putSearchSettings(body: { search_interval_hours: number; allowed_qualities: string[]; import_root: string; + seed_ratio: number; + seed_days: number; + stall_hours: number; }): Promise { return api.put('/api/admin/acquisition/settings', body); } @@ -199,6 +202,12 @@ export async function refuseDownload(id: string): Promise { await api.post(`/api/admin/downloads/${id}/refuse`, {}); } +// A running download given up: the release is set aside for good, the +// download removed and the album searched for again (#5523). +export async function abandonDownload(id: string): Promise { + await api.post(`/api/admin/downloads/${id}/abandon`, {}); +} + export async function listReleaseBlocks(offset: number = 0): Promise { return api.get(`/api/admin/release-blocks?offset=${offset}`); } diff --git a/web/src/lib/components/SearchSettingsCard.svelte b/web/src/lib/components/SearchSettingsCard.svelte index f7831722..f256d50a 100644 --- a/web/src/lib/components/SearchSettingsCard.svelte +++ b/web/src/lib/components/SearchSettingsCard.svelte @@ -11,13 +11,23 @@ // for, whether the best release is taken without asking, and which // qualities are taken at all. The schedule wakes every ten minutes and // keeps to each indexer's searches-per-hour. A finished download is - // checked and filed into one of the library's folders (#5522). + // checked and filed into one of the library's folders (#5522); it then + // seeds until the ratio or the days below, and a download nobody shares + // is given up after the hours below (#5523). const client = useQueryClient(); const store = createSearchSettingsQuery(); const query = $derived($store); - type Form = { auto_grab: boolean; search_interval_hours: number; allowed: string[]; import_root: string }; + type Form = { + auto_grab: boolean; + search_interval_hours: number; + allowed: string[]; + import_root: string; + seed_ratio: number; + seed_days: number; + stall_hours: number; + }; // The form is the server's settings until edited; saved holds what a save // returned until the query catches up. @@ -32,7 +42,10 @@ auto_grab: s.auto_grab, search_interval_hours: s.search_interval_hours, allowed: [...s.allowed_qualities], - import_root: s.import_root + import_root: s.import_root, + seed_ratio: s.seed_ratio, + seed_days: s.seed_days, + stall_hours: s.stall_hours }; } @@ -55,7 +68,10 @@ auto_grab: form.auto_grab, search_interval_hours: Number(form.search_interval_hours), allowed_qualities: form.allowed, - import_root: form.import_root + import_root: form.import_root, + seed_ratio: Number(form.seed_ratio), + seed_days: Number(form.seed_days), + stall_hours: Number(form.stall_hours) }); edited = null; await client.invalidateQueries({ queryKey: qk.searchSettings() }); @@ -158,7 +174,7 @@ File new music into A download that checks out is copied here as Artist/Album (Year)/Artist - Album - NN - - Title, tagged with its MusicBrainz ids. The download itself keeps seeding. + Title, tagged with its MusicBrainz ids. edit({ seed_ratio: Number(e.currentTarget.value) })} + class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary + focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent" + /> + or for + edit({ seed_days: Number(e.currentTarget.value) })} + class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary + focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent" + /> + days + + + + +

{lastPass(query.data)}

diff --git a/web/src/lib/components/SearchSettingsCard.test.ts b/web/src/lib/components/SearchSettingsCard.test.ts index b93a0c86..b3cf0c4f 100644 --- a/web/src/lib/components/SearchSettingsCard.test.ts +++ b/web/src/lib/components/SearchSettingsCard.test.ts @@ -24,7 +24,10 @@ const settings: SearchSettings = { last_pass_at: new Date(Date.now() - 5 * 60_000).toISOString(), last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true }, import_root: '', - scan_paths: ['/music', '/music2'] + scan_paths: ['/music', '/music2'], + seed_ratio: 2, + seed_days: 14, + stall_hours: 6 }; afterEach(() => vi.clearAllMocks()); @@ -60,7 +63,10 @@ describe('SearchSettingsCard', () => { auto_grab: false, search_interval_hours: 24, allowed_qualities: ['lossless', 'lossy_high', 'unknown'], - import_root: '' + import_root: '', + seed_ratio: 2, + seed_days: 14, + stall_hours: 6 }) ); }); @@ -82,6 +88,20 @@ describe('SearchSettingsCard', () => { expect(screen.getByText('/music')).toBeTruthy(); }); + test('seeding and giving up on a stalled download are set here', async () => { + setup(); + expect((screen.getByLabelText(/seed ratio/i) as HTMLInputElement).value).toBe('2'); + await fireEvent.input(screen.getByLabelText(/seed ratio/i), { target: { value: '1.5' } }); + await fireEvent.input(screen.getByLabelText(/seed days/i), { target: { value: '7' } }); + await fireEvent.input(screen.getByLabelText(/stall hours/i), { target: { value: '0' } }); + await fireEvent.click(screen.getByRole('button', { name: /^save$/i })); + await waitFor(() => + expect(putSearchSettings).toHaveBeenCalledWith( + expect.objectContaining({ seed_ratio: 1.5, seed_days: 7, stall_hours: 0 }) + ) + ); + }); + test('before the first pass it says so', () => { setup({ ...settings, last_pass_at: null, last_pass: {} }); expect(screen.getByTestId('last-pass').textContent).toMatch(/hasn’t run yet/); diff --git a/web/src/lib/components/wanted/BlockList.svelte b/web/src/lib/components/wanted/BlockList.svelte index cd41332a..33b80bd3 100644 --- a/web/src/lib/components/wanted/BlockList.svelte +++ b/web/src/lib/components/wanted/BlockList.svelte @@ -8,9 +8,10 @@ import ListContinuation from '#lib/components/ListContinuation.svelte'; import type { ReleaseBlock } from '#lib/api/types.js'; - // The Blocked tab (M502 #5521–#5522): releases the search will never take + // The Blocked tab (M502 #5521–#5523): releases the search will not take // again — set aside by a person, by what their files were, or by an - // import's check that found they weren't the album. + // import's check that found they weren't the album; and, for a week, + // releases given up on because they would not finish. const client = useQueryClient(); const blocksStore = createReleaseBlocksQuery(); @@ -35,8 +36,13 @@ function by(b: ReleaseBlock): string { if (b.reason === 'operator') return 'Set aside by you'; if (b.reason === 'files') return 'Set aside for its files'; + if (b.reason === 'stalled') return 'Given up on: it would not finish'; return 'Refused on import'; } + + // A release given up on is held back for a while, then may be taken again. + const until = (b: ReleaseBlock) => + b.expires_at ? ` · held back until ${new Date(b.expires_at).toLocaleDateString()}` : ''; {#if blocks.isPending} @@ -61,7 +67,7 @@ {#if b.want_title}
For {b.want_artist} – {b.want_title}
{/if} -
{by(b)} {relativeTime(b.blocked_at)}
+
{by(b)} {relativeTime(b.blocked_at)}{until(b)}
{#if b.detail}

{b.detail}

{/if} diff --git a/web/src/lib/components/wanted/BlockList.test.ts b/web/src/lib/components/wanted/BlockList.test.ts index bd3574c3..ae4817e0 100644 --- a/web/src/lib/components/wanted/BlockList.test.ts +++ b/web/src/lib/components/wanted/BlockList.test.ts @@ -18,6 +18,7 @@ const block: ReleaseBlock = { reason: 'import', detail: 'The album tag says another version.', blocked_at: new Date(Date.now() - 3_600_000).toISOString(), + expires_at: null, want_title: 'Random Access Memories', want_artist: 'Daft Punk' }; @@ -43,6 +44,12 @@ describe('BlockList', () => { await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('abc123')); }); + test('a release given up on says until when it is held back', () => { + setup([{ ...block, reason: 'stalled', detail: 'Nobody was sharing it.', expires_at: '2026-10-17T12:00:00Z' }]); + const line = screen.getByText(/given up on: it would not finish/i).textContent ?? ''; + expect(line).toMatch(/held back until/); + }); + test('with nothing blocked it says so', () => { setup([]); expect(screen.getByText(/no releases are blocked/i)).toBeTruthy(); diff --git a/web/src/lib/components/wanted/DownloadList.svelte b/web/src/lib/components/wanted/DownloadList.svelte index 183babbb..7d6d0150 100644 --- a/web/src/lib/components/wanted/DownloadList.svelte +++ b/web/src/lib/components/wanted/DownloadList.svelte @@ -1,23 +1,32 @@ +{#each clientsDown as c (c.client)} +

+ Can't reach the download client {c.client} since {relativeTime(c.since)}: {c.detail}. Its + downloads are left as they are until it answers. +

+{/each} + {#if downloads.isPending}

Reading the downloads…

{:else if downloads.isError && !downloads.isFetchNextPageError} @@ -147,6 +203,12 @@
{/if} + {#if running(d) && d.condition} +

{condition(d.condition)}

+ {/if} + {#if ending(d)} +

{ending(d)}

+ {/if} {#if d.import_reason}

{d.import_reason} @@ -158,12 +220,21 @@ {#if filed(d)}

{filed(d)}

{/if} + {#if seeded(d)} +

{seeded(d)}

+ {/if} {#if d.rehomed_from}

Taken for {d.rehomed_from}; it turned out to be this album.

{/if}

{when(d)}

- {#if needsYou(d)} + {#if running(d)} +
+ +
+ {:else if needsYou(d)}
{#if d.import_state === 'review'}