feat: download lifecycle — stalled and failed downloads are given up and searched again; imported ones seed, then go (M502 #5523)
release / govulncheck (push) Successful in 17s
release / web (push) Failing after 17s
release / go (push) Successful in 1m11s
release / integration (push) Successful in 4m50s
release / android (push) Successful in 4m54s
release / Build signed APK (releases and dev) (push) Successful in 5m9s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 17s
release / web (push) Failing after 17s
release / go (push) Successful in 1m11s
release / integration (push) Successful in 4m50s
release / android (push) Successful in 4m54s
release / Build signed APK (releases and dev) (push) Successful in 5m9s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
Ported from Fabled Librarian's System 104. A new lifecycle poller, every ten minutes: - a download the client fails is given up at once; - one nobody shares, or still fetching its file list, past the stall setting (6 h) is asked for again three times, then given up; - giving up removes the download with its files, holds the release back for a week (release_blocks.expires_at), ends the grab and searches the album again, with no limit per album; - a running download the client stops listing for half an hour ends and is searched again, not held back; an empty list is not taken as gone; - an imported download seeds to ratio 2.0 or 14 days, then is removed; - a download the client would not remove is asked again every pass. Every condition and act is one open row per (client, download, condition) in download_interventions. Activity shows what is wrong with a download and how often it was retried, why one ended, its seeding, and a banner for a client that does not answer. A running download can be given up by hand (set aside for good). The seed ratio, seed days and stall hours are on the search settings card. The download client lookup moves to acquire.DownloadClientFor, shared by the importer and the lifecycle. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -24,6 +24,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/gc"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/importer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/logging"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/notifications"
|
||||
@@ -341,6 +342,11 @@ func run() error {
|
||||
}
|
||||
go importer.New(pool, scanner, cat, printer, cfg.Library.ScanPaths, logger.With("component", "import")).Run(ctx)
|
||||
|
||||
// Download lifecycle (M502 #5523): every ten minutes, a download that
|
||||
// failed or that nobody shares is retried, then given up and its album
|
||||
// searched again; an imported one seeds, then is removed.
|
||||
go lifecycle.New(pool, logger.With("component", "lifecycle")).Run(ctx)
|
||||
|
||||
// Duplicate resolver (M498, M502): classifies the sweep's groups and, with
|
||||
// the operator's auto-resolve on, merges the copies that are safe to merge.
|
||||
// Started after SetNotifier so its summaries reach admins. See
|
||||
|
||||
@@ -21,6 +21,9 @@ const (
|
||||
BlockFiles = "files"
|
||||
// BlockImport: the import found its files were not the album (#5522).
|
||||
BlockImport = "import"
|
||||
// BlockStalled: given up on because it would not finish (#5523); held
|
||||
// back until the block's expires_at.
|
||||
BlockStalled = "stalled"
|
||||
)
|
||||
|
||||
// Grab origins, matching grabs_origin_check.
|
||||
@@ -92,11 +95,22 @@ func (m *Memory) blocked(ctx context.Context) map[string]string {
|
||||
|
||||
// BlockSentence is what a blocked candidate says.
|
||||
func BlockSentence(reason, detail string) string {
|
||||
if reason == BlockFiles {
|
||||
switch reason {
|
||||
case BlockFiles:
|
||||
if detail != "" {
|
||||
return "Set aside for what its files hold: " + detail
|
||||
}
|
||||
return "Set aside for what its files hold."
|
||||
case BlockImport:
|
||||
if detail != "" {
|
||||
return "Refused on import: " + detail
|
||||
}
|
||||
return "Refused on import: its files were not the album."
|
||||
case BlockStalled:
|
||||
if detail != "" {
|
||||
return "Given up on for now: " + detail
|
||||
}
|
||||
return "Given up on for now: it would not finish."
|
||||
}
|
||||
return "You set this release aside."
|
||||
}
|
||||
|
||||
@@ -12,8 +12,11 @@ package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
@@ -154,6 +157,19 @@ func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) {
|
||||
return downloader.Mapped(d, downloader.PathMap{Local: row.LocalPath, Remote: row.RemotePath}), nil
|
||||
}
|
||||
|
||||
// DownloadClientFor builds the download client a grab went to, by the code
|
||||
// of its acquisition_services row.
|
||||
func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (downloader.Driver, error) {
|
||||
row, err := dbq.New(pool).GetAcquisitionService(ctx, code)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("the download client %q is no longer set up", code)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return DownloadClient(row)
|
||||
}
|
||||
|
||||
// Sources builds every enabled search source.
|
||||
func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
|
||||
rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/importer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle"
|
||||
)
|
||||
|
||||
// Downloads and their imports (M502 #5522): the Wanted view's Activity tab,
|
||||
@@ -40,6 +41,24 @@ type downloadView struct {
|
||||
Files []string `json:"files"`
|
||||
Skipped int `json:"skipped"`
|
||||
RehomedFrom string `json:"rehomed_from,omitempty"`
|
||||
// EndDetail says why an ended download ended (#5523).
|
||||
EndDetail string `json:"end_detail"`
|
||||
// Seeding: imported and still sharing; Ratio its share ratio as last
|
||||
// read; SeedReleasedAt when it was removed once it had shared enough.
|
||||
Seeding bool `json:"seeding"`
|
||||
Ratio float32 `json:"ratio"`
|
||||
SeedReleasedAt *string `json:"seed_released_at"`
|
||||
// Condition is what is wrong with it now, if anything.
|
||||
Condition *conditionView `json:"condition"`
|
||||
}
|
||||
|
||||
// conditionView is an open row of the intervention ledger: what was seen,
|
||||
// since when, and how many times it was retried.
|
||||
type conditionView struct {
|
||||
Trigger string `json:"trigger"`
|
||||
Detail string `json:"detail"`
|
||||
Retries int32 `json:"retries"`
|
||||
Since string `json:"since"`
|
||||
}
|
||||
|
||||
func downloadFrom(g dbq.ListGrabsRow) downloadView {
|
||||
@@ -48,6 +67,18 @@ func downloadFrom(g dbq.ListGrabsRow) downloadView {
|
||||
Origin: g.Origin, GrabbedAt: formatTimestamp(g.GrabbedAt), EndReason: g.EndReason,
|
||||
Progress: g.Progress, ImportState: g.ImportState, ImportReason: g.ImportReason,
|
||||
ImportedFiles: g.ImportedFiles, Destination: g.Destination, Files: []string{},
|
||||
EndDetail: g.EndDetail, Seeding: g.Seeding, Ratio: g.Ratio,
|
||||
}
|
||||
if g.SeedReleasedAt.Valid {
|
||||
at := formatTimestamp(g.SeedReleasedAt)
|
||||
v.SeedReleasedAt = &at
|
||||
}
|
||||
if g.Condition != nil {
|
||||
v.Condition = &conditionView{Trigger: *g.Condition, Detail: deref(g.ConditionDetail),
|
||||
Since: formatTimestamp(g.ConditionSince)}
|
||||
if g.ConditionRetries != nil {
|
||||
v.Condition.Retries = *g.ConditionRetries
|
||||
}
|
||||
}
|
||||
if g.WantID.Valid {
|
||||
id := uuidToString(g.WantID)
|
||||
@@ -82,6 +113,14 @@ type downloadsPage struct {
|
||||
Items []downloadView `json:"items"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
// ClientsDown are the download clients not answering now (#5523).
|
||||
ClientsDown []clientDownView `json:"clients_down"`
|
||||
}
|
||||
|
||||
type clientDownView struct {
|
||||
Client string `json:"client"`
|
||||
Detail string `json:"detail"`
|
||||
Since string `json:"since"`
|
||||
}
|
||||
|
||||
// handleListDownloads implements GET /api/admin/downloads?offset=: running
|
||||
@@ -97,13 +136,47 @@ func (h *handlers) handleListDownloads(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrWithLog(w, h.logger, "admin downloads: list", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset}
|
||||
down, err := dbq.New(h.pool).ListClientsDown(r.Context())
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin downloads: clients down", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset,
|
||||
ClientsDown: make([]clientDownView, 0, len(down))}
|
||||
for _, g := range rows {
|
||||
out.Items = append(out.Items, downloadFrom(g))
|
||||
}
|
||||
for _, c := range down {
|
||||
out.ClientsDown = append(out.ClientsDown, clientDownView{Client: c.Client, Detail: c.Detail,
|
||||
Since: formatTimestamp(c.FirstSeen)})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// handleAbandonDownload implements POST /api/admin/downloads/{id}/abandon: a
|
||||
// running download is given up, its release set aside, the download removed
|
||||
// and the album searched again.
|
||||
func (h *handlers) handleAbandonDownload(w http.ResponseWriter, r *http.Request) {
|
||||
admin, ok := requireUser(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
id, ok := requireURLUUID(w, r, "id")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if _, err := h.lifecycle.Abandon(r.Context(), id, admin.ID); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows), errors.Is(err, lifecycle.ErrNotAbandonable):
|
||||
writeErr(w, apierror.Conflict("not_running", "that download is no longer running"))
|
||||
default:
|
||||
writeErrWithLog(w, h.logger, "admin downloads: abandon", apierror.Internal(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleImportAnyway implements POST /api/admin/downloads/{id}/import: a
|
||||
// download the check held for a person (or one that failed) is imported on
|
||||
// the next pass whatever the check says.
|
||||
@@ -160,13 +233,16 @@ func (h *handlers) writeImportErr(w http.ResponseWriter, what string, err error)
|
||||
}
|
||||
|
||||
type releaseBlockView struct {
|
||||
InfoHash string `json:"info_hash"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Detail string `json:"detail"`
|
||||
BlockedAt string `json:"blocked_at"`
|
||||
WantTitle string `json:"want_title"`
|
||||
WantArtist string `json:"want_artist"`
|
||||
InfoHash string `json:"info_hash"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Detail string `json:"detail"`
|
||||
BlockedAt string `json:"blocked_at"`
|
||||
// ExpiresAt is when a release given up on may be taken again; null is
|
||||
// for good (#5523).
|
||||
ExpiresAt *string `json:"expires_at"`
|
||||
WantTitle string `json:"want_title"`
|
||||
WantArtist string `json:"want_artist"`
|
||||
}
|
||||
|
||||
type releaseBlocksPage struct {
|
||||
@@ -193,6 +269,10 @@ func (h *handlers) handleListReleaseBlocks(w http.ResponseWriter, r *http.Reques
|
||||
for _, b := range rows {
|
||||
v := releaseBlockView{InfoHash: b.InfoHash, Name: b.Name, Reason: b.Reason, Detail: b.Detail,
|
||||
BlockedAt: formatTimestamp(b.BlockedAt)}
|
||||
if b.ExpiresAt.Valid {
|
||||
at := formatTimestamp(b.ExpiresAt)
|
||||
v.ExpiresAt = &at
|
||||
}
|
||||
if b.WantTitle != nil {
|
||||
v.WantTitle = *b.WantTitle
|
||||
}
|
||||
|
||||
@@ -37,6 +37,12 @@ type searchSettingsView struct {
|
||||
// is the first of ScanPaths, the library's folders.
|
||||
ImportRoot string `json:"import_root"`
|
||||
ScanPaths []string `json:"scan_paths"`
|
||||
// The download lifecycle (#5523): seed an imported download to this
|
||||
// ratio or for this many days; give up on one nobody shares after this
|
||||
// many hours. 0 is: remove once imported, no time limit, never.
|
||||
SeedRatio float32 `json:"seed_ratio"`
|
||||
SeedDays int32 `json:"seed_days"`
|
||||
StallHours int32 `json:"stall_hours"`
|
||||
}
|
||||
|
||||
func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView {
|
||||
@@ -44,6 +50,7 @@ func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView
|
||||
AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours,
|
||||
AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass,
|
||||
ImportRoot: s.ImportRoot, ScanPaths: roots,
|
||||
SeedRatio: s.SeedRatio, SeedDays: s.SeedDays, StallHours: s.StallHours,
|
||||
}
|
||||
if v.ScanPaths == nil {
|
||||
v.ScanPaths = []string{}
|
||||
@@ -76,6 +83,9 @@ type putSearchSettingsBody struct {
|
||||
SearchIntervalHours int32 `json:"search_interval_hours"`
|
||||
AllowedQualities []string `json:"allowed_qualities"`
|
||||
ImportRoot string `json:"import_root"`
|
||||
SeedRatio float32 `json:"seed_ratio"`
|
||||
SeedDays int32 `json:"seed_days"`
|
||||
StallHours int32 `json:"stall_hours"`
|
||||
}
|
||||
|
||||
// handlePutSearchSettings implements PUT /api/admin/acquisition/settings.
|
||||
@@ -102,9 +112,15 @@ func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Reques
|
||||
writeErr(w, apierror.BadRequest("bad_import_root", "file new music into one of the library's folders"))
|
||||
return
|
||||
}
|
||||
if body.SeedRatio < 0 || body.SeedRatio > 100 || body.SeedDays < 0 || body.SeedDays > 365 ||
|
||||
body.StallHours < 0 || body.StallHours > 720 {
|
||||
writeErr(w, apierror.BadRequest("bad_lifecycle",
|
||||
"seed ratio must be 0 to 100, seed days 0 to 365 and stall hours 0 to 720"))
|
||||
return
|
||||
}
|
||||
s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{
|
||||
AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities,
|
||||
ImportRoot: body.ImportRoot,
|
||||
ImportRoot: body.ImportRoot, SeedRatio: body.SeedRatio, SeedDays: body.SeedDays, StallHours: body.StallHours,
|
||||
})
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err))
|
||||
|
||||
@@ -62,6 +62,7 @@ type wantGrabView struct {
|
||||
Progress float32 `json:"progress"`
|
||||
ImportState string `json:"import_state"`
|
||||
ImportReason string `json:"import_reason"`
|
||||
EndDetail string `json:"end_detail"`
|
||||
ImportedAt *string `json:"imported_at"`
|
||||
ImportedFiles int32 `json:"imported_files"`
|
||||
}
|
||||
@@ -164,7 +165,8 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) {
|
||||
v.Grab = &wantGrabView{ID: uuidToString(row.GrabID), Name: *row.GrabName,
|
||||
GrabbedAt: formatTimestamp(row.GrabbedAt), Live: !row.GrabEndedAt.Valid,
|
||||
Origin: deref(row.GrabOrigin), Quality: deref(row.GrabQuality), EndReason: deref(row.GrabEndReason),
|
||||
ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason)}
|
||||
ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason),
|
||||
EndDetail: deref(row.GrabEndDetail)}
|
||||
if row.GrabProgress != nil {
|
||||
v.Grab.Progress = *row.GrabProgress
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lidarrconfig"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lidarrquarantine"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/netsettings"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/notifications"
|
||||
@@ -80,6 +81,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
||||
wanted: wanted.New(pool, cat, logger),
|
||||
searcher: wanted.NewSearcher(pool, cat, logger),
|
||||
importer: importer.New(pool, scanner, cat, nil, scannerRoots(scanner), logger),
|
||||
lifecycle: lifecycle.New(pool, logger),
|
||||
catalogueSearches: newSearchCache(),
|
||||
catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)),
|
||||
leveled: newLeveledRenderer(dataDir, loudSettings, logger),
|
||||
@@ -264,6 +266,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
||||
admin.Post("/downloads/{id}/import", h.handleImportAnyway)
|
||||
admin.Post("/downloads/{id}/retry", h.handleRetryImport)
|
||||
admin.Post("/downloads/{id}/refuse", h.handleRefuseDownload)
|
||||
admin.Post("/downloads/{id}/abandon", h.handleAbandonDownload)
|
||||
admin.Get("/follows", h.handleListFollows)
|
||||
admin.Get("/follows/{mbid}", h.handleGetFollow)
|
||||
admin.Put("/follows/{mbid}", h.handlePutFollow)
|
||||
@@ -437,6 +440,9 @@ type handlers struct {
|
||||
// importer refuses a download at a person's word (#5522). The import
|
||||
// loop itself runs in main; this one only acts on a request.
|
||||
importer *importer.Importer
|
||||
// lifecycle gives up on a download at a person's word (#5523); its loop
|
||||
// runs in main.
|
||||
lifecycle *lifecycle.Service
|
||||
// catalogueSearches reuses recent MusicBrainz answers; nil asks every
|
||||
// time, which is what handler tests get.
|
||||
catalogueSearches *searchCache
|
||||
|
||||
@@ -16,7 +16,7 @@ UPDATE grabs
|
||||
SET import_state = 'importing', import_attempts = import_attempts + 1,
|
||||
import_updated_at = now(), progress = 1
|
||||
WHERE id = $1 AND ended_at IS NULL AND import_state IN ('', 'downloading', 'waiting')
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
// Takes a finished download for import. The update is the lock: a second
|
||||
@@ -52,6 +52,9 @@ func (q *Queries) ClaimGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -60,7 +63,7 @@ const forceGrabImport = `-- name: ForceGrabImport :one
|
||||
UPDATE grabs
|
||||
SET import_state = '', import_forced = true, import_reason = '', import_updated_at = now()
|
||||
WHERE id = $1 AND ended_at IS NULL AND import_state IN ('review', 'failed')
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
// "Import anyway": the next pass imports the download whatever the check said.
|
||||
@@ -95,12 +98,15 @@ func (q *Queries) ForceGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getGrab = `-- name: GetGrab :one
|
||||
SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE id = $1
|
||||
SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at FROM grabs WHERE id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) {
|
||||
@@ -134,6 +140,9 @@ func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) {
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -202,9 +211,16 @@ const listGrabs = `-- name: ListGrabs :many
|
||||
SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced,
|
||||
g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state,
|
||||
g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files,
|
||||
g.destination, w.title AS want_title, w.artist_name AS want_artist
|
||||
g.destination, g.seeding, g.ratio, g.seed_released_at,
|
||||
w.title AS want_title, w.artist_name AS want_artist,
|
||||
di.trigger AS condition, di.detail AS condition_detail, di.remediations AS condition_retries,
|
||||
di.first_seen AS condition_since
|
||||
FROM grabs g
|
||||
LEFT JOIN wants w ON w.id = g.want_id
|
||||
LEFT JOIN download_interventions di
|
||||
ON di.id = (SELECT x.id FROM download_interventions x
|
||||
WHERE x.grab_id = g.id AND x.outcome = 'open'
|
||||
ORDER BY x.last_seen DESC LIMIT 1)
|
||||
ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC,
|
||||
g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id
|
||||
LIMIT $2 OFFSET $1
|
||||
@@ -216,32 +232,40 @@ type ListGrabsParams struct {
|
||||
}
|
||||
|
||||
type ListGrabsRow struct {
|
||||
ID pgtype.UUID
|
||||
WantID pgtype.UUID
|
||||
Name string
|
||||
Indexer string
|
||||
Client string
|
||||
Quality string
|
||||
Origin string
|
||||
Forced bool
|
||||
GrabbedAt pgtype.Timestamptz
|
||||
EndedAt pgtype.Timestamptz
|
||||
EndReason string
|
||||
EndDetail string
|
||||
Progress float32
|
||||
ImportState string
|
||||
ImportReason string
|
||||
ImportDetail []byte
|
||||
ImportUpdatedAt pgtype.Timestamptz
|
||||
ImportedAt pgtype.Timestamptz
|
||||
ImportedFiles int32
|
||||
Destination string
|
||||
WantTitle *string
|
||||
WantArtist *string
|
||||
ID pgtype.UUID
|
||||
WantID pgtype.UUID
|
||||
Name string
|
||||
Indexer string
|
||||
Client string
|
||||
Quality string
|
||||
Origin string
|
||||
Forced bool
|
||||
GrabbedAt pgtype.Timestamptz
|
||||
EndedAt pgtype.Timestamptz
|
||||
EndReason string
|
||||
EndDetail string
|
||||
Progress float32
|
||||
ImportState string
|
||||
ImportReason string
|
||||
ImportDetail []byte
|
||||
ImportUpdatedAt pgtype.Timestamptz
|
||||
ImportedAt pgtype.Timestamptz
|
||||
ImportedFiles int32
|
||||
Destination string
|
||||
Seeding bool
|
||||
Ratio float32
|
||||
SeedReleasedAt pgtype.Timestamptz
|
||||
WantTitle *string
|
||||
WantArtist *string
|
||||
Condition *string
|
||||
ConditionDetail *string
|
||||
ConditionRetries *int32
|
||||
ConditionSince pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// The Wanted view's Activity tab: downloads waiting on a person first, then
|
||||
// the rest still running, then the ended ones newest first.
|
||||
// the rest still running, then the ended ones newest first. Each carries
|
||||
// what is wrong with it now, if anything (#5523).
|
||||
func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGrabsRow, error) {
|
||||
rows, err := q.db.Query(ctx, listGrabs, arg.Skip, arg.MaxRows)
|
||||
if err != nil {
|
||||
@@ -272,8 +296,15 @@ func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGra
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
&i.WantTitle,
|
||||
&i.WantArtist,
|
||||
&i.Condition,
|
||||
&i.ConditionDetail,
|
||||
&i.ConditionRetries,
|
||||
&i.ConditionSince,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -287,7 +318,7 @@ func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGra
|
||||
|
||||
const listLiveGrabs = `-- name: ListLiveGrabs :many
|
||||
|
||||
SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title,
|
||||
SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, g.seeding, g.ratio, g.seed_released_at, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title,
|
||||
w.reason AS want_reason, w.album_id AS want_album_id
|
||||
FROM grabs g
|
||||
LEFT JOIN wants w ON w.id = g.want_id
|
||||
@@ -323,6 +354,9 @@ type ListLiveGrabsRow struct {
|
||||
ImportedAt pgtype.Timestamptz
|
||||
ImportedFiles int32
|
||||
Destination string
|
||||
Seeding bool
|
||||
Ratio float32
|
||||
SeedReleasedAt pgtype.Timestamptz
|
||||
ReleaseGroupMbid *string
|
||||
ArtistMbid *string
|
||||
ArtistName *string
|
||||
@@ -370,6 +404,9 @@ func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error)
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
&i.ReleaseGroupMbid,
|
||||
&i.ArtistMbid,
|
||||
&i.ArtistName,
|
||||
@@ -388,10 +425,11 @@ func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error)
|
||||
}
|
||||
|
||||
const listReleaseBlocksPage = `-- name: ListReleaseBlocksPage :many
|
||||
SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title,
|
||||
SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, b.expires_at, w.title AS want_title,
|
||||
w.artist_name AS want_artist
|
||||
FROM release_blocks b
|
||||
LEFT JOIN wants w ON w.id = b.want_id
|
||||
WHERE b.expires_at IS NULL OR b.expires_at > now()
|
||||
ORDER BY b.blocked_at DESC, b.info_hash
|
||||
LIMIT $2 OFFSET $1
|
||||
`
|
||||
@@ -407,6 +445,7 @@ type ListReleaseBlocksPageRow struct {
|
||||
Reason string
|
||||
Detail string
|
||||
BlockedAt pgtype.Timestamptz
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
WantTitle *string
|
||||
WantArtist *string
|
||||
}
|
||||
@@ -426,6 +465,7 @@ func (q *Queries) ListReleaseBlocksPage(ctx context.Context, arg ListReleaseBloc
|
||||
&i.Reason,
|
||||
&i.Detail,
|
||||
&i.BlockedAt,
|
||||
&i.ExpiresAt,
|
||||
&i.WantTitle,
|
||||
&i.WantArtist,
|
||||
); err != nil {
|
||||
@@ -522,13 +562,16 @@ UPDATE grabs
|
||||
import_updated_at = now(),
|
||||
import_forced = false,
|
||||
imported_at = CASE WHEN $1::text = 'imported' THEN now() ELSE imported_at END,
|
||||
-- The files were copied, so an imported download goes on sharing
|
||||
-- until the lifecycle removes it (#5523).
|
||||
seeding = $1::text = 'imported',
|
||||
imported_files = $4,
|
||||
destination = $5,
|
||||
ended_at = CASE WHEN $6::text <> '' THEN now() ELSE ended_at END,
|
||||
end_reason = $6,
|
||||
end_detail = CASE WHEN $6::text <> '' THEN $2::text ELSE end_detail END
|
||||
WHERE id = $7
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
type RecordImportParams struct {
|
||||
@@ -582,6 +625,9 @@ func (q *Queries) RecordImport(ctx context.Context, arg RecordImportParams) (Gra
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -591,7 +637,7 @@ UPDATE grabs g
|
||||
SET want_id = $1
|
||||
WHERE g.id = $2 AND g.ended_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM grabs o WHERE o.want_id = $1 AND o.ended_at IS NULL)
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
type RehomeGrabParams struct {
|
||||
@@ -632,6 +678,9 @@ func (q *Queries) RehomeGrab(ctx context.Context, arg RehomeGrabParams) (Grab, e
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -640,7 +689,7 @@ const retryGrabImport = `-- name: RetryGrabImport :one
|
||||
UPDATE grabs
|
||||
SET import_state = '', import_reason = '', import_updated_at = now()
|
||||
WHERE id = $1 AND ended_at IS NULL AND import_state = 'failed'
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) {
|
||||
@@ -674,6 +723,9 @@ func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: lifecycle.sql
|
||||
|
||||
package dbq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const countRemediation = `-- name: CountRemediation :one
|
||||
UPDATE download_interventions
|
||||
SET remediations = remediations + 1,
|
||||
action = $1,
|
||||
last_seen = now(),
|
||||
outcome = CASE WHEN $1::text IN ('given_up', 'removed') THEN 'resolved' ELSE outcome END,
|
||||
settled_at = CASE WHEN $1::text IN ('given_up', 'removed') THEN now() ELSE settled_at END
|
||||
WHERE client = $2 AND info_hash = $3
|
||||
AND trigger = $4 AND outcome = 'open'
|
||||
RETURNING remediations
|
||||
`
|
||||
|
||||
type CountRemediationParams struct {
|
||||
Action string
|
||||
Client string
|
||||
InfoHash string
|
||||
Trigger string
|
||||
}
|
||||
|
||||
// Minstrel acted on an open condition. A retry climbs the ladder; giving up
|
||||
// or removing ends the condition, since what it was about is gone. Written
|
||||
// before the act, so a retry that happened is never uncounted.
|
||||
func (q *Queries) CountRemediation(ctx context.Context, arg CountRemediationParams) (int32, error) {
|
||||
row := q.db.QueryRow(ctx, countRemediation,
|
||||
arg.Action,
|
||||
arg.Client,
|
||||
arg.InfoHash,
|
||||
arg.Trigger,
|
||||
)
|
||||
var remediations int32
|
||||
err := row.Scan(&remediations)
|
||||
return remediations, err
|
||||
}
|
||||
|
||||
const deleteExpiredBlocks = `-- name: DeleteExpiredBlocks :execrows
|
||||
DELETE FROM release_blocks WHERE expires_at IS NOT NULL AND expires_at <= now()
|
||||
`
|
||||
|
||||
// A release given up on is held back for a while; after that the search may
|
||||
// take it again.
|
||||
func (q *Queries) DeleteExpiredBlocks(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, deleteExpiredBlocks)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const giveUpGrab = `-- name: GiveUpGrab :one
|
||||
UPDATE grabs
|
||||
SET ended_at = now(), end_reason = $1, end_detail = $2,
|
||||
import_updated_at = now()
|
||||
WHERE id = $3 AND ended_at IS NULL AND import_state = ANY($4::text[])
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
type GiveUpGrabParams struct {
|
||||
EndReason string
|
||||
EndDetail string
|
||||
ID pgtype.UUID
|
||||
States []string
|
||||
}
|
||||
|
||||
// Ends a running grab, given up on. Only from the states named: the update
|
||||
// is the lock against an import that claimed the download meanwhile.
|
||||
func (q *Queries) GiveUpGrab(ctx context.Context, arg GiveUpGrabParams) (Grab, error) {
|
||||
row := q.db.QueryRow(ctx, giveUpGrab,
|
||||
arg.EndReason,
|
||||
arg.EndDetail,
|
||||
arg.ID,
|
||||
arg.States,
|
||||
)
|
||||
var i Grab
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.WantID,
|
||||
&i.InfoHash,
|
||||
&i.Name,
|
||||
&i.Key,
|
||||
&i.Indexer,
|
||||
&i.Source,
|
||||
&i.Client,
|
||||
&i.SavePath,
|
||||
&i.Quality,
|
||||
&i.Origin,
|
||||
&i.Forced,
|
||||
&i.GrabbedBy,
|
||||
&i.GrabbedAt,
|
||||
&i.EndedAt,
|
||||
&i.EndReason,
|
||||
&i.EndDetail,
|
||||
&i.Progress,
|
||||
&i.ImportState,
|
||||
&i.ImportReason,
|
||||
&i.ImportDetail,
|
||||
&i.ImportForced,
|
||||
&i.ImportAttempts,
|
||||
&i.ImportUpdatedAt,
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const listClientsDown = `-- name: ListClientsDown :many
|
||||
SELECT client, detail, first_seen FROM download_interventions
|
||||
WHERE trigger = 'client_unreachable' AND outcome = 'open'
|
||||
ORDER BY client
|
||||
`
|
||||
|
||||
type ListClientsDownRow struct {
|
||||
Client string
|
||||
Detail string
|
||||
FirstSeen pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// Download clients not answering now, for the Activity tab.
|
||||
func (q *Queries) ListClientsDown(ctx context.Context) ([]ListClientsDownRow, error) {
|
||||
rows, err := q.db.Query(ctx, listClientsDown)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListClientsDownRow
|
||||
for rows.Next() {
|
||||
var i ListClientsDownRow
|
||||
if err := rows.Scan(&i.Client, &i.Detail, &i.FirstSeen); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listLeftovers = `-- name: ListLeftovers :many
|
||||
SELECT id, client, info_hash, grab_id, name, trigger, action, outcome, detail, observations, remediations, first_seen, last_seen, settled_at FROM download_interventions
|
||||
WHERE trigger = 'not_removed' AND outcome = 'open' AND info_hash <> ''
|
||||
ORDER BY first_seen
|
||||
`
|
||||
|
||||
// Downloads given up on that the client would not remove; each pass asks
|
||||
// again.
|
||||
func (q *Queries) ListLeftovers(ctx context.Context) ([]DownloadIntervention, error) {
|
||||
rows, err := q.db.Query(ctx, listLeftovers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []DownloadIntervention
|
||||
for rows.Next() {
|
||||
var i DownloadIntervention
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Client,
|
||||
&i.InfoHash,
|
||||
&i.GrabID,
|
||||
&i.Name,
|
||||
&i.Trigger,
|
||||
&i.Action,
|
||||
&i.Outcome,
|
||||
&i.Detail,
|
||||
&i.Observations,
|
||||
&i.Remediations,
|
||||
&i.FirstSeen,
|
||||
&i.LastSeen,
|
||||
&i.SettledAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listSeedingGrabs = `-- name: ListSeedingGrabs :many
|
||||
SELECT id, client, info_hash, name, imported_at FROM grabs
|
||||
WHERE seeding AND seed_released_at IS NULL
|
||||
ORDER BY imported_at, id
|
||||
`
|
||||
|
||||
type ListSeedingGrabsRow struct {
|
||||
ID pgtype.UUID
|
||||
Client string
|
||||
InfoHash string
|
||||
Name string
|
||||
ImportedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// Imported downloads still sharing, oldest import first.
|
||||
func (q *Queries) ListSeedingGrabs(ctx context.Context) ([]ListSeedingGrabsRow, error) {
|
||||
rows, err := q.db.Query(ctx, listSeedingGrabs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListSeedingGrabsRow
|
||||
for rows.Next() {
|
||||
var i ListSeedingGrabsRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Client,
|
||||
&i.InfoHash,
|
||||
&i.Name,
|
||||
&i.ImportedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const liveGrabOnHash = `-- name: LiveGrabOnHash :one
|
||||
SELECT EXISTS (SELECT 1 FROM grabs WHERE info_hash = $1 AND ended_at IS NULL)::bool
|
||||
`
|
||||
|
||||
func (q *Queries) LiveGrabOnHash(ctx context.Context, infoHash string) (bool, error) {
|
||||
row := q.db.QueryRow(ctx, liveGrabOnHash, infoHash)
|
||||
var column_1 bool
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const raiseIntervention = `-- name: RaiseIntervention :one
|
||||
|
||||
INSERT INTO download_interventions (client, info_hash, grab_id, name, trigger, detail)
|
||||
VALUES ($1, $2, $3, $4,
|
||||
$5, $6)
|
||||
ON CONFLICT (client, info_hash, trigger) WHERE outcome = 'open' DO UPDATE SET
|
||||
observations = download_interventions.observations + 1,
|
||||
last_seen = now(),
|
||||
detail = EXCLUDED.detail,
|
||||
grab_id = COALESCE(EXCLUDED.grab_id, download_interventions.grab_id),
|
||||
name = CASE WHEN EXCLUDED.name <> '' THEN EXCLUDED.name ELSE download_interventions.name END
|
||||
RETURNING id, client, info_hash, grab_id, name, trigger, action, outcome, detail, observations, remediations, first_seen, last_seen, settled_at
|
||||
`
|
||||
|
||||
type RaiseInterventionParams struct {
|
||||
Client string
|
||||
InfoHash string
|
||||
GrabID pgtype.UUID
|
||||
Name string
|
||||
Trigger string
|
||||
Detail string
|
||||
}
|
||||
|
||||
// The download lifecycle (M502 #5523): the intervention ledger, giving a
|
||||
// download up, and seeding after import.
|
||||
// A condition seen, or seen again: a second pass lands on the open row, so
|
||||
// a download stuck for a week is one row that says how long. The name only
|
||||
// improves, and the detail is the latest word.
|
||||
func (q *Queries) RaiseIntervention(ctx context.Context, arg RaiseInterventionParams) (DownloadIntervention, error) {
|
||||
row := q.db.QueryRow(ctx, raiseIntervention,
|
||||
arg.Client,
|
||||
arg.InfoHash,
|
||||
arg.GrabID,
|
||||
arg.Name,
|
||||
arg.Trigger,
|
||||
arg.Detail,
|
||||
)
|
||||
var i DownloadIntervention
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Client,
|
||||
&i.InfoHash,
|
||||
&i.GrabID,
|
||||
&i.Name,
|
||||
&i.Trigger,
|
||||
&i.Action,
|
||||
&i.Outcome,
|
||||
&i.Detail,
|
||||
&i.Observations,
|
||||
&i.Remediations,
|
||||
&i.FirstSeen,
|
||||
&i.LastSeen,
|
||||
&i.SettledAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const releaseSeed = `-- name: ReleaseSeed :exec
|
||||
UPDATE grabs SET seed_released_at = now() WHERE id = $1 AND seed_released_at IS NULL
|
||||
`
|
||||
|
||||
// The seeding download was removed, or found gone.
|
||||
func (q *Queries) ReleaseSeed(ctx context.Context, id pgtype.UUID) error {
|
||||
_, err := q.db.Exec(ctx, releaseSeed, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const setGrabRatio = `-- name: SetGrabRatio :exec
|
||||
UPDATE grabs SET ratio = $1 WHERE id = $2
|
||||
`
|
||||
|
||||
type SetGrabRatioParams struct {
|
||||
Ratio float32
|
||||
ID pgtype.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) SetGrabRatio(ctx context.Context, arg SetGrabRatioParams) error {
|
||||
_, err := q.db.Exec(ctx, setGrabRatio, arg.Ratio, arg.ID)
|
||||
return err
|
||||
}
|
||||
|
||||
const settleEndedGrabs = `-- name: SettleEndedGrabs :execrows
|
||||
UPDATE download_interventions di
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
FROM grabs g
|
||||
WHERE g.id = di.grab_id AND g.ended_at IS NOT NULL
|
||||
AND di.outcome = 'open' AND di.trigger <> 'not_removed'
|
||||
`
|
||||
|
||||
// What was wrong with a download whose grab has ended (imported, refused,
|
||||
// given up) is over, except a download left in the client to remove.
|
||||
func (q *Queries) SettleEndedGrabs(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, settleEndedGrabs)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const settleIntervention = `-- name: SettleIntervention :execrows
|
||||
UPDATE download_interventions
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
WHERE client = $1 AND info_hash = $2
|
||||
AND trigger = $3 AND outcome = 'open'
|
||||
`
|
||||
|
||||
type SettleInterventionParams struct {
|
||||
Client string
|
||||
InfoHash string
|
||||
Trigger string
|
||||
}
|
||||
|
||||
// The condition cleared.
|
||||
func (q *Queries) SettleIntervention(ctx context.Context, arg SettleInterventionParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, settleIntervention, arg.Client, arg.InfoHash, arg.Trigger)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const settleInterventionsFor = `-- name: SettleInterventionsFor :execrows
|
||||
UPDATE download_interventions
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
WHERE client = $1 AND info_hash = $2 AND outcome = 'open'
|
||||
`
|
||||
|
||||
type SettleInterventionsForParams struct {
|
||||
Client string
|
||||
InfoHash string
|
||||
}
|
||||
|
||||
// Everything open about one download is over: it is gone from the client.
|
||||
func (q *Queries) SettleInterventionsFor(ctx context.Context, arg SettleInterventionsForParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, settleInterventionsFor, arg.Client, arg.InfoHash)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
@@ -226,6 +226,9 @@ type AcquisitionSetting struct {
|
||||
LastPass []byte
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
ImportRoot string
|
||||
SeedRatio float32
|
||||
SeedDays int32
|
||||
StallHours int32
|
||||
}
|
||||
|
||||
type Album struct {
|
||||
@@ -355,6 +358,23 @@ type DiscoverTuning struct {
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type DownloadIntervention struct {
|
||||
ID pgtype.UUID
|
||||
Client string
|
||||
InfoHash string
|
||||
GrabID pgtype.UUID
|
||||
Name string
|
||||
Trigger string
|
||||
Action string
|
||||
Outcome string
|
||||
Detail string
|
||||
Observations int32
|
||||
Remediations int32
|
||||
FirstSeen pgtype.Timestamptz
|
||||
LastSeen pgtype.Timestamptz
|
||||
SettledAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type DuplicateGroup struct {
|
||||
ID pgtype.UUID
|
||||
MemberKey string
|
||||
@@ -459,6 +479,9 @@ type Grab struct {
|
||||
ImportedAt pgtype.Timestamptz
|
||||
ImportedFiles int32
|
||||
Destination string
|
||||
Seeding bool
|
||||
Ratio float32
|
||||
SeedReleasedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type LibraryChange struct {
|
||||
@@ -701,6 +724,7 @@ type ReleaseBlock struct {
|
||||
WantID pgtype.UUID
|
||||
BlockedBy pgtype.UUID
|
||||
BlockedAt pgtype.Timestamptz
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ReleaseTypeProfile struct {
|
||||
|
||||
@@ -12,11 +12,13 @@ import (
|
||||
)
|
||||
|
||||
const blockRelease = `-- name: BlockRelease :exec
|
||||
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by)
|
||||
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by, expires_at)
|
||||
VALUES ($1, $2, $3, $4,
|
||||
$5, $6)
|
||||
$5, $6, $7)
|
||||
ON CONFLICT (info_hash) DO UPDATE
|
||||
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now()
|
||||
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(),
|
||||
expires_at = EXCLUDED.expires_at, blocked_by = EXCLUDED.blocked_by
|
||||
WHERE release_blocks.expires_at IS NOT NULL OR EXCLUDED.expires_at IS NULL
|
||||
`
|
||||
|
||||
type BlockReleaseParams struct {
|
||||
@@ -26,8 +28,12 @@ type BlockReleaseParams struct {
|
||||
Detail string
|
||||
WantID pgtype.UUID
|
||||
BlockedBy pgtype.UUID
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// A release set aside. expires_at NULL is for good. A block for good is never
|
||||
// shortened by a timed one: only a timed block, or another for good, is
|
||||
// replaced.
|
||||
func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) error {
|
||||
_, err := q.db.Exec(ctx, blockRelease,
|
||||
arg.InfoHash,
|
||||
@@ -36,6 +42,7 @@ func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) erro
|
||||
arg.Detail,
|
||||
arg.WantID,
|
||||
arg.BlockedBy,
|
||||
arg.ExpiresAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -89,7 +96,7 @@ func (q *Queries) EndGrab(ctx context.Context, arg EndGrabParams) error {
|
||||
|
||||
const getAcquisitionSettings = `-- name: GetAcquisitionSettings :one
|
||||
|
||||
SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root FROM acquisition_settings WHERE id = true
|
||||
SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root, seed_ratio, seed_days, stall_hours FROM acquisition_settings WHERE id = true
|
||||
`
|
||||
|
||||
// Searching for wants and grabbing releases (M502 #5521).
|
||||
@@ -105,12 +112,15 @@ func (q *Queries) GetAcquisitionSettings(ctx context.Context) (AcquisitionSettin
|
||||
&i.LastPass,
|
||||
&i.UpdatedAt,
|
||||
&i.ImportRoot,
|
||||
&i.SeedRatio,
|
||||
&i.SeedDays,
|
||||
&i.StallHours,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getLiveGrab = `-- name: GetLiveGrab :one
|
||||
SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE want_id = $1 AND ended_at IS NULL
|
||||
SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at FROM grabs WHERE want_id = $1 AND ended_at IS NULL
|
||||
`
|
||||
|
||||
func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, error) {
|
||||
@@ -144,6 +154,9 @@ func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, er
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -230,7 +243,7 @@ INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_
|
||||
VALUES ($1, $2, $3, $4, $5,
|
||||
$6, $7, $8, $9,
|
||||
$10, $11, $12)
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
type InsertGrabParams struct {
|
||||
@@ -292,6 +305,9 @@ func (q *Queries) InsertGrab(ctx context.Context, arg InsertGrabParams) (Grab, e
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -350,9 +366,13 @@ func (q *Queries) InsertWantCandidate(ctx context.Context, arg InsertWantCandida
|
||||
}
|
||||
|
||||
const listReleaseBlocks = `-- name: ListReleaseBlocks :many
|
||||
SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at FROM release_blocks ORDER BY blocked_at DESC
|
||||
SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at, expires_at FROM release_blocks
|
||||
WHERE expires_at IS NULL OR expires_at > now()
|
||||
ORDER BY blocked_at DESC
|
||||
`
|
||||
|
||||
// Every release set aside and still held back: a stalled one lifts by itself
|
||||
// at expires_at (#5523).
|
||||
func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error) {
|
||||
rows, err := q.db.Query(ctx, listReleaseBlocks)
|
||||
if err != nil {
|
||||
@@ -370,6 +390,7 @@ func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error)
|
||||
&i.WantID,
|
||||
&i.BlockedBy,
|
||||
&i.BlockedAt,
|
||||
&i.ExpiresAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -582,9 +603,12 @@ UPDATE acquisition_settings
|
||||
search_interval_hours = $2,
|
||||
allowed_qualities = $3,
|
||||
import_root = $4,
|
||||
seed_ratio = $5,
|
||||
seed_days = $6,
|
||||
stall_hours = $7,
|
||||
updated_at = now()
|
||||
WHERE id = true
|
||||
RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root
|
||||
RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root, seed_ratio, seed_days, stall_hours
|
||||
`
|
||||
|
||||
type UpdateAcquisitionSettingsParams struct {
|
||||
@@ -592,6 +616,9 @@ type UpdateAcquisitionSettingsParams struct {
|
||||
SearchIntervalHours int32
|
||||
AllowedQualities []string
|
||||
ImportRoot string
|
||||
SeedRatio float32
|
||||
SeedDays int32
|
||||
StallHours int32
|
||||
}
|
||||
|
||||
func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcquisitionSettingsParams) (AcquisitionSetting, error) {
|
||||
@@ -600,6 +627,9 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui
|
||||
arg.SearchIntervalHours,
|
||||
arg.AllowedQualities,
|
||||
arg.ImportRoot,
|
||||
arg.SeedRatio,
|
||||
arg.SeedDays,
|
||||
arg.StallHours,
|
||||
)
|
||||
var i AcquisitionSetting
|
||||
err := row.Scan(
|
||||
@@ -611,6 +641,9 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui
|
||||
&i.LastPass,
|
||||
&i.UpdatedAt,
|
||||
&i.ImportRoot,
|
||||
&i.SeedRatio,
|
||||
&i.SeedDays,
|
||||
&i.StallHours,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
@@ -30,7 +30,8 @@ SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::big
|
||||
(SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity,
|
||||
(SELECT COUNT(*) FROM grabs
|
||||
WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention,
|
||||
(SELECT COUNT(*) FROM release_blocks)::bigint AS blocked
|
||||
(SELECT COUNT(*) FROM release_blocks b
|
||||
WHERE b.expires_at IS NULL OR b.expires_at > now())::bigint AS blocked
|
||||
FROM wants
|
||||
`
|
||||
|
||||
@@ -537,7 +538,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas
|
||||
gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin,
|
||||
gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason,
|
||||
gr.progress AS grab_progress, gr.import_state AS grab_import_state,
|
||||
gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at,
|
||||
gr.import_reason AS grab_import_reason, gr.end_detail AS grab_end_detail, gr.imported_at AS grab_imported_at,
|
||||
gr.imported_files AS grab_imported_files
|
||||
FROM wants w
|
||||
LEFT JOIN users u ON u.id = w.created_by
|
||||
@@ -590,6 +591,7 @@ type ListWantsRow struct {
|
||||
GrabProgress *float32
|
||||
GrabImportState *string
|
||||
GrabImportReason *string
|
||||
GrabEndDetail *string
|
||||
GrabImportedAt pgtype.Timestamptz
|
||||
GrabImportedFiles *int32
|
||||
}
|
||||
@@ -636,6 +638,7 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan
|
||||
&i.GrabProgress,
|
||||
&i.GrabImportState,
|
||||
&i.GrabImportReason,
|
||||
&i.GrabEndDetail,
|
||||
&i.GrabImportedAt,
|
||||
&i.GrabImportedFiles,
|
||||
); err != nil {
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
ALTER TABLE acquisition_settings
|
||||
DROP COLUMN seed_ratio,
|
||||
DROP COLUMN seed_days,
|
||||
DROP COLUMN stall_hours;
|
||||
|
||||
ALTER TABLE release_blocks DROP COLUMN expires_at;
|
||||
DELETE FROM release_blocks WHERE reason = 'stalled';
|
||||
ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check;
|
||||
ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check
|
||||
CHECK (reason IN ('operator', 'files', 'import'));
|
||||
|
||||
DROP INDEX grabs_seeding_idx;
|
||||
ALTER TABLE grabs
|
||||
DROP COLUMN seeding,
|
||||
DROP COLUMN ratio,
|
||||
DROP COLUMN seed_released_at;
|
||||
|
||||
UPDATE grabs SET end_reason = 'superseded' WHERE end_reason IN ('stalled', 'failed', 'abandoned', 'gone');
|
||||
ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check;
|
||||
ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check
|
||||
CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held'));
|
||||
|
||||
DROP TABLE download_interventions;
|
||||
@@ -0,0 +1,90 @@
|
||||
-- Download lifecycle (M502 #5523), ported from Fabled Librarian's System 104:
|
||||
-- a download that stalls or fails is given up and its album searched again;
|
||||
-- an imported download seeds, then is removed.
|
||||
--
|
||||
-- download_interventions what was seen wrong with a download or a client,
|
||||
-- what Minstrel did about it, and whether it
|
||||
-- cleared. One open row per (client, download,
|
||||
-- condition), updated in place.
|
||||
-- grabs (columns) a grab also ends by being given up (stalled,
|
||||
-- failed), set aside by a person, or leaving the
|
||||
-- client; an imported one records its seeding.
|
||||
-- release_blocks a release given up on is held back for a while,
|
||||
-- not for ever: expires_at.
|
||||
-- acquisition_settings how long to seed, and how long a download may sit
|
||||
-- with nobody sharing it.
|
||||
CREATE TABLE download_interventions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
-- The acquisition_services code of the download client.
|
||||
client text NOT NULL,
|
||||
-- The torrent, or '' for a condition of the client itself.
|
||||
info_hash text NOT NULL DEFAULT '',
|
||||
grab_id uuid REFERENCES grabs (id) ON DELETE SET NULL,
|
||||
name text NOT NULL DEFAULT '',
|
||||
-- no_peers nobody is sharing an unfinished download.
|
||||
-- no_metadata still fetching the torrent's file list.
|
||||
-- client_error the client gave up on it, in its own words.
|
||||
-- client_unreachable the client does not answer, or its engine is down.
|
||||
-- gone the client no longer lists a running download.
|
||||
-- not_removed given up on, and the client would not remove it.
|
||||
trigger text NOT NULL CONSTRAINT download_interventions_trigger_check CHECK (trigger IN (
|
||||
'no_peers', 'no_metadata', 'client_error', 'client_unreachable', 'gone', 'not_removed')),
|
||||
-- escalated seen and shown; nothing done yet.
|
||||
-- retried asked the trackers again and re-read the files.
|
||||
-- given_up the download was removed and its album searched again.
|
||||
-- removed a download left behind was removed.
|
||||
action text NOT NULL DEFAULT 'escalated'
|
||||
CONSTRAINT download_interventions_action_check CHECK (action IN ('escalated', 'retried', 'given_up', 'removed')),
|
||||
outcome text NOT NULL DEFAULT 'open'
|
||||
CONSTRAINT download_interventions_outcome_check CHECK (outcome IN ('open', 'resolved')),
|
||||
detail text NOT NULL DEFAULT '',
|
||||
-- How many passes have seen it, and how many times Minstrel acted on it.
|
||||
-- The second is the retry ladder's rung, kept here so a restart does not
|
||||
-- hand a stuck download fresh retries.
|
||||
observations integer NOT NULL DEFAULT 1,
|
||||
remediations integer NOT NULL DEFAULT 0,
|
||||
first_seen timestamptz NOT NULL DEFAULT now(),
|
||||
last_seen timestamptz NOT NULL DEFAULT now(),
|
||||
settled_at timestamptz,
|
||||
CONSTRAINT download_interventions_settled_together CHECK ((outcome = 'open') = (settled_at IS NULL))
|
||||
);
|
||||
|
||||
-- One open row per condition: a pass that sees it again lands on the same row.
|
||||
CREATE UNIQUE INDEX download_interventions_one_open
|
||||
ON download_interventions (client, info_hash, trigger) WHERE outcome = 'open';
|
||||
CREATE INDEX download_interventions_grab_idx ON download_interventions (grab_id) WHERE grab_id IS NOT NULL;
|
||||
|
||||
ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check;
|
||||
ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check
|
||||
CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held', 'stalled', 'failed', 'abandoned', 'gone'));
|
||||
|
||||
-- An imported download goes on sharing (the files were copied) until its
|
||||
-- ratio reaches the setting or its time is up; seed_released_at is when it
|
||||
-- was removed, or found gone.
|
||||
ALTER TABLE grabs
|
||||
ADD COLUMN seeding boolean NOT NULL DEFAULT false,
|
||||
ADD COLUMN ratio real NOT NULL DEFAULT 0,
|
||||
ADD COLUMN seed_released_at timestamptz;
|
||||
|
||||
CREATE INDEX grabs_seeding_idx ON grabs (imported_at) WHERE seeding AND seed_released_at IS NULL;
|
||||
|
||||
-- stalled: given up on because it would not finish; held back until
|
||||
-- expires_at. NULL is for good.
|
||||
ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check;
|
||||
ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check
|
||||
CHECK (reason IN ('operator', 'files', 'import', 'stalled'));
|
||||
ALTER TABLE release_blocks ADD COLUMN expires_at timestamptz;
|
||||
|
||||
-- FL's rulings: seed to ratio 2.0 or for 14 days at most; a download nobody
|
||||
-- has shared for 6 hours is retried, then given up.
|
||||
-- seed_ratio 0 the download is removed once imported.
|
||||
-- seed_days 0 no time limit: only the ratio ends it.
|
||||
-- stall_hours 0 never give up on a download for want of peers (a failed
|
||||
-- one is still given up).
|
||||
ALTER TABLE acquisition_settings
|
||||
ADD COLUMN seed_ratio real NOT NULL DEFAULT 2.0
|
||||
CONSTRAINT acquisition_settings_seed_ratio_check CHECK (seed_ratio BETWEEN 0 AND 100),
|
||||
ADD COLUMN seed_days integer NOT NULL DEFAULT 14
|
||||
CONSTRAINT acquisition_settings_seed_days_check CHECK (seed_days BETWEEN 0 AND 365),
|
||||
ADD COLUMN stall_hours integer NOT NULL DEFAULT 6
|
||||
CONSTRAINT acquisition_settings_stall_hours_check CHECK (stall_hours BETWEEN 0 AND 720);
|
||||
@@ -51,6 +51,9 @@ UPDATE grabs
|
||||
import_updated_at = now(),
|
||||
import_forced = false,
|
||||
imported_at = CASE WHEN sqlc.arg(import_state)::text = 'imported' THEN now() ELSE imported_at END,
|
||||
-- The files were copied, so an imported download goes on sharing
|
||||
-- until the lifecycle removes it (#5523).
|
||||
seeding = sqlc.arg(import_state)::text = 'imported',
|
||||
imported_files = sqlc.arg(imported_files),
|
||||
destination = sqlc.arg(destination),
|
||||
ended_at = CASE WHEN sqlc.arg(end_reason)::text <> '' THEN now() ELSE ended_at END,
|
||||
@@ -110,21 +113,30 @@ SELECT DISTINCT COALESCE(t.disc_number, 1)::int AS disc, t.track_number::int AS
|
||||
|
||||
-- name: ListGrabs :many
|
||||
-- The Wanted view's Activity tab: downloads waiting on a person first, then
|
||||
-- the rest still running, then the ended ones newest first.
|
||||
-- the rest still running, then the ended ones newest first. Each carries
|
||||
-- what is wrong with it now, if anything (#5523).
|
||||
SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced,
|
||||
g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state,
|
||||
g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files,
|
||||
g.destination, w.title AS want_title, w.artist_name AS want_artist
|
||||
g.destination, g.seeding, g.ratio, g.seed_released_at,
|
||||
w.title AS want_title, w.artist_name AS want_artist,
|
||||
di.trigger AS condition, di.detail AS condition_detail, di.remediations AS condition_retries,
|
||||
di.first_seen AS condition_since
|
||||
FROM grabs g
|
||||
LEFT JOIN wants w ON w.id = g.want_id
|
||||
LEFT JOIN download_interventions di
|
||||
ON di.id = (SELECT x.id FROM download_interventions x
|
||||
WHERE x.grab_id = g.id AND x.outcome = 'open'
|
||||
ORDER BY x.last_seen DESC LIMIT 1)
|
||||
ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC,
|
||||
g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id
|
||||
LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip);
|
||||
|
||||
-- name: ListReleaseBlocksPage :many
|
||||
SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title,
|
||||
SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, b.expires_at, w.title AS want_title,
|
||||
w.artist_name AS want_artist
|
||||
FROM release_blocks b
|
||||
LEFT JOIN wants w ON w.id = b.want_id
|
||||
WHERE b.expires_at IS NULL OR b.expires_at > now()
|
||||
ORDER BY b.blocked_at DESC, b.info_hash
|
||||
LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip);
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
-- The download lifecycle (M502 #5523): the intervention ledger, giving a
|
||||
-- download up, and seeding after import.
|
||||
|
||||
-- name: RaiseIntervention :one
|
||||
-- A condition seen, or seen again: a second pass lands on the open row, so
|
||||
-- a download stuck for a week is one row that says how long. The name only
|
||||
-- improves, and the detail is the latest word.
|
||||
INSERT INTO download_interventions (client, info_hash, grab_id, name, trigger, detail)
|
||||
VALUES (sqlc.arg(client), sqlc.arg(info_hash), sqlc.narg(grab_id), sqlc.arg(name),
|
||||
sqlc.arg(trigger), sqlc.arg(detail))
|
||||
ON CONFLICT (client, info_hash, trigger) WHERE outcome = 'open' DO UPDATE SET
|
||||
observations = download_interventions.observations + 1,
|
||||
last_seen = now(),
|
||||
detail = EXCLUDED.detail,
|
||||
grab_id = COALESCE(EXCLUDED.grab_id, download_interventions.grab_id),
|
||||
name = CASE WHEN EXCLUDED.name <> '' THEN EXCLUDED.name ELSE download_interventions.name END
|
||||
RETURNING *;
|
||||
|
||||
-- name: SettleIntervention :execrows
|
||||
-- The condition cleared.
|
||||
UPDATE download_interventions
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash)
|
||||
AND trigger = sqlc.arg(trigger) AND outcome = 'open';
|
||||
|
||||
-- name: SettleInterventionsFor :execrows
|
||||
-- Everything open about one download is over: it is gone from the client.
|
||||
UPDATE download_interventions
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash) AND outcome = 'open';
|
||||
|
||||
-- name: SettleEndedGrabs :execrows
|
||||
-- What was wrong with a download whose grab has ended (imported, refused,
|
||||
-- given up) is over, except a download left in the client to remove.
|
||||
UPDATE download_interventions di
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
FROM grabs g
|
||||
WHERE g.id = di.grab_id AND g.ended_at IS NOT NULL
|
||||
AND di.outcome = 'open' AND di.trigger <> 'not_removed';
|
||||
|
||||
-- name: CountRemediation :one
|
||||
-- Minstrel acted on an open condition. A retry climbs the ladder; giving up
|
||||
-- or removing ends the condition, since what it was about is gone. Written
|
||||
-- before the act, so a retry that happened is never uncounted.
|
||||
UPDATE download_interventions
|
||||
SET remediations = remediations + 1,
|
||||
action = sqlc.arg(action),
|
||||
last_seen = now(),
|
||||
outcome = CASE WHEN sqlc.arg(action)::text IN ('given_up', 'removed') THEN 'resolved' ELSE outcome END,
|
||||
settled_at = CASE WHEN sqlc.arg(action)::text IN ('given_up', 'removed') THEN now() ELSE settled_at END
|
||||
WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash)
|
||||
AND trigger = sqlc.arg(trigger) AND outcome = 'open'
|
||||
RETURNING remediations;
|
||||
|
||||
-- name: ListLeftovers :many
|
||||
-- Downloads given up on that the client would not remove; each pass asks
|
||||
-- again.
|
||||
SELECT * FROM download_interventions
|
||||
WHERE trigger = 'not_removed' AND outcome = 'open' AND info_hash <> ''
|
||||
ORDER BY first_seen;
|
||||
|
||||
-- name: ListClientsDown :many
|
||||
-- Download clients not answering now, for the Activity tab.
|
||||
SELECT client, detail, first_seen FROM download_interventions
|
||||
WHERE trigger = 'client_unreachable' AND outcome = 'open'
|
||||
ORDER BY client;
|
||||
|
||||
-- name: LiveGrabOnHash :one
|
||||
SELECT EXISTS (SELECT 1 FROM grabs WHERE info_hash = sqlc.arg(info_hash) AND ended_at IS NULL)::bool;
|
||||
|
||||
-- name: GiveUpGrab :one
|
||||
-- Ends a running grab, given up on. Only from the states named: the update
|
||||
-- is the lock against an import that claimed the download meanwhile.
|
||||
UPDATE grabs
|
||||
SET ended_at = now(), end_reason = sqlc.arg(end_reason), end_detail = sqlc.arg(end_detail),
|
||||
import_updated_at = now()
|
||||
WHERE id = sqlc.arg(id) AND ended_at IS NULL AND import_state = ANY(sqlc.arg(states)::text[])
|
||||
RETURNING *;
|
||||
|
||||
-- name: ListSeedingGrabs :many
|
||||
-- Imported downloads still sharing, oldest import first.
|
||||
SELECT id, client, info_hash, name, imported_at FROM grabs
|
||||
WHERE seeding AND seed_released_at IS NULL
|
||||
ORDER BY imported_at, id;
|
||||
|
||||
-- name: SetGrabRatio :exec
|
||||
UPDATE grabs SET ratio = sqlc.arg(ratio) WHERE id = sqlc.arg(id);
|
||||
|
||||
-- name: ReleaseSeed :exec
|
||||
-- The seeding download was removed, or found gone.
|
||||
UPDATE grabs SET seed_released_at = now() WHERE id = $1 AND seed_released_at IS NULL;
|
||||
|
||||
-- name: DeleteExpiredBlocks :execrows
|
||||
-- A release given up on is held back for a while; after that the search may
|
||||
-- take it again.
|
||||
DELETE FROM release_blocks WHERE expires_at IS NOT NULL AND expires_at <= now();
|
||||
@@ -9,6 +9,9 @@ UPDATE acquisition_settings
|
||||
search_interval_hours = sqlc.arg(search_interval_hours),
|
||||
allowed_qualities = sqlc.arg(allowed_qualities),
|
||||
import_root = sqlc.arg(import_root),
|
||||
seed_ratio = sqlc.arg(seed_ratio),
|
||||
seed_days = sqlc.arg(seed_days),
|
||||
stall_hours = sqlc.arg(stall_hours),
|
||||
updated_at = now()
|
||||
WHERE id = true
|
||||
RETURNING *;
|
||||
@@ -109,14 +112,23 @@ UPDATE grabs
|
||||
WHERE id = sqlc.arg(id) AND ended_at IS NULL;
|
||||
|
||||
-- name: ListReleaseBlocks :many
|
||||
SELECT * FROM release_blocks ORDER BY blocked_at DESC;
|
||||
-- Every release set aside and still held back: a stalled one lifts by itself
|
||||
-- at expires_at (#5523).
|
||||
SELECT * FROM release_blocks
|
||||
WHERE expires_at IS NULL OR expires_at > now()
|
||||
ORDER BY blocked_at DESC;
|
||||
|
||||
-- name: BlockRelease :exec
|
||||
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by)
|
||||
-- A release set aside. expires_at NULL is for good. A block for good is never
|
||||
-- shortened by a timed one: only a timed block, or another for good, is
|
||||
-- replaced.
|
||||
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by, expires_at)
|
||||
VALUES (sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(reason), sqlc.arg(detail),
|
||||
sqlc.narg(want_id), sqlc.narg(blocked_by))
|
||||
sqlc.narg(want_id), sqlc.narg(blocked_by), sqlc.narg(expires_at))
|
||||
ON CONFLICT (info_hash) DO UPDATE
|
||||
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now();
|
||||
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(),
|
||||
expires_at = EXCLUDED.expires_at, blocked_by = EXCLUDED.blocked_by
|
||||
WHERE release_blocks.expires_at IS NOT NULL OR EXCLUDED.expires_at IS NULL;
|
||||
|
||||
-- name: UnblockRelease :execrows
|
||||
DELETE FROM release_blocks WHERE info_hash = $1;
|
||||
|
||||
@@ -194,7 +194,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas
|
||||
gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin,
|
||||
gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason,
|
||||
gr.progress AS grab_progress, gr.import_state AS grab_import_state,
|
||||
gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at,
|
||||
gr.import_reason AS grab_import_reason, gr.end_detail AS grab_end_detail, gr.imported_at AS grab_imported_at,
|
||||
gr.imported_files AS grab_imported_files
|
||||
FROM wants w
|
||||
LEFT JOIN users u ON u.id = w.created_by
|
||||
@@ -219,5 +219,6 @@ SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::big
|
||||
(SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity,
|
||||
(SELECT COUNT(*) FROM grabs
|
||||
WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention,
|
||||
(SELECT COUNT(*) FROM release_blocks)::bigint AS blocked
|
||||
(SELECT COUNT(*) FROM release_blocks b
|
||||
WHERE b.expires_at IS NULL OR b.expires_at > now())::bigint AS blocked
|
||||
FROM wants;
|
||||
|
||||
@@ -109,6 +109,7 @@ var dataTables = []string{
|
||||
"mb_release_groups", // M502 #5519
|
||||
"mb_artists", // M502 #5519
|
||||
"requests", // M502 #5520 (was lidarr_requests)
|
||||
"download_interventions", // M502 #5523
|
||||
"release_blocks", // M502 #5521
|
||||
"grabs", // M502 #5521
|
||||
"want_candidates", // M502 #5521
|
||||
@@ -187,6 +188,7 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) {
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE acquisition_settings
|
||||
SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, import_root = DEFAULT,
|
||||
seed_ratio = DEFAULT, seed_days = DEFAULT, stall_hours = DEFAULT,
|
||||
last_pass_at = NULL, last_pass = DEFAULT, updated_at = DEFAULT`,
|
||||
); err != nil {
|
||||
t.Fatalf("dbtest.ResetDB reset acquisition settings: %v", err)
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
@@ -67,8 +66,8 @@ type Importer struct {
|
||||
// Roots are the library's folders (the scan paths). New music is filed
|
||||
// under the one the settings name, or the first.
|
||||
Roots []string
|
||||
// ClientFor builds the download client a grab went to. Defaults to the
|
||||
// acquisition_services row of that code.
|
||||
// ClientFor builds the download client a grab went to. Defaults to
|
||||
// acquire.DownloadClientFor.
|
||||
ClientFor func(ctx context.Context, code string) (downloader.Driver, error)
|
||||
logger *slog.Logger
|
||||
wake chan struct{}
|
||||
@@ -83,19 +82,10 @@ func New(pool *pgxpool.Pool, scanner Scanner, cat Catalogue, printer Printer, ro
|
||||
}
|
||||
im := &Importer{pool: pool, Scanner: scanner, Catalogue: cat, Printer: printer, Roots: roots,
|
||||
logger: logger, wake: make(chan struct{}, 1)}
|
||||
im.ClientFor = im.clientFor
|
||||
return im
|
||||
}
|
||||
|
||||
func (im *Importer) clientFor(ctx context.Context, code string) (downloader.Driver, error) {
|
||||
row, err := dbq.New(im.pool).GetAcquisitionService(ctx, code)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("the download client %q is no longer set up", code)
|
||||
}
|
||||
return nil, err
|
||||
im.ClientFor = func(ctx context.Context, code string) (downloader.Driver, error) {
|
||||
return acquire.DownloadClientFor(ctx, pool, code)
|
||||
}
|
||||
return acquire.DownloadClient(row)
|
||||
return im
|
||||
}
|
||||
|
||||
// Wake asks for a pass now: after a grab, or a person's decision.
|
||||
@@ -201,7 +191,7 @@ func (im *Importer) passClient(ctx context.Context, q *dbq.Queries, code string,
|
||||
}
|
||||
t, ok := byHash[indexer.NormalizeHash(g.InfoHash)]
|
||||
if !ok {
|
||||
continue // gone from the client: the download lifecycle's (#5523)
|
||||
continue // gone from the client: the lifecycle settles it (#5523)
|
||||
}
|
||||
if !g.WantID.Valid {
|
||||
wait(g, "Its album is no longer wanted.")
|
||||
|
||||
@@ -0,0 +1,659 @@
|
||||
// Package lifecycle looks after a download from the grab to its removal
|
||||
// (M502 #5523), ported from Fabled Librarian's internal/health and its seed
|
||||
// release (System 104).
|
||||
//
|
||||
// - A download the client has failed is given up at once.
|
||||
// - One nobody has shared, or still fetching its file list, after the
|
||||
// stall setting is retried: the trackers asked again and the files
|
||||
// re-read, three times a pass apart. Then it is given up.
|
||||
// - Giving up removes the download with its files, holds the release back
|
||||
// for a week, ends the grab and searches for the album again. There is
|
||||
// no limit per album: what stops it churning is the hold on each release.
|
||||
// - A running download the client no longer lists, for half an hour, ends
|
||||
// and its album is searched again. The release is not held back: nothing
|
||||
// was wrong with it.
|
||||
// - An imported download seeds (its files were copied) until its share
|
||||
// ratio reaches the setting or its days are up, then is removed.
|
||||
//
|
||||
// Every condition and every act goes in the intervention ledger, one open
|
||||
// row per (client, download, condition), so the Activity tab can say what
|
||||
// is wrong with a download and what was done.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/downloader"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
)
|
||||
|
||||
// Conditions, matching download_interventions_trigger_check.
|
||||
const (
|
||||
NoPeers = "no_peers"
|
||||
NoMetadata = "no_metadata"
|
||||
ClientError = "client_error"
|
||||
ClientUnreachable = "client_unreachable"
|
||||
Gone = "gone"
|
||||
NotRemoved = "not_removed"
|
||||
)
|
||||
|
||||
// Acts, matching download_interventions_action_check.
|
||||
const (
|
||||
actRetried = "retried"
|
||||
actGivenUp = "given_up"
|
||||
actRemoved = "removed"
|
||||
)
|
||||
|
||||
// End reasons a grab is given with, matching grabs_end_reason_check.
|
||||
const (
|
||||
EndStalled = "stalled"
|
||||
EndFailed = "failed"
|
||||
EndAbandoned = "abandoned"
|
||||
EndGone = "gone"
|
||||
)
|
||||
|
||||
const (
|
||||
// interval: three retries a pass apart is half an hour of asking before
|
||||
// a stalled download is given up.
|
||||
interval = 10 * time.Minute
|
||||
passTimeout = 5 * time.Minute
|
||||
// Retries is how many times one stuck download is nudged first.
|
||||
Retries = 3
|
||||
// StallBlock is how long a release given up on is held back.
|
||||
StallBlock = 7 * 24 * time.Hour
|
||||
// goneAfter: a client restarting can answer without a download for a
|
||||
// while; half an hour missing is gone.
|
||||
goneAfter = 30 * time.Minute
|
||||
)
|
||||
|
||||
// running are the import states of a grab still downloading: the lifecycle
|
||||
// acts only on these, so an import under way is never pulled from under.
|
||||
var running = []string{"", "downloading", "waiting"}
|
||||
|
||||
// Settings are the limits, from acquisition_settings.
|
||||
type Settings struct {
|
||||
// SeedRatio is the share ratio an imported download seeds to; 0 removes
|
||||
// it once imported.
|
||||
SeedRatio float64
|
||||
// SeedTime is the longest it seeds; 0 is no limit.
|
||||
SeedTime time.Duration
|
||||
// Patience is how long a download may sit with nobody sharing it before
|
||||
// it is retried, then given up; 0 never gives up for that.
|
||||
Patience time.Duration
|
||||
}
|
||||
|
||||
// Defaults are FL's rulings.
|
||||
var Defaults = Settings{SeedRatio: 2, SeedTime: 14 * 24 * time.Hour, Patience: 6 * time.Hour}
|
||||
|
||||
// Service runs the lifecycle passes.
|
||||
type Service struct {
|
||||
pool *pgxpool.Pool
|
||||
// ClientFor builds a download client by its code. Defaults to
|
||||
// acquire.DownloadClientFor.
|
||||
ClientFor func(ctx context.Context, code string) (downloader.Driver, error)
|
||||
// Now is the clock; nil is time.Now.
|
||||
Now func() time.Time
|
||||
logger *slog.Logger
|
||||
mu sync.Mutex // one pass at a time
|
||||
}
|
||||
|
||||
// New returns a Service.
|
||||
func New(pool *pgxpool.Pool, logger *slog.Logger) *Service {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Service{pool: pool, logger: logger,
|
||||
ClientFor: func(ctx context.Context, code string) (downloader.Driver, error) {
|
||||
return acquire.DownloadClientFor(ctx, pool, code)
|
||||
}}
|
||||
}
|
||||
|
||||
func (s *Service) now() time.Time {
|
||||
if s.Now != nil {
|
||||
return s.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// Run passes a minute after start, then every ten minutes, until ctx ends.
|
||||
// Each pass is bounded, so a client that never answers costs one pass.
|
||||
func (s *Service) Run(ctx context.Context) {
|
||||
first := time.NewTimer(time.Minute)
|
||||
defer first.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-first.C:
|
||||
}
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
rep := s.Pass(ctx)
|
||||
if rep.Retried+rep.GivenUp+rep.Gone+rep.Released+rep.Removed > 0 || rep.Error != "" {
|
||||
s.logger.Info("lifecycle: pass", "retried", rep.Retried, "given_up", rep.GivenUp, "gone", rep.Gone,
|
||||
"released", rep.Released, "removed", rep.Removed, "raised", rep.Raised, "settled", rep.Settled,
|
||||
"error", rep.Error)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Report is one pass.
|
||||
type Report struct {
|
||||
// Raised and Settled count ledger writes; the rest count acts.
|
||||
Raised, Settled int
|
||||
Retried, GivenUp, Gone, Released, Removed int
|
||||
Error string
|
||||
}
|
||||
|
||||
// Pass looks at every running download, every seeding one and every one
|
||||
// left behind, once.
|
||||
func (s *Service) Pass(ctx context.Context) Report {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ctx, cancel := context.WithTimeout(ctx, passTimeout)
|
||||
defer cancel()
|
||||
var rep Report
|
||||
q := dbq.New(s.pool)
|
||||
set := s.settings(ctx, q)
|
||||
|
||||
if _, err := q.DeleteExpiredBlocks(ctx); err != nil {
|
||||
s.logger.Warn("lifecycle: lift expired holds", "err", err)
|
||||
}
|
||||
if n, err := q.SettleEndedGrabs(ctx); err == nil {
|
||||
rep.Settled += int(n)
|
||||
}
|
||||
live, err := q.ListLiveGrabs(ctx)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
seeds, err := q.ListSeedingGrabs(ctx)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
left, err := q.ListLeftovers(ctx)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
|
||||
work := map[string]*clientWork{}
|
||||
var order []string
|
||||
of := func(code string) *clientWork {
|
||||
if w, ok := work[code]; ok {
|
||||
return w
|
||||
}
|
||||
order = append(order, code)
|
||||
work[code] = &clientWork{}
|
||||
return work[code]
|
||||
}
|
||||
for _, g := range live {
|
||||
if g.InfoHash != "" {
|
||||
of(g.Client).live = append(of(g.Client).live, g)
|
||||
}
|
||||
}
|
||||
for _, g := range seeds {
|
||||
if g.InfoHash != "" {
|
||||
of(g.Client).seeds = append(of(g.Client).seeds, g)
|
||||
}
|
||||
}
|
||||
for _, l := range left {
|
||||
of(l.Client).left = append(of(l.Client).left, l)
|
||||
}
|
||||
for _, code := range order {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
s.passClient(ctx, q, code, work[code], set, &rep)
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
type clientWork struct {
|
||||
live []dbq.ListLiveGrabsRow
|
||||
seeds []dbq.ListSeedingGrabsRow
|
||||
left []dbq.DownloadIntervention
|
||||
}
|
||||
|
||||
func (s *Service) settings(ctx context.Context, q *dbq.Queries) Settings {
|
||||
row, err := q.GetAcquisitionSettings(ctx)
|
||||
if err != nil {
|
||||
// A blip keeps the defaults rather than stopping the pass.
|
||||
return Defaults
|
||||
}
|
||||
return Settings{
|
||||
SeedRatio: float64(row.SeedRatio),
|
||||
SeedTime: time.Duration(row.SeedDays) * 24 * time.Hour,
|
||||
Patience: time.Duration(row.StallHours) * time.Hour,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) passClient(ctx context.Context, q *dbq.Queries, code string, w *clientWork, set Settings, rep *Report) {
|
||||
d, err := s.ClientFor(ctx, code)
|
||||
if err != nil {
|
||||
s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, firstLine(err.Error()), rep)
|
||||
return
|
||||
}
|
||||
var hashes []string
|
||||
for _, g := range w.live {
|
||||
hashes = append(hashes, g.InfoHash)
|
||||
}
|
||||
for _, g := range w.seeds {
|
||||
hashes = append(hashes, g.InfoHash)
|
||||
}
|
||||
for _, l := range w.left {
|
||||
hashes = append(hashes, l.InfoHash)
|
||||
}
|
||||
ts, err := d.Torrents(ctx, hashes...)
|
||||
if err != nil {
|
||||
// Nothing about any one download is observed through a client that
|
||||
// cannot answer.
|
||||
s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, firstLine(err.Error()), rep)
|
||||
return
|
||||
}
|
||||
for _, t := range ts {
|
||||
if t.EngineDown != "" {
|
||||
s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, t.EngineDown, rep)
|
||||
return
|
||||
}
|
||||
}
|
||||
s.settle(ctx, q, code, "", ClientUnreachable, rep)
|
||||
|
||||
byHash := make(map[string]downloader.Torrent, len(ts))
|
||||
for _, t := range ts {
|
||||
byHash[indexer.NormalizeHash(t.Hash)] = t
|
||||
}
|
||||
// ABSENCE IS ONLY A MEASUREMENT FROM A CLIENT THAT LISTS SOMETHING. An
|
||||
// engine switched off, or a store that would not read, answers with an
|
||||
// empty list; then nothing is concluded from what is missing.
|
||||
listed := len(ts) > 0
|
||||
if !listed {
|
||||
all, err := d.Torrents(ctx)
|
||||
listed = err == nil && len(all) > 0
|
||||
}
|
||||
|
||||
for _, l := range w.left {
|
||||
t, ok := byHash[indexer.NormalizeHash(l.InfoHash)]
|
||||
s.removeAgain(ctx, q, d, l, t, ok, listed, rep)
|
||||
}
|
||||
for _, g := range w.live {
|
||||
t, ok := byHash[indexer.NormalizeHash(g.InfoHash)]
|
||||
s.judge(ctx, q, d, g, t, ok, listed, set, rep)
|
||||
}
|
||||
for _, g := range w.seeds {
|
||||
t, ok := byHash[indexer.NormalizeHash(g.InfoHash)]
|
||||
s.seed(ctx, q, d, g, t, ok, set, rep)
|
||||
}
|
||||
}
|
||||
|
||||
// judge decides what, if anything, is wrong with one running download, and
|
||||
// answers each condition either way: raised when true, settled when not.
|
||||
func (s *Service) judge(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow,
|
||||
t downloader.Torrent, ok, listed bool, set Settings, rep *Report) {
|
||||
if !isRunning(g.ImportState) {
|
||||
return // the importer's, or a person's
|
||||
}
|
||||
if !ok {
|
||||
if !listed {
|
||||
return
|
||||
}
|
||||
row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, Gone,
|
||||
"The download client no longer lists it.", rep)
|
||||
if raised && s.now().Sub(row.FirstSeen.Time) >= goneAfter {
|
||||
s.endGone(ctx, q, g, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, Gone, rep)
|
||||
|
||||
if t.Done() {
|
||||
// Finished: whatever was wrong while it downloaded is over, and the
|
||||
// import is the importer's.
|
||||
for _, c := range []string{ClientError, NoMetadata, NoPeers} {
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, c, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// The client's own verdict, given up on at once: it has already made
|
||||
// the decision a retry would ask about (FL #4749).
|
||||
if t.Failed() {
|
||||
why := t.FailReason
|
||||
if why == "" {
|
||||
why = fmt.Sprintf("the download client reports it %q", t.State)
|
||||
}
|
||||
if _, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, ClientError, why, rep); raised {
|
||||
s.giveUp(ctx, q, d, g, ClientError, EndFailed, why, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, ClientError, rep)
|
||||
|
||||
// Below here the stall setting applies, measured from the grab. With it
|
||||
// off nothing is raised and what was raised clears.
|
||||
age := s.now().Sub(g.GrabbedAt.Time)
|
||||
patient := set.Patience > 0 && age > set.Patience
|
||||
|
||||
if patient && t.Fetching() {
|
||||
row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NoMetadata,
|
||||
fmt.Sprintf("Still fetching its file list after %s.", span(age)), rep)
|
||||
if raised {
|
||||
s.ladder(ctx, q, d, g, NoMetadata, row.Remediations, age, rep)
|
||||
}
|
||||
} else {
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, NoMetadata, rep)
|
||||
}
|
||||
// Not while fetching: a torrent with no file list has no swarm to be
|
||||
// alone in.
|
||||
if patient && !t.Fetching() && t.Alone() {
|
||||
row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NoPeers,
|
||||
fmt.Sprintf("Nobody has shared it in %s; %.0f%% downloaded.", span(age), t.Progress*100), rep)
|
||||
if raised {
|
||||
s.ladder(ctx, q, d, g, NoPeers, row.Remediations, age, rep)
|
||||
}
|
||||
} else {
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, NoPeers, rep)
|
||||
}
|
||||
}
|
||||
|
||||
// ladder retries a stuck download until the retries are spent, then gives
|
||||
// it up. The rung is the ledger row's, so a restart hands out no more.
|
||||
func (s *Service) ladder(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow,
|
||||
trigger string, done int32, age time.Duration, rep *Report) {
|
||||
if int(done) < Retries {
|
||||
// Counted before it happens: a retry that happened and was not
|
||||
// counted is a bound that never binds.
|
||||
n, err := q.CountRemediation(ctx, dbq.CountRemediationParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash, Trigger: trigger, Action: actRetried})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Exchange schedules its own announces and has no recheck: for it a
|
||||
// retry is the time between passes, and the ladder still climbs.
|
||||
if err := d.Reannounce(ctx, g.InfoHash); err != nil && !errors.Is(err, downloader.ErrUnsupported) {
|
||||
s.logger.Warn("lifecycle: reannounce", "grab", g.Name, "err", err)
|
||||
}
|
||||
if err := d.Recheck(ctx, g.InfoHash); err != nil && !errors.Is(err, downloader.ErrUnsupported) {
|
||||
s.logger.Warn("lifecycle: recheck", "grab", g.Name, "err", err)
|
||||
}
|
||||
s.logger.Info("lifecycle: retried a stalled download", "grab", g.Name, "trigger", trigger, "attempt", n)
|
||||
rep.Retried++
|
||||
return
|
||||
}
|
||||
why := fmt.Sprintf("taken %s ago and never finished: %s", span(age), reasonFor(trigger))
|
||||
s.giveUp(ctx, q, d, g, trigger, EndStalled, why, rep)
|
||||
}
|
||||
|
||||
// giveUp ends the grab, holds the release back for StallBlock, removes the
|
||||
// download with its files and searches for the album again.
|
||||
func (s *Service) giveUp(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow,
|
||||
trigger, end, why string, rep *Report) {
|
||||
until := s.now().Add(StallBlock)
|
||||
detail := capital(why) + fmt.Sprintf(". Held back until %s; the album is searched for again.", until.Format("2 Jan"))
|
||||
// The update is the lock: an import that claimed the download meanwhile
|
||||
// keeps it.
|
||||
if _, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: end, EndDetail: detail, States: running}); err != nil {
|
||||
return
|
||||
}
|
||||
// Held back before it is removed: if the pass dies between the two, the
|
||||
// release is still not taken again.
|
||||
if err := acquire.NewMemory(s.pool).Block(ctx, dbq.BlockReleaseParams{
|
||||
InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockStalled, Detail: capital(why) + ".",
|
||||
WantID: g.WantID, ExpiresAt: pgtype.Timestamptz{Time: until, Valid: true},
|
||||
}); err != nil {
|
||||
s.logger.Warn("lifecycle: hold the release back", "grab", g.Name, "err", err)
|
||||
}
|
||||
removed := s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, rep)
|
||||
if g.WantID.Valid {
|
||||
if err := q.SearchWantAgain(ctx, g.WantID); err != nil {
|
||||
s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err)
|
||||
}
|
||||
}
|
||||
if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash, Trigger: trigger, Action: actGivenUp}); err != nil {
|
||||
s.logger.Warn("lifecycle: stamp the ledger", "grab", g.Name, "err", err)
|
||||
}
|
||||
if removed {
|
||||
if n, err := q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash}); err == nil {
|
||||
rep.Settled += int(n)
|
||||
}
|
||||
}
|
||||
s.logger.Info("lifecycle: gave up on a download", "grab", g.Name, "trigger", trigger, "why", why)
|
||||
rep.GivenUp++
|
||||
}
|
||||
|
||||
// endGone ends a grab whose download left the client, and searches again.
|
||||
// The release is not held back: nothing was found wrong with it.
|
||||
func (s *Service) endGone(ctx context.Context, q *dbq.Queries, g dbq.ListLiveGrabsRow, rep *Report) {
|
||||
detail := "The download client no longer lists it: removed outside Minstrel. The album is searched for again."
|
||||
if _, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: EndGone, EndDetail: detail, States: running}); err != nil {
|
||||
return
|
||||
}
|
||||
if g.WantID.Valid {
|
||||
if err := q.SearchWantAgain(ctx, g.WantID); err != nil {
|
||||
s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err)
|
||||
}
|
||||
}
|
||||
if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash, Trigger: Gone, Action: actGivenUp}); err != nil {
|
||||
s.logger.Warn("lifecycle: stamp the ledger", "grab", g.Name, "err", err)
|
||||
}
|
||||
if n, err := q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash}); err == nil {
|
||||
rep.Settled += int(n)
|
||||
}
|
||||
s.logger.Info("lifecycle: a download left its client", "grab", g.Name)
|
||||
rep.Gone++
|
||||
}
|
||||
|
||||
// seed removes an imported download once it has shared enough.
|
||||
//
|
||||
// Absence is not a verdict: a restarting client answers with a partial list.
|
||||
// A seeding download missing from it is left until its time is up, then
|
||||
// recorded as released; there is nothing to delete.
|
||||
func (s *Service) seed(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListSeedingGrabsRow,
|
||||
t downloader.Torrent, ok bool, set Settings, rep *Report) {
|
||||
overdue := set.SeedTime > 0 && g.ImportedAt.Valid && s.now().Sub(g.ImportedAt.Time) >= set.SeedTime
|
||||
if !ok {
|
||||
if overdue {
|
||||
s.release(ctx, q, g.ID, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := q.SetGrabRatio(ctx, dbq.SetGrabRatioParams{ID: g.ID, Ratio: float32(t.Ratio)}); err != nil {
|
||||
s.logger.Warn("lifecycle: record the ratio", "grab", g.Name, "err", err)
|
||||
}
|
||||
if t.Ratio < set.SeedRatio && !overdue {
|
||||
return
|
||||
}
|
||||
s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, rep)
|
||||
s.logger.Info("lifecycle: seeded enough, removed the download", "grab", g.Name, "ratio", t.Ratio, "overdue", overdue)
|
||||
s.release(ctx, q, g.ID, rep)
|
||||
}
|
||||
|
||||
func (s *Service) release(ctx context.Context, q *dbq.Queries, id pgtype.UUID, rep *Report) {
|
||||
if err := q.ReleaseSeed(ctx, id); err != nil {
|
||||
s.logger.Warn("lifecycle: record the seeding ended", "err", err)
|
||||
return
|
||||
}
|
||||
rep.Released++
|
||||
}
|
||||
|
||||
// remove deletes a download and its files. A client that refuses leaves an
|
||||
// open not_removed row, which every pass tries again.
|
||||
func (s *Service) remove(ctx context.Context, q *dbq.Queries, d downloader.Driver, code, hash string,
|
||||
grab pgtype.UUID, name string, rep *Report) bool {
|
||||
if err := d.Delete(ctx, true, hash); err != nil {
|
||||
s.logger.Warn("lifecycle: remove the download", "grab", name, "err", err)
|
||||
s.raise(ctx, q, code, hash, grab, name, NotRemoved, notRemoved(err), rep)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// removeAgain asks the client once more to remove a download left behind,
|
||||
// unless it has been taken again since.
|
||||
func (s *Service) removeAgain(ctx context.Context, q *dbq.Queries, d downloader.Driver, l dbq.DownloadIntervention,
|
||||
t downloader.Torrent, ok, listed bool, rep *Report) {
|
||||
if !ok {
|
||||
if listed {
|
||||
s.settle(ctx, q, l.Client, l.InfoHash, NotRemoved, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
live, err := q.LiveGrabOnHash(ctx, l.InfoHash)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if live {
|
||||
s.settle(ctx, q, l.Client, l.InfoHash, NotRemoved, rep)
|
||||
return
|
||||
}
|
||||
if err := d.Delete(ctx, true, l.InfoHash); err != nil {
|
||||
s.raise(ctx, q, l.Client, l.InfoHash, l.GrabID, t.Name, NotRemoved, notRemoved(err), rep)
|
||||
return
|
||||
}
|
||||
if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{
|
||||
Client: l.Client, InfoHash: l.InfoHash, Trigger: NotRemoved, Action: actRemoved}); err != nil {
|
||||
s.logger.Warn("lifecycle: stamp the ledger", "hash", l.InfoHash, "err", err)
|
||||
}
|
||||
s.logger.Info("lifecycle: removed a download left behind", "name", l.Name)
|
||||
rep.Removed++
|
||||
}
|
||||
|
||||
// raise records a condition or agrees with the open row that says it. The
|
||||
// bool is false when the write failed: then nothing is acted on, since
|
||||
// without the row's count there is no bound.
|
||||
func (s *Service) raise(ctx context.Context, q *dbq.Queries, code, hash string, grab pgtype.UUID, name, trigger,
|
||||
detail string, rep *Report) (dbq.DownloadIntervention, bool) {
|
||||
row, err := q.RaiseIntervention(ctx, dbq.RaiseInterventionParams{
|
||||
Client: code, InfoHash: hash, GrabID: grab, Name: name, Trigger: trigger, Detail: detail})
|
||||
if err != nil {
|
||||
s.logger.Warn("lifecycle: record", "client", code, "hash", hash, "trigger", trigger, "err", err)
|
||||
return row, false
|
||||
}
|
||||
rep.Raised++
|
||||
return row, true
|
||||
}
|
||||
|
||||
func (s *Service) settle(ctx context.Context, q *dbq.Queries, code, hash, trigger string, rep *Report) {
|
||||
n, err := q.SettleIntervention(ctx, dbq.SettleInterventionParams{Client: code, InfoHash: hash, Trigger: trigger})
|
||||
if err != nil {
|
||||
s.logger.Warn("lifecycle: settle", "client", code, "hash", hash, "trigger", trigger, "err", err)
|
||||
return
|
||||
}
|
||||
rep.Settled += int(n)
|
||||
}
|
||||
|
||||
// ErrNotAbandonable: the download is not running; an import, or a person's
|
||||
// decision on one, has it.
|
||||
var ErrNotAbandonable = errors.New("lifecycle: that download is not running")
|
||||
|
||||
// Abandon is a person giving up on a running download: the release is set
|
||||
// aside for good (their word about it), the download removed with its files,
|
||||
// and the album searched for again.
|
||||
func (s *Service) Abandon(ctx context.Context, grabID, by pgtype.UUID) (dbq.Grab, error) {
|
||||
q := dbq.New(s.pool)
|
||||
g, err := q.GetGrab(ctx, grabID)
|
||||
if err != nil {
|
||||
return dbq.Grab{}, err
|
||||
}
|
||||
if g.EndedAt.Valid || !isRunning(g.ImportState) {
|
||||
return dbq.Grab{}, ErrNotAbandonable
|
||||
}
|
||||
const why = "Given up by you before it finished. The album is searched for again."
|
||||
ended, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: EndAbandoned, EndDetail: why, States: running})
|
||||
if err != nil {
|
||||
return dbq.Grab{}, ErrNotAbandonable
|
||||
}
|
||||
if g.InfoHash != "" {
|
||||
if err := acquire.NewMemory(s.pool).Block(ctx, dbq.BlockReleaseParams{
|
||||
InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockOperator, Detail: "Given up by you before it finished.",
|
||||
WantID: g.WantID, BlockedBy: by,
|
||||
}); err != nil {
|
||||
s.logger.Warn("lifecycle: set the release aside", "grab", g.Name, "err", err)
|
||||
}
|
||||
var rep Report
|
||||
if d, err := s.ClientFor(ctx, g.Client); err != nil {
|
||||
s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NotRemoved, notRemoved(err), &rep)
|
||||
} else if s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, &rep) {
|
||||
_, _ = q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{Client: g.Client, InfoHash: g.InfoHash})
|
||||
}
|
||||
}
|
||||
if g.WantID.Valid {
|
||||
if err := q.SearchWantAgain(ctx, g.WantID); err != nil {
|
||||
s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err)
|
||||
}
|
||||
}
|
||||
return ended, nil
|
||||
}
|
||||
|
||||
func isRunning(state string) bool {
|
||||
for _, r := range running {
|
||||
if state == r {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// reasonFor is a condition in a person's words.
|
||||
func reasonFor(trigger string) string {
|
||||
switch trigger {
|
||||
case NoPeers:
|
||||
return "nobody was sharing it"
|
||||
case NoMetadata:
|
||||
return "its file list never arrived"
|
||||
case ClientError:
|
||||
return "the download client gave up on it"
|
||||
}
|
||||
return "it stalled"
|
||||
}
|
||||
|
||||
func notRemoved(err error) string {
|
||||
return "Given up on, but the download client would not remove it: " + firstLine(err.Error()) +
|
||||
". Minstrel asks again every ten minutes."
|
||||
}
|
||||
|
||||
// span is a duration as a person says it.
|
||||
func span(d time.Duration) string {
|
||||
switch {
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%d hours", int(d.Hours()))
|
||||
}
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
|
||||
func capital(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + s[1:]
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
return strings.TrimSpace(s[:i])
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/downloader"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/wanted"
|
||||
)
|
||||
|
||||
func newPool(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in -short mode")
|
||||
}
|
||||
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
||||
}
|
||||
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("pool: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
dbtest.ResetDB(t, pool)
|
||||
return pool
|
||||
}
|
||||
|
||||
const (
|
||||
hash = "0123456789abcdef0123456789abcdef01234567"
|
||||
other = "fedcba9876543210fedcba9876543210fedcba98"
|
||||
)
|
||||
|
||||
// fakeClient lists the torrents it holds and records what it is asked.
|
||||
type fakeClient struct {
|
||||
downloader.Driver
|
||||
held map[string]downloader.Torrent
|
||||
err error
|
||||
deleteErr error
|
||||
deleted []string
|
||||
reannounce int
|
||||
recheck int
|
||||
}
|
||||
|
||||
func (c *fakeClient) Torrents(_ context.Context, hashes ...string) ([]downloader.Torrent, error) {
|
||||
if c.err != nil {
|
||||
return nil, c.err
|
||||
}
|
||||
var out []downloader.Torrent
|
||||
if len(hashes) == 0 {
|
||||
for _, t := range c.held {
|
||||
out = append(out, t)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
for _, h := range hashes {
|
||||
if t, ok := c.held[h]; ok {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error {
|
||||
if c.deleteErr != nil {
|
||||
return c.deleteErr
|
||||
}
|
||||
c.deleted = append(c.deleted, hashes...)
|
||||
for _, h := range hashes {
|
||||
delete(c.held, h)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) Reannounce(context.Context, ...string) error { c.reannounce++; return nil }
|
||||
func (c *fakeClient) Recheck(context.Context, ...string) error { c.recheck++; return nil }
|
||||
|
||||
type stack struct {
|
||||
q *dbq.Queries
|
||||
s *Service
|
||||
client *fakeClient
|
||||
want dbq.Want
|
||||
grab dbq.Grab
|
||||
clock time.Time
|
||||
}
|
||||
|
||||
// newStack wants RAM and has a grab of it running in a fake client holding
|
||||
// t under hash.
|
||||
func newStack(t *testing.T, tor downloader.Torrent) *stack {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
pool := newPool(t)
|
||||
q := dbq.New(pool)
|
||||
w, err := wanted.WantWith(ctx, q, wanted.WantParams{GroupMBID: "rg-ram", ArtistMBID: "mb-dp", ArtistName: "Daft Punk",
|
||||
Title: "Random Access Memories", PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: wanted.ReasonRequest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g, err := acquire.NewMemory(pool).Grabbed(ctx, dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash,
|
||||
Name: "Daft Punk - RAM [FLAC]", Key: hash, Client: "qbittorrent", SavePath: "/downloads", Origin: acquire.OriginSchedule},
|
||||
pgtype.UUID{}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := q.MarkWantSearched(ctx, w.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tor.Hash = hash
|
||||
st := &stack{q: q, client: &fakeClient{held: map[string]downloader.Torrent{hash: tor}}, want: w, grab: g,
|
||||
clock: time.Now()}
|
||||
st.s = New(pool, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
st.s.ClientFor = func(context.Context, string) (downloader.Driver, error) { return st.client, nil }
|
||||
st.s.Now = func() time.Time { return st.clock }
|
||||
return st
|
||||
}
|
||||
|
||||
func (st *stack) grabNow(t *testing.T) dbq.Grab {
|
||||
t.Helper()
|
||||
g, err := st.q.GetGrab(context.Background(), st.grab.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func (st *stack) searchedAgain(t *testing.T) bool {
|
||||
t.Helper()
|
||||
w, err := st.q.GetWantByReleaseGroup(context.Background(), "rg-ram")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return !w.SearchedAt.Valid
|
||||
}
|
||||
|
||||
func (st *stack) ledger(t *testing.T) []dbq.DownloadIntervention {
|
||||
t.Helper()
|
||||
rows, err := st.s.pool.Query(context.Background(), `SELECT * FROM download_interventions ORDER BY first_seen, id`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := pgx.CollectRows(rows, pgx.RowToStructByPos[dbq.DownloadIntervention])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestLifecycle_AFailedDownloadIsGivenUpAtOnce_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "error",
|
||||
FailReason: "No space left on device"})
|
||||
rep := st.s.Pass(context.Background())
|
||||
if rep.GivenUp != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
g := st.grabNow(t)
|
||||
if !g.EndedAt.Valid || g.EndReason != EndFailed {
|
||||
t.Fatalf("grab ended=%v reason=%q", g.EndedAt.Valid, g.EndReason)
|
||||
}
|
||||
if len(st.client.deleted) != 1 || !st.searchedAgain(t) {
|
||||
t.Errorf("deleted %v, searched again %v", st.client.deleted, st.searchedAgain(t))
|
||||
}
|
||||
blocks, err := st.q.ListReleaseBlocks(context.Background())
|
||||
if err != nil || len(blocks) != 1 || blocks[0].Reason != acquire.BlockStalled || !blocks[0].ExpiresAt.Valid {
|
||||
t.Fatalf("blocks = %+v %v", blocks, err)
|
||||
}
|
||||
if d := blocks[0].ExpiresAt.Time.Sub(st.clock); d < StallBlock-time.Minute || d > StallBlock+time.Minute {
|
||||
t.Errorf("held back for %s", d)
|
||||
}
|
||||
rows := st.ledger(t)
|
||||
if len(rows) != 1 || rows[0].Trigger != ClientError || rows[0].Action != actGivenUp || rows[0].Outcome != "resolved" ||
|
||||
rows[0].Detail != "No space left on device" {
|
||||
t.Errorf("ledger = %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_ADownloadNobodySharesIsRetriedThenGivenUp_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"})
|
||||
ctx := context.Background()
|
||||
|
||||
// Within the stall setting: nothing is wrong yet.
|
||||
if rep := st.s.Pass(ctx); rep.Retried+rep.GivenUp != 0 || len(st.ledger(t)) != 0 {
|
||||
t.Fatalf("a young download was judged: %+v", rep)
|
||||
}
|
||||
|
||||
st.clock = st.clock.Add(7 * time.Hour)
|
||||
for i := 1; i <= Retries; i++ {
|
||||
if rep := st.s.Pass(ctx); rep.Retried != 1 || rep.GivenUp != 0 {
|
||||
t.Fatalf("pass %d = %+v", i, rep)
|
||||
}
|
||||
}
|
||||
if st.client.reannounce != Retries || st.client.recheck != Retries {
|
||||
t.Errorf("reannounced %d, rechecked %d", st.client.reannounce, st.client.recheck)
|
||||
}
|
||||
if st.grabNow(t).EndedAt.Valid {
|
||||
t.Fatal("given up before the retries ran out")
|
||||
}
|
||||
if rep := st.s.Pass(ctx); rep.GivenUp != 1 {
|
||||
t.Fatalf("last pass = %+v", rep)
|
||||
}
|
||||
g := st.grabNow(t)
|
||||
if g.EndReason != EndStalled || len(st.client.deleted) != 1 || !st.searchedAgain(t) {
|
||||
t.Errorf("grab %q: %s; deleted %v", g.EndReason, g.EndDetail, st.client.deleted)
|
||||
}
|
||||
rows := st.ledger(t)
|
||||
if len(rows) != 1 || rows[0].Trigger != NoPeers || rows[0].Remediations != Retries+1 || rows[0].Outcome != "resolved" {
|
||||
t.Errorf("ledger = %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AStalledDownloadThatRecoversIsSettled_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"})
|
||||
ctx := context.Background()
|
||||
st.clock = st.clock.Add(7 * time.Hour)
|
||||
st.s.Pass(ctx)
|
||||
t2 := st.client.held[hash]
|
||||
t2.NumSeeds = 3
|
||||
st.client.held[hash] = t2
|
||||
st.s.Pass(ctx)
|
||||
rows := st.ledger(t)
|
||||
if len(rows) != 1 || rows[0].Outcome != "resolved" || rows[0].Action != actRetried {
|
||||
t.Fatalf("ledger = %+v", rows)
|
||||
}
|
||||
if st.grabNow(t).EndedAt.Valid {
|
||||
t.Error("a download that found peers was given up")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_TheStallSettingOffNeverGivesUp_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"})
|
||||
ctx := context.Background()
|
||||
if _, err := st.s.pool.Exec(ctx, `UPDATE acquisition_settings SET stall_hours = 0`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st.clock = st.clock.Add(30 * 24 * time.Hour)
|
||||
for range Retries + 2 {
|
||||
if rep := st.s.Pass(ctx); rep.Retried+rep.GivenUp != 0 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_ADownloadGoneFromTheClientEndsAfterHalfAnHour_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "downloading", NumSeeds: 4})
|
||||
ctx := context.Background()
|
||||
delete(st.client.held, hash)
|
||||
st.client.held[other] = downloader.Torrent{Hash: other, Name: "Someone else's", Progress: 1, State: "uploading"}
|
||||
|
||||
if rep := st.s.Pass(ctx); rep.Gone != 0 {
|
||||
t.Fatalf("ended at first sight: %+v", rep)
|
||||
}
|
||||
st.clock = st.clock.Add(31 * time.Minute)
|
||||
if rep := st.s.Pass(ctx); rep.Gone != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
g := st.grabNow(t)
|
||||
if g.EndReason != EndGone || !st.searchedAgain(t) {
|
||||
t.Errorf("grab %q, searched again %v", g.EndReason, st.searchedAgain(t))
|
||||
}
|
||||
if blocks, _ := st.q.ListReleaseBlocks(ctx); len(blocks) != 0 {
|
||||
t.Errorf("a release nothing was wrong with was held back: %+v", blocks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AnEmptyListIsNotAMeasurement_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "downloading"})
|
||||
ctx := context.Background()
|
||||
delete(st.client.held, hash)
|
||||
st.clock = st.clock.Add(24 * time.Hour)
|
||||
for range 3 {
|
||||
if rep := st.s.Pass(ctx); rep.Gone != 0 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
}
|
||||
if len(st.ledger(t)) != 0 || st.grabNow(t).EndedAt.Valid {
|
||||
t.Error("a client listing nothing was taken to have lost the download")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AnUnreachableClientIsRecordedAndSettled_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"})
|
||||
ctx := context.Background()
|
||||
st.clock = st.clock.Add(7 * time.Hour)
|
||||
st.client.err = errors.New("dial tcp: connection refused")
|
||||
st.s.Pass(ctx)
|
||||
down, err := st.q.ListClientsDown(ctx)
|
||||
if err != nil || len(down) != 1 || down[0].Client != "qbittorrent" {
|
||||
t.Fatalf("clients down = %+v %v", down, err)
|
||||
}
|
||||
if st.client.reannounce != 0 {
|
||||
t.Error("a download was judged through a client that did not answer")
|
||||
}
|
||||
st.client.err = nil
|
||||
st.s.Pass(ctx)
|
||||
if down, _ := st.q.ListClientsDown(ctx); len(down) != 0 {
|
||||
t.Errorf("still down: %+v", down)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AnImportedDownloadSeedsThenIsRemoved_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", Ratio: 0.8})
|
||||
ctx := context.Background()
|
||||
if _, err := st.q.RecordImport(ctx, dbq.RecordImportParams{ID: st.grab.ID, ImportState: "imported",
|
||||
ImportReason: "Imported.", ImportDetail: []byte("{}"), ImportedFiles: 3, EndReason: "imported"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !st.grabNow(t).Seeding {
|
||||
t.Fatal("an import did not leave its download seeding")
|
||||
}
|
||||
if rep := st.s.Pass(ctx); rep.Released != 0 || len(st.client.deleted) != 0 {
|
||||
t.Fatalf("removed before the ratio: %+v", rep)
|
||||
}
|
||||
if g := st.grabNow(t); g.Ratio < 0.79 || g.Ratio > 0.81 {
|
||||
t.Errorf("ratio recorded %v", g.Ratio)
|
||||
}
|
||||
tor := st.client.held[hash]
|
||||
tor.Ratio = 2.1
|
||||
st.client.held[hash] = tor
|
||||
if rep := st.s.Pass(ctx); rep.Released != 1 || len(st.client.deleted) != 1 {
|
||||
t.Fatalf("pass = %+v, deleted %v", rep, st.client.deleted)
|
||||
}
|
||||
if !st.grabNow(t).SeedReleasedAt.Valid {
|
||||
t.Error("the release was not recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AnImportedDownloadIsRemovedWhenItsDaysAreUp_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", Ratio: 0.1})
|
||||
ctx := context.Background()
|
||||
if _, err := st.q.RecordImport(ctx, dbq.RecordImportParams{ID: st.grab.ID, ImportState: "imported",
|
||||
ImportReason: "Imported.", ImportDetail: []byte("{}"), ImportedFiles: 3, EndReason: "imported"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st.clock = st.clock.Add(15 * 24 * time.Hour)
|
||||
if rep := st.s.Pass(ctx); rep.Released != 1 || len(st.client.deleted) != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_ADownloadTheClientWouldNotRemoveIsAskedAgain_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "error"})
|
||||
ctx := context.Background()
|
||||
st.client.deleteErr = errors.New("HTTP 500")
|
||||
if rep := st.s.Pass(ctx); rep.GivenUp != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
if !st.grabNow(t).EndedAt.Valid {
|
||||
t.Fatal("a refused removal held the grab")
|
||||
}
|
||||
open := 0
|
||||
for _, r := range st.ledger(t) {
|
||||
if r.Trigger == NotRemoved && r.Outcome == "open" {
|
||||
open++
|
||||
}
|
||||
}
|
||||
if open != 1 {
|
||||
t.Fatalf("ledger = %+v", st.ledger(t))
|
||||
}
|
||||
st.client.deleteErr = nil
|
||||
if rep := st.s.Pass(ctx); rep.Removed != 1 || len(st.client.deleted) != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
for _, r := range st.ledger(t) {
|
||||
if r.Outcome == "open" {
|
||||
t.Errorf("still open: %+v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_APersonGivingUpSetsTheReleaseAsideForGood_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "downloading"})
|
||||
ctx := context.Background()
|
||||
if _, err := st.s.Abandon(ctx, st.grab.ID, pgtype.UUID{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := st.grabNow(t)
|
||||
if g.EndReason != EndAbandoned || len(st.client.deleted) != 1 || !st.searchedAgain(t) {
|
||||
t.Errorf("grab %q, deleted %v", g.EndReason, st.client.deleted)
|
||||
}
|
||||
blocks, _ := st.q.ListReleaseBlocks(ctx)
|
||||
if len(blocks) != 1 || blocks[0].Reason != acquire.BlockOperator || blocks[0].ExpiresAt.Valid {
|
||||
t.Errorf("blocks = %+v", blocks)
|
||||
}
|
||||
if _, err := st.s.Abandon(ctx, st.grab.ID, pgtype.UUID{}); !errors.Is(err, ErrNotAbandonable) {
|
||||
t.Errorf("abandoned twice: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AHoldExpiresButABlockForGoodIsNeverShortened_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "x", Progress: 0.4, State: "downloading"})
|
||||
ctx := context.Background()
|
||||
mem := acquire.NewMemory(st.s.pool)
|
||||
past := pgtype.Timestamptz{Time: time.Now().Add(-time.Hour), Valid: true}
|
||||
if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: other, Name: "old", Reason: acquire.BlockStalled, ExpiresAt: past}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if blocks, _ := st.q.ListReleaseBlocks(ctx); len(blocks) != 0 {
|
||||
t.Errorf("an expired hold still holds: %+v", blocks)
|
||||
}
|
||||
if n, err := st.q.DeleteExpiredBlocks(ctx); err != nil || n != 1 {
|
||||
t.Errorf("lifted %d %v", n, err)
|
||||
}
|
||||
|
||||
if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: hash, Name: "x", Reason: acquire.BlockOperator}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
soon := pgtype.Timestamptz{Time: time.Now().Add(time.Hour), Valid: true}
|
||||
if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: hash, Name: "x", Reason: acquire.BlockStalled, ExpiresAt: soon}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocks, _ := st.q.ListReleaseBlocks(ctx)
|
||||
if len(blocks) != 1 || blocks[0].Reason != acquire.BlockOperator || blocks[0].ExpiresAt.Valid {
|
||||
t.Errorf("a block for good was shortened: %+v", blocks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpan(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
d time.Duration
|
||||
want string
|
||||
}{{45 * time.Minute, "45 minutes"}, {7 * time.Hour, "7 hours"}, {3 * 24 * time.Hour, "3 days"}} {
|
||||
if got := span(c.d); got != c.want {
|
||||
t.Errorf("span(%s) = %q, want %q", c.d, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -630,6 +630,8 @@ export type WantGrab = {
|
||||
progress: number;
|
||||
import_state: ImportState;
|
||||
import_reason: string;
|
||||
// Why an ended download ended (#5523).
|
||||
end_detail: string;
|
||||
imported_at: string | null;
|
||||
imported_files: number;
|
||||
};
|
||||
@@ -710,6 +712,12 @@ export type SearchSettings = {
|
||||
// scan_paths), and the library's folders (#5522).
|
||||
import_root: string;
|
||||
scan_paths: string[];
|
||||
// The download lifecycle (#5523): an imported download seeds to this
|
||||
// ratio or for this many days (0: removed once imported; no time limit),
|
||||
// and one nobody shares is given up after this many hours (0: never).
|
||||
seed_ratio: number;
|
||||
seed_days: number;
|
||||
stall_hours: number;
|
||||
};
|
||||
|
||||
export type WantCounts = {
|
||||
@@ -761,17 +769,42 @@ export type Download = {
|
||||
files: string[];
|
||||
skipped: number;
|
||||
rehomed_from?: string;
|
||||
// Why an ended download ended; for an imported one, its seeding (#5523).
|
||||
end_detail: string;
|
||||
seeding: boolean;
|
||||
ratio: number;
|
||||
seed_released_at: string | null;
|
||||
// What is wrong with it now, if anything.
|
||||
condition: DownloadCondition | null;
|
||||
};
|
||||
|
||||
export type DownloadsPage = { items: Download[]; limit: number; offset: number };
|
||||
// An open condition on a download, from the lifecycle's ledger (#5523).
|
||||
export type DownloadCondition = {
|
||||
trigger: 'no_peers' | 'no_metadata' | 'client_error' | 'client_unreachable' | 'gone' | 'not_removed';
|
||||
detail: string;
|
||||
retries: number;
|
||||
since: string;
|
||||
};
|
||||
|
||||
// A download client not answering now.
|
||||
export type ClientDown = { client: string; detail: string; since: string };
|
||||
|
||||
export type DownloadsPage = {
|
||||
items: Download[];
|
||||
limit: number;
|
||||
offset: number;
|
||||
clients_down: ClientDown[];
|
||||
};
|
||||
|
||||
// A release set aside: by a person, by its files, or by an import's check.
|
||||
export type ReleaseBlock = {
|
||||
info_hash: string;
|
||||
name: string;
|
||||
reason: 'operator' | 'files' | 'import';
|
||||
reason: 'operator' | 'files' | 'import' | 'stalled';
|
||||
detail: string;
|
||||
blocked_at: string;
|
||||
// When a release given up on may be taken again; null is for good.
|
||||
expires_at: string | null;
|
||||
want_title: string;
|
||||
want_artist: string;
|
||||
};
|
||||
|
||||
@@ -160,6 +160,9 @@ export async function putSearchSettings(body: {
|
||||
search_interval_hours: number;
|
||||
allowed_qualities: string[];
|
||||
import_root: string;
|
||||
seed_ratio: number;
|
||||
seed_days: number;
|
||||
stall_hours: number;
|
||||
}): Promise<SearchSettings> {
|
||||
return api.put<SearchSettings>('/api/admin/acquisition/settings', body);
|
||||
}
|
||||
@@ -199,6 +202,12 @@ export async function refuseDownload(id: string): Promise<void> {
|
||||
await api.post(`/api/admin/downloads/${id}/refuse`, {});
|
||||
}
|
||||
|
||||
// A running download given up: the release is set aside for good, the
|
||||
// download removed and the album searched for again (#5523).
|
||||
export async function abandonDownload(id: string): Promise<void> {
|
||||
await api.post(`/api/admin/downloads/${id}/abandon`, {});
|
||||
}
|
||||
|
||||
export async function listReleaseBlocks(offset: number = 0): Promise<ReleaseBlocksPage> {
|
||||
return api.get<ReleaseBlocksPage>(`/api/admin/release-blocks?offset=${offset}`);
|
||||
}
|
||||
|
||||
@@ -11,13 +11,23 @@
|
||||
// for, whether the best release is taken without asking, and which
|
||||
// qualities are taken at all. The schedule wakes every ten minutes and
|
||||
// keeps to each indexer's searches-per-hour. A finished download is
|
||||
// checked and filed into one of the library's folders (#5522).
|
||||
// checked and filed into one of the library's folders (#5522); it then
|
||||
// seeds until the ratio or the days below, and a download nobody shares
|
||||
// is given up after the hours below (#5523).
|
||||
|
||||
const client = useQueryClient();
|
||||
const store = createSearchSettingsQuery();
|
||||
const query = $derived($store);
|
||||
|
||||
type Form = { auto_grab: boolean; search_interval_hours: number; allowed: string[]; import_root: string };
|
||||
type Form = {
|
||||
auto_grab: boolean;
|
||||
search_interval_hours: number;
|
||||
allowed: string[];
|
||||
import_root: string;
|
||||
seed_ratio: number;
|
||||
seed_days: number;
|
||||
stall_hours: number;
|
||||
};
|
||||
|
||||
// The form is the server's settings until edited; saved holds what a save
|
||||
// returned until the query catches up.
|
||||
@@ -32,7 +42,10 @@
|
||||
auto_grab: s.auto_grab,
|
||||
search_interval_hours: s.search_interval_hours,
|
||||
allowed: [...s.allowed_qualities],
|
||||
import_root: s.import_root
|
||||
import_root: s.import_root,
|
||||
seed_ratio: s.seed_ratio,
|
||||
seed_days: s.seed_days,
|
||||
stall_hours: s.stall_hours
|
||||
};
|
||||
}
|
||||
|
||||
@@ -55,7 +68,10 @@
|
||||
auto_grab: form.auto_grab,
|
||||
search_interval_hours: Number(form.search_interval_hours),
|
||||
allowed_qualities: form.allowed,
|
||||
import_root: form.import_root
|
||||
import_root: form.import_root,
|
||||
seed_ratio: Number(form.seed_ratio),
|
||||
seed_days: Number(form.seed_days),
|
||||
stall_hours: Number(form.stall_hours)
|
||||
});
|
||||
edited = null;
|
||||
await client.invalidateQueries({ queryKey: qk.searchSettings() });
|
||||
@@ -158,7 +174,7 @@
|
||||
<span class="text-sm text-text-primary">File new music into</span>
|
||||
<span class="block text-xs text-text-secondary">
|
||||
A download that checks out is copied here as Artist/Album (Year)/Artist - Album - NN -
|
||||
Title, tagged with its MusicBrainz ids. The download itself keeps seeding.
|
||||
Title, tagged with its MusicBrainz ids.
|
||||
</span>
|
||||
<select
|
||||
value={form.import_root || query.data.scan_paths[0]}
|
||||
@@ -174,11 +190,68 @@
|
||||
{:else if query.data.scan_paths.length === 1}
|
||||
<p class="text-xs text-text-secondary">
|
||||
A download that checks out is copied into <span class="font-mono">{query.data.scan_paths[0]}</span>
|
||||
as Artist/Album (Year)/Artist - Album - NN - Title, tagged with its MusicBrainz ids. The
|
||||
download itself keeps seeding.
|
||||
as Artist/Album (Year)/Artist - Album - NN - Title, tagged with its MusicBrainz ids.
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
<fieldset class="space-y-3">
|
||||
<legend class="text-sm text-text-primary">After the download</legend>
|
||||
<label class="block">
|
||||
<span class="block text-xs text-text-secondary">
|
||||
The files are copied into the library, so the download goes on sharing until either limit,
|
||||
then is removed with its files. A ratio of 0 removes it once imported; 0 days is no time
|
||||
limit.
|
||||
</span>
|
||||
<span class="mt-1 flex flex-wrap items-center gap-2 text-sm text-text-secondary">
|
||||
Seed to a ratio of
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="100"
|
||||
step="0.1"
|
||||
aria-label="Seed ratio"
|
||||
value={form.seed_ratio}
|
||||
oninput={(e) => edit({ seed_ratio: Number(e.currentTarget.value) })}
|
||||
class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary
|
||||
focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent"
|
||||
/>
|
||||
or for
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="365"
|
||||
aria-label="Seed days"
|
||||
value={form.seed_days}
|
||||
oninput={(e) => edit({ seed_days: Number(e.currentTarget.value) })}
|
||||
class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary
|
||||
focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent"
|
||||
/>
|
||||
days
|
||||
</span>
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="block text-xs text-text-secondary">
|
||||
A download nobody is sharing is asked for again three times, ten minutes apart, then given
|
||||
up: removed, held back for a week, and the album searched for again. 0 never gives up for
|
||||
that; a download the client fails is always given up.
|
||||
</span>
|
||||
<span class="mt-1 flex flex-wrap items-center gap-2 text-sm text-text-secondary">
|
||||
Give up on a stalled download after
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="720"
|
||||
aria-label="Stall hours"
|
||||
value={form.stall_hours}
|
||||
oninput={(e) => edit({ stall_hours: Number(e.currentTarget.value) })}
|
||||
class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary
|
||||
focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent"
|
||||
/>
|
||||
hours
|
||||
</span>
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<p class="text-xs text-text-secondary" data-testid="last-pass">{lastPass(query.data)}</p>
|
||||
|
||||
<div class="flex justify-end">
|
||||
|
||||
@@ -24,7 +24,10 @@ const settings: SearchSettings = {
|
||||
last_pass_at: new Date(Date.now() - 5 * 60_000).toISOString(),
|
||||
last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true },
|
||||
import_root: '',
|
||||
scan_paths: ['/music', '/music2']
|
||||
scan_paths: ['/music', '/music2'],
|
||||
seed_ratio: 2,
|
||||
seed_days: 14,
|
||||
stall_hours: 6
|
||||
};
|
||||
|
||||
afterEach(() => vi.clearAllMocks());
|
||||
@@ -60,7 +63,10 @@ describe('SearchSettingsCard', () => {
|
||||
auto_grab: false,
|
||||
search_interval_hours: 24,
|
||||
allowed_qualities: ['lossless', 'lossy_high', 'unknown'],
|
||||
import_root: ''
|
||||
import_root: '',
|
||||
seed_ratio: 2,
|
||||
seed_days: 14,
|
||||
stall_hours: 6
|
||||
})
|
||||
);
|
||||
});
|
||||
@@ -82,6 +88,20 @@ describe('SearchSettingsCard', () => {
|
||||
expect(screen.getByText('/music')).toBeTruthy();
|
||||
});
|
||||
|
||||
test('seeding and giving up on a stalled download are set here', async () => {
|
||||
setup();
|
||||
expect((screen.getByLabelText(/seed ratio/i) as HTMLInputElement).value).toBe('2');
|
||||
await fireEvent.input(screen.getByLabelText(/seed ratio/i), { target: { value: '1.5' } });
|
||||
await fireEvent.input(screen.getByLabelText(/seed days/i), { target: { value: '7' } });
|
||||
await fireEvent.input(screen.getByLabelText(/stall hours/i), { target: { value: '0' } });
|
||||
await fireEvent.click(screen.getByRole('button', { name: /^save$/i }));
|
||||
await waitFor(() =>
|
||||
expect(putSearchSettings).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ seed_ratio: 1.5, seed_days: 7, stall_hours: 0 })
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
test('before the first pass it says so', () => {
|
||||
setup({ ...settings, last_pass_at: null, last_pass: {} });
|
||||
expect(screen.getByTestId('last-pass').textContent).toMatch(/hasn’t run yet/);
|
||||
|
||||
@@ -8,9 +8,10 @@
|
||||
import ListContinuation from '#lib/components/ListContinuation.svelte';
|
||||
import type { ReleaseBlock } from '#lib/api/types.js';
|
||||
|
||||
// The Blocked tab (M502 #5521–#5522): releases the search will never take
|
||||
// The Blocked tab (M502 #5521–#5523): releases the search will not take
|
||||
// again — set aside by a person, by what their files were, or by an
|
||||
// import's check that found they weren't the album.
|
||||
// import's check that found they weren't the album; and, for a week,
|
||||
// releases given up on because they would not finish.
|
||||
|
||||
const client = useQueryClient();
|
||||
const blocksStore = createReleaseBlocksQuery();
|
||||
@@ -35,8 +36,13 @@
|
||||
function by(b: ReleaseBlock): string {
|
||||
if (b.reason === 'operator') return 'Set aside by you';
|
||||
if (b.reason === 'files') return 'Set aside for its files';
|
||||
if (b.reason === 'stalled') return 'Given up on: it would not finish';
|
||||
return 'Refused on import';
|
||||
}
|
||||
|
||||
// A release given up on is held back for a while, then may be taken again.
|
||||
const until = (b: ReleaseBlock) =>
|
||||
b.expires_at ? ` · held back until ${new Date(b.expires_at).toLocaleDateString()}` : '';
|
||||
</script>
|
||||
|
||||
{#if blocks.isPending}
|
||||
@@ -61,7 +67,7 @@
|
||||
{#if b.want_title}
|
||||
<div class="truncate text-xs text-text-secondary">For {b.want_artist} – {b.want_title}</div>
|
||||
{/if}
|
||||
<div class="text-xs text-text-muted">{by(b)} {relativeTime(b.blocked_at)}</div>
|
||||
<div class="text-xs text-text-muted">{by(b)} {relativeTime(b.blocked_at)}{until(b)}</div>
|
||||
{#if b.detail}
|
||||
<p class="text-xs text-text-secondary">{b.detail}</p>
|
||||
{/if}
|
||||
|
||||
@@ -18,6 +18,7 @@ const block: ReleaseBlock = {
|
||||
reason: 'import',
|
||||
detail: 'The album tag says another version.',
|
||||
blocked_at: new Date(Date.now() - 3_600_000).toISOString(),
|
||||
expires_at: null,
|
||||
want_title: 'Random Access Memories',
|
||||
want_artist: 'Daft Punk'
|
||||
};
|
||||
@@ -43,6 +44,12 @@ describe('BlockList', () => {
|
||||
await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('abc123'));
|
||||
});
|
||||
|
||||
test('a release given up on says until when it is held back', () => {
|
||||
setup([{ ...block, reason: 'stalled', detail: 'Nobody was sharing it.', expires_at: '2026-10-17T12:00:00Z' }]);
|
||||
const line = screen.getByText(/given up on: it would not finish/i).textContent ?? '';
|
||||
expect(line).toMatch(/held back until/);
|
||||
});
|
||||
|
||||
test('with nothing blocked it says so', () => {
|
||||
setup([]);
|
||||
expect(screen.getByText(/no releases are blocked/i)).toBeTruthy();
|
||||
|
||||
@@ -1,23 +1,32 @@
|
||||
<script lang="ts">
|
||||
import { Download as DownloadIcon } from 'lucide-svelte';
|
||||
import { useQueryClient } from '@tanstack/svelte-query';
|
||||
import { createDownloadsQuery, importAnyway, refuseDownload, retryImport } from '#lib/api/wanted.js';
|
||||
import {
|
||||
abandonDownload,
|
||||
createDownloadsQuery,
|
||||
importAnyway,
|
||||
refuseDownload,
|
||||
retryImport
|
||||
} from '#lib/api/wanted.js';
|
||||
import { errMessage } from '#lib/api/errors.js';
|
||||
import { pushToast } from '#lib/stores/toast.svelte.js';
|
||||
import { relativeTime } from '#lib/utils/relativeTime.js';
|
||||
import ListContinuation from '#lib/components/ListContinuation.svelte';
|
||||
import type { Download, ImportEvidence } from '#lib/api/types.js';
|
||||
import type { Download, DownloadCondition, ImportEvidence } from '#lib/api/types.js';
|
||||
|
||||
// The Activity tab (M502 #5522): every download Minstrel took, running
|
||||
// ones first. A finished download is checked against MusicBrainz before it
|
||||
// is filed; one the check could not settle waits here for a person, who
|
||||
// imports it anyway or refuses it. A refused release is set aside (the
|
||||
// Blocked tab) and its album searched for again.
|
||||
// Blocked tab) and its album searched for again. A download that fails, or
|
||||
// that nobody shares, is retried and then given up the same way, held back
|
||||
// for a week; an imported one seeds, then is removed (#5523).
|
||||
|
||||
const client = useQueryClient();
|
||||
const downloadsStore = createDownloadsQuery();
|
||||
const downloads = $derived($downloadsStore);
|
||||
const items = $derived((downloads.data?.pages ?? []).flatMap((p) => p.items) as Download[]);
|
||||
const clientsDown = $derived(downloads.data?.pages[0]?.clients_down ?? []);
|
||||
|
||||
let busy = $state<string | null>(null);
|
||||
|
||||
@@ -39,9 +48,25 @@
|
||||
const onRetry = (d: Download) => act(d, retryImport, `${d.name} will be tried again within the minute.`);
|
||||
const onRefuse = (d: Download) =>
|
||||
act(d, refuseDownload, `Refused ${d.name}. It's set aside and the album will be searched for again.`);
|
||||
const onAbandon = (d: Download) =>
|
||||
act(d, abandonDownload, `Gave up on ${d.name}. It's set aside and the album will be searched for again.`);
|
||||
|
||||
// A download still downloading, or waiting to be: one a person may give up.
|
||||
const running = (d: Download) =>
|
||||
!d.ended_at && (d.import_state === '' || d.import_state === 'downloading' || d.import_state === 'waiting');
|
||||
|
||||
// How a download that ended without an import ended.
|
||||
const endings: Record<string, string> = {
|
||||
stalled: 'Given up',
|
||||
failed: 'Failed',
|
||||
abandoned: 'Given up by you',
|
||||
gone: 'Left the client',
|
||||
superseded: 'Replaced'
|
||||
};
|
||||
|
||||
// The state of a download as a short label, and whether it needs a person.
|
||||
function label(d: Download): string {
|
||||
if (d.ended_at && endings[d.end_reason]) return endings[d.end_reason];
|
||||
switch (d.import_state) {
|
||||
case 'downloading':
|
||||
case '':
|
||||
@@ -68,7 +93,7 @@
|
||||
|
||||
function badge(d: Download): string {
|
||||
if (needsYou(d)) return 'border-accent text-text-primary';
|
||||
if (d.import_state === 'refused') return 'border-error-fg text-error-fg';
|
||||
if (d.import_state === 'refused' || (d.ended_at && endings[d.end_reason])) return 'border-error-fg text-error-fg';
|
||||
return 'border-border text-text-secondary';
|
||||
}
|
||||
|
||||
@@ -82,6 +107,26 @@
|
||||
return parts.join(' · ');
|
||||
}
|
||||
|
||||
// What the lifecycle sees wrong with a running download, and what it did.
|
||||
function condition(c: DownloadCondition): string {
|
||||
let said = c.detail;
|
||||
if (c.trigger === 'no_peers' || c.trigger === 'no_metadata') {
|
||||
said += c.retries
|
||||
? ` Asked for again ${c.retries} of 3 times; given up after that.`
|
||||
: ' It will be asked for again three times, then given up.';
|
||||
}
|
||||
return said;
|
||||
}
|
||||
|
||||
// What ended a download that was not imported, in the lifecycle's words.
|
||||
const ending = (d: Download) => (d.ended_at && endings[d.end_reason] ? d.end_detail : '');
|
||||
|
||||
function seeded(d: Download): string | null {
|
||||
if (d.seed_released_at) return `Seeded, then removed from the download client ${relativeTime(d.seed_released_at)}.`;
|
||||
if (d.seeding) return `Seeding: share ratio ${d.ratio.toFixed(2)}.`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function when(d: Download): string {
|
||||
const who = d.origin === 'operator' ? 'by you' : 'by the schedule';
|
||||
let said = `Taken ${who} ${relativeTime(d.grabbed_at)}`;
|
||||
@@ -103,6 +148,17 @@
|
||||
'rounded-md border border-border px-2.5 py-1 text-xs text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50';
|
||||
</script>
|
||||
|
||||
{#each clientsDown as c (c.client)}
|
||||
<p
|
||||
class="mb-3 rounded-lg border border-error-fg bg-surface p-3 text-sm text-text-primary"
|
||||
role="status"
|
||||
data-testid="client-down"
|
||||
>
|
||||
Can't reach the download client {c.client} since {relativeTime(c.since)}: {c.detail}. Its
|
||||
downloads are left as they are until it answers.
|
||||
</p>
|
||||
{/each}
|
||||
|
||||
{#if downloads.isPending}
|
||||
<p class="text-text-secondary">Reading the downloads…</p>
|
||||
{:else if downloads.isError && !downloads.isFetchNextPageError}
|
||||
@@ -147,6 +203,12 @@
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if running(d) && d.condition}
|
||||
<p class="text-xs text-text-primary" data-testid="download-condition">{condition(d.condition)}</p>
|
||||
{/if}
|
||||
{#if ending(d)}
|
||||
<p class="text-xs text-text-secondary" data-testid="download-ending">{ending(d)}</p>
|
||||
{/if}
|
||||
{#if d.import_reason}
|
||||
<p class="text-xs {needsYou(d) ? 'text-text-primary' : 'text-text-secondary'}" data-testid="download-reason">
|
||||
{d.import_reason}
|
||||
@@ -158,12 +220,21 @@
|
||||
{#if filed(d)}
|
||||
<p class="truncate text-xs text-text-secondary" title={d.destination}>{filed(d)}</p>
|
||||
{/if}
|
||||
{#if seeded(d)}
|
||||
<p class="text-xs text-text-muted" data-testid="download-seeding">{seeded(d)}</p>
|
||||
{/if}
|
||||
{#if d.rehomed_from}
|
||||
<p class="text-xs text-text-muted">Taken for {d.rehomed_from}; it turned out to be this album.</p>
|
||||
{/if}
|
||||
<p class="text-xs text-text-muted">{when(d)}</p>
|
||||
|
||||
{#if needsYou(d)}
|
||||
{#if running(d)}
|
||||
<div class="flex flex-wrap gap-1.5 pt-1">
|
||||
<button type="button" class={button} disabled={busy === d.id} onclick={() => onAbandon(d)}>
|
||||
Give up
|
||||
</button>
|
||||
</div>
|
||||
{:else if needsYou(d)}
|
||||
<div class="flex flex-wrap gap-1.5 pt-1">
|
||||
{#if d.import_state === 'review'}
|
||||
<button type="button" class={button} disabled={busy === d.id} onclick={() => onImport(d)}>
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
import { afterEach, describe, expect, test, vi } from 'vitest';
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/svelte';
|
||||
import { mockInfiniteQuery } from '../../../test-utils/query';
|
||||
import type { Download } from '#lib/api/types.js';
|
||||
import type { ClientDown, Download } from '#lib/api/types.js';
|
||||
|
||||
vi.mock('#lib/api/wanted.js', () => ({
|
||||
createDownloadsQuery: vi.fn(),
|
||||
importAnyway: vi.fn().mockResolvedValue(undefined),
|
||||
retryImport: vi.fn().mockResolvedValue(undefined),
|
||||
refuseDownload: vi.fn().mockResolvedValue(undefined)
|
||||
refuseDownload: vi.fn().mockResolvedValue(undefined),
|
||||
abandonDownload: vi.fn().mockResolvedValue(undefined)
|
||||
}));
|
||||
vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() }));
|
||||
|
||||
import DownloadList from './DownloadList.svelte';
|
||||
import { createDownloadsQuery, importAnyway, refuseDownload, retryImport } from '#lib/api/wanted.js';
|
||||
import { abandonDownload, createDownloadsQuery, importAnyway, refuseDownload, retryImport } from '#lib/api/wanted.js';
|
||||
|
||||
const ago = (ms: number) => new Date(Date.now() - ms).toISOString();
|
||||
|
||||
@@ -38,15 +39,22 @@ function download(id: string, extra: Partial<Download> = {}): Download {
|
||||
destination: '',
|
||||
files: [],
|
||||
skipped: 0,
|
||||
end_detail: '',
|
||||
seeding: false,
|
||||
ratio: 0,
|
||||
seed_released_at: null,
|
||||
condition: null,
|
||||
...extra
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => vi.clearAllMocks());
|
||||
|
||||
function setup(items: Download[]) {
|
||||
function setup(items: Download[], clientsDown: ClientDown[] = []) {
|
||||
vi.mocked(createDownloadsQuery).mockReturnValue(
|
||||
mockInfiniteQuery({ pages: [{ items, total: items.length, limit: 50, offset: 0 }] }) as unknown as ReturnType<
|
||||
mockInfiniteQuery({
|
||||
pages: [{ items, total: items.length, limit: 50, offset: 0, clients_down: clientsDown }]
|
||||
}) as unknown as ReturnType<
|
||||
typeof createDownloadsQuery
|
||||
>
|
||||
);
|
||||
@@ -54,13 +62,45 @@ function setup(items: Download[]) {
|
||||
}
|
||||
|
||||
describe('DownloadList', () => {
|
||||
test('a running download shows how far it has got', () => {
|
||||
test('a running download shows how far it has got, and can be given up', async () => {
|
||||
setup([download('d1', { progress: 0.375 })]);
|
||||
expect(screen.getByTestId('download-state').textContent).toMatch(/downloading/i);
|
||||
expect(screen.getByRole('progressbar', { name: /downloaded/i }).getAttribute('aria-valuenow')).toBe('37');
|
||||
expect(screen.queryByTestId('download-condition')).toBeNull();
|
||||
await fireEvent.click(screen.getByRole('button', { name: /give up/i }));
|
||||
await waitFor(() => expect(abandonDownload).toHaveBeenCalledWith('d1'));
|
||||
});
|
||||
|
||||
test('a stalled download says what is wrong and how often it was asked for again', () => {
|
||||
setup([
|
||||
download('d1', {
|
||||
progress: 0.4,
|
||||
condition: { trigger: 'no_peers', detail: 'Nobody has shared it in 7 hours; 40% downloaded.', retries: 2, since: ago(1_200_000) }
|
||||
})
|
||||
]);
|
||||
expect(screen.getByTestId('download-condition').textContent).toMatch(
|
||||
/nobody has shared it in 7 hours; 40% downloaded\. asked for again 2 of 3 times/i
|
||||
);
|
||||
});
|
||||
|
||||
test('a download given up on says why it ended', () => {
|
||||
setup([
|
||||
download('d1', {
|
||||
ended_at: ago(60_000),
|
||||
end_reason: 'stalled',
|
||||
end_detail: 'Taken 7 hours ago and never finished: nobody was sharing it. Held back until 17 Oct.'
|
||||
})
|
||||
]);
|
||||
expect(screen.getByTestId('download-state').textContent).toMatch(/given up/i);
|
||||
expect(screen.getByTestId('download-ending').textContent).toMatch(/nobody was sharing it/);
|
||||
expect(screen.queryByRole('button')).toBeNull();
|
||||
});
|
||||
|
||||
test('a download client that does not answer is named above the list', () => {
|
||||
setup([download('d1')], [{ client: 'qbittorrent', detail: 'connection refused', since: ago(600_000) }]);
|
||||
expect(screen.getByTestId('client-down').textContent).toMatch(/can't reach the download client qbittorrent .*connection refused/i);
|
||||
});
|
||||
|
||||
test('a doubtful download says why and can be imported anyway or refused', async () => {
|
||||
setup([
|
||||
download('d1', {
|
||||
@@ -102,6 +142,17 @@ describe('DownloadList', () => {
|
||||
expect(screen.queryByRole('button')).toBeNull();
|
||||
});
|
||||
|
||||
test('an imported download says how its seeding goes', () => {
|
||||
const imported = { ended_at: ago(60_000), end_reason: 'imported', import_state: 'imported' as const };
|
||||
setup([
|
||||
download('d1', { ...imported, seeding: true, ratio: 0.8 }),
|
||||
download('d2', { ...imported, seeding: true, ratio: 2.1, seed_released_at: ago(60_000) })
|
||||
]);
|
||||
const lines = screen.getAllByTestId('download-seeding').map((e) => e.textContent);
|
||||
expect(lines[0]).toMatch(/seeding: share ratio 0\.80/i);
|
||||
expect(lines[1]).toMatch(/seeded, then removed/i);
|
||||
});
|
||||
|
||||
test('with nothing downloaded it says what lands here', () => {
|
||||
setup([]);
|
||||
expect(screen.getByText(/nothing has been downloaded yet/i)).toBeTruthy();
|
||||
|
||||
@@ -117,7 +117,10 @@
|
||||
said = `${when}: ${s.error}`;
|
||||
}
|
||||
}
|
||||
if (w.grab?.import_state === 'refused') said += `. Last download refused: ${w.grab.import_reason}`;
|
||||
const g = w.grab;
|
||||
if (g?.import_state === 'refused') said += `. Last download refused: ${g.import_reason}`;
|
||||
else if (g && !g.live && ['stalled', 'failed', 'gone'].includes(g.end_reason) && g.end_detail)
|
||||
said += `. Last download given up: ${g.end_detail}`;
|
||||
return said;
|
||||
}
|
||||
|
||||
|
||||
@@ -65,6 +65,7 @@ function grab(extra: Partial<WantGrab> = {}): WantGrab {
|
||||
progress: 0,
|
||||
import_state: '',
|
||||
import_reason: '',
|
||||
end_detail: '',
|
||||
imported_at: null,
|
||||
imported_files: 0,
|
||||
...extra
|
||||
@@ -155,6 +156,16 @@ describe('WantList', () => {
|
||||
expect(screen.getByTestId('want-search').textContent).toMatch(/last download refused: another album/i);
|
||||
});
|
||||
|
||||
test('a download given up on is named beside the search that follows it', () => {
|
||||
setup([
|
||||
want('w1', {
|
||||
search: { searched_at: ago(3_600_000), outcome: 'waiting', considered: 4, best_release: '', best_verdict: '', error: '' },
|
||||
grab: grab({ live: false, end_reason: 'stalled', import_state: 'downloading', end_detail: 'Nobody was sharing it.' })
|
||||
})
|
||||
]);
|
||||
expect(screen.getByTestId('want-search').textContent).toMatch(/last download given up: nobody was sharing it/i);
|
||||
});
|
||||
|
||||
test('a held album says how it arrived', () => {
|
||||
setup(
|
||||
[
|
||||
|
||||
@@ -76,5 +76,7 @@
|
||||
"candidate_not_found": "That release is no longer in the list. Search again.",
|
||||
"no_infohash": "That release can't be set aside: its indexer gave no infohash to know it by.",
|
||||
"want_held": "The library already holds that album.",
|
||||
"block_not_found": "That release isn't set aside any more."
|
||||
"block_not_found": "That release isn't set aside any more.",
|
||||
"not_running": "That download isn't running any more. Refresh the list.",
|
||||
"bad_lifecycle": "Seed ratio is 0 to 100, seed days 0 to 365, and stall hours 0 to 720."
|
||||
}
|
||||
|
||||
@@ -59,7 +59,10 @@ vi.mock('#lib/api/wanted.js', async () => {
|
||||
last_pass: {},
|
||||
import_root: '',
|
||||
// Not "/music": a Lidarr root folder on this page is named that.
|
||||
scan_paths: ['/library']
|
||||
scan_paths: ['/library'],
|
||||
seed_ratio: 2,
|
||||
seed_days: 14,
|
||||
stall_hours: 6
|
||||
}
|
||||
})
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user