DLNA renderers, app updates told like Librarian, acquisition without Lidarr, Collection and admin rework (M502–M505, M542) #156
@@ -24,6 +24,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/gc"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/importer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/logging"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/notifications"
|
||||
@@ -341,6 +342,11 @@ func run() error {
|
||||
}
|
||||
go importer.New(pool, scanner, cat, printer, cfg.Library.ScanPaths, logger.With("component", "import")).Run(ctx)
|
||||
|
||||
// Download lifecycle (M502 #5523): every ten minutes, a download that
|
||||
// failed or that nobody shares is retried, then given up and its album
|
||||
// searched again; an imported one seeds, then is removed.
|
||||
go lifecycle.New(pool, logger.With("component", "lifecycle")).Run(ctx)
|
||||
|
||||
// Duplicate resolver (M498, M502): classifies the sweep's groups and, with
|
||||
// the operator's auto-resolve on, merges the copies that are safe to merge.
|
||||
// Started after SetNotifier so its summaries reach admins. See
|
||||
|
||||
@@ -21,6 +21,9 @@ const (
|
||||
BlockFiles = "files"
|
||||
// BlockImport: the import found its files were not the album (#5522).
|
||||
BlockImport = "import"
|
||||
// BlockStalled: given up on because it would not finish (#5523); held
|
||||
// back until the block's expires_at.
|
||||
BlockStalled = "stalled"
|
||||
)
|
||||
|
||||
// Grab origins, matching grabs_origin_check.
|
||||
@@ -92,11 +95,22 @@ func (m *Memory) blocked(ctx context.Context) map[string]string {
|
||||
|
||||
// BlockSentence is what a blocked candidate says.
|
||||
func BlockSentence(reason, detail string) string {
|
||||
if reason == BlockFiles {
|
||||
switch reason {
|
||||
case BlockFiles:
|
||||
if detail != "" {
|
||||
return "Set aside for what its files hold: " + detail
|
||||
}
|
||||
return "Set aside for what its files hold."
|
||||
case BlockImport:
|
||||
if detail != "" {
|
||||
return "Refused on import: " + detail
|
||||
}
|
||||
return "Refused on import: its files were not the album."
|
||||
case BlockStalled:
|
||||
if detail != "" {
|
||||
return "Given up on for now: " + detail
|
||||
}
|
||||
return "Given up on for now: it would not finish."
|
||||
}
|
||||
return "You set this release aside."
|
||||
}
|
||||
|
||||
@@ -12,8 +12,11 @@ package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
@@ -154,6 +157,19 @@ func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) {
|
||||
return downloader.Mapped(d, downloader.PathMap{Local: row.LocalPath, Remote: row.RemotePath}), nil
|
||||
}
|
||||
|
||||
// DownloadClientFor builds the download client a grab went to, by the code
|
||||
// of its acquisition_services row.
|
||||
func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (downloader.Driver, error) {
|
||||
row, err := dbq.New(pool).GetAcquisitionService(ctx, code)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("the download client %q is no longer set up", code)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return DownloadClient(row)
|
||||
}
|
||||
|
||||
// Sources builds every enabled search source.
|
||||
func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) {
|
||||
rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/importer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle"
|
||||
)
|
||||
|
||||
// Downloads and their imports (M502 #5522): the Wanted view's Activity tab,
|
||||
@@ -40,6 +41,24 @@ type downloadView struct {
|
||||
Files []string `json:"files"`
|
||||
Skipped int `json:"skipped"`
|
||||
RehomedFrom string `json:"rehomed_from,omitempty"`
|
||||
// EndDetail says why an ended download ended (#5523).
|
||||
EndDetail string `json:"end_detail"`
|
||||
// Seeding: imported and still sharing; Ratio its share ratio as last
|
||||
// read; SeedReleasedAt when it was removed once it had shared enough.
|
||||
Seeding bool `json:"seeding"`
|
||||
Ratio float32 `json:"ratio"`
|
||||
SeedReleasedAt *string `json:"seed_released_at"`
|
||||
// Condition is what is wrong with it now, if anything.
|
||||
Condition *conditionView `json:"condition"`
|
||||
}
|
||||
|
||||
// conditionView is an open row of the intervention ledger: what was seen,
|
||||
// since when, and how many times it was retried.
|
||||
type conditionView struct {
|
||||
Trigger string `json:"trigger"`
|
||||
Detail string `json:"detail"`
|
||||
Retries int32 `json:"retries"`
|
||||
Since string `json:"since"`
|
||||
}
|
||||
|
||||
func downloadFrom(g dbq.ListGrabsRow) downloadView {
|
||||
@@ -48,6 +67,18 @@ func downloadFrom(g dbq.ListGrabsRow) downloadView {
|
||||
Origin: g.Origin, GrabbedAt: formatTimestamp(g.GrabbedAt), EndReason: g.EndReason,
|
||||
Progress: g.Progress, ImportState: g.ImportState, ImportReason: g.ImportReason,
|
||||
ImportedFiles: g.ImportedFiles, Destination: g.Destination, Files: []string{},
|
||||
EndDetail: g.EndDetail, Seeding: g.Seeding, Ratio: g.Ratio,
|
||||
}
|
||||
if g.SeedReleasedAt.Valid {
|
||||
at := formatTimestamp(g.SeedReleasedAt)
|
||||
v.SeedReleasedAt = &at
|
||||
}
|
||||
if g.Condition != nil {
|
||||
v.Condition = &conditionView{Trigger: *g.Condition, Detail: deref(g.ConditionDetail),
|
||||
Since: formatTimestamp(g.ConditionSince)}
|
||||
if g.ConditionRetries != nil {
|
||||
v.Condition.Retries = *g.ConditionRetries
|
||||
}
|
||||
}
|
||||
if g.WantID.Valid {
|
||||
id := uuidToString(g.WantID)
|
||||
@@ -82,6 +113,14 @@ type downloadsPage struct {
|
||||
Items []downloadView `json:"items"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
// ClientsDown are the download clients not answering now (#5523).
|
||||
ClientsDown []clientDownView `json:"clients_down"`
|
||||
}
|
||||
|
||||
type clientDownView struct {
|
||||
Client string `json:"client"`
|
||||
Detail string `json:"detail"`
|
||||
Since string `json:"since"`
|
||||
}
|
||||
|
||||
// handleListDownloads implements GET /api/admin/downloads?offset=: running
|
||||
@@ -97,13 +136,47 @@ func (h *handlers) handleListDownloads(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrWithLog(w, h.logger, "admin downloads: list", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset}
|
||||
down, err := dbq.New(h.pool).ListClientsDown(r.Context())
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin downloads: clients down", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset,
|
||||
ClientsDown: make([]clientDownView, 0, len(down))}
|
||||
for _, g := range rows {
|
||||
out.Items = append(out.Items, downloadFrom(g))
|
||||
}
|
||||
for _, c := range down {
|
||||
out.ClientsDown = append(out.ClientsDown, clientDownView{Client: c.Client, Detail: c.Detail,
|
||||
Since: formatTimestamp(c.FirstSeen)})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// handleAbandonDownload implements POST /api/admin/downloads/{id}/abandon: a
|
||||
// running download is given up, its release set aside, the download removed
|
||||
// and the album searched again.
|
||||
func (h *handlers) handleAbandonDownload(w http.ResponseWriter, r *http.Request) {
|
||||
admin, ok := requireUser(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
id, ok := requireURLUUID(w, r, "id")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if _, err := h.lifecycle.Abandon(r.Context(), id, admin.ID); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows), errors.Is(err, lifecycle.ErrNotAbandonable):
|
||||
writeErr(w, apierror.Conflict("not_running", "that download is no longer running"))
|
||||
default:
|
||||
writeErrWithLog(w, h.logger, "admin downloads: abandon", apierror.Internal(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleImportAnyway implements POST /api/admin/downloads/{id}/import: a
|
||||
// download the check held for a person (or one that failed) is imported on
|
||||
// the next pass whatever the check says.
|
||||
@@ -160,13 +233,16 @@ func (h *handlers) writeImportErr(w http.ResponseWriter, what string, err error)
|
||||
}
|
||||
|
||||
type releaseBlockView struct {
|
||||
InfoHash string `json:"info_hash"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Detail string `json:"detail"`
|
||||
BlockedAt string `json:"blocked_at"`
|
||||
WantTitle string `json:"want_title"`
|
||||
WantArtist string `json:"want_artist"`
|
||||
InfoHash string `json:"info_hash"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Detail string `json:"detail"`
|
||||
BlockedAt string `json:"blocked_at"`
|
||||
// ExpiresAt is when a release given up on may be taken again; null is
|
||||
// for good (#5523).
|
||||
ExpiresAt *string `json:"expires_at"`
|
||||
WantTitle string `json:"want_title"`
|
||||
WantArtist string `json:"want_artist"`
|
||||
}
|
||||
|
||||
type releaseBlocksPage struct {
|
||||
@@ -193,6 +269,10 @@ func (h *handlers) handleListReleaseBlocks(w http.ResponseWriter, r *http.Reques
|
||||
for _, b := range rows {
|
||||
v := releaseBlockView{InfoHash: b.InfoHash, Name: b.Name, Reason: b.Reason, Detail: b.Detail,
|
||||
BlockedAt: formatTimestamp(b.BlockedAt)}
|
||||
if b.ExpiresAt.Valid {
|
||||
at := formatTimestamp(b.ExpiresAt)
|
||||
v.ExpiresAt = &at
|
||||
}
|
||||
if b.WantTitle != nil {
|
||||
v.WantTitle = *b.WantTitle
|
||||
}
|
||||
|
||||
@@ -37,6 +37,12 @@ type searchSettingsView struct {
|
||||
// is the first of ScanPaths, the library's folders.
|
||||
ImportRoot string `json:"import_root"`
|
||||
ScanPaths []string `json:"scan_paths"`
|
||||
// The download lifecycle (#5523): seed an imported download to this
|
||||
// ratio or for this many days; give up on one nobody shares after this
|
||||
// many hours. 0 is: remove once imported, no time limit, never.
|
||||
SeedRatio float32 `json:"seed_ratio"`
|
||||
SeedDays int32 `json:"seed_days"`
|
||||
StallHours int32 `json:"stall_hours"`
|
||||
}
|
||||
|
||||
func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView {
|
||||
@@ -44,6 +50,7 @@ func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView
|
||||
AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours,
|
||||
AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass,
|
||||
ImportRoot: s.ImportRoot, ScanPaths: roots,
|
||||
SeedRatio: s.SeedRatio, SeedDays: s.SeedDays, StallHours: s.StallHours,
|
||||
}
|
||||
if v.ScanPaths == nil {
|
||||
v.ScanPaths = []string{}
|
||||
@@ -76,6 +83,9 @@ type putSearchSettingsBody struct {
|
||||
SearchIntervalHours int32 `json:"search_interval_hours"`
|
||||
AllowedQualities []string `json:"allowed_qualities"`
|
||||
ImportRoot string `json:"import_root"`
|
||||
SeedRatio float32 `json:"seed_ratio"`
|
||||
SeedDays int32 `json:"seed_days"`
|
||||
StallHours int32 `json:"stall_hours"`
|
||||
}
|
||||
|
||||
// handlePutSearchSettings implements PUT /api/admin/acquisition/settings.
|
||||
@@ -102,9 +112,15 @@ func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Reques
|
||||
writeErr(w, apierror.BadRequest("bad_import_root", "file new music into one of the library's folders"))
|
||||
return
|
||||
}
|
||||
if body.SeedRatio < 0 || body.SeedRatio > 100 || body.SeedDays < 0 || body.SeedDays > 365 ||
|
||||
body.StallHours < 0 || body.StallHours > 720 {
|
||||
writeErr(w, apierror.BadRequest("bad_lifecycle",
|
||||
"seed ratio must be 0 to 100, seed days 0 to 365 and stall hours 0 to 720"))
|
||||
return
|
||||
}
|
||||
s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{
|
||||
AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities,
|
||||
ImportRoot: body.ImportRoot,
|
||||
ImportRoot: body.ImportRoot, SeedRatio: body.SeedRatio, SeedDays: body.SeedDays, StallHours: body.StallHours,
|
||||
})
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err))
|
||||
|
||||
@@ -62,6 +62,7 @@ type wantGrabView struct {
|
||||
Progress float32 `json:"progress"`
|
||||
ImportState string `json:"import_state"`
|
||||
ImportReason string `json:"import_reason"`
|
||||
EndDetail string `json:"end_detail"`
|
||||
ImportedAt *string `json:"imported_at"`
|
||||
ImportedFiles int32 `json:"imported_files"`
|
||||
}
|
||||
@@ -164,7 +165,8 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) {
|
||||
v.Grab = &wantGrabView{ID: uuidToString(row.GrabID), Name: *row.GrabName,
|
||||
GrabbedAt: formatTimestamp(row.GrabbedAt), Live: !row.GrabEndedAt.Valid,
|
||||
Origin: deref(row.GrabOrigin), Quality: deref(row.GrabQuality), EndReason: deref(row.GrabEndReason),
|
||||
ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason)}
|
||||
ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason),
|
||||
EndDetail: deref(row.GrabEndDetail)}
|
||||
if row.GrabProgress != nil {
|
||||
v.Grab.Progress = *row.GrabProgress
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lidarrconfig"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lidarrquarantine"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/mailer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/netsettings"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/notifications"
|
||||
@@ -80,6 +81,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
||||
wanted: wanted.New(pool, cat, logger),
|
||||
searcher: wanted.NewSearcher(pool, cat, logger),
|
||||
importer: importer.New(pool, scanner, cat, nil, scannerRoots(scanner), logger),
|
||||
lifecycle: lifecycle.New(pool, logger),
|
||||
catalogueSearches: newSearchCache(),
|
||||
catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)),
|
||||
leveled: newLeveledRenderer(dataDir, loudSettings, logger),
|
||||
@@ -264,6 +266,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
||||
admin.Post("/downloads/{id}/import", h.handleImportAnyway)
|
||||
admin.Post("/downloads/{id}/retry", h.handleRetryImport)
|
||||
admin.Post("/downloads/{id}/refuse", h.handleRefuseDownload)
|
||||
admin.Post("/downloads/{id}/abandon", h.handleAbandonDownload)
|
||||
admin.Get("/follows", h.handleListFollows)
|
||||
admin.Get("/follows/{mbid}", h.handleGetFollow)
|
||||
admin.Put("/follows/{mbid}", h.handlePutFollow)
|
||||
@@ -437,6 +440,9 @@ type handlers struct {
|
||||
// importer refuses a download at a person's word (#5522). The import
|
||||
// loop itself runs in main; this one only acts on a request.
|
||||
importer *importer.Importer
|
||||
// lifecycle gives up on a download at a person's word (#5523); its loop
|
||||
// runs in main.
|
||||
lifecycle *lifecycle.Service
|
||||
// catalogueSearches reuses recent MusicBrainz answers; nil asks every
|
||||
// time, which is what handler tests get.
|
||||
catalogueSearches *searchCache
|
||||
|
||||
@@ -16,7 +16,7 @@ UPDATE grabs
|
||||
SET import_state = 'importing', import_attempts = import_attempts + 1,
|
||||
import_updated_at = now(), progress = 1
|
||||
WHERE id = $1 AND ended_at IS NULL AND import_state IN ('', 'downloading', 'waiting')
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
// Takes a finished download for import. The update is the lock: a second
|
||||
@@ -52,6 +52,9 @@ func (q *Queries) ClaimGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -60,7 +63,7 @@ const forceGrabImport = `-- name: ForceGrabImport :one
|
||||
UPDATE grabs
|
||||
SET import_state = '', import_forced = true, import_reason = '', import_updated_at = now()
|
||||
WHERE id = $1 AND ended_at IS NULL AND import_state IN ('review', 'failed')
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
// "Import anyway": the next pass imports the download whatever the check said.
|
||||
@@ -95,12 +98,15 @@ func (q *Queries) ForceGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getGrab = `-- name: GetGrab :one
|
||||
SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE id = $1
|
||||
SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at FROM grabs WHERE id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) {
|
||||
@@ -134,6 +140,9 @@ func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) {
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -202,9 +211,16 @@ const listGrabs = `-- name: ListGrabs :many
|
||||
SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced,
|
||||
g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state,
|
||||
g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files,
|
||||
g.destination, w.title AS want_title, w.artist_name AS want_artist
|
||||
g.destination, g.seeding, g.ratio, g.seed_released_at,
|
||||
w.title AS want_title, w.artist_name AS want_artist,
|
||||
di.trigger AS condition, di.detail AS condition_detail, di.remediations AS condition_retries,
|
||||
di.first_seen AS condition_since
|
||||
FROM grabs g
|
||||
LEFT JOIN wants w ON w.id = g.want_id
|
||||
LEFT JOIN download_interventions di
|
||||
ON di.id = (SELECT x.id FROM download_interventions x
|
||||
WHERE x.grab_id = g.id AND x.outcome = 'open'
|
||||
ORDER BY x.last_seen DESC LIMIT 1)
|
||||
ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC,
|
||||
g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id
|
||||
LIMIT $2 OFFSET $1
|
||||
@@ -216,32 +232,40 @@ type ListGrabsParams struct {
|
||||
}
|
||||
|
||||
type ListGrabsRow struct {
|
||||
ID pgtype.UUID
|
||||
WantID pgtype.UUID
|
||||
Name string
|
||||
Indexer string
|
||||
Client string
|
||||
Quality string
|
||||
Origin string
|
||||
Forced bool
|
||||
GrabbedAt pgtype.Timestamptz
|
||||
EndedAt pgtype.Timestamptz
|
||||
EndReason string
|
||||
EndDetail string
|
||||
Progress float32
|
||||
ImportState string
|
||||
ImportReason string
|
||||
ImportDetail []byte
|
||||
ImportUpdatedAt pgtype.Timestamptz
|
||||
ImportedAt pgtype.Timestamptz
|
||||
ImportedFiles int32
|
||||
Destination string
|
||||
WantTitle *string
|
||||
WantArtist *string
|
||||
ID pgtype.UUID
|
||||
WantID pgtype.UUID
|
||||
Name string
|
||||
Indexer string
|
||||
Client string
|
||||
Quality string
|
||||
Origin string
|
||||
Forced bool
|
||||
GrabbedAt pgtype.Timestamptz
|
||||
EndedAt pgtype.Timestamptz
|
||||
EndReason string
|
||||
EndDetail string
|
||||
Progress float32
|
||||
ImportState string
|
||||
ImportReason string
|
||||
ImportDetail []byte
|
||||
ImportUpdatedAt pgtype.Timestamptz
|
||||
ImportedAt pgtype.Timestamptz
|
||||
ImportedFiles int32
|
||||
Destination string
|
||||
Seeding bool
|
||||
Ratio float32
|
||||
SeedReleasedAt pgtype.Timestamptz
|
||||
WantTitle *string
|
||||
WantArtist *string
|
||||
Condition *string
|
||||
ConditionDetail *string
|
||||
ConditionRetries *int32
|
||||
ConditionSince pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// The Wanted view's Activity tab: downloads waiting on a person first, then
|
||||
// the rest still running, then the ended ones newest first.
|
||||
// the rest still running, then the ended ones newest first. Each carries
|
||||
// what is wrong with it now, if anything (#5523).
|
||||
func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGrabsRow, error) {
|
||||
rows, err := q.db.Query(ctx, listGrabs, arg.Skip, arg.MaxRows)
|
||||
if err != nil {
|
||||
@@ -272,8 +296,15 @@ func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGra
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
&i.WantTitle,
|
||||
&i.WantArtist,
|
||||
&i.Condition,
|
||||
&i.ConditionDetail,
|
||||
&i.ConditionRetries,
|
||||
&i.ConditionSince,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -287,7 +318,7 @@ func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGra
|
||||
|
||||
const listLiveGrabs = `-- name: ListLiveGrabs :many
|
||||
|
||||
SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title,
|
||||
SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, g.seeding, g.ratio, g.seed_released_at, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title,
|
||||
w.reason AS want_reason, w.album_id AS want_album_id
|
||||
FROM grabs g
|
||||
LEFT JOIN wants w ON w.id = g.want_id
|
||||
@@ -323,6 +354,9 @@ type ListLiveGrabsRow struct {
|
||||
ImportedAt pgtype.Timestamptz
|
||||
ImportedFiles int32
|
||||
Destination string
|
||||
Seeding bool
|
||||
Ratio float32
|
||||
SeedReleasedAt pgtype.Timestamptz
|
||||
ReleaseGroupMbid *string
|
||||
ArtistMbid *string
|
||||
ArtistName *string
|
||||
@@ -370,6 +404,9 @@ func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error)
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
&i.ReleaseGroupMbid,
|
||||
&i.ArtistMbid,
|
||||
&i.ArtistName,
|
||||
@@ -388,10 +425,11 @@ func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error)
|
||||
}
|
||||
|
||||
const listReleaseBlocksPage = `-- name: ListReleaseBlocksPage :many
|
||||
SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title,
|
||||
SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, b.expires_at, w.title AS want_title,
|
||||
w.artist_name AS want_artist
|
||||
FROM release_blocks b
|
||||
LEFT JOIN wants w ON w.id = b.want_id
|
||||
WHERE b.expires_at IS NULL OR b.expires_at > now()
|
||||
ORDER BY b.blocked_at DESC, b.info_hash
|
||||
LIMIT $2 OFFSET $1
|
||||
`
|
||||
@@ -407,6 +445,7 @@ type ListReleaseBlocksPageRow struct {
|
||||
Reason string
|
||||
Detail string
|
||||
BlockedAt pgtype.Timestamptz
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
WantTitle *string
|
||||
WantArtist *string
|
||||
}
|
||||
@@ -426,6 +465,7 @@ func (q *Queries) ListReleaseBlocksPage(ctx context.Context, arg ListReleaseBloc
|
||||
&i.Reason,
|
||||
&i.Detail,
|
||||
&i.BlockedAt,
|
||||
&i.ExpiresAt,
|
||||
&i.WantTitle,
|
||||
&i.WantArtist,
|
||||
); err != nil {
|
||||
@@ -522,13 +562,16 @@ UPDATE grabs
|
||||
import_updated_at = now(),
|
||||
import_forced = false,
|
||||
imported_at = CASE WHEN $1::text = 'imported' THEN now() ELSE imported_at END,
|
||||
-- The files were copied, so an imported download goes on sharing
|
||||
-- until the lifecycle removes it (#5523).
|
||||
seeding = $1::text = 'imported',
|
||||
imported_files = $4,
|
||||
destination = $5,
|
||||
ended_at = CASE WHEN $6::text <> '' THEN now() ELSE ended_at END,
|
||||
end_reason = $6,
|
||||
end_detail = CASE WHEN $6::text <> '' THEN $2::text ELSE end_detail END
|
||||
WHERE id = $7
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
type RecordImportParams struct {
|
||||
@@ -582,6 +625,9 @@ func (q *Queries) RecordImport(ctx context.Context, arg RecordImportParams) (Gra
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -591,7 +637,7 @@ UPDATE grabs g
|
||||
SET want_id = $1
|
||||
WHERE g.id = $2 AND g.ended_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM grabs o WHERE o.want_id = $1 AND o.ended_at IS NULL)
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
type RehomeGrabParams struct {
|
||||
@@ -632,6 +678,9 @@ func (q *Queries) RehomeGrab(ctx context.Context, arg RehomeGrabParams) (Grab, e
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -640,7 +689,7 @@ const retryGrabImport = `-- name: RetryGrabImport :one
|
||||
UPDATE grabs
|
||||
SET import_state = '', import_reason = '', import_updated_at = now()
|
||||
WHERE id = $1 AND ended_at IS NULL AND import_state = 'failed'
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) {
|
||||
@@ -674,6 +723,9 @@ func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: lifecycle.sql
|
||||
|
||||
package dbq
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const countRemediation = `-- name: CountRemediation :one
|
||||
UPDATE download_interventions
|
||||
SET remediations = remediations + 1,
|
||||
action = $1,
|
||||
last_seen = now(),
|
||||
outcome = CASE WHEN $1::text IN ('given_up', 'removed') THEN 'resolved' ELSE outcome END,
|
||||
settled_at = CASE WHEN $1::text IN ('given_up', 'removed') THEN now() ELSE settled_at END
|
||||
WHERE client = $2 AND info_hash = $3
|
||||
AND trigger = $4 AND outcome = 'open'
|
||||
RETURNING remediations
|
||||
`
|
||||
|
||||
type CountRemediationParams struct {
|
||||
Action string
|
||||
Client string
|
||||
InfoHash string
|
||||
Trigger string
|
||||
}
|
||||
|
||||
// Minstrel acted on an open condition. A retry climbs the ladder; giving up
|
||||
// or removing ends the condition, since what it was about is gone. Written
|
||||
// before the act, so a retry that happened is never uncounted.
|
||||
func (q *Queries) CountRemediation(ctx context.Context, arg CountRemediationParams) (int32, error) {
|
||||
row := q.db.QueryRow(ctx, countRemediation,
|
||||
arg.Action,
|
||||
arg.Client,
|
||||
arg.InfoHash,
|
||||
arg.Trigger,
|
||||
)
|
||||
var remediations int32
|
||||
err := row.Scan(&remediations)
|
||||
return remediations, err
|
||||
}
|
||||
|
||||
const deleteExpiredBlocks = `-- name: DeleteExpiredBlocks :execrows
|
||||
DELETE FROM release_blocks WHERE expires_at IS NOT NULL AND expires_at <= now()
|
||||
`
|
||||
|
||||
// A release given up on is held back for a while; after that the search may
|
||||
// take it again.
|
||||
func (q *Queries) DeleteExpiredBlocks(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, deleteExpiredBlocks)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const giveUpGrab = `-- name: GiveUpGrab :one
|
||||
UPDATE grabs
|
||||
SET ended_at = now(), end_reason = $1, end_detail = $2,
|
||||
import_updated_at = now()
|
||||
WHERE id = $3 AND ended_at IS NULL AND import_state = ANY($4::text[])
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
type GiveUpGrabParams struct {
|
||||
EndReason string
|
||||
EndDetail string
|
||||
ID pgtype.UUID
|
||||
States []string
|
||||
}
|
||||
|
||||
// Ends a running grab, given up on. Only from the states named: the update
|
||||
// is the lock against an import that claimed the download meanwhile.
|
||||
func (q *Queries) GiveUpGrab(ctx context.Context, arg GiveUpGrabParams) (Grab, error) {
|
||||
row := q.db.QueryRow(ctx, giveUpGrab,
|
||||
arg.EndReason,
|
||||
arg.EndDetail,
|
||||
arg.ID,
|
||||
arg.States,
|
||||
)
|
||||
var i Grab
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.WantID,
|
||||
&i.InfoHash,
|
||||
&i.Name,
|
||||
&i.Key,
|
||||
&i.Indexer,
|
||||
&i.Source,
|
||||
&i.Client,
|
||||
&i.SavePath,
|
||||
&i.Quality,
|
||||
&i.Origin,
|
||||
&i.Forced,
|
||||
&i.GrabbedBy,
|
||||
&i.GrabbedAt,
|
||||
&i.EndedAt,
|
||||
&i.EndReason,
|
||||
&i.EndDetail,
|
||||
&i.Progress,
|
||||
&i.ImportState,
|
||||
&i.ImportReason,
|
||||
&i.ImportDetail,
|
||||
&i.ImportForced,
|
||||
&i.ImportAttempts,
|
||||
&i.ImportUpdatedAt,
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const listClientsDown = `-- name: ListClientsDown :many
|
||||
SELECT client, detail, first_seen FROM download_interventions
|
||||
WHERE trigger = 'client_unreachable' AND outcome = 'open'
|
||||
ORDER BY client
|
||||
`
|
||||
|
||||
type ListClientsDownRow struct {
|
||||
Client string
|
||||
Detail string
|
||||
FirstSeen pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// Download clients not answering now, for the Activity tab.
|
||||
func (q *Queries) ListClientsDown(ctx context.Context) ([]ListClientsDownRow, error) {
|
||||
rows, err := q.db.Query(ctx, listClientsDown)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListClientsDownRow
|
||||
for rows.Next() {
|
||||
var i ListClientsDownRow
|
||||
if err := rows.Scan(&i.Client, &i.Detail, &i.FirstSeen); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listLeftovers = `-- name: ListLeftovers :many
|
||||
SELECT id, client, info_hash, grab_id, name, trigger, action, outcome, detail, observations, remediations, first_seen, last_seen, settled_at FROM download_interventions
|
||||
WHERE trigger = 'not_removed' AND outcome = 'open' AND info_hash <> ''
|
||||
ORDER BY first_seen
|
||||
`
|
||||
|
||||
// Downloads given up on that the client would not remove; each pass asks
|
||||
// again.
|
||||
func (q *Queries) ListLeftovers(ctx context.Context) ([]DownloadIntervention, error) {
|
||||
rows, err := q.db.Query(ctx, listLeftovers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []DownloadIntervention
|
||||
for rows.Next() {
|
||||
var i DownloadIntervention
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Client,
|
||||
&i.InfoHash,
|
||||
&i.GrabID,
|
||||
&i.Name,
|
||||
&i.Trigger,
|
||||
&i.Action,
|
||||
&i.Outcome,
|
||||
&i.Detail,
|
||||
&i.Observations,
|
||||
&i.Remediations,
|
||||
&i.FirstSeen,
|
||||
&i.LastSeen,
|
||||
&i.SettledAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listSeedingGrabs = `-- name: ListSeedingGrabs :many
|
||||
SELECT id, client, info_hash, name, imported_at FROM grabs
|
||||
WHERE seeding AND seed_released_at IS NULL
|
||||
ORDER BY imported_at, id
|
||||
`
|
||||
|
||||
type ListSeedingGrabsRow struct {
|
||||
ID pgtype.UUID
|
||||
Client string
|
||||
InfoHash string
|
||||
Name string
|
||||
ImportedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// Imported downloads still sharing, oldest import first.
|
||||
func (q *Queries) ListSeedingGrabs(ctx context.Context) ([]ListSeedingGrabsRow, error) {
|
||||
rows, err := q.db.Query(ctx, listSeedingGrabs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListSeedingGrabsRow
|
||||
for rows.Next() {
|
||||
var i ListSeedingGrabsRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Client,
|
||||
&i.InfoHash,
|
||||
&i.Name,
|
||||
&i.ImportedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const liveGrabOnHash = `-- name: LiveGrabOnHash :one
|
||||
SELECT EXISTS (SELECT 1 FROM grabs WHERE info_hash = $1 AND ended_at IS NULL)::bool
|
||||
`
|
||||
|
||||
func (q *Queries) LiveGrabOnHash(ctx context.Context, infoHash string) (bool, error) {
|
||||
row := q.db.QueryRow(ctx, liveGrabOnHash, infoHash)
|
||||
var column_1 bool
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const raiseIntervention = `-- name: RaiseIntervention :one
|
||||
|
||||
INSERT INTO download_interventions (client, info_hash, grab_id, name, trigger, detail)
|
||||
VALUES ($1, $2, $3, $4,
|
||||
$5, $6)
|
||||
ON CONFLICT (client, info_hash, trigger) WHERE outcome = 'open' DO UPDATE SET
|
||||
observations = download_interventions.observations + 1,
|
||||
last_seen = now(),
|
||||
detail = EXCLUDED.detail,
|
||||
grab_id = COALESCE(EXCLUDED.grab_id, download_interventions.grab_id),
|
||||
name = CASE WHEN EXCLUDED.name <> '' THEN EXCLUDED.name ELSE download_interventions.name END
|
||||
RETURNING id, client, info_hash, grab_id, name, trigger, action, outcome, detail, observations, remediations, first_seen, last_seen, settled_at
|
||||
`
|
||||
|
||||
type RaiseInterventionParams struct {
|
||||
Client string
|
||||
InfoHash string
|
||||
GrabID pgtype.UUID
|
||||
Name string
|
||||
Trigger string
|
||||
Detail string
|
||||
}
|
||||
|
||||
// The download lifecycle (M502 #5523): the intervention ledger, giving a
|
||||
// download up, and seeding after import.
|
||||
// A condition seen, or seen again: a second pass lands on the open row, so
|
||||
// a download stuck for a week is one row that says how long. The name only
|
||||
// improves, and the detail is the latest word.
|
||||
func (q *Queries) RaiseIntervention(ctx context.Context, arg RaiseInterventionParams) (DownloadIntervention, error) {
|
||||
row := q.db.QueryRow(ctx, raiseIntervention,
|
||||
arg.Client,
|
||||
arg.InfoHash,
|
||||
arg.GrabID,
|
||||
arg.Name,
|
||||
arg.Trigger,
|
||||
arg.Detail,
|
||||
)
|
||||
var i DownloadIntervention
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Client,
|
||||
&i.InfoHash,
|
||||
&i.GrabID,
|
||||
&i.Name,
|
||||
&i.Trigger,
|
||||
&i.Action,
|
||||
&i.Outcome,
|
||||
&i.Detail,
|
||||
&i.Observations,
|
||||
&i.Remediations,
|
||||
&i.FirstSeen,
|
||||
&i.LastSeen,
|
||||
&i.SettledAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const releaseSeed = `-- name: ReleaseSeed :exec
|
||||
UPDATE grabs SET seed_released_at = now() WHERE id = $1 AND seed_released_at IS NULL
|
||||
`
|
||||
|
||||
// The seeding download was removed, or found gone.
|
||||
func (q *Queries) ReleaseSeed(ctx context.Context, id pgtype.UUID) error {
|
||||
_, err := q.db.Exec(ctx, releaseSeed, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const setGrabRatio = `-- name: SetGrabRatio :exec
|
||||
UPDATE grabs SET ratio = $1 WHERE id = $2
|
||||
`
|
||||
|
||||
type SetGrabRatioParams struct {
|
||||
Ratio float32
|
||||
ID pgtype.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) SetGrabRatio(ctx context.Context, arg SetGrabRatioParams) error {
|
||||
_, err := q.db.Exec(ctx, setGrabRatio, arg.Ratio, arg.ID)
|
||||
return err
|
||||
}
|
||||
|
||||
const settleEndedGrabs = `-- name: SettleEndedGrabs :execrows
|
||||
UPDATE download_interventions di
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
FROM grabs g
|
||||
WHERE g.id = di.grab_id AND g.ended_at IS NOT NULL
|
||||
AND di.outcome = 'open' AND di.trigger <> 'not_removed'
|
||||
`
|
||||
|
||||
// What was wrong with a download whose grab has ended (imported, refused,
|
||||
// given up) is over, except a download left in the client to remove.
|
||||
func (q *Queries) SettleEndedGrabs(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, settleEndedGrabs)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const settleIntervention = `-- name: SettleIntervention :execrows
|
||||
UPDATE download_interventions
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
WHERE client = $1 AND info_hash = $2
|
||||
AND trigger = $3 AND outcome = 'open'
|
||||
`
|
||||
|
||||
type SettleInterventionParams struct {
|
||||
Client string
|
||||
InfoHash string
|
||||
Trigger string
|
||||
}
|
||||
|
||||
// The condition cleared.
|
||||
func (q *Queries) SettleIntervention(ctx context.Context, arg SettleInterventionParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, settleIntervention, arg.Client, arg.InfoHash, arg.Trigger)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const settleInterventionsFor = `-- name: SettleInterventionsFor :execrows
|
||||
UPDATE download_interventions
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
WHERE client = $1 AND info_hash = $2 AND outcome = 'open'
|
||||
`
|
||||
|
||||
type SettleInterventionsForParams struct {
|
||||
Client string
|
||||
InfoHash string
|
||||
}
|
||||
|
||||
// Everything open about one download is over: it is gone from the client.
|
||||
func (q *Queries) SettleInterventionsFor(ctx context.Context, arg SettleInterventionsForParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, settleInterventionsFor, arg.Client, arg.InfoHash)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
@@ -226,6 +226,9 @@ type AcquisitionSetting struct {
|
||||
LastPass []byte
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
ImportRoot string
|
||||
SeedRatio float32
|
||||
SeedDays int32
|
||||
StallHours int32
|
||||
}
|
||||
|
||||
type Album struct {
|
||||
@@ -355,6 +358,23 @@ type DiscoverTuning struct {
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type DownloadIntervention struct {
|
||||
ID pgtype.UUID
|
||||
Client string
|
||||
InfoHash string
|
||||
GrabID pgtype.UUID
|
||||
Name string
|
||||
Trigger string
|
||||
Action string
|
||||
Outcome string
|
||||
Detail string
|
||||
Observations int32
|
||||
Remediations int32
|
||||
FirstSeen pgtype.Timestamptz
|
||||
LastSeen pgtype.Timestamptz
|
||||
SettledAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type DuplicateGroup struct {
|
||||
ID pgtype.UUID
|
||||
MemberKey string
|
||||
@@ -459,6 +479,9 @@ type Grab struct {
|
||||
ImportedAt pgtype.Timestamptz
|
||||
ImportedFiles int32
|
||||
Destination string
|
||||
Seeding bool
|
||||
Ratio float32
|
||||
SeedReleasedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type LibraryChange struct {
|
||||
@@ -701,6 +724,7 @@ type ReleaseBlock struct {
|
||||
WantID pgtype.UUID
|
||||
BlockedBy pgtype.UUID
|
||||
BlockedAt pgtype.Timestamptz
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ReleaseTypeProfile struct {
|
||||
|
||||
@@ -12,11 +12,13 @@ import (
|
||||
)
|
||||
|
||||
const blockRelease = `-- name: BlockRelease :exec
|
||||
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by)
|
||||
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by, expires_at)
|
||||
VALUES ($1, $2, $3, $4,
|
||||
$5, $6)
|
||||
$5, $6, $7)
|
||||
ON CONFLICT (info_hash) DO UPDATE
|
||||
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now()
|
||||
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(),
|
||||
expires_at = EXCLUDED.expires_at, blocked_by = EXCLUDED.blocked_by
|
||||
WHERE release_blocks.expires_at IS NOT NULL OR EXCLUDED.expires_at IS NULL
|
||||
`
|
||||
|
||||
type BlockReleaseParams struct {
|
||||
@@ -26,8 +28,12 @@ type BlockReleaseParams struct {
|
||||
Detail string
|
||||
WantID pgtype.UUID
|
||||
BlockedBy pgtype.UUID
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
// A release set aside. expires_at NULL is for good. A block for good is never
|
||||
// shortened by a timed one: only a timed block, or another for good, is
|
||||
// replaced.
|
||||
func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) error {
|
||||
_, err := q.db.Exec(ctx, blockRelease,
|
||||
arg.InfoHash,
|
||||
@@ -36,6 +42,7 @@ func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) erro
|
||||
arg.Detail,
|
||||
arg.WantID,
|
||||
arg.BlockedBy,
|
||||
arg.ExpiresAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -89,7 +96,7 @@ func (q *Queries) EndGrab(ctx context.Context, arg EndGrabParams) error {
|
||||
|
||||
const getAcquisitionSettings = `-- name: GetAcquisitionSettings :one
|
||||
|
||||
SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root FROM acquisition_settings WHERE id = true
|
||||
SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root, seed_ratio, seed_days, stall_hours FROM acquisition_settings WHERE id = true
|
||||
`
|
||||
|
||||
// Searching for wants and grabbing releases (M502 #5521).
|
||||
@@ -105,12 +112,15 @@ func (q *Queries) GetAcquisitionSettings(ctx context.Context) (AcquisitionSettin
|
||||
&i.LastPass,
|
||||
&i.UpdatedAt,
|
||||
&i.ImportRoot,
|
||||
&i.SeedRatio,
|
||||
&i.SeedDays,
|
||||
&i.StallHours,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getLiveGrab = `-- name: GetLiveGrab :one
|
||||
SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE want_id = $1 AND ended_at IS NULL
|
||||
SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at FROM grabs WHERE want_id = $1 AND ended_at IS NULL
|
||||
`
|
||||
|
||||
func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, error) {
|
||||
@@ -144,6 +154,9 @@ func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, er
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -230,7 +243,7 @@ INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_
|
||||
VALUES ($1, $2, $3, $4, $5,
|
||||
$6, $7, $8, $9,
|
||||
$10, $11, $12)
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination
|
||||
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at
|
||||
`
|
||||
|
||||
type InsertGrabParams struct {
|
||||
@@ -292,6 +305,9 @@ func (q *Queries) InsertGrab(ctx context.Context, arg InsertGrabParams) (Grab, e
|
||||
&i.ImportedAt,
|
||||
&i.ImportedFiles,
|
||||
&i.Destination,
|
||||
&i.Seeding,
|
||||
&i.Ratio,
|
||||
&i.SeedReleasedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -350,9 +366,13 @@ func (q *Queries) InsertWantCandidate(ctx context.Context, arg InsertWantCandida
|
||||
}
|
||||
|
||||
const listReleaseBlocks = `-- name: ListReleaseBlocks :many
|
||||
SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at FROM release_blocks ORDER BY blocked_at DESC
|
||||
SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at, expires_at FROM release_blocks
|
||||
WHERE expires_at IS NULL OR expires_at > now()
|
||||
ORDER BY blocked_at DESC
|
||||
`
|
||||
|
||||
// Every release set aside and still held back: a stalled one lifts by itself
|
||||
// at expires_at (#5523).
|
||||
func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error) {
|
||||
rows, err := q.db.Query(ctx, listReleaseBlocks)
|
||||
if err != nil {
|
||||
@@ -370,6 +390,7 @@ func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error)
|
||||
&i.WantID,
|
||||
&i.BlockedBy,
|
||||
&i.BlockedAt,
|
||||
&i.ExpiresAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -582,9 +603,12 @@ UPDATE acquisition_settings
|
||||
search_interval_hours = $2,
|
||||
allowed_qualities = $3,
|
||||
import_root = $4,
|
||||
seed_ratio = $5,
|
||||
seed_days = $6,
|
||||
stall_hours = $7,
|
||||
updated_at = now()
|
||||
WHERE id = true
|
||||
RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root
|
||||
RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root, seed_ratio, seed_days, stall_hours
|
||||
`
|
||||
|
||||
type UpdateAcquisitionSettingsParams struct {
|
||||
@@ -592,6 +616,9 @@ type UpdateAcquisitionSettingsParams struct {
|
||||
SearchIntervalHours int32
|
||||
AllowedQualities []string
|
||||
ImportRoot string
|
||||
SeedRatio float32
|
||||
SeedDays int32
|
||||
StallHours int32
|
||||
}
|
||||
|
||||
func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcquisitionSettingsParams) (AcquisitionSetting, error) {
|
||||
@@ -600,6 +627,9 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui
|
||||
arg.SearchIntervalHours,
|
||||
arg.AllowedQualities,
|
||||
arg.ImportRoot,
|
||||
arg.SeedRatio,
|
||||
arg.SeedDays,
|
||||
arg.StallHours,
|
||||
)
|
||||
var i AcquisitionSetting
|
||||
err := row.Scan(
|
||||
@@ -611,6 +641,9 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui
|
||||
&i.LastPass,
|
||||
&i.UpdatedAt,
|
||||
&i.ImportRoot,
|
||||
&i.SeedRatio,
|
||||
&i.SeedDays,
|
||||
&i.StallHours,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
@@ -30,7 +30,8 @@ SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::big
|
||||
(SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity,
|
||||
(SELECT COUNT(*) FROM grabs
|
||||
WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention,
|
||||
(SELECT COUNT(*) FROM release_blocks)::bigint AS blocked
|
||||
(SELECT COUNT(*) FROM release_blocks b
|
||||
WHERE b.expires_at IS NULL OR b.expires_at > now())::bigint AS blocked
|
||||
FROM wants
|
||||
`
|
||||
|
||||
@@ -537,7 +538,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas
|
||||
gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin,
|
||||
gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason,
|
||||
gr.progress AS grab_progress, gr.import_state AS grab_import_state,
|
||||
gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at,
|
||||
gr.import_reason AS grab_import_reason, gr.end_detail AS grab_end_detail, gr.imported_at AS grab_imported_at,
|
||||
gr.imported_files AS grab_imported_files
|
||||
FROM wants w
|
||||
LEFT JOIN users u ON u.id = w.created_by
|
||||
@@ -590,6 +591,7 @@ type ListWantsRow struct {
|
||||
GrabProgress *float32
|
||||
GrabImportState *string
|
||||
GrabImportReason *string
|
||||
GrabEndDetail *string
|
||||
GrabImportedAt pgtype.Timestamptz
|
||||
GrabImportedFiles *int32
|
||||
}
|
||||
@@ -636,6 +638,7 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan
|
||||
&i.GrabProgress,
|
||||
&i.GrabImportState,
|
||||
&i.GrabImportReason,
|
||||
&i.GrabEndDetail,
|
||||
&i.GrabImportedAt,
|
||||
&i.GrabImportedFiles,
|
||||
); err != nil {
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
ALTER TABLE acquisition_settings
|
||||
DROP COLUMN seed_ratio,
|
||||
DROP COLUMN seed_days,
|
||||
DROP COLUMN stall_hours;
|
||||
|
||||
ALTER TABLE release_blocks DROP COLUMN expires_at;
|
||||
DELETE FROM release_blocks WHERE reason = 'stalled';
|
||||
ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check;
|
||||
ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check
|
||||
CHECK (reason IN ('operator', 'files', 'import'));
|
||||
|
||||
DROP INDEX grabs_seeding_idx;
|
||||
ALTER TABLE grabs
|
||||
DROP COLUMN seeding,
|
||||
DROP COLUMN ratio,
|
||||
DROP COLUMN seed_released_at;
|
||||
|
||||
UPDATE grabs SET end_reason = 'superseded' WHERE end_reason IN ('stalled', 'failed', 'abandoned', 'gone');
|
||||
ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check;
|
||||
ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check
|
||||
CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held'));
|
||||
|
||||
DROP TABLE download_interventions;
|
||||
@@ -0,0 +1,90 @@
|
||||
-- Download lifecycle (M502 #5523), ported from Fabled Librarian's System 104:
|
||||
-- a download that stalls or fails is given up and its album searched again;
|
||||
-- an imported download seeds, then is removed.
|
||||
--
|
||||
-- download_interventions what was seen wrong with a download or a client,
|
||||
-- what Minstrel did about it, and whether it
|
||||
-- cleared. One open row per (client, download,
|
||||
-- condition), updated in place.
|
||||
-- grabs (columns) a grab also ends by being given up (stalled,
|
||||
-- failed), set aside by a person, or leaving the
|
||||
-- client; an imported one records its seeding.
|
||||
-- release_blocks a release given up on is held back for a while,
|
||||
-- not for ever: expires_at.
|
||||
-- acquisition_settings how long to seed, and how long a download may sit
|
||||
-- with nobody sharing it.
|
||||
CREATE TABLE download_interventions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
-- The acquisition_services code of the download client.
|
||||
client text NOT NULL,
|
||||
-- The torrent, or '' for a condition of the client itself.
|
||||
info_hash text NOT NULL DEFAULT '',
|
||||
grab_id uuid REFERENCES grabs (id) ON DELETE SET NULL,
|
||||
name text NOT NULL DEFAULT '',
|
||||
-- no_peers nobody is sharing an unfinished download.
|
||||
-- no_metadata still fetching the torrent's file list.
|
||||
-- client_error the client gave up on it, in its own words.
|
||||
-- client_unreachable the client does not answer, or its engine is down.
|
||||
-- gone the client no longer lists a running download.
|
||||
-- not_removed given up on, and the client would not remove it.
|
||||
trigger text NOT NULL CONSTRAINT download_interventions_trigger_check CHECK (trigger IN (
|
||||
'no_peers', 'no_metadata', 'client_error', 'client_unreachable', 'gone', 'not_removed')),
|
||||
-- escalated seen and shown; nothing done yet.
|
||||
-- retried asked the trackers again and re-read the files.
|
||||
-- given_up the download was removed and its album searched again.
|
||||
-- removed a download left behind was removed.
|
||||
action text NOT NULL DEFAULT 'escalated'
|
||||
CONSTRAINT download_interventions_action_check CHECK (action IN ('escalated', 'retried', 'given_up', 'removed')),
|
||||
outcome text NOT NULL DEFAULT 'open'
|
||||
CONSTRAINT download_interventions_outcome_check CHECK (outcome IN ('open', 'resolved')),
|
||||
detail text NOT NULL DEFAULT '',
|
||||
-- How many passes have seen it, and how many times Minstrel acted on it.
|
||||
-- The second is the retry ladder's rung, kept here so a restart does not
|
||||
-- hand a stuck download fresh retries.
|
||||
observations integer NOT NULL DEFAULT 1,
|
||||
remediations integer NOT NULL DEFAULT 0,
|
||||
first_seen timestamptz NOT NULL DEFAULT now(),
|
||||
last_seen timestamptz NOT NULL DEFAULT now(),
|
||||
settled_at timestamptz,
|
||||
CONSTRAINT download_interventions_settled_together CHECK ((outcome = 'open') = (settled_at IS NULL))
|
||||
);
|
||||
|
||||
-- One open row per condition: a pass that sees it again lands on the same row.
|
||||
CREATE UNIQUE INDEX download_interventions_one_open
|
||||
ON download_interventions (client, info_hash, trigger) WHERE outcome = 'open';
|
||||
CREATE INDEX download_interventions_grab_idx ON download_interventions (grab_id) WHERE grab_id IS NOT NULL;
|
||||
|
||||
ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check;
|
||||
ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check
|
||||
CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held', 'stalled', 'failed', 'abandoned', 'gone'));
|
||||
|
||||
-- An imported download goes on sharing (the files were copied) until its
|
||||
-- ratio reaches the setting or its time is up; seed_released_at is when it
|
||||
-- was removed, or found gone.
|
||||
ALTER TABLE grabs
|
||||
ADD COLUMN seeding boolean NOT NULL DEFAULT false,
|
||||
ADD COLUMN ratio real NOT NULL DEFAULT 0,
|
||||
ADD COLUMN seed_released_at timestamptz;
|
||||
|
||||
CREATE INDEX grabs_seeding_idx ON grabs (imported_at) WHERE seeding AND seed_released_at IS NULL;
|
||||
|
||||
-- stalled: given up on because it would not finish; held back until
|
||||
-- expires_at. NULL is for good.
|
||||
ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check;
|
||||
ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check
|
||||
CHECK (reason IN ('operator', 'files', 'import', 'stalled'));
|
||||
ALTER TABLE release_blocks ADD COLUMN expires_at timestamptz;
|
||||
|
||||
-- FL's rulings: seed to ratio 2.0 or for 14 days at most; a download nobody
|
||||
-- has shared for 6 hours is retried, then given up.
|
||||
-- seed_ratio 0 the download is removed once imported.
|
||||
-- seed_days 0 no time limit: only the ratio ends it.
|
||||
-- stall_hours 0 never give up on a download for want of peers (a failed
|
||||
-- one is still given up).
|
||||
ALTER TABLE acquisition_settings
|
||||
ADD COLUMN seed_ratio real NOT NULL DEFAULT 2.0
|
||||
CONSTRAINT acquisition_settings_seed_ratio_check CHECK (seed_ratio BETWEEN 0 AND 100),
|
||||
ADD COLUMN seed_days integer NOT NULL DEFAULT 14
|
||||
CONSTRAINT acquisition_settings_seed_days_check CHECK (seed_days BETWEEN 0 AND 365),
|
||||
ADD COLUMN stall_hours integer NOT NULL DEFAULT 6
|
||||
CONSTRAINT acquisition_settings_stall_hours_check CHECK (stall_hours BETWEEN 0 AND 720);
|
||||
@@ -51,6 +51,9 @@ UPDATE grabs
|
||||
import_updated_at = now(),
|
||||
import_forced = false,
|
||||
imported_at = CASE WHEN sqlc.arg(import_state)::text = 'imported' THEN now() ELSE imported_at END,
|
||||
-- The files were copied, so an imported download goes on sharing
|
||||
-- until the lifecycle removes it (#5523).
|
||||
seeding = sqlc.arg(import_state)::text = 'imported',
|
||||
imported_files = sqlc.arg(imported_files),
|
||||
destination = sqlc.arg(destination),
|
||||
ended_at = CASE WHEN sqlc.arg(end_reason)::text <> '' THEN now() ELSE ended_at END,
|
||||
@@ -110,21 +113,30 @@ SELECT DISTINCT COALESCE(t.disc_number, 1)::int AS disc, t.track_number::int AS
|
||||
|
||||
-- name: ListGrabs :many
|
||||
-- The Wanted view's Activity tab: downloads waiting on a person first, then
|
||||
-- the rest still running, then the ended ones newest first.
|
||||
-- the rest still running, then the ended ones newest first. Each carries
|
||||
-- what is wrong with it now, if anything (#5523).
|
||||
SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced,
|
||||
g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state,
|
||||
g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files,
|
||||
g.destination, w.title AS want_title, w.artist_name AS want_artist
|
||||
g.destination, g.seeding, g.ratio, g.seed_released_at,
|
||||
w.title AS want_title, w.artist_name AS want_artist,
|
||||
di.trigger AS condition, di.detail AS condition_detail, di.remediations AS condition_retries,
|
||||
di.first_seen AS condition_since
|
||||
FROM grabs g
|
||||
LEFT JOIN wants w ON w.id = g.want_id
|
||||
LEFT JOIN download_interventions di
|
||||
ON di.id = (SELECT x.id FROM download_interventions x
|
||||
WHERE x.grab_id = g.id AND x.outcome = 'open'
|
||||
ORDER BY x.last_seen DESC LIMIT 1)
|
||||
ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC,
|
||||
g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id
|
||||
LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip);
|
||||
|
||||
-- name: ListReleaseBlocksPage :many
|
||||
SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title,
|
||||
SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, b.expires_at, w.title AS want_title,
|
||||
w.artist_name AS want_artist
|
||||
FROM release_blocks b
|
||||
LEFT JOIN wants w ON w.id = b.want_id
|
||||
WHERE b.expires_at IS NULL OR b.expires_at > now()
|
||||
ORDER BY b.blocked_at DESC, b.info_hash
|
||||
LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip);
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
-- The download lifecycle (M502 #5523): the intervention ledger, giving a
|
||||
-- download up, and seeding after import.
|
||||
|
||||
-- name: RaiseIntervention :one
|
||||
-- A condition seen, or seen again: a second pass lands on the open row, so
|
||||
-- a download stuck for a week is one row that says how long. The name only
|
||||
-- improves, and the detail is the latest word.
|
||||
INSERT INTO download_interventions (client, info_hash, grab_id, name, trigger, detail)
|
||||
VALUES (sqlc.arg(client), sqlc.arg(info_hash), sqlc.narg(grab_id), sqlc.arg(name),
|
||||
sqlc.arg(trigger), sqlc.arg(detail))
|
||||
ON CONFLICT (client, info_hash, trigger) WHERE outcome = 'open' DO UPDATE SET
|
||||
observations = download_interventions.observations + 1,
|
||||
last_seen = now(),
|
||||
detail = EXCLUDED.detail,
|
||||
grab_id = COALESCE(EXCLUDED.grab_id, download_interventions.grab_id),
|
||||
name = CASE WHEN EXCLUDED.name <> '' THEN EXCLUDED.name ELSE download_interventions.name END
|
||||
RETURNING *;
|
||||
|
||||
-- name: SettleIntervention :execrows
|
||||
-- The condition cleared.
|
||||
UPDATE download_interventions
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash)
|
||||
AND trigger = sqlc.arg(trigger) AND outcome = 'open';
|
||||
|
||||
-- name: SettleInterventionsFor :execrows
|
||||
-- Everything open about one download is over: it is gone from the client.
|
||||
UPDATE download_interventions
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash) AND outcome = 'open';
|
||||
|
||||
-- name: SettleEndedGrabs :execrows
|
||||
-- What was wrong with a download whose grab has ended (imported, refused,
|
||||
-- given up) is over, except a download left in the client to remove.
|
||||
UPDATE download_interventions di
|
||||
SET outcome = 'resolved', settled_at = now(), last_seen = now()
|
||||
FROM grabs g
|
||||
WHERE g.id = di.grab_id AND g.ended_at IS NOT NULL
|
||||
AND di.outcome = 'open' AND di.trigger <> 'not_removed';
|
||||
|
||||
-- name: CountRemediation :one
|
||||
-- Minstrel acted on an open condition. A retry climbs the ladder; giving up
|
||||
-- or removing ends the condition, since what it was about is gone. Written
|
||||
-- before the act, so a retry that happened is never uncounted.
|
||||
UPDATE download_interventions
|
||||
SET remediations = remediations + 1,
|
||||
action = sqlc.arg(action),
|
||||
last_seen = now(),
|
||||
outcome = CASE WHEN sqlc.arg(action)::text IN ('given_up', 'removed') THEN 'resolved' ELSE outcome END,
|
||||
settled_at = CASE WHEN sqlc.arg(action)::text IN ('given_up', 'removed') THEN now() ELSE settled_at END
|
||||
WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash)
|
||||
AND trigger = sqlc.arg(trigger) AND outcome = 'open'
|
||||
RETURNING remediations;
|
||||
|
||||
-- name: ListLeftovers :many
|
||||
-- Downloads given up on that the client would not remove; each pass asks
|
||||
-- again.
|
||||
SELECT * FROM download_interventions
|
||||
WHERE trigger = 'not_removed' AND outcome = 'open' AND info_hash <> ''
|
||||
ORDER BY first_seen;
|
||||
|
||||
-- name: ListClientsDown :many
|
||||
-- Download clients not answering now, for the Activity tab.
|
||||
SELECT client, detail, first_seen FROM download_interventions
|
||||
WHERE trigger = 'client_unreachable' AND outcome = 'open'
|
||||
ORDER BY client;
|
||||
|
||||
-- name: LiveGrabOnHash :one
|
||||
SELECT EXISTS (SELECT 1 FROM grabs WHERE info_hash = sqlc.arg(info_hash) AND ended_at IS NULL)::bool;
|
||||
|
||||
-- name: GiveUpGrab :one
|
||||
-- Ends a running grab, given up on. Only from the states named: the update
|
||||
-- is the lock against an import that claimed the download meanwhile.
|
||||
UPDATE grabs
|
||||
SET ended_at = now(), end_reason = sqlc.arg(end_reason), end_detail = sqlc.arg(end_detail),
|
||||
import_updated_at = now()
|
||||
WHERE id = sqlc.arg(id) AND ended_at IS NULL AND import_state = ANY(sqlc.arg(states)::text[])
|
||||
RETURNING *;
|
||||
|
||||
-- name: ListSeedingGrabs :many
|
||||
-- Imported downloads still sharing, oldest import first.
|
||||
SELECT id, client, info_hash, name, imported_at FROM grabs
|
||||
WHERE seeding AND seed_released_at IS NULL
|
||||
ORDER BY imported_at, id;
|
||||
|
||||
-- name: SetGrabRatio :exec
|
||||
UPDATE grabs SET ratio = sqlc.arg(ratio) WHERE id = sqlc.arg(id);
|
||||
|
||||
-- name: ReleaseSeed :exec
|
||||
-- The seeding download was removed, or found gone.
|
||||
UPDATE grabs SET seed_released_at = now() WHERE id = $1 AND seed_released_at IS NULL;
|
||||
|
||||
-- name: DeleteExpiredBlocks :execrows
|
||||
-- A release given up on is held back for a while; after that the search may
|
||||
-- take it again.
|
||||
DELETE FROM release_blocks WHERE expires_at IS NOT NULL AND expires_at <= now();
|
||||
@@ -9,6 +9,9 @@ UPDATE acquisition_settings
|
||||
search_interval_hours = sqlc.arg(search_interval_hours),
|
||||
allowed_qualities = sqlc.arg(allowed_qualities),
|
||||
import_root = sqlc.arg(import_root),
|
||||
seed_ratio = sqlc.arg(seed_ratio),
|
||||
seed_days = sqlc.arg(seed_days),
|
||||
stall_hours = sqlc.arg(stall_hours),
|
||||
updated_at = now()
|
||||
WHERE id = true
|
||||
RETURNING *;
|
||||
@@ -109,14 +112,23 @@ UPDATE grabs
|
||||
WHERE id = sqlc.arg(id) AND ended_at IS NULL;
|
||||
|
||||
-- name: ListReleaseBlocks :many
|
||||
SELECT * FROM release_blocks ORDER BY blocked_at DESC;
|
||||
-- Every release set aside and still held back: a stalled one lifts by itself
|
||||
-- at expires_at (#5523).
|
||||
SELECT * FROM release_blocks
|
||||
WHERE expires_at IS NULL OR expires_at > now()
|
||||
ORDER BY blocked_at DESC;
|
||||
|
||||
-- name: BlockRelease :exec
|
||||
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by)
|
||||
-- A release set aside. expires_at NULL is for good. A block for good is never
|
||||
-- shortened by a timed one: only a timed block, or another for good, is
|
||||
-- replaced.
|
||||
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by, expires_at)
|
||||
VALUES (sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(reason), sqlc.arg(detail),
|
||||
sqlc.narg(want_id), sqlc.narg(blocked_by))
|
||||
sqlc.narg(want_id), sqlc.narg(blocked_by), sqlc.narg(expires_at))
|
||||
ON CONFLICT (info_hash) DO UPDATE
|
||||
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now();
|
||||
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(),
|
||||
expires_at = EXCLUDED.expires_at, blocked_by = EXCLUDED.blocked_by
|
||||
WHERE release_blocks.expires_at IS NOT NULL OR EXCLUDED.expires_at IS NULL;
|
||||
|
||||
-- name: UnblockRelease :execrows
|
||||
DELETE FROM release_blocks WHERE info_hash = $1;
|
||||
|
||||
@@ -194,7 +194,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas
|
||||
gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin,
|
||||
gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason,
|
||||
gr.progress AS grab_progress, gr.import_state AS grab_import_state,
|
||||
gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at,
|
||||
gr.import_reason AS grab_import_reason, gr.end_detail AS grab_end_detail, gr.imported_at AS grab_imported_at,
|
||||
gr.imported_files AS grab_imported_files
|
||||
FROM wants w
|
||||
LEFT JOIN users u ON u.id = w.created_by
|
||||
@@ -219,5 +219,6 @@ SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::big
|
||||
(SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity,
|
||||
(SELECT COUNT(*) FROM grabs
|
||||
WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention,
|
||||
(SELECT COUNT(*) FROM release_blocks)::bigint AS blocked
|
||||
(SELECT COUNT(*) FROM release_blocks b
|
||||
WHERE b.expires_at IS NULL OR b.expires_at > now())::bigint AS blocked
|
||||
FROM wants;
|
||||
|
||||
@@ -109,6 +109,7 @@ var dataTables = []string{
|
||||
"mb_release_groups", // M502 #5519
|
||||
"mb_artists", // M502 #5519
|
||||
"requests", // M502 #5520 (was lidarr_requests)
|
||||
"download_interventions", // M502 #5523
|
||||
"release_blocks", // M502 #5521
|
||||
"grabs", // M502 #5521
|
||||
"want_candidates", // M502 #5521
|
||||
@@ -187,6 +188,7 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) {
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE acquisition_settings
|
||||
SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, import_root = DEFAULT,
|
||||
seed_ratio = DEFAULT, seed_days = DEFAULT, stall_hours = DEFAULT,
|
||||
last_pass_at = NULL, last_pass = DEFAULT, updated_at = DEFAULT`,
|
||||
); err != nil {
|
||||
t.Fatalf("dbtest.ResetDB reset acquisition settings: %v", err)
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
@@ -67,8 +66,8 @@ type Importer struct {
|
||||
// Roots are the library's folders (the scan paths). New music is filed
|
||||
// under the one the settings name, or the first.
|
||||
Roots []string
|
||||
// ClientFor builds the download client a grab went to. Defaults to the
|
||||
// acquisition_services row of that code.
|
||||
// ClientFor builds the download client a grab went to. Defaults to
|
||||
// acquire.DownloadClientFor.
|
||||
ClientFor func(ctx context.Context, code string) (downloader.Driver, error)
|
||||
logger *slog.Logger
|
||||
wake chan struct{}
|
||||
@@ -83,19 +82,10 @@ func New(pool *pgxpool.Pool, scanner Scanner, cat Catalogue, printer Printer, ro
|
||||
}
|
||||
im := &Importer{pool: pool, Scanner: scanner, Catalogue: cat, Printer: printer, Roots: roots,
|
||||
logger: logger, wake: make(chan struct{}, 1)}
|
||||
im.ClientFor = im.clientFor
|
||||
return im
|
||||
}
|
||||
|
||||
func (im *Importer) clientFor(ctx context.Context, code string) (downloader.Driver, error) {
|
||||
row, err := dbq.New(im.pool).GetAcquisitionService(ctx, code)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("the download client %q is no longer set up", code)
|
||||
}
|
||||
return nil, err
|
||||
im.ClientFor = func(ctx context.Context, code string) (downloader.Driver, error) {
|
||||
return acquire.DownloadClientFor(ctx, pool, code)
|
||||
}
|
||||
return acquire.DownloadClient(row)
|
||||
return im
|
||||
}
|
||||
|
||||
// Wake asks for a pass now: after a grab, or a person's decision.
|
||||
@@ -201,7 +191,7 @@ func (im *Importer) passClient(ctx context.Context, q *dbq.Queries, code string,
|
||||
}
|
||||
t, ok := byHash[indexer.NormalizeHash(g.InfoHash)]
|
||||
if !ok {
|
||||
continue // gone from the client: the download lifecycle's (#5523)
|
||||
continue // gone from the client: the lifecycle settles it (#5523)
|
||||
}
|
||||
if !g.WantID.Valid {
|
||||
wait(g, "Its album is no longer wanted.")
|
||||
|
||||
@@ -0,0 +1,659 @@
|
||||
// Package lifecycle looks after a download from the grab to its removal
|
||||
// (M502 #5523), ported from Fabled Librarian's internal/health and its seed
|
||||
// release (System 104).
|
||||
//
|
||||
// - A download the client has failed is given up at once.
|
||||
// - One nobody has shared, or still fetching its file list, after the
|
||||
// stall setting is retried: the trackers asked again and the files
|
||||
// re-read, three times a pass apart. Then it is given up.
|
||||
// - Giving up removes the download with its files, holds the release back
|
||||
// for a week, ends the grab and searches for the album again. There is
|
||||
// no limit per album: what stops it churning is the hold on each release.
|
||||
// - A running download the client no longer lists, for half an hour, ends
|
||||
// and its album is searched again. The release is not held back: nothing
|
||||
// was wrong with it.
|
||||
// - An imported download seeds (its files were copied) until its share
|
||||
// ratio reaches the setting or its days are up, then is removed.
|
||||
//
|
||||
// Every condition and every act goes in the intervention ledger, one open
|
||||
// row per (client, download, condition), so the Activity tab can say what
|
||||
// is wrong with a download and what was done.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/downloader"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
)
|
||||
|
||||
// Conditions, matching download_interventions_trigger_check.
|
||||
const (
|
||||
NoPeers = "no_peers"
|
||||
NoMetadata = "no_metadata"
|
||||
ClientError = "client_error"
|
||||
ClientUnreachable = "client_unreachable"
|
||||
Gone = "gone"
|
||||
NotRemoved = "not_removed"
|
||||
)
|
||||
|
||||
// Acts, matching download_interventions_action_check.
|
||||
const (
|
||||
actRetried = "retried"
|
||||
actGivenUp = "given_up"
|
||||
actRemoved = "removed"
|
||||
)
|
||||
|
||||
// End reasons a grab is given with, matching grabs_end_reason_check.
|
||||
const (
|
||||
EndStalled = "stalled"
|
||||
EndFailed = "failed"
|
||||
EndAbandoned = "abandoned"
|
||||
EndGone = "gone"
|
||||
)
|
||||
|
||||
const (
|
||||
// interval: three retries a pass apart is half an hour of asking before
|
||||
// a stalled download is given up.
|
||||
interval = 10 * time.Minute
|
||||
passTimeout = 5 * time.Minute
|
||||
// Retries is how many times one stuck download is nudged first.
|
||||
Retries = 3
|
||||
// StallBlock is how long a release given up on is held back.
|
||||
StallBlock = 7 * 24 * time.Hour
|
||||
// goneAfter: a client restarting can answer without a download for a
|
||||
// while; half an hour missing is gone.
|
||||
goneAfter = 30 * time.Minute
|
||||
)
|
||||
|
||||
// running are the import states of a grab still downloading: the lifecycle
|
||||
// acts only on these, so an import under way is never pulled from under.
|
||||
var running = []string{"", "downloading", "waiting"}
|
||||
|
||||
// Settings are the limits, from acquisition_settings.
|
||||
type Settings struct {
|
||||
// SeedRatio is the share ratio an imported download seeds to; 0 removes
|
||||
// it once imported.
|
||||
SeedRatio float64
|
||||
// SeedTime is the longest it seeds; 0 is no limit.
|
||||
SeedTime time.Duration
|
||||
// Patience is how long a download may sit with nobody sharing it before
|
||||
// it is retried, then given up; 0 never gives up for that.
|
||||
Patience time.Duration
|
||||
}
|
||||
|
||||
// Defaults are FL's rulings.
|
||||
var Defaults = Settings{SeedRatio: 2, SeedTime: 14 * 24 * time.Hour, Patience: 6 * time.Hour}
|
||||
|
||||
// Service runs the lifecycle passes.
|
||||
type Service struct {
|
||||
pool *pgxpool.Pool
|
||||
// ClientFor builds a download client by its code. Defaults to
|
||||
// acquire.DownloadClientFor.
|
||||
ClientFor func(ctx context.Context, code string) (downloader.Driver, error)
|
||||
// Now is the clock; nil is time.Now.
|
||||
Now func() time.Time
|
||||
logger *slog.Logger
|
||||
mu sync.Mutex // one pass at a time
|
||||
}
|
||||
|
||||
// New returns a Service.
|
||||
func New(pool *pgxpool.Pool, logger *slog.Logger) *Service {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Service{pool: pool, logger: logger,
|
||||
ClientFor: func(ctx context.Context, code string) (downloader.Driver, error) {
|
||||
return acquire.DownloadClientFor(ctx, pool, code)
|
||||
}}
|
||||
}
|
||||
|
||||
func (s *Service) now() time.Time {
|
||||
if s.Now != nil {
|
||||
return s.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// Run passes a minute after start, then every ten minutes, until ctx ends.
|
||||
// Each pass is bounded, so a client that never answers costs one pass.
|
||||
func (s *Service) Run(ctx context.Context) {
|
||||
first := time.NewTimer(time.Minute)
|
||||
defer first.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-first.C:
|
||||
}
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
rep := s.Pass(ctx)
|
||||
if rep.Retried+rep.GivenUp+rep.Gone+rep.Released+rep.Removed > 0 || rep.Error != "" {
|
||||
s.logger.Info("lifecycle: pass", "retried", rep.Retried, "given_up", rep.GivenUp, "gone", rep.Gone,
|
||||
"released", rep.Released, "removed", rep.Removed, "raised", rep.Raised, "settled", rep.Settled,
|
||||
"error", rep.Error)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Report is one pass.
|
||||
type Report struct {
|
||||
// Raised and Settled count ledger writes; the rest count acts.
|
||||
Raised, Settled int
|
||||
Retried, GivenUp, Gone, Released, Removed int
|
||||
Error string
|
||||
}
|
||||
|
||||
// Pass looks at every running download, every seeding one and every one
|
||||
// left behind, once.
|
||||
func (s *Service) Pass(ctx context.Context) Report {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ctx, cancel := context.WithTimeout(ctx, passTimeout)
|
||||
defer cancel()
|
||||
var rep Report
|
||||
q := dbq.New(s.pool)
|
||||
set := s.settings(ctx, q)
|
||||
|
||||
if _, err := q.DeleteExpiredBlocks(ctx); err != nil {
|
||||
s.logger.Warn("lifecycle: lift expired holds", "err", err)
|
||||
}
|
||||
if n, err := q.SettleEndedGrabs(ctx); err == nil {
|
||||
rep.Settled += int(n)
|
||||
}
|
||||
live, err := q.ListLiveGrabs(ctx)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
seeds, err := q.ListSeedingGrabs(ctx)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
left, err := q.ListLeftovers(ctx)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
|
||||
work := map[string]*clientWork{}
|
||||
var order []string
|
||||
of := func(code string) *clientWork {
|
||||
if w, ok := work[code]; ok {
|
||||
return w
|
||||
}
|
||||
order = append(order, code)
|
||||
work[code] = &clientWork{}
|
||||
return work[code]
|
||||
}
|
||||
for _, g := range live {
|
||||
if g.InfoHash != "" {
|
||||
of(g.Client).live = append(of(g.Client).live, g)
|
||||
}
|
||||
}
|
||||
for _, g := range seeds {
|
||||
if g.InfoHash != "" {
|
||||
of(g.Client).seeds = append(of(g.Client).seeds, g)
|
||||
}
|
||||
}
|
||||
for _, l := range left {
|
||||
of(l.Client).left = append(of(l.Client).left, l)
|
||||
}
|
||||
for _, code := range order {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
s.passClient(ctx, q, code, work[code], set, &rep)
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
type clientWork struct {
|
||||
live []dbq.ListLiveGrabsRow
|
||||
seeds []dbq.ListSeedingGrabsRow
|
||||
left []dbq.DownloadIntervention
|
||||
}
|
||||
|
||||
func (s *Service) settings(ctx context.Context, q *dbq.Queries) Settings {
|
||||
row, err := q.GetAcquisitionSettings(ctx)
|
||||
if err != nil {
|
||||
// A blip keeps the defaults rather than stopping the pass.
|
||||
return Defaults
|
||||
}
|
||||
return Settings{
|
||||
SeedRatio: float64(row.SeedRatio),
|
||||
SeedTime: time.Duration(row.SeedDays) * 24 * time.Hour,
|
||||
Patience: time.Duration(row.StallHours) * time.Hour,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) passClient(ctx context.Context, q *dbq.Queries, code string, w *clientWork, set Settings, rep *Report) {
|
||||
d, err := s.ClientFor(ctx, code)
|
||||
if err != nil {
|
||||
s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, firstLine(err.Error()), rep)
|
||||
return
|
||||
}
|
||||
var hashes []string
|
||||
for _, g := range w.live {
|
||||
hashes = append(hashes, g.InfoHash)
|
||||
}
|
||||
for _, g := range w.seeds {
|
||||
hashes = append(hashes, g.InfoHash)
|
||||
}
|
||||
for _, l := range w.left {
|
||||
hashes = append(hashes, l.InfoHash)
|
||||
}
|
||||
ts, err := d.Torrents(ctx, hashes...)
|
||||
if err != nil {
|
||||
// Nothing about any one download is observed through a client that
|
||||
// cannot answer.
|
||||
s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, firstLine(err.Error()), rep)
|
||||
return
|
||||
}
|
||||
for _, t := range ts {
|
||||
if t.EngineDown != "" {
|
||||
s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, t.EngineDown, rep)
|
||||
return
|
||||
}
|
||||
}
|
||||
s.settle(ctx, q, code, "", ClientUnreachable, rep)
|
||||
|
||||
byHash := make(map[string]downloader.Torrent, len(ts))
|
||||
for _, t := range ts {
|
||||
byHash[indexer.NormalizeHash(t.Hash)] = t
|
||||
}
|
||||
// ABSENCE IS ONLY A MEASUREMENT FROM A CLIENT THAT LISTS SOMETHING. An
|
||||
// engine switched off, or a store that would not read, answers with an
|
||||
// empty list; then nothing is concluded from what is missing.
|
||||
listed := len(ts) > 0
|
||||
if !listed {
|
||||
all, err := d.Torrents(ctx)
|
||||
listed = err == nil && len(all) > 0
|
||||
}
|
||||
|
||||
for _, l := range w.left {
|
||||
t, ok := byHash[indexer.NormalizeHash(l.InfoHash)]
|
||||
s.removeAgain(ctx, q, d, l, t, ok, listed, rep)
|
||||
}
|
||||
for _, g := range w.live {
|
||||
t, ok := byHash[indexer.NormalizeHash(g.InfoHash)]
|
||||
s.judge(ctx, q, d, g, t, ok, listed, set, rep)
|
||||
}
|
||||
for _, g := range w.seeds {
|
||||
t, ok := byHash[indexer.NormalizeHash(g.InfoHash)]
|
||||
s.seed(ctx, q, d, g, t, ok, set, rep)
|
||||
}
|
||||
}
|
||||
|
||||
// judge decides what, if anything, is wrong with one running download, and
|
||||
// answers each condition either way: raised when true, settled when not.
|
||||
func (s *Service) judge(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow,
|
||||
t downloader.Torrent, ok, listed bool, set Settings, rep *Report) {
|
||||
if !isRunning(g.ImportState) {
|
||||
return // the importer's, or a person's
|
||||
}
|
||||
if !ok {
|
||||
if !listed {
|
||||
return
|
||||
}
|
||||
row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, Gone,
|
||||
"The download client no longer lists it.", rep)
|
||||
if raised && s.now().Sub(row.FirstSeen.Time) >= goneAfter {
|
||||
s.endGone(ctx, q, g, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, Gone, rep)
|
||||
|
||||
if t.Done() {
|
||||
// Finished: whatever was wrong while it downloaded is over, and the
|
||||
// import is the importer's.
|
||||
for _, c := range []string{ClientError, NoMetadata, NoPeers} {
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, c, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// The client's own verdict, given up on at once: it has already made
|
||||
// the decision a retry would ask about (FL #4749).
|
||||
if t.Failed() {
|
||||
why := t.FailReason
|
||||
if why == "" {
|
||||
why = fmt.Sprintf("the download client reports it %q", t.State)
|
||||
}
|
||||
if _, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, ClientError, why, rep); raised {
|
||||
s.giveUp(ctx, q, d, g, ClientError, EndFailed, why, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, ClientError, rep)
|
||||
|
||||
// Below here the stall setting applies, measured from the grab. With it
|
||||
// off nothing is raised and what was raised clears.
|
||||
age := s.now().Sub(g.GrabbedAt.Time)
|
||||
patient := set.Patience > 0 && age > set.Patience
|
||||
|
||||
if patient && t.Fetching() {
|
||||
row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NoMetadata,
|
||||
fmt.Sprintf("Still fetching its file list after %s.", span(age)), rep)
|
||||
if raised {
|
||||
s.ladder(ctx, q, d, g, NoMetadata, row.Remediations, age, rep)
|
||||
}
|
||||
} else {
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, NoMetadata, rep)
|
||||
}
|
||||
// Not while fetching: a torrent with no file list has no swarm to be
|
||||
// alone in.
|
||||
if patient && !t.Fetching() && t.Alone() {
|
||||
row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NoPeers,
|
||||
fmt.Sprintf("Nobody has shared it in %s; %.0f%% downloaded.", span(age), t.Progress*100), rep)
|
||||
if raised {
|
||||
s.ladder(ctx, q, d, g, NoPeers, row.Remediations, age, rep)
|
||||
}
|
||||
} else {
|
||||
s.settle(ctx, q, g.Client, g.InfoHash, NoPeers, rep)
|
||||
}
|
||||
}
|
||||
|
||||
// ladder retries a stuck download until the retries are spent, then gives
|
||||
// it up. The rung is the ledger row's, so a restart hands out no more.
|
||||
func (s *Service) ladder(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow,
|
||||
trigger string, done int32, age time.Duration, rep *Report) {
|
||||
if int(done) < Retries {
|
||||
// Counted before it happens: a retry that happened and was not
|
||||
// counted is a bound that never binds.
|
||||
n, err := q.CountRemediation(ctx, dbq.CountRemediationParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash, Trigger: trigger, Action: actRetried})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Exchange schedules its own announces and has no recheck: for it a
|
||||
// retry is the time between passes, and the ladder still climbs.
|
||||
if err := d.Reannounce(ctx, g.InfoHash); err != nil && !errors.Is(err, downloader.ErrUnsupported) {
|
||||
s.logger.Warn("lifecycle: reannounce", "grab", g.Name, "err", err)
|
||||
}
|
||||
if err := d.Recheck(ctx, g.InfoHash); err != nil && !errors.Is(err, downloader.ErrUnsupported) {
|
||||
s.logger.Warn("lifecycle: recheck", "grab", g.Name, "err", err)
|
||||
}
|
||||
s.logger.Info("lifecycle: retried a stalled download", "grab", g.Name, "trigger", trigger, "attempt", n)
|
||||
rep.Retried++
|
||||
return
|
||||
}
|
||||
why := fmt.Sprintf("taken %s ago and never finished: %s", span(age), reasonFor(trigger))
|
||||
s.giveUp(ctx, q, d, g, trigger, EndStalled, why, rep)
|
||||
}
|
||||
|
||||
// giveUp ends the grab, holds the release back for StallBlock, removes the
|
||||
// download with its files and searches for the album again.
|
||||
func (s *Service) giveUp(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow,
|
||||
trigger, end, why string, rep *Report) {
|
||||
until := s.now().Add(StallBlock)
|
||||
detail := capital(why) + fmt.Sprintf(". Held back until %s; the album is searched for again.", until.Format("2 Jan"))
|
||||
// The update is the lock: an import that claimed the download meanwhile
|
||||
// keeps it.
|
||||
if _, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: end, EndDetail: detail, States: running}); err != nil {
|
||||
return
|
||||
}
|
||||
// Held back before it is removed: if the pass dies between the two, the
|
||||
// release is still not taken again.
|
||||
if err := acquire.NewMemory(s.pool).Block(ctx, dbq.BlockReleaseParams{
|
||||
InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockStalled, Detail: capital(why) + ".",
|
||||
WantID: g.WantID, ExpiresAt: pgtype.Timestamptz{Time: until, Valid: true},
|
||||
}); err != nil {
|
||||
s.logger.Warn("lifecycle: hold the release back", "grab", g.Name, "err", err)
|
||||
}
|
||||
removed := s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, rep)
|
||||
if g.WantID.Valid {
|
||||
if err := q.SearchWantAgain(ctx, g.WantID); err != nil {
|
||||
s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err)
|
||||
}
|
||||
}
|
||||
if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash, Trigger: trigger, Action: actGivenUp}); err != nil {
|
||||
s.logger.Warn("lifecycle: stamp the ledger", "grab", g.Name, "err", err)
|
||||
}
|
||||
if removed {
|
||||
if n, err := q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash}); err == nil {
|
||||
rep.Settled += int(n)
|
||||
}
|
||||
}
|
||||
s.logger.Info("lifecycle: gave up on a download", "grab", g.Name, "trigger", trigger, "why", why)
|
||||
rep.GivenUp++
|
||||
}
|
||||
|
||||
// endGone ends a grab whose download left the client, and searches again.
|
||||
// The release is not held back: nothing was found wrong with it.
|
||||
func (s *Service) endGone(ctx context.Context, q *dbq.Queries, g dbq.ListLiveGrabsRow, rep *Report) {
|
||||
detail := "The download client no longer lists it: removed outside Minstrel. The album is searched for again."
|
||||
if _, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: EndGone, EndDetail: detail, States: running}); err != nil {
|
||||
return
|
||||
}
|
||||
if g.WantID.Valid {
|
||||
if err := q.SearchWantAgain(ctx, g.WantID); err != nil {
|
||||
s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err)
|
||||
}
|
||||
}
|
||||
if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash, Trigger: Gone, Action: actGivenUp}); err != nil {
|
||||
s.logger.Warn("lifecycle: stamp the ledger", "grab", g.Name, "err", err)
|
||||
}
|
||||
if n, err := q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{
|
||||
Client: g.Client, InfoHash: g.InfoHash}); err == nil {
|
||||
rep.Settled += int(n)
|
||||
}
|
||||
s.logger.Info("lifecycle: a download left its client", "grab", g.Name)
|
||||
rep.Gone++
|
||||
}
|
||||
|
||||
// seed removes an imported download once it has shared enough.
|
||||
//
|
||||
// Absence is not a verdict: a restarting client answers with a partial list.
|
||||
// A seeding download missing from it is left until its time is up, then
|
||||
// recorded as released; there is nothing to delete.
|
||||
func (s *Service) seed(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListSeedingGrabsRow,
|
||||
t downloader.Torrent, ok bool, set Settings, rep *Report) {
|
||||
overdue := set.SeedTime > 0 && g.ImportedAt.Valid && s.now().Sub(g.ImportedAt.Time) >= set.SeedTime
|
||||
if !ok {
|
||||
if overdue {
|
||||
s.release(ctx, q, g.ID, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := q.SetGrabRatio(ctx, dbq.SetGrabRatioParams{ID: g.ID, Ratio: float32(t.Ratio)}); err != nil {
|
||||
s.logger.Warn("lifecycle: record the ratio", "grab", g.Name, "err", err)
|
||||
}
|
||||
if t.Ratio < set.SeedRatio && !overdue {
|
||||
return
|
||||
}
|
||||
s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, rep)
|
||||
s.logger.Info("lifecycle: seeded enough, removed the download", "grab", g.Name, "ratio", t.Ratio, "overdue", overdue)
|
||||
s.release(ctx, q, g.ID, rep)
|
||||
}
|
||||
|
||||
func (s *Service) release(ctx context.Context, q *dbq.Queries, id pgtype.UUID, rep *Report) {
|
||||
if err := q.ReleaseSeed(ctx, id); err != nil {
|
||||
s.logger.Warn("lifecycle: record the seeding ended", "err", err)
|
||||
return
|
||||
}
|
||||
rep.Released++
|
||||
}
|
||||
|
||||
// remove deletes a download and its files. A client that refuses leaves an
|
||||
// open not_removed row, which every pass tries again.
|
||||
func (s *Service) remove(ctx context.Context, q *dbq.Queries, d downloader.Driver, code, hash string,
|
||||
grab pgtype.UUID, name string, rep *Report) bool {
|
||||
if err := d.Delete(ctx, true, hash); err != nil {
|
||||
s.logger.Warn("lifecycle: remove the download", "grab", name, "err", err)
|
||||
s.raise(ctx, q, code, hash, grab, name, NotRemoved, notRemoved(err), rep)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// removeAgain asks the client once more to remove a download left behind,
|
||||
// unless it has been taken again since.
|
||||
func (s *Service) removeAgain(ctx context.Context, q *dbq.Queries, d downloader.Driver, l dbq.DownloadIntervention,
|
||||
t downloader.Torrent, ok, listed bool, rep *Report) {
|
||||
if !ok {
|
||||
if listed {
|
||||
s.settle(ctx, q, l.Client, l.InfoHash, NotRemoved, rep)
|
||||
}
|
||||
return
|
||||
}
|
||||
live, err := q.LiveGrabOnHash(ctx, l.InfoHash)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if live {
|
||||
s.settle(ctx, q, l.Client, l.InfoHash, NotRemoved, rep)
|
||||
return
|
||||
}
|
||||
if err := d.Delete(ctx, true, l.InfoHash); err != nil {
|
||||
s.raise(ctx, q, l.Client, l.InfoHash, l.GrabID, t.Name, NotRemoved, notRemoved(err), rep)
|
||||
return
|
||||
}
|
||||
if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{
|
||||
Client: l.Client, InfoHash: l.InfoHash, Trigger: NotRemoved, Action: actRemoved}); err != nil {
|
||||
s.logger.Warn("lifecycle: stamp the ledger", "hash", l.InfoHash, "err", err)
|
||||
}
|
||||
s.logger.Info("lifecycle: removed a download left behind", "name", l.Name)
|
||||
rep.Removed++
|
||||
}
|
||||
|
||||
// raise records a condition or agrees with the open row that says it. The
|
||||
// bool is false when the write failed: then nothing is acted on, since
|
||||
// without the row's count there is no bound.
|
||||
func (s *Service) raise(ctx context.Context, q *dbq.Queries, code, hash string, grab pgtype.UUID, name, trigger,
|
||||
detail string, rep *Report) (dbq.DownloadIntervention, bool) {
|
||||
row, err := q.RaiseIntervention(ctx, dbq.RaiseInterventionParams{
|
||||
Client: code, InfoHash: hash, GrabID: grab, Name: name, Trigger: trigger, Detail: detail})
|
||||
if err != nil {
|
||||
s.logger.Warn("lifecycle: record", "client", code, "hash", hash, "trigger", trigger, "err", err)
|
||||
return row, false
|
||||
}
|
||||
rep.Raised++
|
||||
return row, true
|
||||
}
|
||||
|
||||
func (s *Service) settle(ctx context.Context, q *dbq.Queries, code, hash, trigger string, rep *Report) {
|
||||
n, err := q.SettleIntervention(ctx, dbq.SettleInterventionParams{Client: code, InfoHash: hash, Trigger: trigger})
|
||||
if err != nil {
|
||||
s.logger.Warn("lifecycle: settle", "client", code, "hash", hash, "trigger", trigger, "err", err)
|
||||
return
|
||||
}
|
||||
rep.Settled += int(n)
|
||||
}
|
||||
|
||||
// ErrNotAbandonable: the download is not running; an import, or a person's
|
||||
// decision on one, has it.
|
||||
var ErrNotAbandonable = errors.New("lifecycle: that download is not running")
|
||||
|
||||
// Abandon is a person giving up on a running download: the release is set
|
||||
// aside for good (their word about it), the download removed with its files,
|
||||
// and the album searched for again.
|
||||
func (s *Service) Abandon(ctx context.Context, grabID, by pgtype.UUID) (dbq.Grab, error) {
|
||||
q := dbq.New(s.pool)
|
||||
g, err := q.GetGrab(ctx, grabID)
|
||||
if err != nil {
|
||||
return dbq.Grab{}, err
|
||||
}
|
||||
if g.EndedAt.Valid || !isRunning(g.ImportState) {
|
||||
return dbq.Grab{}, ErrNotAbandonable
|
||||
}
|
||||
const why = "Given up by you before it finished. The album is searched for again."
|
||||
ended, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: EndAbandoned, EndDetail: why, States: running})
|
||||
if err != nil {
|
||||
return dbq.Grab{}, ErrNotAbandonable
|
||||
}
|
||||
if g.InfoHash != "" {
|
||||
if err := acquire.NewMemory(s.pool).Block(ctx, dbq.BlockReleaseParams{
|
||||
InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockOperator, Detail: "Given up by you before it finished.",
|
||||
WantID: g.WantID, BlockedBy: by,
|
||||
}); err != nil {
|
||||
s.logger.Warn("lifecycle: set the release aside", "grab", g.Name, "err", err)
|
||||
}
|
||||
var rep Report
|
||||
if d, err := s.ClientFor(ctx, g.Client); err != nil {
|
||||
s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NotRemoved, notRemoved(err), &rep)
|
||||
} else if s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, &rep) {
|
||||
_, _ = q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{Client: g.Client, InfoHash: g.InfoHash})
|
||||
}
|
||||
}
|
||||
if g.WantID.Valid {
|
||||
if err := q.SearchWantAgain(ctx, g.WantID); err != nil {
|
||||
s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err)
|
||||
}
|
||||
}
|
||||
return ended, nil
|
||||
}
|
||||
|
||||
func isRunning(state string) bool {
|
||||
for _, r := range running {
|
||||
if state == r {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// reasonFor is a condition in a person's words.
|
||||
func reasonFor(trigger string) string {
|
||||
switch trigger {
|
||||
case NoPeers:
|
||||
return "nobody was sharing it"
|
||||
case NoMetadata:
|
||||
return "its file list never arrived"
|
||||
case ClientError:
|
||||
return "the download client gave up on it"
|
||||
}
|
||||
return "it stalled"
|
||||
}
|
||||
|
||||
func notRemoved(err error) string {
|
||||
return "Given up on, but the download client would not remove it: " + firstLine(err.Error()) +
|
||||
". Minstrel asks again every ten minutes."
|
||||
}
|
||||
|
||||
// span is a duration as a person says it.
|
||||
func span(d time.Duration) string {
|
||||
switch {
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%d hours", int(d.Hours()))
|
||||
}
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
|
||||
func capital(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + s[1:]
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
return strings.TrimSpace(s[:i])
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/downloader"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/wanted"
|
||||
)
|
||||
|
||||
func newPool(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in -short mode")
|
||||
}
|
||||
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
||||
}
|
||||
if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
pool, err := pgxpool.New(context.Background(), dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("pool: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
dbtest.ResetDB(t, pool)
|
||||
return pool
|
||||
}
|
||||
|
||||
const (
|
||||
hash = "0123456789abcdef0123456789abcdef01234567"
|
||||
other = "fedcba9876543210fedcba9876543210fedcba98"
|
||||
)
|
||||
|
||||
// fakeClient lists the torrents it holds and records what it is asked.
|
||||
type fakeClient struct {
|
||||
downloader.Driver
|
||||
held map[string]downloader.Torrent
|
||||
err error
|
||||
deleteErr error
|
||||
deleted []string
|
||||
reannounce int
|
||||
recheck int
|
||||
}
|
||||
|
||||
func (c *fakeClient) Torrents(_ context.Context, hashes ...string) ([]downloader.Torrent, error) {
|
||||
if c.err != nil {
|
||||
return nil, c.err
|
||||
}
|
||||
var out []downloader.Torrent
|
||||
if len(hashes) == 0 {
|
||||
for _, t := range c.held {
|
||||
out = append(out, t)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
for _, h := range hashes {
|
||||
if t, ok := c.held[h]; ok {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error {
|
||||
if c.deleteErr != nil {
|
||||
return c.deleteErr
|
||||
}
|
||||
c.deleted = append(c.deleted, hashes...)
|
||||
for _, h := range hashes {
|
||||
delete(c.held, h)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) Reannounce(context.Context, ...string) error { c.reannounce++; return nil }
|
||||
func (c *fakeClient) Recheck(context.Context, ...string) error { c.recheck++; return nil }
|
||||
|
||||
type stack struct {
|
||||
q *dbq.Queries
|
||||
s *Service
|
||||
client *fakeClient
|
||||
want dbq.Want
|
||||
grab dbq.Grab
|
||||
clock time.Time
|
||||
}
|
||||
|
||||
// newStack wants RAM and has a grab of it running in a fake client holding
|
||||
// t under hash.
|
||||
func newStack(t *testing.T, tor downloader.Torrent) *stack {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
pool := newPool(t)
|
||||
q := dbq.New(pool)
|
||||
w, err := wanted.WantWith(ctx, q, wanted.WantParams{GroupMBID: "rg-ram", ArtistMBID: "mb-dp", ArtistName: "Daft Punk",
|
||||
Title: "Random Access Memories", PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: wanted.ReasonRequest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g, err := acquire.NewMemory(pool).Grabbed(ctx, dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash,
|
||||
Name: "Daft Punk - RAM [FLAC]", Key: hash, Client: "qbittorrent", SavePath: "/downloads", Origin: acquire.OriginSchedule},
|
||||
pgtype.UUID{}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := q.MarkWantSearched(ctx, w.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tor.Hash = hash
|
||||
st := &stack{q: q, client: &fakeClient{held: map[string]downloader.Torrent{hash: tor}}, want: w, grab: g,
|
||||
clock: time.Now()}
|
||||
st.s = New(pool, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
st.s.ClientFor = func(context.Context, string) (downloader.Driver, error) { return st.client, nil }
|
||||
st.s.Now = func() time.Time { return st.clock }
|
||||
return st
|
||||
}
|
||||
|
||||
func (st *stack) grabNow(t *testing.T) dbq.Grab {
|
||||
t.Helper()
|
||||
g, err := st.q.GetGrab(context.Background(), st.grab.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func (st *stack) searchedAgain(t *testing.T) bool {
|
||||
t.Helper()
|
||||
w, err := st.q.GetWantByReleaseGroup(context.Background(), "rg-ram")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return !w.SearchedAt.Valid
|
||||
}
|
||||
|
||||
func (st *stack) ledger(t *testing.T) []dbq.DownloadIntervention {
|
||||
t.Helper()
|
||||
rows, err := st.s.pool.Query(context.Background(), `SELECT * FROM download_interventions ORDER BY first_seen, id`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := pgx.CollectRows(rows, pgx.RowToStructByPos[dbq.DownloadIntervention])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestLifecycle_AFailedDownloadIsGivenUpAtOnce_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "error",
|
||||
FailReason: "No space left on device"})
|
||||
rep := st.s.Pass(context.Background())
|
||||
if rep.GivenUp != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
g := st.grabNow(t)
|
||||
if !g.EndedAt.Valid || g.EndReason != EndFailed {
|
||||
t.Fatalf("grab ended=%v reason=%q", g.EndedAt.Valid, g.EndReason)
|
||||
}
|
||||
if len(st.client.deleted) != 1 || !st.searchedAgain(t) {
|
||||
t.Errorf("deleted %v, searched again %v", st.client.deleted, st.searchedAgain(t))
|
||||
}
|
||||
blocks, err := st.q.ListReleaseBlocks(context.Background())
|
||||
if err != nil || len(blocks) != 1 || blocks[0].Reason != acquire.BlockStalled || !blocks[0].ExpiresAt.Valid {
|
||||
t.Fatalf("blocks = %+v %v", blocks, err)
|
||||
}
|
||||
if d := blocks[0].ExpiresAt.Time.Sub(st.clock); d < StallBlock-time.Minute || d > StallBlock+time.Minute {
|
||||
t.Errorf("held back for %s", d)
|
||||
}
|
||||
rows := st.ledger(t)
|
||||
if len(rows) != 1 || rows[0].Trigger != ClientError || rows[0].Action != actGivenUp || rows[0].Outcome != "resolved" ||
|
||||
rows[0].Detail != "No space left on device" {
|
||||
t.Errorf("ledger = %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_ADownloadNobodySharesIsRetriedThenGivenUp_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"})
|
||||
ctx := context.Background()
|
||||
|
||||
// Within the stall setting: nothing is wrong yet.
|
||||
if rep := st.s.Pass(ctx); rep.Retried+rep.GivenUp != 0 || len(st.ledger(t)) != 0 {
|
||||
t.Fatalf("a young download was judged: %+v", rep)
|
||||
}
|
||||
|
||||
st.clock = st.clock.Add(7 * time.Hour)
|
||||
for i := 1; i <= Retries; i++ {
|
||||
if rep := st.s.Pass(ctx); rep.Retried != 1 || rep.GivenUp != 0 {
|
||||
t.Fatalf("pass %d = %+v", i, rep)
|
||||
}
|
||||
}
|
||||
if st.client.reannounce != Retries || st.client.recheck != Retries {
|
||||
t.Errorf("reannounced %d, rechecked %d", st.client.reannounce, st.client.recheck)
|
||||
}
|
||||
if st.grabNow(t).EndedAt.Valid {
|
||||
t.Fatal("given up before the retries ran out")
|
||||
}
|
||||
if rep := st.s.Pass(ctx); rep.GivenUp != 1 {
|
||||
t.Fatalf("last pass = %+v", rep)
|
||||
}
|
||||
g := st.grabNow(t)
|
||||
if g.EndReason != EndStalled || len(st.client.deleted) != 1 || !st.searchedAgain(t) {
|
||||
t.Errorf("grab %q: %s; deleted %v", g.EndReason, g.EndDetail, st.client.deleted)
|
||||
}
|
||||
rows := st.ledger(t)
|
||||
if len(rows) != 1 || rows[0].Trigger != NoPeers || rows[0].Remediations != Retries+1 || rows[0].Outcome != "resolved" {
|
||||
t.Errorf("ledger = %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AStalledDownloadThatRecoversIsSettled_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"})
|
||||
ctx := context.Background()
|
||||
st.clock = st.clock.Add(7 * time.Hour)
|
||||
st.s.Pass(ctx)
|
||||
t2 := st.client.held[hash]
|
||||
t2.NumSeeds = 3
|
||||
st.client.held[hash] = t2
|
||||
st.s.Pass(ctx)
|
||||
rows := st.ledger(t)
|
||||
if len(rows) != 1 || rows[0].Outcome != "resolved" || rows[0].Action != actRetried {
|
||||
t.Fatalf("ledger = %+v", rows)
|
||||
}
|
||||
if st.grabNow(t).EndedAt.Valid {
|
||||
t.Error("a download that found peers was given up")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_TheStallSettingOffNeverGivesUp_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"})
|
||||
ctx := context.Background()
|
||||
if _, err := st.s.pool.Exec(ctx, `UPDATE acquisition_settings SET stall_hours = 0`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st.clock = st.clock.Add(30 * 24 * time.Hour)
|
||||
for range Retries + 2 {
|
||||
if rep := st.s.Pass(ctx); rep.Retried+rep.GivenUp != 0 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_ADownloadGoneFromTheClientEndsAfterHalfAnHour_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "downloading", NumSeeds: 4})
|
||||
ctx := context.Background()
|
||||
delete(st.client.held, hash)
|
||||
st.client.held[other] = downloader.Torrent{Hash: other, Name: "Someone else's", Progress: 1, State: "uploading"}
|
||||
|
||||
if rep := st.s.Pass(ctx); rep.Gone != 0 {
|
||||
t.Fatalf("ended at first sight: %+v", rep)
|
||||
}
|
||||
st.clock = st.clock.Add(31 * time.Minute)
|
||||
if rep := st.s.Pass(ctx); rep.Gone != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
g := st.grabNow(t)
|
||||
if g.EndReason != EndGone || !st.searchedAgain(t) {
|
||||
t.Errorf("grab %q, searched again %v", g.EndReason, st.searchedAgain(t))
|
||||
}
|
||||
if blocks, _ := st.q.ListReleaseBlocks(ctx); len(blocks) != 0 {
|
||||
t.Errorf("a release nothing was wrong with was held back: %+v", blocks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AnEmptyListIsNotAMeasurement_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "downloading"})
|
||||
ctx := context.Background()
|
||||
delete(st.client.held, hash)
|
||||
st.clock = st.clock.Add(24 * time.Hour)
|
||||
for range 3 {
|
||||
if rep := st.s.Pass(ctx); rep.Gone != 0 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
}
|
||||
if len(st.ledger(t)) != 0 || st.grabNow(t).EndedAt.Valid {
|
||||
t.Error("a client listing nothing was taken to have lost the download")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AnUnreachableClientIsRecordedAndSettled_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"})
|
||||
ctx := context.Background()
|
||||
st.clock = st.clock.Add(7 * time.Hour)
|
||||
st.client.err = errors.New("dial tcp: connection refused")
|
||||
st.s.Pass(ctx)
|
||||
down, err := st.q.ListClientsDown(ctx)
|
||||
if err != nil || len(down) != 1 || down[0].Client != "qbittorrent" {
|
||||
t.Fatalf("clients down = %+v %v", down, err)
|
||||
}
|
||||
if st.client.reannounce != 0 {
|
||||
t.Error("a download was judged through a client that did not answer")
|
||||
}
|
||||
st.client.err = nil
|
||||
st.s.Pass(ctx)
|
||||
if down, _ := st.q.ListClientsDown(ctx); len(down) != 0 {
|
||||
t.Errorf("still down: %+v", down)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AnImportedDownloadSeedsThenIsRemoved_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", Ratio: 0.8})
|
||||
ctx := context.Background()
|
||||
if _, err := st.q.RecordImport(ctx, dbq.RecordImportParams{ID: st.grab.ID, ImportState: "imported",
|
||||
ImportReason: "Imported.", ImportDetail: []byte("{}"), ImportedFiles: 3, EndReason: "imported"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !st.grabNow(t).Seeding {
|
||||
t.Fatal("an import did not leave its download seeding")
|
||||
}
|
||||
if rep := st.s.Pass(ctx); rep.Released != 0 || len(st.client.deleted) != 0 {
|
||||
t.Fatalf("removed before the ratio: %+v", rep)
|
||||
}
|
||||
if g := st.grabNow(t); g.Ratio < 0.79 || g.Ratio > 0.81 {
|
||||
t.Errorf("ratio recorded %v", g.Ratio)
|
||||
}
|
||||
tor := st.client.held[hash]
|
||||
tor.Ratio = 2.1
|
||||
st.client.held[hash] = tor
|
||||
if rep := st.s.Pass(ctx); rep.Released != 1 || len(st.client.deleted) != 1 {
|
||||
t.Fatalf("pass = %+v, deleted %v", rep, st.client.deleted)
|
||||
}
|
||||
if !st.grabNow(t).SeedReleasedAt.Valid {
|
||||
t.Error("the release was not recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AnImportedDownloadIsRemovedWhenItsDaysAreUp_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", Ratio: 0.1})
|
||||
ctx := context.Background()
|
||||
if _, err := st.q.RecordImport(ctx, dbq.RecordImportParams{ID: st.grab.ID, ImportState: "imported",
|
||||
ImportReason: "Imported.", ImportDetail: []byte("{}"), ImportedFiles: 3, EndReason: "imported"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st.clock = st.clock.Add(15 * 24 * time.Hour)
|
||||
if rep := st.s.Pass(ctx); rep.Released != 1 || len(st.client.deleted) != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_ADownloadTheClientWouldNotRemoveIsAskedAgain_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "error"})
|
||||
ctx := context.Background()
|
||||
st.client.deleteErr = errors.New("HTTP 500")
|
||||
if rep := st.s.Pass(ctx); rep.GivenUp != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
if !st.grabNow(t).EndedAt.Valid {
|
||||
t.Fatal("a refused removal held the grab")
|
||||
}
|
||||
open := 0
|
||||
for _, r := range st.ledger(t) {
|
||||
if r.Trigger == NotRemoved && r.Outcome == "open" {
|
||||
open++
|
||||
}
|
||||
}
|
||||
if open != 1 {
|
||||
t.Fatalf("ledger = %+v", st.ledger(t))
|
||||
}
|
||||
st.client.deleteErr = nil
|
||||
if rep := st.s.Pass(ctx); rep.Removed != 1 || len(st.client.deleted) != 1 {
|
||||
t.Fatalf("pass = %+v", rep)
|
||||
}
|
||||
for _, r := range st.ledger(t) {
|
||||
if r.Outcome == "open" {
|
||||
t.Errorf("still open: %+v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_APersonGivingUpSetsTheReleaseAsideForGood_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "downloading"})
|
||||
ctx := context.Background()
|
||||
if _, err := st.s.Abandon(ctx, st.grab.ID, pgtype.UUID{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := st.grabNow(t)
|
||||
if g.EndReason != EndAbandoned || len(st.client.deleted) != 1 || !st.searchedAgain(t) {
|
||||
t.Errorf("grab %q, deleted %v", g.EndReason, st.client.deleted)
|
||||
}
|
||||
blocks, _ := st.q.ListReleaseBlocks(ctx)
|
||||
if len(blocks) != 1 || blocks[0].Reason != acquire.BlockOperator || blocks[0].ExpiresAt.Valid {
|
||||
t.Errorf("blocks = %+v", blocks)
|
||||
}
|
||||
if _, err := st.s.Abandon(ctx, st.grab.ID, pgtype.UUID{}); !errors.Is(err, ErrNotAbandonable) {
|
||||
t.Errorf("abandoned twice: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycle_AHoldExpiresButABlockForGoodIsNeverShortened_Integration(t *testing.T) {
|
||||
st := newStack(t, downloader.Torrent{Name: "x", Progress: 0.4, State: "downloading"})
|
||||
ctx := context.Background()
|
||||
mem := acquire.NewMemory(st.s.pool)
|
||||
past := pgtype.Timestamptz{Time: time.Now().Add(-time.Hour), Valid: true}
|
||||
if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: other, Name: "old", Reason: acquire.BlockStalled, ExpiresAt: past}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if blocks, _ := st.q.ListReleaseBlocks(ctx); len(blocks) != 0 {
|
||||
t.Errorf("an expired hold still holds: %+v", blocks)
|
||||
}
|
||||
if n, err := st.q.DeleteExpiredBlocks(ctx); err != nil || n != 1 {
|
||||
t.Errorf("lifted %d %v", n, err)
|
||||
}
|
||||
|
||||
if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: hash, Name: "x", Reason: acquire.BlockOperator}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
soon := pgtype.Timestamptz{Time: time.Now().Add(time.Hour), Valid: true}
|
||||
if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: hash, Name: "x", Reason: acquire.BlockStalled, ExpiresAt: soon}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocks, _ := st.q.ListReleaseBlocks(ctx)
|
||||
if len(blocks) != 1 || blocks[0].Reason != acquire.BlockOperator || blocks[0].ExpiresAt.Valid {
|
||||
t.Errorf("a block for good was shortened: %+v", blocks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpan(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
d time.Duration
|
||||
want string
|
||||
}{{45 * time.Minute, "45 minutes"}, {7 * time.Hour, "7 hours"}, {3 * 24 * time.Hour, "3 days"}} {
|
||||
if got := span(c.d); got != c.want {
|
||||
t.Errorf("span(%s) = %q, want %q", c.d, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -630,6 +630,8 @@ export type WantGrab = {
|
||||
progress: number;
|
||||
import_state: ImportState;
|
||||
import_reason: string;
|
||||
// Why an ended download ended (#5523).
|
||||
end_detail: string;
|
||||
imported_at: string | null;
|
||||
imported_files: number;
|
||||
};
|
||||
@@ -710,6 +712,12 @@ export type SearchSettings = {
|
||||
// scan_paths), and the library's folders (#5522).
|
||||
import_root: string;
|
||||
scan_paths: string[];
|
||||
// The download lifecycle (#5523): an imported download seeds to this
|
||||
// ratio or for this many days (0: removed once imported; no time limit),
|
||||
// and one nobody shares is given up after this many hours (0: never).
|
||||
seed_ratio: number;
|
||||
seed_days: number;
|
||||
stall_hours: number;
|
||||
};
|
||||
|
||||
export type WantCounts = {
|
||||
@@ -761,17 +769,42 @@ export type Download = {
|
||||
files: string[];
|
||||
skipped: number;
|
||||
rehomed_from?: string;
|
||||
// Why an ended download ended; for an imported one, its seeding (#5523).
|
||||
end_detail: string;
|
||||
seeding: boolean;
|
||||
ratio: number;
|
||||
seed_released_at: string | null;
|
||||
// What is wrong with it now, if anything.
|
||||
condition: DownloadCondition | null;
|
||||
};
|
||||
|
||||
export type DownloadsPage = { items: Download[]; limit: number; offset: number };
|
||||
// An open condition on a download, from the lifecycle's ledger (#5523).
|
||||
export type DownloadCondition = {
|
||||
trigger: 'no_peers' | 'no_metadata' | 'client_error' | 'client_unreachable' | 'gone' | 'not_removed';
|
||||
detail: string;
|
||||
retries: number;
|
||||
since: string;
|
||||
};
|
||||
|
||||
// A download client not answering now.
|
||||
export type ClientDown = { client: string; detail: string; since: string };
|
||||
|
||||
export type DownloadsPage = {
|
||||
items: Download[];
|
||||
limit: number;
|
||||
offset: number;
|
||||
clients_down: ClientDown[];
|
||||
};
|
||||
|
||||
// A release set aside: by a person, by its files, or by an import's check.
|
||||
export type ReleaseBlock = {
|
||||
info_hash: string;
|
||||
name: string;
|
||||
reason: 'operator' | 'files' | 'import';
|
||||
reason: 'operator' | 'files' | 'import' | 'stalled';
|
||||
detail: string;
|
||||
blocked_at: string;
|
||||
// When a release given up on may be taken again; null is for good.
|
||||
expires_at: string | null;
|
||||
want_title: string;
|
||||
want_artist: string;
|
||||
};
|
||||
|
||||
@@ -160,6 +160,9 @@ export async function putSearchSettings(body: {
|
||||
search_interval_hours: number;
|
||||
allowed_qualities: string[];
|
||||
import_root: string;
|
||||
seed_ratio: number;
|
||||
seed_days: number;
|
||||
stall_hours: number;
|
||||
}): Promise<SearchSettings> {
|
||||
return api.put<SearchSettings>('/api/admin/acquisition/settings', body);
|
||||
}
|
||||
@@ -199,6 +202,12 @@ export async function refuseDownload(id: string): Promise<void> {
|
||||
await api.post(`/api/admin/downloads/${id}/refuse`, {});
|
||||
}
|
||||
|
||||
// A running download given up: the release is set aside for good, the
|
||||
// download removed and the album searched for again (#5523).
|
||||
export async function abandonDownload(id: string): Promise<void> {
|
||||
await api.post(`/api/admin/downloads/${id}/abandon`, {});
|
||||
}
|
||||
|
||||
export async function listReleaseBlocks(offset: number = 0): Promise<ReleaseBlocksPage> {
|
||||
return api.get<ReleaseBlocksPage>(`/api/admin/release-blocks?offset=${offset}`);
|
||||
}
|
||||
|
||||
@@ -11,13 +11,23 @@
|
||||
// for, whether the best release is taken without asking, and which
|
||||
// qualities are taken at all. The schedule wakes every ten minutes and
|
||||
// keeps to each indexer's searches-per-hour. A finished download is
|
||||
// checked and filed into one of the library's folders (#5522).
|
||||
// checked and filed into one of the library's folders (#5522); it then
|
||||
// seeds until the ratio or the days below, and a download nobody shares
|
||||
// is given up after the hours below (#5523).
|
||||
|
||||
const client = useQueryClient();
|
||||
const store = createSearchSettingsQuery();
|
||||
const query = $derived($store);
|
||||
|
||||
type Form = { auto_grab: boolean; search_interval_hours: number; allowed: string[]; import_root: string };
|
||||
type Form = {
|
||||
auto_grab: boolean;
|
||||
search_interval_hours: number;
|
||||
allowed: string[];
|
||||
import_root: string;
|
||||
seed_ratio: number;
|
||||
seed_days: number;
|
||||
stall_hours: number;
|
||||
};
|
||||
|
||||
// The form is the server's settings until edited; saved holds what a save
|
||||
// returned until the query catches up.
|
||||
@@ -32,7 +42,10 @@
|
||||
auto_grab: s.auto_grab,
|
||||
search_interval_hours: s.search_interval_hours,
|
||||
allowed: [...s.allowed_qualities],
|
||||
import_root: s.import_root
|
||||
import_root: s.import_root,
|
||||
seed_ratio: s.seed_ratio,
|
||||
seed_days: s.seed_days,
|
||||
stall_hours: s.stall_hours
|
||||
};
|
||||
}
|
||||
|
||||
@@ -55,7 +68,10 @@
|
||||
auto_grab: form.auto_grab,
|
||||
search_interval_hours: Number(form.search_interval_hours),
|
||||
allowed_qualities: form.allowed,
|
||||
import_root: form.import_root
|
||||
import_root: form.import_root,
|
||||
seed_ratio: Number(form.seed_ratio),
|
||||
seed_days: Number(form.seed_days),
|
||||
stall_hours: Number(form.stall_hours)
|
||||
});
|
||||
edited = null;
|
||||
await client.invalidateQueries({ queryKey: qk.searchSettings() });
|
||||
@@ -158,7 +174,7 @@
|
||||
<span class="text-sm text-text-primary">File new music into</span>
|
||||
<span class="block text-xs text-text-secondary">
|
||||
A download that checks out is copied here as Artist/Album (Year)/Artist - Album - NN -
|
||||
Title, tagged with its MusicBrainz ids. The download itself keeps seeding.
|
||||
Title, tagged with its MusicBrainz ids.
|
||||
</span>
|
||||
<select
|
||||
value={form.import_root || query.data.scan_paths[0]}
|
||||
@@ -174,11 +190,68 @@
|
||||
{:else if query.data.scan_paths.length === 1}
|
||||
<p class="text-xs text-text-secondary">
|
||||
A download that checks out is copied into <span class="font-mono">{query.data.scan_paths[0]}</span>
|
||||
as Artist/Album (Year)/Artist - Album - NN - Title, tagged with its MusicBrainz ids. The
|
||||
download itself keeps seeding.
|
||||
as Artist/Album (Year)/Artist - Album - NN - Title, tagged with its MusicBrainz ids.
|
||||
</p>
|
||||
{/if}
|
||||
|
||||
<fieldset class="space-y-3">
|
||||
<legend class="text-sm text-text-primary">After the download</legend>
|
||||
<label class="block">
|
||||
<span class="block text-xs text-text-secondary">
|
||||
The files are copied into the library, so the download goes on sharing until either limit,
|
||||
then is removed with its files. A ratio of 0 removes it once imported; 0 days is no time
|
||||
limit.
|
||||
</span>
|
||||
<span class="mt-1 flex flex-wrap items-center gap-2 text-sm text-text-secondary">
|
||||
Seed to a ratio of
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="100"
|
||||
step="0.1"
|
||||
aria-label="Seed ratio"
|
||||
value={form.seed_ratio}
|
||||
oninput={(e) => edit({ seed_ratio: Number(e.currentTarget.value) })}
|
||||
class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary
|
||||
focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent"
|
||||
/>
|
||||
or for
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="365"
|
||||
aria-label="Seed days"
|
||||
value={form.seed_days}
|
||||
oninput={(e) => edit({ seed_days: Number(e.currentTarget.value) })}
|
||||
class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary
|
||||
focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent"
|
||||
/>
|
||||
days
|
||||
</span>
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="block text-xs text-text-secondary">
|
||||
A download nobody is sharing is asked for again three times, ten minutes apart, then given
|
||||
up: removed, held back for a week, and the album searched for again. 0 never gives up for
|
||||
that; a download the client fails is always given up.
|
||||
</span>
|
||||
<span class="mt-1 flex flex-wrap items-center gap-2 text-sm text-text-secondary">
|
||||
Give up on a stalled download after
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="720"
|
||||
aria-label="Stall hours"
|
||||
value={form.stall_hours}
|
||||
oninput={(e) => edit({ stall_hours: Number(e.currentTarget.value) })}
|
||||
class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary
|
||||
focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent"
|
||||
/>
|
||||
hours
|
||||
</span>
|
||||
</label>
|
||||
</fieldset>
|
||||
|
||||
<p class="text-xs text-text-secondary" data-testid="last-pass">{lastPass(query.data)}</p>
|
||||
|
||||
<div class="flex justify-end">
|
||||
|
||||
@@ -24,7 +24,10 @@ const settings: SearchSettings = {
|
||||
last_pass_at: new Date(Date.now() - 5 * 60_000).toISOString(),
|
||||
last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true },
|
||||
import_root: '',
|
||||
scan_paths: ['/music', '/music2']
|
||||
scan_paths: ['/music', '/music2'],
|
||||
seed_ratio: 2,
|
||||
seed_days: 14,
|
||||
stall_hours: 6
|
||||
};
|
||||
|
||||
afterEach(() => vi.clearAllMocks());
|
||||
@@ -60,7 +63,10 @@ describe('SearchSettingsCard', () => {
|
||||
auto_grab: false,
|
||||
search_interval_hours: 24,
|
||||
allowed_qualities: ['lossless', 'lossy_high', 'unknown'],
|
||||
import_root: ''
|
||||
import_root: '',
|
||||
seed_ratio: 2,
|
||||
seed_days: 14,
|
||||
stall_hours: 6
|
||||
})
|
||||
);
|
||||
});
|
||||
@@ -82,6 +88,20 @@ describe('SearchSettingsCard', () => {
|
||||
expect(screen.getByText('/music')).toBeTruthy();
|
||||
});
|
||||
|
||||
test('seeding and giving up on a stalled download are set here', async () => {
|
||||
setup();
|
||||
expect((screen.getByLabelText(/seed ratio/i) as HTMLInputElement).value).toBe('2');
|
||||
await fireEvent.input(screen.getByLabelText(/seed ratio/i), { target: { value: '1.5' } });
|
||||
await fireEvent.input(screen.getByLabelText(/seed days/i), { target: { value: '7' } });
|
||||
await fireEvent.input(screen.getByLabelText(/stall hours/i), { target: { value: '0' } });
|
||||
await fireEvent.click(screen.getByRole('button', { name: /^save$/i }));
|
||||
await waitFor(() =>
|
||||
expect(putSearchSettings).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ seed_ratio: 1.5, seed_days: 7, stall_hours: 0 })
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
test('before the first pass it says so', () => {
|
||||
setup({ ...settings, last_pass_at: null, last_pass: {} });
|
||||
expect(screen.getByTestId('last-pass').textContent).toMatch(/hasn’t run yet/);
|
||||
|
||||
@@ -8,9 +8,10 @@
|
||||
import ListContinuation from '#lib/components/ListContinuation.svelte';
|
||||
import type { ReleaseBlock } from '#lib/api/types.js';
|
||||
|
||||
// The Blocked tab (M502 #5521–#5522): releases the search will never take
|
||||
// The Blocked tab (M502 #5521–#5523): releases the search will not take
|
||||
// again — set aside by a person, by what their files were, or by an
|
||||
// import's check that found they weren't the album.
|
||||
// import's check that found they weren't the album; and, for a week,
|
||||
// releases given up on because they would not finish.
|
||||
|
||||
const client = useQueryClient();
|
||||
const blocksStore = createReleaseBlocksQuery();
|
||||
@@ -35,8 +36,13 @@
|
||||
function by(b: ReleaseBlock): string {
|
||||
if (b.reason === 'operator') return 'Set aside by you';
|
||||
if (b.reason === 'files') return 'Set aside for its files';
|
||||
if (b.reason === 'stalled') return 'Given up on: it would not finish';
|
||||
return 'Refused on import';
|
||||
}
|
||||
|
||||
// A release given up on is held back for a while, then may be taken again.
|
||||
const until = (b: ReleaseBlock) =>
|
||||
b.expires_at ? ` · held back until ${new Date(b.expires_at).toLocaleDateString()}` : '';
|
||||
</script>
|
||||
|
||||
{#if blocks.isPending}
|
||||
@@ -61,7 +67,7 @@
|
||||
{#if b.want_title}
|
||||
<div class="truncate text-xs text-text-secondary">For {b.want_artist} – {b.want_title}</div>
|
||||
{/if}
|
||||
<div class="text-xs text-text-muted">{by(b)} {relativeTime(b.blocked_at)}</div>
|
||||
<div class="text-xs text-text-muted">{by(b)} {relativeTime(b.blocked_at)}{until(b)}</div>
|
||||
{#if b.detail}
|
||||
<p class="text-xs text-text-secondary">{b.detail}</p>
|
||||
{/if}
|
||||
|
||||
@@ -18,6 +18,7 @@ const block: ReleaseBlock = {
|
||||
reason: 'import',
|
||||
detail: 'The album tag says another version.',
|
||||
blocked_at: new Date(Date.now() - 3_600_000).toISOString(),
|
||||
expires_at: null,
|
||||
want_title: 'Random Access Memories',
|
||||
want_artist: 'Daft Punk'
|
||||
};
|
||||
@@ -43,6 +44,12 @@ describe('BlockList', () => {
|
||||
await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('abc123'));
|
||||
});
|
||||
|
||||
test('a release given up on says until when it is held back', () => {
|
||||
setup([{ ...block, reason: 'stalled', detail: 'Nobody was sharing it.', expires_at: '2026-10-17T12:00:00Z' }]);
|
||||
const line = screen.getByText(/given up on: it would not finish/i).textContent ?? '';
|
||||
expect(line).toMatch(/held back until/);
|
||||
});
|
||||
|
||||
test('with nothing blocked it says so', () => {
|
||||
setup([]);
|
||||
expect(screen.getByText(/no releases are blocked/i)).toBeTruthy();
|
||||
|
||||
@@ -1,23 +1,32 @@
|
||||
<script lang="ts">
|
||||
import { Download as DownloadIcon } from 'lucide-svelte';
|
||||
import { useQueryClient } from '@tanstack/svelte-query';
|
||||
import { createDownloadsQuery, importAnyway, refuseDownload, retryImport } from '#lib/api/wanted.js';
|
||||
import {
|
||||
abandonDownload,
|
||||
createDownloadsQuery,
|
||||
importAnyway,
|
||||
refuseDownload,
|
||||
retryImport
|
||||
} from '#lib/api/wanted.js';
|
||||
import { errMessage } from '#lib/api/errors.js';
|
||||
import { pushToast } from '#lib/stores/toast.svelte.js';
|
||||
import { relativeTime } from '#lib/utils/relativeTime.js';
|
||||
import ListContinuation from '#lib/components/ListContinuation.svelte';
|
||||
import type { Download, ImportEvidence } from '#lib/api/types.js';
|
||||
import type { Download, DownloadCondition, ImportEvidence } from '#lib/api/types.js';
|
||||
|
||||
// The Activity tab (M502 #5522): every download Minstrel took, running
|
||||
// ones first. A finished download is checked against MusicBrainz before it
|
||||
// is filed; one the check could not settle waits here for a person, who
|
||||
// imports it anyway or refuses it. A refused release is set aside (the
|
||||
// Blocked tab) and its album searched for again.
|
||||
// Blocked tab) and its album searched for again. A download that fails, or
|
||||
// that nobody shares, is retried and then given up the same way, held back
|
||||
// for a week; an imported one seeds, then is removed (#5523).
|
||||
|
||||
const client = useQueryClient();
|
||||
const downloadsStore = createDownloadsQuery();
|
||||
const downloads = $derived($downloadsStore);
|
||||
const items = $derived((downloads.data?.pages ?? []).flatMap((p) => p.items) as Download[]);
|
||||
const clientsDown = $derived(downloads.data?.pages[0]?.clients_down ?? []);
|
||||
|
||||
let busy = $state<string | null>(null);
|
||||
|
||||
@@ -39,9 +48,25 @@
|
||||
const onRetry = (d: Download) => act(d, retryImport, `${d.name} will be tried again within the minute.`);
|
||||
const onRefuse = (d: Download) =>
|
||||
act(d, refuseDownload, `Refused ${d.name}. It's set aside and the album will be searched for again.`);
|
||||
const onAbandon = (d: Download) =>
|
||||
act(d, abandonDownload, `Gave up on ${d.name}. It's set aside and the album will be searched for again.`);
|
||||
|
||||
// A download still downloading, or waiting to be: one a person may give up.
|
||||
const running = (d: Download) =>
|
||||
!d.ended_at && (d.import_state === '' || d.import_state === 'downloading' || d.import_state === 'waiting');
|
||||
|
||||
// How a download that ended without an import ended.
|
||||
const endings: Record<string, string> = {
|
||||
stalled: 'Given up',
|
||||
failed: 'Failed',
|
||||
abandoned: 'Given up by you',
|
||||
gone: 'Left the client',
|
||||
superseded: 'Replaced'
|
||||
};
|
||||
|
||||
// The state of a download as a short label, and whether it needs a person.
|
||||
function label(d: Download): string {
|
||||
if (d.ended_at && endings[d.end_reason]) return endings[d.end_reason];
|
||||
switch (d.import_state) {
|
||||
case 'downloading':
|
||||
case '':
|
||||
@@ -68,7 +93,7 @@
|
||||
|
||||
function badge(d: Download): string {
|
||||
if (needsYou(d)) return 'border-accent text-text-primary';
|
||||
if (d.import_state === 'refused') return 'border-error-fg text-error-fg';
|
||||
if (d.import_state === 'refused' || (d.ended_at && endings[d.end_reason])) return 'border-error-fg text-error-fg';
|
||||
return 'border-border text-text-secondary';
|
||||
}
|
||||
|
||||
@@ -82,6 +107,26 @@
|
||||
return parts.join(' · ');
|
||||
}
|
||||
|
||||
// What the lifecycle sees wrong with a running download, and what it did.
|
||||
function condition(c: DownloadCondition): string {
|
||||
let said = c.detail;
|
||||
if (c.trigger === 'no_peers' || c.trigger === 'no_metadata') {
|
||||
said += c.retries
|
||||
? ` Asked for again ${c.retries} of 3 times; given up after that.`
|
||||
: ' It will be asked for again three times, then given up.';
|
||||
}
|
||||
return said;
|
||||
}
|
||||
|
||||
// What ended a download that was not imported, in the lifecycle's words.
|
||||
const ending = (d: Download) => (d.ended_at && endings[d.end_reason] ? d.end_detail : '');
|
||||
|
||||
function seeded(d: Download): string | null {
|
||||
if (d.seed_released_at) return `Seeded, then removed from the download client ${relativeTime(d.seed_released_at)}.`;
|
||||
if (d.seeding) return `Seeding: share ratio ${d.ratio.toFixed(2)}.`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function when(d: Download): string {
|
||||
const who = d.origin === 'operator' ? 'by you' : 'by the schedule';
|
||||
let said = `Taken ${who} ${relativeTime(d.grabbed_at)}`;
|
||||
@@ -103,6 +148,17 @@
|
||||
'rounded-md border border-border px-2.5 py-1 text-xs text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50';
|
||||
</script>
|
||||
|
||||
{#each clientsDown as c (c.client)}
|
||||
<p
|
||||
class="mb-3 rounded-lg border border-error-fg bg-surface p-3 text-sm text-text-primary"
|
||||
role="status"
|
||||
data-testid="client-down"
|
||||
>
|
||||
Can't reach the download client {c.client} since {relativeTime(c.since)}: {c.detail}. Its
|
||||
downloads are left as they are until it answers.
|
||||
</p>
|
||||
{/each}
|
||||
|
||||
{#if downloads.isPending}
|
||||
<p class="text-text-secondary">Reading the downloads…</p>
|
||||
{:else if downloads.isError && !downloads.isFetchNextPageError}
|
||||
@@ -147,6 +203,12 @@
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if running(d) && d.condition}
|
||||
<p class="text-xs text-text-primary" data-testid="download-condition">{condition(d.condition)}</p>
|
||||
{/if}
|
||||
{#if ending(d)}
|
||||
<p class="text-xs text-text-secondary" data-testid="download-ending">{ending(d)}</p>
|
||||
{/if}
|
||||
{#if d.import_reason}
|
||||
<p class="text-xs {needsYou(d) ? 'text-text-primary' : 'text-text-secondary'}" data-testid="download-reason">
|
||||
{d.import_reason}
|
||||
@@ -158,12 +220,21 @@
|
||||
{#if filed(d)}
|
||||
<p class="truncate text-xs text-text-secondary" title={d.destination}>{filed(d)}</p>
|
||||
{/if}
|
||||
{#if seeded(d)}
|
||||
<p class="text-xs text-text-muted" data-testid="download-seeding">{seeded(d)}</p>
|
||||
{/if}
|
||||
{#if d.rehomed_from}
|
||||
<p class="text-xs text-text-muted">Taken for {d.rehomed_from}; it turned out to be this album.</p>
|
||||
{/if}
|
||||
<p class="text-xs text-text-muted">{when(d)}</p>
|
||||
|
||||
{#if needsYou(d)}
|
||||
{#if running(d)}
|
||||
<div class="flex flex-wrap gap-1.5 pt-1">
|
||||
<button type="button" class={button} disabled={busy === d.id} onclick={() => onAbandon(d)}>
|
||||
Give up
|
||||
</button>
|
||||
</div>
|
||||
{:else if needsYou(d)}
|
||||
<div class="flex flex-wrap gap-1.5 pt-1">
|
||||
{#if d.import_state === 'review'}
|
||||
<button type="button" class={button} disabled={busy === d.id} onclick={() => onImport(d)}>
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
import { afterEach, describe, expect, test, vi } from 'vitest';
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/svelte';
|
||||
import { mockInfiniteQuery } from '../../../test-utils/query';
|
||||
import type { Download } from '#lib/api/types.js';
|
||||
import type { ClientDown, Download } from '#lib/api/types.js';
|
||||
|
||||
vi.mock('#lib/api/wanted.js', () => ({
|
||||
createDownloadsQuery: vi.fn(),
|
||||
importAnyway: vi.fn().mockResolvedValue(undefined),
|
||||
retryImport: vi.fn().mockResolvedValue(undefined),
|
||||
refuseDownload: vi.fn().mockResolvedValue(undefined)
|
||||
refuseDownload: vi.fn().mockResolvedValue(undefined),
|
||||
abandonDownload: vi.fn().mockResolvedValue(undefined)
|
||||
}));
|
||||
vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() }));
|
||||
|
||||
import DownloadList from './DownloadList.svelte';
|
||||
import { createDownloadsQuery, importAnyway, refuseDownload, retryImport } from '#lib/api/wanted.js';
|
||||
import { abandonDownload, createDownloadsQuery, importAnyway, refuseDownload, retryImport } from '#lib/api/wanted.js';
|
||||
|
||||
const ago = (ms: number) => new Date(Date.now() - ms).toISOString();
|
||||
|
||||
@@ -38,15 +39,22 @@ function download(id: string, extra: Partial<Download> = {}): Download {
|
||||
destination: '',
|
||||
files: [],
|
||||
skipped: 0,
|
||||
end_detail: '',
|
||||
seeding: false,
|
||||
ratio: 0,
|
||||
seed_released_at: null,
|
||||
condition: null,
|
||||
...extra
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => vi.clearAllMocks());
|
||||
|
||||
function setup(items: Download[]) {
|
||||
function setup(items: Download[], clientsDown: ClientDown[] = []) {
|
||||
vi.mocked(createDownloadsQuery).mockReturnValue(
|
||||
mockInfiniteQuery({ pages: [{ items, total: items.length, limit: 50, offset: 0 }] }) as unknown as ReturnType<
|
||||
mockInfiniteQuery({
|
||||
pages: [{ items, total: items.length, limit: 50, offset: 0, clients_down: clientsDown }]
|
||||
}) as unknown as ReturnType<
|
||||
typeof createDownloadsQuery
|
||||
>
|
||||
);
|
||||
@@ -54,13 +62,45 @@ function setup(items: Download[]) {
|
||||
}
|
||||
|
||||
describe('DownloadList', () => {
|
||||
test('a running download shows how far it has got', () => {
|
||||
test('a running download shows how far it has got, and can be given up', async () => {
|
||||
setup([download('d1', { progress: 0.375 })]);
|
||||
expect(screen.getByTestId('download-state').textContent).toMatch(/downloading/i);
|
||||
expect(screen.getByRole('progressbar', { name: /downloaded/i }).getAttribute('aria-valuenow')).toBe('37');
|
||||
expect(screen.queryByTestId('download-condition')).toBeNull();
|
||||
await fireEvent.click(screen.getByRole('button', { name: /give up/i }));
|
||||
await waitFor(() => expect(abandonDownload).toHaveBeenCalledWith('d1'));
|
||||
});
|
||||
|
||||
test('a stalled download says what is wrong and how often it was asked for again', () => {
|
||||
setup([
|
||||
download('d1', {
|
||||
progress: 0.4,
|
||||
condition: { trigger: 'no_peers', detail: 'Nobody has shared it in 7 hours; 40% downloaded.', retries: 2, since: ago(1_200_000) }
|
||||
})
|
||||
]);
|
||||
expect(screen.getByTestId('download-condition').textContent).toMatch(
|
||||
/nobody has shared it in 7 hours; 40% downloaded\. asked for again 2 of 3 times/i
|
||||
);
|
||||
});
|
||||
|
||||
test('a download given up on says why it ended', () => {
|
||||
setup([
|
||||
download('d1', {
|
||||
ended_at: ago(60_000),
|
||||
end_reason: 'stalled',
|
||||
end_detail: 'Taken 7 hours ago and never finished: nobody was sharing it. Held back until 17 Oct.'
|
||||
})
|
||||
]);
|
||||
expect(screen.getByTestId('download-state').textContent).toMatch(/given up/i);
|
||||
expect(screen.getByTestId('download-ending').textContent).toMatch(/nobody was sharing it/);
|
||||
expect(screen.queryByRole('button')).toBeNull();
|
||||
});
|
||||
|
||||
test('a download client that does not answer is named above the list', () => {
|
||||
setup([download('d1')], [{ client: 'qbittorrent', detail: 'connection refused', since: ago(600_000) }]);
|
||||
expect(screen.getByTestId('client-down').textContent).toMatch(/can't reach the download client qbittorrent .*connection refused/i);
|
||||
});
|
||||
|
||||
test('a doubtful download says why and can be imported anyway or refused', async () => {
|
||||
setup([
|
||||
download('d1', {
|
||||
@@ -102,6 +142,17 @@ describe('DownloadList', () => {
|
||||
expect(screen.queryByRole('button')).toBeNull();
|
||||
});
|
||||
|
||||
test('an imported download says how its seeding goes', () => {
|
||||
const imported = { ended_at: ago(60_000), end_reason: 'imported', import_state: 'imported' as const };
|
||||
setup([
|
||||
download('d1', { ...imported, seeding: true, ratio: 0.8 }),
|
||||
download('d2', { ...imported, seeding: true, ratio: 2.1, seed_released_at: ago(60_000) })
|
||||
]);
|
||||
const lines = screen.getAllByTestId('download-seeding').map((e) => e.textContent);
|
||||
expect(lines[0]).toMatch(/seeding: share ratio 0\.80/i);
|
||||
expect(lines[1]).toMatch(/seeded, then removed/i);
|
||||
});
|
||||
|
||||
test('with nothing downloaded it says what lands here', () => {
|
||||
setup([]);
|
||||
expect(screen.getByText(/nothing has been downloaded yet/i)).toBeTruthy();
|
||||
|
||||
@@ -117,7 +117,10 @@
|
||||
said = `${when}: ${s.error}`;
|
||||
}
|
||||
}
|
||||
if (w.grab?.import_state === 'refused') said += `. Last download refused: ${w.grab.import_reason}`;
|
||||
const g = w.grab;
|
||||
if (g?.import_state === 'refused') said += `. Last download refused: ${g.import_reason}`;
|
||||
else if (g && !g.live && ['stalled', 'failed', 'gone'].includes(g.end_reason) && g.end_detail)
|
||||
said += `. Last download given up: ${g.end_detail}`;
|
||||
return said;
|
||||
}
|
||||
|
||||
|
||||
@@ -65,6 +65,7 @@ function grab(extra: Partial<WantGrab> = {}): WantGrab {
|
||||
progress: 0,
|
||||
import_state: '',
|
||||
import_reason: '',
|
||||
end_detail: '',
|
||||
imported_at: null,
|
||||
imported_files: 0,
|
||||
...extra
|
||||
@@ -155,6 +156,16 @@ describe('WantList', () => {
|
||||
expect(screen.getByTestId('want-search').textContent).toMatch(/last download refused: another album/i);
|
||||
});
|
||||
|
||||
test('a download given up on is named beside the search that follows it', () => {
|
||||
setup([
|
||||
want('w1', {
|
||||
search: { searched_at: ago(3_600_000), outcome: 'waiting', considered: 4, best_release: '', best_verdict: '', error: '' },
|
||||
grab: grab({ live: false, end_reason: 'stalled', import_state: 'downloading', end_detail: 'Nobody was sharing it.' })
|
||||
})
|
||||
]);
|
||||
expect(screen.getByTestId('want-search').textContent).toMatch(/last download given up: nobody was sharing it/i);
|
||||
});
|
||||
|
||||
test('a held album says how it arrived', () => {
|
||||
setup(
|
||||
[
|
||||
|
||||
@@ -76,5 +76,7 @@
|
||||
"candidate_not_found": "That release is no longer in the list. Search again.",
|
||||
"no_infohash": "That release can't be set aside: its indexer gave no infohash to know it by.",
|
||||
"want_held": "The library already holds that album.",
|
||||
"block_not_found": "That release isn't set aside any more."
|
||||
"block_not_found": "That release isn't set aside any more.",
|
||||
"not_running": "That download isn't running any more. Refresh the list.",
|
||||
"bad_lifecycle": "Seed ratio is 0 to 100, seed days 0 to 365, and stall hours 0 to 720."
|
||||
}
|
||||
|
||||
@@ -59,7 +59,10 @@ vi.mock('#lib/api/wanted.js', async () => {
|
||||
last_pass: {},
|
||||
import_root: '',
|
||||
// Not "/music": a Lidarr root folder on this page is named that.
|
||||
scan_paths: ['/library']
|
||||
scan_paths: ['/library'],
|
||||
seed_ratio: 2,
|
||||
seed_days: 14,
|
||||
stall_hours: 6
|
||||
}
|
||||
})
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user