feat: download lifecycle — stalled and failed downloads are given up and searched again; imported ones seed, then go (M502 #5523)
release / govulncheck (push) Successful in 17s
release / web (push) Failing after 17s
release / go (push) Successful in 1m11s
release / integration (push) Successful in 4m50s
release / android (push) Successful in 4m54s
release / Build signed APK (releases and dev) (push) Successful in 5m9s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 17s
release / web (push) Failing after 17s
release / go (push) Successful in 1m11s
release / integration (push) Successful in 4m50s
release / android (push) Successful in 4m54s
release / Build signed APK (releases and dev) (push) Successful in 5m9s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
Ported from Fabled Librarian's System 104. A new lifecycle poller, every ten minutes: - a download the client fails is given up at once; - one nobody shares, or still fetching its file list, past the stall setting (6 h) is asked for again three times, then given up; - giving up removes the download with its files, holds the release back for a week (release_blocks.expires_at), ends the grab and searches the album again, with no limit per album; - a running download the client stops listing for half an hour ends and is searched again, not held back; an empty list is not taken as gone; - an imported download seeds to ratio 2.0 or 14 days, then is removed; - a download the client would not remove is asked again every pass. Every condition and act is one open row per (client, download, condition) in download_interventions. Activity shows what is wrong with a download and how often it was retried, why one ended, its seeding, and a banner for a client that does not answer. A running download can be given up by hand (set aside for good). The seed ratio, seed days and stall hours are on the search settings card. The download client lookup moves to acquire.DownloadClientFor, shared by the importer and the lifecycle. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/importer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle"
|
||||
)
|
||||
|
||||
// Downloads and their imports (M502 #5522): the Wanted view's Activity tab,
|
||||
@@ -40,6 +41,24 @@ type downloadView struct {
|
||||
Files []string `json:"files"`
|
||||
Skipped int `json:"skipped"`
|
||||
RehomedFrom string `json:"rehomed_from,omitempty"`
|
||||
// EndDetail says why an ended download ended (#5523).
|
||||
EndDetail string `json:"end_detail"`
|
||||
// Seeding: imported and still sharing; Ratio its share ratio as last
|
||||
// read; SeedReleasedAt when it was removed once it had shared enough.
|
||||
Seeding bool `json:"seeding"`
|
||||
Ratio float32 `json:"ratio"`
|
||||
SeedReleasedAt *string `json:"seed_released_at"`
|
||||
// Condition is what is wrong with it now, if anything.
|
||||
Condition *conditionView `json:"condition"`
|
||||
}
|
||||
|
||||
// conditionView is an open row of the intervention ledger: what was seen,
|
||||
// since when, and how many times it was retried.
|
||||
type conditionView struct {
|
||||
Trigger string `json:"trigger"`
|
||||
Detail string `json:"detail"`
|
||||
Retries int32 `json:"retries"`
|
||||
Since string `json:"since"`
|
||||
}
|
||||
|
||||
func downloadFrom(g dbq.ListGrabsRow) downloadView {
|
||||
@@ -48,6 +67,18 @@ func downloadFrom(g dbq.ListGrabsRow) downloadView {
|
||||
Origin: g.Origin, GrabbedAt: formatTimestamp(g.GrabbedAt), EndReason: g.EndReason,
|
||||
Progress: g.Progress, ImportState: g.ImportState, ImportReason: g.ImportReason,
|
||||
ImportedFiles: g.ImportedFiles, Destination: g.Destination, Files: []string{},
|
||||
EndDetail: g.EndDetail, Seeding: g.Seeding, Ratio: g.Ratio,
|
||||
}
|
||||
if g.SeedReleasedAt.Valid {
|
||||
at := formatTimestamp(g.SeedReleasedAt)
|
||||
v.SeedReleasedAt = &at
|
||||
}
|
||||
if g.Condition != nil {
|
||||
v.Condition = &conditionView{Trigger: *g.Condition, Detail: deref(g.ConditionDetail),
|
||||
Since: formatTimestamp(g.ConditionSince)}
|
||||
if g.ConditionRetries != nil {
|
||||
v.Condition.Retries = *g.ConditionRetries
|
||||
}
|
||||
}
|
||||
if g.WantID.Valid {
|
||||
id := uuidToString(g.WantID)
|
||||
@@ -82,6 +113,14 @@ type downloadsPage struct {
|
||||
Items []downloadView `json:"items"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
// ClientsDown are the download clients not answering now (#5523).
|
||||
ClientsDown []clientDownView `json:"clients_down"`
|
||||
}
|
||||
|
||||
type clientDownView struct {
|
||||
Client string `json:"client"`
|
||||
Detail string `json:"detail"`
|
||||
Since string `json:"since"`
|
||||
}
|
||||
|
||||
// handleListDownloads implements GET /api/admin/downloads?offset=: running
|
||||
@@ -97,13 +136,47 @@ func (h *handlers) handleListDownloads(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrWithLog(w, h.logger, "admin downloads: list", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset}
|
||||
down, err := dbq.New(h.pool).ListClientsDown(r.Context())
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin downloads: clients down", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset,
|
||||
ClientsDown: make([]clientDownView, 0, len(down))}
|
||||
for _, g := range rows {
|
||||
out.Items = append(out.Items, downloadFrom(g))
|
||||
}
|
||||
for _, c := range down {
|
||||
out.ClientsDown = append(out.ClientsDown, clientDownView{Client: c.Client, Detail: c.Detail,
|
||||
Since: formatTimestamp(c.FirstSeen)})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// handleAbandonDownload implements POST /api/admin/downloads/{id}/abandon: a
|
||||
// running download is given up, its release set aside, the download removed
|
||||
// and the album searched again.
|
||||
func (h *handlers) handleAbandonDownload(w http.ResponseWriter, r *http.Request) {
|
||||
admin, ok := requireUser(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
id, ok := requireURLUUID(w, r, "id")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if _, err := h.lifecycle.Abandon(r.Context(), id, admin.ID); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, pgx.ErrNoRows), errors.Is(err, lifecycle.ErrNotAbandonable):
|
||||
writeErr(w, apierror.Conflict("not_running", "that download is no longer running"))
|
||||
default:
|
||||
writeErrWithLog(w, h.logger, "admin downloads: abandon", apierror.Internal(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleImportAnyway implements POST /api/admin/downloads/{id}/import: a
|
||||
// download the check held for a person (or one that failed) is imported on
|
||||
// the next pass whatever the check says.
|
||||
@@ -160,13 +233,16 @@ func (h *handlers) writeImportErr(w http.ResponseWriter, what string, err error)
|
||||
}
|
||||
|
||||
type releaseBlockView struct {
|
||||
InfoHash string `json:"info_hash"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Detail string `json:"detail"`
|
||||
BlockedAt string `json:"blocked_at"`
|
||||
WantTitle string `json:"want_title"`
|
||||
WantArtist string `json:"want_artist"`
|
||||
InfoHash string `json:"info_hash"`
|
||||
Name string `json:"name"`
|
||||
Reason string `json:"reason"`
|
||||
Detail string `json:"detail"`
|
||||
BlockedAt string `json:"blocked_at"`
|
||||
// ExpiresAt is when a release given up on may be taken again; null is
|
||||
// for good (#5523).
|
||||
ExpiresAt *string `json:"expires_at"`
|
||||
WantTitle string `json:"want_title"`
|
||||
WantArtist string `json:"want_artist"`
|
||||
}
|
||||
|
||||
type releaseBlocksPage struct {
|
||||
@@ -193,6 +269,10 @@ func (h *handlers) handleListReleaseBlocks(w http.ResponseWriter, r *http.Reques
|
||||
for _, b := range rows {
|
||||
v := releaseBlockView{InfoHash: b.InfoHash, Name: b.Name, Reason: b.Reason, Detail: b.Detail,
|
||||
BlockedAt: formatTimestamp(b.BlockedAt)}
|
||||
if b.ExpiresAt.Valid {
|
||||
at := formatTimestamp(b.ExpiresAt)
|
||||
v.ExpiresAt = &at
|
||||
}
|
||||
if b.WantTitle != nil {
|
||||
v.WantTitle = *b.WantTitle
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user