minstrel/internal/recommendation/library_scale.go

package recommendation

import (
	"context"
	"math"
	"sync"
	"time"

	"github.com/jackc/pgx/v5/pgtype"

	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)

// Scaling the candidate pool to the library (#3880).
//
// DefaultCandidateSourceLimits returns what its own comment calls "the v1
// hardcoded constants per spec" — ~170 candidates, identical for a 500-track
// library and a 100,000-track one. Operator, 2026-09-10: "is the pool that we
// draw from somehow scaled to the amount of music in the library... my earlier
// understanding of the tuning and work may have been skewed by what was in my
// library."
//
// It was not, and the consequence compounds. The pool samples a shrinking
// FRACTION of the library as it grows — 17% of 1,000 tracks, 1.7% of 10,000,
// 0.17% of 100,000 — so the ceiling on how much of a collection can ever
// surface stays flat while the collection does not. RandomFill, whose entire
// job is exploration, becomes a thinner and noisier slice of a more diverse
// corpus at exactly the moment diversity rises.

const (
	// libraryScaleReference is the library size at which the base limits
	// apply unchanged. Below it nothing scales, so small libraries keep
	// today's behaviour exactly.
	//
	// ASSUMED, NOT MEASURED. The size the v1 constants were actually tuned
	// against is unrecorded; 5,000 is a plausible mid-size library and a
	// deliberately conservative place to start growing. This constant is the
	// single thing to change.
	//
	// Measured on the operator's library, 26,005 tracks (scale 2.28), for the
	// 15 most-played seeds and two Moe Shop tracks (#3880, 2026-10-21):
	//   lb_similar       0–6 edges (3 of 17 seeds have any): never reaches
	//                    even the base limit, so scaling it costs nothing
	//   similar_artists  860–6,307 tracks over 7–33 artists: always full
	//   tag_overlap      4,658–19,658 tracks: always full
	//   likes_overlap    193–497 liked tracks sharing a tag: always full
	//   taste_overlap    20,045 tracks over 174 artists: always full
	//   coplay           0 rows: a single-user server
	// So the arms the scale grows all have far more rows than it asks for, and
	// the tuned-against size cannot be recovered from data; the number stays
	// until a library shows a scaled arm running dry or a pool too slow.
	libraryScaleReference = 5000

	// maxLibraryScale bounds growth so a very large library does not turn
	// every recommendation query into a huge scan. At 4x the pool tops out
	// around 500 candidates, which is still cheap to score in memory.
	maxLibraryScale = 4.0
)

// libraryScale is sqrt rather than linear on purpose. Linear growth would put
// a 100,000-track library at a 3,400-candidate pool — a slow query, slow
// scoring, and far past the point where more candidates improve the answer.
// Square-root growth keeps the pool meaningfully proportional while staying
// bounded: 1x at the reference, 2x at four times it, 4x at sixteen times.
//
// Never returns below 1: the base limits are a floor, not a midpoint. That
// also keeps this safe against #3889 — growing an arm ordered by unseeded
// random() is fine, shrinking one is what breaks same-day determinism.
func libraryScale(libraryTracks int64) float64 {
	if libraryTracks <= libraryScaleReference {
		return 1.0
	}
	f := math.Sqrt(float64(libraryTracks) / float64(libraryScaleReference))
	return math.Min(f, maxLibraryScale)
}

// ScaleForLibrary grows the library-bounded arms of a limit set for a library
// of the given size, and leaves the rest alone.
//
// THE SCALING IS NOT UNIFORM, and that is the substance of it rather than a
// refinement. An arm's limit only matters if there are rows for it to cut off,
// so what an arm is BOUNDED BY decides whether library size can help it:
//
//	scaled     LBSimilar, SimilarArtist, TagOverlap — bounded by similarity
//	           and tag data, which grows as the library does. A bigger library
//	           means more of ListenBrainz's returned MBIDs survive the
//	           local-library filter in similarity/worker.go.
//	scaled     RandomFill — the exploration arm, and the one the complaint is
//	           really about. This is where "fraction of the library" lives.
//	unscaled   LikesOverlap — bounded by the USER's likes.
//	unscaled   UserCoplay — bounded by the INSTANCE's co-play graph.
//	unscaled   TasteOverlap — bounded by the taste profile's artists.
//
// Raising the last three would not sample more of the library; it would
// sample more of a set that did not grow, which is churn rather than reach.
// Leaving TasteOverlap alone has a second benefit: it and RandomFill are the
// two arms carrying sim_score 0, so this does not inflate the
// seed-independent share as fast as a uniform scale would.
func ScaleForLibrary(base CandidateSourceLimits, libraryTracks int64) CandidateSourceLimits {
	f := libraryScale(libraryTracks)
	grow := func(n int) int { return int(float64(n) * f) } // f >= 1, so never shrinks
	return CandidateSourceLimits{
		LBSimilar:     grow(base.LBSimilar),
		SimilarArtist: grow(base.SimilarArtist),
		TagOverlap:    grow(base.TagOverlap),
		RandomFill:    grow(base.RandomFill),
		LikesOverlap:  base.LikesOverlap,
		UserCoplay:    base.UserCoplay,
		TasteOverlap:  base.TasteOverlap,
	}
}

// CountLibraryTracks returns the whole library's track count.
//
// Reuses CountTracksMatching with an empty pattern — `title ILIKE '%%'`
// matches every row — rather than adding a query, because a new one would
// need sqlc regeneration (rule 28, and see #3889 for where that blocks).
// The user id is left invalid so the quarantine anti-join is skipped: this
// number sizes a pool, and a per-user view of it would be false precision.
func CountLibraryTracks(ctx context.Context, q *dbq.Queries) (int64, error) {
	return q.CountTracksMatching(ctx, dbq.CountTracksMatchingParams{
		Column1: "",
		UserID:  pgtype.UUID{}, // invalid → NULL → no quarantine filter
	})
}

// librarySizeTTL is how long a counted size is reused. Library size changes
// only when a scan runs, so minutes are plenty — and the count is a full
// table scan (the ILIKE defeats every index), which is why it is not done
// per request.
const librarySizeTTL = 5 * time.Minute

// librarySizeTimeout bounds the count itself — see loadWithDeadline.
const librarySizeTimeout = 3 * time.Second

// LibrarySize memoises the library track count.
//
// Degrades rather than fails: a count that errors or times out leaves the
// previous value in place, and a zero (never yet counted) scales to the base
// limits — which is exactly today's behaviour. Nothing about sizing a
// candidate pool justifies failing the request it is sizing.
type LibrarySize struct {
	mu   sync.Mutex
	now  func() time.Time // injectable for tests
	at   time.Time
	size int64
}

// NewLibrarySize returns an empty cache. The zero value works too; this
// exists so tests can pin the clock.
func NewLibrarySize(now func() time.Time) *LibrarySize {
	return &LibrarySize{now: now}
}

// Get returns the cached size, refreshing through load when stale.
//
// A NIL RECEIVER IS VALID and means "no cache": the count still runs, it is
// just not memoised. That is deliberate rather than defensive habit. The api
// handlers struct is built directly by a dozen tests that cannot know about
// every field, and a nil here previously panicked inside a radio request —
// turning a missing pool-sizing HINT into a 500. Uncached-but-correct is the
// right failure for something whose whole contract is that it degrades.
func (l *LibrarySize) Get(ctx context.Context, load func(context.Context) (int64, error)) int64 {
	if l == nil {
		n, err := loadWithDeadline(ctx, load)
		if err != nil {
			return 0 // scales to the base limits
		}
		return n
	}

	l.mu.Lock()
	defer l.mu.Unlock()

	now := time.Now
	if l.now != nil {
		now = l.now
	}
	if !l.at.IsZero() && now().Sub(l.at) < librarySizeTTL {
		return l.size
	}

	n, err := loadWithDeadline(ctx, load)
	if err != nil {
		// Keep the last known value and re-try at the next call rather than
		// stamping `at`, so a transient failure does not pin a stale number
		// for the whole TTL.
		return l.size
	}
	l.size, l.at = n, now()
	return l.size
}

// loadWithDeadline bounds the count. Rule 156: the caller is a user waiting
// on a radio, and a pool-sizing hint is never worth hanging for.
func loadWithDeadline(ctx context.Context, load func(context.Context) (int64, error)) (int64, error) {
	cctx, cancel := context.WithTimeout(ctx, librarySizeTimeout)
	defer cancel()
	return load(cctx)
}