minstrel/internal/library/fingerprint.go

package library

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"log/slog"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/jackc/pgx/v5/pgtype"

	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)

// Acoustic identity (M400).
//
// Two values per track, because they answer different questions:
//
//	audio_stream_sha256  a SHA-256 of the ENCODED audio packets. Equal means the
//	                     same audio bytes, whatever the tags or container around
//	                     them say. No threshold and no false positives — this is
//	                     what catches two copies of one MP3 that differ only in
//	                     their ID3 (#3885).
//
//	chromaprint          fpcalc's raw fingerprint. Close means the same
//	                     recording, even at another bitrate or in another codec
//	                     — the case an exact hash cannot see.
//
// Both shell out, in the shape probeDurationMs already set: a deadline on every
// call, and a failure that leaves the value unset rather than failing the file.
// A track with no fingerprint is never a duplicate candidate; it is still a
// track.

// fingerprintTimeout bounds one ffmpeg hash or fpcalc call. Longer than
// probeTimeout because both read the audio rather than a header: the hash reads
// every packet and fpcalc decodes up to its -length. 60s leaves room for a large
// lossless file on a slow network mount; a call needing more is a stall, not a
// big file.
const fingerprintTimeout = 60 * time.Second

// fingerprintWaitDelay bounds how long Output may keep waiting on the tool's
// pipes after the deadline has killed it. Without it, a child that left a
// descendant holding stdout open would block the scan past its own timeout.
const fingerprintWaitDelay = 5 * time.Second

// fingerprintVersion stamps how a track_fingerprints row was derived. Bump it
// whenever the derivation changes — the hash arguments or fpcalc's flags — and
// the backfill re-derives every row below it. Fingerprints taken by two methods
// are not comparable, and nothing else would reveal that the library held a mix.
//
// The length is deliberately not part of it: it is an operator setting (#3913),
// so each row records the length it was taken at and readers compare only rows
// at the current one. See fingerprint_settings.go.
const fingerprintVersion int16 = 1

// errFingerprintTimeout marks a tool that ran out of time. Distinct from a
// failed exit because a stall is a fact about the mount, not about the file.
var errFingerprintTimeout = errors.New("fingerprint tool timed out")

// defaultChromaprintLengthSec is the shipped value of the length setting (#3913):
// how many seconds of audio fpcalc fingerprints. 120 is fpcalc's own default.
const defaultChromaprintLengthSec = 120

// errChromaprintSkipped marks a chromaprint not taken because fingerprinting is
// switched off. Inconclusive rather than a verdict: nothing was learned about
// the file.
var errChromaprintSkipped = errors.New("chromaprint skipped: fingerprinting is off")

// fpcalcStderrTail caps how much of a failing tool's stderr reaches the log.
const fpcalcStderrTail = 512

// streamHashArgs hashes the encoded audio packets, never decoded samples.
//
// -c:a copy is the point, not an optimisation. A decoded hash of a lossy file
// depends on the decoder's float maths and sample conversion, which can move
// between ffmpeg releases — so an image upgrade could silently change every
// stored hash, and yesterday's duplicate would stop matching today's copy.
// Packet bytes do not move. It is also far cheaper: demux only, no decode.
//
// -map 0:a keeps embedded cover art (an attached-picture video stream) out of
// the hash, so two copies of one recording carrying different art still match.
func streamHashArgs(path string) []string {
	return []string{
		"-v", "error",
		"-i", path,
		"-map", "0:a",
		"-c:a", "copy",
		"-f", "hash", "-hash", "sha256",
		"-",
	}
}

// fpcalcArgs asks for the raw fingerprint as SIGNED integers.
//
// -raw because the matcher compares items bit by bit, which the compressed form
// cannot do without being unpacked first. -signed because the column is Postgres
// integer[], which is signed: fpcalc's default prints uint32, and half of those
// values do not fit. Signed output is the same 32 bits with no reinterpretation
// step left to get wrong.
func fpcalcArgs(path string, lengthSec int) []string {
	return []string{
		"-raw", "-signed",
		"-length", strconv.Itoa(lengthSec),
		path,
	}
}

// lookupFingerprintLengthSec is how much audio the AcoustID lookup print
// covers (M401). Fixed at fpcalc's default rather than following the
// duplicate-detection length setting: AcoustID's database is built from
// prints that taggers such as Picard take at the default, and a lookup print
// of a different span matches it less well.
const lookupFingerprintLengthSec = 120

// fpcalcLookupArgs asks for the COMPRESSED fingerprint, fpcalc's default
// output and the string AcoustID's lookup takes. A second decode beside the
// raw print, chosen over re-encoding the stored raw array in Go (M401 D1):
// fpcalc is the reference encoder, while a subtle bug in a port would still
// produce a well-formed string that AcoustID answers with "no match".
func fpcalcLookupArgs(path string) []string {
	return []string{
		"-length", strconv.Itoa(lookupFingerprintLengthSec),
		path,
	}
}

// lookupFingerprint is what an AcoustID lookup sends for one file.
type lookupFingerprint struct {
	fingerprint string
	// durationSec is the whole file's length, not the span fingerprinted.
	durationSec int
}

// fingerprintResult is one attempt at both halves of a track's identity. They
// fail independently: a file ffmpeg can demux may still defeat fpcalc.
type fingerprintResult struct {
	streamSHA256 []byte
	chromaprint  []int32
	hashErr      error
	printErr     error
}

// fingerprintOptions is what the settings decide for one attempt. Captured once
// per file, so the length a chromaprint was taken at is the length stored with it
// even if the setting changes mid-attempt.
type fingerprintOptions struct {
	lengthSec int32
	// chromaprint false takes the stream hash alone: a demux, no decode.
	chromaprint bool
}

// computeFingerprint derives the halves opts asks for, for the file at path.
func computeFingerprint(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
	var r fingerprintResult
	r.streamSHA256, r.hashErr = computeAudioStreamSHA256(ctx, path)
	if !opts.chromaprint {
		r.printErr = errChromaprintSkipped
		return r
	}
	r.chromaprint, r.printErr = computeChromaprint(ctx, path, opts.lengthSec)
	return r
}

// inconclusive reports whether either half failed for a reason that says
// nothing about the file. Such a result must never be stored: stamped at the
// current version it would read as "tried, and this file cannot be
// fingerprinted", and the backfill would never try it again.
func (r fingerprintResult) inconclusive() bool {
	return isInconclusive(r.hashErr) || isInconclusive(r.printErr)
}

// isInconclusive names the failures that are not a verdict on the file: a
// stall, a cancelled scan, a tool that is not installed, and a chromaprint
// skipped because fingerprinting is off. A missing tool matters outside the
// image — a dev binary run without fpcalc on PATH must not stamp every track in
// the library as unfingerprintable.
func isInconclusive(err error) bool {
	return errors.Is(err, errFingerprintTimeout) ||
		errors.Is(err, errChromaprintSkipped) ||
		errors.Is(err, context.Canceled) ||
		errors.Is(err, context.DeadlineExceeded) ||
		errors.Is(err, exec.ErrNotFound)
}

// fingerprintFile runs the scanner's fingerprinter. A Scanner built without New
// gets the real tools rather than a nil-func panic halfway through a scan.
func (s *Scanner) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
	if s.fingerprint == nil {
		return computeFingerprint(ctx, path, opts)
	}
	return s.fingerprint(ctx, path, opts)
}

// fingerprintOutcome is what storeFingerprint did with one attempt.
type fingerprintOutcome int

const (
	outcomeFingerprinted fingerprintOutcome = iota // both halves stored
	outcomeRejected                                // stored with a NULL half: a verdict
	outcomeInconclusive                            // nothing stored; worth trying again
	outcomeStoreFailed                             // the write itself failed
)

// storeFingerprint records one attempt, for the scan (new or changed bytes) and
// the backfill (#3908) alike, so there is one rule for what gets written. It
// never fails its caller: a missing fingerprint only keeps a track out of
// duplicate detection, which is not worth dropping a scan or a pass over.
//
// lengthSec is the length fp's chromaprint was taken at, stored with it (#3913).
func storeFingerprint(
	ctx context.Context, q *dbq.Queries, logger *slog.Logger,
	trackID pgtype.UUID, path string, fp fingerprintResult, lengthSec int32,
) fingerprintOutcome {
	if fp.hashErr != nil {
		logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr)
	}
	if fp.printErr != nil {
		logger.Warn("fingerprint: chromaprint failed", "path", path, "err", fp.printErr)
	}
	if fp.inconclusive() {
		// Any row this track holds describes bytes we could not confirm — the
		// previous bytes for the scan, an older derivation for the backfill.
		// Drop it rather than stamp a failure that says nothing about the file.
		if err := q.DeleteTrackFingerprint(ctx, trackID); err != nil {
			logger.Warn("fingerprint: clearing stale fingerprint failed", "path", path, "err", err)
		}
		return outcomeInconclusive
	}
	// A NULL half here is a verdict — the tool ran and rejected this file — and
	// is stamped at the current version so the backfill does not retry it on
	// every pass. It is retried when the file changes.
	if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
		TrackID:              trackID,
		AudioStreamSha256:    fp.streamSHA256,
		Chromaprint:          fp.chromaprint,
		FingerprintVersion:   fingerprintVersion,
		ChromaprintLengthSec: lengthSec,
	}); err != nil {
		logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err)
		return outcomeStoreFailed
	}
	if fp.hashErr != nil || fp.printErr != nil {
		return outcomeRejected
	}
	return outcomeFingerprinted
}

// computeAudioStreamSHA256 returns the SHA-256 of the file's encoded audio.
func computeAudioStreamSHA256(ctx context.Context, path string) ([]byte, error) {
	out, err := runFingerprintTool(ctx, "ffmpeg", streamHashArgs(path))
	if err != nil {
		return nil, err
	}
	return parseStreamHash(out)
}

// computeChromaprint returns the raw acoustic fingerprint of the first
// lengthSec seconds of the file.
func computeChromaprint(ctx context.Context, path string, lengthSec int32) ([]int32, error) {
	out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, int(lengthSec)))
	if err != nil {
		return nil, err
	}
	return parseFpcalcRaw(out)
}

// runFingerprintTool runs one tool under fingerprintTimeout.
//
// Any non-zero exit is an error, and that deliberately includes fpcalc's exit 3:
// "reading failed, but here is a fingerprint of what I got". A partial
// fingerprint of a damaged file is not that file's identity. Stored, it would
// score against a healthy copy over whatever prefix survived, and could group
// or fail to group either way. Absent is better than wrong.
func runFingerprintTool(ctx context.Context, name string, args []string) ([]byte, error) {
	runCtx, cancel := context.WithTimeout(ctx, fingerprintTimeout)
	defer cancel()

	cmd := exec.CommandContext(runCtx, name, args...)
	cmd.WaitDelay = fingerprintWaitDelay
	out, err := cmd.Output()
	if err == nil {
		return out, nil
	}
	// The caller gave up (a cancelled scan). Report that rather than the
	// signal-killed exit it caused, so it is never mistaken for a verdict on
	// the file.
	if ctx.Err() != nil {
		return nil, fmt.Errorf("%s: %w", name, ctx.Err())
	}
	// Named separately so a stall reads as a stall, not as a crash.
	if errors.Is(runCtx.Err(), context.DeadlineExceeded) {
		return nil, fmt.Errorf("%s: no result within %s: %w", name, fingerprintTimeout, errFingerprintTimeout)
	}
	var exitErr *exec.ExitError
	if errors.As(err, &exitErr) {
		return nil, fmt.Errorf("%s exited %d: %s", name, exitErr.ExitCode(), stderrTail(exitErr.Stderr))
	}
	return nil, fmt.Errorf("%s: %w", name, err)
}

// stderrTail keeps the END of a failing tool's stderr. ffmpeg and fpcalc print
// the actual reason last, after any banner or per-frame warnings, so a cap that
// kept the head would log the noise and drop the cause.
func stderrTail(stderr []byte) []byte {
	stderr = bytes.TrimSpace(stderr)
	if len(stderr) > fpcalcStderrTail {
		stderr = stderr[len(stderr)-fpcalcStderrTail:]
	}
	return stderr
}

// parseStreamHash reads the ffmpeg hash muxer's "SHA256=<hex>" line.
func parseStreamHash(out []byte) ([]byte, error) {
	for _, line := range strings.Split(string(out), "\n") {
		hexed, ok := strings.CutPrefix(strings.TrimSpace(line), "SHA256=")
		if !ok {
			continue
		}
		sum, err := hex.DecodeString(hexed)
		if err != nil {
			return nil, fmt.Errorf("stream hash %q: %w", hexed, err)
		}
		if len(sum) != sha256.Size {
			return nil, fmt.Errorf("stream hash is %d bytes, want %d", len(sum), sha256.Size)
		}
		return sum, nil
	}
	return nil, errors.New("ffmpeg printed no SHA256= line")
}

// parseFpcalcRaw reads fpcalc's text output:
//
//	DURATION=<seconds>
//	FINGERPRINT=<int32>,<int32>,...
func parseFpcalcRaw(out []byte) ([]int32, error) {
	for _, line := range strings.Split(string(out), "\n") {
		list, ok := strings.CutPrefix(strings.TrimSpace(line), "FINGERPRINT=")
		if !ok {
			continue
		}
		if list == "" {
			return nil, errors.New("fpcalc returned an empty fingerprint")
		}
		items := strings.Split(list, ",")
		fp := make([]int32, len(items))
		for i, item := range items {
			// ParseInt at 32 bits, not ParseUint: a value past int32 means the
			// output was unsigned — -signed went missing from the invocation —
			// and nothing downstream would reinterpret it. Refuse it here.
			v, err := strconv.ParseInt(item, 10, 32)
			if err != nil {
				return nil, fmt.Errorf("fingerprint item %d %q: %w", i, item, err)
			}
			fp[i] = int32(v)
		}
		return fp, nil
	}
	return nil, errors.New("fpcalc printed no FINGERPRINT= line")
}

// parseFpcalcCompressed reads fpcalc's default text output:
//
//	DURATION=<whole seconds>
//	FINGERPRINT=<URL-safe base64, unpadded>
//
// The fingerprint is kept as the opaque string fpcalc printed. It is checked
// only for being non-empty and in fpcalc's alphabet, which catches a -raw
// list or a truncated line; whether it is a GOOD print is AcoustID's to judge.
func parseFpcalcCompressed(out []byte) (lookupFingerprint, error) {
	var fp lookupFingerprint
	var sawDuration bool
	for _, line := range strings.Split(string(out), "\n") {
		line = strings.TrimSpace(line)
		if v, ok := strings.CutPrefix(line, "DURATION="); ok {
			d, err := strconv.Atoi(v)
			if err != nil {
				return lookupFingerprint{}, fmt.Errorf("fpcalc duration %q: %w", v, err)
			}
			fp.durationSec, sawDuration = d, true
		}
		if v, ok := strings.CutPrefix(line, "FINGERPRINT="); ok {
			fp.fingerprint = v
		}
	}
	switch {
	case fp.fingerprint == "":
		return lookupFingerprint{}, errors.New("fpcalc printed no fingerprint")
	case strings.IndexFunc(fp.fingerprint, notURLBase64) >= 0:
		return lookupFingerprint{}, fmt.Errorf("fpcalc fingerprint %.20q… is not a compressed print", fp.fingerprint)
	case !sawDuration:
		return lookupFingerprint{}, errors.New("fpcalc printed no DURATION= line")
	// AcoustID refuses a duration that is not a positive integer, and a file
	// under a second long has nothing a lookup could match.
	case fp.durationSec <= 0:
		return lookupFingerprint{}, fmt.Errorf("fpcalc duration %d is too short to look up", fp.durationSec)
	}
	return fp, nil
}

func notURLBase64(r rune) bool {
	switch {
	case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_':
		return false
	}
	return true
}