package library import ( "bytes" "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "log/slog" "os/exec" "strconv" "strings" "time" "github.com/jackc/pgx/v5/pgtype" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" ) // Acoustic identity (M400). // // Two values per track, because they answer different questions: // // audio_stream_sha256 a SHA-256 of the ENCODED audio packets. Equal means the // same audio bytes, whatever the tags or container around // them say. No threshold and no false positives — this is // what catches two copies of one MP3 that differ only in // their ID3 (#3885). // // chromaprint fpcalc's raw fingerprint. Close means the same // recording, even at another bitrate or in another codec // — the case an exact hash cannot see. // // Both shell out, in the shape probeDurationMs already set: a deadline on every // call, and a failure that leaves the value unset rather than failing the file. // A track with no fingerprint is never a duplicate candidate; it is still a // track. // fingerprintTimeout bounds one ffmpeg hash or fpcalc call. Longer than // probeTimeout because both read the audio rather than a header: the hash reads // every packet and fpcalc decodes up to its -length. 60s leaves room for a large // lossless file on a slow network mount; a call needing more is a stall, not a // big file. const fingerprintTimeout = 60 * time.Second // fingerprintWaitDelay bounds how long Output may keep waiting on the tool's // pipes after the deadline has killed it. Without it, a child that left a // descendant holding stdout open would block the scan past its own timeout. const fingerprintWaitDelay = 5 * time.Second // fingerprintVersion stamps how a track_fingerprints row was derived. Bump it // whenever the derivation changes — the hash arguments, fpcalc's flags or its // length — and the backfill re-derives every row below it. Fingerprints taken // by two methods are not comparable, and nothing else would reveal that the // library held a mix. const fingerprintVersion int16 = 1 // errFingerprintTimeout marks a tool that ran out of time. Distinct from a // failed exit because a stall is a fact about the mount, not about the file. var errFingerprintTimeout = errors.New("fingerprint tool timed out") // defaultChromaprintLengthSec is how many seconds of audio fpcalc fingerprints. // 120 is fpcalc's own default. Fingerprints taken at different lengths are not // comparable, so changing this has to re-derive every stored one. const defaultChromaprintLengthSec = 120 // fpcalcStderrTail caps how much of a failing tool's stderr reaches the log. const fpcalcStderrTail = 512 // streamHashArgs hashes the encoded audio packets, never decoded samples. // // -c:a copy is the point, not an optimisation. A decoded hash of a lossy file // depends on the decoder's float maths and sample conversion, which can move // between ffmpeg releases — so an image upgrade could silently change every // stored hash, and yesterday's duplicate would stop matching today's copy. // Packet bytes do not move. It is also far cheaper: demux only, no decode. // // -map 0:a keeps embedded cover art (an attached-picture video stream) out of // the hash, so two copies of one recording carrying different art still match. func streamHashArgs(path string) []string { return []string{ "-v", "error", "-i", path, "-map", "0:a", "-c:a", "copy", "-f", "hash", "-hash", "sha256", "-", } } // fpcalcArgs asks for the raw fingerprint as SIGNED integers. // // -raw because the matcher compares items bit by bit, which the compressed form // cannot do without being unpacked first. -signed because the column is Postgres // integer[], which is signed: fpcalc's default prints uint32, and half of those // values do not fit. Signed output is the same 32 bits with no reinterpretation // step left to get wrong. func fpcalcArgs(path string, lengthSec int) []string { return []string{ "-raw", "-signed", "-length", strconv.Itoa(lengthSec), path, } } // fingerprintResult is one attempt at both halves of a track's identity. They // fail independently: a file ffmpeg can demux may still defeat fpcalc. type fingerprintResult struct { streamSHA256 []byte chromaprint []int32 hashErr error printErr error } // computeFingerprint derives both halves for the file at path. func computeFingerprint(ctx context.Context, path string) fingerprintResult { var r fingerprintResult r.streamSHA256, r.hashErr = computeAudioStreamSHA256(ctx, path) r.chromaprint, r.printErr = computeChromaprint(ctx, path, defaultChromaprintLengthSec) return r } // inconclusive reports whether either half failed for a reason that says // nothing about the file. Such a result must never be stored: stamped at the // current version it would read as "tried, and this file cannot be // fingerprinted", and the backfill would never try it again. func (r fingerprintResult) inconclusive() bool { return isInconclusive(r.hashErr) || isInconclusive(r.printErr) } // isInconclusive names the failures that are not a verdict on the file: a // stall, a cancelled scan, and a tool that is not installed. The last matters // outside the image — a dev binary run without fpcalc on PATH must not stamp // every track in the library as unfingerprintable. func isInconclusive(err error) bool { return errors.Is(err, errFingerprintTimeout) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, exec.ErrNotFound) } // fingerprintFile runs the scanner's fingerprinter. A Scanner built without New // gets the real tools rather than a nil-func panic halfway through a scan. func (s *Scanner) fingerprintFile(ctx context.Context, path string) fingerprintResult { if s.fingerprint == nil { return computeFingerprint(ctx, path) } return s.fingerprint(ctx, path) } // fingerprintOutcome is what storeFingerprint did with one attempt. type fingerprintOutcome int const ( outcomeFingerprinted fingerprintOutcome = iota // both halves stored outcomeRejected // stored with a NULL half: a verdict outcomeInconclusive // nothing stored; worth trying again outcomeStoreFailed // the write itself failed ) // storeFingerprint records one attempt, for the scan (new or changed bytes) and // the backfill (#3908) alike, so there is one rule for what gets written. It // never fails its caller: a missing fingerprint only keeps a track out of // duplicate detection, which is not worth dropping a scan or a pass over. func storeFingerprint( ctx context.Context, q *dbq.Queries, logger *slog.Logger, trackID pgtype.UUID, path string, fp fingerprintResult, ) fingerprintOutcome { if fp.hashErr != nil { logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr) } if fp.printErr != nil { logger.Warn("fingerprint: chromaprint failed", "path", path, "err", fp.printErr) } if fp.inconclusive() { // Any row this track holds describes bytes we could not confirm — the // previous bytes for the scan, an older derivation for the backfill. // Drop it rather than stamp a failure that says nothing about the file. if err := q.DeleteTrackFingerprint(ctx, trackID); err != nil { logger.Warn("fingerprint: clearing stale fingerprint failed", "path", path, "err", err) } return outcomeInconclusive } // A NULL half here is a verdict — the tool ran and rejected this file — and // is stamped at the current version so the backfill does not retry it on // every pass. It is retried when the file changes. if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{ TrackID: trackID, AudioStreamSha256: fp.streamSHA256, Chromaprint: fp.chromaprint, FingerprintVersion: fingerprintVersion, }); err != nil { logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err) return outcomeStoreFailed } if fp.hashErr != nil || fp.printErr != nil { return outcomeRejected } return outcomeFingerprinted } // computeAudioStreamSHA256 returns the SHA-256 of the file's encoded audio. func computeAudioStreamSHA256(ctx context.Context, path string) ([]byte, error) { out, err := runFingerprintTool(ctx, "ffmpeg", streamHashArgs(path)) if err != nil { return nil, err } return parseStreamHash(out) } // computeChromaprint returns the raw acoustic fingerprint of the first // lengthSec seconds of the file. func computeChromaprint(ctx context.Context, path string, lengthSec int) ([]int32, error) { out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, lengthSec)) if err != nil { return nil, err } return parseFpcalcRaw(out) } // runFingerprintTool runs one tool under fingerprintTimeout. // // Any non-zero exit is an error, and that deliberately includes fpcalc's exit 3: // "reading failed, but here is a fingerprint of what I got". A partial // fingerprint of a damaged file is not that file's identity. Stored, it would // score against a healthy copy over whatever prefix survived, and could group // or fail to group either way. Absent is better than wrong. func runFingerprintTool(ctx context.Context, name string, args []string) ([]byte, error) { runCtx, cancel := context.WithTimeout(ctx, fingerprintTimeout) defer cancel() cmd := exec.CommandContext(runCtx, name, args...) cmd.WaitDelay = fingerprintWaitDelay out, err := cmd.Output() if err == nil { return out, nil } // The caller gave up (a cancelled scan). Report that rather than the // signal-killed exit it caused, so it is never mistaken for a verdict on // the file. if ctx.Err() != nil { return nil, fmt.Errorf("%s: %w", name, ctx.Err()) } // Named separately so a stall reads as a stall, not as a crash. if errors.Is(runCtx.Err(), context.DeadlineExceeded) { return nil, fmt.Errorf("%s: no result within %s: %w", name, fingerprintTimeout, errFingerprintTimeout) } var exitErr *exec.ExitError if errors.As(err, &exitErr) { return nil, fmt.Errorf("%s exited %d: %s", name, exitErr.ExitCode(), stderrTail(exitErr.Stderr)) } return nil, fmt.Errorf("%s: %w", name, err) } // stderrTail keeps the END of a failing tool's stderr. ffmpeg and fpcalc print // the actual reason last, after any banner or per-frame warnings, so a cap that // kept the head would log the noise and drop the cause. func stderrTail(stderr []byte) []byte { stderr = bytes.TrimSpace(stderr) if len(stderr) > fpcalcStderrTail { stderr = stderr[len(stderr)-fpcalcStderrTail:] } return stderr } // parseStreamHash reads the ffmpeg hash muxer's "SHA256=" line. func parseStreamHash(out []byte) ([]byte, error) { for _, line := range strings.Split(string(out), "\n") { hexed, ok := strings.CutPrefix(strings.TrimSpace(line), "SHA256=") if !ok { continue } sum, err := hex.DecodeString(hexed) if err != nil { return nil, fmt.Errorf("stream hash %q: %w", hexed, err) } if len(sum) != sha256.Size { return nil, fmt.Errorf("stream hash is %d bytes, want %d", len(sum), sha256.Size) } return sum, nil } return nil, errors.New("ffmpeg printed no SHA256= line") } // parseFpcalcRaw reads fpcalc's text output: // // DURATION= // FINGERPRINT=,,... func parseFpcalcRaw(out []byte) ([]int32, error) { for _, line := range strings.Split(string(out), "\n") { list, ok := strings.CutPrefix(strings.TrimSpace(line), "FINGERPRINT=") if !ok { continue } if list == "" { return nil, errors.New("fpcalc returned an empty fingerprint") } items := strings.Split(list, ",") fp := make([]int32, len(items)) for i, item := range items { // ParseInt at 32 bits, not ParseUint: a value past int32 means the // output was unsigned — -signed went missing from the invocation — // and nothing downstream would reinterpret it. Refuse it here. v, err := strconv.ParseInt(item, 10, 32) if err != nil { return nil, fmt.Errorf("fingerprint item %d %q: %w", i, item, err) } fp[i] = int32(v) } return fp, nil } return nil, errors.New("fpcalc printed no FINGERPRINT= line") }