package library import ( "encoding/binary" "errors" "io" "strings" "unicode/utf16" ) // Why this file exists at all: github.com/dhowden/tag reads every other field // we need correctly, but its text-frame reader destroys multi-value frames. // readTFrame does // // strings.Join(strings.Split(txt, string(singleZero)), "") // // — it splits on the ID3v2 null separator and rejoins with the EMPTY string, so // a file tagged "Alternative Rock" + "Rock" comes back as the single token // "Alternative RockRock" (#2499). We stored that verbatim, which corrupted the // genre browse axis and polluted the taste profile's tag vocabulary. // // ffprobe is not an escape hatch either: ffmpeg's read_ttag calls decode_str // exactly once with no loop, so it keeps only the FIRST value and silently // discards the rest. Truncating multi-genre tags would blunt genre similarity, // which is the main thing genre feeds. // // So the TCON frame is parsed here directly. Only the genre frame — everything // else still comes from dhowden/tag, which handles it fine. // maxID3TagSize caps how much of a file we'll buffer looking for TCON. Real // tags are kilobytes; embedded cover art pushes them to a few megabytes. The // cap exists so a corrupt or hostile size field can't make the scanner // allocate wildly on a file it was only asked to index. const maxID3TagSize = 16 << 20 // errNoGenreFrame means the file carries no readable genre frame. It is an // expected outcome (plenty of files are untagged), not a failure. var errNoGenreFrame = errors.New("library: no ID3v2 genre frame") // readID3v2GenreValues returns the raw, still-unnormalised values of the ID3v2 // genre frame — one entry per value the tag actually declares. Numeric ID3v1 // references are left alone here; normaliseGenreValue resolves them. // // rs is seeked to the start, so it is safe to call after dhowden/tag has // already consumed the reader. func readID3v2GenreValues(rs io.ReadSeeker) ([]string, error) { if _, err := rs.Seek(0, io.SeekStart); err != nil { return nil, err } var hdr [10]byte if _, err := io.ReadFull(rs, hdr[:]); err != nil { return nil, errNoGenreFrame } if string(hdr[0:3]) != "ID3" { return nil, errNoGenreFrame } major := hdr[3] // 2.2, 2.3 and 2.4 are the versions in the wild. A future 2.5 would very // likely move the frame layout, so refuse rather than misparse it. if major < 2 || major > 4 { return nil, errNoGenreFrame } tagFlags := hdr[5] size := syncsafeInt(hdr[6:10]) if size <= 0 || size > maxID3TagSize { return nil, errNoGenreFrame } body := make([]byte, size) if _, err := io.ReadFull(rs, body); err != nil { // A truncated tag is still worth parsing as far as it goes — frame // walking stops cleanly at the end of what we managed to read. return nil, errNoGenreFrame } // 2.2 used flag 0x40 for whole-tag compression with a scheme that was // never actually specified. Nothing can read those. if major == 2 && tagFlags&0x40 != 0 { return nil, errNoGenreFrame } if tagFlags&0x80 != 0 { // Whole-tag unsynchronisation (2.2/2.3). 2.4 moved this per-frame, but // some writers still set it at tag level, and undoing it twice is // harmless: after the first pass no 0xFF 0x00 pairs remain. body = undoUnsynchronisation(body) } if major >= 3 && tagFlags&0x40 != 0 { var ok bool if body, ok = skipExtendedHeader(body, major); !ok { return nil, errNoGenreFrame } } return findGenreFrame(body, major) } // findGenreFrame walks the frame list and decodes the genre frame's values. func findGenreFrame(body []byte, major byte) ([]string, error) { // 2.2 frames: 3-byte id + 3-byte size, no flags. 2.3/2.4: 4-byte id + // 4-byte size + 2-byte flags. The size field is the other difference that // matters — see frameSize. idLen, sizeLen, flagLen := 4, 4, 2 wantID := "TCON" if major == 2 { idLen, sizeLen, flagLen = 3, 3, 0 wantID = "TCO" } hdrLen := idLen + sizeLen + flagLen for off := 0; off+hdrLen <= len(body); { id := string(body[off : off+idLen]) // A zero byte where a frame id belongs means we've reached the padding // that fills out the tag. Everything after it is zeros. if body[off] == 0 { break } size := frameSize(body[off+idLen:off+idLen+sizeLen], major) if size <= 0 || off+hdrLen+size > len(body) { // Bogus length — we can't trust any offset past this point. break } if id == wantID { var flags uint16 if flagLen == 2 { flags = binary.BigEndian.Uint16(body[off+idLen+sizeLen : off+hdrLen]) } data, ok := frameData(body[off+hdrLen:off+hdrLen+size], major, flags) if !ok { return nil, errNoGenreFrame } return decodeTextValues(data), nil } off += hdrLen + size } return nil, errNoGenreFrame } // frameSize decodes a frame's length field. 2.4 made it syncsafe (7 bits per // byte); 2.2 and 2.3 are plain big-endian. Reading a 2.3 size as syncsafe (or // the reverse) yields a plausible-looking wrong offset rather than an obvious // error, which is exactly how frame-walking bugs go unnoticed. func frameSize(b []byte, major byte) int { switch { case major == 2: return int(b[0])<<16 | int(b[1])<<8 | int(b[2]) case major == 3: n := binary.BigEndian.Uint32(b) if n > maxID3TagSize { return -1 } return int(n) default: return syncsafeInt(b) } } // frameData strips per-frame wrappers and reports whether the payload is // readable at all. Compressed and encrypted frames are not (we have no // zlib-in-frame or key handling, and neither is meaningful for a genre tag). func frameData(data []byte, major byte, flags uint16) ([]byte, bool) { if major == 3 { // 2.3 flags: %abc00000 %ijk00000 — i compression, j encryption, // k grouping. if flags&0x0080 != 0 || flags&0x0040 != 0 { return nil, false } if flags&0x0020 != 0 { if len(data) < 1 { return nil, false } data = data[1:] // group identifier } return data, true } if major == 4 { // 2.4 flags: %0abc0000 %0h00kmnp — h grouping, k compression, // m encryption, n unsynchronisation, p data-length indicator. if flags&0x0008 != 0 || flags&0x0004 != 0 { return nil, false } if flags&0x0040 != 0 { if len(data) < 1 { return nil, false } data = data[1:] } if flags&0x0001 != 0 { if len(data) < 4 { return nil, false } data = data[4:] // syncsafe expanded size; we don't need it } if flags&0x0002 != 0 { data = undoUnsynchronisation(data) } return data, true } return data, true // 2.2 has no frame flags } // decodeTextValues splits a text frame's payload into its individual values and // decodes each according to the frame's encoding byte. // // This is the whole point of the file: ID3v2 separates multiple values in one // text frame with a null, and that separator is two bytes wide for the UTF-16 // encodings. Splitting a UTF-16 payload on single nulls would cut every ASCII // character in half. func decodeTextValues(data []byte) []string { if len(data) == 0 { return nil } encoding := data[0] payload := data[1:] switch encoding { case 0: // ISO-8859-1 return mapChunks(splitOnNul(payload, 1), decodeLatin1) case 3: // UTF-8 return mapChunks(splitOnNul(payload, 1), func(b []byte) string { return string(b) }) case 1, 2: // UTF-16 with BOM / UTF-16BE without chunks := splitOnNul(payload, 2) // Encoding 2 is big-endian by definition. Encoding 1 carries a byte // order mark, which the spec says must appear on EVERY value in a // multi-value frame — but writers that emit one only on the first value // are common. Take the first BOM found as the default for values that // lack their own, otherwise everything after the first value decodes // byte-swapped into CJK gibberish. defaultBE := true if encoding == 1 { for _, c := range chunks { if be, ok := bomOrder(c); ok { defaultBE = be break } } } out := make([]string, 0, len(chunks)) for _, c := range chunks { be := defaultBE if encoding == 1 { if o, ok := bomOrder(c); ok { be, c = o, c[2:] } } if s := strings.TrimSpace(decodeUTF16(c, be)); s != "" { out = append(out, s) } } return out default: // Unknown encoding byte. Treating it as Latin-1 recovers ASCII text, // which is better than dropping the frame. return mapChunks(splitOnNul(payload, 1), decodeLatin1) } } // splitOnNul splits on a null of the given width, honouring alignment so a // 2-byte-wide separator can't match across a character boundary. func splitOnNul(b []byte, width int) [][]byte { var out [][]byte start := 0 for i := 0; i+width <= len(b); i += width { if !isNul(b[i : i+width]) { continue } out = append(out, b[start:i]) start = i + width } if start < len(b) { out = append(out, b[start:]) } return out } func isNul(b []byte) bool { for _, c := range b { if c != 0 { return false } } return true } func mapChunks(chunks [][]byte, decode func([]byte) string) []string { out := make([]string, 0, len(chunks)) for _, c := range chunks { if s := strings.TrimSpace(decode(c)); s != "" { out = append(out, s) } } return out } // decodeLatin1 widens ISO-8859-1 bytes to runes. A plain string() conversion // would treat the bytes as UTF-8 and mangle every accented character. func decodeLatin1(b []byte) string { runes := make([]rune, len(b)) for i, c := range b { runes[i] = rune(c) } return string(runes) } // bomOrder reports the byte order a UTF-16 byte-order mark declares, and // whether one is present at all. func bomOrder(b []byte) (bigEndian, ok bool) { if len(b) < 2 { return false, false } switch { case b[0] == 0xFE && b[1] == 0xFF: return true, true case b[0] == 0xFF && b[1] == 0xFE: return false, true } return false, false } // decodeUTF16 decodes UTF-16 code units in the given byte order. Any BOM has // already been consumed by the caller. func decodeUTF16(b []byte, bigEndian bool) string { if len(b) < 2 { return "" } units := make([]uint16, 0, len(b)/2) for i := 0; i+1 < len(b); i += 2 { if bigEndian { units = append(units, uint16(b[i])<<8|uint16(b[i+1])) } else { units = append(units, uint16(b[i+1])<<8|uint16(b[i])) } } return string(utf16.Decode(units)) } // skipExtendedHeader advances past the optional extended header. The two // versions disagree about whether the size field counts itself, which is worth // spelling out because getting it wrong offsets the entire frame list by four // bytes and makes every frame id look like padding. func skipExtendedHeader(body []byte, major byte) ([]byte, bool) { if len(body) < 4 { return nil, false } if major == 3 { // 2.3: size EXCLUDES the four size bytes themselves. size := int(binary.BigEndian.Uint32(body[0:4])) if size < 0 || 4+size > len(body) { return nil, false } return body[4+size:], true } // 2.4: syncsafe size INCLUDING the size bytes. size := syncsafeInt(body[0:4]) if size < 4 || size > len(body) { return nil, false } return body[size:], true } // syncsafeInt decodes a 4-byte synchsafe integer (7 significant bits per byte). func syncsafeInt(b []byte) int { if len(b) < 4 { return -1 } // A set high bit means this isn't a valid synchsafe integer. Some writers // emit a plain big-endian size here; refusing is safer than silently // dropping bits and walking to a wrong offset. for _, c := range b[:4] { if c&0x80 != 0 { return -1 } } return int(b[0])<<21 | int(b[1])<<14 | int(b[2])<<7 | int(b[3]) } // undoUnsynchronisation collapses the 0xFF 0x00 pairs that unsynchronisation // inserts to stop a tag from looking like an MPEG frame sync. func undoUnsynchronisation(b []byte) []byte { out := make([]byte, 0, len(b)) for i := 0; i < len(b); i++ { out = append(out, b[i]) if b[i] == 0xFF && i+1 < len(b) && b[i+1] == 0x00 { i++ } } return out }