meta_test.go

package main

import (
	"bytes"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"strings"
	"testing"
)

// localData is used to hold the information about a test file on disk, including its location, metadata size, and frames.
type localData struct {
	name     string
	path     string
	metasize int // (header length + frames length)
	frames   []refFrame
}

// remoteData is used to hold the information about a podcast and one specific episode.
type remoteData struct {
	name   string // podcast name
	url    string // RSS feed URL
	number string // episode number for test
	data   localData
}

// refFrame holds information about an individual frame in the metadata.
type refFrame struct {
	id    string // standard frame ID
	name  string // human-readable frame name
	value string // frame value
}

// We're going to use these generated audio files to test our ability to read metadata.
var localFiles = []localData{
	{"White", "./tests/white.mp3", 10 + 291, []refFrame{
		{"TIT2", "title", "White Title"},
		{"TPE1", "artist", "White Artist"},
		{"TALB", "album", "White Album"},
		{"TRCK", "track", "1"},
		{"TDRC", "date", "1111"}, // ID3v2.4 only
		{"TCON", "genre", "White Noise"},
		{"COMM", "comment", "White noise generated by Audacity"},
		{"TXXX", "Additional Tag", "Additional white noise tag"},
	}},
	{"Pink", "./tests/pink.mp3", 10 + 209, []refFrame{
		{"TIT2", "title", "Pink Title"},
		{"TPE1", "artist", "Pink Artist"},
		{"TALB", "album", "Pink Album"},
		{"TRCK", "track", "2"},
		{"TDRC", "date", "22222"}, // ID3v2.4 only
		{"TCON", "genre", "Country"},
		{"COMM", "comment", "Pink noise"},
		{"TYER", "TYER", "22222"},
		{"TXXX", "Pink", "Noise"},
	}},
	{"Brown", "./tests/brown.mp3", 10 + 117, []refFrame{
		{"TIT2", "title", "Brown Title"},
		{"TPE1", "artist", "Brown Artist"},
		{"TALB", "album", "Brown Album"},
		{"TRCK", "track", "3"},
		{"TDRC", "date", "33"}, // ID3v2.4 only
		{"TYER", "TYER", "33"},
	}},
}

// We're going to use these podcast episodes to test our ability to download an episode and read and write the correct
// metadata. I tried to choose podcasts in which there is a reasonable confidence that the files will remain online for
// a long time and the metadata will not change.
var onlineFiles = []remoteData{
	// This episode uses ID3v2.3 tags.
	{"99% Invisible", "https://feeds.99percentinvisible.org/99percentinvisible", "333",
		localData{"Roman Mars", "333- Mini-Stories- Volume 5.mp3", -1, []refFrame{
			{"TPE1", "artist", "Roman Mars"},
			{"TPE2", "album_artist", "Roman Mars"},
			{"TALB", "album", "99% Invisible"},
			{"TIT2", "title", "333- Mini-Stories: Volume 5"},
			{"TCON", "genre", "Podcast"},
			{"PCST", "podcast", "1"},
			{"TRCK", "track", "333"},
			{"TYER", "year", "2018"},
			{"TDAT", "date", "1812"},
			{"TIME", "time", "2307"},
		}}},

	// This episode uses ID3v2.2 tags and also tests the ability to handle seasons.
	// The pubDate for this episode ends in "GMT".
	{"All Systems Go", "https://anchor.fm/s/f921c24/podcast/rss", "1-10",
		localData{"All Systems Go", "1-10 func 100dofCode(Kofi host, Chris host) challengeResult {.mp3", -1, []refFrame{
			{"TP1", "artist", "Chris Saunders"},
			{"TP2", "album_artist", "Chris Saunders"},
			{"TAL", "album", "All Systems Go"},
			{"TT2", "title", "func 100dofCode(Kofi host, Chris host) challengeResult {"},
			{"TT1", "genre", "Podcast"},
			{"TPA", "season", "1"},
			{"TRK", "track", "10"},
			{"TYE", "year", "2020"},
			{"TDA", "date", "1101"},
			{"TIM", "time", "0530"},
			{"WAF", "url", "https://anchor.fm/s/f921c24/podcast/play/9620184/https%3A%2F%2Fd3ctxlq1ktw2nl.cloudfront.net%2Fproduction%2F2020-0-11%2F42970113-44100-2-d1cd5dae3811c.mp3"},
			{"TSS", "", "Logic Pro X 10.4.8"},
		}}},
}

// Test the ability to read metadata correctly. The mp3 files to read are local files.
func TestReadMeta(t *testing.T) {
	// First, let's make sure that ffprobe is finding the correct metadata in our test files. If that looks good, then
	// we can see how our solution is looking.
	for _, file := range localFiles {
		probeMeta(t, file.name, file.path, file.frames)
	}

	// Now, let's see how our reader stacks up.
	for _, file := range localFiles {
		meta, _, err := splitFile(file.path)
		if err != nil {
			t.Error(file.name, "-", err)
			continue
		}

		if num := readMeta(t, meta, file.frames); num > 0 {
			t.Error(file.name, "-", num, "errors")
			continue
		}
	}
}

// Test the ability to write metadata and files correctly. The files to copy and write are the same files in TestReadMeta.
func TestWriteMeta(t *testing.T) {
	// Read the reference files into memory and write them back out. If they're equal, then the write
	// operation is good.
	for _, file := range localFiles {
		filepath := file.path
		meta, audio, err := splitFile(file.path)
		if err != nil {
			t.Error(file.name, "-", err)
			continue
		}

		// Check that we copied the correct amount of metadata.
		if file.metasize > 0 && len(meta.Bytes()) != file.metasize {
			t.Error(file.name, "- Metadata sizes do not match")
			t.Log("\tExpected:", file.metasize)
			t.Log("\tReceived:", len(meta.Bytes()))
		}

		// If we read the correct amount of metadata out, then the first byte in the audio data should be 0xFF.
		if audio[0] != 0xFF {
			t.Error(file.name, "- Audio data does not start with 0xFF")
		}

		// Test writing everything to disk.
		filepath += "_tmp"
		testWrite(t, file.name, filepath, meta, audio, file.frames)
	}
}

// Test the ability to download and save a podcast episode with the correct file information and metadata.
func TestDownloadEpisode(t *testing.T) {
	for _, podcast := range onlineFiles {
		u, err := url.Parse(podcast.url)
		if err != nil {
			t.Error(podcast.name, "- URL error:", err)
			continue
		}

		// Download the episode.
		show := Show{URL: u}
		if n, _, err := show.Sync("./tests", podcast.number); err != nil {
			t.Error(podcast.name, "- Error syncing:", err)
			continue
		} else if n != 1 {
			t.Error(podcast.name, "- Downloaded", n, "episodes (expected 1)")
			continue
		}

		filepath := path.Join("./tests", podcast.name, podcast.data.path)
		meta, audio, err := splitFile(filepath)
		if err != nil {
			t.Error(podcast.data.name, "-", err)
			continue
		}

		// If we read the correct amount of metadata out, then the first byte in the audio data should be 0xFF.
		if audio[0] != 0xFF {
			t.Error(podcast.data.name, "- Audio data does not start with 0xFF")
			continue
		}

		// Check that we have the correct metadata.
		if num := readMeta(t, meta, podcast.data.frames); num > 0 {
			t.Error(podcast.data.name, "-", num, "errors")
			continue
		}
	}
}

// probeMeta compares the metadata of a file using ffprobe to the expected metadata in the file table. This runs ffprobe
// on the specified file and reads the metadata as key/value pairs. Note that ffprobe does not return the actual tag
// name; it returns a human-readable format. For example, it returns "title" instead of "TIT2".
func probeMeta(t *testing.T, name string, filepath string, frames []refFrame) {
	// Get the frames from ffprobe's output.
	cmd := exec.Command("ffprobe", "-hide_banner", filepath)
	output, err := cmd.CombinedOutput()
	if err != nil {
		t.Error(name, "- Error with ffprobe:", err)
		return
	}

	// To find our metadata, we're going to read everything between the header (Metadata:) and trailer (Duration:).
	start := false
	probedMeta := make(map[string]string)

	lines := strings.Split(string(output), "\n")
	for _, v := range lines {
		if strings.Contains(v, "Metadata:") {
			start = true
		} else if strings.Contains(v, "Duration:") {
			break
		} else if start {
			fields := strings.SplitN(v, ":", 2)
			id := strings.TrimSpace(fields[0])
			value := strings.TrimSpace(fields[1])
			probedMeta[id] = value
		}
	}

	if len(probedMeta) == 0 {
		t.Error(name, "- ffprobe didn't read any metadata")
		return
	}

	for _, frame := range frames {
		want := frame.value
		have := probedMeta[frame.name]
		if want != have {
			t.Error(name, "- Values do not match for frame "+frame.id+" ("+frame.name+")")
			t.Log("\tWant:", want)
			t.Log("\tHave:", have)
		}
		delete(probedMeta, frame.name)
	}
}

// readMeta compares the metadata of a file using our meta reader to the expected metadata in the file table.
func readMeta(t *testing.T, meta *Meta, frames []refFrame) int {
	numErrors := 0

	// Go through all of the known frames and make sure our meta reader found the same values.
	for _, frame := range frames {
		found := false

		// Look for a match in all of the values for this frame ID in the metadata.
		values := meta.GetValues(frame.id)
		if len(values) == 0 {
			t.Error("No values for frame id " + frame.id + " (" + frame.name + ")")
			numErrors++
			continue
		}

		for _, value := range values {
			switch frame.id {
			case "COM", "COMM":
				// If this frame is present, then there are usually 2 instances of it: one that starts with 3 null
				// bytes, and one that starts with three 'X' bytes. Either way, the next byte is a null separator
				// followed by the value.
				value = bytes.TrimLeft(value, string([]byte{0x00, 'X'}))
				if strings.TrimSpace(string(value)) == frame.value {
					found = true
				}
			case "TXX", "TXXX":
				// This is the user-defined field. The frame name and frame value are separated by a null byte.
				fields := bytes.SplitN(value, []byte{0x00}, 2)
				if len(fields) == 2 && string(fields[0]) == frame.name && string(fields[1]) == frame.value {
					found = true
				}
			default:
				if strings.TrimSpace(string(value)) == frame.value {
					found = true
				}
			}

			if found {
				break
			}
		}

		if !found {
			numErrors++
			t.Error("Values do not match for frame " + frame.id + " (" + frame.name + ")")
			t.Log("\tWant:", frame.value)
			for _, value := range values {
				t.Log("\tHave:", string(value))
			}
		}
	}

	return numErrors
}

// testWrite tests the ability to write metadata correctly.
func testWrite(t *testing.T, name string, path string, meta *Meta, audio []byte, frames []refFrame) {
	if !writeData(t, name, path, meta.Build(), audio) {
		t.Error(name, "-", "Error while writing tmp file")
	}

	// Let's use ffprobe to see if all of the metadata was written correctly.
	probeMeta(t, name, path, frames)

	// And then do one more check with our reader.
	newMeta, _, err := splitFile(path)
	if err != nil {
		t.Error(name, "-", err)
	}
	if num := readMeta(t, newMeta, frames); num > 0 {
		t.Error(name, "-", num, "errors")
	}

	// Now that we're done, we can remove the temporary file.
	if err := os.Remove(path); err != nil {
		t.Error(name, "-", err)
	}
}

// writeData writes the metadata and audio data to the specified file.
func writeData(t *testing.T, name string, filepath string, meta, audio []byte) bool {
	file, err := os.Create(filepath)
	if err != nil {
		t.Error(name, "-", err)
		return false
	}
	defer file.Close()

	// Write the metadata first.
	if n, err := file.Write(meta); err != nil {
		t.Error(name, "-", err)
		return false
	} else if n != len(meta) {
		t.Error(name, "- Failed to write correct number of bytes")
		t.Log("\tExpected:", len(meta))
		t.Log("\tActual  :", n)
		return false
	}

	// Then right the audio data.
	if n, err := file.Write(audio); err != nil {
		t.Error(name, "-", err)
		return false
	} else if n != len(audio) {
		t.Error(name, "- Failed to write correct number of bytes")
		t.Log("\tExpected:", len(audio))
		t.Log("\tActual  :", n)
		return false
	}

	return true
}

// splitFile reads the data from the audio file and splits it into metadata and audio data.
func splitFile(path string) (*Meta, []byte, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, nil, err
	}

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	meta := NewMeta(data)
	audio := data[meta.Len():]

	return meta, audio, nil
}