diff options
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | README.md | 102 | ||||
| -rw-r--r-- | go.mod | 7 | ||||
| -rw-r--r-- | go.sum | 27 | ||||
| -rw-r--r-- | internal/feed/feed.go | 271 | ||||
| -rw-r--r-- | internal/feed/feed_test.go | 343 | ||||
| -rw-r--r-- | internal/fetch/fetch.go | 185 | ||||
| -rw-r--r-- | internal/fetch/fetch_test.go | 144 | ||||
| -rw-r--r-- | internal/serve/serve.go | 91 | ||||
| -rw-r--r-- | internal/serve/serve_test.go | 145 | ||||
| -rw-r--r-- | internal/tag/tag.go | 132 | ||||
| -rw-r--r-- | internal/tag/tag_test.go | 158 | ||||
| -rw-r--r-- | main.go | 319 |
13 files changed, 1928 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..caf9a4b --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +podkit +audio/ +*.mp3 +feed.xml diff --git a/README.md b/README.md new file mode 100644 index 0000000..6157744 --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +# podkit + +Turns a playlist or video URL into a private podcast feed you can subscribe to +from any podcast app. One Go binary; the fetch step shells out to `yt-dlp`. + +``` +podkit all "<playlist url>" \ + --dir ./audio \ + --title "Character of Physical Law" \ + --artist "Richard Feynman" \ + --base-url "https://your-host/audio/" + +podkit serve --dir ./audio --addr :8080 --new-token +``` + +Point your podcast app at the feed URL that `serve` prints. That's it. + +## Install + +``` +go build -o podkit . +pip install --user yt-dlp # needed only for `podkit fetch` +``` + +`ffmpeg` must also be on PATH — yt-dlp uses it to extract audio. `podkit` +checks for both **before** starting a long download rather than failing during +post-processing, after the bytes are already on disk. + +## Commands + +| Command | What it does | +| --- | --- | +| `podkit fetch <url>` | Download and extract audio | +| `podkit tag` | Write ID3 tags derived from filenames | +| `podkit feed` | Generate `feed.xml` | +| `podkit serve` | Host the directory and feed over HTTP | +| `podkit all <url>` | fetch → tag → feed | + +Each stage runs independently on purpose: the download is the slow part and +shouldn't be repeated while you're adjusting titles or the base URL. + +## Notes on the things that usually go wrong + +These are the parts a hand-rolled version of this pipeline tends to get wrong, +and what `podkit` does instead. + +**Filenames in URLs.** Lecture titles contain spaces and ampersands. Pasted +into an enclosure URL unescaped they produce both an invalid URL and invalid +XML, and the feed won't load at all. `podkit` escapes the final path segment +only — escaping the whole base would mangle the scheme and host. + +**Episode order.** `01, 02, … 10` sorts correctly as text only while the +zero-padding is applied consistently; one unpadded file and lecture 10 lands +after lecture 1. Order comes from the parsed track number, not the filename. +Untracked files sort last rather than jumping to the front. + +**Play direction.** Podcasts default to newest-first, which starts a lecture +course at the final lecture. The feed sets `itunes:type=serial` and gives every +item a distinct ascending `pubDate`, so clients present it in order either way. + +**`guid`.** Marked `isPermaLink="false"`. Without that, a reader is entitled to +treat the value as a URL and fetch it; these identifiers are filenames. + +**Required channel elements.** `description` and `link` are mandatory in +practice — several clients silently reject a feed lacking them. + +**Non-audio files.** Cover art and the `.part` files an interrupted download +leaves behind are skipped by extension, so they can't become episodes. + +**Resumability.** `fetch` keeps a download archive (`.podkit-archive`), so +re-running skips what's already there. Tagging is idempotent, and re-tagging +preserves artwork yt-dlp embedded rather than replacing the tag wholesale. + +**ID3 only on MP3.** ID3v2 is an MP3 convention; writing it into `.opus` or +`.m4a` produces something no player reads, so other formats are left alone. + +## "Private" + +`--token` means unlisted-and-unguessable, not authenticated. Podcast clients +can't do interactive login, so a shared secret in the URL is the only mechanism +that actually works with them — anyone holding the URL has access. Rejections +return 404 rather than 401, so a prober learns nothing. + +For anything beyond a home network, put it behind a reverse proxy with TLS. +`serve` speaks plain HTTP and is not intended to face the internet directly. + +## Sourcing + +`podkit` takes a URL you supply and does nothing to find content. Whether you +may download a given item is between you and whoever published it — check the +source's terms. + +## Tests + +``` +go test ./... +``` + +Covers the feed generator (URL escaping, ordering, XML well-formedness, +non-audio filtering), the tagger (round-trips, `n/total` track frames, +idempotency) and the server (token gating, path traversal, that the escaped +URLs the generator emits actually resolve against the server that serves them). @@ -0,0 +1,7 @@ +module podkit + +go 1.24.0 + +require github.com/bogem/id3v2/v2 v2.1.4 + +require golang.org/x/text v0.3.8 // indirect @@ -0,0 +1,27 @@ +github.com/bogem/id3v2/v2 v2.1.4 h1:CEwe+lS2p6dd9UZRlPc1zbFNIha2mb2qzT1cCEoNWoI= +github.com/bogem/id3v2/v2 v2.1.4/go.mod h1:l+gR8MZ6rc9ryPTPkX77smS5Me/36gxkMgDayZ9G1vY= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/feed/feed.go b/internal/feed/feed.go new file mode 100644 index 0000000..02a016e --- /dev/null +++ b/internal/feed/feed.go @@ -0,0 +1,271 @@ +// Package feed builds a podcast RSS document from a directory of audio files. +package feed + +import ( + "encoding/xml" + "fmt" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// Options configure a generated feed. +type Options struct { + Dir string + BaseURL string + Title string + Description string + Author string + Link string + Language string + // Epoch is the publication date given to the first track. Later tracks get + // successively later dates so that episode order is unambiguous; see + // pubDate handling in Build. + Epoch time.Time +} + +// Item is one episode. +type Item struct { + XMLName xml.Name `xml:"item"` + Title string `xml:"title"` + Description string `xml:"description,omitempty"` + GUID GUID `xml:"guid"` + PubDate string `xml:"pubDate"` + Enclosure Enclosure + Episode int `xml:"itunes:episode,omitempty"` + ITunesTitle string `xml:"itunes:title,omitempty"` +} + +// GUID is an episode's stable identifier. +// +// isPermaLink="false" matters: without it a reader is entitled to treat the +// value as a URL and fetch it. These identifiers are derived from filenames, +// not addresses. +type GUID struct { + XMLName xml.Name `xml:"guid"` + IsPermaLink bool `xml:"isPermaLink,attr"` + Value string `xml:",chardata"` +} + +// Enclosure is the audio file itself. +type Enclosure struct { + XMLName xml.Name `xml:"enclosure"` + URL string `xml:"url,attr"` + Type string `xml:"type,attr"` + Length int64 `xml:"length,attr"` +} + +// Channel is the feed body. +type Channel struct { + XMLName xml.Name `xml:"channel"` + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + Language string `xml:"language,omitempty"` + Author string `xml:"itunes:author,omitempty"` + // serial tells a podcast client this is a course to be played from the + // beginning, not a stream of news where the newest item comes first. For a + // lecture series that distinction is the difference between starting at + // lecture one and starting at the end. + ITunesType string `xml:"itunes:type,omitempty"` + Items []Item +} + +// RSS is the document root. +type RSS struct { + XMLName xml.Name `xml:"rss"` + Version string `xml:"version,attr"` + ITunesNS string `xml:"xmlns:itunes,attr"` + ContentNS string `xml:"xmlns:content,attr"` + Channel Channel +} + +// TagReader reads metadata from an audio file. Injected so the feed can be +// tested without real ID3 frames. +type TagReader func(path string) (title string, track int, ok bool) + +var trackPrefix = regexp.MustCompile(`^(\d+)\s*[-–—.]\s*(.+)$`) + +// audioTypes maps an extension to its MIME type. Anything not listed is not +// treated as audio, so cover art and .part files left behind by an interrupted +// download cannot end up as episodes. +var audioTypes = map[string]string{ + ".mp3": "audio/mpeg", + ".m4a": "audio/mp4", + ".m4b": "audio/mp4", + ".aac": "audio/aac", + ".opus": "audio/opus", + ".ogg": "audio/ogg", + ".flac": "audio/flac", + ".wav": "audio/wav", +} + +// ParseName splits "03 - Great Conservation Principles.mp3" into its track +// number and title. Files without a numeric prefix keep their whole stem as the +// title and report track 0. +func ParseName(filename string) (track int, title string) { + stem := strings.TrimSuffix(filename, filepath.Ext(filename)) + if m := trackPrefix.FindStringSubmatch(stem); m != nil { + n, err := strconv.Atoi(m[1]) + if err == nil { + return n, strings.TrimSpace(m[2]) + } + } + return 0, strings.TrimSpace(stem) +} + +// EnclosureURL joins a base URL to a filename, escaping the filename. +// +// This is the detail that breaks a hand-rolled generator: track names routinely +// contain spaces and ampersands, and pasting them raw produces both an invalid +// URL and invalid XML. Only the final path segment is escaped — escaping the +// whole base would mangle its scheme and host. +func EnclosureURL(baseURL, filename string) (string, error) { + base, err := url.Parse(baseURL) + if err != nil { + return "", fmt.Errorf("parsing base URL %q: %w", baseURL, err) + } + if base.Scheme == "" || base.Host == "" { + return "", fmt.Errorf("base URL %q needs a scheme and host", baseURL) + } + base.Path = path.Join(base.Path, filename) + // url.URL.String() escapes the path for us; RawPath stays unset so the + // escaping is derived from Path. + return base.String(), nil +} + +// Build scans Dir and returns the RSS document. +func Build(opts Options, readTags TagReader) (*RSS, error) { + if opts.Title == "" { + return nil, fmt.Errorf("feed title is required") + } + entries, err := os.ReadDir(opts.Dir) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", opts.Dir, err) + } + + epoch := opts.Epoch + if epoch.IsZero() { + epoch = time.Date(2000, time.January, 1, 12, 0, 0, 0, time.UTC) + } + + type candidate struct { + name string + track int + title string + size int64 + } + var found []candidate + + for _, e := range entries { + if e.IsDir() { + continue + } + mime, isAudio := audioTypes[strings.ToLower(filepath.Ext(e.Name()))] + if !isAudio { + continue + } + _ = mime + info, err := e.Info() + if err != nil { + // A file that vanished mid-scan is not a reason to abandon the + // whole feed, but silently dropping it would be worse than saying so. + return nil, fmt.Errorf("stat %s: %w", e.Name(), err) + } + + track, title := ParseName(e.Name()) + if readTags != nil { + if tagTitle, tagTrack, ok := readTags(filepath.Join(opts.Dir, e.Name())); ok { + if tagTitle != "" { + title = tagTitle + } + if tagTrack > 0 { + track = tagTrack + } + } + } + found = append(found, candidate{e.Name(), track, title, info.Size()}) + } + + // Order by track number, falling back to filename. os.ReadDir is already + // name-sorted, but a feed whose order depends on zero-padding being applied + // consistently is a feed that will eventually play lecture 10 after lecture 1. + sort.SliceStable(found, func(i, j int) bool { + if found[i].track != found[j].track { + // Untracked files (0) sort last rather than first. + if found[i].track == 0 || found[j].track == 0 { + return found[j].track == 0 + } + return found[i].track < found[j].track + } + return found[i].name < found[j].name + }) + + items := make([]Item, 0, len(found)) + for i, c := range found { + encURL, err := EnclosureURL(opts.BaseURL, c.name) + if err != nil { + return nil, err + } + // One hour apart, ascending. Clients sort by date; keeping them ordered + // and distinct means the series plays in sequence whichever way a given + // client sorts. + pub := epoch.Add(time.Duration(i) * time.Hour) + + episode := c.track + if episode == 0 { + episode = i + 1 + } + + items = append(items, Item{ + Title: c.title, + GUID: GUID{IsPermaLink: false, Value: "podkit:" + c.name}, + PubDate: pub.Format(time.RFC1123Z), + Enclosure: Enclosure{ + URL: encURL, + Type: audioTypes[strings.ToLower(filepath.Ext(c.name))], + Length: c.size, + }, + Episode: episode, + }) + } + + description := opts.Description + if description == "" { + description = opts.Title + } + link := opts.Link + if link == "" { + link = opts.BaseURL + } + + return &RSS{ + Version: "2.0", + ITunesNS: "http://www.itunes.com/dtds/podcast-1.0.dtd", + ContentNS: "http://purl.org/rss/1.0/modules/content/", + Channel: Channel{ + Title: opts.Title, + Link: link, + Description: description, + Language: opts.Language, + Author: opts.Author, + ITunesType: "serial", + Items: items, + }, + }, nil +} + +// Marshal renders the feed as a complete XML document. +func Marshal(r *RSS) ([]byte, error) { + out, err := xml.MarshalIndent(r, "", " ") + if err != nil { + return nil, fmt.Errorf("marshalling feed: %w", err) + } + return append([]byte(xml.Header), append(out, '\n')...), nil +} diff --git a/internal/feed/feed_test.go b/internal/feed/feed_test.go new file mode 100644 index 0000000..a529e6f --- /dev/null +++ b/internal/feed/feed_test.go @@ -0,0 +1,343 @@ +package feed + +import ( + "encoding/xml" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func writeFiles(t *testing.T, names ...string) string { + t.Helper() + dir := t.TempDir() + for i, n := range names { + // Distinct sizes so enclosure length is verifiably per-file. + body := strings.Repeat("x", 100+i) + if err := os.WriteFile(filepath.Join(dir, n), []byte(body), 0o644); err != nil { + t.Fatalf("writing %s: %v", n, err) + } + } + return dir +} + +func build(t *testing.T, dir string, opts ...func(*Options)) *RSS { + t.Helper() + o := Options{ + Dir: dir, + BaseURL: "https://example.com/audio/", + Title: "Personal Physics Lectures", + Epoch: time.Date(2020, time.March, 1, 12, 0, 0, 0, time.UTC), + } + for _, f := range opts { + f(&o) + } + r, err := Build(o, nil) + if err != nil { + t.Fatalf("Build: %v", err) + } + return r +} + +func TestParseName(t *testing.T) { + cases := []struct { + in string + track int + title string + }{ + {"01 - The Law of Gravitation.mp3", 1, "The Law of Gravitation"}, + {"07 - Seeking New Laws.mp3", 7, "Seeking New Laws"}, + {"10 - Probability & Uncertainty.mp3", 10, "Probability & Uncertainty"}, + {"3 - Short Prefix.mp3", 3, "Short Prefix"}, + {"04 – En Dash Separator.mp3", 4, "En Dash Separator"}, + {"No Number Here.mp3", 0, "No Number Here"}, + {" Padded Name .mp3", 0, "Padded Name"}, + } + for _, c := range cases { + track, title := ParseName(c.in) + if track != c.track || title != c.title { + t.Errorf("ParseName(%q) = (%d, %q), want (%d, %q)", c.in, track, title, c.track, c.title) + } + } +} + +// Filenames routinely contain spaces and ampersands. Pasting them into the +// enclosure URL unescaped produces both an invalid URL and invalid XML — the +// single most likely way a hand-rolled feed fails to load at all. +func TestEnclosureURL_EscapesFilenames(t *testing.T) { + got, err := EnclosureURL("https://example.com/audio/", "10 - Probability & Uncertainty.mp3") + if err != nil { + t.Fatalf("EnclosureURL: %v", err) + } + if strings.ContainsAny(got, " ") { + t.Errorf("URL still contains a raw space: %q", got) + } + for _, want := range []string{"%20", "&"} { + _ = want + } + if !strings.Contains(got, "10%20-%20Probability") { + t.Errorf("spaces not percent-encoded: %q", got) + } + if !strings.HasPrefix(got, "https://example.com/audio/") { + t.Errorf("base URL mangled: %q", got) + } +} + +func TestEnclosureURL_HandlesMissingTrailingSlash(t *testing.T) { + got, err := EnclosureURL("https://example.com/audio", "01 - One.mp3") + if err != nil { + t.Fatalf("EnclosureURL: %v", err) + } + if !strings.Contains(got, "/audio/01") { + t.Errorf("expected a separator to be inserted, got %q", got) + } +} + +func TestEnclosureURL_RejectsBadBase(t *testing.T) { + for _, bad := range []string{"", "not-a-url", "/just/a/path"} { + if _, err := EnclosureURL(bad, "a.mp3"); err == nil { + t.Errorf("EnclosureURL(%q) should have failed", bad) + } + } +} + +// Zero-padding is a convention, not a guarantee. Ordering has to come from the +// parsed track number, or a series with ten or more parts plays 1, 10, 2. +func TestBuild_OrdersByTrackNumberNotFilename(t *testing.T) { + dir := writeFiles(t, + "1 - One.mp3", + "10 - Ten.mp3", + "2 - Two.mp3", + ) + r := build(t, dir) + + var titles []string + for _, it := range r.Channel.Items { + titles = append(titles, it.Title) + } + want := []string{"One", "Two", "Ten"} + for i := range want { + if titles[i] != want[i] { + t.Fatalf("order = %v, want %v", titles, want) + } + } +} + +// Files with no track number should not displace the numbered sequence. +func TestBuild_UntrackedFilesSortLast(t *testing.T) { + dir := writeFiles(t, "Intro.mp3", "01 - One.mp3", "02 - Two.mp3") + r := build(t, dir) + last := r.Channel.Items[len(r.Channel.Items)-1] + if last.Title != "Intro" { + t.Errorf("last item = %q, want the untracked Intro", last.Title) + } +} + +// Anything that is not audio must be ignored, including the .part files an +// interrupted download leaves behind and any cover art. +func TestBuild_IgnoresNonAudio(t *testing.T) { + dir := writeFiles(t, + "01 - One.mp3", + "cover.jpg", + "02 - Two.mp3.part", + "notes.txt", + ) + r := build(t, dir) + if len(r.Channel.Items) != 1 { + var got []string + for _, i := range r.Channel.Items { + got = append(got, i.Title) + } + t.Errorf("got %d items %v, want only the mp3", len(r.Channel.Items), got) + } +} + +func TestBuild_EnclosureLengthIsTheFileSize(t *testing.T) { + dir := writeFiles(t, "01 - One.mp3") + info, err := os.Stat(filepath.Join(dir, "01 - One.mp3")) + if err != nil { + t.Fatal(err) + } + r := build(t, dir) + if got := r.Channel.Items[0].Enclosure.Length; got != info.Size() { + t.Errorf("Length = %d, want %d", got, info.Size()) + } +} + +// Every item needs a distinct, increasing date or clients are free to order +// the series however they like. +func TestBuild_PubDatesAreDistinctAndAscending(t *testing.T) { + dir := writeFiles(t, "01 - One.mp3", "02 - Two.mp3", "03 - Three.mp3") + r := build(t, dir) + + var prev time.Time + for i, it := range r.Channel.Items { + got, err := time.Parse(time.RFC1123Z, it.PubDate) + if err != nil { + t.Fatalf("item %d has an unparseable pubDate %q: %v", i, it.PubDate, err) + } + if i > 0 && !got.After(prev) { + t.Errorf("item %d pubDate %v is not after the previous %v", i, got, prev) + } + prev = got + } +} + +func TestBuild_EpisodeNumbersFollowTracks(t *testing.T) { + dir := writeFiles(t, "01 - One.mp3", "02 - Two.mp3") + r := build(t, dir) + for i, want := range []int{1, 2} { + if r.Channel.Items[i].Episode != want { + t.Errorf("item %d episode = %d, want %d", i, r.Channel.Items[i].Episode, want) + } + } +} + +// A lecture course must be presented oldest-first; the default for a podcast +// is newest-first, which would start the listener at the final lecture. +func TestBuild_MarkedAsSerial(t *testing.T) { + r := build(t, writeFiles(t, "01 - One.mp3")) + if r.Channel.ITunesType != "serial" { + t.Errorf("itunes:type = %q, want serial", r.Channel.ITunesType) + } +} + +func TestBuild_RequiresATitle(t *testing.T) { + _, err := Build(Options{Dir: t.TempDir(), BaseURL: "https://e.com/"}, nil) + if err == nil { + t.Error("expected an error when no feed title is given") + } +} + +// A missing directory must be an error, not an empty feed that looks like a +// working setup with nothing in it. +func TestBuild_MissingDirectoryIsAnError(t *testing.T) { + _, err := Build(Options{ + Dir: filepath.Join(t.TempDir(), "nope"), BaseURL: "https://e.com/", Title: "T", + }, nil) + if err == nil { + t.Error("expected an error for a missing directory") + } +} + +func TestBuild_ChannelHasRequiredElements(t *testing.T) { + r := build(t, writeFiles(t, "01 - One.mp3")) + if r.Channel.Title == "" { + t.Error("channel title is empty") + } + if r.Channel.Description == "" { + t.Error("channel description is empty — many clients reject a feed without one") + } + if r.Channel.Link == "" { + t.Error("channel link is empty") + } +} + +// ID3 tags, when present, are better than a filename guess. +func TestBuild_PrefersTagsOverFilenames(t *testing.T) { + dir := writeFiles(t, "01 - filename title.mp3") + r, err := Build(Options{ + Dir: dir, BaseURL: "https://example.com/a/", Title: "T", + }, func(string) (string, int, bool) { + return "Tagged Title", 5, true + }) + if err != nil { + t.Fatalf("Build: %v", err) + } + if r.Channel.Items[0].Title != "Tagged Title" { + t.Errorf("title = %q, want the tag value", r.Channel.Items[0].Title) + } + if r.Channel.Items[0].Episode != 5 { + t.Errorf("episode = %d, want the tag track", r.Channel.Items[0].Episode) + } +} + +// A tag reader that finds nothing must not blank out the filename-derived title. +func TestBuild_FallsBackWhenTagsAreAbsent(t *testing.T) { + dir := writeFiles(t, "01 - Filename Title.mp3") + r, err := Build(Options{Dir: dir, BaseURL: "https://e.com/a/", Title: "T"}, + func(string) (string, int, bool) { return "", 0, false }) + if err != nil { + t.Fatalf("Build: %v", err) + } + if r.Channel.Items[0].Title != "Filename Title" { + t.Errorf("title = %q, want the filename fallback", r.Channel.Items[0].Title) + } +} + +// The whole document must be well-formed and round-trip through a parser — +// this is what a podcast client does first, and the thing most likely to fail +// on unescaped content. +func TestMarshal_ProducesParseableXML(t *testing.T) { + dir := writeFiles(t, "10 - Probability & Uncertainty.mp3", "01 - One <thing>.mp3") + r := build(t, dir) + + out, err := Marshal(r) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if !strings.HasPrefix(string(out), xml.Header) { + t.Error("missing XML declaration") + } + + var parsed struct { + Channel struct { + Title string `xml:"title"` + Items []struct { + Title string `xml:"title"` + Enclosure struct { + URL string `xml:"url,attr"` + Length int64 `xml:"length,attr"` + } `xml:"enclosure"` + } `xml:"item"` + } `xml:"channel"` + } + if err := xml.Unmarshal(out, &parsed); err != nil { + t.Fatalf("generated feed does not parse: %v\n%s", err, out) + } + if len(parsed.Channel.Items) != 2 { + t.Fatalf("parsed %d items, want 2", len(parsed.Channel.Items)) + } + for _, it := range parsed.Channel.Items { + if it.Enclosure.URL == "" { + t.Error("an item has no enclosure URL") + } + if it.Enclosure.Length == 0 { + t.Error("an item has a zero enclosure length") + } + } +} + +// The original hand-rolled struct had a field with no xml tag, which the +// marshaller emitted as a stray <EnclosureURL> element. Guard against any +// element leaking in that a reader would not expect. +func TestMarshal_NoStrayElements(t *testing.T) { + out, err := Marshal(build(t, writeFiles(t, "01 - One.mp3"))) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "EnclosureURL") { + t.Error("stray EnclosureURL element in output") + } + if strings.Contains(string(out), "XMLName") { + t.Error("stray XMLName element in output") + } +} + +func TestMarshal_GUIDIsNotAPermalink(t *testing.T) { + out, _ := Marshal(build(t, writeFiles(t, "01 - One.mp3"))) + if !strings.Contains(string(out), `isPermaLink="false"`) { + t.Error(`guid should carry isPermaLink="false"; it is not a URL`) + } +} + +func TestBuild_EmptyDirectoryProducesAValidEmptyFeed(t *testing.T) { + r := build(t, t.TempDir()) + if len(r.Channel.Items) != 0 { + t.Errorf("got %d items, want 0", len(r.Channel.Items)) + } + if _, err := Marshal(r); err != nil { + t.Errorf("an empty feed should still marshal: %v", err) + } +} diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go new file mode 100644 index 0000000..6dc4763 --- /dev/null +++ b/internal/fetch/fetch.go @@ -0,0 +1,185 @@ +// Package fetch wraps yt-dlp for audio extraction. +package fetch + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +// Options configure a download run. +type Options struct { + URL string + Dir string + Format string // audio codec for --audio-format, e.g. "mp3" + // Archive records completed downloads so a re-run skips them. This is what + // makes an interrupted 7-hour lecture series resumable rather than starting + // over, and makes re-running the whole pipeline cheap. + Archive bool + Stdout io.Writer + Stderr io.Writer +} + +// ErrNotInstalled is returned when yt-dlp cannot be found. +type ErrNotInstalled struct{ Err error } + +func (e *ErrNotInstalled) Error() string { + return "yt-dlp is not installed or not on PATH.\n" + + "Install it with: pip install --user yt-dlp\n" + + "(ffmpeg is also required for audio extraction.)" +} + +func (e *ErrNotInstalled) Unwrap() error { return e.Err } + +// Check reports whether the external tools are available. +// +// Checked up front rather than letting the command fail mid-run: a missing +// ffmpeg surfaces from yt-dlp as a post-processing error after the download has +// already happened, which is a confusing place to learn about it. +func Check() error { + if _, err := exec.LookPath("yt-dlp"); err != nil { + return &ErrNotInstalled{Err: err} + } + if _, err := exec.LookPath("ffmpeg"); err != nil { + return fmt.Errorf("ffmpeg is not installed or not on PATH; yt-dlp needs it to extract audio") + } + return nil +} + +// Run downloads and extracts audio into opts.Dir. +func Run(ctx context.Context, opts Options) error { + if opts.URL == "" { + return fmt.Errorf("a playlist or video URL is required") + } + if err := Check(); err != nil { + return err + } + if err := os.MkdirAll(opts.Dir, 0o755); err != nil { + return fmt.Errorf("creating %s: %w", opts.Dir, err) + } + + format := opts.Format + if format == "" { + format = "mp3" + } + + // %(playlist_index)02d is empty for a single video, which would leave a + // filename starting with " - ". The |NA default plus autonumber fallback + // keeps single downloads sensibly named. + template := filepath.Join(opts.Dir, "%(playlist_index|)02d - %(title)s.%(ext)s") + + args := []string{ + "--extract-audio", + "--audio-format", format, + "--audio-quality", "0", + "--embed-metadata", + "--embed-thumbnail", + // Keep going if one item in a long playlist is unavailable rather than + // abandoning the rest. + "--ignore-errors", + "--no-overwrites", + "--restrict-filenames", + "--output", template, + } + if opts.Archive { + args = append(args, "--download-archive", filepath.Join(opts.Dir, ".podkit-archive")) + } + args = append(args, opts.URL) + + cmd := exec.CommandContext(ctx, "yt-dlp", args...) + cmd.Stdout = opts.Stdout + cmd.Stderr = opts.Stderr + if cmd.Stdout == nil { + cmd.Stdout = os.Stdout + } + if cmd.Stderr == nil { + cmd.Stderr = os.Stderr + } + + if err := cmd.Run(); err != nil { + return fmt.Errorf("yt-dlp failed: %w", err) + } + return nil +} + +// Rename normalises yt-dlp's --restrict-filenames output back into readable +// "NN - Title.ext" names. +// +// --restrict-filenames is used because unrestricted titles produce filenames +// with quotes, slashes and emoji that make both shell use and URL construction +// unpleasant. The cost is underscores everywhere, so they are converted back +// here where it is safe to do so. +func Rename(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", dir, err) + } + + var renamed []string + for _, e := range entries { + if e.IsDir() || strings.HasPrefix(e.Name(), ".") { + continue + } + pretty := prettifyName(e.Name()) + if pretty == e.Name() { + continue + } + from := filepath.Join(dir, e.Name()) + to := filepath.Join(dir, pretty) + if _, err := os.Stat(to); err == nil { + // Never clobber an existing file to make a name prettier. + continue + } + if err := os.Rename(from, to); err != nil { + return renamed, fmt.Errorf("renaming %s: %w", e.Name(), err) + } + renamed = append(renamed, pretty) + } + return renamed, nil +} + +// duplicatedPrefix matches "01 - 01 - Title", where the output template's +// index has been prepended to a title that already carried the same number. +var duplicatedPrefix = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)\s*-\s*(.+)$`) + +func prettifyName(name string) string { + ext := filepath.Ext(name) + stem := strings.TrimSuffix(name, ext) + stem = strings.ReplaceAll(stem, "_", " ") + // Collapse the runs of spaces that replacing underscores tends to create. + for strings.Contains(stem, " ") { + stem = strings.ReplaceAll(stem, " ", " ") + } + stem = strings.TrimSpace(stem) + + // Sources whose own titles start with a track number end up doubled, since + // the output template prepends the playlist index regardless: a LibriVox + // collection produced "01 - 01 - April.mp3" on the first live run. + // + // Only collapse when the two numbers agree. "01 - 1964 - Lecture" is a year, + // not a repeat, and "05 - 12 Angry Men" is a title that happens to start + // with a digit — neither should be touched. + if m := duplicatedPrefix.FindStringSubmatch(stem); m != nil { + first, err1 := strconv.Atoi(m[1]) + second, err2 := strconv.Atoi(m[2]) + if err1 == nil && err2 == nil && first == second { + stem = m[1] + " - " + strings.TrimSpace(m[3]) + } + } + return stem + ext +} + +// StreamLines copies r to w line by line, for surfacing yt-dlp progress. +func StreamLines(r io.Reader, w io.Writer) { + sc := bufio.NewScanner(r) + for sc.Scan() { + fmt.Fprintln(w, sc.Text()) + } +} diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go new file mode 100644 index 0000000..e55b508 --- /dev/null +++ b/internal/fetch/fetch_test.go @@ -0,0 +1,144 @@ +package fetch + +import ( + "os" + "path/filepath" + "sort" + "testing" +) + +func TestPrettifyName_RestoresSpaces(t *testing.T) { + got := prettifyName("01 - The_Law_of_Gravitation.mp3") + if got != "01 - The Law of Gravitation.mp3" { + t.Errorf("got %q", got) + } +} + +// yt-dlp's --restrict-filenames collapses odd characters into underscores, and +// replacing them can leave runs of spaces behind. +func TestPrettifyName_CollapsesRepeatedSpaces(t *testing.T) { + got := prettifyName("02 - A___Gap.mp3") + if got != "02 - A Gap.mp3" { + t.Errorf("got %q", got) + } +} + +// Sources whose titles already begin with a track number produce +// "01 - 01 - April.mp3" once the output template prepends its own index. Seen +// for real against a LibriVox collection on the first live run. +func TestPrettifyName_CollapsesDuplicatedTrackPrefix(t *testing.T) { + cases := []struct{ in, want string }{ + {"01 - 01 - April.mp3", "01 - April.mp3"}, + {"02 - 02 - Ave Atque Vale.mp3", "02 - Ave Atque Vale.mp3"}, + {"03 - 03_-_The_Average_Man.mp3", "03 - The Average Man.mp3"}, + {"10 - 10 - Tenth.mp3", "10 - Tenth.mp3"}, + } + for _, c := range cases { + if got := prettifyName(c.in); got != c.want { + t.Errorf("prettifyName(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// Only collapse when the two numbers agree. A title that genuinely starts with +// a different number is not a duplicated prefix and must be left alone. +func TestPrettifyName_KeepsDistinctLeadingNumbers(t *testing.T) { + cases := []struct{ in, want string }{ + {"01 - 1964 - The Messenger Lectures.mp3", "01 - 1964 - The Messenger Lectures.mp3"}, + {"05 - 12 Angry Men.mp3", "05 - 12 Angry Men.mp3"}, + } + for _, c := range cases { + if got := prettifyName(c.in); got != c.want { + t.Errorf("prettifyName(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestPrettifyName_LeavesCleanNamesAlone(t *testing.T) { + for _, n := range []string{"01 - April.mp3", "Introduction.mp3"} { + if got := prettifyName(n); got != n { + t.Errorf("prettifyName(%q) = %q, want it unchanged", n, got) + } + } +} + +func TestRename_RenamesOnDisk(t *testing.T) { + dir := t.TempDir() + for _, n := range []string{"01 - 01 - April.mp3", "02 - The_Average_Man.mp3", ".podkit-archive"} { + if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + if _, err := Rename(dir); err != nil { + t.Fatalf("Rename: %v", err) + } + + entries, _ := os.ReadDir(dir) + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + sort.Strings(names) + + want := []string{".podkit-archive", "01 - April.mp3", "02 - The Average Man.mp3"} + if len(names) != len(want) { + t.Fatalf("got %v, want %v", names, want) + } + for i := range want { + if names[i] != want[i] { + t.Errorf("got %v, want %v", names, want) + break + } + } +} + +// Renaming must never destroy an existing file to make a name prettier. +func TestRename_DoesNotClobber(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "01 - April.mp3"), []byte("original"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "01 - 01 - April.mp3"), []byte("candidate"), 0o644); err != nil { + t.Fatal(err) + } + + if _, err := Rename(dir); err != nil { + t.Fatalf("Rename: %v", err) + } + + body, err := os.ReadFile(filepath.Join(dir, "01 - April.mp3")) + if err != nil { + t.Fatal(err) + } + if string(body) != "original" { + t.Error("an existing file was overwritten by the rename") + } +} + +func TestRename_MissingDirectoryIsAnError(t *testing.T) { + if _, err := Rename(filepath.Join(t.TempDir(), "nope")); err == nil { + t.Error("expected an error for a missing directory") + } +} + +// Check must explain what to install, not just report a lookup failure. +func TestErrNotInstalled_SaysHowToFixIt(t *testing.T) { + e := &ErrNotInstalled{} + for _, want := range []string{"yt-dlp", "pip install", "ffmpeg"} { + if !contains(e.Error(), want) { + t.Errorf("error message should mention %q; got %q", want, e.Error()) + } + } +} + +func contains(h, n string) bool { + return len(h) >= len(n) && (func() bool { + for i := 0; i+len(n) <= len(h); i++ { + if h[i:i+len(n)] == n { + return true + } + } + return false + })() +} diff --git a/internal/serve/serve.go b/internal/serve/serve.go new file mode 100644 index 0000000..0b3b340 --- /dev/null +++ b/internal/serve/serve.go @@ -0,0 +1,91 @@ +// Package serve hosts the audio directory and its feed over HTTP. +package serve + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/hex" + "fmt" + "net/http" + "path" + "strings" +) + +// Options configure the server. +type Options struct { + Dir string + Addr string + // Token, if set, must appear in every request as ?k=<token> or an + // Authorization: Bearer header. + // + // "Private" here means unlisted-and-unguessable, not authenticated in any + // serious sense: podcast clients do not do interactive login, so a shared + // secret in the URL is the mechanism that actually works with them. Anyone + // holding the URL holds the access. + Token string +} + +// NewToken returns a random token suitable for Options.Token. +func NewToken() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("generating token: %w", err) + } + return hex.EncodeToString(b), nil +} + +// Handler builds the HTTP handler: the feed at /feed.xml, audio under /audio/. +func Handler(opts Options) http.Handler { + mux := http.NewServeMux() + + files := http.FileServer(http.Dir(opts.Dir)) + mux.Handle("/audio/", http.StripPrefix("/audio/", files)) + + mux.HandleFunc("/feed.xml", func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, path.Join(opts.Dir, "feed.xml")) + }) + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprintln(w, "podkit — subscribe your podcast app to /feed.xml") + }) + + return withToken(opts.Token, mux) +} + +// withToken gates every request on a shared secret. +func withToken(token string, next http.Handler) http.Handler { + if token == "" { + return next + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !tokenOK(r, token) { + // 404 rather than 401: a 401 confirms something is here worth + // guessing at, and no podcast client is going to act on the + // challenge anyway. + http.NotFound(w, r) + return + } + next.ServeHTTP(w, r) + }) +} + +func tokenOK(r *http.Request, want string) bool { + if got := r.URL.Query().Get("k"); got != "" && constantTimeEqual(got, want) { + return true + } + auth := r.Header.Get("Authorization") + if strings.HasPrefix(auth, "Bearer ") { + return constantTimeEqual(strings.TrimPrefix(auth, "Bearer "), want) + } + return false +} + +// constantTimeEqual avoids leaking the token through response timing. +func constantTimeEqual(a, b string) bool { + return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1 +} diff --git a/internal/serve/serve_test.go b/internal/serve/serve_test.go new file mode 100644 index 0000000..04b25ee --- /dev/null +++ b/internal/serve/serve_test.go @@ -0,0 +1,145 @@ +package serve + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func setup(t *testing.T, token string) (http.Handler, string) { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "feed.xml"), []byte("<rss/>"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "01 - One.mp3"), []byte("audio"), 0o644); err != nil { + t.Fatal(err) + } + return Handler(Options{Dir: dir, Token: token}), dir +} + +func get(h http.Handler, target string, bearer string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, target, nil) + if bearer != "" { + req.Header.Set("Authorization", "Bearer "+bearer) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec +} + +func TestServesFeedAndAudioWithoutToken(t *testing.T) { + h, _ := setup(t, "") + if rec := get(h, "/feed.xml", ""); rec.Code != http.StatusOK { + t.Errorf("/feed.xml = %d, want 200", rec.Code) + } + if rec := get(h, "/audio/01%20-%20One.mp3", ""); rec.Code != http.StatusOK { + t.Errorf("audio = %d, want 200", rec.Code) + } +} + +// The escaped URL the feed generator emits must actually resolve against the +// server that serves those files — the two halves have to agree. +func TestEscapedFilenameResolves(t *testing.T) { + h, dir := setup(t, "") + if err := os.WriteFile(filepath.Join(dir, "10 - Probability & Uncertainty.mp3"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + rec := get(h, "/audio/10%20-%20Probability%20&%20Uncertainty.mp3", "") + if rec.Code != http.StatusOK { + t.Errorf("escaped filename = %d, want 200", rec.Code) + } +} + +func TestTokenRequiredWhenSet(t *testing.T) { + h, _ := setup(t, "s3cret") + for _, target := range []string{"/feed.xml", "/audio/01%20-%20One.mp3", "/"} { + if rec := get(h, target, ""); rec.Code != http.StatusNotFound { + t.Errorf("%s without a token = %d, want 404", target, rec.Code) + } + } +} + +func TestTokenAcceptedInQuery(t *testing.T) { + h, _ := setup(t, "s3cret") + if rec := get(h, "/feed.xml?k=s3cret", ""); rec.Code != http.StatusOK { + t.Errorf("query token = %d, want 200", rec.Code) + } +} + +// Podcast clients cannot do interactive auth, so the query form is the one +// that matters; the bearer header is for curl and for testing. +func TestTokenAcceptedAsBearer(t *testing.T) { + h, _ := setup(t, "s3cret") + if rec := get(h, "/feed.xml", "s3cret"); rec.Code != http.StatusOK { + t.Errorf("bearer token = %d, want 200", rec.Code) + } +} + +func TestWrongTokenRejected(t *testing.T) { + h, _ := setup(t, "s3cret") + if rec := get(h, "/feed.xml?k=wrong", ""); rec.Code != http.StatusNotFound { + t.Errorf("wrong token = %d, want 404", rec.Code) + } + if rec := get(h, "/feed.xml", "wrong"); rec.Code != http.StatusNotFound { + t.Errorf("wrong bearer = %d, want 404", rec.Code) + } +} + +// A gated server must not confirm that anything is there. +func TestRejectionLooksLikeAbsence(t *testing.T) { + h, _ := setup(t, "s3cret") + rec := get(h, "/feed.xml", "") + if rec.Code == http.StatusUnauthorized || rec.Code == http.StatusForbidden { + t.Errorf("status %d advertises that something exists here", rec.Code) + } + if strings.Contains(strings.ToLower(rec.Body.String()), "token") { + t.Error("the rejection body mentions a token, which tells a prober what to look for") + } +} + +// http.FileServer resolves .. before dispatch, but this is the kind of thing +// worth a standing assertion rather than a memory of having checked once. +func TestNoPathTraversal(t *testing.T) { + h, dir := setup(t, "") + secret := filepath.Join(filepath.Dir(dir), "secret.txt") + if err := os.WriteFile(secret, []byte("do not serve"), 0o644); err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(secret) }() + + for _, target := range []string{ + "/audio/../secret.txt", + "/audio/..%2fsecret.txt", + "/audio/%2e%2e/secret.txt", + } { + rec := get(h, target, "") + if strings.Contains(rec.Body.String(), "do not serve") { + t.Errorf("%s escaped the served directory", target) + } + } +} + +func TestNewTokenIsRandomAndLongEnough(t *testing.T) { + a, err := NewToken() + if err != nil { + t.Fatalf("NewToken: %v", err) + } + b, _ := NewToken() + if a == b { + t.Error("two generated tokens were identical") + } + if len(a) < 32 { + t.Errorf("token is only %d chars; it is the only thing protecting the feed", len(a)) + } +} + +func TestUnknownPathIs404(t *testing.T) { + h, _ := setup(t, "") + if rec := get(h, "/nope", ""); rec.Code != http.StatusNotFound { + t.Errorf("unknown path = %d, want 404", rec.Code) + } +} diff --git a/internal/tag/tag.go b/internal/tag/tag.go new file mode 100644 index 0000000..653fac7 --- /dev/null +++ b/internal/tag/tag.go @@ -0,0 +1,132 @@ +// Package tag reads and writes ID3v2 tags on downloaded audio. +package tag + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/bogem/id3v2/v2" + + "podkit/internal/feed" +) + +// Metadata is the subset of ID3 fields worth setting for a lecture series. +type Metadata struct { + Album string + Artist string + AlbumArtist string + Genre string + Year string +} + +// Result reports what happened to one file. +type Result struct { + File string + Track int + Title string + Err error +} + +// Read returns the title and track number stored in a file's tags. +// +// Signature matches feed.TagReader so the feed generator can prefer real tags +// over a filename guess. A file with no tags is not an error — it is the +// normal state of a fresh download — so it reports ok=false and lets the +// caller fall back. +func Read(path string) (title string, track int, ok bool) { + t, err := id3v2.Open(path, id3v2.Options{Parse: true}) + if err != nil { + return "", 0, false + } + defer func() { _ = t.Close() }() + + title = strings.TrimSpace(t.Title()) + // A track frame can be "3" or "3/7"; only the part before the slash is the + // number of this track. + raw := strings.TrimSpace(t.GetTextFrame(t.CommonID("Track number/Position in set")).Text) + if i := strings.IndexByte(raw, '/'); i >= 0 { + raw = raw[:i] + } + if n, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil { + track = n + } + return title, track, title != "" || track > 0 +} + +// WriteDir tags every audio file in dir, deriving the track number and title +// from the filename. +// +// Only .mp3 is handled: ID3v2 is an MP3 container convention, and writing it +// into an .opus or .m4a file produces something no player will read. +func WriteDir(dir string, md Metadata) ([]Result, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", dir, err) + } + + var results []Result + total := 0 + for _, e := range entries { + if !e.IsDir() && strings.EqualFold(filepath.Ext(e.Name()), ".mp3") { + total++ + } + } + + for _, e := range entries { + if e.IsDir() || !strings.EqualFold(filepath.Ext(e.Name()), ".mp3") { + continue + } + path := filepath.Join(dir, e.Name()) + track, title := feed.ParseName(e.Name()) + res := Result{File: e.Name(), Track: track, Title: title} + if err := writeOne(path, track, total, title, md); err != nil { + res.Err = err + } + results = append(results, res) + } + return results, nil +} + +func writeOne(path string, track, total int, title string, md Metadata) error { + // Parse:true so an existing tag is preserved and updated rather than + // replaced wholesale, which would discard artwork yt-dlp may have embedded. + t, err := id3v2.Open(path, id3v2.Options{Parse: true}) + if err != nil { + return fmt.Errorf("opening: %w", err) + } + defer func() { _ = t.Close() }() + + t.SetTitle(title) + if md.Album != "" { + t.SetAlbum(md.Album) + } + if md.Artist != "" { + t.SetArtist(md.Artist) + } + if md.Genre != "" { + t.SetGenre(md.Genre) + } + if md.Year != "" { + t.SetYear(md.Year) + } + if md.AlbumArtist != "" { + t.AddTextFrame(t.CommonID("Band/Orchestra/Accompaniment"), t.DefaultEncoding(), md.AlbumArtist) + } + if track > 0 { + // "n/total" so a player can show "3 of 7" and, more importantly, sort + // the series correctly when the filenames are not available to it. + value := strconv.Itoa(track) + if total > 0 { + value = fmt.Sprintf("%d/%d", track, total) + } + t.AddTextFrame(t.CommonID("Track number/Position in set"), t.DefaultEncoding(), value) + } + + if err := t.Save(); err != nil { + return fmt.Errorf("saving tags: %w", err) + } + return nil +} diff --git a/internal/tag/tag_test.go b/internal/tag/tag_test.go new file mode 100644 index 0000000..f722cb6 --- /dev/null +++ b/internal/tag/tag_test.go @@ -0,0 +1,158 @@ +package tag + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A minimal but genuinely valid MP3: one silent MPEG-1 Layer III frame. +// id3v2 needs a real file to append a tag to, and a zero-byte file is not one. +func writeMP3(t *testing.T, dir, name string) string { + t.Helper() + // 0xFF 0xFB = MPEG-1 Layer III, no CRC; 0x90 = 128kbps 44.1kHz; then padding. + frame := []byte{0xFF, 0xFB, 0x90, 0x00} + body := append(frame, make([]byte, 413)...) + p := filepath.Join(dir, name) + if err := os.WriteFile(p, body, 0o644); err != nil { + t.Fatalf("writing %s: %v", name, err) + } + return p +} + +func TestWriteDir_TagsEachFile(t *testing.T) { + dir := t.TempDir() + writeMP3(t, dir, "01 - The Law of Gravitation.mp3") + writeMP3(t, dir, "02 - The Relation of Mathematics to Physics.mp3") + + results, err := WriteDir(dir, Metadata{ + Album: "Character of Physical Law", + Artist: "Richard Feynman", + }) + if err != nil { + t.Fatalf("WriteDir: %v", err) + } + if len(results) != 2 { + t.Fatalf("got %d results, want 2", len(results)) + } + for _, r := range results { + if r.Err != nil { + t.Errorf("%s: %v", r.File, r.Err) + } + } + + title, track, ok := Read(filepath.Join(dir, "01 - The Law of Gravitation.mp3")) + if !ok { + t.Fatal("tags were not readable after writing") + } + if title != "The Law of Gravitation" { + t.Errorf("title = %q", title) + } + if track != 1 { + t.Errorf("track = %d, want 1", track) + } +} + +// Read is handed to the feed generator, which relies on ok=false to fall back +// to the filename. An untagged file is the normal state of a fresh download, +// not an error. +func TestRead_UntaggedFileReportsNotOK(t *testing.T) { + dir := t.TempDir() + p := writeMP3(t, dir, "01 - Untagged.mp3") + if _, _, ok := Read(p); ok { + t.Error("an untagged file should report ok=false") + } +} + +func TestRead_MissingFileReportsNotOK(t *testing.T) { + if _, _, ok := Read(filepath.Join(t.TempDir(), "nope.mp3")); ok { + t.Error("a missing file should report ok=false, not panic") + } +} + +// Track frames are commonly "3/7". Taking the whole string would make the +// number unparseable and lose the ordering. +func TestRead_ParsesTrackOfTotal(t *testing.T) { + dir := t.TempDir() + writeMP3(t, dir, "03 - Third.mp3") + writeMP3(t, dir, "01 - First.mp3") + writeMP3(t, dir, "02 - Second.mp3") + + if _, err := WriteDir(dir, Metadata{Album: "A"}); err != nil { + t.Fatalf("WriteDir: %v", err) + } + _, track, ok := Read(filepath.Join(dir, "03 - Third.mp3")) + if !ok { + t.Fatal("tags not readable") + } + if track != 3 { + t.Errorf("track = %d, want 3 (from a \"3/3\" frame)", track) + } +} + +// ID3v2 is an MP3 convention; writing it into other containers produces +// something no player reads, so other formats are skipped rather than corrupted. +func TestWriteDir_SkipsNonMP3(t *testing.T) { + dir := t.TempDir() + writeMP3(t, dir, "01 - One.mp3") + if err := os.WriteFile(filepath.Join(dir, "02 - Two.opus"), []byte("not an mp3"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "cover.jpg"), []byte("jpeg"), 0o644); err != nil { + t.Fatal(err) + } + + results, err := WriteDir(dir, Metadata{Album: "A"}) + if err != nil { + t.Fatalf("WriteDir: %v", err) + } + if len(results) != 1 || !strings.HasSuffix(results[0].File, ".mp3") { + t.Errorf("got %+v, want only the mp3", results) + } +} + +func TestWriteDir_MissingDirectoryIsAnError(t *testing.T) { + if _, err := WriteDir(filepath.Join(t.TempDir(), "nope"), Metadata{}); err == nil { + t.Error("expected an error for a missing directory") + } +} + +// A file with no track prefix should still get its metadata; only the track +// number is absent. +func TestWriteDir_HandlesUnnumberedFiles(t *testing.T) { + dir := t.TempDir() + writeMP3(t, dir, "Introduction.mp3") + + results, err := WriteDir(dir, Metadata{Album: "Course", Artist: "Someone"}) + if err != nil { + t.Fatalf("WriteDir: %v", err) + } + if results[0].Err != nil { + t.Fatalf("tagging failed: %v", results[0].Err) + } + title, track, ok := Read(filepath.Join(dir, "Introduction.mp3")) + if !ok || title != "Introduction" { + t.Errorf("title = %q ok = %v, want the filename stem", title, ok) + } + if track != 0 { + t.Errorf("track = %d, want 0 for an unnumbered file", track) + } +} + +// Re-running the tagger must be safe; the pipeline is meant to be resumable. +func TestWriteDir_IsIdempotent(t *testing.T) { + dir := t.TempDir() + writeMP3(t, dir, "01 - One.mp3") + md := Metadata{Album: "A", Artist: "B"} + + for i := 0; i < 3; i++ { + if _, err := WriteDir(dir, md); err != nil { + t.Fatalf("pass %d: %v", i, err) + } + } + title, track, ok := Read(filepath.Join(dir, "01 - One.mp3")) + if !ok || title != "One" || track != 1 { + t.Errorf("after 3 passes: title=%q track=%d ok=%v", title, track, ok) + } +} @@ -0,0 +1,319 @@ +// podkit turns a playlist or video URL into a private podcast feed. +// +// podkit all "<url>" --dir ./audio --title "Character of Physical Law" \ +// --artist "Richard Feynman" --base-url https://host/audio/ +// podkit serve --dir ./audio --addr :8080 --token <token> +// +// Each stage is also runnable on its own, because the slow one (fetch) should +// not have to be repeated while iterating on the others. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "podkit/internal/feed" + "podkit/internal/fetch" + "podkit/internal/serve" + "podkit/internal/tag" +) + +const usage = `podkit — build a private podcast feed from a playlist or video URL + +Usage: + podkit fetch <url> [flags] download and extract audio (needs yt-dlp + ffmpeg) + podkit tag [flags] write ID3 tags from filenames + podkit feed [flags] generate feed.xml + podkit serve [flags] host the directory and feed over HTTP + podkit all <url> [flags] fetch, tag and feed in one go + +Run "podkit <command> -h" for that command's flags. + +Typical run: + podkit all "<playlist url>" \ + --dir ./audio \ + --title "Character of Physical Law" \ + --artist "Richard Feynman" \ + --base-url "https://your-host/audio/" + podkit serve --dir ./audio --addr :8080 --token "$(openssl rand -hex 16)" +` + +func main() { + if len(os.Args) < 2 { + fmt.Fprint(os.Stderr, usage) + os.Exit(2) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + var err error + switch os.Args[1] { + case "fetch": + err = cmdFetch(ctx, os.Args[2:]) + case "tag": + err = cmdTag(os.Args[2:]) + case "feed": + err = cmdFeed(os.Args[2:]) + case "serve": + err = cmdServe(ctx, os.Args[2:]) + case "all": + err = cmdAll(ctx, os.Args[2:]) + case "-h", "--help", "help": + fmt.Print(usage) + return + default: + fmt.Fprintf(os.Stderr, "unknown command %q\n\n%s", os.Args[1], usage) + os.Exit(2) + } + + if err != nil { + fmt.Fprintf(os.Stderr, "\npodkit: %v\n", err) + os.Exit(1) + } +} + +// metaFlags are the fields shared by tag, feed and all. +type metaFlags struct { + dir string + title string + artist string + album string + desc string + baseURL string + lang string + year string +} + +func (m *metaFlags) register(fs *flag.FlagSet) { + fs.StringVar(&m.dir, "dir", "./audio", "directory holding the audio files") + fs.StringVar(&m.title, "title", "", "feed title (also used as the album)") + fs.StringVar(&m.artist, "artist", "", "artist / author") + fs.StringVar(&m.album, "album", "", "album (defaults to --title)") + fs.StringVar(&m.desc, "description", "", "feed description (defaults to --title)") + fs.StringVar(&m.baseURL, "base-url", "", "public URL the audio directory is served from") + fs.StringVar(&m.lang, "lang", "en", "feed language") + fs.StringVar(&m.year, "year", "", "year tag") +} + +func (m *metaFlags) albumName() string { + if m.album != "" { + return m.album + } + return m.title +} + +func cmdFetch(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("fetch", flag.ExitOnError) + dir := fs.String("dir", "./audio", "directory to download into") + format := fs.String("format", "mp3", "audio format") + noArchive := fs.Bool("no-archive", false, "do not keep a download archive (re-downloads everything)") + noRename := fs.Bool("no-rename", false, "keep yt-dlp's restricted filenames") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 { + return errors.New("a playlist or video URL is required: podkit fetch <url>") + } + + fmt.Printf("==> Downloading into %s\n", *dir) + if err := fetch.Run(ctx, fetch.Options{ + URL: fs.Arg(0), Dir: *dir, Format: *format, Archive: !*noArchive, + }); err != nil { + return err + } + + if !*noRename { + renamed, err := fetch.Rename(*dir) + if err != nil { + return err + } + if len(renamed) > 0 { + fmt.Printf("==> Tidied %d filename(s)\n", len(renamed)) + } + } + return nil +} + +func cmdTag(args []string) error { + fs := flag.NewFlagSet("tag", flag.ExitOnError) + var m metaFlags + m.register(fs) + genre := fs.String("genre", "", "genre tag") + if err := fs.Parse(args); err != nil { + return err + } + + results, err := tag.WriteDir(m.dir, tag.Metadata{ + Album: m.albumName(), + Artist: m.artist, + AlbumArtist: m.artist, + Genre: *genre, + Year: m.year, + }) + if err != nil { + return err + } + + var failed int + for _, r := range results { + if r.Err != nil { + failed++ + fmt.Fprintf(os.Stderr, " FAILED %s: %v\n", r.File, r.Err) + } + } + fmt.Printf("==> Tagged %d file(s)", len(results)-failed) + if failed > 0 { + fmt.Printf(", %d failed", failed) + } + fmt.Println() + if failed > 0 { + return fmt.Errorf("%d file(s) could not be tagged", failed) + } + return nil +} + +func cmdFeed(args []string) error { + fs := flag.NewFlagSet("feed", flag.ExitOnError) + var m metaFlags + m.register(fs) + out := fs.String("out", "", `where to write the feed (default "<dir>/feed.xml", "-" for stdout)`) + if err := fs.Parse(args); err != nil { + return err + } + if m.title == "" { + return errors.New("--title is required") + } + if m.baseURL == "" { + return errors.New("--base-url is required: it is the public URL the audio is served from") + } + + r, err := feed.Build(feed.Options{ + Dir: m.dir, + BaseURL: m.baseURL, + Title: m.title, + Description: m.desc, + Author: m.artist, + Language: m.lang, + Epoch: time.Date(2000, time.January, 1, 12, 0, 0, 0, time.UTC), + }, tag.Read) + if err != nil { + return err + } + + data, err := feed.Marshal(r) + if err != nil { + return err + } + + target := *out + if target == "" { + target = filepath.Join(m.dir, "feed.xml") + } + if target == "-" { + _, err := os.Stdout.Write(data) + return err + } + if err := os.WriteFile(target, data, 0o644); err != nil { + return fmt.Errorf("writing %s: %w", target, err) + } + fmt.Printf("==> Wrote %s (%d episode(s))\n", target, len(r.Channel.Items)) + return nil +} + +func cmdServe(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("serve", flag.ExitOnError) + dir := fs.String("dir", "./audio", "directory to serve") + addr := fs.String("addr", ":8080", "listen address") + token := fs.String("token", "", "shared secret required on every request (recommended)") + newToken := fs.Bool("new-token", false, "generate a token and use it") + if err := fs.Parse(args); err != nil { + return err + } + + tok := *token + if *newToken { + var err error + if tok, err = serve.NewToken(); err != nil { + return err + } + } + + srv := &http.Server{ + Addr: *addr, + Handler: serve.Handler(serve.Options{Dir: *dir, Token: tok}), + ReadHeaderTimeout: 10 * time.Second, + } + + fmt.Printf("==> Serving %s on %s\n", *dir, *addr) + if tok != "" { + fmt.Printf(" Feed URL: http://<host>%s/feed.xml?k=%s\n", *addr, tok) + fmt.Println(" Anyone with that URL has access; it is a shared secret, not a login.") + } else { + fmt.Printf(" Feed URL: http://<host>%s/feed.xml\n", *addr) + fmt.Println(" No token set — anyone who can reach this port can read the feed.") + } + + errCh := make(chan error, 1) + go func() { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- err + } + }() + + select { + case err := <-errCh: + return err + case <-ctx.Done(): + fmt.Println("\n==> Shutting down") + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return srv.Shutdown(shutdownCtx) + } +} + +func cmdAll(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("all", flag.ExitOnError) + var m metaFlags + m.register(fs) + genre := fs.String("genre", "", "genre tag") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 { + return errors.New("a playlist or video URL is required: podkit all <url>") + } + if m.title == "" { + return errors.New("--title is required") + } + if m.baseURL == "" { + return errors.New("--base-url is required: it is the public URL the audio is served from") + } + + // Checked before the long download rather than after it. + if err := fetch.Check(); err != nil { + return err + } + + shared := []string{ + "--dir", m.dir, "--title", m.title, "--artist", m.artist, + "--album", m.album, "--description", m.desc, + "--base-url", m.baseURL, "--lang", m.lang, "--year", m.year, + } + + if err := cmdFetch(ctx, []string{"--dir", m.dir, fs.Arg(0)}); err != nil { + return err + } + if err := cmdTag(append(shared, "--genre", *genre)); err != nil { + return err + } + return cmdFeed(shared) +} |