From 293a627358553c9f61802caf9d4bc76af9c2e239 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Mon, 21 Sep 2026 06:50:42 +0000 Subject: podkit: playlist URL to private podcast feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One Go binary with fetch / tag / feed / serve / all. The stages run separately on purpose: the download is the slow part and should not be repeated while adjusting titles or the base URL. Built from a shell + Python + Go sketch. The pieces that needed fixing on the way, all of which produce a feed that loads wrong or not at all: - Enclosure URLs were unescaped. Track titles routinely contain spaces and ampersands, which make both the URL and the XML invalid — the likeliest way for a feed to fail to load entirely. Only the final path segment is escaped; escaping the base would mangle scheme and host. - Ordering came from filename sort, which is correct only while zero-padding is applied consistently. One unpadded file puts lecture 10 after lecture 1. Order now comes from the parsed track number. - No pubDate and no itunes:type. Podcast clients default to newest-first, which starts a lecture course at the final lecture. Items now carry distinct ascending dates and the feed is marked serial. - Missing channel description and link, which several clients reject outright. - A struct field with no xml tag emitted a stray element. - guid had no isPermaLink="false", so a reader may treat it as a fetchable URL. - Errors from ReadDir, Info and Marshal were discarded, so a missing directory produced an empty feed that looked like a working setup with nothing in it. - os.Args indexed without a length check. Beyond the sketch: a download archive so an interrupted series resumes, an up-front check for yt-dlp and ffmpeg before the long download rather than a post-processing failure after the bytes are on disk, non-audio files excluded by extension so cover art and .part files cannot become episodes, and a serve command with token gating that 404s rather than 401s. Found by the first live run against a LibriVox collection: sources whose own titles begin with a track number came out as "01 - 01 - April.mp3", because the output template prepends the playlist index regardless. Collapsed, but only when the two numbers agree — "01 - 1964 - Lecture" is a year and "05 - 12 Angry Men" is a title, and neither is a repeat. 47 tests. Verified end to end against public-domain audio: downloaded, converted, tagged, served, and the delivered file is a playable 128kbps MP3 with correct ID3 title/artist/album/track frames. --- internal/fetch/fetch.go | 185 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 internal/fetch/fetch.go (limited to 'internal/fetch/fetch.go') diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go new file mode 100644 index 0000000..6dc4763 --- /dev/null +++ b/internal/fetch/fetch.go @@ -0,0 +1,185 @@ +// Package fetch wraps yt-dlp for audio extraction. +package fetch + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +// Options configure a download run. +type Options struct { + URL string + Dir string + Format string // audio codec for --audio-format, e.g. "mp3" + // Archive records completed downloads so a re-run skips them. This is what + // makes an interrupted 7-hour lecture series resumable rather than starting + // over, and makes re-running the whole pipeline cheap. + Archive bool + Stdout io.Writer + Stderr io.Writer +} + +// ErrNotInstalled is returned when yt-dlp cannot be found. +type ErrNotInstalled struct{ Err error } + +func (e *ErrNotInstalled) Error() string { + return "yt-dlp is not installed or not on PATH.\n" + + "Install it with: pip install --user yt-dlp\n" + + "(ffmpeg is also required for audio extraction.)" +} + +func (e *ErrNotInstalled) Unwrap() error { return e.Err } + +// Check reports whether the external tools are available. +// +// Checked up front rather than letting the command fail mid-run: a missing +// ffmpeg surfaces from yt-dlp as a post-processing error after the download has +// already happened, which is a confusing place to learn about it. +func Check() error { + if _, err := exec.LookPath("yt-dlp"); err != nil { + return &ErrNotInstalled{Err: err} + } + if _, err := exec.LookPath("ffmpeg"); err != nil { + return fmt.Errorf("ffmpeg is not installed or not on PATH; yt-dlp needs it to extract audio") + } + return nil +} + +// Run downloads and extracts audio into opts.Dir. +func Run(ctx context.Context, opts Options) error { + if opts.URL == "" { + return fmt.Errorf("a playlist or video URL is required") + } + if err := Check(); err != nil { + return err + } + if err := os.MkdirAll(opts.Dir, 0o755); err != nil { + return fmt.Errorf("creating %s: %w", opts.Dir, err) + } + + format := opts.Format + if format == "" { + format = "mp3" + } + + // %(playlist_index)02d is empty for a single video, which would leave a + // filename starting with " - ". The |NA default plus autonumber fallback + // keeps single downloads sensibly named. + template := filepath.Join(opts.Dir, "%(playlist_index|)02d - %(title)s.%(ext)s") + + args := []string{ + "--extract-audio", + "--audio-format", format, + "--audio-quality", "0", + "--embed-metadata", + "--embed-thumbnail", + // Keep going if one item in a long playlist is unavailable rather than + // abandoning the rest. + "--ignore-errors", + "--no-overwrites", + "--restrict-filenames", + "--output", template, + } + if opts.Archive { + args = append(args, "--download-archive", filepath.Join(opts.Dir, ".podkit-archive")) + } + args = append(args, opts.URL) + + cmd := exec.CommandContext(ctx, "yt-dlp", args...) + cmd.Stdout = opts.Stdout + cmd.Stderr = opts.Stderr + if cmd.Stdout == nil { + cmd.Stdout = os.Stdout + } + if cmd.Stderr == nil { + cmd.Stderr = os.Stderr + } + + if err := cmd.Run(); err != nil { + return fmt.Errorf("yt-dlp failed: %w", err) + } + return nil +} + +// Rename normalises yt-dlp's --restrict-filenames output back into readable +// "NN - Title.ext" names. +// +// --restrict-filenames is used because unrestricted titles produce filenames +// with quotes, slashes and emoji that make both shell use and URL construction +// unpleasant. The cost is underscores everywhere, so they are converted back +// here where it is safe to do so. +func Rename(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", dir, err) + } + + var renamed []string + for _, e := range entries { + if e.IsDir() || strings.HasPrefix(e.Name(), ".") { + continue + } + pretty := prettifyName(e.Name()) + if pretty == e.Name() { + continue + } + from := filepath.Join(dir, e.Name()) + to := filepath.Join(dir, pretty) + if _, err := os.Stat(to); err == nil { + // Never clobber an existing file to make a name prettier. + continue + } + if err := os.Rename(from, to); err != nil { + return renamed, fmt.Errorf("renaming %s: %w", e.Name(), err) + } + renamed = append(renamed, pretty) + } + return renamed, nil +} + +// duplicatedPrefix matches "01 - 01 - Title", where the output template's +// index has been prepended to a title that already carried the same number. +var duplicatedPrefix = regexp.MustCompile(`^(\d+)\s*-\s*(\d+)\s*-\s*(.+)$`) + +func prettifyName(name string) string { + ext := filepath.Ext(name) + stem := strings.TrimSuffix(name, ext) + stem = strings.ReplaceAll(stem, "_", " ") + // Collapse the runs of spaces that replacing underscores tends to create. + for strings.Contains(stem, " ") { + stem = strings.ReplaceAll(stem, " ", " ") + } + stem = strings.TrimSpace(stem) + + // Sources whose own titles start with a track number end up doubled, since + // the output template prepends the playlist index regardless: a LibriVox + // collection produced "01 - 01 - April.mp3" on the first live run. + // + // Only collapse when the two numbers agree. "01 - 1964 - Lecture" is a year, + // not a repeat, and "05 - 12 Angry Men" is a title that happens to start + // with a digit — neither should be touched. + if m := duplicatedPrefix.FindStringSubmatch(stem); m != nil { + first, err1 := strconv.Atoi(m[1]) + second, err2 := strconv.Atoi(m[2]) + if err1 == nil && err2 == nil && first == second { + stem = m[1] + " - " + strings.TrimSpace(m[3]) + } + } + return stem + ext +} + +// StreamLines copies r to w line by line, for surfacing yt-dlp progress. +func StreamLines(r io.Reader, w io.Writer) { + sc := bufio.NewScanner(r) + for sc.Scan() { + fmt.Fprintln(w, sc.Text()) + } +} -- cgit v1.2.3