Split download from hosting; carry the token into enclosure URLsHEADmain

The machine that hosts a feed and the machine that downloads the audio do
not have to be the same, and usually shouldn't be: a datacentre IP is
treated very differently from a home connection by the larger video sites.
Downloading where the person actually is sidesteps that entirely, and means
the server needs no credentials for or route to the source.

- push/ingest: `podkit push` uploads a local library to a remote
  `serve --ingest`, asking for a manifest first so only the difference is
  sent. Uploads land via a temp file and rename, so an interrupted transfer
  cannot be picked up as a short episode. Ingest is off by default and
  refuses to start without a token.
- get: downloads a direct audio URL or every enclosure in an RSS feed,
  with no extractor involved. Numbers RSS items in reverse so a mirrored
  series plays in order rather than backwards.

Two bugs found by running it rather than by testing it:

- A tokened server produced enclosure URLs with no token. The feed loaded,
  listed every episode, and all of them 404'd in a real client — the whole
  point of the tool, broken in the default configuration. Feeds now carry
  the token; regression test asserts enclosures from a tokened feed
  actually resolve against a tokened server.
- Ingest ran filepath.Base before validating, so "../evil.mp3" was stored
  as "evil.mp3" — contained, but under a name the client never asked for,
  leaving push and the server disagreeing about the library. Names are now
  validated raw and rejected.

Also: identify with a real user-agent (libsyn 403s Go's default), write
feed.xml via rename so a polling client never reads a truncated feed, and
match ingested file permissions to downloaded ones.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B

1 files changed, 247 insertions, 0 deletions

diff --git a/internal/fetch/direct.go b/internal/fetch/direct.go
new file mode 100644
index 0000000..9bd7116
--- /dev/null
+++ b/internal/fetch/direct.go
@@ -0,0 +1,247 @@
+package fetch
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"podkit/internal/feed"
+)
+
+// userAgent identifies podkit to feed hosts.
+//
+// Go's default ("Go-http-client/…") is rejected outright by several podcast
+// CDNs — libsyn returns 403 for it. Identifying honestly as what this is gets a
+// normal response and leaves an accurate trace in the host's logs.
+const userAgent = "podkit/1.0 (+https://github.com/; podcast feed mirror)"
+
+func get(ctx context.Context, c *http.Client, rawURL string) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+	return c.Do(req)
+}
+
+// Source is one downloadable audio file discovered from a URL.
+type Source struct {
+	URL   string
+	Name  string
+	Track int
+}
+
+// rssEnvelope is the minimum needed to pull enclosures out of a podcast feed.
+type rssEnvelope struct {
+	Channel struct {
+		Title string `xml:"title"`
+		Items []struct {
+			Title     string `xml:"title"`
+			Enclosure struct {
+				URL  string `xml:"url,attr"`
+				Type string `xml:"type,attr"`
+			} `xml:"enclosure"`
+		} `xml:"item"`
+	} `xml:"channel"`
+}
+
+// Discover resolves a URL into the audio files behind it.
+//
+// Handles two cases that need no extractor at all: a direct link to an audio
+// file, and an RSS feed whose items carry enclosures. Between them these cover
+// archive.org, university lecture pages, and every existing podcast — none of
+// which require yt-dlp, and none of which are subject to YouTube's bot checks.
+func Discover(ctx context.Context, rawURL string, client *http.Client) ([]Source, string, error) {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	u, err := url.Parse(rawURL)
+	if err != nil || u.Scheme == "" || u.Host == "" {
+		return nil, "", fmt.Errorf("%q is not a usable URL", rawURL)
+	}
+
+	// A direct audio link needs no fetch to classify.
+	if feed.IsAudio(path.Base(u.Path)) {
+		return []Source{{URL: rawURL, Name: nameFromURL(u)}}, "", nil
+	}
+
+	resp, err := get(ctx, client, rawURL)
+	if err != nil {
+		return nil, "", fmt.Errorf("fetching %s: %w", rawURL, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, "", fmt.Errorf("fetching %s: %s", rawURL, resp.Status)
+	}
+
+	// Capped: a feed is text, and an unbounded read here would let a wrong URL
+	// pull an arbitrary amount into memory.
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
+	if err != nil {
+		return nil, "", fmt.Errorf("reading %s: %w", rawURL, err)
+	}
+
+	var rss rssEnvelope
+	if err := xml.Unmarshal(body, &rss); err != nil {
+		return nil, "", fmt.Errorf("%s is neither a direct audio file nor an RSS feed", rawURL)
+	}
+	if len(rss.Channel.Items) == 0 {
+		return nil, "", fmt.Errorf("%s parsed as XML but has no items with enclosures", rawURL)
+	}
+
+	var sources []Source
+	// RSS is newest-first by convention, so the last item is episode 1. Numbering
+	// in reverse keeps a series in listening order once it reaches the feed.
+	total := len(rss.Channel.Items)
+	for i, it := range rss.Channel.Items {
+		if it.Enclosure.URL == "" {
+			continue
+		}
+		eu, err := url.Parse(it.Enclosure.URL)
+		if err != nil {
+			continue
+		}
+		track := total - i
+		title := strings.TrimSpace(it.Title)
+		if title == "" {
+			title = nameFromURL(eu)
+		}
+		sources = append(sources, Source{
+			URL:   it.Enclosure.URL,
+			Name:  fmt.Sprintf("%02d - %s%s", track, sanitise(title), extOf(eu, it.Enclosure.Type)),
+			Track: track,
+		})
+	}
+	if len(sources) == 0 {
+		return nil, "", fmt.Errorf("%s has items but none carry an audio enclosure", rawURL)
+	}
+	return sources, strings.TrimSpace(rss.Channel.Title), nil
+}
+
+// Download writes sources into dir, skipping ones already present.
+func Download(ctx context.Context, sources []Source, dir string, client *http.Client, progress io.Writer) error {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("creating %s: %w", dir, err)
+	}
+
+	for _, s := range sources {
+		target := filepath.Join(dir, s.Name)
+		if _, err := os.Stat(target); err == nil {
+			if progress != nil {
+				fmt.Fprintf(progress, "    have %s\n", s.Name)
+			}
+			continue
+		}
+		if progress != nil {
+			fmt.Fprintf(progress, "    get  %s\n", s.Name)
+		}
+		if err := downloadOne(ctx, client, s.URL, target); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func downloadOne(ctx context.Context, c *http.Client, src, target string) error {
+	resp, err := get(ctx, c, src)
+	if err != nil {
+		return fmt.Errorf("downloading %s: %w", src, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("downloading %s: %s", src, resp.Status)
+	}
+
+	// Downloaded to a partial name and renamed on success, so an interrupted
+	// run leaves nothing that looks like a complete episode.
+	tmp := target + ".part"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return err
+	}
+	_, copyErr := io.Copy(f, resp.Body)
+	closeErr := f.Close()
+	if copyErr != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("downloading %s: %w", src, copyErr)
+	}
+	if closeErr != nil {
+		os.Remove(tmp)
+		return closeErr
+	}
+	return os.Rename(tmp, target)
+}
+
+func nameFromURL(u *url.URL) string {
+	name := path.Base(u.Path)
+	if name == "" || name == "." || name == "/" {
+		return "audio.mp3"
+	}
+	if decoded, err := url.PathUnescape(name); err == nil {
+		name = decoded
+	}
+	return sanitise(name)
+}
+
+// extOf picks a file extension from the enclosure URL, falling back to its
+// declared MIME type, then to .mp3.
+func extOf(u *url.URL, mimeType string) string {
+	if ext := path.Ext(u.Path); feed.IsAudio("x" + ext) {
+		return ext
+	}
+	switch strings.ToLower(strings.TrimSpace(mimeType)) {
+	case "audio/mp4", "audio/x-m4a":
+		return ".m4a"
+	case "audio/aac":
+		return ".aac"
+	case "audio/ogg":
+		return ".ogg"
+	case "audio/opus":
+		return ".opus"
+	case "audio/flac":
+		return ".flac"
+	case "audio/wav", "audio/x-wav":
+		return ".wav"
+	default:
+		return ".mp3"
+	}
+}
+
+// sanitise strips the characters that make a filename awkward in a shell or a
+// URL. Spaces are kept — the feed generator escapes them properly.
+func sanitise(s string) string {
+	s = strings.Map(func(r rune) rune {
+		switch r {
+		case '/', '\\', ':', '*', '?', '"', '<', '>', '|', '\n', '\r', '\t':
+			return -1
+		}
+		if r < 0x20 {
+			return -1
+		}
+		return r
+	}, s)
+	for strings.Contains(s, "  ") {
+		s = strings.ReplaceAll(s, "  ", " ")
+	}
+	s = strings.TrimSpace(s)
+	s = strings.TrimLeft(s, ".")
+	if s == "" {
+		return "episode"
+	}
+	// Keep well clear of the 255-byte limit most filesystems impose, leaving
+	// room for the extension and the ".part" suffix used while downloading.
+	if len(s) > 180 {
+		s = strings.TrimSpace(s[:180])
+	}
+	return s
+}