2 files changed, 489 insertions, 0 deletions
diff --git a/internal/fetch/direct.go b/internal/fetch/direct.go
new file mode 100644
index 0000000..9bd7116
--- /dev/null
+++ b/internal/fetch/direct.go
@@ -0,0 +1,247 @@
+package fetch
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"podkit/internal/feed"
+)
+
+// userAgent identifies podkit to feed hosts.
+//
+// Go's default ("Go-http-client/…") is rejected outright by several podcast
+// CDNs — libsyn returns 403 for it. Identifying honestly as what this is gets a
+// normal response and leaves an accurate trace in the host's logs.
+const userAgent = "podkit/1.0 (+https://github.com/; podcast feed mirror)"
+
+func get(ctx context.Context, c *http.Client, rawURL string) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+	return c.Do(req)
+}
+
+// Source is one downloadable audio file discovered from a URL.
+type Source struct {
+	URL   string
+	Name  string
+	Track int
+}
+
+// rssEnvelope is the minimum needed to pull enclosures out of a podcast feed.
+type rssEnvelope struct {
+	Channel struct {
+		Title string `xml:"title"`
+		Items []struct {
+			Title     string `xml:"title"`
+			Enclosure struct {
+				URL  string `xml:"url,attr"`
+				Type string `xml:"type,attr"`
+			} `xml:"enclosure"`
+		} `xml:"item"`
+	} `xml:"channel"`
+}
+
+// Discover resolves a URL into the audio files behind it.
+//
+// Handles two cases that need no extractor at all: a direct link to an audio
+// file, and an RSS feed whose items carry enclosures. Between them these cover
+// archive.org, university lecture pages, and every existing podcast — none of
+// which require yt-dlp, and none of which are subject to YouTube's bot checks.
+func Discover(ctx context.Context, rawURL string, client *http.Client) ([]Source, string, error) {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	u, err := url.Parse(rawURL)
+	if err != nil || u.Scheme == "" || u.Host == "" {
+		return nil, "", fmt.Errorf("%q is not a usable URL", rawURL)
+	}
+
+	// A direct audio link needs no fetch to classify.
+	if feed.IsAudio(path.Base(u.Path)) {
+		return []Source{{URL: rawURL, Name: nameFromURL(u)}}, "", nil
+	}
+
+	resp, err := get(ctx, client, rawURL)
+	if err != nil {
+		return nil, "", fmt.Errorf("fetching %s: %w", rawURL, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, "", fmt.Errorf("fetching %s: %s", rawURL, resp.Status)
+	}
+
+	// Capped: a feed is text, and an unbounded read here would let a wrong URL
+	// pull an arbitrary amount into memory.
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
+	if err != nil {
+		return nil, "", fmt.Errorf("reading %s: %w", rawURL, err)
+	}
+
+	var rss rssEnvelope
+	if err := xml.Unmarshal(body, &rss); err != nil {
+		return nil, "", fmt.Errorf("%s is neither a direct audio file nor an RSS feed", rawURL)
+	}
+	if len(rss.Channel.Items) == 0 {
+		return nil, "", fmt.Errorf("%s parsed as XML but has no items with enclosures", rawURL)
+	}
+
+	var sources []Source
+	// RSS is newest-first by convention, so the last item is episode 1. Numbering
+	// in reverse keeps a series in listening order once it reaches the feed.
+	total := len(rss.Channel.Items)
+	for i, it := range rss.Channel.Items {
+		if it.Enclosure.URL == "" {
+			continue
+		}
+		eu, err := url.Parse(it.Enclosure.URL)
+		if err != nil {
+			continue
+		}
+		track := total - i
+		title := strings.TrimSpace(it.Title)
+		if title == "" {
+			title = nameFromURL(eu)
+		}
+		sources = append(sources, Source{
+			URL:   it.Enclosure.URL,
+			Name:  fmt.Sprintf("%02d - %s%s", track, sanitise(title), extOf(eu, it.Enclosure.Type)),
+			Track: track,
+		})
+	}
+	if len(sources) == 0 {
+		return nil, "", fmt.Errorf("%s has items but none carry an audio enclosure", rawURL)
+	}
+	return sources, strings.TrimSpace(rss.Channel.Title), nil
+}
+
+// Download writes sources into dir, skipping ones already present.
+func Download(ctx context.Context, sources []Source, dir string, client *http.Client, progress io.Writer) error {
+	if client == nil {
+		client = http.DefaultClient
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("creating %s: %w", dir, err)
+	}
+
+	for _, s := range sources {
+		target := filepath.Join(dir, s.Name)
+		if _, err := os.Stat(target); err == nil {
+			if progress != nil {
+				fmt.Fprintf(progress, "    have %s\n", s.Name)
+			}
+			continue
+		}
+		if progress != nil {
+			fmt.Fprintf(progress, "    get  %s\n", s.Name)
+		}
+		if err := downloadOne(ctx, client, s.URL, target); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func downloadOne(ctx context.Context, c *http.Client, src, target string) error {
+	resp, err := get(ctx, c, src)
+	if err != nil {
+		return fmt.Errorf("downloading %s: %w", src, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("downloading %s: %s", src, resp.Status)
+	}
+
+	// Downloaded to a partial name and renamed on success, so an interrupted
+	// run leaves nothing that looks like a complete episode.
+	tmp := target + ".part"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return err
+	}
+	_, copyErr := io.Copy(f, resp.Body)
+	closeErr := f.Close()
+	if copyErr != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("downloading %s: %w", src, copyErr)
+	}
+	if closeErr != nil {
+		os.Remove(tmp)
+		return closeErr
+	}
+	return os.Rename(tmp, target)
+}
+
+func nameFromURL(u *url.URL) string {
+	name := path.Base(u.Path)
+	if name == "" || name == "." || name == "/" {
+		return "audio.mp3"
+	}
+	if decoded, err := url.PathUnescape(name); err == nil {
+		name = decoded
+	}
+	return sanitise(name)
+}
+
+// extOf picks a file extension from the enclosure URL, falling back to its
+// declared MIME type, then to .mp3.
+func extOf(u *url.URL, mimeType string) string {
+	if ext := path.Ext(u.Path); feed.IsAudio("x" + ext) {
+		return ext
+	}
+	switch strings.ToLower(strings.TrimSpace(mimeType)) {
+	case "audio/mp4", "audio/x-m4a":
+		return ".m4a"
+	case "audio/aac":
+		return ".aac"
+	case "audio/ogg":
+		return ".ogg"
+	case "audio/opus":
+		return ".opus"
+	case "audio/flac":
+		return ".flac"
+	case "audio/wav", "audio/x-wav":
+		return ".wav"
+	default:
+		return ".mp3"
+	}
+}
+
+// sanitise strips the characters that make a filename awkward in a shell or a
+// URL. Spaces are kept — the feed generator escapes them properly.
+func sanitise(s string) string {
+	s = strings.Map(func(r rune) rune {
+		switch r {
+		case '/', '\\', ':', '*', '?', '"', '<', '>', '|', '\n', '\r', '\t':
+			return -1
+		}
+		if r < 0x20 {
+			return -1
+		}
+		return r
+	}, s)
+	for strings.Contains(s, "  ") {
+		s = strings.ReplaceAll(s, "  ", " ")
+	}
+	s = strings.TrimSpace(s)
+	s = strings.TrimLeft(s, ".")
+	if s == "" {
+		return "episode"
+	}
+	// Keep well clear of the 255-byte limit most filesystems impose, leaving
+	// room for the extension and the ".part" suffix used while downloading.
+	if len(s) > 180 {
+		s = strings.TrimSpace(s[:180])
+	}
+	return s
+}
diff --git a/internal/fetch/direct_test.go b/internal/fetch/direct_test.go
new file mode 100644
index 0000000..86b9e31
--- /dev/null
+++ b/internal/fetch/direct_test.go
@@ -0,0 +1,242 @@
+package fetch
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+const sampleRSS = `<?xml version="1.0"?>
+<rss version="2.0">
+  <channel>
+    <title>Lectures on Physics</title>
+    <item>
+      <title>Three - The Great Conservation Principles</title>
+      <enclosure url="%s/three.mp3" type="audio/mpeg" length="10"/>
+    </item>
+    <item>
+      <title>Two - The Relation of Mathematics &amp; Physics</title>
+      <enclosure url="%s/two.mp3" type="audio/mpeg" length="10"/>
+    </item>
+    <item>
+      <title>One - The Law of Gravitation</title>
+      <enclosure url="%s/one.mp3" type="audio/mpeg" length="10"/>
+    </item>
+  </channel>
+</rss>`
+
+func rssServer(t *testing.T) *httptest.Server {
+	t.Helper()
+	var srv *httptest.Server
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, ".mp3") {
+			fmt.Fprintf(w, "audio for %s", r.URL.Path)
+			return
+		}
+		w.Header().Set("Content-Type", "application/rss+xml")
+		fmt.Fprintf(w, sampleRSS, srv.URL, srv.URL, srv.URL)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestDiscoverDirectAudioURL(t *testing.T) {
+	sources, _, err := Discover(context.Background(), "https://example.org/talks/01%20-%20Intro.mp3", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(sources) != 1 {
+		t.Fatalf("got %d sources", len(sources))
+	}
+	if sources[0].Name != "01 - Intro.mp3" {
+		t.Errorf("name %q — the URL escape should be decoded", sources[0].Name)
+	}
+}
+
+func TestDiscoverRSSEnclosures(t *testing.T) {
+	srv := rssServer(t)
+	sources, title, err := Discover(context.Background(), srv.URL+"/feed.xml", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if title != "Lectures on Physics" {
+		t.Errorf("title %q", title)
+	}
+	if len(sources) != 3 {
+		t.Fatalf("got %d sources", len(sources))
+	}
+}
+
+// RSS is newest-first by convention, so the last item is episode 1. Numbering
+// has to run in reverse or a mirrored course plays backwards.
+func TestDiscoverNumbersRSSInListeningOrder(t *testing.T) {
+	srv := rssServer(t)
+	sources, _, err := Discover(context.Background(), srv.URL+"/feed.xml", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	byTrack := map[int]string{}
+	for _, s := range sources {
+		byTrack[s.Track] = s.Name
+	}
+	if !strings.Contains(byTrack[1], "One - The Law of Gravitation") {
+		t.Errorf("track 1 is %q, want the feed's last item", byTrack[1])
+	}
+	if !strings.Contains(byTrack[3], "Three - The Great Conservation") {
+		t.Errorf("track 3 is %q, want the feed's first item", byTrack[3])
+	}
+	if !strings.HasPrefix(byTrack[1], "01 - ") {
+		t.Errorf("track 1 name %q is not zero-padded", byTrack[1])
+	}
+}
+
+func TestDiscoverRejectsNonFeed(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "<html><body>not a feed</body></html>")
+	}))
+	defer srv.Close()
+
+	if _, _, err := Discover(context.Background(), srv.URL, nil); err == nil {
+		t.Error("expected an error for a page that is not a feed")
+	}
+}
+
+func TestDiscoverRejectsBadURL(t *testing.T) {
+	for _, u := range []string{"", "not-a-url", "/just/a/path"} {
+		if _, _, err := Discover(context.Background(), u, nil); err == nil {
+			t.Errorf("%q was accepted", u)
+		}
+	}
+}
+
+func TestDownloadWritesFiles(t *testing.T) {
+	srv := rssServer(t)
+	dir := t.TempDir()
+
+	sources, _, err := Discover(context.Background(), srv.URL+"/feed.xml", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := Download(context.Background(), sources, dir, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	entries, _ := os.ReadDir(dir)
+	if len(entries) != 3 {
+		t.Fatalf("got %d files", len(entries))
+	}
+	for _, e := range entries {
+		if strings.HasSuffix(e.Name(), ".part") {
+			t.Errorf("partial file %q left behind", e.Name())
+		}
+	}
+}
+
+// Re-running must not re-download; this is what makes a long series resumable.
+func TestDownloadSkipsExisting(t *testing.T) {
+	var hits int
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits++
+		fmt.Fprint(w, "audio")
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	sources := []Source{{URL: srv.URL + "/a.mp3", Name: "01 - A.mp3", Track: 1}}
+
+	for i := 0; i < 3; i++ {
+		if err := Download(context.Background(), sources, dir, nil, nil); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if hits != 1 {
+		t.Errorf("downloaded %d times, want 1", hits)
+	}
+}
+
+// A failed download must not leave something the feed generator would treat as
+// a complete episode.
+func TestDownloadLeavesNothingOnFailure(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "gone", http.StatusNotFound)
+	}))
+	defer srv.Close()
+
+	dir := t.TempDir()
+	err := Download(context.Background(), []Source{{URL: srv.URL + "/a.mp3", Name: "01 - A.mp3"}}, dir, nil, nil)
+	if err == nil {
+		t.Fatal("expected an error")
+	}
+	if entries, _ := os.ReadDir(dir); len(entries) != 0 {
+		t.Errorf("left %d file(s) behind after a failed download", len(entries))
+	}
+}
+
+func TestSanitiseRemovesPathCharacters(t *testing.T) {
+	cases := []struct{ in, want string }{
+		{"A/B: The Sequel", "AB The Sequel"},
+		{"What? Why!", "What Why!"},
+		{"  spaced   out  ", "spaced out"},
+		// Separators are dropped and the leading dots trimmed, so a name that
+		// tried to traverse comes out as an ordinary filename.
+		{"../../etc/passwd", "etcpasswd"},
+		{"", "episode"},
+		{"...", "episode"},
+	}
+	for _, c := range cases {
+		if got := sanitise(c.in); got != c.want {
+			t.Errorf("sanitise(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+// A sanitised name must never be able to escape the download directory.
+func TestSanitisedNamesStayInDirectory(t *testing.T) {
+	dir := t.TempDir()
+	for _, in := range []string{"../escape", "../../escape", "/absolute", "a/b/c"} {
+		joined := filepath.Join(dir, sanitise(in)+".mp3")
+		if !strings.HasPrefix(joined, dir+string(filepath.Separator)) {
+			t.Errorf("sanitise(%q) produced a path outside the directory: %s", in, joined)
+		}
+	}
+}
+
+func TestExtOfPrefersURLThenMIME(t *testing.T) {
+	sources, _, err := Discover(context.Background(), "https://example.org/a/file.opus", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if filepath.Ext(sources[0].Name) != ".opus" {
+		t.Errorf("name %q", sources[0].Name)
+	}
+}
+
+// An enclosure URL with no useful extension has to fall back to the declared
+// MIME type, or the file lands as .mp3 and no player reads its tags.
+func TestExtOfFallsBackToMIMEType(t *testing.T) {
+	var srv *httptest.Server
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.URL.Path, "/stream") {
+			fmt.Fprint(w, "audio")
+			return
+		}
+		fmt.Fprintf(w, `<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>
+		  <item><title>Only</title><enclosure url="%s/stream?id=7" type="audio/mp4"/></item>
+		</channel></rss>`, srv.URL)
+	}))
+	defer srv.Close()
+
+	sources, _, err := Discover(context.Background(), srv.URL+"/feed.xml", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if filepath.Ext(sources[0].Name) != ".m4a" {
+		t.Errorf("name %q, want a .m4a extension from the audio/mp4 type", sources[0].Name)
+	}
+}