From e3ecae12d7a309eb5a6c8e62910257096cb99375 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 07:05:24 +0000 Subject: Split download from hosting; carry the token into enclosure URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The machine that hosts a feed and the machine that downloads the audio do not have to be the same, and usually shouldn't be: a datacentre IP is treated very differently from a home connection by the larger video sites. Downloading where the person actually is sidesteps that entirely, and means the server needs no credentials for or route to the source. - push/ingest: `podkit push` uploads a local library to a remote `serve --ingest`, asking for a manifest first so only the difference is sent. Uploads land via a temp file and rename, so an interrupted transfer cannot be picked up as a short episode. Ingest is off by default and refuses to start without a token. - get: downloads a direct audio URL or every enclosure in an RSS feed, with no extractor involved. Numbers RSS items in reverse so a mirrored series plays in order rather than backwards. Two bugs found by running it rather than by testing it: - A tokened server produced enclosure URLs with no token. The feed loaded, listed every episode, and all of them 404'd in a real client — the whole point of the tool, broken in the default configuration. Feeds now carry the token; regression test asserts enclosures from a tokened feed actually resolve against a tokened server. - Ingest ran filepath.Base before validating, so "../evil.mp3" was stored as "evil.mp3" — contained, but under a name the client never asked for, leaving push and the server disagreeing about the library. Names are now validated raw and rejected. Also: identify with a real user-agent (libsyn 403s Go's default), write feed.xml via rename so a polling client never reads a truncated feed, and match ingested file permissions to downloaded ones. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- internal/fetch/direct.go | 247 ++++++++++++++++++++++++++++++++++++++++++ internal/fetch/direct_test.go | 242 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 489 insertions(+) create mode 100644 internal/fetch/direct.go create mode 100644 internal/fetch/direct_test.go (limited to 'internal/fetch') diff --git a/internal/fetch/direct.go b/internal/fetch/direct.go new file mode 100644 index 0000000..9bd7116 --- /dev/null +++ b/internal/fetch/direct.go @@ -0,0 +1,247 @@ +package fetch + +import ( + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "podkit/internal/feed" +) + +// userAgent identifies podkit to feed hosts. +// +// Go's default ("Go-http-client/…") is rejected outright by several podcast +// CDNs — libsyn returns 403 for it. Identifying honestly as what this is gets a +// normal response and leaves an accurate trace in the host's logs. +const userAgent = "podkit/1.0 (+https://github.com/; podcast feed mirror)" + +func get(ctx context.Context, c *http.Client, rawURL string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + return c.Do(req) +} + +// Source is one downloadable audio file discovered from a URL. +type Source struct { + URL string + Name string + Track int +} + +// rssEnvelope is the minimum needed to pull enclosures out of a podcast feed. +type rssEnvelope struct { + Channel struct { + Title string `xml:"title"` + Items []struct { + Title string `xml:"title"` + Enclosure struct { + URL string `xml:"url,attr"` + Type string `xml:"type,attr"` + } `xml:"enclosure"` + } `xml:"item"` + } `xml:"channel"` +} + +// Discover resolves a URL into the audio files behind it. +// +// Handles two cases that need no extractor at all: a direct link to an audio +// file, and an RSS feed whose items carry enclosures. Between them these cover +// archive.org, university lecture pages, and every existing podcast — none of +// which require yt-dlp, and none of which are subject to YouTube's bot checks. +func Discover(ctx context.Context, rawURL string, client *http.Client) ([]Source, string, error) { + if client == nil { + client = http.DefaultClient + } + u, err := url.Parse(rawURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return nil, "", fmt.Errorf("%q is not a usable URL", rawURL) + } + + // A direct audio link needs no fetch to classify. + if feed.IsAudio(path.Base(u.Path)) { + return []Source{{URL: rawURL, Name: nameFromURL(u)}}, "", nil + } + + resp, err := get(ctx, client, rawURL) + if err != nil { + return nil, "", fmt.Errorf("fetching %s: %w", rawURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("fetching %s: %s", rawURL, resp.Status) + } + + // Capped: a feed is text, and an unbounded read here would let a wrong URL + // pull an arbitrary amount into memory. + body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + if err != nil { + return nil, "", fmt.Errorf("reading %s: %w", rawURL, err) + } + + var rss rssEnvelope + if err := xml.Unmarshal(body, &rss); err != nil { + return nil, "", fmt.Errorf("%s is neither a direct audio file nor an RSS feed", rawURL) + } + if len(rss.Channel.Items) == 0 { + return nil, "", fmt.Errorf("%s parsed as XML but has no items with enclosures", rawURL) + } + + var sources []Source + // RSS is newest-first by convention, so the last item is episode 1. Numbering + // in reverse keeps a series in listening order once it reaches the feed. + total := len(rss.Channel.Items) + for i, it := range rss.Channel.Items { + if it.Enclosure.URL == "" { + continue + } + eu, err := url.Parse(it.Enclosure.URL) + if err != nil { + continue + } + track := total - i + title := strings.TrimSpace(it.Title) + if title == "" { + title = nameFromURL(eu) + } + sources = append(sources, Source{ + URL: it.Enclosure.URL, + Name: fmt.Sprintf("%02d - %s%s", track, sanitise(title), extOf(eu, it.Enclosure.Type)), + Track: track, + }) + } + if len(sources) == 0 { + return nil, "", fmt.Errorf("%s has items but none carry an audio enclosure", rawURL) + } + return sources, strings.TrimSpace(rss.Channel.Title), nil +} + +// Download writes sources into dir, skipping ones already present. +func Download(ctx context.Context, sources []Source, dir string, client *http.Client, progress io.Writer) error { + if client == nil { + client = http.DefaultClient + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating %s: %w", dir, err) + } + + for _, s := range sources { + target := filepath.Join(dir, s.Name) + if _, err := os.Stat(target); err == nil { + if progress != nil { + fmt.Fprintf(progress, " have %s\n", s.Name) + } + continue + } + if progress != nil { + fmt.Fprintf(progress, " get %s\n", s.Name) + } + if err := downloadOne(ctx, client, s.URL, target); err != nil { + return err + } + } + return nil +} + +func downloadOne(ctx context.Context, c *http.Client, src, target string) error { + resp, err := get(ctx, c, src) + if err != nil { + return fmt.Errorf("downloading %s: %w", src, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("downloading %s: %s", src, resp.Status) + } + + // Downloaded to a partial name and renamed on success, so an interrupted + // run leaves nothing that looks like a complete episode. + tmp := target + ".part" + f, err := os.Create(tmp) + if err != nil { + return err + } + _, copyErr := io.Copy(f, resp.Body) + closeErr := f.Close() + if copyErr != nil { + os.Remove(tmp) + return fmt.Errorf("downloading %s: %w", src, copyErr) + } + if closeErr != nil { + os.Remove(tmp) + return closeErr + } + return os.Rename(tmp, target) +} + +func nameFromURL(u *url.URL) string { + name := path.Base(u.Path) + if name == "" || name == "." || name == "/" { + return "audio.mp3" + } + if decoded, err := url.PathUnescape(name); err == nil { + name = decoded + } + return sanitise(name) +} + +// extOf picks a file extension from the enclosure URL, falling back to its +// declared MIME type, then to .mp3. +func extOf(u *url.URL, mimeType string) string { + if ext := path.Ext(u.Path); feed.IsAudio("x" + ext) { + return ext + } + switch strings.ToLower(strings.TrimSpace(mimeType)) { + case "audio/mp4", "audio/x-m4a": + return ".m4a" + case "audio/aac": + return ".aac" + case "audio/ogg": + return ".ogg" + case "audio/opus": + return ".opus" + case "audio/flac": + return ".flac" + case "audio/wav", "audio/x-wav": + return ".wav" + default: + return ".mp3" + } +} + +// sanitise strips the characters that make a filename awkward in a shell or a +// URL. Spaces are kept — the feed generator escapes them properly. +func sanitise(s string) string { + s = strings.Map(func(r rune) rune { + switch r { + case '/', '\\', ':', '*', '?', '"', '<', '>', '|', '\n', '\r', '\t': + return -1 + } + if r < 0x20 { + return -1 + } + return r + }, s) + for strings.Contains(s, " ") { + s = strings.ReplaceAll(s, " ", " ") + } + s = strings.TrimSpace(s) + s = strings.TrimLeft(s, ".") + if s == "" { + return "episode" + } + // Keep well clear of the 255-byte limit most filesystems impose, leaving + // room for the extension and the ".part" suffix used while downloading. + if len(s) > 180 { + s = strings.TrimSpace(s[:180]) + } + return s +} diff --git a/internal/fetch/direct_test.go b/internal/fetch/direct_test.go new file mode 100644 index 0000000..86b9e31 --- /dev/null +++ b/internal/fetch/direct_test.go @@ -0,0 +1,242 @@ +package fetch + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +const sampleRSS = ` + + + Lectures on Physics + + Three - The Great Conservation Principles + + + + Two - The Relation of Mathematics & Physics + + + + One - The Law of Gravitation + + + +` + +func rssServer(t *testing.T) *httptest.Server { + t.Helper() + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".mp3") { + fmt.Fprintf(w, "audio for %s", r.URL.Path) + return + } + w.Header().Set("Content-Type", "application/rss+xml") + fmt.Fprintf(w, sampleRSS, srv.URL, srv.URL, srv.URL) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestDiscoverDirectAudioURL(t *testing.T) { + sources, _, err := Discover(context.Background(), "https://example.org/talks/01%20-%20Intro.mp3", nil) + if err != nil { + t.Fatal(err) + } + if len(sources) != 1 { + t.Fatalf("got %d sources", len(sources)) + } + if sources[0].Name != "01 - Intro.mp3" { + t.Errorf("name %q — the URL escape should be decoded", sources[0].Name) + } +} + +func TestDiscoverRSSEnclosures(t *testing.T) { + srv := rssServer(t) + sources, title, err := Discover(context.Background(), srv.URL+"/feed.xml", nil) + if err != nil { + t.Fatal(err) + } + if title != "Lectures on Physics" { + t.Errorf("title %q", title) + } + if len(sources) != 3 { + t.Fatalf("got %d sources", len(sources)) + } +} + +// RSS is newest-first by convention, so the last item is episode 1. Numbering +// has to run in reverse or a mirrored course plays backwards. +func TestDiscoverNumbersRSSInListeningOrder(t *testing.T) { + srv := rssServer(t) + sources, _, err := Discover(context.Background(), srv.URL+"/feed.xml", nil) + if err != nil { + t.Fatal(err) + } + + byTrack := map[int]string{} + for _, s := range sources { + byTrack[s.Track] = s.Name + } + if !strings.Contains(byTrack[1], "One - The Law of Gravitation") { + t.Errorf("track 1 is %q, want the feed's last item", byTrack[1]) + } + if !strings.Contains(byTrack[3], "Three - The Great Conservation") { + t.Errorf("track 3 is %q, want the feed's first item", byTrack[3]) + } + if !strings.HasPrefix(byTrack[1], "01 - ") { + t.Errorf("track 1 name %q is not zero-padded", byTrack[1]) + } +} + +func TestDiscoverRejectsNonFeed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "not a feed") + })) + defer srv.Close() + + if _, _, err := Discover(context.Background(), srv.URL, nil); err == nil { + t.Error("expected an error for a page that is not a feed") + } +} + +func TestDiscoverRejectsBadURL(t *testing.T) { + for _, u := range []string{"", "not-a-url", "/just/a/path"} { + if _, _, err := Discover(context.Background(), u, nil); err == nil { + t.Errorf("%q was accepted", u) + } + } +} + +func TestDownloadWritesFiles(t *testing.T) { + srv := rssServer(t) + dir := t.TempDir() + + sources, _, err := Discover(context.Background(), srv.URL+"/feed.xml", nil) + if err != nil { + t.Fatal(err) + } + if err := Download(context.Background(), sources, dir, nil, nil); err != nil { + t.Fatal(err) + } + + entries, _ := os.ReadDir(dir) + if len(entries) != 3 { + t.Fatalf("got %d files", len(entries)) + } + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".part") { + t.Errorf("partial file %q left behind", e.Name()) + } + } +} + +// Re-running must not re-download; this is what makes a long series resumable. +func TestDownloadSkipsExisting(t *testing.T) { + var hits int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + fmt.Fprint(w, "audio") + })) + defer srv.Close() + + dir := t.TempDir() + sources := []Source{{URL: srv.URL + "/a.mp3", Name: "01 - A.mp3", Track: 1}} + + for i := 0; i < 3; i++ { + if err := Download(context.Background(), sources, dir, nil, nil); err != nil { + t.Fatal(err) + } + } + if hits != 1 { + t.Errorf("downloaded %d times, want 1", hits) + } +} + +// A failed download must not leave something the feed generator would treat as +// a complete episode. +func TestDownloadLeavesNothingOnFailure(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "gone", http.StatusNotFound) + })) + defer srv.Close() + + dir := t.TempDir() + err := Download(context.Background(), []Source{{URL: srv.URL + "/a.mp3", Name: "01 - A.mp3"}}, dir, nil, nil) + if err == nil { + t.Fatal("expected an error") + } + if entries, _ := os.ReadDir(dir); len(entries) != 0 { + t.Errorf("left %d file(s) behind after a failed download", len(entries)) + } +} + +func TestSanitiseRemovesPathCharacters(t *testing.T) { + cases := []struct{ in, want string }{ + {"A/B: The Sequel", "AB The Sequel"}, + {"What? Why!", "What Why!"}, + {" spaced out ", "spaced out"}, + // Separators are dropped and the leading dots trimmed, so a name that + // tried to traverse comes out as an ordinary filename. + {"../../etc/passwd", "etcpasswd"}, + {"", "episode"}, + {"...", "episode"}, + } + for _, c := range cases { + if got := sanitise(c.in); got != c.want { + t.Errorf("sanitise(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// A sanitised name must never be able to escape the download directory. +func TestSanitisedNamesStayInDirectory(t *testing.T) { + dir := t.TempDir() + for _, in := range []string{"../escape", "../../escape", "/absolute", "a/b/c"} { + joined := filepath.Join(dir, sanitise(in)+".mp3") + if !strings.HasPrefix(joined, dir+string(filepath.Separator)) { + t.Errorf("sanitise(%q) produced a path outside the directory: %s", in, joined) + } + } +} + +func TestExtOfPrefersURLThenMIME(t *testing.T) { + sources, _, err := Discover(context.Background(), "https://example.org/a/file.opus", nil) + if err != nil { + t.Fatal(err) + } + if filepath.Ext(sources[0].Name) != ".opus" { + t.Errorf("name %q", sources[0].Name) + } +} + +// An enclosure URL with no useful extension has to fall back to the declared +// MIME type, or the file lands as .mp3 and no player reads its tags. +func TestExtOfFallsBackToMIMEType(t *testing.T) { + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/stream") { + fmt.Fprint(w, "audio") + return + } + fmt.Fprintf(w, `T + Only + `, srv.URL) + })) + defer srv.Close() + + sources, _, err := Discover(context.Background(), srv.URL+"/feed.xml", nil) + if err != nil { + t.Fatal(err) + } + if filepath.Ext(sources[0].Name) != ".m4a" { + t.Errorf("name %q, want a .m4a extension from the audio/mp4 type", sources[0].Name) + } +} -- cgit v1.2.3