package fetch import ( "context" "encoding/xml" "fmt" "io" "net/http" "net/url" "os" "path" "path/filepath" "strings" "podkit/internal/feed" ) // userAgent identifies podkit to feed hosts. // // Go's default ("Go-http-client/…") is rejected outright by several podcast // CDNs — libsyn returns 403 for it. Identifying honestly as what this is gets a // normal response and leaves an accurate trace in the host's logs. const userAgent = "podkit/1.0 (+https://github.com/; podcast feed mirror)" func get(ctx context.Context, c *http.Client, rawURL string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", userAgent) return c.Do(req) } // Source is one downloadable audio file discovered from a URL. type Source struct { URL string Name string Track int } // rssEnvelope is the minimum needed to pull enclosures out of a podcast feed. type rssEnvelope struct { Channel struct { Title string `xml:"title"` Items []struct { Title string `xml:"title"` Enclosure struct { URL string `xml:"url,attr"` Type string `xml:"type,attr"` } `xml:"enclosure"` } `xml:"item"` } `xml:"channel"` } // Discover resolves a URL into the audio files behind it. // // Handles two cases that need no extractor at all: a direct link to an audio // file, and an RSS feed whose items carry enclosures. Between them these cover // archive.org, university lecture pages, and every existing podcast — none of // which require yt-dlp, and none of which are subject to YouTube's bot checks. func Discover(ctx context.Context, rawURL string, client *http.Client) ([]Source, string, error) { if client == nil { client = http.DefaultClient } u, err := url.Parse(rawURL) if err != nil || u.Scheme == "" || u.Host == "" { return nil, "", fmt.Errorf("%q is not a usable URL", rawURL) } // A direct audio link needs no fetch to classify. if feed.IsAudio(path.Base(u.Path)) { return []Source{{URL: rawURL, Name: nameFromURL(u)}}, "", nil } resp, err := get(ctx, client, rawURL) if err != nil { return nil, "", fmt.Errorf("fetching %s: %w", rawURL, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, "", fmt.Errorf("fetching %s: %s", rawURL, resp.Status) } // Capped: a feed is text, and an unbounded read here would let a wrong URL // pull an arbitrary amount into memory. body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) if err != nil { return nil, "", fmt.Errorf("reading %s: %w", rawURL, err) } var rss rssEnvelope if err := xml.Unmarshal(body, &rss); err != nil { return nil, "", fmt.Errorf("%s is neither a direct audio file nor an RSS feed", rawURL) } if len(rss.Channel.Items) == 0 { return nil, "", fmt.Errorf("%s parsed as XML but has no items with enclosures", rawURL) } var sources []Source // RSS is newest-first by convention, so the last item is episode 1. Numbering // in reverse keeps a series in listening order once it reaches the feed. total := len(rss.Channel.Items) for i, it := range rss.Channel.Items { if it.Enclosure.URL == "" { continue } eu, err := url.Parse(it.Enclosure.URL) if err != nil { continue } track := total - i title := strings.TrimSpace(it.Title) if title == "" { title = nameFromURL(eu) } sources = append(sources, Source{ URL: it.Enclosure.URL, Name: fmt.Sprintf("%02d - %s%s", track, sanitise(title), extOf(eu, it.Enclosure.Type)), Track: track, }) } if len(sources) == 0 { return nil, "", fmt.Errorf("%s has items but none carry an audio enclosure", rawURL) } return sources, strings.TrimSpace(rss.Channel.Title), nil } // Download writes sources into dir, skipping ones already present. func Download(ctx context.Context, sources []Source, dir string, client *http.Client, progress io.Writer) error { if client == nil { client = http.DefaultClient } if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("creating %s: %w", dir, err) } for _, s := range sources { target := filepath.Join(dir, s.Name) if _, err := os.Stat(target); err == nil { if progress != nil { fmt.Fprintf(progress, " have %s\n", s.Name) } continue } if progress != nil { fmt.Fprintf(progress, " get %s\n", s.Name) } if err := downloadOne(ctx, client, s.URL, target); err != nil { return err } } return nil } func downloadOne(ctx context.Context, c *http.Client, src, target string) error { resp, err := get(ctx, c, src) if err != nil { return fmt.Errorf("downloading %s: %w", src, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("downloading %s: %s", src, resp.Status) } // Downloaded to a partial name and renamed on success, so an interrupted // run leaves nothing that looks like a complete episode. tmp := target + ".part" f, err := os.Create(tmp) if err != nil { return err } _, copyErr := io.Copy(f, resp.Body) closeErr := f.Close() if copyErr != nil { os.Remove(tmp) return fmt.Errorf("downloading %s: %w", src, copyErr) } if closeErr != nil { os.Remove(tmp) return closeErr } return os.Rename(tmp, target) } func nameFromURL(u *url.URL) string { name := path.Base(u.Path) if name == "" || name == "." || name == "/" { return "audio.mp3" } if decoded, err := url.PathUnescape(name); err == nil { name = decoded } return sanitise(name) } // extOf picks a file extension from the enclosure URL, falling back to its // declared MIME type, then to .mp3. func extOf(u *url.URL, mimeType string) string { if ext := path.Ext(u.Path); feed.IsAudio("x" + ext) { return ext } switch strings.ToLower(strings.TrimSpace(mimeType)) { case "audio/mp4", "audio/x-m4a": return ".m4a" case "audio/aac": return ".aac" case "audio/ogg": return ".ogg" case "audio/opus": return ".opus" case "audio/flac": return ".flac" case "audio/wav", "audio/x-wav": return ".wav" default: return ".mp3" } } // sanitise strips the characters that make a filename awkward in a shell or a // URL. Spaces are kept — the feed generator escapes them properly. func sanitise(s string) string { s = strings.Map(func(r rune) rune { switch r { case '/', '\\', ':', '*', '?', '"', '<', '>', '|', '\n', '\r', '\t': return -1 } if r < 0x20 { return -1 } return r }, s) for strings.Contains(s, " ") { s = strings.ReplaceAll(s, " ", " ") } s = strings.TrimSpace(s) s = strings.TrimLeft(s, ".") if s == "" { return "episode" } // Keep well clear of the 255-byte limit most filesystems impose, leaving // room for the extension and the ".part" suffix used while downloading. if len(s) > 180 { s = strings.TrimSpace(s[:180]) } return s }