Split download from hosting; carry the token into enclosure URLsHEADmain

The machine that hosts a feed and the machine that downloads the audio do
not have to be the same, and usually shouldn't be: a datacentre IP is
treated very differently from a home connection by the larger video sites.
Downloading where the person actually is sidesteps that entirely, and means
the server needs no credentials for or route to the source.

- push/ingest: `podkit push` uploads a local library to a remote
  `serve --ingest`, asking for a manifest first so only the difference is
  sent. Uploads land via a temp file and rename, so an interrupted transfer
  cannot be picked up as a short episode. Ingest is off by default and
  refuses to start without a token.
- get: downloads a direct audio URL or every enclosure in an RSS feed,
  with no extractor involved. Numbers RSS items in reverse so a mirrored
  series plays in order rather than backwards.

Two bugs found by running it rather than by testing it:

- A tokened server produced enclosure URLs with no token. The feed loaded,
  listed every episode, and all of them 404'd in a real client — the whole
  point of the tool, broken in the default configuration. Feeds now carry
  the token; regression test asserts enclosures from a tokened feed
  actually resolve against a tokened server.
- Ingest ran filepath.Base before validating, so "../evil.mp3" was stored
  as "evil.mp3" — contained, but under a name the client never asked for,
  leaving push and the server disagreeing about the library. Names are now
  validated raw and rejected.

Also: identify with a real user-agent (libsyn 403s Go's default), write
feed.xml via rename so a polling client never reads a truncated feed, and
match ingested file permissions to downloaded ones.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B

4 files changed, 489 insertions, 0 deletions

diff --git a/internal/serve/ingest.go b/internal/serve/ingest.go
new file mode 100644
index 0000000..a949e1e
--- /dev/null
+++ b/internal/serve/ingest.go
@@ -0,0 +1,162 @@
+package serve
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"podkit/internal/feed"
+)
+
+// Manifest is what the server already holds, used by `podkit push` to decide
+// what still needs uploading. Keyed by filename.
+type Manifest struct {
+	Files map[string]int64 `json:"files"`
+}
+
+// maxUpload caps a single uploaded file. Generous enough for a long lecture at
+// a high bitrate; present so a runaway or hostile client cannot fill the disk
+// in one request.
+const maxUpload = 500 << 20 // 500 MiB
+
+// readManifest lists the audio already present in dir.
+func readManifest(dir string) (Manifest, error) {
+	m := Manifest{Files: map[string]int64{}}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return m, err
+	}
+	for _, e := range entries {
+		if e.IsDir() || !feed.IsAudio(e.Name()) {
+			continue
+		}
+		info, err := e.Info()
+		if err != nil {
+			// A file that vanished mid-scan just isn't in the manifest; push
+			// will then offer it, which is the safe direction to be wrong in.
+			continue
+		}
+		m.Files[e.Name()] = info.Size()
+	}
+	return m, nil
+}
+
+func handleManifest(dir string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		m, err := readManifest(dir)
+		if err != nil {
+			http.Error(w, "cannot read library", http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(m)
+	}
+}
+
+// handleIngest accepts an uploaded audio file and rebuilds the feed.
+//
+// This is the half of podkit that runs on the server. The download half runs
+// wherever the person doing it actually is — which is the point: the machine
+// hosting the feed never needs to talk to the source at all.
+func handleIngest(dir string, rebuild func() error) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			w.Header().Set("Allow", http.MethodPost)
+			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+
+		// Validated raw, not via filepath.Base: Base would quietly turn
+		// "../evil.mp3" into "evil.mp3" and store it under a name the client
+		// never asked for. That is safe from traversal but leaves push and the
+		// server disagreeing about what the library contains, which breaks the
+		// manifest comparison on the next run. A name with a separator in it is
+		// a bug or an attack either way, so reject it.
+		name := r.URL.Query().Get("name")
+		if !validUploadName(name) {
+			http.Error(w, "invalid or missing name parameter", http.StatusBadRequest)
+			return
+		}
+		if !feed.IsAudio(name) {
+			http.Error(w, "not an audio file", http.StatusUnsupportedMediaType)
+			return
+		}
+
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			http.Error(w, "cannot write to library", http.StatusInternalServerError)
+			return
+		}
+
+		// Written to a temporary name and renamed into place, so an upload cut
+		// off halfway can never be picked up as a half-length episode.
+		tmp, err := os.CreateTemp(dir, ".upload-*")
+		if err != nil {
+			http.Error(w, "cannot write to library", http.StatusInternalServerError)
+			return
+		}
+		tmpName := tmp.Name()
+		defer os.Remove(tmpName) // no-op once the rename below succeeds
+
+		written, err := io.Copy(tmp, http.MaxBytesReader(w, r.Body, maxUpload))
+		closeErr := tmp.Close()
+		if err != nil {
+			http.Error(w, "upload failed or too large", http.StatusBadRequest)
+			return
+		}
+		if closeErr != nil {
+			http.Error(w, "cannot write to library", http.StatusInternalServerError)
+			return
+		}
+		if written == 0 {
+			http.Error(w, "empty upload", http.StatusBadRequest)
+			return
+		}
+
+		// CreateTemp makes the file 0600; match the 0644 that downloaded files
+		// get, so a library's permissions don't depend on how each episode
+		// happened to arrive.
+		if err := os.Chmod(tmpName, 0o644); err != nil {
+			http.Error(w, "cannot write to library", http.StatusInternalServerError)
+			return
+		}
+
+		if err := os.Rename(tmpName, filepath.Join(dir, name)); err != nil {
+			http.Error(w, "cannot write to library", http.StatusInternalServerError)
+			return
+		}
+
+		// A new file that isn't in the feed is invisible to the podcast app, so
+		// a rebuild failure has to be reported rather than logged and forgotten.
+		if rebuild != nil {
+			if err := rebuild(); err != nil {
+				http.Error(w, fmt.Sprintf("stored %s but the feed rebuild failed: %v", name, err),
+					http.StatusInternalServerError)
+				return
+			}
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"stored": name,
+			"bytes":  written,
+		})
+	}
+}
+
+// validUploadName rejects names that would escape the directory or collide with
+// podkit's own files.
+func validUploadName(name string) bool {
+	switch name {
+	case "", ".", "..", "/", "feed.xml":
+		return false
+	}
+	if strings.ContainsAny(name, `/\`) {
+		return false
+	}
+	// Dotfiles are podkit's own bookkeeping (.podkit-archive, .upload-*).
+	return !strings.HasPrefix(name, ".")
+}
diff --git a/internal/serve/ingest_test.go b/internal/serve/ingest_test.go
new file mode 100644
index 0000000..38f7a86
--- /dev/null
+++ b/internal/serve/ingest_test.go
@@ -0,0 +1,210 @@
+package serve
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func ingestServer(t *testing.T, dir string, rebuild func() error) http.Handler {
+	t.Helper()
+	return Handler(Options{Dir: dir, Token: "secret", AllowIngest: true, Rebuild: rebuild})
+}
+
+func post(t *testing.T, h http.Handler, target, body string) *httptest.ResponseRecorder {
+	t.Helper()
+	req := httptest.NewRequest(http.MethodPost, target, strings.NewReader(body))
+	req.Header.Set("Authorization", "Bearer secret")
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+	return rec
+}
+
+func TestIngestStoresFileAndRebuilds(t *testing.T) {
+	dir := t.TempDir()
+	rebuilt := 0
+	h := ingestServer(t, dir, func() error { rebuilt++; return nil })
+
+	rec := post(t, h, "/ingest?name=01+-+April.mp3", "audio-bytes")
+	if rec.Code != http.StatusOK {
+		t.Fatalf("got %d: %s", rec.Code, rec.Body.String())
+	}
+
+	body, err := os.ReadFile(filepath.Join(dir, "01 - April.mp3"))
+	if err != nil {
+		t.Fatalf("file not stored: %v", err)
+	}
+	if string(body) != "audio-bytes" {
+		t.Errorf("stored %q", body)
+	}
+	if rebuilt != 1 {
+		t.Errorf("rebuild ran %d times, want 1", rebuilt)
+	}
+}
+
+// A stored file that never reaches the feed is invisible to the podcast app,
+// so a rebuild failure must surface as an error rather than a quiet success.
+func TestIngestReportsRebuildFailure(t *testing.T) {
+	dir := t.TempDir()
+	h := ingestServer(t, dir, func() error { return errors.New("no title configured") })
+
+	rec := post(t, h, "/ingest?name=01+-+April.mp3", "audio-bytes")
+	if rec.Code != http.StatusInternalServerError {
+		t.Fatalf("got %d, want 500", rec.Code)
+	}
+	if !strings.Contains(rec.Body.String(), "no title configured") {
+		t.Errorf("underlying cause not reported: %s", rec.Body.String())
+	}
+}
+
+func TestIngestRejectsPathTraversal(t *testing.T) {
+	dir := t.TempDir()
+	h := ingestServer(t, dir, nil)
+
+	for _, name := range []string{
+		"../escaped.mp3",
+		"..%2Fescaped.mp3",
+		"/etc/cron.d/evil.mp3",
+		"sub/dir.mp3",
+	} {
+		rec := post(t, h, "/ingest?name="+name, "x")
+		if rec.Code == http.StatusOK {
+			t.Errorf("name %q was accepted", name)
+		}
+	}
+
+	// Nothing may have been created outside the library directory.
+	if _, err := os.Stat(filepath.Join(filepath.Dir(dir), "escaped.mp3")); err == nil {
+		t.Error("a file was written outside the library directory")
+	}
+}
+
+func TestIngestRejectsNonAudio(t *testing.T) {
+	h := ingestServer(t, t.TempDir(), nil)
+	for _, name := range []string{"notes.txt", "cover.jpg", "script.sh", "feed.xml"} {
+		if rec := post(t, h, "/ingest?name="+name, "x"); rec.Code == http.StatusOK {
+			t.Errorf("name %q was accepted", name)
+		}
+	}
+}
+
+// Dotfiles are podkit's own bookkeeping; an upload must not be able to replace
+// the download archive or masquerade as a temporary upload.
+func TestIngestRejectsDotfiles(t *testing.T) {
+	h := ingestServer(t, t.TempDir(), nil)
+	for _, name := range []string{".podkit-archive", ".upload-x.mp3", ".mp3"} {
+		if rec := post(t, h, "/ingest?name="+name, "x"); rec.Code == http.StatusOK {
+			t.Errorf("name %q was accepted", name)
+		}
+	}
+}
+
+func TestIngestRejectsEmptyUpload(t *testing.T) {
+	dir := t.TempDir()
+	h := ingestServer(t, dir, nil)
+
+	rec := post(t, h, "/ingest?name=01+-+April.mp3", "")
+	if rec.Code == http.StatusOK {
+		t.Fatal("an empty upload was accepted")
+	}
+	if _, err := os.Stat(filepath.Join(dir, "01 - April.mp3")); err == nil {
+		t.Error("an empty file was left in the library")
+	}
+}
+
+// A half-written upload must never be left behind under its real name, where
+// the feed generator would pick it up as a complete episode.
+func TestIngestLeavesNoPartialFile(t *testing.T) {
+	dir := t.TempDir()
+	h := ingestServer(t, dir, nil)
+
+	post(t, h, "/ingest?name=01+-+April.mp3", "")
+
+	entries, _ := os.ReadDir(dir)
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), ".upload-") {
+			t.Errorf("temporary upload file %q was left behind", e.Name())
+		}
+	}
+}
+
+func TestIngestRequiresPost(t *testing.T) {
+	h := ingestServer(t, t.TempDir(), nil)
+	req := httptest.NewRequest(http.MethodGet, "/ingest?name=a.mp3", nil)
+	req.Header.Set("Authorization", "Bearer secret")
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+	if rec.Code != http.StatusMethodNotAllowed {
+		t.Errorf("got %d, want 405", rec.Code)
+	}
+}
+
+// Without the token, the write endpoints must be as invisible as the feed.
+func TestIngestRequiresToken(t *testing.T) {
+	dir := t.TempDir()
+	h := ingestServer(t, dir, nil)
+
+	for _, target := range []string{"/ingest?name=01+-+April.mp3", "/manifest"} {
+		req := httptest.NewRequest(http.MethodPost, target, strings.NewReader("x"))
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, req)
+		if rec.Code != http.StatusNotFound {
+			t.Errorf("%s without a token returned %d, want 404", target, rec.Code)
+		}
+	}
+	if _, err := os.Stat(filepath.Join(dir, "01 - April.mp3")); err == nil {
+		t.Error("an untokened upload was stored")
+	}
+}
+
+// A server started without --ingest must not expose the write endpoints at all.
+func TestIngestDisabledByDefault(t *testing.T) {
+	dir := t.TempDir()
+	h := Handler(Options{Dir: dir, Token: "secret"})
+
+	rec := post(t, h, "/ingest?name=01+-+April.mp3", "x")
+	if rec.Code == http.StatusOK {
+		t.Error("ingest was reachable on a server started without it")
+	}
+	if _, err := os.Stat(filepath.Join(dir, "01 - April.mp3")); err == nil {
+		t.Error("a file was stored by a server with ingest disabled")
+	}
+}
+
+func TestManifestListsAudioOnly(t *testing.T) {
+	dir := t.TempDir()
+	write := func(name, body string) {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("01 - April.mp3", "aaa")
+	write("02 - May.mp3", "bbbb")
+	write("feed.xml", "<rss/>")
+	write("cover.jpg", "img")
+
+	h := ingestServer(t, dir, nil)
+	req := httptest.NewRequest(http.MethodGet, "/manifest", nil)
+	req.Header.Set("Authorization", "Bearer secret")
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("got %d", rec.Code)
+	}
+	var m Manifest
+	if err := json.Unmarshal(rec.Body.Bytes(), &m); err != nil {
+		t.Fatal(err)
+	}
+	if len(m.Files) != 2 {
+		t.Fatalf("got %v, want 2 audio files", m.Files)
+	}
+	if m.Files["01 - April.mp3"] != 3 || m.Files["02 - May.mp3"] != 4 {
+		t.Errorf("sizes wrong: %v", m.Files)
+	}
+}
diff --git a/internal/serve/roundtrip_test.go b/internal/serve/roundtrip_test.go
new file mode 100644
index 0000000..c18de79
--- /dev/null
+++ b/internal/serve/roundtrip_test.go
@@ -0,0 +1,102 @@
+package serve
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"podkit/internal/feed"
+)
+
+// A podcast client fetches the feed, then fetches each enclosure as a bare GET
+// carrying nothing from the first request. If the server wants a token and the
+// enclosure URLs do not have one, the feed loads, lists every episode, and not
+// one of them will ever download — a failure that looks like success right up
+// until playback.
+//
+// Found live: a tokened server produced enclosure URLs without the token, and
+// every episode 404'd.
+func TestTokenedFeedEnclosuresAreFetchable(t *testing.T) {
+	const token = "s3cret"
+
+	dir := t.TempDir()
+	for _, name := range []string{"01 - April.mp3", "10 - Probability & Uncertainty.mp3"} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte("audio"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	srv := httptest.NewServer(Handler(Options{Dir: dir, Token: token}))
+	defer srv.Close()
+
+	// Build the feed exactly as the server would, pointed at itself.
+	rss, err := feed.Build(feed.Options{
+		Dir:     dir,
+		BaseURL: srv.URL + "/audio/",
+		Title:   "Test",
+		Token:   token,
+		Epoch:   time.Date(2000, time.January, 1, 12, 0, 0, 0, time.UTC),
+	}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rss.Channel.Items) != 2 {
+		t.Fatalf("got %d items", len(rss.Channel.Items))
+	}
+
+	for _, item := range rss.Channel.Items {
+		resp, err := http.Get(item.Enclosure.URL)
+		if err != nil {
+			t.Fatalf("GET %s: %v", item.Enclosure.URL, err)
+		}
+		resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			t.Errorf("enclosure %s returned %d — a podcast client could not download this episode",
+				item.Enclosure.URL, resp.StatusCode)
+		}
+	}
+}
+
+// The converse: without a token configured anywhere, enclosures must still
+// resolve, and must not carry a stray empty k= parameter.
+func TestUntokenedFeedEnclosuresAreFetchable(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "01 - April.mp3"), []byte("audio"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	srv := httptest.NewServer(Handler(Options{Dir: dir}))
+	defer srv.Close()
+
+	rss, err := feed.Build(feed.Options{
+		Dir: dir, BaseURL: srv.URL + "/audio/", Title: "Test",
+	}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	url := rss.Channel.Items[0].Enclosure.URL
+	if got := rss.Channel.Items[0].Enclosure.URL; len(got) > 0 && containsQ(got, "k=") {
+		t.Errorf("enclosure %s carries a token parameter when none was set", got)
+	}
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("enclosure %s returned %d", url, resp.StatusCode)
+	}
+}
+
+func containsQ(h, n string) bool {
+	for i := 0; i+len(n) <= len(h); i++ {
+		if h[i:i+len(n)] == n {
+			return true
+		}
+	}
+	return false
+}
diff --git a/internal/serve/serve.go b/internal/serve/serve.go
index 0b3b340..396babe 100644
--- a/internal/serve/serve.go
+++ b/internal/serve/serve.go
@@ -23,6 +23,16 @@ type Options struct {
 	// secret in the URL is the mechanism that actually works with them. Anyone
 	// holding the URL holds the access.
 	Token string
+
+	// AllowIngest enables /manifest and /ingest, letting `podkit push` upload
+	// episodes from elsewhere. Off by default: a feed host that also accepts
+	// writes is a different security proposition from one that only reads.
+	AllowIngest bool
+
+	// Rebuild regenerates feed.xml after an upload. Injected rather than built
+	// here so the server does not need to know the feed's metadata, and so the
+	// ingest path is testable without a real feed.
+	Rebuild func() error
 }
 
 // NewToken returns a random token suitable for Options.Token.
@@ -45,6 +55,11 @@ func Handler(opts Options) http.Handler {
 		http.ServeFile(w, r, path.Join(opts.Dir, "feed.xml"))
 	})
 
+	if opts.AllowIngest {
+		mux.HandleFunc("/manifest", handleManifest(opts.Dir))
+		mux.HandleFunc("/ingest", handleIngest(opts.Dir, opts.Rebuild))
+	}
+
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path != "/" {
 			http.NotFound(w, r)