Split download from hosting; carry the token into enclosure URLsHEADmain

The machine that hosts a feed and the machine that downloads the audio do
not have to be the same, and usually shouldn't be: a datacentre IP is
treated very differently from a home connection by the larger video sites.
Downloading where the person actually is sidesteps that entirely, and means
the server needs no credentials for or route to the source.

- push/ingest: `podkit push` uploads a local library to a remote
  `serve --ingest`, asking for a manifest first so only the difference is
  sent. Uploads land via a temp file and rename, so an interrupted transfer
  cannot be picked up as a short episode. Ingest is off by default and
  refuses to start without a token.
- get: downloads a direct audio URL or every enclosure in an RSS feed,
  with no extractor involved. Numbers RSS items in reverse so a mirrored
  series plays in order rather than backwards.

Two bugs found by running it rather than by testing it:

- A tokened server produced enclosure URLs with no token. The feed loaded,
  listed every episode, and all of them 404'd in a real client — the whole
  point of the tool, broken in the default configuration. Feeds now carry
  the token; regression test asserts enclosures from a tokened feed
  actually resolve against a tokened server.
- Ingest ran filepath.Base before validating, so "../evil.mp3" was stored
  as "evil.mp3" — contained, but under a name the client never asked for,
  leaving push and the server disagreeing about the library. Names are now
  validated raw and rejected.

Also: identify with a real user-agent (libsyn 403s Go's default), write
feed.xml via rename so a polling client never reads a truncated feed, and
match ingested file permissions to downloaded ones.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B

2 files changed, 478 insertions, 0 deletions

diff --git a/internal/push/push.go b/internal/push/push.go
new file mode 100644
index 0000000..b640127
--- /dev/null
+++ b/internal/push/push.go
@@ -0,0 +1,216 @@
+// Package push uploads locally-downloaded audio to a remote podkit server.
+//
+// This exists because the download and the hosting do not have to happen on the
+// same machine, and usually shouldn't. Downloading is something a person does
+// from their own connection; hosting is something a server does. Splitting them
+// means the server never needs credentials for, or a working route to, whatever
+// the audio came from.
+package push
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"podkit/internal/feed"
+)
+
+// Options configure an upload run.
+type Options struct {
+	Dir    string // local directory holding the audio
+	Server string // base URL of the remote podkit server
+	Token  string // the remote server's token
+	Force  bool   // upload even when the server already has a matching file
+	Client *http.Client
+}
+
+// Result describes what happened to one file.
+type Result struct {
+	Name    string
+	Bytes   int64
+	Skipped bool
+	Err     error
+}
+
+// Run uploads every audio file in Dir that the server does not already have.
+func Run(ctx context.Context, opts Options) ([]Result, error) {
+	if opts.Server == "" {
+		return nil, fmt.Errorf("a server URL is required")
+	}
+	base, err := url.Parse(opts.Server)
+	if err != nil || base.Scheme == "" || base.Host == "" {
+		return nil, fmt.Errorf("server %q needs a scheme and host, e.g. https://host/podkit", opts.Server)
+	}
+
+	client := opts.Client
+	if client == nil {
+		// No overall timeout: a large episode on a slow uplink is a legitimate
+		// long request. Cancellation comes from ctx instead.
+		client = &http.Client{Timeout: 0}
+	}
+
+	local, err := localFiles(opts.Dir)
+	if err != nil {
+		return nil, err
+	}
+	if len(local) == 0 {
+		return nil, fmt.Errorf("no audio files in %s", opts.Dir)
+	}
+
+	remote := map[string]int64{}
+	if !opts.Force {
+		remote, err = fetchManifest(ctx, client, base, opts.Token)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var results []Result
+	for _, f := range local {
+		if size, ok := remote[f.name]; ok && size == f.size {
+			results = append(results, Result{Name: f.name, Bytes: f.size, Skipped: true})
+			continue
+		}
+		err := upload(ctx, client, base, opts.Token, opts.Dir, f.name, f.size)
+		results = append(results, Result{Name: f.name, Bytes: f.size, Err: err})
+		if err != nil && ctx.Err() != nil {
+			// Interrupted rather than rejected; stop instead of hammering
+			// through the rest of the list with a dead context.
+			break
+		}
+	}
+	return results, nil
+}
+
+type localFile struct {
+	name string
+	size int64
+}
+
+func localFiles(dir string) ([]localFile, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("reading %s: %w", dir, err)
+	}
+	var out []localFile
+	for _, e := range entries {
+		if e.IsDir() || !feed.IsAudio(e.Name()) {
+			continue
+		}
+		info, err := e.Info()
+		if err != nil {
+			return nil, fmt.Errorf("stat %s: %w", e.Name(), err)
+		}
+		out = append(out, localFile{e.Name(), info.Size()})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].name < out[j].name })
+	return out, nil
+}
+
+func fetchManifest(ctx context.Context, c *http.Client, base *url.URL, token string) (map[string]int64, error) {
+	u := *base
+	u.Path = path.Join(u.Path, "manifest")
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	auth(req, token)
+
+	resp, err := c.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("contacting %s: %w", u.String(), err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		// The server 404s both a wrong token and a server without --ingest, on
+		// purpose. Say so, rather than reporting a bare 404 the user then has
+		// to guess the meaning of.
+		return nil, fmt.Errorf("manifest request returned %s — check the token, and that the server was started with --ingest",
+			resp.Status)
+	}
+
+	var m struct {
+		Files map[string]int64 `json:"files"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
+		return nil, fmt.Errorf("parsing manifest from %s: %w", u.String(), err)
+	}
+	if m.Files == nil {
+		m.Files = map[string]int64{}
+	}
+	return m.Files, nil
+}
+
+func upload(ctx context.Context, c *http.Client, base *url.URL, token, dir, name string, size int64) error {
+	f, err := os.Open(filepath.Join(dir, name))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	u := *base
+	u.Path = path.Join(u.Path, "ingest")
+	q := u.Query()
+	q.Set("name", name)
+	u.RawQuery = q.Encode()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), f)
+	if err != nil {
+		return err
+	}
+	auth(req, token)
+	req.Header.Set("Content-Type", feed.MIMEType(name))
+	// Set explicitly so the server sees a length rather than a chunked body,
+	// and so MaxBytesReader can reject an oversized file before the transfer.
+	req.ContentLength = size
+
+	resp, err := c.Do(req)
+	if err != nil {
+		return fmt.Errorf("uploading %s: %w", name, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
+		msg := strings.TrimSpace(string(body))
+		if msg == "" {
+			msg = resp.Status
+		}
+		return fmt.Errorf("uploading %s: %s", name, msg)
+	}
+	_, _ = io.Copy(io.Discard, resp.Body)
+	return nil
+}
+
+func auth(req *http.Request, token string) {
+	if token == "" {
+		return
+	}
+	// Header rather than query string: it keeps the secret out of the server's
+	// access log, and push is not a podcast client so it has the choice.
+	req.Header.Set("Authorization", "Bearer "+token)
+}
+
+// Summarise renders results for the terminal.
+func Summarise(results []Result) (uploaded, skipped, failed int) {
+	for _, r := range results {
+		switch {
+		case r.Err != nil:
+			failed++
+		case r.Skipped:
+			skipped++
+		default:
+			uploaded++
+		}
+	}
+	return
+}
diff --git a/internal/push/push_test.go b/internal/push/push_test.go
new file mode 100644
index 0000000..8e79739
--- /dev/null
+++ b/internal/push/push_test.go
@@ -0,0 +1,262 @@
+package push
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+// fakeServer stands in for a remote podkit server with ingest enabled.
+type fakeServer struct {
+	mu       sync.Mutex
+	existing map[string]int64
+	received map[string]string
+	srv      *httptest.Server
+}
+
+func newFakeServer(t *testing.T, existing map[string]int64) *fakeServer {
+	t.Helper()
+	f := &fakeServer{existing: existing, received: map[string]string{}}
+	if f.existing == nil {
+		f.existing = map[string]int64{}
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/manifest", func(w http.ResponseWriter, r *http.Request) {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		_ = json.NewEncoder(w).Encode(map[string]any{"files": f.existing})
+	})
+	mux.HandleFunc("/ingest", func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		f.received[r.URL.Query().Get("name")] = string(body)
+		w.WriteHeader(http.StatusOK)
+	})
+
+	f.srv = httptest.NewServer(mux)
+	t.Cleanup(f.srv.Close)
+	return f
+}
+
+func libraryDir(t *testing.T, files map[string]string) string {
+	t.Helper()
+	dir := t.TempDir()
+	for name, body := range files {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return dir
+}
+
+func TestPushUploadsAudio(t *testing.T) {
+	dir := libraryDir(t, map[string]string{
+		"01 - April.mp3": "aaa",
+		"02 - May.mp3":   "bbbb",
+	})
+	f := newFakeServer(t, nil)
+
+	results, err := Run(context.Background(), Options{Dir: dir, Server: f.srv.URL, Token: "t"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	uploaded, skipped, failed := Summarise(results)
+	if uploaded != 2 || skipped != 0 || failed != 0 {
+		t.Fatalf("uploaded=%d skipped=%d failed=%d", uploaded, skipped, failed)
+	}
+	if f.received["01 - April.mp3"] != "aaa" || f.received["02 - May.mp3"] != "bbbb" {
+		t.Errorf("server received %v", f.received)
+	}
+}
+
+// Re-running a push must not re-send what is already there; a lecture series is
+// gigabytes and the second run should be nearly free.
+func TestPushSkipsFilesTheServerHas(t *testing.T) {
+	dir := libraryDir(t, map[string]string{
+		"01 - April.mp3": "aaa",
+		"02 - May.mp3":   "bbbb",
+	})
+	f := newFakeServer(t, map[string]int64{"01 - April.mp3": 3})
+
+	results, err := Run(context.Background(), Options{Dir: dir, Server: f.srv.URL})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	uploaded, skipped, _ := Summarise(results)
+	if uploaded != 1 || skipped != 1 {
+		t.Fatalf("uploaded=%d skipped=%d", uploaded, skipped)
+	}
+	if _, sent := f.received["01 - April.mp3"]; sent {
+		t.Error("a file the server already had was re-uploaded")
+	}
+}
+
+// A same-named file of a different size is a different file — most likely a
+// truncated earlier upload — and must be re-sent rather than assumed good.
+func TestPushResendsOnSizeMismatch(t *testing.T) {
+	dir := libraryDir(t, map[string]string{"01 - April.mp3": "aaa"})
+	f := newFakeServer(t, map[string]int64{"01 - April.mp3": 99})
+
+	results, err := Run(context.Background(), Options{Dir: dir, Server: f.srv.URL})
+	if err != nil {
+		t.Fatal(err)
+	}
+	uploaded, skipped, _ := Summarise(results)
+	if uploaded != 1 || skipped != 0 {
+		t.Fatalf("uploaded=%d skipped=%d", uploaded, skipped)
+	}
+}
+
+func TestPushForceIgnoresManifest(t *testing.T) {
+	dir := libraryDir(t, map[string]string{"01 - April.mp3": "aaa"})
+	f := newFakeServer(t, map[string]int64{"01 - April.mp3": 3})
+
+	results, err := Run(context.Background(), Options{Dir: dir, Server: f.srv.URL, Force: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if uploaded, _, _ := Summarise(results); uploaded != 1 {
+		t.Fatalf("uploaded=%d, want 1 under --force", uploaded)
+	}
+}
+
+func TestPushIgnoresNonAudio(t *testing.T) {
+	dir := libraryDir(t, map[string]string{
+		"01 - April.mp3": "aaa",
+		"feed.xml":       "<rss/>",
+		"cover.jpg":      "img",
+		"notes.txt":      "hello",
+	})
+	f := newFakeServer(t, nil)
+
+	if _, err := Run(context.Background(), Options{Dir: dir, Server: f.srv.URL}); err != nil {
+		t.Fatal(err)
+	}
+	if len(f.received) != 1 {
+		t.Errorf("server received %v, want only the mp3", f.received)
+	}
+}
+
+func TestPushSendsToken(t *testing.T) {
+	dir := libraryDir(t, map[string]string{"01 - April.mp3": "aaa"})
+
+	var gotAuth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/manifest" {
+			gotAuth = r.Header.Get("Authorization")
+			_ = json.NewEncoder(w).Encode(map[string]any{"files": map[string]int64{}})
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	if _, err := Run(context.Background(), Options{Dir: dir, Server: srv.URL, Token: "sekrit"}); err != nil {
+		t.Fatal(err)
+	}
+	if gotAuth != "Bearer sekrit" {
+		t.Errorf("Authorization header was %q", gotAuth)
+	}
+}
+
+// A 404 from the manifest means either a bad token or a server without
+// --ingest. The error has to say so, since a bare "404" is unactionable.
+func TestPushExplainsManifest404(t *testing.T) {
+	dir := libraryDir(t, map[string]string{"01 - April.mp3": "aaa"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.NotFound(w, r)
+	}))
+	defer srv.Close()
+
+	_, err := Run(context.Background(), Options{Dir: dir, Server: srv.URL})
+	if err == nil {
+		t.Fatal("expected an error")
+	}
+	for _, want := range []string{"token", "--ingest"} {
+		if !contains(err.Error(), want) {
+			t.Errorf("error should mention %q; got %q", want, err)
+		}
+	}
+}
+
+func TestPushReportsUploadFailure(t *testing.T) {
+	dir := libraryDir(t, map[string]string{"01 - April.mp3": "aaa"})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/manifest" {
+			_ = json.NewEncoder(w).Encode(map[string]any{"files": map[string]int64{}})
+			return
+		}
+		http.Error(w, "disk full", http.StatusInsufficientStorage)
+	}))
+	defer srv.Close()
+
+	results, err := Run(context.Background(), Options{Dir: dir, Server: srv.URL})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, _, failed := Summarise(results); failed != 1 {
+		t.Fatalf("failed=%d, want 1", failed)
+	}
+	if !contains(results[0].Err.Error(), "disk full") {
+		t.Errorf("server's reason was lost: %v", results[0].Err)
+	}
+}
+
+func TestPushRejectsBadServerURL(t *testing.T) {
+	dir := libraryDir(t, map[string]string{"01 - April.mp3": "aaa"})
+	for _, s := range []string{"", "not-a-url", "/relative/path"} {
+		if _, err := Run(context.Background(), Options{Dir: dir, Server: s}); err == nil {
+			t.Errorf("server %q was accepted", s)
+		}
+	}
+}
+
+func TestPushRejectsEmptyLibrary(t *testing.T) {
+	f := newFakeServer(t, nil)
+	if _, err := Run(context.Background(), Options{Dir: t.TempDir(), Server: f.srv.URL}); err == nil {
+		t.Error("expected an error for a directory with no audio")
+	}
+}
+
+// The base URL may include a path, since the server will usually sit behind a
+// reverse proxy at something like /podkit.
+func TestPushHonoursServerPathPrefix(t *testing.T) {
+	dir := libraryDir(t, map[string]string{"01 - April.mp3": "aaa"})
+
+	var paths []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		paths = append(paths, r.URL.Path)
+		if r.URL.Path == "/podkit/manifest" {
+			_ = json.NewEncoder(w).Encode(map[string]any{"files": map[string]int64{}})
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	if _, err := Run(context.Background(), Options{Dir: dir, Server: srv.URL + "/podkit"}); err != nil {
+		t.Fatal(err)
+	}
+	if len(paths) != 2 || paths[0] != "/podkit/manifest" || paths[1] != "/podkit/ingest" {
+		t.Errorf("requested %v", paths)
+	}
+}
+
+func contains(h, n string) bool {
+	for i := 0; i+len(n) <= len(h); i++ {
+		if h[i:i+len(n)] == n {
+			return true
+		}
+	}
+	return false
+}