summaryrefslogtreecommitdiff
path: root/server/internal
diff options
context:
space:
mode:
authorPeter Stone <thepeterstone@gmail.com>2026-10-05 09:54:59 +0000
committerPeter Stone <thepeterstone@gmail.com>2026-10-05 09:54:59 +0000
commit217c982403a105e986fb7ce13b922d9f08583632 (patch)
tree38f2d3a9556b3dfa1b835b9b2c1f7efcb547e719 /server/internal
parent7cc88394458e9d03e3a6a6e58cdff9c8dd4ad7a3 (diff)
Move skip detection to a server, and find that it detects ads
The server is Peter's design and it works: listend polls the phone's feed list, mirrors the newest 8 episodes per show, detects across all of them at once, and serves the offsets *and the audio they belong to*. Deployed on tungsten behind Apache at https://listen.terst.org. FingerprintWorker and MidrollWorker are deleted -- the battery drain is gone because the work is. Serving the audio is load-bearing. Dynamic ad insertion turned out to be on every episode: Grass Apes declares 22:05 and the server was served 28:18. An offset measured on one copy is minutes wrong for another, so the phone has to play the bytes the detector examined. The offsets land in the same columns the on-device detector wrote, so playback needed no changes and there is no migration. **The headline finding is a failure, and it is proved rather than suspected.** Detection reported the intro confirmed on 8 of 8 Radiolab episodes and passed its self-check. It had locked onto an advertisement. The chosen segment was searched for in an ad-free copy of the same episode fetched hours earlier: r = 0.0621. It is in the ad break, not the programme. The confirmation threshold assumes three *independently fetched* episodes. Mirroring 8 within one minute from the same ad server makes them maximally dependent -- whatever campaign is running is in all of them, identical -- and SelfCheck cannot catch it, because the segments genuinely are the same audio. Moving to a server fixed the cost problem and broke the independence the safety mechanism rests on. The published offsets are head-window matches, so they happen to skip pre-roll advertising, which is useful by accident and will move whenever the campaign rotates. It is not intro detection and is not described as such anywhere. The fix, not made: record when each episode was fetched and require the agreeing episodes to span more than one fetch occasion. Reasoning, and an argument for detecting the ads directly instead, in docs/detection-findings.md. Also recorded there: two tooling traps that each cost real time. `android-ui tap` can land on the soft keyboard, because the dump reports layout coordinates the IME does not move -- a Save button and a text field both appeared tapped and neither fired. And the server's token is alphanumeric now because `adb shell input text` ate the leading "-" of a base64url one and produced a silent 401 that looked like broken wiring. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B
Diffstat (limited to 'server/internal')
-rw-r--r--server/internal/api/api.go198
-rw-r--r--server/internal/api/api_test.go325
-rw-r--r--server/internal/feed/feed.go143
-rw-r--r--server/internal/feed/feed_test.go145
-rw-r--r--server/internal/mirror/mirror.go156
-rw-r--r--server/internal/mirror/mirror_test.go162
-rw-r--r--server/internal/store/id.go20
-rw-r--r--server/internal/store/store.go295
-rw-r--r--server/internal/store/store_test.go239
9 files changed, 1683 insertions, 0 deletions
diff --git a/server/internal/api/api.go b/server/internal/api/api.go
new file mode 100644
index 0000000..b6f14ff
--- /dev/null
+++ b/server/internal/api/api.go
@@ -0,0 +1,198 @@
+// Package api serves the phone: what to skip, and the audio those offsets belong to.
+package api
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/terst/listen/server/internal/store"
+)
+
+// Fetcher is asked to mirror an episode the phone wants and the server does not have.
+//
+// Asynchronous on purpose: the phone gets an immediate "pending" rather than holding a request
+// open for the length of a download.
+type Fetcher interface {
+ Request(feedURL, guid, enclosureURL, title string)
+}
+
+// Options configures the server.
+type Options struct {
+ // Token guards everything but health. This serves a private library over the open
+ // internet.
+ Token string
+ // AudioDir is where mirrored episodes live.
+ AudioDir string
+ // OnSubscriptions is called when the phone sends its list, so a newly-subscribed show is
+ // polled at once rather than at the next interval. Optional.
+ OnSubscriptions func(feeds []string)
+}
+
+// Server is the HTTP surface.
+type Server struct {
+ state *store.State
+ fetcher Fetcher
+ opts Options
+}
+
+// New builds the server.
+func New(state *store.State, fetcher Fetcher, opts Options) *Server {
+ return &Server{state: state, fetcher: fetcher, opts: opts}
+}
+
+type episodeResponse struct {
+ // Status is "ready" when the audio can be played now, or "pending" when the server has
+ // been asked to fetch it. There is deliberately no "unknown": an episode the phone knows
+ // about is one the server can be told to go and get.
+ Status string `json:"status"`
+ AudioURL string `json:"audioUrl,omitempty"`
+ DurationMs int64 `json:"durationMs,omitempty"`
+ IntroMs *int64 `json:"introMs,omitempty"`
+ OutroMs *int64 `json:"outroMs,omitempty"`
+ // Detected says whether detection has run over this episode's show, so the phone can tell
+ // "looked and found nothing" apart from "not looked at yet" -- a distinction the app got
+ // wrong when the readout said "None" for both.
+ Detected bool `json:"detected"`
+}
+
+// Handler returns the routed, authenticated handler.
+func (s *Server) Handler() http.Handler {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/api/health", s.health)
+ mux.HandleFunc("/api/subscriptions", s.authed(s.subscriptions))
+ mux.HandleFunc("/api/episode", s.authed(s.episode))
+ mux.HandleFunc("/audio/", s.authed(s.audio))
+ return mux
+}
+
+func (s *Server) health(w http.ResponseWriter, _ *http.Request) {
+ writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+}
+
+// authed accepts the token in the Authorization header or the query string.
+//
+// The query string is not laziness: a media player fetching audio cannot always be made to set
+// a header, and an audio URL the phone cannot use as given would be useless.
+func (s *Server) authed(next http.HandlerFunc) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ if s.opts.Token == "" || !s.tokenOK(r) {
+ w.Header().Set("WWW-Authenticate", `Bearer realm="listen"`)
+ writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
+ return
+ }
+ next(w, r)
+ }
+}
+
+func (s *Server) tokenOK(r *http.Request) bool {
+ if header := r.Header.Get("Authorization"); strings.HasPrefix(header, "Bearer ") {
+ if strings.TrimPrefix(header, "Bearer ") == s.opts.Token {
+ return true
+ }
+ }
+ return r.URL.Query().Get("token") == s.opts.Token
+}
+
+func (s *Server) subscriptions(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "post only"})
+ return
+ }
+ var body struct {
+ Feeds []string `json:"feeds"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ writeJSON(w, http.StatusBadRequest, map[string]string{"error": "expected {\"feeds\":[...]}"})
+ return
+ }
+ removed := s.state.SetFeeds(body.Feeds)
+ if s.opts.OnSubscriptions != nil {
+ s.opts.OnSubscriptions(s.state.Feeds())
+ }
+ writeJSON(w, http.StatusOK, map[string]any{
+ "feeds": len(s.state.Feeds()),
+ "removed": removed,
+ })
+}
+
+func (s *Server) episode(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+ feedURL, guid := q.Get("feed"), q.Get("guid")
+ enclosure, title := q.Get("enclosure"), q.Get("title")
+
+ if feedURL == "" || (guid == "" && enclosure == "" && title == "") {
+ writeJSON(w, http.StatusBadRequest,
+ map[string]string{"error": "feed and at least one of guid, enclosure or title are required"})
+ return
+ }
+
+ found := s.state.Match(feedURL, guid, enclosure, title)
+ if found == nil || !found.Mirrored() {
+ // Ask for it rather than refusing. Everything Peter actually listens to should end up
+ // with offsets, including an episode older than the ones kept by default.
+ s.fetcher.Request(feedURL, guid, enclosure, title)
+ writeJSON(w, http.StatusOK, episodeResponse{Status: "pending"})
+ return
+ }
+ if _, err := os.Stat(filepath.Join(s.opts.AudioDir, found.Path)); err != nil {
+ // The record claims a file that has gone. Treat it as not mirrored and fetch again.
+ s.fetcher.Request(feedURL, found.GUID, found.EnclosureURL, found.Title)
+ writeJSON(w, http.StatusOK, episodeResponse{Status: "pending"})
+ return
+ }
+
+ writeJSON(w, http.StatusOK, episodeResponse{
+ Status: "ready",
+ AudioURL: "/audio/" + found.ID + "?token=" + url.QueryEscape(s.opts.Token),
+ DurationMs: found.DurationMs,
+ IntroMs: found.IntroMs,
+ OutroMs: found.OutroMs,
+ Detected: found.DetectedAtUnix != 0,
+ })
+}
+
+func (s *Server) audio(w http.ResponseWriter, r *http.Request) {
+ id := strings.TrimPrefix(r.URL.Path, "/audio/")
+ // No separators: an id is a flat hex string, and anything else is an attempt to read
+ // outside the audio directory.
+ if id == "" || strings.ContainsAny(id, "/\\") || strings.Contains(id, "..") {
+ http.NotFound(w, r)
+ return
+ }
+
+ found := s.episodeByID(id)
+ if found == nil {
+ http.NotFound(w, r)
+ return
+ }
+ path := filepath.Join(s.opts.AudioDir, found.Path)
+ info, err := os.Stat(path)
+ if err != nil || info.IsDir() {
+ http.NotFound(w, r)
+ return
+ }
+ // ServeFile handles Range, If-Modified-Since and the content type, which a podcast player
+ // needs for seeking.
+ http.ServeFile(w, r, path)
+}
+
+func (s *Server) episodeByID(id string) *store.Episode {
+ for _, show := range s.state.Shows() {
+ for _, e := range show.Episodes {
+ if e.ID == id && e.Mirrored() {
+ return e
+ }
+ }
+ }
+ return nil
+}
+
+func writeJSON(w http.ResponseWriter, status int, body any) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ _ = json.NewEncoder(w).Encode(body)
+}
diff --git a/server/internal/api/api_test.go b/server/internal/api/api_test.go
new file mode 100644
index 0000000..21ef1de
--- /dev/null
+++ b/server/internal/api/api_test.go
@@ -0,0 +1,325 @@
+package api
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/terst/listen/server/internal/feed"
+ "github.com/terst/listen/server/internal/store"
+)
+
+const token = "s3cret"
+
+type fakeFetcher struct{ asked []string }
+
+func (f *fakeFetcher) Request(feedURL, guid, enclosureURL, title string) {
+ f.asked = append(f.asked, feedURL+" "+guid+" "+enclosureURL)
+}
+
+func newServer(t *testing.T) (*Server, *store.State, *fakeFetcher, string) {
+ t.Helper()
+ audioDir := t.TempDir()
+ st := store.New()
+ fetcher := &fakeFetcher{}
+ return New(st, fetcher, Options{Token: token, AudioDir: audioDir}), st, fetcher, audioDir
+}
+
+func get(t *testing.T, s *Server, path string) *httptest.ResponseRecorder {
+ t.Helper()
+ req := httptest.NewRequest(http.MethodGet, path, nil)
+ req.Header.Set("Authorization", "Bearer "+token)
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, req)
+ return rec
+}
+
+func post(t *testing.T, s *Server, path, body string) *httptest.ResponseRecorder {
+ t.Helper()
+ req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))
+ req.Header.Set("Authorization", "Bearer "+token)
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, req)
+ return rec
+}
+
+// This serves a private library over the public internet, so nothing but health is reachable
+// without the token.
+
+func TestRejectsAnUnauthenticatedRequest(t *testing.T) {
+ s, _, _, _ := newServer(t)
+ for _, path := range []string{"/api/episode?feed=f&guid=g", "/api/subscriptions", "/audio/abc"} {
+ req := httptest.NewRequest(http.MethodGet, path, nil)
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, req)
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("%s: status %d, want 401", path, rec.Code)
+ }
+ }
+}
+
+func TestRejectsAWrongToken(t *testing.T) {
+ s, _, _, _ := newServer(t)
+ req := httptest.NewRequest(http.MethodGet, "/api/episode?feed=f&guid=g", nil)
+ req.Header.Set("Authorization", "Bearer wrong")
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, req)
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("status %d, want 401", rec.Code)
+ }
+}
+
+// A media player cannot always set headers on the request it makes for audio, so the token is
+// accepted in the query string too -- which is what makes the returned audio URL usable as-is.
+func TestAcceptsTheTokenAsAQueryParameter(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ seedMirrored(t, st, audioDir, "hello")
+
+ req := httptest.NewRequest(http.MethodGet, "/audio/"+episodeID(st)+"?token="+token, nil)
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status %d, want 200", rec.Code)
+ }
+ if rec.Body.String() != "hello" {
+ t.Errorf("body %q", rec.Body.String())
+ }
+}
+
+func TestHealthNeedsNoToken(t *testing.T) {
+ s, _, _, _ := newServer(t)
+ req := httptest.NewRequest(http.MethodGet, "/api/health", nil)
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Errorf("status %d, want 200", rec.Code)
+ }
+}
+
+func TestSubscriptionsReplacesTheFeedList(t *testing.T) {
+ s, st, _, _ := newServer(t)
+ rec := post(t, s, "/api/subscriptions", `{"feeds":["https://a","https://b"]}`)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
+ }
+ if got := st.Feeds(); len(got) != 2 {
+ t.Errorf("feeds %v, want 2", got)
+ }
+}
+
+// The phone sends its list when it refreshes, and waiting up to a poll interval before acting
+// on it would mean a newly-subscribed show has nothing for hours.
+func TestSubscriptionsTriggersAPollImmediately(t *testing.T) {
+ audioDir := t.TempDir()
+ st := store.New()
+ notified := make(chan []string, 1)
+ s := New(st, &fakeFetcher{}, Options{
+ Token: token,
+ AudioDir: audioDir,
+ OnSubscriptions: func(feeds []string) {
+ notified <- feeds
+ },
+ })
+
+ req := httptest.NewRequest(http.MethodPost, "/api/subscriptions", strings.NewReader(`{"feeds":["https://a"]}`))
+ req.Header.Set("Authorization", "Bearer "+token)
+ s.Handler().ServeHTTP(httptest.NewRecorder(), req)
+
+ select {
+ case feeds := <-notified:
+ if len(feeds) != 1 || feeds[0] != "https://a" {
+ t.Errorf("notified with %v", feeds)
+ }
+ default:
+ t.Error("subscriptions did not trigger a poll")
+ }
+}
+
+func TestSubscriptionsRejectsRubbish(t *testing.T) {
+ s, _, _, _ := newServer(t)
+ if rec := post(t, s, "/api/subscriptions", `not json`); rec.Code != http.StatusBadRequest {
+ t.Errorf("status %d, want 400", rec.Code)
+ }
+}
+
+// An episode nobody has heard of still gets a useful answer: the server is told enough to go
+// and fetch it, and says so, rather than returning a bare 404 the phone can only give up on.
+func TestAnUnknownEpisodeIsRequestedAndReportedPending(t *testing.T) {
+ s, _, fetcher, _ := newServer(t)
+
+ rec := get(t, s, "/api/episode?feed=https://f&guid=g1&enclosure=https://cdn/1.mp3&title=One")
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
+ }
+ var got episodeResponse
+ if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+ t.Fatal(err)
+ }
+ if got.Status != "pending" {
+ t.Errorf("status %q, want pending", got.Status)
+ }
+ if len(fetcher.asked) != 1 {
+ t.Errorf("fetcher asked %d times, want 1", len(fetcher.asked))
+ }
+}
+
+func TestAKnownButUnmirroredEpisodeIsRequestedAndReportedPending(t *testing.T) {
+ s, st, fetcher, _ := newServer(t)
+ st.Merge("https://f", "A Show", []feed.Episode{
+ {GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100},
+ })
+
+ rec := get(t, s, "/api/episode?feed=https://f&guid=g1")
+ var got episodeResponse
+ if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+ t.Fatal(err)
+ }
+ if got.Status != "pending" {
+ t.Errorf("status %q, want pending", got.Status)
+ }
+ if len(fetcher.asked) != 1 {
+ t.Errorf("fetcher asked %d times, want 1", len(fetcher.asked))
+ }
+}
+
+// Audio becomes usable as soon as it is mirrored, whether or not detection has run. Waiting for
+// offsets before handing over playable audio would make the server strictly worse than
+// downloading from the publisher.
+func TestAMirroredEpisodeIsReadyEvenWithoutOffsets(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ seedMirrored(t, st, audioDir, "audio-bytes")
+
+ rec := get(t, s, "/api/episode?feed=https://f&guid=g1")
+ var got episodeResponse
+ if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+ t.Fatal(err)
+ }
+ if got.Status != "ready" {
+ t.Fatalf("status %q, want ready", got.Status)
+ }
+ if got.AudioURL == "" {
+ t.Error("no audio url")
+ }
+ if got.IntroMs != nil || got.OutroMs != nil {
+ t.Errorf("reported offsets that were never detected: %v %v", got.IntroMs, got.OutroMs)
+ }
+}
+
+func TestAMirroredEpisodeReportsItsDetectedOffsets(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ seedMirrored(t, st, audioDir, "audio-bytes")
+ e := st.Match("https://f", "g1", "", "")
+ intro, outro := int64(16652), int64(11000)
+ e.IntroMs, e.OutroMs, e.DurationMs = &intro, &outro, 1325113
+
+ rec := get(t, s, "/api/episode?feed=https://f&guid=g1")
+ var got episodeResponse
+ if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+ t.Fatal(err)
+ }
+ if got.IntroMs == nil || *got.IntroMs != 16652 {
+ t.Errorf("introMs %v, want 16652", got.IntroMs)
+ }
+ if got.OutroMs == nil || *got.OutroMs != 11000 {
+ t.Errorf("outroMs %v, want 11000", got.OutroMs)
+ }
+ if got.DurationMs != 1325113 {
+ t.Errorf("durationMs %d, want the measured length", got.DurationMs)
+ }
+}
+
+// The audio URL has to carry the token, or the phone gets a link it cannot use.
+func TestTheAudioUrlIsUsableAsGiven(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ seedMirrored(t, st, audioDir, "audio-bytes")
+
+ rec := get(t, s, "/api/episode?feed=https://f&guid=g1")
+ var got episodeResponse
+ if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+ t.Fatal(err)
+ }
+
+ req := httptest.NewRequest(http.MethodGet, got.AudioURL, nil)
+ rec2 := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec2, req)
+ if rec2.Code != http.StatusOK {
+ t.Fatalf("fetching the url it gave returned %d", rec2.Code)
+ }
+ if rec2.Body.String() != "audio-bytes" {
+ t.Errorf("body %q", rec2.Body.String())
+ }
+}
+
+// Seeking is the normal case for a podcast, so byte ranges have to work.
+func TestAudioSupportsRangeRequests(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ seedMirrored(t, st, audioDir, "0123456789")
+
+ req := httptest.NewRequest(http.MethodGet, "/audio/"+episodeID(st)+"?token="+token, nil)
+ req.Header.Set("Range", "bytes=3-5")
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusPartialContent {
+ t.Fatalf("status %d, want 206", rec.Code)
+ }
+ if rec.Body.String() != "345" {
+ t.Errorf("body %q, want %q", rec.Body.String(), "345")
+ }
+}
+
+func TestAudioForAnUnknownIdIs404(t *testing.T) {
+ s, _, _, _ := newServer(t)
+ if rec := get(t, s, "/audio/nope"); rec.Code != http.StatusNotFound {
+ t.Errorf("status %d, want 404", rec.Code)
+ }
+}
+
+// A record that claims a file which is no longer there must not 500.
+func TestAudioForAMissingFileIs404(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ seedMirrored(t, st, audioDir, "gone")
+ os.Remove(filepath.Join(audioDir, st.Match("https://f", "g1", "", "").Path))
+
+ if rec := get(t, s, "/audio/"+episodeID(st)); rec.Code != http.StatusNotFound {
+ t.Errorf("status %d, want 404", rec.Code)
+ }
+}
+
+// An id containing path separators must not be able to read outside the audio directory.
+func TestAudioIdCannotEscapeTheAudioDirectory(t *testing.T) {
+ s, _, _, _ := newServer(t)
+ for _, id := range []string{"..%2f..%2fetc%2fpasswd", "../../etc/passwd"} {
+ rec := get(t, s, "/audio/"+id)
+ if rec.Code == http.StatusOK {
+ t.Errorf("%q was served", id)
+ }
+ }
+}
+
+func TestEpisodeNeedsAFeedAndSomeIdentity(t *testing.T) {
+ s, _, _, _ := newServer(t)
+ if rec := get(t, s, "/api/episode"); rec.Code != http.StatusBadRequest {
+ t.Errorf("status %d, want 400", rec.Code)
+ }
+}
+
+func seedMirrored(t *testing.T, st *store.State, audioDir, contents string) {
+ t.Helper()
+ st.Merge("https://f", "A Show", []feed.Episode{
+ {GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100},
+ })
+ e := st.Match("https://f", "g1", "", "")
+ e.Path = e.ID + ".mp3"
+ if err := os.WriteFile(filepath.Join(audioDir, e.Path), []byte(contents), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func episodeID(st *store.State) string {
+ return st.Match("https://f", "g1", "", "").ID
+}
diff --git a/server/internal/feed/feed.go b/server/internal/feed/feed.go
new file mode 100644
index 0000000..edc58be
--- /dev/null
+++ b/server/internal/feed/feed.go
@@ -0,0 +1,143 @@
+// Package feed parses a podcast RSS feed down to the few fields the server needs.
+package feed
+
+import (
+ "encoding/xml"
+ "fmt"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// Episode is one audio episode as its feed describes it.
+type Episode struct {
+ GUID string
+ Title string
+ // EnclosureURL is where the audio is, and the fallback identity when a GUID moves.
+ EnclosureURL string
+ EnclosureType string
+ PublishedUnix int64
+ // DeclaredDurationMs is the feed's claim, which is routinely wrong -- a show using dynamic
+ // ad insertion declares the ad-free length. Useful for ordering, not for arithmetic.
+ DeclaredDurationMs int64
+}
+
+// Channel is a show and its audio episodes, newest first.
+type Channel struct {
+ Title string
+ Episodes []Episode
+}
+
+type rss struct {
+ Channel struct {
+ Title string `xml:"title"`
+ Items []struct {
+ Title string `xml:"title"`
+ GUID string `xml:"guid"`
+ PubDate string `xml:"pubDate"`
+ Duration string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd duration"`
+ Enclosure struct {
+ URL string `xml:"url,attr"`
+ Type string `xml:"type,attr"`
+ } `xml:"enclosure"`
+ } `xml:"item"`
+ } `xml:"channel"`
+}
+
+// Parse reads a feed and returns its audio episodes, newest first.
+//
+// Audio only: video was cut from the app's scope, and a YouTube or Facebook episode has no
+// enclosure the server could fetch regardless -- its audio only exists once yt-dlp has resolved
+// it, which stays the phone's job.
+func Parse(raw []byte) (*Channel, error) {
+ var doc rss
+ if err := xml.Unmarshal(raw, &doc); err != nil {
+ return nil, fmt.Errorf("parse feed: %w", err)
+ }
+ if doc.Channel.Title == "" && len(doc.Channel.Items) == 0 {
+ return nil, fmt.Errorf("parse feed: no channel found")
+ }
+
+ out := &Channel{Title: strings.TrimSpace(doc.Channel.Title)}
+ for _, item := range doc.Channel.Items {
+ if !isAudio(item.Enclosure.Type, item.Enclosure.URL) {
+ continue
+ }
+ out.Episodes = append(out.Episodes, Episode{
+ GUID: strings.TrimSpace(item.GUID),
+ Title: strings.TrimSpace(item.Title),
+ EnclosureURL: strings.TrimSpace(item.Enclosure.URL),
+ EnclosureType: strings.TrimSpace(item.Enclosure.Type),
+ PublishedUnix: parseDate(item.PubDate),
+ DeclaredDurationMs: parseDuration(item.Duration),
+ })
+ }
+
+ // Newest first. Feeds are not reliably ordered, and the retention rule depends on this.
+ sort.SliceStable(out.Episodes, func(i, j int) bool {
+ return out.Episodes[i].PublishedUnix > out.Episodes[j].PublishedUnix
+ })
+ return out, nil
+}
+
+func isAudio(mediaType, url string) bool {
+ if url == "" {
+ return false
+ }
+ if strings.HasPrefix(strings.ToLower(mediaType), "audio/") {
+ return true
+ }
+ // Some feeds omit or mistype the enclosure type; fall back to the extension rather than
+ // dropping a real episode over a publisher's metadata slip.
+ if mediaType == "" {
+ lower := strings.ToLower(url)
+ for _, ext := range []string{".mp3", ".m4a", ".aac", ".ogg", ".opus", ".flac", ".wav"} {
+ if strings.Contains(lower, ext) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// dateLayouts covers what podcast feeds actually emit, which is RFC 1123 with and without a
+// named zone plus a few near-misses.
+var dateLayouts = []string{
+ time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, time.RFC3339,
+ "Mon, 2 Jan 2006 15:04:05 -0700",
+ "Mon, 2 Jan 2006 15:04:05 MST",
+ "Mon, 02 Jan 2006 15:04 -0700",
+ "2 Jan 2006 15:04:05 -0700",
+}
+
+func parseDate(raw string) int64 {
+ raw = strings.TrimSpace(raw)
+ if raw == "" {
+ return 0
+ }
+ for _, layout := range dateLayouts {
+ if t, err := time.Parse(layout, raw); err == nil {
+ return t.Unix()
+ }
+ }
+ return 0
+}
+
+// parseDuration reads itunes:duration, which appears as bare seconds, mm:ss, or hh:mm:ss.
+func parseDuration(raw string) int64 {
+ raw = strings.TrimSpace(raw)
+ if raw == "" {
+ return 0
+ }
+ parts := strings.Split(raw, ":")
+ var total int64
+ for _, p := range parts {
+ n, err := strconv.ParseInt(strings.TrimSpace(p), 10, 64)
+ if err != nil {
+ return 0
+ }
+ total = total*60 + n
+ }
+ return total * 1000
+}
diff --git a/server/internal/feed/feed_test.go b/server/internal/feed/feed_test.go
new file mode 100644
index 0000000..e62558f
--- /dev/null
+++ b/server/internal/feed/feed_test.go
@@ -0,0 +1,145 @@
+package feed
+
+import "testing"
+
+// Real podcast feeds, trimmed. The shapes here are the ones that actually caused trouble in
+// the Android parser and are worth not relearning: a GUID that is not a URL, an enclosure whose
+// type is video, CDATA titles, and an itunes:duration in three different formats.
+
+const sample = `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+<channel>
+ <title>A Show</title>
+ <item>
+ <title><![CDATA[Newest Episode]]></title>
+ <guid isPermaLink="false">tag:example,2026:1</guid>
+ <pubDate>Fri, 03 Oct 2026 10:00:00 -0000</pubDate>
+ <itunes:duration>00:22:05</itunes:duration>
+ <enclosure url="https://cdn.example/1.mp3" type="audio/mpeg" length="21000000"/>
+ </item>
+ <item>
+ <title>Second Episode</title>
+ <guid>https://example.com/2</guid>
+ <pubDate>Fri, 26 Sep 2026 10:00:00 -0000</pubDate>
+ <itunes:duration>1325</itunes:duration>
+ <enclosure url="https://cdn.example/2.mp3" type="audio/mpeg"/>
+ </item>
+ <item>
+ <title>A Video One</title>
+ <guid>https://example.com/3</guid>
+ <pubDate>Fri, 19 Sep 2026 10:00:00 -0000</pubDate>
+ <enclosure url="https://cdn.example/3.mp4" type="video/mp4"/>
+ </item>
+ <item>
+ <title>No Enclosure At All</title>
+ <guid>https://example.com/4</guid>
+ </item>
+ <item>
+ <title>Minutes And Seconds</title>
+ <guid>https://example.com/5</guid>
+ <pubDate>Fri, 12 Sep 2026 10:00:00 -0000</pubDate>
+ <itunes:duration>31:02</itunes:duration>
+ <enclosure url="https://cdn.example/5.mp3" type="audio/mpeg"/>
+ </item>
+</channel>
+</rss>`
+
+func TestParsesTheChannelTitle(t *testing.T) {
+ ch, err := Parse([]byte(sample))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ch.Title != "A Show" {
+ t.Errorf("title %q, want %q", ch.Title, "A Show")
+ }
+}
+
+// Audio only. Video was cut from the app's scope, and an episode whose audio comes from yt-dlp
+// has no enclosure the server could fetch anyway.
+func TestKeepsOnlyAudioEnclosures(t *testing.T) {
+ ch, err := Parse([]byte(sample))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(ch.Episodes) != 3 {
+ var got []string
+ for _, e := range ch.Episodes {
+ got = append(got, e.Title)
+ }
+ t.Fatalf("kept %d episodes (%v), want 3 audio ones", len(ch.Episodes), got)
+ }
+ for _, e := range ch.Episodes {
+ if e.EnclosureURL == "https://cdn.example/3.mp4" {
+ t.Error("kept a video enclosure")
+ }
+ }
+}
+
+func TestReadsTheFieldsNeededToIdentifyAnEpisode(t *testing.T) {
+ ch, err := Parse([]byte(sample))
+ if err != nil {
+ t.Fatal(err)
+ }
+ first := ch.Episodes[0]
+ if first.Title != "Newest Episode" {
+ t.Errorf("title %q, want CDATA content unwrapped", first.Title)
+ }
+ if first.GUID != "tag:example,2026:1" {
+ t.Errorf("guid %q, want the non-URL guid verbatim", first.GUID)
+ }
+ if first.EnclosureURL != "https://cdn.example/1.mp3" {
+ t.Errorf("enclosure %q", first.EnclosureURL)
+ }
+ if first.PublishedUnix == 0 {
+ t.Error("published date not parsed")
+ }
+}
+
+// A declared duration is the feed's claim and routinely wrong, but it is still what decides
+// which episodes look newest before anything has been downloaded.
+func TestParsesEveryDurationFormatFeedsActuallyUse(t *testing.T) {
+ ch, err := Parse([]byte(sample))
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := map[string]int64{
+ "Newest Episode": 1325000, // 00:22:05
+ "Second Episode": 1325000, // bare seconds
+ "Minutes And Seconds": 1862000, // 31:02
+ }
+ for _, e := range ch.Episodes {
+ if w, ok := want[e.Title]; ok && e.DeclaredDurationMs != w {
+ t.Errorf("%s: duration %dms, want %dms", e.Title, e.DeclaredDurationMs, w)
+ }
+ }
+}
+
+// Newest first, because that is the order the retention rule needs and feeds are not reliably
+// sorted.
+func TestOrdersEpisodesNewestFirst(t *testing.T) {
+ ch, err := Parse([]byte(sample))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i := 1; i < len(ch.Episodes); i++ {
+ if ch.Episodes[i-1].PublishedUnix < ch.Episodes[i].PublishedUnix {
+ t.Errorf("episode %d is newer than %d", i, i-1)
+ }
+ }
+}
+
+func TestRejectsSomethingThatIsNotAFeed(t *testing.T) {
+ if _, err := Parse([]byte("not xml at all")); err == nil {
+ t.Error("expected an error")
+ }
+}
+
+func TestAFeedWithNoItemsIsNotAnError(t *testing.T) {
+ ch, err := Parse([]byte(`<rss><channel><title>Empty</title></channel></rss>`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(ch.Episodes) != 0 {
+ t.Errorf("got %d episodes", len(ch.Episodes))
+ }
+}
diff --git a/server/internal/mirror/mirror.go b/server/internal/mirror/mirror.go
new file mode 100644
index 0000000..9e91041
--- /dev/null
+++ b/server/internal/mirror/mirror.go
@@ -0,0 +1,156 @@
+// Package mirror keeps a local copy of the audio the server detects against and the phone
+// plays.
+package mirror
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/terst/listen/server/internal/store"
+)
+
+// Probe measures a downloaded file's real length.
+type Probe interface {
+ DurationMs(ctx context.Context, path string) (int64, error)
+}
+
+// Mirror downloads episodes and reclaims them.
+//
+// The reason the server holds the audio at all, rather than only timestamps: an episode fetched
+// twice is not the same file when a show uses dynamic ad insertion, so an offset measured on one
+// copy points at the wrong moment in another. The phone playing the same bytes the detector
+// examined is what makes the offsets correct.
+type Mirror struct {
+ AudioDir string
+ Probe Probe
+ // Client defaults to a client with a generous timeout: episodes are tens of megabytes.
+ Client *http.Client
+ // UserAgent identifies this server to publishers.
+ UserAgent string
+}
+
+// DefaultTimeout bounds one episode download.
+const DefaultTimeout = 15 * time.Minute
+
+func (m *Mirror) client() *http.Client {
+ if m.Client != nil {
+ return m.Client
+ }
+ return &http.Client{Timeout: DefaultTimeout}
+}
+
+func (m *Mirror) userAgent() string {
+ if m.UserAgent != "" {
+ return m.UserAgent
+ }
+ return "listen-server/1.0 (+https://terst.org)"
+}
+
+// Fetch downloads an episode's audio and records where it went and how long it really is.
+//
+// Writes to a temporary file and renames, so an interrupted download cannot leave a truncated
+// file that looks mirrored -- which would have the phone play a cut-off episode and the
+// detector correlate against noise.
+func (m *Mirror) Fetch(ctx context.Context, e *store.Episode) error {
+ if e.EnclosureURL == "" {
+ return fmt.Errorf("episode %s: nothing to download", e.ID)
+ }
+ if err := os.MkdirAll(m.AudioDir, 0o755); err != nil {
+ return fmt.Errorf("episode %s: %w", e.ID, err)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, e.EnclosureURL, nil)
+ if err != nil {
+ return fmt.Errorf("episode %s: %w", e.ID, err)
+ }
+ req.Header.Set("User-Agent", m.userAgent())
+
+ resp, err := m.client().Do(req)
+ if err != nil {
+ return fmt.Errorf("episode %s: %w", e.ID, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("episode %s: %s returned %s", e.ID, e.EnclosureURL, resp.Status)
+ }
+
+ name := e.ID + extensionFor(e.EnclosureURL, resp.Header.Get("Content-Type"))
+ tmp, err := os.CreateTemp(m.AudioDir, ".download-*")
+ if err != nil {
+ return fmt.Errorf("episode %s: %w", e.ID, err)
+ }
+ tmpName := tmp.Name()
+ defer os.Remove(tmpName)
+
+ if _, err := io.Copy(tmp, resp.Body); err != nil {
+ tmp.Close()
+ return fmt.Errorf("episode %s: %w", e.ID, err)
+ }
+ if err := tmp.Close(); err != nil {
+ return fmt.Errorf("episode %s: %w", e.ID, err)
+ }
+
+ final := filepath.Join(m.AudioDir, name)
+ if err := os.Rename(tmpName, final); err != nil {
+ return fmt.Errorf("episode %s: %w", e.ID, err)
+ }
+
+ // The real length, which is routinely not what the feed declared.
+ if ms, err := m.Probe.DurationMs(ctx, final); err == nil {
+ e.DurationMs = ms
+ }
+ e.Path = name
+ return nil
+}
+
+// Prune deletes an episode's audio, leaving everything detection decided about it in place.
+//
+// The offsets describe the episode, not the file, and re-deriving them would mean downloading
+// it again -- so they outlive the audio.
+func (m *Mirror) Prune(e *store.Episode) error {
+ if e.Path == "" {
+ return nil
+ }
+ err := os.Remove(filepath.Join(m.AudioDir, e.Path))
+ e.Path = ""
+ if err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("episode %s: %w", e.ID, err)
+ }
+ return nil
+}
+
+// extensionFor picks a file extension, preferring the URL's since a player chooses its decoder
+// from it.
+func extensionFor(rawURL, contentType string) string {
+ if u, err := url.Parse(rawURL); err == nil {
+ if ext := path.Ext(u.Path); isAudioExt(ext) {
+ return ext
+ }
+ }
+ switch {
+ case strings.Contains(contentType, "mp4"), strings.Contains(contentType, "m4a"):
+ return ".m4a"
+ case strings.Contains(contentType, "aac"):
+ return ".aac"
+ case strings.Contains(contentType, "ogg"), strings.Contains(contentType, "opus"):
+ return ".ogg"
+ default:
+ return ".mp3"
+ }
+}
+
+func isAudioExt(ext string) bool {
+ switch strings.ToLower(ext) {
+ case ".mp3", ".m4a", ".m4b", ".aac", ".ogg", ".opus", ".flac", ".wav":
+ return true
+ }
+ return false
+}
diff --git a/server/internal/mirror/mirror_test.go b/server/internal/mirror/mirror_test.go
new file mode 100644
index 0000000..b510451
--- /dev/null
+++ b/server/internal/mirror/mirror_test.go
@@ -0,0 +1,162 @@
+package mirror
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/terst/listen/server/internal/store"
+)
+
+type stubProbe struct{ ms int64 }
+
+func (s stubProbe) DurationMs(context.Context, string) (int64, error) { return s.ms, nil }
+
+func episode(id, url string) *store.Episode {
+ return &store.Episode{ID: id, GUID: "g-" + id, EnclosureURL: url, Title: "An Episode"}
+}
+
+func TestFetchWritesTheAudioAndRecordsWhereItWent(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "audio/mpeg")
+ _, _ = w.Write([]byte("fake-mp3-bytes"))
+ }))
+ defer srv.Close()
+
+ dir := t.TempDir()
+ m := &Mirror{AudioDir: dir, Probe: stubProbe{ms: 1325113}}
+ e := episode("abc", srv.URL+"/1.mp3")
+
+ if err := m.Fetch(context.Background(), e); err != nil {
+ t.Fatal(err)
+ }
+ if e.Path == "" {
+ t.Fatal("Path not recorded")
+ }
+ raw, err := os.ReadFile(filepath.Join(dir, e.Path))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(raw) != "fake-mp3-bytes" {
+ t.Errorf("contents %q", raw)
+ }
+ // The measured length, not the feed's claim: everything downstream reads this.
+ if e.DurationMs != 1325113 {
+ t.Errorf("DurationMs %d, want 1325113", e.DurationMs)
+ }
+}
+
+// The file is named by the episode id, which is stable and is what the audio URL already uses.
+func TestFetchNamesTheFileByEpisodeId(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte("x"))
+ }))
+ defer srv.Close()
+
+ m := &Mirror{AudioDir: t.TempDir(), Probe: stubProbe{ms: 1}}
+ e := episode("deadbeef", srv.URL+"/whatever.m4a")
+ if err := m.Fetch(context.Background(), e); err != nil {
+ t.Fatal(err)
+ }
+ if !strings.HasPrefix(e.Path, "deadbeef") {
+ t.Errorf("Path %q does not start with the episode id", e.Path)
+ }
+ if filepath.Ext(e.Path) != ".m4a" {
+ t.Errorf("Path %q lost the extension, which the player needs to pick a decoder", e.Path)
+ }
+}
+
+// A failed or interrupted download must not leave a half file that looks mirrored. That is the
+// state that would have the phone play a truncated episode and the detector read noise.
+func TestAFailedFetchLeavesNothingBehind(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ http.Error(w, "gone", http.StatusNotFound)
+ }))
+ defer srv.Close()
+
+ dir := t.TempDir()
+ m := &Mirror{AudioDir: dir, Probe: stubProbe{ms: 1}}
+ e := episode("abc", srv.URL+"/1.mp3")
+
+ if err := m.Fetch(context.Background(), e); err == nil {
+ t.Fatal("expected an error")
+ }
+ if e.Path != "" {
+ t.Errorf("recorded a path for a download that failed: %q", e.Path)
+ }
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(entries) != 0 {
+ t.Errorf("left %d files behind", len(entries))
+ }
+}
+
+func TestFetchRefusesAnEpisodeWithNoEnclosure(t *testing.T) {
+ m := &Mirror{AudioDir: t.TempDir(), Probe: stubProbe{ms: 1}}
+ if err := m.Fetch(context.Background(), episode("abc", "")); err == nil {
+ t.Error("expected an error for an episode with nothing to download")
+ }
+}
+
+func TestPruneRemovesTheFileAndForgetsThePath(t *testing.T) {
+ dir := t.TempDir()
+ m := &Mirror{AudioDir: dir, Probe: stubProbe{ms: 1}}
+ e := episode("abc", "https://cdn/1.mp3")
+ e.Path = "abc.mp3"
+ if err := os.WriteFile(filepath.Join(dir, e.Path), []byte("bytes"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := m.Prune(e); err != nil {
+ t.Fatal(err)
+ }
+ if e.Path != "" {
+ t.Errorf("Path still set: %q", e.Path)
+ }
+ if _, err := os.Stat(filepath.Join(dir, "abc.mp3")); !os.IsNotExist(err) {
+ t.Error("file still on disk")
+ }
+}
+
+// Detected offsets outlive the audio: they describe the episode, and re-deriving them would
+// mean downloading it again.
+func TestPruneKeepsWhatDetectionDecided(t *testing.T) {
+ dir := t.TempDir()
+ m := &Mirror{AudioDir: dir, Probe: stubProbe{ms: 1}}
+ e := episode("abc", "https://cdn/1.mp3")
+ e.Path = "abc.mp3"
+ intro := int64(16652)
+ e.IntroMs = &intro
+ e.DurationMs = 1325113
+ if err := os.WriteFile(filepath.Join(dir, e.Path), []byte("bytes"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := m.Prune(e); err != nil {
+ t.Fatal(err)
+ }
+ if e.IntroMs == nil || *e.IntroMs != 16652 {
+ t.Errorf("lost the detected intro: %v", e.IntroMs)
+ }
+ if e.DurationMs != 1325113 {
+ t.Errorf("lost the measured duration: %d", e.DurationMs)
+ }
+}
+
+func TestPruningSomethingAlreadyGoneIsFine(t *testing.T) {
+ m := &Mirror{AudioDir: t.TempDir(), Probe: stubProbe{ms: 1}}
+ e := episode("abc", "https://cdn/1.mp3")
+ e.Path = "not-there.mp3"
+ if err := m.Prune(e); err != nil {
+ t.Errorf("pruning a missing file errored: %v", err)
+ }
+ if e.Path != "" {
+ t.Error("Path not cleared")
+ }
+}
diff --git a/server/internal/store/id.go b/server/internal/store/id.go
new file mode 100644
index 0000000..52bab46
--- /dev/null
+++ b/server/internal/store/id.go
@@ -0,0 +1,20 @@
+package store
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+)
+
+// newID is a stable identifier for an episode, derived from what identifies it rather than
+// from a counter or a random value.
+//
+// Stable matters because it names the audio file on disk and appears in the URL the phone
+// fetches: a regenerated id would orphan the file and break a link the phone already holds.
+func newID(feedURL, guid, enclosureURL string) string {
+ key := guid
+ if key == "" {
+ key = enclosureURL
+ }
+ sum := sha256.Sum256([]byte(feedURL + "\x00" + key))
+ return hex.EncodeToString(sum[:12])
+}
diff --git a/server/internal/store/store.go b/server/internal/store/store.go
new file mode 100644
index 0000000..cfe6f40
--- /dev/null
+++ b/server/internal/store/store.go
@@ -0,0 +1,295 @@
+// Package store holds what the server knows: which feeds to poll, which episodes exist, where
+// their audio is, and what detection decided about them.
+package store
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync"
+
+ "github.com/terst/listen/server/internal/feed"
+)
+
+// Episode is one episode's full server-side record.
+type Episode struct {
+ ID string `json:"id"`
+ GUID string `json:"guid"`
+ Title string `json:"title"`
+ EnclosureURL string `json:"enclosureUrl"`
+ PublishedUnix int64 `json:"publishedUnix"`
+ // DeclaredDurationMs is the feed's claim; DurationMs is what the file turned out to be.
+ DeclaredDurationMs int64 `json:"declaredDurationMs"`
+ DurationMs int64 `json:"durationMs,omitempty"`
+
+ // Path is where the mirrored audio sits, relative to the audio directory. Empty until
+ // the episode has been downloaded.
+ Path string `json:"path,omitempty"`
+
+ // IntroMs and OutroMs are what the phone needs, in this episode's own timeline. Nil until
+ // detection has confirmed something that passed its self-check.
+ IntroMs *int64 `json:"introMs,omitempty"`
+ OutroMs *int64 `json:"outroMs,omitempty"`
+
+ // DetectedAtUnix records when detection last ran over the show this belongs to, so a
+ // result can be told apart from never having looked.
+ DetectedAtUnix int64 `json:"detectedAtUnix,omitempty"`
+}
+
+// Mirrored reports whether this episode's audio is on disk.
+func (e *Episode) Mirrored() bool { return e.Path != "" }
+
+// Show is a subscribed feed and everything known about its episodes.
+type Show struct {
+ FeedURL string `json:"feedUrl"`
+ Title string `json:"title"`
+ Episodes []*Episode `json:"episodes"`
+
+ // LastPolledUnix and LastError record the most recent refresh, so a feed that is failing
+ // says so instead of silently having nothing.
+ LastPolledUnix int64 `json:"lastPolledUnix,omitempty"`
+ LastError string `json:"lastError,omitempty"`
+}
+
+// Partition splits a show's episodes into the newest keep and everything else, newest first.
+//
+// By publish date rather than feed order, because feeds are not reliably sorted and the whole
+// point is to mirror what is current.
+func (s *Show) Partition(keep int) (kept, pruned []*Episode) {
+ ordered := make([]*Episode, len(s.Episodes))
+ copy(ordered, s.Episodes)
+ sort.SliceStable(ordered, func(i, j int) bool {
+ return ordered[i].PublishedUnix > ordered[j].PublishedUnix
+ })
+ if keep >= len(ordered) {
+ return ordered, nil
+ }
+ return ordered[:keep], ordered[keep:]
+}
+
+// State is the server's whole persistent world.
+type State struct {
+ mu sync.RWMutex
+ feeds []string
+ shows map[string]*Show
+}
+
+type persisted struct {
+ Feeds []string `json:"feeds"`
+ Shows map[string]*Show `json:"shows"`
+}
+
+// New returns an empty state.
+func New() *State {
+ return &State{shows: map[string]*Show{}}
+}
+
+// Load reads state from disk. A missing file is a first run, not an error.
+func Load(path string) (*State, error) {
+ raw, err := os.ReadFile(path)
+ if os.IsNotExist(err) {
+ return New(), nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("load state: %w", err)
+ }
+ var p persisted
+ if err := json.Unmarshal(raw, &p); err != nil {
+ return nil, fmt.Errorf("load state: %w", err)
+ }
+ s := New()
+ s.feeds = p.Feeds
+ if p.Shows != nil {
+ s.shows = p.Shows
+ }
+ return s, nil
+}
+
+// Save writes state to disk atomically.
+//
+// Via a temporary file and a rename, because this holds the only record of where mirrored audio
+// is and what detection decided: a truncated write would orphan every file on disk and throw
+// away hours of detection.
+func (s *State) Save(path string) error {
+ s.mu.RLock()
+ raw, err := json.MarshalIndent(persisted{Feeds: s.feeds, Shows: s.shows}, "", " ")
+ s.mu.RUnlock()
+ if err != nil {
+ return fmt.Errorf("save state: %w", err)
+ }
+
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ return fmt.Errorf("save state: %w", err)
+ }
+ tmp, err := os.CreateTemp(filepath.Dir(path), ".state-*.tmp")
+ if err != nil {
+ return fmt.Errorf("save state: %w", err)
+ }
+ tmpName := tmp.Name()
+ defer os.Remove(tmpName)
+
+ if _, err := tmp.Write(raw); err != nil {
+ tmp.Close()
+ return fmt.Errorf("save state: %w", err)
+ }
+ if err := tmp.Sync(); err != nil {
+ tmp.Close()
+ return fmt.Errorf("save state: %w", err)
+ }
+ if err := tmp.Close(); err != nil {
+ return fmt.Errorf("save state: %w", err)
+ }
+ if err := os.Rename(tmpName, path); err != nil {
+ return fmt.Errorf("save state: %w", err)
+ }
+ return nil
+}
+
+// SetFeeds replaces the subscription list and reports the feeds that are no longer wanted.
+//
+// Wholesale replacement because the phone owns this list; reconciling two editable copies of
+// something changed this rarely would cost more in sync bugs than it saves.
+func (s *State) SetFeeds(feeds []string) (removed []string) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ wanted := make(map[string]bool, len(feeds))
+ clean := make([]string, 0, len(feeds))
+ for _, f := range feeds {
+ f = strings.TrimSpace(f)
+ if f == "" || wanted[f] {
+ continue
+ }
+ wanted[f] = true
+ clean = append(clean, f)
+ }
+ for url := range s.shows {
+ if !wanted[url] {
+ removed = append(removed, url)
+ }
+ }
+ sort.Strings(removed)
+ s.feeds = clean
+ return removed
+}
+
+// Feeds is the current subscription list.
+func (s *State) Feeds() []string {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ return append([]string(nil), s.feeds...)
+}
+
+// Forget drops a show and returns it, so its files can be reclaimed.
+func (s *State) Forget(feedURL string) *Show {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ show := s.shows[feedURL]
+ delete(s.shows, feedURL)
+ return show
+}
+
+// Show returns a feed's show, or nil.
+func (s *State) Show(feedURL string) *Show {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ return s.shows[feedURL]
+}
+
+// Shows returns every tracked show.
+func (s *State) Shows() []*Show {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ out := make([]*Show, 0, len(s.shows))
+ for _, sh := range s.shows {
+ out = append(out, sh)
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].FeedURL < out[j].FeedURL })
+ return out
+}
+
+// Merge folds a parsed feed into stored state, adding episodes and refreshing feed-owned
+// fields while leaving everything the server learned alone.
+//
+// The distinction matters and has bitten the Android side twice: feed metadata is rewritten on
+// every refresh, so anything measured or detected has to live in fields the merge does not
+// touch, or it silently reverts.
+func (s *State) Merge(feedURL, title string, episodes []feed.Episode) *Show {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ show := s.shows[feedURL]
+ if show == nil {
+ show = &Show{FeedURL: feedURL}
+ s.shows[feedURL] = show
+ }
+ if title != "" {
+ show.Title = title
+ }
+
+ for _, parsed := range episodes {
+ existing := matchIn(show, parsed.GUID, parsed.EnclosureURL, parsed.Title)
+ if existing == nil {
+ show.Episodes = append(show.Episodes, &Episode{
+ ID: newID(feedURL, parsed.GUID, parsed.EnclosureURL),
+ GUID: parsed.GUID,
+ Title: parsed.Title,
+ EnclosureURL: parsed.EnclosureURL,
+ PublishedUnix: parsed.PublishedUnix,
+ DeclaredDurationMs: parsed.DeclaredDurationMs,
+ })
+ continue
+ }
+ // Feed-owned fields only. Path, DurationMs, IntroMs, OutroMs are the server's.
+ existing.GUID = parsed.GUID
+ existing.Title = parsed.Title
+ existing.EnclosureURL = parsed.EnclosureURL
+ if parsed.PublishedUnix != 0 {
+ existing.PublishedUnix = parsed.PublishedUnix
+ }
+ existing.DeclaredDurationMs = parsed.DeclaredDurationMs
+ }
+ return show
+}
+
+// Match finds an episode within a show, on GUID, then enclosure URL, then title.
+//
+// That cascade exists because publishers rewrite feeds: an episode that loses its identity
+// loses its playback position on the phone and its detected offsets here.
+func (s *State) Match(feedURL, guid, enclosureURL, title string) *Episode {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ show := s.shows[feedURL]
+ if show == nil {
+ return nil
+ }
+ return matchIn(show, guid, enclosureURL, title)
+}
+
+func matchIn(show *Show, guid, enclosureURL, title string) *Episode {
+ if guid != "" {
+ for _, e := range show.Episodes {
+ if e.GUID == guid {
+ return e
+ }
+ }
+ }
+ if enclosureURL != "" {
+ for _, e := range show.Episodes {
+ if e.EnclosureURL == enclosureURL {
+ return e
+ }
+ }
+ }
+ if title != "" {
+ for _, e := range show.Episodes {
+ if e.Title == title {
+ return e
+ }
+ }
+ }
+ return nil
+}
diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go
new file mode 100644
index 0000000..cd5cf6f
--- /dev/null
+++ b/server/internal/store/store_test.go
@@ -0,0 +1,239 @@
+package store
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/terst/listen/server/internal/feed"
+)
+
+func ep(guid, title, enclosure string, published int64) feed.Episode {
+ return feed.Episode{GUID: guid, Title: title, EnclosureURL: enclosure, PublishedUnix: published}
+}
+
+// Merging a refreshed feed into stored state must not lose what the server has learned. The
+// Android side has already had this bug twice in different forms -- a corrected duration
+// overwritten on every refresh, and episode rows cascading away carrying the only record of
+// where their files were.
+
+func TestMergeKeepsWhatTheServerLearnedAboutAnEpisode(t *testing.T) {
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
+
+ got := s.Match("https://f", "g1", "", "")
+ if got == nil {
+ t.Fatal("episode not stored")
+ }
+ got.Path = "audio/abc.mp3"
+ got.DurationMs = 1325113
+ intro := int64(16652)
+ got.IntroMs = &intro
+
+ // The same feed again, as a refresh would deliver it.
+ s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
+
+ after := s.Match("https://f", "g1", "", "")
+ if after == nil {
+ t.Fatal("episode lost on refresh")
+ }
+ if after.Path != "audio/abc.mp3" {
+ t.Errorf("mirrored path lost: %q", after.Path)
+ }
+ if after.DurationMs != 1325113 {
+ t.Errorf("measured duration lost: %d", after.DurationMs)
+ }
+ if after.IntroMs == nil || *after.IntroMs != 16652 {
+ t.Errorf("detected intro lost: %v", after.IntroMs)
+ }
+}
+
+func TestMergeAddsNewEpisodesAndKeepsOldOnes(t *testing.T) {
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
+ s.Merge("https://f", "A Show", []feed.Episode{
+ ep("g2", "Two", "https://cdn/2.mp3", 200),
+ ep("g1", "One", "https://cdn/1.mp3", 100),
+ })
+
+ if n := len(s.Show("https://f").Episodes); n != 2 {
+ t.Errorf("%d episodes, want 2", n)
+ }
+}
+
+// GUIDs move: publishers rewrite feeds. Matching on the GUID alone loses an episode's
+// identity, and with it its playback position on the phone and its detected offsets here.
+func TestMatchFallsBackToTheEnclosureUrlWhenTheGuidChanged(t *testing.T) {
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{ep("old-guid", "One", "https://cdn/1.mp3", 100)})
+
+ got := s.Match("https://f", "new-guid", "https://cdn/1.mp3", "")
+ if got == nil {
+ t.Fatal("not matched by enclosure url")
+ }
+ if got.GUID != "old-guid" {
+ t.Errorf("matched the wrong episode: %q", got.GUID)
+ }
+}
+
+func TestMatchFallsBackToTheTitleWhenBothGuidAndUrlChanged(t *testing.T) {
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{ep("old-guid", "Grass Apes", "https://cdn/old.mp3", 100)})
+
+ got := s.Match("https://f", "new-guid", "https://cdn/new.mp3", "Grass Apes")
+ if got == nil {
+ t.Fatal("not matched by title")
+ }
+ if got.GUID != "old-guid" {
+ t.Errorf("matched the wrong episode: %q", got.GUID)
+ }
+}
+
+// The cascade has to be in that order: a GUID match must win over an enclosure match, or a
+// re-uploaded file would steal another episode's identity.
+func TestMatchPrefersTheGuidOverTheEnclosureUrl(t *testing.T) {
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{
+ ep("g1", "One", "https://cdn/shared.mp3", 100),
+ ep("g2", "Two", "https://cdn/shared.mp3", 200),
+ })
+
+ got := s.Match("https://f", "g2", "https://cdn/shared.mp3", "")
+ if got == nil || got.GUID != "g2" {
+ t.Fatalf("matched %v, want g2", got)
+ }
+}
+
+func TestMatchReturnsNothingForAnUnknownEpisode(t *testing.T) {
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
+
+ if got := s.Match("https://f", "nope", "https://cdn/nope.mp3", "Nope"); got != nil {
+ t.Errorf("matched %q for an unknown episode", got.GUID)
+ }
+}
+
+func TestMatchIsScopedToTheShow(t *testing.T) {
+ s := New()
+ s.Merge("https://a", "Show A", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
+ s.Merge("https://b", "Show B", []feed.Episode{})
+
+ if got := s.Match("https://b", "g1", "", ""); got != nil {
+ t.Error("matched an episode from a different show")
+ }
+}
+
+// Retention: newest N by publish date. Anything the server is mid-way through using has to
+// survive, which is the lesson from the phone's own retention reclaiming the file detection
+// was holding.
+func TestKeepNewestSelectsByPublishDateNotFeedOrder(t *testing.T) {
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{
+ ep("g1", "Oldest", "https://cdn/1.mp3", 100),
+ ep("g3", "Newest", "https://cdn/3.mp3", 300),
+ ep("g2", "Middle", "https://cdn/2.mp3", 200),
+ })
+
+ keep, prune := s.Show("https://f").Partition(2)
+ if len(keep) != 2 || len(prune) != 1 {
+ t.Fatalf("keep %d prune %d, want 2 and 1", len(keep), len(prune))
+ }
+ if keep[0].GUID != "g3" || keep[1].GUID != "g2" {
+ t.Errorf("kept %s and %s, want g3 and g2", keep[0].GUID, keep[1].GUID)
+ }
+ if prune[0].GUID != "g1" {
+ t.Errorf("pruned %s, want g1", prune[0].GUID)
+ }
+}
+
+func TestKeepNewestKeepsEverythingWhenThereAreFewerThanTheLimit(t *testing.T) {
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
+
+ keep, prune := s.Show("https://f").Partition(8)
+ if len(keep) != 1 || len(prune) != 0 {
+ t.Errorf("keep %d prune %d, want 1 and 0", len(keep), len(prune))
+ }
+}
+
+// Subscriptions are replaced wholesale by the phone, which owns the list. A show that is gone
+// from the list stops being polled, but its files are the mirror's business to clean up.
+func TestSetFeedsReplacesTheSubscriptionList(t *testing.T) {
+ s := New()
+ s.Merge("https://a", "A", nil)
+ s.Merge("https://b", "B", nil)
+
+ removed := s.SetFeeds([]string{"https://b", "https://c"})
+
+ if len(removed) != 1 || removed[0] != "https://a" {
+ t.Errorf("removed %v, want [https://a]", removed)
+ }
+ got := s.Feeds()
+ if len(got) != 2 {
+ t.Errorf("feeds %v, want 2", got)
+ }
+}
+
+func TestSaveAndLoadRoundTripsEverything(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "state.json")
+
+ s := New()
+ s.SetFeeds([]string{"https://f"})
+ s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
+ e := s.Match("https://f", "g1", "", "")
+ e.Path = "audio/abc.mp3"
+ intro := int64(16652)
+ e.IntroMs = &intro
+
+ if err := s.Save(path); err != nil {
+ t.Fatal(err)
+ }
+ loaded, err := Load(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ got := loaded.Match("https://f", "g1", "", "")
+ if got == nil {
+ t.Fatal("episode did not survive the round trip")
+ }
+ if got.Path != "audio/abc.mp3" || got.IntroMs == nil || *got.IntroMs != 16652 {
+ t.Errorf("round trip lost data: %+v", got)
+ }
+ if len(loaded.Feeds()) != 1 {
+ t.Errorf("feeds did not survive: %v", loaded.Feeds())
+ }
+}
+
+// A missing state file is a first run, not a failure.
+func TestLoadingAMissingFileGivesAnEmptyState(t *testing.T) {
+ s, err := Load(filepath.Join(t.TempDir(), "absent.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(s.Feeds()) != 0 {
+ t.Errorf("expected no feeds, got %v", s.Feeds())
+ }
+}
+
+// Saving must not be able to truncate a good state file into a broken one.
+func TestSaveIsAtomic(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "state.json")
+ s := New()
+ s.SetFeeds([]string{"https://f"})
+ if err := s.Save(path); err != nil {
+ t.Fatal(err)
+ }
+
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, entry := range entries {
+ if entry.Name() != "state.json" {
+ t.Errorf("left a stray file behind: %s", entry.Name())
+ }
+ }
+}