summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--server/cmd/listend/main.go76
-rw-r--r--server/internal/api/api.go52
-rw-r--r--server/internal/api/api_test.go3
-rw-r--r--server/internal/api/transcript_test.go182
-rw-r--r--server/internal/feed/feed.go58
-rw-r--r--server/internal/feed/feed_test.go125
-rw-r--r--server/internal/store/store.go34
-rw-r--r--server/internal/store/store_test.go68
-rw-r--r--server/internal/transcript/fits.go73
-rw-r--r--server/internal/transcript/fits_test.go94
-rw-r--r--server/internal/transcript/harvest.go101
-rw-r--r--server/internal/transcript/harvest_test.go145
-rw-r--r--server/internal/transcript/parse.go138
-rw-r--r--server/internal/transcript/parse_test.go158
-rw-r--r--server/internal/transcript/store.go131
-rw-r--r--server/internal/transcript/store_test.go101
16 files changed, 1534 insertions, 5 deletions
diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go
index a4400d2..3ca2727 100644
--- a/server/cmd/listend/main.go
+++ b/server/cmd/listend/main.go
@@ -34,6 +34,7 @@ import (
"github.com/terst/listen/server/internal/signature"
"github.com/terst/listen/server/internal/skips"
"github.com/terst/listen/server/internal/store"
+ "github.com/terst/listen/server/internal/transcript"
)
func main() {
@@ -69,6 +70,8 @@ func main() {
}
decoder := decode.Decoder{}
+ transcriptDir := filepath.Join(*dataDir, "transcripts")
+ transcripts := transcript.Store{Dir: transcriptDir}
worker := &worker{
state: state,
statePath: statePath,
@@ -77,13 +80,16 @@ func main() {
keep: *keep,
mirror: &mirror.Mirror{AudioDir: audioDir, Probe: decoder},
decoder: decoder,
+ transcripts: transcripts,
+ harvester: transcript.Harvester{Store: transcripts},
requests: make(chan request, 64),
wakeups: make(chan struct{}, 1),
}
server := api.New(state, worker, api.Options{
- Token: token,
- AudioDir: audioDir,
+ Token: token,
+ AudioDir: audioDir,
+ TranscriptDir: transcriptDir,
// Persist the list the moment it arrives, then poll.
//
// Both halves were learned the hard way. The list is the one thing the server cannot
@@ -156,8 +162,12 @@ type worker struct {
keep int
mirror *mirror.Mirror
decoder decode.Decoder
- requests chan request
- wakeups chan struct{}
+ // transcripts holds each episode's words, and harvester takes the publishers' own where
+ // they describe the copy that was mirrored. See transcript.Fits for when they do not.
+ transcripts transcript.Store
+ harvester transcript.Harvester
+ requests chan request
+ wakeups chan struct{}
}
// wake asks for a poll now. Coalescing, so a burst of subscription updates is one poll.
@@ -291,6 +301,14 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) {
if e.Mirrored() {
_ = w.mirror.Prune(e)
}
+ // The words go with the audio. A transcript of a file nobody holds any more is of no
+ // use to the phone, which plays the server's bytes, and the directory would otherwise
+ // grow for ever.
+ if e.TranscriptExamined() {
+ _ = w.transcripts.Prune(e.ID)
+ e.TranscriptSource, e.TranscriptCues, e.TranscriptNote = "", 0, ""
+ e.TranscriptAtUnix = 0
+ }
}
fetched := 0
@@ -311,9 +329,59 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) {
if fetched > 0 {
log.Printf("%s: mirrored %d new episode(s)", channel.Title, fetched)
}
+ w.harvestTranscripts(ctx, show, kept)
w.detect(ctx, show, kept)
}
+// harvestTranscripts takes the publishers' own transcripts for a show's mirrored episodes.
+//
+// Only where the publisher's master is what was mirrored, which transcript.Fits decides from the
+// declared-versus-served length gap rather than from the words. A show using dynamic ad
+// insertion publishes a transcript of the ad-free programme, and reading along with it would be
+// minutes out of step with the audio the phone plays.
+//
+// Each episode is examined once. A refusal is a judgement about this copy, so re-fetching it on
+// every poll would be work done for ever to reach the same answer -- and the refusal is itself
+// the record of which episodes speech recognition would have to do.
+func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candidates []*store.Episode) {
+ taken, declined := 0, 0
+ for _, e := range candidates {
+ if ctx.Err() != nil {
+ return
+ }
+ if !e.Mirrored() || e.TranscriptExamined() || e.TranscriptURL == "" {
+ continue
+ }
+ result, err := w.harvester.Harvest(ctx, transcript.Offer{
+ EpisodeID: e.ID,
+ URL: e.TranscriptURL,
+ Type: e.TranscriptType,
+ ServedMs: e.DurationMs,
+ DeclaredMs: e.DeclaredDurationMs,
+ })
+ if err != nil {
+ // The download failed, so nothing has been decided and the next poll tries again.
+ log.Printf("%s: transcript for %q: %v", show.Title, e.Title, err)
+ continue
+ }
+ switch {
+ case result.Stored:
+ e.TranscriptSource = string(transcript.SourcePublished)
+ e.TranscriptCues = result.Cues
+ e.TranscriptAtUnix = time.Now().Unix()
+ e.TranscriptNote = ""
+ taken++
+ case result.Note != "":
+ e.TranscriptNote = result.Note
+ e.TranscriptAtUnix = time.Now().Unix()
+ declined++
+ }
+ }
+ if taken > 0 || declined > 0 {
+ log.Printf("%s: transcripts -- took %d, declined %d", show.Title, taken, declined)
+ }
+}
+
// detect runs the intro/outro search over a show's mirrored episodes and records the result.
//
// Every mirrored episode at once, which is the whole reason this is a server: the phone could
diff --git a/server/internal/api/api.go b/server/internal/api/api.go
index 969e69d..bcfe6a2 100644
--- a/server/internal/api/api.go
+++ b/server/internal/api/api.go
@@ -10,6 +10,7 @@ import (
"strings"
"github.com/terst/listen/server/internal/store"
+ "github.com/terst/listen/server/internal/transcript"
)
// Fetcher is asked to mirror an episode the phone wants and the server does not have.
@@ -27,6 +28,10 @@ type Options struct {
Token string
// AudioDir is where mirrored episodes live.
AudioDir string
+ // TranscriptDir is where episodes' words live, one file per episode. Separate from the
+ // state file because a transcript is a thousand cues and state.json is rewritten whole
+ // every time anything at all changes.
+ TranscriptDir string
// OnSubscriptions is called when the phone sends its list, so a newly-subscribed show is
// polled at once rather than at the next interval. Optional.
OnSubscriptions func(feeds []string)
@@ -71,6 +76,22 @@ type episodeResponse struct {
// AdsRefused: matches were found but were too extensive to be advertising, so none were
// published. Distinct from nothing having matched.
AdsRefused bool `json:"adsRefused,omitempty"`
+
+ // Transcript says whether this episode has words and where they came from, or why it has
+ // none. Absent entirely when nobody has looked yet, which is a third state again.
+ Transcript *transcriptNote `json:"transcript,omitempty"`
+}
+
+// transcriptNote is the line the phone needs on every episode: is there a transcript, whose
+// words are they, and if there is none, why not.
+//
+// The source travels with it because the two kinds are not the same thing to read along with.
+// A published transcript is the publisher's exact words; a recognised one is a machine's guess
+// at the same audio, and somebody reading it should know which they are looking at.
+type transcriptNote struct {
+ Source string `json:"source,omitempty"`
+ Cues int `json:"cues,omitempty"`
+ Note string `json:"note,omitempty"`
}
type breakRange struct {
@@ -90,6 +111,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("/api/subscriptions", s.authed(s.subscriptions))
mux.HandleFunc("/api/episode", s.authed(s.episode))
mux.HandleFunc("/api/labels", s.authed(s.labels))
+ mux.HandleFunc("/api/transcript", s.authed(s.transcript))
mux.HandleFunc("/audio/", s.authed(s.audio))
return mux
}
@@ -190,9 +212,39 @@ func (s *Server) episode(w http.ResponseWriter, r *http.Request) {
FoundAdMs: found.FoundAdMs,
InsertedAdMs: found.InsertedAdMs,
AdsRefused: found.AdsRefused,
+ Transcript: transcriptNoteFor(found),
})
}
+func transcriptNoteFor(e *store.Episode) *transcriptNote {
+ if !e.TranscriptExamined() {
+ return nil
+ }
+ return &transcriptNote{
+ Source: e.TranscriptSource,
+ Cues: e.TranscriptCues,
+ Note: e.TranscriptNote,
+ }
+}
+
+// transcript serves one episode's words, which are fetched only when somebody opens them --
+// a thousand cues is not something to attach to every episode lookup.
+func (s *Server) transcript(w http.ResponseWriter, r *http.Request) {
+ id := r.URL.Query().Get("episode")
+ // Same rule as the audio route: the id decides a path, so anything with a separator or a
+ // dot segment in it is an attempt to read elsewhere.
+ if id == "" || strings.ContainsAny(id, "/\\") || strings.Contains(id, "..") {
+ http.NotFound(w, r)
+ return
+ }
+ held, err := (transcript.Store{Dir: s.opts.TranscriptDir}).Get(id)
+ if err != nil {
+ http.NotFound(w, r)
+ return
+ }
+ writeJSON(w, http.StatusOK, held)
+}
+
func (s *Server) audio(w http.ResponseWriter, r *http.Request) {
id := strings.TrimPrefix(r.URL.Path, "/audio/")
// No separators: an id is a flat hex string, and anything else is an attempt to read
diff --git a/server/internal/api/api_test.go b/server/internal/api/api_test.go
index d4d3ee6..19e40db 100644
--- a/server/internal/api/api_test.go
+++ b/server/internal/api/api_test.go
@@ -26,7 +26,8 @@ func newServer(t *testing.T) (*Server, *store.State, *fakeFetcher, string) {
audioDir := t.TempDir()
st := store.New()
fetcher := &fakeFetcher{}
- return New(st, fetcher, Options{Token: token, AudioDir: audioDir}), st, fetcher, audioDir
+ opts := Options{Token: token, AudioDir: audioDir, TranscriptDir: t.TempDir()}
+ return New(st, fetcher, opts), st, fetcher, audioDir
}
func get(t *testing.T, s *Server, path string) *httptest.ResponseRecorder {
diff --git a/server/internal/api/transcript_test.go b/server/internal/api/transcript_test.go
new file mode 100644
index 0000000..b208f7f
--- /dev/null
+++ b/server/internal/api/transcript_test.go
@@ -0,0 +1,182 @@
+package api
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/terst/listen/server/internal/feed"
+ "github.com/terst/listen/server/internal/store"
+ "github.com/terst/listen/server/internal/transcript"
+)
+
+/*
+Serving an episode's words.
+
+Two pieces, deliberately separate. The episode response says whether there are words and where
+they came from, which is a line the phone needs on every episode it asks about; the transcript
+itself is a thousand cues and is fetched only when somebody opens it.
+
+Where they came from is not decoration. A published transcript is the publisher's exact words;
+a recognised one is a machine's guess at the same audio, and reading along with it means seeing
+the difference. The app says which.
+*/
+
+func writeAudio(t *testing.T, audioDir, name string) {
+ t.Helper()
+ if err := os.WriteFile(filepath.Join(audioDir, name), []byte("audio"), 0o644); err != nil {
+ t.Fatalf("write audio: %v", err)
+ }
+}
+
+func recorder() *httptest.ResponseRecorder { return httptest.NewRecorder() }
+
+func seedTranscribedEpisode(t *testing.T, s *Server, st *store.State, audioDir string) *store.Episode {
+ t.Helper()
+ st.Merge("https://f", "A Show", []feed.Episode{{
+ GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100,
+ }})
+ e := st.Match("https://f", "g1", "", "")
+ e.Path = e.ID + ".mp3"
+ e.DurationMs = 1_217_000
+ writeAudio(t, audioDir, e.Path)
+
+ held := transcript.Store{Dir: s.opts.TranscriptDir}
+ if err := held.Put(transcript.Transcript{
+ EpisodeID: e.ID,
+ Source: transcript.SourcePublished,
+ Cues: []transcript.Cue{
+ {StartMs: 0, EndMs: 3120, Text: "Welcome back."},
+ {StartMs: 3500, EndMs: 7000, Text: "This week's sponsor is a mattress."},
+ },
+ }); err != nil {
+ t.Fatalf("seed transcript: %v", err)
+ }
+ e.TranscriptSource = string(transcript.SourcePublished)
+ e.TranscriptCues = 2
+ return e
+}
+
+func TestTheEpisodeResponseSaysWhetherThereAreWords(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ seedTranscribedEpisode(t, s, st, audioDir)
+
+ rec := get(t, s, "/api/episode?feed=https://f&guid=g1")
+ var got struct {
+ Transcript *struct {
+ Source string `json:"source"`
+ Cues int `json:"cues"`
+ } `json:"transcript"`
+ }
+ if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if got.Transcript == nil {
+ t.Fatal("the episode has a transcript and the response does not mention it")
+ }
+ if got.Transcript.Source != "published" || got.Transcript.Cues != 2 {
+ t.Errorf("got %+v", *got.Transcript)
+ }
+}
+
+// Refused is not the same as absent, and the phone shows the difference: an episode whose
+// publisher's transcript does not describe the copy being played has been looked at.
+func TestTheEpisodeResponseCarriesARefusal(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ st.Merge("https://f", "A Show", []feed.Episode{{
+ GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100,
+ }})
+ e := st.Match("https://f", "g1", "", "")
+ e.Path = e.ID + ".mp3"
+ writeAudio(t, audioDir, e.Path)
+ e.TranscriptNote = "this copy carries 4:16 of inserted advertising"
+
+ rec := get(t, s, "/api/episode?feed=https://f&guid=g1")
+ var got struct {
+ Transcript *struct {
+ Cues int `json:"cues"`
+ Note string `json:"note"`
+ } `json:"transcript"`
+ }
+ if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if got.Transcript == nil || got.Transcript.Note == "" || got.Transcript.Cues != 0 {
+ t.Errorf("got %+v, want a refusal with no cues", got.Transcript)
+ }
+}
+
+func TestAnEpisodeNobodyHasLookedAtMentionsNoTranscript(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ st.Merge("https://f", "A Show", []feed.Episode{{
+ GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100,
+ }})
+ e := st.Match("https://f", "g1", "", "")
+ e.Path = e.ID + ".mp3"
+ writeAudio(t, audioDir, e.Path)
+
+ rec := get(t, s, "/api/episode?feed=https://f&guid=g1")
+ var got struct {
+ Transcript *json.RawMessage `json:"transcript"`
+ }
+ if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if got.Transcript != nil {
+ t.Errorf("invented a transcript field: %s", *got.Transcript)
+ }
+}
+
+func TestTheTranscriptItselfIsServedByEpisodeId(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ e := seedTranscribedEpisode(t, s, st, audioDir)
+
+ rec := get(t, s, "/api/transcript?episode="+e.ID)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
+ }
+ var got transcript.Transcript
+ if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if len(got.Cues) != 2 || got.Cues[1].Text != "This week's sponsor is a mattress." {
+ t.Errorf("got %+v", got)
+ }
+}
+
+func TestAskingForATranscriptThatDoesNotExistIsNotFound(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ st.Merge("https://f", "A Show", []feed.Episode{{
+ GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100,
+ }})
+ e := st.Match("https://f", "g1", "", "")
+ e.Path = e.ID + ".mp3"
+ writeAudio(t, audioDir, e.Path)
+
+ if rec := get(t, s, "/api/transcript?episode="+e.ID); rec.Code != http.StatusNotFound {
+ t.Errorf("status %d, want 404", rec.Code)
+ }
+}
+
+// Same rule as the audio route: an id decides a path, so it is checked rather than trusted.
+func TestATranscriptIdCannotReachOutsideTheDirectory(t *testing.T) {
+ s, _, _, _ := newServer(t)
+ for _, id := range []string{"../state", "a/b", ""} {
+ if rec := get(t, s, "/api/transcript?episode="+id); rec.Code == http.StatusOK {
+ t.Errorf("%q: served something", id)
+ }
+ }
+}
+
+func TestTheTranscriptRouteNeedsTheToken(t *testing.T) {
+ s, _, _, _ := newServer(t)
+ req, _ := http.NewRequest(http.MethodGet, "/api/transcript?episode=abc", nil)
+ rec := recorder()
+ s.Handler().ServeHTTP(rec, req)
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("status %d, want 401", rec.Code)
+ }
+}
diff --git a/server/internal/feed/feed.go b/server/internal/feed/feed.go
index edc58be..493fbdd 100644
--- a/server/internal/feed/feed.go
+++ b/server/internal/feed/feed.go
@@ -21,6 +21,12 @@ type Episode struct {
// DeclaredDurationMs is the feed's claim, which is routinely wrong -- a show using dynamic
// ad insertion declares the ad-free length. Useful for ordering, not for arithmetic.
DeclaredDurationMs int64
+ // TranscriptURL is a Podcasting 2.0 transcript in a format that carries timestamps, or
+ // empty. The publisher's own words, which is exact where speech recognition is a guess --
+ // but it describes the publisher's master, so whether it describes the file the server
+ // mirrored is a separate question. See transcript.Fits.
+ TranscriptURL string
+ TranscriptType string
}
// Channel is a show and its audio episodes, newest first.
@@ -41,6 +47,12 @@ type rss struct {
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
} `xml:"enclosure"`
+ // Both spellings of the Podcasting 2.0 namespace. The specification was published
+ // under http:// and the feeds in the wild now declare https://; Go matches a
+ // namespace by exact string, so taking only one silently drops the other's
+ // transcripts rather than failing.
+ Transcripts []transcriptTag `xml:"https://podcastindex.org/namespace/1.0 transcript"`
+ OldNamespace []transcriptTag `xml:"http://podcastindex.org/namespace/1.0 transcript"`
} `xml:"item"`
} `xml:"channel"`
}
@@ -64,6 +76,7 @@ func Parse(raw []byte) (*Channel, error) {
if !isAudio(item.Enclosure.Type, item.Enclosure.URL) {
continue
}
+ transcriptURL, transcriptType := bestTranscript(append(item.Transcripts, item.OldNamespace...))
out.Episodes = append(out.Episodes, Episode{
GUID: strings.TrimSpace(item.GUID),
Title: strings.TrimSpace(item.Title),
@@ -71,6 +84,8 @@ func Parse(raw []byte) (*Channel, error) {
EnclosureType: strings.TrimSpace(item.Enclosure.Type),
PublishedUnix: parseDate(item.PubDate),
DeclaredDurationMs: parseDuration(item.Duration),
+ TranscriptURL: transcriptURL,
+ TranscriptType: transcriptType,
})
}
@@ -81,6 +96,49 @@ func Parse(raw []byte) (*Channel, error) {
return out, nil
}
+// transcriptRank orders the transcript formats a publisher may offer. An item routinely carries
+// several at once, and the MIME types in the wild disagree about the same format: both
+// application/srt and application/x-subrip are SRT.
+//
+// WebVTT first only because it is the format with a specification that says what a cue is; SRT
+// is equally usable and ranks next. text/plain, text/html and application/json are absent
+// deliberately -- they carry no timestamps, so a cue cannot be seeked to or placed, and a
+// transcript that can do neither is worse than none at all because storing it stops the episode
+// being looked at again.
+var transcriptRank = map[string]int{
+ "text/vtt": 1,
+ "text/webvtt": 1,
+ "application/srt": 2,
+ "application/x-subrip": 2,
+ "text/srt": 2,
+}
+
+// transcriptTag is one <podcast:transcript> element.
+type transcriptTag struct {
+ URL string `xml:"url,attr"`
+ Type string `xml:"type,attr"`
+}
+
+func bestTranscript(offered []transcriptTag) (string, string) {
+ bestRank := 0
+ var bestURL, bestType string
+ for _, t := range offered {
+ url := strings.TrimSpace(t.URL)
+ mediaType := strings.ToLower(strings.TrimSpace(t.Type))
+ if i := strings.IndexByte(mediaType, ';'); i >= 0 {
+ mediaType = strings.TrimSpace(mediaType[:i])
+ }
+ rank, ok := transcriptRank[mediaType]
+ if !ok || url == "" {
+ continue
+ }
+ if bestRank == 0 || rank < bestRank {
+ bestRank, bestURL, bestType = rank, url, mediaType
+ }
+ }
+ return bestURL, bestType
+}
+
func isAudio(mediaType, url string) bool {
if url == "" {
return false
diff --git a/server/internal/feed/feed_test.go b/server/internal/feed/feed_test.go
index e62558f..ce68404 100644
--- a/server/internal/feed/feed_test.go
+++ b/server/internal/feed/feed_test.go
@@ -143,3 +143,128 @@ func TestAFeedWithNoItemsIsNotAnError(t *testing.T) {
t.Errorf("got %d episodes", len(ch.Episodes))
}
}
+
+/*
+Podcasting 2.0 transcripts.
+
+Eleven of Peter's 44 feeds carry <podcast:transcript>, and the shapes below are the ones they
+actually use (surveyed 2026-10-28). An item routinely carries the same transcript in several
+formats at once, and the MIME types disagree with each other: application/srt and
+application/x-subrip are both SRT, and text/plain, text/html and application/json carry no
+timestamps at all.
+
+Only a timestamped one is worth taking: the app seeks to a cue and the detector places a break
+at one, and neither can be done with a wall of text.
+*/
+
+const withTranscripts = `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:podcast="https://podcastindex.org/namespace/1.0">
+<channel>
+ <title>A Show</title>
+ <item>
+ <title>Every Format At Once</title>
+ <guid>https://example.com/1</guid>
+ <pubDate>Fri, 03 Oct 2026 10:00:00 -0000</pubDate>
+ <enclosure url="https://cdn.example/1.mp3" type="audio/mpeg"/>
+ <podcast:transcript url="https://cdn.example/1.txt" type="text/plain"/>
+ <podcast:transcript url="https://cdn.example/1.srt" type="application/srt"/>
+ <podcast:transcript url="https://cdn.example/1.vtt" type="text/vtt"/>
+ </item>
+ <item>
+ <title>Subrip Spelling</title>
+ <guid>https://example.com/2</guid>
+ <pubDate>Fri, 26 Sep 2026 10:00:00 -0000</pubDate>
+ <enclosure url="https://cdn.example/2.mp3" type="audio/mpeg"/>
+ <podcast:transcript url="https://cdn.example/2.srt" type="application/x-subrip"/>
+ </item>
+ <item>
+ <title>Nothing Timestamped</title>
+ <guid>https://example.com/3</guid>
+ <pubDate>Fri, 19 Sep 2026 10:00:00 -0000</pubDate>
+ <enclosure url="https://cdn.example/3.mp3" type="audio/mpeg"/>
+ <podcast:transcript url="https://cdn.example/3.html" type="text/html"/>
+ <podcast:transcript url="https://cdn.example/3.json" type="application/json"/>
+ </item>
+ <item>
+ <title>None At All</title>
+ <guid>https://example.com/4</guid>
+ <pubDate>Fri, 12 Sep 2026 10:00:00 -0000</pubDate>
+ <enclosure url="https://cdn.example/4.mp3" type="audio/mpeg"/>
+ </item>
+</channel>
+</rss>`
+
+func TestPrefersAWebVTTTranscriptWhenSeveralAreOffered(t *testing.T) {
+ channel, err := Parse([]byte(withTranscripts))
+ if err != nil {
+ t.Fatalf("Parse: %v", err)
+ }
+ got := channel.Episodes[0]
+ if got.TranscriptURL != "https://cdn.example/1.vtt" || got.TranscriptType != "text/vtt" {
+ t.Errorf("got %q (%q), want the WebVTT one", got.TranscriptURL, got.TranscriptType)
+ }
+}
+
+func TestTakesSRTUnderEitherSpelling(t *testing.T) {
+ channel, err := Parse([]byte(withTranscripts))
+ if err != nil {
+ t.Fatalf("Parse: %v", err)
+ }
+ got := channel.Episodes[1]
+ if got.TranscriptURL != "https://cdn.example/2.srt" {
+ t.Errorf("got %q, want the x-subrip one", got.TranscriptURL)
+ }
+}
+
+func TestIgnoresTranscriptsWithNoTimestamps(t *testing.T) {
+ channel, err := Parse([]byte(withTranscripts))
+ if err != nil {
+ t.Fatalf("Parse: %v", err)
+ }
+ if got := channel.Episodes[2]; got.TranscriptURL != "" {
+ t.Errorf("took %q (%q), which has no times in it", got.TranscriptURL, got.TranscriptType)
+ }
+}
+
+func TestAnEpisodeWithNoTranscriptSaysSo(t *testing.T) {
+ channel, err := Parse([]byte(withTranscripts))
+ if err != nil {
+ t.Fatalf("Parse: %v", err)
+ }
+ if got := channel.Episodes[3]; got.TranscriptURL != "" {
+ t.Errorf("invented %q", got.TranscriptURL)
+ }
+}
+
+/*
+The namespace URI is spelled two ways in the wild.
+
+Every one of Peter's eleven transcript-carrying feeds declares the https:// form today
+(surveyed 2026-10-28), but the Podcasting 2.0 specification was published under http:// and
+older generators still emit it. Go matches a namespace by exact string, so a feed using the
+other spelling would parse as having no transcripts at all -- silently, which is the failure
+worth pinning: nothing errors, the episodes simply never get words.
+*/
+func TestTranscriptsAreFoundUnderEitherNamespaceSpelling(t *testing.T) {
+ const httpNamespace = `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:podcast="http://podcastindex.org/namespace/1.0">
+<channel>
+ <title>A Show</title>
+ <item>
+ <title>Older Generator</title>
+ <guid>https://example.com/1</guid>
+ <pubDate>Fri, 03 Oct 2026 10:00:00 -0000</pubDate>
+ <enclosure url="https://cdn.example/1.mp3" type="audio/mpeg"/>
+ <podcast:transcript url="https://cdn.example/1.vtt" type="text/vtt"/>
+ </item>
+</channel>
+</rss>`
+
+ channel, err := Parse([]byte(httpNamespace))
+ if err != nil {
+ t.Fatalf("Parse: %v", err)
+ }
+ if got := channel.Episodes[0].TranscriptURL; got != "https://cdn.example/1.vtt" {
+ t.Errorf("got %q, want the transcript -- the http:// namespace was dropped", got)
+ }
+}
diff --git a/server/internal/store/store.go b/server/internal/store/store.go
index 7f04fa0..26dab82 100644
--- a/server/internal/store/store.go
+++ b/server/internal/store/store.go
@@ -52,6 +52,36 @@ type Episode struct {
// DetectedAtUnix records when detection last ran over the show this belongs to, so a
// result can be told apart from never having looked.
DetectedAtUnix int64 `json:"detectedAtUnix,omitempty"`
+
+ // TranscriptURL and TranscriptType are where the publisher says its own transcript is, in
+ // a format carrying timestamps. Feed-owned, like the enclosure.
+ TranscriptURL string `json:"transcriptUrl,omitempty"`
+ TranscriptType string `json:"transcriptType,omitempty"`
+
+ // TranscriptSource says where the words the server actually holds came from: "published"
+ // for the publisher's own, "asr" for speech recognition over the mirrored bytes. They are
+ // not interchangeable -- a published transcript is exact but describes the publisher's
+ // master, and recognition is approximate but describes the file the phone will play.
+ TranscriptSource string `json:"transcriptSource,omitempty"`
+ // TranscriptCues is how many cues the stored transcript has, so "does this episode have
+ // words" is answerable without opening the file they live in.
+ TranscriptCues int `json:"transcriptCues,omitempty"`
+ // TranscriptAtUnix is when the transcript landed.
+ TranscriptAtUnix int64 `json:"transcriptAtUnix,omitempty"`
+ // TranscriptNote is why a transcript that was offered was not taken -- most often that this
+ // copy carries inserted advertising the publisher's master does not describe. Kept because
+ // refused and never-looked-at are different states, and because a refusal here is the
+ // signal that this episode is one recognition would have to do.
+ TranscriptNote string `json:"transcriptNote,omitempty"`
+}
+
+// HasTranscript reports whether the server holds words for this episode.
+func (e *Episode) HasTranscript() bool { return e.TranscriptCues > 0 }
+
+// TranscriptExamined reports whether anything has been decided about this episode's words, so a
+// refusal is not re-read as an episode nobody has looked at yet.
+func (e *Episode) TranscriptExamined() bool {
+ return e.HasTranscript() || e.TranscriptNote != ""
}
// Break is one stretch of an episode to skip.
@@ -275,6 +305,8 @@ func (s *State) Merge(feedURL, title string, episodes []feed.Episode) *Show {
EnclosureURL: parsed.EnclosureURL,
PublishedUnix: parsed.PublishedUnix,
DeclaredDurationMs: parsed.DeclaredDurationMs,
+ TranscriptURL: parsed.TranscriptURL,
+ TranscriptType: parsed.TranscriptType,
})
continue
}
@@ -286,6 +318,8 @@ func (s *State) Merge(feedURL, title string, episodes []feed.Episode) *Show {
existing.PublishedUnix = parsed.PublishedUnix
}
existing.DeclaredDurationMs = parsed.DeclaredDurationMs
+ existing.TranscriptURL = parsed.TranscriptURL
+ existing.TranscriptType = parsed.TranscriptType
}
return show
}
diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go
index 0540fbf..61cbb30 100644
--- a/server/internal/store/store_test.go
+++ b/server/internal/store/store_test.go
@@ -319,3 +319,71 @@ func TestWhatHasBeenExaminedSurvivesARestart(t *testing.T) {
t.Fatal("a restart would redo every show")
}
}
+
+/*
+The feed owns where a transcript is; the server owns whether it took it.
+
+Same split as every other field here, and for the same reason the duration pair exists: a
+refresh must not overwrite what the server measured. A transcript that was refused because this
+copy of the episode carries inserted advertising would otherwise look unexamined again on every
+poll, and be re-fetched and re-refused for ever.
+*/
+
+func TestARefreshUpdatesWhereTheTranscriptIsWithoutForgettingTheVerdict(t *testing.T) {
+ state := New()
+ state.Merge("https://f", "A Show", []feed.Episode{{
+ GUID: "g1",
+ Title: "One",
+ EnclosureURL: "https://cdn/1.mp3",
+ PublishedUnix: 100,
+ TranscriptURL: "https://cdn/1.vtt", TranscriptType: "text/vtt",
+ }})
+ episode := state.Match("https://f", "g1", "", "")
+ episode.TranscriptSource = "published"
+ episode.TranscriptCues = 412
+ episode.TranscriptAtUnix = 1_700_000_000
+
+ // The publisher moves the file, as publishers do.
+ state.Merge("https://f", "A Show", []feed.Episode{{
+ GUID: "g1",
+ Title: "One",
+ EnclosureURL: "https://cdn/1.mp3",
+ PublishedUnix: 100,
+ TranscriptURL: "https://cdn/1-v2.vtt", TranscriptType: "text/vtt",
+ }})
+
+ got := state.Match("https://f", "g1", "", "")
+ if got.TranscriptURL != "https://cdn/1-v2.vtt" {
+ t.Errorf("transcript url = %q, want the feed's new one", got.TranscriptURL)
+ }
+ if got.TranscriptSource != "published" || got.TranscriptCues != 412 {
+ t.Errorf("a refresh forgot what the server stored: %+v", got)
+ }
+}
+
+func TestAnEpisodeKnowsWhetherItHasWords(t *testing.T) {
+ var e Episode
+ if e.HasTranscript() {
+ t.Error("an episode with no cues claims a transcript")
+ }
+ e.TranscriptCues = 1
+ if !e.HasTranscript() {
+ t.Error("an episode with cues denies having a transcript")
+ }
+}
+
+// Refused and never-looked-at are different states, which is the readout mistake this codebase
+// has now made twice.
+func TestARefusedTranscriptIsNotTheSameAsNoneFound(t *testing.T) {
+ e := Episode{TranscriptNote: "this copy carries 4:16 of inserted advertising"}
+ if e.HasTranscript() {
+ t.Error("a refusal reads as a transcript")
+ }
+ if !e.TranscriptExamined() {
+ t.Error("an episode that was examined and refused reads as never examined")
+ }
+ empty := Episode{}
+ if empty.TranscriptExamined() {
+ t.Error("an episode nobody has looked at reads as examined")
+ }
+}
diff --git a/server/internal/transcript/fits.go b/server/internal/transcript/fits.go
new file mode 100644
index 0000000..7597039
--- /dev/null
+++ b/server/internal/transcript/fits.go
@@ -0,0 +1,73 @@
+package transcript
+
+import "fmt"
+
+const (
+ // insertionMs is how far a served file may exceed its declared length before the excess is
+ // read as inserted advertising rather than as the feed being approximate. Fifteen seconds,
+ // the same unit skips.Classify uses and for the same reason: advertising is sold in units of
+ // roughly that length, so a smaller gap cannot be a real insertion.
+ insertionMs = 15_000
+ // tailMs is how much of a file may follow the last spoken word before the silence has to be
+ // explained. Closing music runs to fifteen seconds in this corpus (LOW's "Part 5: Bane"
+ // ends 15.2s before its file does); the shortest advertisement does not fit in sixty.
+ tailMs = 60_000
+)
+
+// Verdict is whether a published transcript may be used for a particular mirrored file, and if
+// not, why not.
+//
+// Why, not just no. A transcript refused because the file carries inserted advertising is a
+// different thing from one refused because it describes another recording: the first is an
+// episode that needs speech recognition, and the second is a publisher's mistake.
+type Verdict struct {
+ OK bool
+ Why string
+}
+
+// Fits reports whether a transcript ending at lastCueEndMs describes a mirrored file of
+// servedMs, given the declaredMs the feed claims.
+//
+// The decision is made on the lengths rather than on the words because the lengths are
+// independent of the transcript: the feed declares the ad-free programme's length and the file
+// measures what was served, so their difference is the advertising inserted into this copy.
+// A publisher's transcript always describes the ad-free master, so it aligns exactly when that
+// difference is nothing.
+func Fits(lastCueEndMs, servedMs, declaredMs int64) Verdict {
+ if lastCueEndMs <= 0 {
+ return Verdict{Why: "the transcript has no cues"}
+ }
+ if lastCueEndMs > servedMs {
+ return Verdict{Why: fmt.Sprintf(
+ "the transcript runs %s past the end of the file, so it describes another recording",
+ clock(lastCueEndMs-servedMs))}
+ }
+ if declaredMs > 0 {
+ if gap := servedMs - declaredMs; gap > insertionMs {
+ return Verdict{Why: fmt.Sprintf(
+ "this copy carries %s of inserted advertising the transcript does not describe",
+ clock(gap))}
+ }
+ return Verdict{OK: true}
+ }
+ // No declared length to compare against, so the transcript answers for its own gap.
+ if gap := servedMs - lastCueEndMs; gap > tailMs {
+ return Verdict{Why: fmt.Sprintf(
+ "the transcript stops %s before the file does, and the feed declares no length",
+ clock(gap))}
+ }
+ return Verdict{OK: true}
+}
+
+// LastCueEndMs is where a transcript stops describing the audio.
+func LastCueEndMs(cues []Cue) int64 {
+ if len(cues) == 0 {
+ return 0
+ }
+ return cues[len(cues)-1].EndMs
+}
+
+func clock(ms int64) string {
+ s := ms / 1000
+ return fmt.Sprintf("%d:%02d", s/60, s%60)
+}
diff --git a/server/internal/transcript/fits_test.go b/server/internal/transcript/fits_test.go
new file mode 100644
index 0000000..4df5c5a
--- /dev/null
+++ b/server/internal/transcript/fits_test.go
@@ -0,0 +1,94 @@
+package transcript
+
+import "testing"
+
+/*
+Whether a publisher's transcript describes the file the server actually mirrored.
+
+Measured, not assumed, across the four mirrored shows in Peter's library that publish SRT or
+WebVTT (2026-10-28), comparing each transcript's last cue against the declared length and the
+length the mirrored file measures:
+
+ Darknet Diaries "Conti" declared 3557s served 3813s last cue 3544s
+ Wards Way "A Family Bar" declared 1216s served 1217s last cue 1213s
+ LOW "Afterword" declared 610s served 610s last cue 603s
+ LOW "Part 5: Bane" declared 2822s served 2823s last cue 2807s
+
+The pattern is exact. A published transcript describes the publisher's master, which is the
+ad-free programme -- so it aligns precisely when the show does not use dynamic ad insertion, and
+is useless precisely when it does, which is the case where a transcript would be worth most. The
+declared-versus-served gap says which case an episode is in without reading the transcript at
+all, and it is the same figure the ad scan already validates itself against.
+
+A transcript refused here is not wasted: its text is the ad-free programme, and the served file
+is that programme plus advertising, so the two together say where the advertising is. That needs
+speech recognition over the served bytes to compare against, which is a separate piece of work.
+*/
+
+func TestATranscriptFitsAnEpisodeWithNoInsertedAdvertising(t *testing.T) {
+ // Wards Way: served and declared agree, so the master is what was mirrored.
+ got := Fits(1_213_400, 1_217_000, 1_216_000)
+ if !got.OK {
+ t.Errorf("refused a transcript for an episode with no insertion: %s", got.Why)
+ }
+}
+
+func TestATranscriptIsRefusedForAnEpisodeCarryingInsertedAdvertising(t *testing.T) {
+ // Darknet Diaries "Conti": 4:16 of advertising the transcript knows nothing about.
+ got := Fits(3_544_300, 3_813_000, 3_557_000)
+ if got.OK {
+ t.Error("accepted an ad-free transcript for an ad-inserted file")
+ }
+ if got.Why == "" {
+ t.Error("a refusal has to say why -- it is the signal that this show needs recognition")
+ }
+}
+
+// LOW's Part 5 ends 15 seconds before the file does, which is closing music, not an ad break.
+func TestTrailingMusicDoesNotRefuseATranscript(t *testing.T) {
+ got := Fits(2_807_300, 2_822_500, 2_822_000)
+ if !got.OK {
+ t.Errorf("refused over %s of trailing music: %s", "15s", got.Why)
+ }
+}
+
+// A transcript running past the end of the file describes a different recording -- a publisher
+// pointing at the wrong episode, or a master that was later trimmed.
+func TestATranscriptLongerThanTheFileIsRefused(t *testing.T) {
+ got := Fits(1_300_000, 1_217_000, 1_216_000)
+ if got.OK {
+ t.Error("accepted a transcript that outlasts the audio")
+ }
+}
+
+/*
+Without a declared length there is nothing to compare the file against, so the transcript itself
+has to answer for the gap. A whole ad break's worth of silence at the end of a file is not
+closing music.
+*/
+func TestWithNoDeclaredLengthTheTranscriptsOwnGapDecides(t *testing.T) {
+ if got := Fits(1_213_400, 1_217_000, 0); !got.OK {
+ t.Errorf("refused a transcript that reaches the end of the file: %s", got.Why)
+ }
+ if got := Fits(3_544_300, 3_813_000, 0); got.OK {
+ t.Error("accepted a transcript ending 4:29 before the file does")
+ }
+}
+
+func TestATranscriptWithNoCuesFitsNothing(t *testing.T) {
+ if got := Fits(0, 1_217_000, 1_216_000); got.OK {
+ t.Error("accepted an empty transcript")
+ }
+}
+
+// The published length is what Fits judges, so it has to come from the cues rather than from a
+// caller's arithmetic.
+func TestLastCueEndIsTheEndOfTheLastCue(t *testing.T) {
+ cues := []Cue{{StartMs: 0, EndMs: 1000}, {StartMs: 2000, EndMs: 3120}}
+ if got := LastCueEndMs(cues); got != 3120 {
+ t.Errorf("got %d, want 3120", got)
+ }
+ if got := LastCueEndMs(nil); got != 0 {
+ t.Errorf("got %d, want 0", got)
+ }
+}
diff --git a/server/internal/transcript/harvest.go b/server/internal/transcript/harvest.go
new file mode 100644
index 0000000..ed149e3
--- /dev/null
+++ b/server/internal/transcript/harvest.go
@@ -0,0 +1,101 @@
+package transcript
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "time"
+)
+
+// maxTranscriptBytes caps a download. The largest in this corpus is Darknet Diaries' 1784-cue
+// SRT at well under a megabyte; eight is room for an outlier and a refusal for a mistake.
+const maxTranscriptBytes = 8 << 20
+
+// Offer is a publisher's transcript and the file it would have to describe.
+type Offer struct {
+ EpisodeID string
+ URL string
+ Type string
+ // ServedMs is the mirrored file's measured length, DeclaredMs the feed's claim. Their
+ // difference is the advertising inserted into this copy, which is what decides whether the
+ // publisher's master is what was mirrored.
+ ServedMs int64
+ DeclaredMs int64
+}
+
+// Result is what came of an offer: a transcript stored, or a recorded reason it was not.
+type Result struct {
+ Stored bool
+ Cues int
+ // Note is why nothing was stored, and is empty when something was. It is a verdict about
+ // this copy of the episode, so it is recorded rather than retried -- and for an episode
+ // carrying inserted advertising it is also the signal that this is one recognition has to do.
+ Note string
+}
+
+// Harvester takes publishers' transcripts, when they describe the file the server mirrored.
+type Harvester struct {
+ Store Store
+ Client *http.Client
+}
+
+func (h Harvester) client() *http.Client {
+ if h.Client != nil {
+ return h.Client
+ }
+ return &http.Client{Timeout: 60 * time.Second}
+}
+
+// Harvest fetches an offered transcript and stores it if it describes the mirrored file.
+//
+// An error means try again -- the download failed, so nothing has been decided. A Result with a
+// Note means do not try again: the transcript arrived and was judged.
+func (h Harvester) Harvest(ctx context.Context, offer Offer) (Result, error) {
+ if offer.URL == "" {
+ return Result{}, nil
+ }
+
+ body, err := h.download(ctx, offer.URL)
+ if err != nil {
+ return Result{}, err
+ }
+
+ cues, err := Parse(offer.Type, body)
+ if err != nil {
+ // The feed said this URL is the transcript and it is not one. Nothing about fetching it
+ // again changes that, so it is a verdict.
+ return Result{Note: err.Error()}, nil
+ }
+
+ if verdict := Fits(LastCueEndMs(cues), offer.ServedMs, offer.DeclaredMs); !verdict.OK {
+ return Result{Note: verdict.Why}, nil
+ }
+
+ stored := Transcript{EpisodeID: offer.EpisodeID, Source: SourcePublished, Cues: cues}
+ if err := h.Store.Put(stored); err != nil {
+ return Result{}, err
+ }
+ return Result{Stored: true, Cues: len(cues)}, nil
+}
+
+func (h Harvester) download(ctx context.Context, url string) (string, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return "", fmt.Errorf("transcript: %w", err)
+ }
+ req.Header.Set("User-Agent", "listend/1.0")
+ resp, err := h.client().Do(req)
+ if err != nil {
+ return "", fmt.Errorf("transcript: %w", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("transcript: %s returned %s", url, resp.Status)
+ }
+ raw, err := io.ReadAll(io.LimitReader(resp.Body, maxTranscriptBytes))
+ if err != nil {
+ return "", fmt.Errorf("transcript: %w", err)
+ }
+ return string(raw), nil
+}
diff --git a/server/internal/transcript/harvest_test.go b/server/internal/transcript/harvest_test.go
new file mode 100644
index 0000000..ec55681
--- /dev/null
+++ b/server/internal/transcript/harvest_test.go
@@ -0,0 +1,145 @@
+package transcript
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+/*
+Taking a publisher's transcript, or declining to.
+
+The decision is the point. A published transcript is free and exact, and it describes the
+publisher's master -- so for an episode whose served copy carries inserted advertising it is
+minutes out of step with the audio the phone will play, and using it would put the app's
+follow-along and any break placed from it in the wrong place.
+
+A refusal is recorded rather than retried. It is a judgement about this copy of the episode, not
+a transient failure, and an episode re-fetched and re-refused on every poll would be work done
+for ever to reach the same answer.
+*/
+
+const harvestSRT = `1
+00:00:00,000 --> 00:00:03,120
+Welcome back.
+
+2
+00:20:10,000 --> 00:20:16,000
+That's all for this week.
+`
+
+func serving(t *testing.T, status int, body, contentType string) *httptest.Server {
+ t.Helper()
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if contentType != "" {
+ w.Header().Set("Content-Type", contentType)
+ }
+ w.WriteHeader(status)
+ _, _ = w.Write([]byte(body))
+ }))
+ t.Cleanup(server.Close)
+ return server
+}
+
+func TestHarvestStoresAPublishedTranscriptThatFitsTheFile(t *testing.T) {
+ server := serving(t, 200, harvestSRT, "application/srt")
+ store := Store{Dir: t.TempDir()}
+ harvester := Harvester{Store: store}
+
+ got, err := harvester.Harvest(context.Background(), Offer{
+ EpisodeID: "ep1",
+ URL: server.URL,
+ Type: "application/srt",
+ ServedMs: 1_217_000,
+ DeclaredMs: 1_216_000,
+ })
+ if err != nil {
+ t.Fatalf("Harvest: %v", err)
+ }
+ if !got.Stored || got.Cues != 2 {
+ t.Fatalf("got %+v, want it stored with 2 cues", got)
+ }
+ held, err := store.Get("ep1")
+ if err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+ if held.Source != SourcePublished {
+ t.Errorf("source = %q, want %q", held.Source, SourcePublished)
+ }
+}
+
+func TestHarvestDeclinesATranscriptForAnAdInsertedCopy(t *testing.T) {
+ server := serving(t, 200, harvestSRT, "application/srt")
+ store := Store{Dir: t.TempDir()}
+ harvester := Harvester{Store: store}
+
+ // Darknet Diaries' shape: the file is minutes longer than the feed declares.
+ got, err := harvester.Harvest(context.Background(), Offer{
+ EpisodeID: "ep1",
+ URL: server.URL,
+ Type: "application/srt",
+ ServedMs: 1_480_000,
+ DeclaredMs: 1_216_000,
+ })
+ if err != nil {
+ t.Fatalf("Harvest returned an error for a decision: %v", err)
+ }
+ if got.Stored {
+ t.Error("stored an ad-free transcript against an ad-inserted file")
+ }
+ if !strings.Contains(got.Note, "advertising") {
+ t.Errorf("note = %q, want it to say why", got.Note)
+ }
+ if store.Has("ep1") {
+ t.Error("a declined transcript was written anyway")
+ }
+}
+
+// A 404 is the publisher's file having moved, and the next poll gets the feed's new URL. That
+// is worth another attempt; a refusal is not.
+func TestAFailedDownloadIsAnErrorAndNotAVerdict(t *testing.T) {
+ server := serving(t, 404, "nope", "text/plain")
+ harvester := Harvester{Store: Store{Dir: t.TempDir()}}
+
+ got, err := harvester.Harvest(context.Background(), Offer{
+ EpisodeID: "ep1", URL: server.URL, Type: "application/srt",
+ ServedMs: 1_217_000, DeclaredMs: 1_216_000,
+ })
+ if err == nil {
+ t.Fatal("a 404 returned no error, so it would be recorded as a decision")
+ }
+ if got.Note != "" {
+ t.Errorf("note = %q, want nothing -- this is not a verdict", got.Note)
+ }
+}
+
+// A publisher that serves HTML from a URL it labelled as SRT has given us nothing usable, and
+// that is a verdict: the feed says this is the transcript, and it is not one.
+func TestSomethingThatIsNotCuesIsDeclinedNotRetried(t *testing.T) {
+ server := serving(t, 200, "<html><body>Transcript coming soon</body></html>", "text/html")
+ harvester := Harvester{Store: Store{Dir: t.TempDir()}}
+
+ got, err := harvester.Harvest(context.Background(), Offer{
+ EpisodeID: "ep1", URL: server.URL, Type: "application/srt",
+ ServedMs: 1_217_000, DeclaredMs: 1_216_000,
+ })
+ if err != nil {
+ t.Fatalf("Harvest: %v", err)
+ }
+ if got.Stored || got.Note == "" {
+ t.Errorf("got %+v, want a recorded refusal", got)
+ }
+}
+
+func TestAnEpisodeOfferingNoTranscriptIsNotFetched(t *testing.T) {
+ harvester := Harvester{Store: Store{Dir: t.TempDir()}}
+ got, err := harvester.Harvest(context.Background(), Offer{EpisodeID: "ep1", ServedMs: 1_000})
+ if err != nil {
+ t.Fatalf("Harvest: %v", err)
+ }
+ if got.Stored || got.Note != "" {
+ t.Errorf("got %+v, want nothing done and nothing said", got)
+ }
+}
diff --git a/server/internal/transcript/parse.go b/server/internal/transcript/parse.go
new file mode 100644
index 0000000..e5c39cb
--- /dev/null
+++ b/server/internal/transcript/parse.go
@@ -0,0 +1,138 @@
+// Package transcript reads an episode's words with the times they were said at, from whatever
+// shape the words arrived in.
+//
+// Two sources, and they are not interchangeable. A publisher's own transcript is exact and free
+// but describes the publisher's master -- which for a show using dynamic ad insertion is the
+// ad-free programme, minutes shorter than the copy the server mirrored. Speech recognition over
+// the mirrored bytes is slower and approximate but describes the audio the phone will actually
+// play. [Fits] is what decides whether a published transcript may be used for a given file.
+package transcript
+
+import (
+ "fmt"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+// Cue is a run of speech and when it was said, in milliseconds from the start of the file.
+type Cue struct {
+ StartMs int64 `json:"startMs"`
+ EndMs int64 `json:"endMs"`
+ Text string `json:"text"`
+}
+
+// Parse reads cues from a published transcript of the declared MIME type.
+//
+// The types in the wild disagree with each other about the same format: application/srt and
+// application/x-subrip are both SRT. text/plain, text/html and application/json carry no
+// timestamps in any publisher's usage seen here, so they are refused rather than stored -- a
+// transcript with no times can neither be seeked to nor used to place a break, and accepting
+// one would mean the episode never gets another attempt.
+func Parse(mimeType, body string) ([]Cue, error) {
+ var (
+ cues []Cue
+ err error
+ )
+ switch normaliseType(mimeType) {
+ case "application/srt", "application/x-subrip", "text/srt":
+ cues, err = ParseSRT(body)
+ case "text/vtt", "text/webvtt":
+ cues, err = ParseVTT(body)
+ default:
+ return nil, fmt.Errorf("transcript: %q carries no timestamps", mimeType)
+ }
+ if err != nil {
+ return nil, err
+ }
+ if len(cues) == 0 {
+ return nil, fmt.Errorf("transcript: %q parsed to no cues", mimeType)
+ }
+ return cues, nil
+}
+
+func normaliseType(mimeType string) string {
+ t := strings.ToLower(strings.TrimSpace(mimeType))
+ if i := strings.IndexByte(t, ';'); i >= 0 {
+ t = strings.TrimSpace(t[:i])
+ }
+ return t
+}
+
+// timing matches both shapes of cue timing line: SRT's comma and WebVTT's full stop, with the
+// hours field optional as WebVTT allows.
+var timing = regexp.MustCompile(
+ `^(?:(\d+):)?(\d{1,2}):(\d{2})[.,](\d{1,3})\s*-->\s*(?:(\d+):)?(\d{1,2}):(\d{2})[.,](\d{1,3})`,
+)
+
+// markup strips WebVTT's cue spans -- <v Speaker>, <i>, <00:00:01.000> -- which are there to be
+// displayed, not to be read as something somebody said.
+var markup = regexp.MustCompile(`</?[^>]*>`)
+
+// ParseSRT reads SubRip cues. The cue numbers are ignored: they are an index, and a publisher
+// that misnumbers them has still said when the words were spoken.
+func ParseSRT(body string) ([]Cue, error) { return parseCues(body) }
+
+// ParseVTT reads WebVTT cues, skipping the header, NOTE and STYLE blocks and cue identifiers.
+//
+// Everything that is not a timing line is either furniture or the text of the cue that a timing
+// line just opened, which is why one reader serves both formats.
+func ParseVTT(body string) ([]Cue, error) { return parseCues(body) }
+
+func parseCues(body string) ([]Cue, error) {
+ var (
+ cues []Cue
+ current *Cue
+ text []string
+ )
+ flush := func() {
+ if current == nil {
+ return
+ }
+ current.Text = strings.TrimSpace(strings.Join(text, " "))
+ if current.Text != "" {
+ cues = append(cues, *current)
+ }
+ current, text = nil, nil
+ }
+
+ for _, line := range strings.Split(strings.ReplaceAll(body, "\r\n", "\n"), "\n") {
+ trimmed := strings.TrimSpace(line)
+ if m := timing.FindStringSubmatch(trimmed); m != nil {
+ flush()
+ current = &Cue{StartMs: millis(m[1], m[2], m[3], m[4]), EndMs: millis(m[5], m[6], m[7], m[8])}
+ continue
+ }
+ if trimmed == "" {
+ flush()
+ continue
+ }
+ if current == nil {
+ // A cue number, a cue identifier, the WEBVTT header, or a NOTE or STYLE block --
+ // all of it precedes a timing line rather than following one.
+ continue
+ }
+ text = append(text, strings.TrimSpace(markup.ReplaceAllString(trimmed, "")))
+ }
+ flush()
+ return cues, nil
+}
+
+func millis(hours, minutes, seconds, fraction string) int64 {
+ h, _ := strconv.ParseInt(emptyAsZero(hours), 10, 64)
+ m, _ := strconv.ParseInt(minutes, 10, 64)
+ s, _ := strconv.ParseInt(seconds, 10, 64)
+ // A fraction of fewer than three digits is tenths or hundredths, not milliseconds.
+ f, _ := strconv.ParseInt(fraction, 10, 64)
+ for i := len(fraction); i < 3; i++ {
+ f *= 10
+ }
+ return ((h*60+m)*60+s)*1000 + f
+}
+
+func emptyAsZero(s string) string {
+ if s == "" {
+ return "0"
+ }
+ return s
+}
diff --git a/server/internal/transcript/parse_test.go b/server/internal/transcript/parse_test.go
new file mode 100644
index 0000000..4dcbc9a
--- /dev/null
+++ b/server/internal/transcript/parse_test.go
@@ -0,0 +1,158 @@
+package transcript
+
+import "testing"
+
+/*
+Reading a transcript a publisher supplied.
+
+Eleven of Peter's 44 feeds carry <podcast:transcript>, in SRT or WebVTT. Both are cue lists with
+a start, an end and some text, and the differences between them are entirely in the furniture:
+SRT numbers its cues and separates the timestamp fields with a comma, WebVTT writes a header,
+allows a cue identifier, uses a full stop, and may carry styling blocks that are not cues at all.
+
+Parsed rather than passed through because the app seeks to a cue and the detector reads the
+words, and neither wants to know which of the two shapes a given publisher chose.
+*/
+
+const sampleSRT = `1
+00:00:00,000 --> 00:00:03,120
+Welcome back to the show.
+
+2
+00:00:03,500 --> 00:00:07,000
+This episode is brought to you
+by a mattress company.
+
+3
+01:02:03,400 --> 01:02:08,000
+And we're back.
+`
+
+func TestSRTCuesCarryTheirTimesAndText(t *testing.T) {
+ got, err := ParseSRT(sampleSRT)
+ if err != nil {
+ t.Fatalf("ParseSRT: %v", err)
+ }
+ if len(got) != 3 {
+ t.Fatalf("got %d cues, want 3", len(got))
+ }
+ if got[0] != (Cue{StartMs: 0, EndMs: 3120, Text: "Welcome back to the show."}) {
+ t.Errorf("first cue = %+v", got[0])
+ }
+}
+
+// A cue's text is routinely wrapped across lines for display. The words are one sentence.
+func TestAWrappedCueIsOneRunOfText(t *testing.T) {
+ got, err := ParseSRT(sampleSRT)
+ if err != nil {
+ t.Fatalf("ParseSRT: %v", err)
+ }
+ want := "This episode is brought to you by a mattress company."
+ if got[1].Text != want {
+ t.Errorf("got %q, want %q", got[1].Text, want)
+ }
+}
+
+func TestSRTReadsHoursCorrectly(t *testing.T) {
+ got, err := ParseSRT(sampleSRT)
+ if err != nil {
+ t.Fatalf("ParseSRT: %v", err)
+ }
+ if got[2].StartMs != 3_723_400 {
+ t.Errorf("got %d ms, want 3723400 -- 01:02:03.400", got[2].StartMs)
+ }
+}
+
+const sampleVTT = `WEBVTT
+
+NOTE this file was generated by a robot
+
+STYLE
+::cue { color: peachpuff }
+
+intro-1
+00:00.000 --> 00:03.120
+Welcome back to the show.
+
+00:01:03.500 --> 00:01:07.000
+<v Speaker>And we're back.</v>
+`
+
+// WebVTT's header, notes and styling blocks are not cues, and a cue may be named.
+func TestVTTSkipsEverythingThatIsNotACue(t *testing.T) {
+ got, err := ParseVTT(sampleVTT)
+ if err != nil {
+ t.Fatalf("ParseVTT: %v", err)
+ }
+ if len(got) != 2 {
+ t.Fatalf("got %d cues, want 2: %+v", len(got), got)
+ }
+ if got[0].Text != "Welcome back to the show." {
+ t.Errorf("first cue text = %q", got[0].Text)
+ }
+}
+
+// WebVTT allows mm:ss.mmm as well as hh:mm:ss.mmm, and the app seeks to these numbers.
+func TestVTTAcceptsATimestampWithNoHours(t *testing.T) {
+ got, err := ParseVTT(sampleVTT)
+ if err != nil {
+ t.Fatalf("ParseVTT: %v", err)
+ }
+ if got[0].EndMs != 3120 {
+ t.Errorf("got %d ms, want 3120 -- 00:03.120", got[0].EndMs)
+ }
+ if got[1].StartMs != 63_500 {
+ t.Errorf("got %d ms, want 63500 -- 00:01:03.500", got[1].StartMs)
+ }
+}
+
+// Speaker tags are display markup, not words anybody said.
+func TestVTTStripsCueMarkup(t *testing.T) {
+ got, err := ParseVTT(sampleVTT)
+ if err != nil {
+ t.Fatalf("ParseVTT: %v", err)
+ }
+ if got[1].Text != "And we're back." {
+ t.Errorf("got %q", got[1].Text)
+ }
+}
+
+// A publisher's URL says which shape it is, and the types in the wild are not consistent:
+// application/srt and application/x-subrip both appear, for the same format.
+func TestParseChoosesByDeclaredType(t *testing.T) {
+ for _, mime := range []string{"application/srt", "application/x-subrip"} {
+ got, err := Parse(mime, sampleSRT)
+ if err != nil {
+ t.Fatalf("%s: %v", mime, err)
+ }
+ if len(got) != 3 {
+ t.Errorf("%s: got %d cues, want 3", mime, len(got))
+ }
+ }
+ got, err := Parse("text/vtt", sampleVTT)
+ if err != nil {
+ t.Fatalf("text/vtt: %v", err)
+ }
+ if len(got) != 2 {
+ t.Errorf("text/vtt: got %d cues, want 2", len(got))
+ }
+}
+
+// text/plain and text/html carry no timestamps, so there is nothing to seek to and nothing to
+// place an ad break at. Refusing them is the honest answer; taking them would produce a
+// transcript that silently could not do either job.
+func TestParseRefusesAFormatWithNoTimestamps(t *testing.T) {
+ for _, mime := range []string{"text/plain", "text/html", "application/json"} {
+ if _, err := Parse(mime, sampleSRT); err == nil {
+ t.Errorf("%s: parsed without error, want a refusal", mime)
+ }
+ }
+}
+
+// A truncated or empty download is not a transcript, and storing it would mean the episode
+// never gets another attempt.
+func TestParseRefusesSomethingWithNoCues(t *testing.T) {
+ if _, err := Parse("text/vtt", "WEBVTT\n\n"); err == nil {
+ t.Error("parsed an empty file without error")
+ }
+}
diff --git a/server/internal/transcript/store.go b/server/internal/transcript/store.go
new file mode 100644
index 0000000..d7e86a4
--- /dev/null
+++ b/server/internal/transcript/store.go
@@ -0,0 +1,131 @@
+package transcript
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// Source says where a transcript's words came from. The two are not interchangeable, which is
+// why every stored transcript carries it.
+type Source string
+
+const (
+ // SourcePublished is the publisher's own transcript: exact words, but of the publisher's
+ // master, so it is only used where that master is what was mirrored.
+ SourcePublished Source = "published"
+ // SourceASR is speech recognition over the mirrored bytes: approximate words, of exactly
+ // the audio the phone will play.
+ SourceASR Source = "asr"
+)
+
+// Transcript is one episode's words and when they were said.
+type Transcript struct {
+ EpisodeID string `json:"episodeId"`
+ Source Source `json:"source"`
+ // LanguageCode is empty when nobody said. Recognition reports what it detected.
+ LanguageCode string `json:"languageCode,omitempty"`
+ Cues []Cue `json:"cues"`
+}
+
+// LastEndMs is where the words stop.
+func (t Transcript) LastEndMs() int64 { return LastCueEndMs(t.Cues) }
+
+// Store holds transcripts as one file per episode, beside the audio rather than inside the
+// state file.
+//
+// state.json is the only record of where mirrored audio is and is rewritten atomically whenever
+// anything changes -- a label, a poll, a detection result. A transcript runs to well over a
+// thousand cues, so folding 342 of them into that file would mean megabytes rewritten for every
+// tap the listener makes.
+type Store struct{ Dir string }
+
+func (s Store) path(episodeID string) (string, error) {
+ // The id decides a filename, and it arrives from the state file, so it is checked rather
+ // than trusted: anything with a separator or a dot segment in it could write elsewhere.
+ if episodeID == "" ||
+ strings.ContainsAny(episodeID, `/\`) ||
+ episodeID == "." || episodeID == ".." {
+ return "", fmt.Errorf("transcript: %q is not an episode id", episodeID)
+ }
+ return filepath.Join(s.Dir, episodeID+".json"), nil
+}
+
+// Put writes an episode's transcript, replacing any earlier one.
+func (s Store) Put(t Transcript) error {
+ if len(t.Cues) == 0 {
+ return fmt.Errorf("transcript: %s has no cues", t.EpisodeID)
+ }
+ name, err := s.path(t.EpisodeID)
+ if err != nil {
+ return err
+ }
+ if err := os.MkdirAll(s.Dir, 0o755); err != nil {
+ return fmt.Errorf("transcript: %w", err)
+ }
+ raw, err := json.Marshal(t)
+ if err != nil {
+ return fmt.Errorf("transcript: %w", err)
+ }
+ // Written through a temporary file and renamed, the same way the state file is: a
+ // half-written transcript would read as a short one rather than as a failure.
+ tmp, err := os.CreateTemp(s.Dir, ".transcript-*.tmp")
+ if err != nil {
+ return fmt.Errorf("transcript: %w", err)
+ }
+ tmpName := tmp.Name()
+ defer os.Remove(tmpName)
+ if _, err := tmp.Write(raw); err != nil {
+ tmp.Close()
+ return fmt.Errorf("transcript: %w", err)
+ }
+ if err := tmp.Close(); err != nil {
+ return fmt.Errorf("transcript: %w", err)
+ }
+ if err := os.Rename(tmpName, name); err != nil {
+ return fmt.Errorf("transcript: %w", err)
+ }
+ return nil
+}
+
+// Get reads an episode's transcript.
+func (s Store) Get(episodeID string) (Transcript, error) {
+ name, err := s.path(episodeID)
+ if err != nil {
+ return Transcript{}, err
+ }
+ raw, err := os.ReadFile(name)
+ if err != nil {
+ return Transcript{}, fmt.Errorf("transcript: %w", err)
+ }
+ var t Transcript
+ if err := json.Unmarshal(raw, &t); err != nil {
+ return Transcript{}, fmt.Errorf("transcript %s: %w", episodeID, err)
+ }
+ return t, nil
+}
+
+// Has reports whether an episode's transcript is on disk.
+func (s Store) Has(episodeID string) bool {
+ name, err := s.path(episodeID)
+ if err != nil {
+ return false
+ }
+ _, err = os.Stat(name)
+ return err == nil
+}
+
+// Prune removes an episode's transcript, which is what has to happen when its audio is
+// reclaimed. Removing one that is already gone is a retry, not a failure.
+func (s Store) Prune(episodeID string) error {
+ name, err := s.path(episodeID)
+ if err != nil {
+ return err
+ }
+ if err := os.Remove(name); err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("transcript: %w", err)
+ }
+ return nil
+}
diff --git a/server/internal/transcript/store_test.go b/server/internal/transcript/store_test.go
new file mode 100644
index 0000000..5b29d14
--- /dev/null
+++ b/server/internal/transcript/store_test.go
@@ -0,0 +1,101 @@
+package transcript
+
+import (
+ "path/filepath"
+ "testing"
+)
+
+/*
+Where an episode's words live.
+
+Beside the audio rather than in state.json, which is the one file holding the only record of
+where mirrored audio is. A transcript runs to well over a thousand cues -- Darknet Diaries'
+"Conti" has 1784 -- and 342 mirrored episodes of that would turn a file that must be written
+atomically on every label into megabytes rewritten for every change.
+*/
+
+func TestATranscriptSurvivesARoundTrip(t *testing.T) {
+ dir := t.TempDir()
+ store := Store{Dir: dir}
+ want := Transcript{
+ EpisodeID: "abc123",
+ Source: SourcePublished,
+ Cues: []Cue{
+ {StartMs: 0, EndMs: 3120, Text: "Welcome back to the show."},
+ {StartMs: 3500, EndMs: 7000, Text: "This episode is brought to you by a mattress."},
+ },
+ }
+ if err := store.Put(want); err != nil {
+ t.Fatalf("Put: %v", err)
+ }
+
+ got, err := store.Get("abc123")
+ if err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+ if got.Source != SourcePublished || len(got.Cues) != 2 || got.Cues[1].Text != want.Cues[1].Text {
+ t.Errorf("got %+v", got)
+ }
+}
+
+func TestAnEpisodeWithNoTranscriptReportsThatPlainly(t *testing.T) {
+ store := Store{Dir: t.TempDir()}
+ if _, err := store.Get("nothing-here"); err == nil {
+ t.Error("read a transcript that was never written")
+ }
+ if store.Has("nothing-here") {
+ t.Error("claims to have a transcript it never wrote")
+ }
+}
+
+// An id reaches this from the state file, so it decides a path. A traversal in one would write
+// outside the data directory.
+func TestAnIdCannotEscapeTheTranscriptDirectory(t *testing.T) {
+ store := Store{Dir: t.TempDir()}
+ for _, id := range []string{"../escape", "a/b", "", "."} {
+ if err := store.Put(Transcript{EpisodeID: id, Cues: []Cue{{Text: "x"}}}); err == nil {
+ t.Errorf("accepted %q as an episode id", id)
+ }
+ }
+}
+
+// Pruning an episode's audio has to take its words with it, or the directory grows for ever
+// with transcripts of files nobody holds any more.
+func TestPruneRemovesATranscript(t *testing.T) {
+ dir := t.TempDir()
+ store := Store{Dir: dir}
+ if err := store.Put(Transcript{EpisodeID: "gone", Cues: []Cue{{Text: "x"}}}); err != nil {
+ t.Fatalf("Put: %v", err)
+ }
+ if err := store.Prune("gone"); err != nil {
+ t.Fatalf("Prune: %v", err)
+ }
+ if store.Has("gone") {
+ t.Error("the transcript outlived the episode")
+ }
+ // Pruning something already gone is what a retry looks like, not a failure.
+ if err := store.Prune("gone"); err != nil {
+ t.Errorf("Prune on a missing transcript: %v", err)
+ }
+}
+
+func TestPutRefusesATranscriptWithNoCues(t *testing.T) {
+ store := Store{Dir: t.TempDir()}
+ if err := store.Put(Transcript{EpisodeID: "empty"}); err == nil {
+ t.Error("wrote a transcript with nothing in it")
+ }
+}
+
+func TestTheFileIsNamedForTheEpisode(t *testing.T) {
+ dir := t.TempDir()
+ store := Store{Dir: dir}
+ if err := store.Put(Transcript{EpisodeID: "abc123", Cues: []Cue{{Text: "x"}}}); err != nil {
+ t.Fatalf("Put: %v", err)
+ }
+ if _, err := filepath.Glob(filepath.Join(dir, "abc123.json")); err != nil {
+ t.Fatalf("glob: %v", err)
+ }
+ if !store.Has("abc123") {
+ t.Error("Has disagrees with Put")
+ }
+}
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-14 21:33:20 +0000