From f2b5c8f471096ce467c0b606dfaaa88ebf46648a Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Wed, 28 Oct 2026 07:50:54 +0000 Subject: Take publishers' transcripts, where they describe the copy we mirrored Eleven of the 44 subscribed feeds carry , in SRT or WebVTT. Those are the publisher's exact words with the times they were said, which is free and better than anything recognition would produce. **They are only usable when the show does not insert advertising**, and that turns out to be exact rather than approximate. A published transcript describes the publisher's master. Measured against the live library on 2026-10-28: Darknet Diaries "Conti" declared 3557s served 3813s last cue 3544s Wards Way "A Family Bar" declared 1216s served 1217s last cue 1213s LOW "Afterword" declared 610s served 610s last cue 603s LOW "Part 5: Bane" declared 2822s served 2823s last cue 2807s The transcript tracks the *declared* length every time. So it aligns precisely when served and declared agree, and is minutes out of step precisely when they do not -- which is the case where a transcript would be worth most. Reading along with one of those would put every line in the wrong place, and a break placed from it would seek into the programme. transcript.Fits makes that call from the two lengths rather than from the words, because the lengths are independent of the transcript: the feed declares the ad-free programme and the file measures what was served, so their difference is this copy's advertising. It is the same figure and the same fifteen-second unit the ad scan already validates itself against. A refusal is recorded, not retried. It is a judgement about this copy, so re-fetching every poll would be work done for ever to reach the same answer -- and the refusal is the record of which episodes speech recognition will have to do. Refused, taken and never-looked-at are three states, kept apart here and in the API, because conflating the first and third is the readout mistake this project has already made twice. Run against the live library before committing: 342 mirrored episodes, 57 offered a timestamped transcript, 38 taken, 19 declined -- and every decline is a show with dynamic insertion (Heavyweight 10:03, Incels 11:54, Darknet Diaries 4:23). Transcripts live one file per episode beside the audio, not in state.json: that file is the only record of where mirrored audio is and is rewritten whole on every label and every poll, and Darknet Diaries' "Conti" alone is 1784 cues. They are pruned with the audio they describe. Both spellings of the Podcasting 2.0 namespace are read. Every feed here declares https:// today, but the specification was published under http:// and Go matches a namespace by exact string -- so the wrong one would mean episodes silently never getting words, with nothing failing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- server/cmd/listend/main.go | 76 +++++++++++- server/internal/api/api.go | 52 +++++++++ server/internal/api/api_test.go | 3 +- server/internal/api/transcript_test.go | 182 +++++++++++++++++++++++++++++ server/internal/feed/feed.go | 58 +++++++++ server/internal/feed/feed_test.go | 125 ++++++++++++++++++++ server/internal/store/store.go | 34 ++++++ server/internal/store/store_test.go | 68 +++++++++++ server/internal/transcript/fits.go | 73 ++++++++++++ server/internal/transcript/fits_test.go | 94 +++++++++++++++ server/internal/transcript/harvest.go | 101 ++++++++++++++++ server/internal/transcript/harvest_test.go | 145 +++++++++++++++++++++++ server/internal/transcript/parse.go | 138 ++++++++++++++++++++++ server/internal/transcript/parse_test.go | 158 +++++++++++++++++++++++++ server/internal/transcript/store.go | 131 +++++++++++++++++++++ server/internal/transcript/store_test.go | 101 ++++++++++++++++ 16 files changed, 1534 insertions(+), 5 deletions(-) create mode 100644 server/internal/api/transcript_test.go create mode 100644 server/internal/transcript/fits.go create mode 100644 server/internal/transcript/fits_test.go create mode 100644 server/internal/transcript/harvest.go create mode 100644 server/internal/transcript/harvest_test.go create mode 100644 server/internal/transcript/parse.go create mode 100644 server/internal/transcript/parse_test.go create mode 100644 server/internal/transcript/store.go create mode 100644 server/internal/transcript/store_test.go (limited to 'server') diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index a4400d2..3ca2727 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -34,6 +34,7 @@ import ( "github.com/terst/listen/server/internal/signature" "github.com/terst/listen/server/internal/skips" "github.com/terst/listen/server/internal/store" + "github.com/terst/listen/server/internal/transcript" ) func main() { @@ -69,6 +70,8 @@ func main() { } decoder := decode.Decoder{} + transcriptDir := filepath.Join(*dataDir, "transcripts") + transcripts := transcript.Store{Dir: transcriptDir} worker := &worker{ state: state, statePath: statePath, @@ -77,13 +80,16 @@ func main() { keep: *keep, mirror: &mirror.Mirror{AudioDir: audioDir, Probe: decoder}, decoder: decoder, + transcripts: transcripts, + harvester: transcript.Harvester{Store: transcripts}, requests: make(chan request, 64), wakeups: make(chan struct{}, 1), } server := api.New(state, worker, api.Options{ - Token: token, - AudioDir: audioDir, + Token: token, + AudioDir: audioDir, + TranscriptDir: transcriptDir, // Persist the list the moment it arrives, then poll. // // Both halves were learned the hard way. The list is the one thing the server cannot @@ -156,8 +162,12 @@ type worker struct { keep int mirror *mirror.Mirror decoder decode.Decoder - requests chan request - wakeups chan struct{} + // transcripts holds each episode's words, and harvester takes the publishers' own where + // they describe the copy that was mirrored. See transcript.Fits for when they do not. + transcripts transcript.Store + harvester transcript.Harvester + requests chan request + wakeups chan struct{} } // wake asks for a poll now. Coalescing, so a burst of subscription updates is one poll. @@ -291,6 +301,14 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) { if e.Mirrored() { _ = w.mirror.Prune(e) } + // The words go with the audio. A transcript of a file nobody holds any more is of no + // use to the phone, which plays the server's bytes, and the directory would otherwise + // grow for ever. + if e.TranscriptExamined() { + _ = w.transcripts.Prune(e.ID) + e.TranscriptSource, e.TranscriptCues, e.TranscriptNote = "", 0, "" + e.TranscriptAtUnix = 0 + } } fetched := 0 @@ -311,9 +329,59 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) { if fetched > 0 { log.Printf("%s: mirrored %d new episode(s)", channel.Title, fetched) } + w.harvestTranscripts(ctx, show, kept) w.detect(ctx, show, kept) } +// harvestTranscripts takes the publishers' own transcripts for a show's mirrored episodes. +// +// Only where the publisher's master is what was mirrored, which transcript.Fits decides from the +// declared-versus-served length gap rather than from the words. A show using dynamic ad +// insertion publishes a transcript of the ad-free programme, and reading along with it would be +// minutes out of step with the audio the phone plays. +// +// Each episode is examined once. A refusal is a judgement about this copy, so re-fetching it on +// every poll would be work done for ever to reach the same answer -- and the refusal is itself +// the record of which episodes speech recognition would have to do. +func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candidates []*store.Episode) { + taken, declined := 0, 0 + for _, e := range candidates { + if ctx.Err() != nil { + return + } + if !e.Mirrored() || e.TranscriptExamined() || e.TranscriptURL == "" { + continue + } + result, err := w.harvester.Harvest(ctx, transcript.Offer{ + EpisodeID: e.ID, + URL: e.TranscriptURL, + Type: e.TranscriptType, + ServedMs: e.DurationMs, + DeclaredMs: e.DeclaredDurationMs, + }) + if err != nil { + // The download failed, so nothing has been decided and the next poll tries again. + log.Printf("%s: transcript for %q: %v", show.Title, e.Title, err) + continue + } + switch { + case result.Stored: + e.TranscriptSource = string(transcript.SourcePublished) + e.TranscriptCues = result.Cues + e.TranscriptAtUnix = time.Now().Unix() + e.TranscriptNote = "" + taken++ + case result.Note != "": + e.TranscriptNote = result.Note + e.TranscriptAtUnix = time.Now().Unix() + declined++ + } + } + if taken > 0 || declined > 0 { + log.Printf("%s: transcripts -- took %d, declined %d", show.Title, taken, declined) + } +} + // detect runs the intro/outro search over a show's mirrored episodes and records the result. // // Every mirrored episode at once, which is the whole reason this is a server: the phone could diff --git a/server/internal/api/api.go b/server/internal/api/api.go index 969e69d..bcfe6a2 100644 --- a/server/internal/api/api.go +++ b/server/internal/api/api.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/terst/listen/server/internal/store" + "github.com/terst/listen/server/internal/transcript" ) // Fetcher is asked to mirror an episode the phone wants and the server does not have. @@ -27,6 +28,10 @@ type Options struct { Token string // AudioDir is where mirrored episodes live. AudioDir string + // TranscriptDir is where episodes' words live, one file per episode. Separate from the + // state file because a transcript is a thousand cues and state.json is rewritten whole + // every time anything at all changes. + TranscriptDir string // OnSubscriptions is called when the phone sends its list, so a newly-subscribed show is // polled at once rather than at the next interval. Optional. OnSubscriptions func(feeds []string) @@ -71,6 +76,22 @@ type episodeResponse struct { // AdsRefused: matches were found but were too extensive to be advertising, so none were // published. Distinct from nothing having matched. AdsRefused bool `json:"adsRefused,omitempty"` + + // Transcript says whether this episode has words and where they came from, or why it has + // none. Absent entirely when nobody has looked yet, which is a third state again. + Transcript *transcriptNote `json:"transcript,omitempty"` +} + +// transcriptNote is the line the phone needs on every episode: is there a transcript, whose +// words are they, and if there is none, why not. +// +// The source travels with it because the two kinds are not the same thing to read along with. +// A published transcript is the publisher's exact words; a recognised one is a machine's guess +// at the same audio, and somebody reading it should know which they are looking at. +type transcriptNote struct { + Source string `json:"source,omitempty"` + Cues int `json:"cues,omitempty"` + Note string `json:"note,omitempty"` } type breakRange struct { @@ -90,6 +111,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("/api/subscriptions", s.authed(s.subscriptions)) mux.HandleFunc("/api/episode", s.authed(s.episode)) mux.HandleFunc("/api/labels", s.authed(s.labels)) + mux.HandleFunc("/api/transcript", s.authed(s.transcript)) mux.HandleFunc("/audio/", s.authed(s.audio)) return mux } @@ -190,9 +212,39 @@ func (s *Server) episode(w http.ResponseWriter, r *http.Request) { FoundAdMs: found.FoundAdMs, InsertedAdMs: found.InsertedAdMs, AdsRefused: found.AdsRefused, + Transcript: transcriptNoteFor(found), }) } +func transcriptNoteFor(e *store.Episode) *transcriptNote { + if !e.TranscriptExamined() { + return nil + } + return &transcriptNote{ + Source: e.TranscriptSource, + Cues: e.TranscriptCues, + Note: e.TranscriptNote, + } +} + +// transcript serves one episode's words, which are fetched only when somebody opens them -- +// a thousand cues is not something to attach to every episode lookup. +func (s *Server) transcript(w http.ResponseWriter, r *http.Request) { + id := r.URL.Query().Get("episode") + // Same rule as the audio route: the id decides a path, so anything with a separator or a + // dot segment in it is an attempt to read elsewhere. + if id == "" || strings.ContainsAny(id, "/\\") || strings.Contains(id, "..") { + http.NotFound(w, r) + return + } + held, err := (transcript.Store{Dir: s.opts.TranscriptDir}).Get(id) + if err != nil { + http.NotFound(w, r) + return + } + writeJSON(w, http.StatusOK, held) +} + func (s *Server) audio(w http.ResponseWriter, r *http.Request) { id := strings.TrimPrefix(r.URL.Path, "/audio/") // No separators: an id is a flat hex string, and anything else is an attempt to read diff --git a/server/internal/api/api_test.go b/server/internal/api/api_test.go index d4d3ee6..19e40db 100644 --- a/server/internal/api/api_test.go +++ b/server/internal/api/api_test.go @@ -26,7 +26,8 @@ func newServer(t *testing.T) (*Server, *store.State, *fakeFetcher, string) { audioDir := t.TempDir() st := store.New() fetcher := &fakeFetcher{} - return New(st, fetcher, Options{Token: token, AudioDir: audioDir}), st, fetcher, audioDir + opts := Options{Token: token, AudioDir: audioDir, TranscriptDir: t.TempDir()} + return New(st, fetcher, opts), st, fetcher, audioDir } func get(t *testing.T, s *Server, path string) *httptest.ResponseRecorder { diff --git a/server/internal/api/transcript_test.go b/server/internal/api/transcript_test.go new file mode 100644 index 0000000..b208f7f --- /dev/null +++ b/server/internal/api/transcript_test.go @@ -0,0 +1,182 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/terst/listen/server/internal/feed" + "github.com/terst/listen/server/internal/store" + "github.com/terst/listen/server/internal/transcript" +) + +/* +Serving an episode's words. + +Two pieces, deliberately separate. The episode response says whether there are words and where +they came from, which is a line the phone needs on every episode it asks about; the transcript +itself is a thousand cues and is fetched only when somebody opens it. + +Where they came from is not decoration. A published transcript is the publisher's exact words; +a recognised one is a machine's guess at the same audio, and reading along with it means seeing +the difference. The app says which. +*/ + +func writeAudio(t *testing.T, audioDir, name string) { + t.Helper() + if err := os.WriteFile(filepath.Join(audioDir, name), []byte("audio"), 0o644); err != nil { + t.Fatalf("write audio: %v", err) + } +} + +func recorder() *httptest.ResponseRecorder { return httptest.NewRecorder() } + +func seedTranscribedEpisode(t *testing.T, s *Server, st *store.State, audioDir string) *store.Episode { + t.Helper() + st.Merge("https://f", "A Show", []feed.Episode{{ + GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100, + }}) + e := st.Match("https://f", "g1", "", "") + e.Path = e.ID + ".mp3" + e.DurationMs = 1_217_000 + writeAudio(t, audioDir, e.Path) + + held := transcript.Store{Dir: s.opts.TranscriptDir} + if err := held.Put(transcript.Transcript{ + EpisodeID: e.ID, + Source: transcript.SourcePublished, + Cues: []transcript.Cue{ + {StartMs: 0, EndMs: 3120, Text: "Welcome back."}, + {StartMs: 3500, EndMs: 7000, Text: "This week's sponsor is a mattress."}, + }, + }); err != nil { + t.Fatalf("seed transcript: %v", err) + } + e.TranscriptSource = string(transcript.SourcePublished) + e.TranscriptCues = 2 + return e +} + +func TestTheEpisodeResponseSaysWhetherThereAreWords(t *testing.T) { + s, st, _, audioDir := newServer(t) + seedTranscribedEpisode(t, s, st, audioDir) + + rec := get(t, s, "/api/episode?feed=https://f&guid=g1") + var got struct { + Transcript *struct { + Source string `json:"source"` + Cues int `json:"cues"` + } `json:"transcript"` + } + if err := json.NewDecoder(rec.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.Transcript == nil { + t.Fatal("the episode has a transcript and the response does not mention it") + } + if got.Transcript.Source != "published" || got.Transcript.Cues != 2 { + t.Errorf("got %+v", *got.Transcript) + } +} + +// Refused is not the same as absent, and the phone shows the difference: an episode whose +// publisher's transcript does not describe the copy being played has been looked at. +func TestTheEpisodeResponseCarriesARefusal(t *testing.T) { + s, st, _, audioDir := newServer(t) + st.Merge("https://f", "A Show", []feed.Episode{{ + GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100, + }}) + e := st.Match("https://f", "g1", "", "") + e.Path = e.ID + ".mp3" + writeAudio(t, audioDir, e.Path) + e.TranscriptNote = "this copy carries 4:16 of inserted advertising" + + rec := get(t, s, "/api/episode?feed=https://f&guid=g1") + var got struct { + Transcript *struct { + Cues int `json:"cues"` + Note string `json:"note"` + } `json:"transcript"` + } + if err := json.NewDecoder(rec.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.Transcript == nil || got.Transcript.Note == "" || got.Transcript.Cues != 0 { + t.Errorf("got %+v, want a refusal with no cues", got.Transcript) + } +} + +func TestAnEpisodeNobodyHasLookedAtMentionsNoTranscript(t *testing.T) { + s, st, _, audioDir := newServer(t) + st.Merge("https://f", "A Show", []feed.Episode{{ + GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100, + }}) + e := st.Match("https://f", "g1", "", "") + e.Path = e.ID + ".mp3" + writeAudio(t, audioDir, e.Path) + + rec := get(t, s, "/api/episode?feed=https://f&guid=g1") + var got struct { + Transcript *json.RawMessage `json:"transcript"` + } + if err := json.NewDecoder(rec.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.Transcript != nil { + t.Errorf("invented a transcript field: %s", *got.Transcript) + } +} + +func TestTheTranscriptItselfIsServedByEpisodeId(t *testing.T) { + s, st, _, audioDir := newServer(t) + e := seedTranscribedEpisode(t, s, st, audioDir) + + rec := get(t, s, "/api/transcript?episode="+e.ID) + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + var got transcript.Transcript + if err := json.NewDecoder(rec.Body).Decode(&got); err != nil { + t.Fatalf("decode: %v", err) + } + if len(got.Cues) != 2 || got.Cues[1].Text != "This week's sponsor is a mattress." { + t.Errorf("got %+v", got) + } +} + +func TestAskingForATranscriptThatDoesNotExistIsNotFound(t *testing.T) { + s, st, _, audioDir := newServer(t) + st.Merge("https://f", "A Show", []feed.Episode{{ + GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100, + }}) + e := st.Match("https://f", "g1", "", "") + e.Path = e.ID + ".mp3" + writeAudio(t, audioDir, e.Path) + + if rec := get(t, s, "/api/transcript?episode="+e.ID); rec.Code != http.StatusNotFound { + t.Errorf("status %d, want 404", rec.Code) + } +} + +// Same rule as the audio route: an id decides a path, so it is checked rather than trusted. +func TestATranscriptIdCannotReachOutsideTheDirectory(t *testing.T) { + s, _, _, _ := newServer(t) + for _, id := range []string{"../state", "a/b", ""} { + if rec := get(t, s, "/api/transcript?episode="+id); rec.Code == http.StatusOK { + t.Errorf("%q: served something", id) + } + } +} + +func TestTheTranscriptRouteNeedsTheToken(t *testing.T) { + s, _, _, _ := newServer(t) + req, _ := http.NewRequest(http.MethodGet, "/api/transcript?episode=abc", nil) + rec := recorder() + s.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Errorf("status %d, want 401", rec.Code) + } +} diff --git a/server/internal/feed/feed.go b/server/internal/feed/feed.go index edc58be..493fbdd 100644 --- a/server/internal/feed/feed.go +++ b/server/internal/feed/feed.go @@ -21,6 +21,12 @@ type Episode struct { // DeclaredDurationMs is the feed's claim, which is routinely wrong -- a show using dynamic // ad insertion declares the ad-free length. Useful for ordering, not for arithmetic. DeclaredDurationMs int64 + // TranscriptURL is a Podcasting 2.0 transcript in a format that carries timestamps, or + // empty. The publisher's own words, which is exact where speech recognition is a guess -- + // but it describes the publisher's master, so whether it describes the file the server + // mirrored is a separate question. See transcript.Fits. + TranscriptURL string + TranscriptType string } // Channel is a show and its audio episodes, newest first. @@ -41,6 +47,12 @@ type rss struct { URL string `xml:"url,attr"` Type string `xml:"type,attr"` } `xml:"enclosure"` + // Both spellings of the Podcasting 2.0 namespace. The specification was published + // under http:// and the feeds in the wild now declare https://; Go matches a + // namespace by exact string, so taking only one silently drops the other's + // transcripts rather than failing. + Transcripts []transcriptTag `xml:"https://podcastindex.org/namespace/1.0 transcript"` + OldNamespace []transcriptTag `xml:"http://podcastindex.org/namespace/1.0 transcript"` } `xml:"item"` } `xml:"channel"` } @@ -64,6 +76,7 @@ func Parse(raw []byte) (*Channel, error) { if !isAudio(item.Enclosure.Type, item.Enclosure.URL) { continue } + transcriptURL, transcriptType := bestTranscript(append(item.Transcripts, item.OldNamespace...)) out.Episodes = append(out.Episodes, Episode{ GUID: strings.TrimSpace(item.GUID), Title: strings.TrimSpace(item.Title), @@ -71,6 +84,8 @@ func Parse(raw []byte) (*Channel, error) { EnclosureType: strings.TrimSpace(item.Enclosure.Type), PublishedUnix: parseDate(item.PubDate), DeclaredDurationMs: parseDuration(item.Duration), + TranscriptURL: transcriptURL, + TranscriptType: transcriptType, }) } @@ -81,6 +96,49 @@ func Parse(raw []byte) (*Channel, error) { return out, nil } +// transcriptRank orders the transcript formats a publisher may offer. An item routinely carries +// several at once, and the MIME types in the wild disagree about the same format: both +// application/srt and application/x-subrip are SRT. +// +// WebVTT first only because it is the format with a specification that says what a cue is; SRT +// is equally usable and ranks next. text/plain, text/html and application/json are absent +// deliberately -- they carry no timestamps, so a cue cannot be seeked to or placed, and a +// transcript that can do neither is worse than none at all because storing it stops the episode +// being looked at again. +var transcriptRank = map[string]int{ + "text/vtt": 1, + "text/webvtt": 1, + "application/srt": 2, + "application/x-subrip": 2, + "text/srt": 2, +} + +// transcriptTag is one element. +type transcriptTag struct { + URL string `xml:"url,attr"` + Type string `xml:"type,attr"` +} + +func bestTranscript(offered []transcriptTag) (string, string) { + bestRank := 0 + var bestURL, bestType string + for _, t := range offered { + url := strings.TrimSpace(t.URL) + mediaType := strings.ToLower(strings.TrimSpace(t.Type)) + if i := strings.IndexByte(mediaType, ';'); i >= 0 { + mediaType = strings.TrimSpace(mediaType[:i]) + } + rank, ok := transcriptRank[mediaType] + if !ok || url == "" { + continue + } + if bestRank == 0 || rank < bestRank { + bestRank, bestURL, bestType = rank, url, mediaType + } + } + return bestURL, bestType +} + func isAudio(mediaType, url string) bool { if url == "" { return false diff --git a/server/internal/feed/feed_test.go b/server/internal/feed/feed_test.go index e62558f..ce68404 100644 --- a/server/internal/feed/feed_test.go +++ b/server/internal/feed/feed_test.go @@ -143,3 +143,128 @@ func TestAFeedWithNoItemsIsNotAnError(t *testing.T) { t.Errorf("got %d episodes", len(ch.Episodes)) } } + +/* +Podcasting 2.0 transcripts. + +Eleven of Peter's 44 feeds carry , and the shapes below are the ones they +actually use (surveyed 2026-10-28). An item routinely carries the same transcript in several +formats at once, and the MIME types disagree with each other: application/srt and +application/x-subrip are both SRT, and text/plain, text/html and application/json carry no +timestamps at all. + +Only a timestamped one is worth taking: the app seeks to a cue and the detector places a break +at one, and neither can be done with a wall of text. +*/ + +const withTranscripts = ` + + + A Show + + Every Format At Once + https://example.com/1 + Fri, 03 Oct 2026 10:00:00 -0000 + + + + + + + Subrip Spelling + https://example.com/2 + Fri, 26 Sep 2026 10:00:00 -0000 + + + + + Nothing Timestamped + https://example.com/3 + Fri, 19 Sep 2026 10:00:00 -0000 + + + + + + None At All + https://example.com/4 + Fri, 12 Sep 2026 10:00:00 -0000 + + + +` + +func TestPrefersAWebVTTTranscriptWhenSeveralAreOffered(t *testing.T) { + channel, err := Parse([]byte(withTranscripts)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + got := channel.Episodes[0] + if got.TranscriptURL != "https://cdn.example/1.vtt" || got.TranscriptType != "text/vtt" { + t.Errorf("got %q (%q), want the WebVTT one", got.TranscriptURL, got.TranscriptType) + } +} + +func TestTakesSRTUnderEitherSpelling(t *testing.T) { + channel, err := Parse([]byte(withTranscripts)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + got := channel.Episodes[1] + if got.TranscriptURL != "https://cdn.example/2.srt" { + t.Errorf("got %q, want the x-subrip one", got.TranscriptURL) + } +} + +func TestIgnoresTranscriptsWithNoTimestamps(t *testing.T) { + channel, err := Parse([]byte(withTranscripts)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if got := channel.Episodes[2]; got.TranscriptURL != "" { + t.Errorf("took %q (%q), which has no times in it", got.TranscriptURL, got.TranscriptType) + } +} + +func TestAnEpisodeWithNoTranscriptSaysSo(t *testing.T) { + channel, err := Parse([]byte(withTranscripts)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if got := channel.Episodes[3]; got.TranscriptURL != "" { + t.Errorf("invented %q", got.TranscriptURL) + } +} + +/* +The namespace URI is spelled two ways in the wild. + +Every one of Peter's eleven transcript-carrying feeds declares the https:// form today +(surveyed 2026-10-28), but the Podcasting 2.0 specification was published under http:// and +older generators still emit it. Go matches a namespace by exact string, so a feed using the +other spelling would parse as having no transcripts at all -- silently, which is the failure +worth pinning: nothing errors, the episodes simply never get words. +*/ +func TestTranscriptsAreFoundUnderEitherNamespaceSpelling(t *testing.T) { + const httpNamespace = ` + + + A Show + + Older Generator + https://example.com/1 + Fri, 03 Oct 2026 10:00:00 -0000 + + + + +` + + channel, err := Parse([]byte(httpNamespace)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if got := channel.Episodes[0].TranscriptURL; got != "https://cdn.example/1.vtt" { + t.Errorf("got %q, want the transcript -- the http:// namespace was dropped", got) + } +} diff --git a/server/internal/store/store.go b/server/internal/store/store.go index 7f04fa0..26dab82 100644 --- a/server/internal/store/store.go +++ b/server/internal/store/store.go @@ -52,6 +52,36 @@ type Episode struct { // DetectedAtUnix records when detection last ran over the show this belongs to, so a // result can be told apart from never having looked. DetectedAtUnix int64 `json:"detectedAtUnix,omitempty"` + + // TranscriptURL and TranscriptType are where the publisher says its own transcript is, in + // a format carrying timestamps. Feed-owned, like the enclosure. + TranscriptURL string `json:"transcriptUrl,omitempty"` + TranscriptType string `json:"transcriptType,omitempty"` + + // TranscriptSource says where the words the server actually holds came from: "published" + // for the publisher's own, "asr" for speech recognition over the mirrored bytes. They are + // not interchangeable -- a published transcript is exact but describes the publisher's + // master, and recognition is approximate but describes the file the phone will play. + TranscriptSource string `json:"transcriptSource,omitempty"` + // TranscriptCues is how many cues the stored transcript has, so "does this episode have + // words" is answerable without opening the file they live in. + TranscriptCues int `json:"transcriptCues,omitempty"` + // TranscriptAtUnix is when the transcript landed. + TranscriptAtUnix int64 `json:"transcriptAtUnix,omitempty"` + // TranscriptNote is why a transcript that was offered was not taken -- most often that this + // copy carries inserted advertising the publisher's master does not describe. Kept because + // refused and never-looked-at are different states, and because a refusal here is the + // signal that this episode is one recognition would have to do. + TranscriptNote string `json:"transcriptNote,omitempty"` +} + +// HasTranscript reports whether the server holds words for this episode. +func (e *Episode) HasTranscript() bool { return e.TranscriptCues > 0 } + +// TranscriptExamined reports whether anything has been decided about this episode's words, so a +// refusal is not re-read as an episode nobody has looked at yet. +func (e *Episode) TranscriptExamined() bool { + return e.HasTranscript() || e.TranscriptNote != "" } // Break is one stretch of an episode to skip. @@ -275,6 +305,8 @@ func (s *State) Merge(feedURL, title string, episodes []feed.Episode) *Show { EnclosureURL: parsed.EnclosureURL, PublishedUnix: parsed.PublishedUnix, DeclaredDurationMs: parsed.DeclaredDurationMs, + TranscriptURL: parsed.TranscriptURL, + TranscriptType: parsed.TranscriptType, }) continue } @@ -286,6 +318,8 @@ func (s *State) Merge(feedURL, title string, episodes []feed.Episode) *Show { existing.PublishedUnix = parsed.PublishedUnix } existing.DeclaredDurationMs = parsed.DeclaredDurationMs + existing.TranscriptURL = parsed.TranscriptURL + existing.TranscriptType = parsed.TranscriptType } return show } diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go index 0540fbf..61cbb30 100644 --- a/server/internal/store/store_test.go +++ b/server/internal/store/store_test.go @@ -319,3 +319,71 @@ func TestWhatHasBeenExaminedSurvivesARestart(t *testing.T) { t.Fatal("a restart would redo every show") } } + +/* +The feed owns where a transcript is; the server owns whether it took it. + +Same split as every other field here, and for the same reason the duration pair exists: a +refresh must not overwrite what the server measured. A transcript that was refused because this +copy of the episode carries inserted advertising would otherwise look unexamined again on every +poll, and be re-fetched and re-refused for ever. +*/ + +func TestARefreshUpdatesWhereTheTranscriptIsWithoutForgettingTheVerdict(t *testing.T) { + state := New() + state.Merge("https://f", "A Show", []feed.Episode{{ + GUID: "g1", + Title: "One", + EnclosureURL: "https://cdn/1.mp3", + PublishedUnix: 100, + TranscriptURL: "https://cdn/1.vtt", TranscriptType: "text/vtt", + }}) + episode := state.Match("https://f", "g1", "", "") + episode.TranscriptSource = "published" + episode.TranscriptCues = 412 + episode.TranscriptAtUnix = 1_700_000_000 + + // The publisher moves the file, as publishers do. + state.Merge("https://f", "A Show", []feed.Episode{{ + GUID: "g1", + Title: "One", + EnclosureURL: "https://cdn/1.mp3", + PublishedUnix: 100, + TranscriptURL: "https://cdn/1-v2.vtt", TranscriptType: "text/vtt", + }}) + + got := state.Match("https://f", "g1", "", "") + if got.TranscriptURL != "https://cdn/1-v2.vtt" { + t.Errorf("transcript url = %q, want the feed's new one", got.TranscriptURL) + } + if got.TranscriptSource != "published" || got.TranscriptCues != 412 { + t.Errorf("a refresh forgot what the server stored: %+v", got) + } +} + +func TestAnEpisodeKnowsWhetherItHasWords(t *testing.T) { + var e Episode + if e.HasTranscript() { + t.Error("an episode with no cues claims a transcript") + } + e.TranscriptCues = 1 + if !e.HasTranscript() { + t.Error("an episode with cues denies having a transcript") + } +} + +// Refused and never-looked-at are different states, which is the readout mistake this codebase +// has now made twice. +func TestARefusedTranscriptIsNotTheSameAsNoneFound(t *testing.T) { + e := Episode{TranscriptNote: "this copy carries 4:16 of inserted advertising"} + if e.HasTranscript() { + t.Error("a refusal reads as a transcript") + } + if !e.TranscriptExamined() { + t.Error("an episode that was examined and refused reads as never examined") + } + empty := Episode{} + if empty.TranscriptExamined() { + t.Error("an episode nobody has looked at reads as examined") + } +} diff --git a/server/internal/transcript/fits.go b/server/internal/transcript/fits.go new file mode 100644 index 0000000..7597039 --- /dev/null +++ b/server/internal/transcript/fits.go @@ -0,0 +1,73 @@ +package transcript + +import "fmt" + +const ( + // insertionMs is how far a served file may exceed its declared length before the excess is + // read as inserted advertising rather than as the feed being approximate. Fifteen seconds, + // the same unit skips.Classify uses and for the same reason: advertising is sold in units of + // roughly that length, so a smaller gap cannot be a real insertion. + insertionMs = 15_000 + // tailMs is how much of a file may follow the last spoken word before the silence has to be + // explained. Closing music runs to fifteen seconds in this corpus (LOW's "Part 5: Bane" + // ends 15.2s before its file does); the shortest advertisement does not fit in sixty. + tailMs = 60_000 +) + +// Verdict is whether a published transcript may be used for a particular mirrored file, and if +// not, why not. +// +// Why, not just no. A transcript refused because the file carries inserted advertising is a +// different thing from one refused because it describes another recording: the first is an +// episode that needs speech recognition, and the second is a publisher's mistake. +type Verdict struct { + OK bool + Why string +} + +// Fits reports whether a transcript ending at lastCueEndMs describes a mirrored file of +// servedMs, given the declaredMs the feed claims. +// +// The decision is made on the lengths rather than on the words because the lengths are +// independent of the transcript: the feed declares the ad-free programme's length and the file +// measures what was served, so their difference is the advertising inserted into this copy. +// A publisher's transcript always describes the ad-free master, so it aligns exactly when that +// difference is nothing. +func Fits(lastCueEndMs, servedMs, declaredMs int64) Verdict { + if lastCueEndMs <= 0 { + return Verdict{Why: "the transcript has no cues"} + } + if lastCueEndMs > servedMs { + return Verdict{Why: fmt.Sprintf( + "the transcript runs %s past the end of the file, so it describes another recording", + clock(lastCueEndMs-servedMs))} + } + if declaredMs > 0 { + if gap := servedMs - declaredMs; gap > insertionMs { + return Verdict{Why: fmt.Sprintf( + "this copy carries %s of inserted advertising the transcript does not describe", + clock(gap))} + } + return Verdict{OK: true} + } + // No declared length to compare against, so the transcript answers for its own gap. + if gap := servedMs - lastCueEndMs; gap > tailMs { + return Verdict{Why: fmt.Sprintf( + "the transcript stops %s before the file does, and the feed declares no length", + clock(gap))} + } + return Verdict{OK: true} +} + +// LastCueEndMs is where a transcript stops describing the audio. +func LastCueEndMs(cues []Cue) int64 { + if len(cues) == 0 { + return 0 + } + return cues[len(cues)-1].EndMs +} + +func clock(ms int64) string { + s := ms / 1000 + return fmt.Sprintf("%d:%02d", s/60, s%60) +} diff --git a/server/internal/transcript/fits_test.go b/server/internal/transcript/fits_test.go new file mode 100644 index 0000000..4df5c5a --- /dev/null +++ b/server/internal/transcript/fits_test.go @@ -0,0 +1,94 @@ +package transcript + +import "testing" + +/* +Whether a publisher's transcript describes the file the server actually mirrored. + +Measured, not assumed, across the four mirrored shows in Peter's library that publish SRT or +WebVTT (2026-10-28), comparing each transcript's last cue against the declared length and the +length the mirrored file measures: + + Darknet Diaries "Conti" declared 3557s served 3813s last cue 3544s + Wards Way "A Family Bar" declared 1216s served 1217s last cue 1213s + LOW "Afterword" declared 610s served 610s last cue 603s + LOW "Part 5: Bane" declared 2822s served 2823s last cue 2807s + +The pattern is exact. A published transcript describes the publisher's master, which is the +ad-free programme -- so it aligns precisely when the show does not use dynamic ad insertion, and +is useless precisely when it does, which is the case where a transcript would be worth most. The +declared-versus-served gap says which case an episode is in without reading the transcript at +all, and it is the same figure the ad scan already validates itself against. + +A transcript refused here is not wasted: its text is the ad-free programme, and the served file +is that programme plus advertising, so the two together say where the advertising is. That needs +speech recognition over the served bytes to compare against, which is a separate piece of work. +*/ + +func TestATranscriptFitsAnEpisodeWithNoInsertedAdvertising(t *testing.T) { + // Wards Way: served and declared agree, so the master is what was mirrored. + got := Fits(1_213_400, 1_217_000, 1_216_000) + if !got.OK { + t.Errorf("refused a transcript for an episode with no insertion: %s", got.Why) + } +} + +func TestATranscriptIsRefusedForAnEpisodeCarryingInsertedAdvertising(t *testing.T) { + // Darknet Diaries "Conti": 4:16 of advertising the transcript knows nothing about. + got := Fits(3_544_300, 3_813_000, 3_557_000) + if got.OK { + t.Error("accepted an ad-free transcript for an ad-inserted file") + } + if got.Why == "" { + t.Error("a refusal has to say why -- it is the signal that this show needs recognition") + } +} + +// LOW's Part 5 ends 15 seconds before the file does, which is closing music, not an ad break. +func TestTrailingMusicDoesNotRefuseATranscript(t *testing.T) { + got := Fits(2_807_300, 2_822_500, 2_822_000) + if !got.OK { + t.Errorf("refused over %s of trailing music: %s", "15s", got.Why) + } +} + +// A transcript running past the end of the file describes a different recording -- a publisher +// pointing at the wrong episode, or a master that was later trimmed. +func TestATranscriptLongerThanTheFileIsRefused(t *testing.T) { + got := Fits(1_300_000, 1_217_000, 1_216_000) + if got.OK { + t.Error("accepted a transcript that outlasts the audio") + } +} + +/* +Without a declared length there is nothing to compare the file against, so the transcript itself +has to answer for the gap. A whole ad break's worth of silence at the end of a file is not +closing music. +*/ +func TestWithNoDeclaredLengthTheTranscriptsOwnGapDecides(t *testing.T) { + if got := Fits(1_213_400, 1_217_000, 0); !got.OK { + t.Errorf("refused a transcript that reaches the end of the file: %s", got.Why) + } + if got := Fits(3_544_300, 3_813_000, 0); got.OK { + t.Error("accepted a transcript ending 4:29 before the file does") + } +} + +func TestATranscriptWithNoCuesFitsNothing(t *testing.T) { + if got := Fits(0, 1_217_000, 1_216_000); got.OK { + t.Error("accepted an empty transcript") + } +} + +// The published length is what Fits judges, so it has to come from the cues rather than from a +// caller's arithmetic. +func TestLastCueEndIsTheEndOfTheLastCue(t *testing.T) { + cues := []Cue{{StartMs: 0, EndMs: 1000}, {StartMs: 2000, EndMs: 3120}} + if got := LastCueEndMs(cues); got != 3120 { + t.Errorf("got %d, want 3120", got) + } + if got := LastCueEndMs(nil); got != 0 { + t.Errorf("got %d, want 0", got) + } +} diff --git a/server/internal/transcript/harvest.go b/server/internal/transcript/harvest.go new file mode 100644 index 0000000..ed149e3 --- /dev/null +++ b/server/internal/transcript/harvest.go @@ -0,0 +1,101 @@ +package transcript + +import ( + "context" + "fmt" + "io" + "net/http" + "time" +) + +// maxTranscriptBytes caps a download. The largest in this corpus is Darknet Diaries' 1784-cue +// SRT at well under a megabyte; eight is room for an outlier and a refusal for a mistake. +const maxTranscriptBytes = 8 << 20 + +// Offer is a publisher's transcript and the file it would have to describe. +type Offer struct { + EpisodeID string + URL string + Type string + // ServedMs is the mirrored file's measured length, DeclaredMs the feed's claim. Their + // difference is the advertising inserted into this copy, which is what decides whether the + // publisher's master is what was mirrored. + ServedMs int64 + DeclaredMs int64 +} + +// Result is what came of an offer: a transcript stored, or a recorded reason it was not. +type Result struct { + Stored bool + Cues int + // Note is why nothing was stored, and is empty when something was. It is a verdict about + // this copy of the episode, so it is recorded rather than retried -- and for an episode + // carrying inserted advertising it is also the signal that this is one recognition has to do. + Note string +} + +// Harvester takes publishers' transcripts, when they describe the file the server mirrored. +type Harvester struct { + Store Store + Client *http.Client +} + +func (h Harvester) client() *http.Client { + if h.Client != nil { + return h.Client + } + return &http.Client{Timeout: 60 * time.Second} +} + +// Harvest fetches an offered transcript and stores it if it describes the mirrored file. +// +// An error means try again -- the download failed, so nothing has been decided. A Result with a +// Note means do not try again: the transcript arrived and was judged. +func (h Harvester) Harvest(ctx context.Context, offer Offer) (Result, error) { + if offer.URL == "" { + return Result{}, nil + } + + body, err := h.download(ctx, offer.URL) + if err != nil { + return Result{}, err + } + + cues, err := Parse(offer.Type, body) + if err != nil { + // The feed said this URL is the transcript and it is not one. Nothing about fetching it + // again changes that, so it is a verdict. + return Result{Note: err.Error()}, nil + } + + if verdict := Fits(LastCueEndMs(cues), offer.ServedMs, offer.DeclaredMs); !verdict.OK { + return Result{Note: verdict.Why}, nil + } + + stored := Transcript{EpisodeID: offer.EpisodeID, Source: SourcePublished, Cues: cues} + if err := h.Store.Put(stored); err != nil { + return Result{}, err + } + return Result{Stored: true, Cues: len(cues)}, nil +} + +func (h Harvester) download(ctx context.Context, url string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", fmt.Errorf("transcript: %w", err) + } + req.Header.Set("User-Agent", "listend/1.0") + resp, err := h.client().Do(req) + if err != nil { + return "", fmt.Errorf("transcript: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("transcript: %s returned %s", url, resp.Status) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, maxTranscriptBytes)) + if err != nil { + return "", fmt.Errorf("transcript: %w", err) + } + return string(raw), nil +} diff --git a/server/internal/transcript/harvest_test.go b/server/internal/transcript/harvest_test.go new file mode 100644 index 0000000..ec55681 --- /dev/null +++ b/server/internal/transcript/harvest_test.go @@ -0,0 +1,145 @@ +package transcript + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +/* +Taking a publisher's transcript, or declining to. + +The decision is the point. A published transcript is free and exact, and it describes the +publisher's master -- so for an episode whose served copy carries inserted advertising it is +minutes out of step with the audio the phone will play, and using it would put the app's +follow-along and any break placed from it in the wrong place. + +A refusal is recorded rather than retried. It is a judgement about this copy of the episode, not +a transient failure, and an episode re-fetched and re-refused on every poll would be work done +for ever to reach the same answer. +*/ + +const harvestSRT = `1 +00:00:00,000 --> 00:00:03,120 +Welcome back. + +2 +00:20:10,000 --> 00:20:16,000 +That's all for this week. +` + +func serving(t *testing.T, status int, body, contentType string) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if contentType != "" { + w.Header().Set("Content-Type", contentType) + } + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} + +func TestHarvestStoresAPublishedTranscriptThatFitsTheFile(t *testing.T) { + server := serving(t, 200, harvestSRT, "application/srt") + store := Store{Dir: t.TempDir()} + harvester := Harvester{Store: store} + + got, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", + URL: server.URL, + Type: "application/srt", + ServedMs: 1_217_000, + DeclaredMs: 1_216_000, + }) + if err != nil { + t.Fatalf("Harvest: %v", err) + } + if !got.Stored || got.Cues != 2 { + t.Fatalf("got %+v, want it stored with 2 cues", got) + } + held, err := store.Get("ep1") + if err != nil { + t.Fatalf("Get: %v", err) + } + if held.Source != SourcePublished { + t.Errorf("source = %q, want %q", held.Source, SourcePublished) + } +} + +func TestHarvestDeclinesATranscriptForAnAdInsertedCopy(t *testing.T) { + server := serving(t, 200, harvestSRT, "application/srt") + store := Store{Dir: t.TempDir()} + harvester := Harvester{Store: store} + + // Darknet Diaries' shape: the file is minutes longer than the feed declares. + got, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", + URL: server.URL, + Type: "application/srt", + ServedMs: 1_480_000, + DeclaredMs: 1_216_000, + }) + if err != nil { + t.Fatalf("Harvest returned an error for a decision: %v", err) + } + if got.Stored { + t.Error("stored an ad-free transcript against an ad-inserted file") + } + if !strings.Contains(got.Note, "advertising") { + t.Errorf("note = %q, want it to say why", got.Note) + } + if store.Has("ep1") { + t.Error("a declined transcript was written anyway") + } +} + +// A 404 is the publisher's file having moved, and the next poll gets the feed's new URL. That +// is worth another attempt; a refusal is not. +func TestAFailedDownloadIsAnErrorAndNotAVerdict(t *testing.T) { + server := serving(t, 404, "nope", "text/plain") + harvester := Harvester{Store: Store{Dir: t.TempDir()}} + + got, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", URL: server.URL, Type: "application/srt", + ServedMs: 1_217_000, DeclaredMs: 1_216_000, + }) + if err == nil { + t.Fatal("a 404 returned no error, so it would be recorded as a decision") + } + if got.Note != "" { + t.Errorf("note = %q, want nothing -- this is not a verdict", got.Note) + } +} + +// A publisher that serves HTML from a URL it labelled as SRT has given us nothing usable, and +// that is a verdict: the feed says this is the transcript, and it is not one. +func TestSomethingThatIsNotCuesIsDeclinedNotRetried(t *testing.T) { + server := serving(t, 200, "Transcript coming soon", "text/html") + harvester := Harvester{Store: Store{Dir: t.TempDir()}} + + got, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", URL: server.URL, Type: "application/srt", + ServedMs: 1_217_000, DeclaredMs: 1_216_000, + }) + if err != nil { + t.Fatalf("Harvest: %v", err) + } + if got.Stored || got.Note == "" { + t.Errorf("got %+v, want a recorded refusal", got) + } +} + +func TestAnEpisodeOfferingNoTranscriptIsNotFetched(t *testing.T) { + harvester := Harvester{Store: Store{Dir: t.TempDir()}} + got, err := harvester.Harvest(context.Background(), Offer{EpisodeID: "ep1", ServedMs: 1_000}) + if err != nil { + t.Fatalf("Harvest: %v", err) + } + if got.Stored || got.Note != "" { + t.Errorf("got %+v, want nothing done and nothing said", got) + } +} diff --git a/server/internal/transcript/parse.go b/server/internal/transcript/parse.go new file mode 100644 index 0000000..e5c39cb --- /dev/null +++ b/server/internal/transcript/parse.go @@ -0,0 +1,138 @@ +// Package transcript reads an episode's words with the times they were said at, from whatever +// shape the words arrived in. +// +// Two sources, and they are not interchangeable. A publisher's own transcript is exact and free +// but describes the publisher's master -- which for a show using dynamic ad insertion is the +// ad-free programme, minutes shorter than the copy the server mirrored. Speech recognition over +// the mirrored bytes is slower and approximate but describes the audio the phone will actually +// play. [Fits] is what decides whether a published transcript may be used for a given file. +package transcript + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// Cue is a run of speech and when it was said, in milliseconds from the start of the file. +type Cue struct { + StartMs int64 `json:"startMs"` + EndMs int64 `json:"endMs"` + Text string `json:"text"` +} + +// Parse reads cues from a published transcript of the declared MIME type. +// +// The types in the wild disagree with each other about the same format: application/srt and +// application/x-subrip are both SRT. text/plain, text/html and application/json carry no +// timestamps in any publisher's usage seen here, so they are refused rather than stored -- a +// transcript with no times can neither be seeked to nor used to place a break, and accepting +// one would mean the episode never gets another attempt. +func Parse(mimeType, body string) ([]Cue, error) { + var ( + cues []Cue + err error + ) + switch normaliseType(mimeType) { + case "application/srt", "application/x-subrip", "text/srt": + cues, err = ParseSRT(body) + case "text/vtt", "text/webvtt": + cues, err = ParseVTT(body) + default: + return nil, fmt.Errorf("transcript: %q carries no timestamps", mimeType) + } + if err != nil { + return nil, err + } + if len(cues) == 0 { + return nil, fmt.Errorf("transcript: %q parsed to no cues", mimeType) + } + return cues, nil +} + +func normaliseType(mimeType string) string { + t := strings.ToLower(strings.TrimSpace(mimeType)) + if i := strings.IndexByte(t, ';'); i >= 0 { + t = strings.TrimSpace(t[:i]) + } + return t +} + +// timing matches both shapes of cue timing line: SRT's comma and WebVTT's full stop, with the +// hours field optional as WebVTT allows. +var timing = regexp.MustCompile( + `^(?:(\d+):)?(\d{1,2}):(\d{2})[.,](\d{1,3})\s*-->\s*(?:(\d+):)?(\d{1,2}):(\d{2})[.,](\d{1,3})`, +) + +// markup strips WebVTT's cue spans -- , , <00:00:01.000> -- which are there to be +// displayed, not to be read as something somebody said. +var markup = regexp.MustCompile(`]*>`) + +// ParseSRT reads SubRip cues. The cue numbers are ignored: they are an index, and a publisher +// that misnumbers them has still said when the words were spoken. +func ParseSRT(body string) ([]Cue, error) { return parseCues(body) } + +// ParseVTT reads WebVTT cues, skipping the header, NOTE and STYLE blocks and cue identifiers. +// +// Everything that is not a timing line is either furniture or the text of the cue that a timing +// line just opened, which is why one reader serves both formats. +func ParseVTT(body string) ([]Cue, error) { return parseCues(body) } + +func parseCues(body string) ([]Cue, error) { + var ( + cues []Cue + current *Cue + text []string + ) + flush := func() { + if current == nil { + return + } + current.Text = strings.TrimSpace(strings.Join(text, " ")) + if current.Text != "" { + cues = append(cues, *current) + } + current, text = nil, nil + } + + for _, line := range strings.Split(strings.ReplaceAll(body, "\r\n", "\n"), "\n") { + trimmed := strings.TrimSpace(line) + if m := timing.FindStringSubmatch(trimmed); m != nil { + flush() + current = &Cue{StartMs: millis(m[1], m[2], m[3], m[4]), EndMs: millis(m[5], m[6], m[7], m[8])} + continue + } + if trimmed == "" { + flush() + continue + } + if current == nil { + // A cue number, a cue identifier, the WEBVTT header, or a NOTE or STYLE block -- + // all of it precedes a timing line rather than following one. + continue + } + text = append(text, strings.TrimSpace(markup.ReplaceAllString(trimmed, ""))) + } + flush() + return cues, nil +} + +func millis(hours, minutes, seconds, fraction string) int64 { + h, _ := strconv.ParseInt(emptyAsZero(hours), 10, 64) + m, _ := strconv.ParseInt(minutes, 10, 64) + s, _ := strconv.ParseInt(seconds, 10, 64) + // A fraction of fewer than three digits is tenths or hundredths, not milliseconds. + f, _ := strconv.ParseInt(fraction, 10, 64) + for i := len(fraction); i < 3; i++ { + f *= 10 + } + return ((h*60+m)*60+s)*1000 + f +} + +func emptyAsZero(s string) string { + if s == "" { + return "0" + } + return s +} diff --git a/server/internal/transcript/parse_test.go b/server/internal/transcript/parse_test.go new file mode 100644 index 0000000..4dcbc9a --- /dev/null +++ b/server/internal/transcript/parse_test.go @@ -0,0 +1,158 @@ +package transcript + +import "testing" + +/* +Reading a transcript a publisher supplied. + +Eleven of Peter's 44 feeds carry , in SRT or WebVTT. Both are cue lists with +a start, an end and some text, and the differences between them are entirely in the furniture: +SRT numbers its cues and separates the timestamp fields with a comma, WebVTT writes a header, +allows a cue identifier, uses a full stop, and may carry styling blocks that are not cues at all. + +Parsed rather than passed through because the app seeks to a cue and the detector reads the +words, and neither wants to know which of the two shapes a given publisher chose. +*/ + +const sampleSRT = `1 +00:00:00,000 --> 00:00:03,120 +Welcome back to the show. + +2 +00:00:03,500 --> 00:00:07,000 +This episode is brought to you +by a mattress company. + +3 +01:02:03,400 --> 01:02:08,000 +And we're back. +` + +func TestSRTCuesCarryTheirTimesAndText(t *testing.T) { + got, err := ParseSRT(sampleSRT) + if err != nil { + t.Fatalf("ParseSRT: %v", err) + } + if len(got) != 3 { + t.Fatalf("got %d cues, want 3", len(got)) + } + if got[0] != (Cue{StartMs: 0, EndMs: 3120, Text: "Welcome back to the show."}) { + t.Errorf("first cue = %+v", got[0]) + } +} + +// A cue's text is routinely wrapped across lines for display. The words are one sentence. +func TestAWrappedCueIsOneRunOfText(t *testing.T) { + got, err := ParseSRT(sampleSRT) + if err != nil { + t.Fatalf("ParseSRT: %v", err) + } + want := "This episode is brought to you by a mattress company." + if got[1].Text != want { + t.Errorf("got %q, want %q", got[1].Text, want) + } +} + +func TestSRTReadsHoursCorrectly(t *testing.T) { + got, err := ParseSRT(sampleSRT) + if err != nil { + t.Fatalf("ParseSRT: %v", err) + } + if got[2].StartMs != 3_723_400 { + t.Errorf("got %d ms, want 3723400 -- 01:02:03.400", got[2].StartMs) + } +} + +const sampleVTT = `WEBVTT + +NOTE this file was generated by a robot + +STYLE +::cue { color: peachpuff } + +intro-1 +00:00.000 --> 00:03.120 +Welcome back to the show. + +00:01:03.500 --> 00:01:07.000 +And we're back. +` + +// WebVTT's header, notes and styling blocks are not cues, and a cue may be named. +func TestVTTSkipsEverythingThatIsNotACue(t *testing.T) { + got, err := ParseVTT(sampleVTT) + if err != nil { + t.Fatalf("ParseVTT: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d cues, want 2: %+v", len(got), got) + } + if got[0].Text != "Welcome back to the show." { + t.Errorf("first cue text = %q", got[0].Text) + } +} + +// WebVTT allows mm:ss.mmm as well as hh:mm:ss.mmm, and the app seeks to these numbers. +func TestVTTAcceptsATimestampWithNoHours(t *testing.T) { + got, err := ParseVTT(sampleVTT) + if err != nil { + t.Fatalf("ParseVTT: %v", err) + } + if got[0].EndMs != 3120 { + t.Errorf("got %d ms, want 3120 -- 00:03.120", got[0].EndMs) + } + if got[1].StartMs != 63_500 { + t.Errorf("got %d ms, want 63500 -- 00:01:03.500", got[1].StartMs) + } +} + +// Speaker tags are display markup, not words anybody said. +func TestVTTStripsCueMarkup(t *testing.T) { + got, err := ParseVTT(sampleVTT) + if err != nil { + t.Fatalf("ParseVTT: %v", err) + } + if got[1].Text != "And we're back." { + t.Errorf("got %q", got[1].Text) + } +} + +// A publisher's URL says which shape it is, and the types in the wild are not consistent: +// application/srt and application/x-subrip both appear, for the same format. +func TestParseChoosesByDeclaredType(t *testing.T) { + for _, mime := range []string{"application/srt", "application/x-subrip"} { + got, err := Parse(mime, sampleSRT) + if err != nil { + t.Fatalf("%s: %v", mime, err) + } + if len(got) != 3 { + t.Errorf("%s: got %d cues, want 3", mime, len(got)) + } + } + got, err := Parse("text/vtt", sampleVTT) + if err != nil { + t.Fatalf("text/vtt: %v", err) + } + if len(got) != 2 { + t.Errorf("text/vtt: got %d cues, want 2", len(got)) + } +} + +// text/plain and text/html carry no timestamps, so there is nothing to seek to and nothing to +// place an ad break at. Refusing them is the honest answer; taking them would produce a +// transcript that silently could not do either job. +func TestParseRefusesAFormatWithNoTimestamps(t *testing.T) { + for _, mime := range []string{"text/plain", "text/html", "application/json"} { + if _, err := Parse(mime, sampleSRT); err == nil { + t.Errorf("%s: parsed without error, want a refusal", mime) + } + } +} + +// A truncated or empty download is not a transcript, and storing it would mean the episode +// never gets another attempt. +func TestParseRefusesSomethingWithNoCues(t *testing.T) { + if _, err := Parse("text/vtt", "WEBVTT\n\n"); err == nil { + t.Error("parsed an empty file without error") + } +} diff --git a/server/internal/transcript/store.go b/server/internal/transcript/store.go new file mode 100644 index 0000000..d7e86a4 --- /dev/null +++ b/server/internal/transcript/store.go @@ -0,0 +1,131 @@ +package transcript + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Source says where a transcript's words came from. The two are not interchangeable, which is +// why every stored transcript carries it. +type Source string + +const ( + // SourcePublished is the publisher's own transcript: exact words, but of the publisher's + // master, so it is only used where that master is what was mirrored. + SourcePublished Source = "published" + // SourceASR is speech recognition over the mirrored bytes: approximate words, of exactly + // the audio the phone will play. + SourceASR Source = "asr" +) + +// Transcript is one episode's words and when they were said. +type Transcript struct { + EpisodeID string `json:"episodeId"` + Source Source `json:"source"` + // LanguageCode is empty when nobody said. Recognition reports what it detected. + LanguageCode string `json:"languageCode,omitempty"` + Cues []Cue `json:"cues"` +} + +// LastEndMs is where the words stop. +func (t Transcript) LastEndMs() int64 { return LastCueEndMs(t.Cues) } + +// Store holds transcripts as one file per episode, beside the audio rather than inside the +// state file. +// +// state.json is the only record of where mirrored audio is and is rewritten atomically whenever +// anything changes -- a label, a poll, a detection result. A transcript runs to well over a +// thousand cues, so folding 342 of them into that file would mean megabytes rewritten for every +// tap the listener makes. +type Store struct{ Dir string } + +func (s Store) path(episodeID string) (string, error) { + // The id decides a filename, and it arrives from the state file, so it is checked rather + // than trusted: anything with a separator or a dot segment in it could write elsewhere. + if episodeID == "" || + strings.ContainsAny(episodeID, `/\`) || + episodeID == "." || episodeID == ".." { + return "", fmt.Errorf("transcript: %q is not an episode id", episodeID) + } + return filepath.Join(s.Dir, episodeID+".json"), nil +} + +// Put writes an episode's transcript, replacing any earlier one. +func (s Store) Put(t Transcript) error { + if len(t.Cues) == 0 { + return fmt.Errorf("transcript: %s has no cues", t.EpisodeID) + } + name, err := s.path(t.EpisodeID) + if err != nil { + return err + } + if err := os.MkdirAll(s.Dir, 0o755); err != nil { + return fmt.Errorf("transcript: %w", err) + } + raw, err := json.Marshal(t) + if err != nil { + return fmt.Errorf("transcript: %w", err) + } + // Written through a temporary file and renamed, the same way the state file is: a + // half-written transcript would read as a short one rather than as a failure. + tmp, err := os.CreateTemp(s.Dir, ".transcript-*.tmp") + if err != nil { + return fmt.Errorf("transcript: %w", err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return fmt.Errorf("transcript: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("transcript: %w", err) + } + if err := os.Rename(tmpName, name); err != nil { + return fmt.Errorf("transcript: %w", err) + } + return nil +} + +// Get reads an episode's transcript. +func (s Store) Get(episodeID string) (Transcript, error) { + name, err := s.path(episodeID) + if err != nil { + return Transcript{}, err + } + raw, err := os.ReadFile(name) + if err != nil { + return Transcript{}, fmt.Errorf("transcript: %w", err) + } + var t Transcript + if err := json.Unmarshal(raw, &t); err != nil { + return Transcript{}, fmt.Errorf("transcript %s: %w", episodeID, err) + } + return t, nil +} + +// Has reports whether an episode's transcript is on disk. +func (s Store) Has(episodeID string) bool { + name, err := s.path(episodeID) + if err != nil { + return false + } + _, err = os.Stat(name) + return err == nil +} + +// Prune removes an episode's transcript, which is what has to happen when its audio is +// reclaimed. Removing one that is already gone is a retry, not a failure. +func (s Store) Prune(episodeID string) error { + name, err := s.path(episodeID) + if err != nil { + return err + } + if err := os.Remove(name); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("transcript: %w", err) + } + return nil +} diff --git a/server/internal/transcript/store_test.go b/server/internal/transcript/store_test.go new file mode 100644 index 0000000..5b29d14 --- /dev/null +++ b/server/internal/transcript/store_test.go @@ -0,0 +1,101 @@ +package transcript + +import ( + "path/filepath" + "testing" +) + +/* +Where an episode's words live. + +Beside the audio rather than in state.json, which is the one file holding the only record of +where mirrored audio is. A transcript runs to well over a thousand cues -- Darknet Diaries' +"Conti" has 1784 -- and 342 mirrored episodes of that would turn a file that must be written +atomically on every label into megabytes rewritten for every change. +*/ + +func TestATranscriptSurvivesARoundTrip(t *testing.T) { + dir := t.TempDir() + store := Store{Dir: dir} + want := Transcript{ + EpisodeID: "abc123", + Source: SourcePublished, + Cues: []Cue{ + {StartMs: 0, EndMs: 3120, Text: "Welcome back to the show."}, + {StartMs: 3500, EndMs: 7000, Text: "This episode is brought to you by a mattress."}, + }, + } + if err := store.Put(want); err != nil { + t.Fatalf("Put: %v", err) + } + + got, err := store.Get("abc123") + if err != nil { + t.Fatalf("Get: %v", err) + } + if got.Source != SourcePublished || len(got.Cues) != 2 || got.Cues[1].Text != want.Cues[1].Text { + t.Errorf("got %+v", got) + } +} + +func TestAnEpisodeWithNoTranscriptReportsThatPlainly(t *testing.T) { + store := Store{Dir: t.TempDir()} + if _, err := store.Get("nothing-here"); err == nil { + t.Error("read a transcript that was never written") + } + if store.Has("nothing-here") { + t.Error("claims to have a transcript it never wrote") + } +} + +// An id reaches this from the state file, so it decides a path. A traversal in one would write +// outside the data directory. +func TestAnIdCannotEscapeTheTranscriptDirectory(t *testing.T) { + store := Store{Dir: t.TempDir()} + for _, id := range []string{"../escape", "a/b", "", "."} { + if err := store.Put(Transcript{EpisodeID: id, Cues: []Cue{{Text: "x"}}}); err == nil { + t.Errorf("accepted %q as an episode id", id) + } + } +} + +// Pruning an episode's audio has to take its words with it, or the directory grows for ever +// with transcripts of files nobody holds any more. +func TestPruneRemovesATranscript(t *testing.T) { + dir := t.TempDir() + store := Store{Dir: dir} + if err := store.Put(Transcript{EpisodeID: "gone", Cues: []Cue{{Text: "x"}}}); err != nil { + t.Fatalf("Put: %v", err) + } + if err := store.Prune("gone"); err != nil { + t.Fatalf("Prune: %v", err) + } + if store.Has("gone") { + t.Error("the transcript outlived the episode") + } + // Pruning something already gone is what a retry looks like, not a failure. + if err := store.Prune("gone"); err != nil { + t.Errorf("Prune on a missing transcript: %v", err) + } +} + +func TestPutRefusesATranscriptWithNoCues(t *testing.T) { + store := Store{Dir: t.TempDir()} + if err := store.Put(Transcript{EpisodeID: "empty"}); err == nil { + t.Error("wrote a transcript with nothing in it") + } +} + +func TestTheFileIsNamedForTheEpisode(t *testing.T) { + dir := t.TempDir() + store := Store{Dir: dir} + if err := store.Put(Transcript{EpisodeID: "abc123", Cues: []Cue{{Text: "x"}}}); err != nil { + t.Fatalf("Put: %v", err) + } + if _, err := filepath.Glob(filepath.Join(dir, "abc123.json")); err != nil { + t.Fatalf("glob: %v", err) + } + if !store.Has("abc123") { + t.Error("Has disagrees with Put") + } +} -- cgit v1.2.3