From 1985b45e13d8493cb9a2104f5c69c3fdc7a9bf26 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Fri, 30 Oct 2026 07:05:09 +0000 Subject: Fetch a second copy of an episode to subtract from the first Subtraction measures itself at 92-104% against the length gap and finds every pod, but it needs a publisher's transcript, and only 18 of 336 mirrored episodes have a usable one. So the best detector in the project covers about five per cent of what Peter listens to. This is Peter's idea for the other ninety-five. Advertising is inserted per request, which this project measured the hard way: the same Radiolab episode came back 7:29 of advertising on one fetch and 5:34 on another. So two fetches of an episode agree on the programme and differ on the advertising, and subtracting one from the other says where the advertising in the other one is. Inserted needs no change at all -- it already takes two cue lists and reports what the second has that the first does not. Confirmed before building the pass: a second fetch of Tangle's "The corruption is still the story" came back at 986 seconds against the mirrored copy's 1056, with both declaring 690. Seventy seconds of different advertising, from a request made minutes apart. What it cannot see is a campaign both copies happened to carry. That is the exact opposite blind spot to the audio scan, whose entire method is finding what a batch of episodes shares -- so between them the two cover each other, which is worth more than either being better. The second copy's audio is deleted the moment it is recognised, and it is fetched into its own directory under its own id, so nothing can mistake it for the bytes the phone plays even for the moment it exists. Only episodes whose served length exceeds what the feed declares are fetched at all: without a gap nothing was inserted, and the second copy would be the same recording. Bounded to twenty minutes a cycle, abandoned after three failures in a row with the attempts given back, and ordered newest first -- the same shape as recognition, for the same reasons. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- server/internal/store/store.go | 18 +++++++++++++ server/internal/store/store_test.go | 50 +++++++++++++++++++++++++++++++++++++ server/internal/transcript/fits.go | 10 ++++++++ server/internal/transcript/store.go | 15 ++++++++++- 4 files changed, 92 insertions(+), 1 deletion(-) (limited to 'server/internal') diff --git a/server/internal/store/store.go b/server/internal/store/store.go index 7f81bd5..2a6992b 100644 --- a/server/internal/store/store.go +++ b/server/internal/store/store.go @@ -73,6 +73,14 @@ type Episode struct { // states as everywhere else: words, a reason, or nobody has looked. MasterCues int `json:"masterCues,omitempty"` MasterNote string `json:"masterNote,omitempty"` + // MasterSource says what kind of master it is: "published" for the publisher's own ad-free + // transcript, "refetch" for a recognised second fetch of the same episode. They are not + // equally good and the readout should not pretend otherwise -- a refetch carries its own + // advertising, so anything both copies happen to share is invisible to it. + MasterSource string `json:"masterSource,omitempty"` + // RefetchAttemptedAtUnix is when a second copy was last fetched for this episode, so a + // failure is not retried every cycle. Recorded only when an attempt finished. + RefetchAttemptedAtUnix int64 `json:"refetchAttemptedAtUnix,omitempty"` // TranscribeAttemptedAtUnix is when speech recognition last ran over this episode, whether // it worked or not, so a failure is not retried on every poll for ever. Recorded only when @@ -110,6 +118,16 @@ func (e *Episode) NeedsMaster() bool { e.MasterCues == 0 && e.MasterNote == "" } +// NeedsRefetch reports whether a second copy of this episode is worth fetching. +// +// Only for an episode that has its own words but no master to subtract them from, and only +// where the lengths say advertising was inserted: without a gap the two fetches would be the +// same recording and there would be nothing to find. +func (e *Episode) NeedsRefetch() bool { + return e.Mirrored() && e.HasTranscript() && e.EnclosureURL != "" && + e.MasterCues == 0 && e.MasterNote == "" && e.RefetchAttemptedAtUnix == 0 +} + // NeedsRecognising reports whether speech recognition should be run over this episode. // // Deliberately not the negation of [Episode.TranscriptExamined]. An episode whose publisher's diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go index c04dc29..9020b17 100644 --- a/server/internal/store/store_test.go +++ b/server/internal/store/store_test.go @@ -435,3 +435,53 @@ func TestAFinishedAttemptIsNotRepeated(t *testing.T) { t.Error("an episode already attempted would be attempted again") } } + +/* +Which episodes are worth fetching a second time. + +Advertising is inserted per request, so two fetches of the same episode carry different +campaigns and agree on the programme. Subtracting one from the other therefore says where the +advertising is -- the same arithmetic as a publisher's master, with a second recording standing +in for one. + +Only worth doing where there is advertising to find. An episode whose served length matches what +the feed declares has none inserted, and the second fetch would be the same recording. +*/ + +func TestAnEpisodeWithItsOwnWordsAndNoMasterIsWorthRefetching(t *testing.T) { + e := Episode{ + Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3", TranscriptCues: 400, + } + if !e.NeedsRefetch() { + t.Error("an episode with words and nothing to subtract them from was passed over") + } +} + +func TestAnEpisodeThatAlreadyHasAMasterIsNotRefetched(t *testing.T) { + e := Episode{ + Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3", TranscriptCues: 400, MasterCues: 300, + } + if e.NeedsRefetch() { + t.Error("would fetch a second copy for an episode that already has a master") + } +} + +// Nothing to subtract from until its own words exist. +func TestAnEpisodeWithNoWordsYetIsNotRefetched(t *testing.T) { + e := Episode{Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3"} + if e.NeedsRefetch() { + t.Error("would fetch a second copy before there was a first to compare it against") + } +} + +// A finished attempt is not repeated: a publisher that will not serve it again will not serve +// it again next cycle either. +func TestAFinishedRefetchIsNotRepeated(t *testing.T) { + e := Episode{ + Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3", TranscriptCues: 400, + RefetchAttemptedAtUnix: 1_700_000_000, + } + if e.NeedsRefetch() { + t.Error("would fetch a second copy again") + } +} diff --git a/server/internal/transcript/fits.go b/server/internal/transcript/fits.go index b0495d0..c22958b 100644 --- a/server/internal/transcript/fits.go +++ b/server/internal/transcript/fits.go @@ -76,6 +76,16 @@ func Fits(lastCueEndMs, servedMs, declaredMs int64) Verdict { return Verdict{OK: true} } +// HasInsertedAdvertising reports whether this copy is longer than the feed says the programme +// is, by more than the feed could be approximate about. +// +// The one fact about an episode that comes from neither a detector nor a transcript. An episode +// without it has nothing inserted to find, so there is no point fetching a second copy to +// compare against -- the two would be the same recording. +func HasInsertedAdvertising(servedMs, declaredMs int64) bool { + return declaredMs > 0 && servedMs-declaredMs > insertionMs +} + // LastCueEndMs is where a transcript stops describing the audio. func LastCueEndMs(cues []Cue) int64 { if len(cues) == 0 { diff --git a/server/internal/transcript/store.go b/server/internal/transcript/store.go index 9450b3a..50feae4 100644 --- a/server/internal/transcript/store.go +++ b/server/internal/transcript/store.go @@ -19,6 +19,15 @@ const ( // SourceASR is speech recognition over the mirrored bytes: approximate words, of exactly // the audio the phone will play. SourceASR Source = "asr" + // SourceRefetch is speech recognition over a *second* fetch of the same episode, kept as a + // master for subtraction. It is not ad-free -- it carries whatever campaign was running + // when it was fetched -- but advertising is inserted per request, so the two copies differ + // in exactly the places the advertising sits. What they share is the programme. + // + // Subtracting it therefore reports the mirrored copy's advertising minus anything both + // copies happened to carry, which is the one thing it cannot see. That is the opposite + // blind spot to the audio scan, which only finds what a batch of episodes shares. + SourceRefetch Source = "refetch" ) // Transcript is one episode's words and when they were said. @@ -110,9 +119,13 @@ func (s Store) Put(t Transcript) error { // inserted into, and subtracting it from a recognised reading of the served file is what says // where. See Inserted. func (s Store) PutMaster(t Transcript) error { + source := t.Source + if source == "" { + source = SourcePublished + } return s.Put(Transcript{ EpisodeID: t.EpisodeID + masterSuffix, - Source: SourcePublished, + Source: source, LanguageCode: t.LanguageCode, Cues: t.Cues, }) -- cgit v1.2.3