diff options
| author | Peter Stone <thepeterstone@gmail.com> | 2026-10-31 03:59:56 +0000 |
|---|---|---|
| committer | Peter Stone <thepeterstone@gmail.com> | 2026-10-31 03:59:56 +0000 |
| commit | 4d4a337bfd56ee43cc63c21e721a4baf9f6c487f (patch) | |
| tree | b67c8a45f073304f569fdc1bce221bee2def9f85 /server | |
| parent | dbcdfa0f5549984646221ec7c582fd81ea106a66 (diff) | |
Find what a show says every week, and publish it as the show's own audio
The text analogue of the audio scan, and it costs nothing: the server already
holds transcripts for eight episodes of each show, mirrored within a minute of
each other, carrying one campaign and eight different programmes. Text several
of them share is not the programme.
It reaches where subtraction cannot. Subtraction needs a second description of
one episode -- a publisher's transcript or a second fetch -- and 18 of 337
mirrored episodes have one. This needs only the episodes already held. Measured
against the length gap on the live library: Heavyweight 111%, Incels 105%, Ezra
Klein 92%, Tangle 83%, against refetching's 21% on the same show.
**It is published as the show's own audio, not as advertising, and that is a
measured limit rather than caution.** What recurs is a show's sponsor reads and
its furniture together -- its welcome, its break cue, its credits -- and I tried
to separate them and could not. Scored against the 404 recurring stretches in
episodes that also hold a publisher's master, which says which is which:
ad copy markers 5% of the advertising, 4% of the furniture
length mean 51s against 29s, overlapping throughout
position ads 62 open / 176 mid / 41 close
furniture 12 open / 61 mid / 52 close
A first classifier built on ad copy agreed with the master 33% of the time and
called 263 advertisements furniture. None of those signals separates the two.
So the detector claims only what it measured: this is text the show says every
week. That is what KindCue means, and it leaves the listener's setting in charge
-- a show's own audio is skipped by preference, advertising unconditionally, and
claiming the stronger of the two on a coin flip would be the worse mistake.
Refused when more than 35% of an episode recurs, the same share the ad scan
refuses at: a re-aired episode or a formulaic segment would otherwise be eaten
whole. Guarded by share rather than by the length gap, because the episodes this
is worth most on are the ones with no gap at all -- a show that bakes its reads
into what it publishes declares the length it serves, so the gap is blind to
every advertisement in it.
Skipped entirely for an episode that already has measured advertising. The
better answer must not be overwritten by the vaguer one.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B
Diffstat (limited to 'server')
| -rw-r--r-- | server/cmd/listend/main.go | 87 | ||||
| -rwxr-xr-x | server/furniture-probe | bin | 0 -> 4762590 bytes | |||
| -rw-r--r-- | server/internal/transcript/publish.go | 46 | ||||
| -rw-r--r-- | server/internal/transcript/publish_test.go | 69 | ||||
| -rw-r--r-- | server/internal/transcript/recurring.go | 114 | ||||
| -rw-r--r-- | server/internal/transcript/recurring_test.go | 207 |
6 files changed, 522 insertions, 1 deletions
diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index a98e01f..6cbfb65 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -40,6 +40,10 @@ import ( "github.com/terst/listen/server/internal/transcript" ) +// minRecurringCorpus is how many of a show's episodes must have words before what they share +// means anything. Three: the target and the two others a stretch has to turn up in. +const minRecurringCorpus = 3 + func main() { var ( addr = flag.String("addr", "127.0.0.1:38090", "address to listen on") @@ -378,6 +382,7 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) { log.Printf("%s: mirrored %d new episode(s)", channel.Title, fetched) } w.harvestTranscripts(ctx, show, kept) + w.findRecurring(show) w.detect(ctx, show, kept) } @@ -569,7 +574,13 @@ func (w *worker) rederiveAds() { // Publishing is decided every time, even when the candidates have not moved: what // is fit to act on is a separate judgement from what was found, and it changes on // its own. Only the file is left alone when nothing moved. - w.publishWordBreaks(e, found) + // + // Only for episodes this pass owns. An episode with no master has its breaks from + // findRecurring, and re-deriving subtraction for it would clear them -- it has + // nothing to subtract. + if e.MasterCues > 0 { + w.publishWordBreaks(e, found) + } if sameCandidates(found, held.Ads) { continue } @@ -607,6 +618,80 @@ func sameCandidates(left, right []transcript.Candidate) bool { } /* +findRecurring works out what each of a show's episodes says every week. + +Per show rather than per episode, because the comparison is against the show's other episodes. +Eight of them are mirrored within a minute of each other, so they carry one advertising campaign +and eight different programmes -- the text analogue of what makes the audio scan work, using +transcripts already on disk and costing no download and no recognition. + +Published as the show's own audio rather than as advertising. What recurs is the sponsor reads +and the furniture together, and nothing measurable separates them: ad copy turns up in 5% of the +advertising and 4% of the furniture, and length and position overlap throughout. The listener's +own setting governs a show's own audio, so claiming the weaker of the two is the safe mistake. + +Skipped for an episode that already has measured advertising. Subtraction knows which stretches +are inserted; this only knows that they repeat, and the better answer should not be overwritten +by the vaguer one. +*/ +func (w *worker) findRecurring(show *store.Show) { + type held struct { + episode *store.Episode + cues []transcript.Cue + } + var corpus []held + for _, e := range show.Episodes { + if !e.Mirrored() || !e.HasTranscript() { + continue + } + stored, err := w.transcripts.Get(e.ID) + if err != nil { + continue + } + corpus = append(corpus, held{e, stored.Cues}) + } + if len(corpus) < minRecurringCorpus { + return + } + + published, refused := 0, 0 + for i, target := range corpus { + if target.episode.MasterCues > 0 { + continue + } + others := make([][]transcript.Cue, 0, len(corpus)-1) + for j, other := range corpus { + if i != j { + others = append(others, other.cues) + } + } + found := transcript.Recurring(target.cues, others) + fit, why := transcript.PublishableRecurring(found, target.episode.DurationMs) + target.episode.WordAdsRefused = why + target.episode.WordBreaks = nil + for _, c := range fit { + target.episode.WordBreaks = append(target.episode.WordBreaks, store.Break{ + StartMs: c.StartMs, + EndMs: c.EndMs, + // What recurs is the show's own recurring audio, which is what this kind + // means. The listener decides whether to skip it. + Kind: string(skips.KindCue), + }) + } + if len(fit) > 0 { + published++ + } + if why != "" { + refused++ + } + } + if published > 0 || refused > 0 { + log.Printf("%s: recurring text -- %d episode(s) publish it, %d refused, from a corpus of %d", + show.Title, published, refused, len(corpus)) + } +} + +/* publishWordBreaks turns what the words found into stretches the phone will actually seek past. The only place anything read out of a transcript reaches playback. Everything else in this diff --git a/server/furniture-probe b/server/furniture-probe Binary files differnew file mode 100755 index 0000000..daf1701 --- /dev/null +++ b/server/furniture-probe diff --git a/server/internal/transcript/publish.go b/server/internal/transcript/publish.go index 5d6958c..670a8d5 100644 --- a/server/internal/transcript/publish.go +++ b/server/internal/transcript/publish.go @@ -59,3 +59,49 @@ func Publishable(found []Candidate, servedMs, declaredMs int64) ([]Candidate, st } return measured, "" } + +// recurringShareLimit is how much of an episode may recur before the result is refused, as a +// percentage. Thirty-five, the same share the ad scan refuses at and for the same reason: a +// re-aired episode, a two-parter or a show built around a formulaic segment would otherwise be +// eaten whole. +const recurringShareLimit = 35 + +/* +PublishableRecurring returns the recurring text fit to skip as the show's own audio. + +Separate from [Publishable] because it publishes a different thing. Subtraction measures +advertising and the phone skips advertising unconditionally; this measures only that a show says +something every week, which is its sponsor reads and its furniture together, and nothing +available separates them -- ad copy appears in 5% of the advertising and 4% of the furniture, +and length and position overlap throughout. So it is published as the show's own audio, which +the listener's own setting governs, and the stronger claim is left unmade. + +Guarded by share of the episode rather than by the length gap, because the episodes this is +worth most on are the ones with no gap at all: a show that bakes its reads into what it +publishes declares the length it serves, so the gap is blind to every advertisement in it. +*/ +func PublishableRecurring(found []Candidate, episodeMs int64) ([]Candidate, string) { + recurring := make([]Candidate, 0, len(found)) + for _, c := range found { + if len(c.Signals) == 1 && c.Signals[0] == SignalRecurring { + recurring = append(recurring, c) + } + } + if len(recurring) == 0 { + return nil, "" + } + if episodeMs <= 0 { + return nil, "the episode's length is unknown, so there is no share to judge this against" + } + + var total int64 + for _, c := range recurring { + total += c.EndMs - c.StartMs + } + if total*100 > episodeMs*recurringShareLimit { + return nil, fmt.Sprintf( + "%s of a %s episode recurs, which is too much of it to have been understood", + clock(total), clock(episodeMs)) + } + return recurring, "" +} diff --git a/server/internal/transcript/publish_test.go b/server/internal/transcript/publish_test.go index edcf9c9..8c98a8f 100644 --- a/server/internal/transcript/publish_test.go +++ b/server/internal/transcript/publish_test.go @@ -116,3 +116,72 @@ func TestNothingFoundPublishesNothingAndSaysNothing(t *testing.T) { t.Errorf("refused %q, but there was nothing to refuse", why) } } + +/* +Recurring text is published as the show's own audio, governed by the listener's preference. + +It is not advertising as far as anything here can tell -- see the measurement in +recurring_test.go -- so it is published as what it demonstrably is: text this show says every +week. The phone skips a show's own audio only if asked, and advertising always, so claiming the +weaker of the two is the safe mistake. + +Guarded by share of the episode rather than by the length gap, because the episodes this is +worth most on are the ones with no gap at all: a show that bakes its reads into what it +publishes declares the length it serves, and the gap is blind to every advertisement in it. +*/ + +func TestRecurringTextIsPublishableAsTheShowsOwn(t *testing.T) { + found := candidates(SignalRecurring, [2]int64{0, 60_000}, [2]int64{600_000, 700_000}) + + got, why := PublishableRecurring(found, 3_600_000) + if why != "" { + t.Fatalf("refused: %s", why) + } + if len(got) != 2 { + t.Errorf("got %d, want 2", len(got)) + } +} + +/* +A show that recurs through a third of itself has not been understood. + +The same share the ad scan refuses at, and for the same reason: a re-aired episode, a two-parter +or a show with a formulaic segment would otherwise be eaten whole. +*/ +func TestTooMuchOfTheEpisodeRecurringIsRefused(t *testing.T) { + found := candidates(SignalRecurring, [2]int64{0, 1_500_000}) + + got, why := PublishableRecurring(found, 3_600_000) + if len(got) != 0 { + t.Errorf("published %d covering %d%% of the episode", len(got), 1_500_000*100/3_600_000) + } + if why == "" { + t.Error("said nothing about why") + } +} + +// An episode of unknown length cannot be judged by share of itself. +func TestRecurringTextNeedsTheEpisodeLength(t *testing.T) { + found := candidates(SignalRecurring, [2]int64{0, 60_000}) + + if got, why := PublishableRecurring(found, 0); len(got) != 0 || why == "" { + t.Errorf("got %d breaks, refused %q", len(got), why) + } +} + +// Measured advertising is published by Publishable and must not come through here as the show's +// own audio, which would weaken it from always-skipped to skipped-by-preference. +func TestMeasuredAdvertisingDoesNotComeThroughAsFurniture(t *testing.T) { + found := candidates(SignalDifference, [2]int64{0, 60_000}) + + if got, _ := PublishableRecurring(found, 3_600_000); len(got) != 0 { + t.Error("measured advertising was republished as the show's own audio") + } +} + +func TestNoRecurringTextPublishesNothing(t *testing.T) { + got, why := PublishableRecurring(nil, 3_600_000) + if len(got) != 0 || why != "" { + t.Errorf("got %d breaks, refused %q", len(got), why) + } +} diff --git a/server/internal/transcript/recurring.go b/server/internal/transcript/recurring.go new file mode 100644 index 0000000..02fd68a --- /dev/null +++ b/server/internal/transcript/recurring.go @@ -0,0 +1,114 @@ +package transcript + +import "strings" + +/* +Finding what a show says in every episode. + +The text analogue of the audio scan, and it costs nothing: the server already holds transcripts +for eight episodes of each show, mirrored within a minute of each other. They carry the same +advertising campaign and different programmes, so text appearing across several of them is not +the programme. + +It reaches where subtraction cannot. Subtraction needs a second description of one episode -- +the publisher's transcript, or a second fetch weeks later -- and only 18 of 337 mirrored +episodes have one. This needs nothing but the episodes the server already holds, which is why +it is worth more than either of the other two: measured against the length gap on the live +library, Heavyweight 111%, Incels 105%, Ezra Klein 92%, Tangle 83%, against refetching's 21% on +the same show. + +What it finds is "what this show says every week", which is advertising *and* the show's own +furniture -- its welcome, its break cue, its credits. Those are not the same thing to a +listener and are not treated as the same thing here; see Furniture. +*/ + +// SignalRecurring marks a candidate found because several of a show's episodes say it. +const SignalRecurring = "recurring" + +const ( + // recurringWords is how long a run must be to count as the same text in two episodes. + // Six: long enough that an ordinary sentence of programme does not collide with another + // episode's, short enough to survive recognition hearing one word differently. + recurringWords = 6 + // minOtherEpisodes is how many other episodes must say it. Two, because two episodes of a + // show share turns of phrase, running jokes and guests' names, and one coincidence should + // not become a skip. A campaign runs across the whole batch. + minOtherEpisodes = 2 +) + +// Recurring reports the stretches of target that at least [minOtherEpisodes] of others also say. +func Recurring(target []Cue, others [][]Cue) []Candidate { + targetWords := tokenise(target) + if len(targetWords) < recurringWords || len(others) < minOtherEpisodes { + return nil + } + + elsewhere := make([]map[string]bool, 0, len(others)) + for _, other := range others { + elsewhere = append(elsewhere, runsOf(tokenise(other), recurringWords)) + } + + // A word is shared when some run covering it is said by enough other episodes. Marking + // every word of a matching run, rather than its first, is what makes the stretches join up + // into one break instead of a chain of six-word fragments. + shared := make([]bool, len(targetWords)) + for i := 0; i+recurringWords <= len(targetWords); i++ { + run := joinWords(targetWords[i : i+recurringWords]) + said := 0 + for _, other := range elsewhere { + if other[run] { + said++ + } + } + if said < minOtherEpisodes { + continue + } + for j := i; j < i+recurringWords; j++ { + shared[j] = true + } + } + + return candidatesFrom(targetWords, shared, SignalRecurring) +} + +// candidatesFrom turns runs of marked words into candidates, dropping anything too brief to be +// a break -- a line both episodes happen to use is not something to seek past. +func candidatesFrom(words []token, marked []bool, signal string) []Candidate { + var found []Candidate + for i := 0; i < len(words); { + if !marked[i] { + i++ + continue + } + start := i + for i < len(words) && marked[i] { + i++ + } + candidate := Candidate{ + StartMs: words[start].startMs, + EndMs: words[i-1].endMs, + Signals: []string{signal}, + } + if candidate.EndMs-candidate.StartMs >= minCandidateMs { + found = append(found, candidate) + } + } + return found +} + +// runsOf indexes every run of n words. +func runsOf(words []token, n int) map[string]bool { + runs := make(map[string]bool, len(words)) + for i := 0; i+n <= len(words); i++ { + runs[joinWords(words[i:i+n])] = true + } + return runs +} + +func joinWords(words []token) string { + parts := make([]string, len(words)) + for i, w := range words { + parts[i] = w.word + } + return strings.Join(parts, " ") +} diff --git a/server/internal/transcript/recurring_test.go b/server/internal/transcript/recurring_test.go new file mode 100644 index 0000000..0de5086 --- /dev/null +++ b/server/internal/transcript/recurring_test.go @@ -0,0 +1,207 @@ +package transcript + +import "testing" + +/* +Finding what a show says in every episode. + +The text analogue of the audio scan, and it costs nothing: the server already holds transcripts +for eight episodes of each show, mirrored within a minute of each other. They carry the same +advertising campaign and different programmes, so text appearing across several of them is not +the programme. + +It reaches where subtraction cannot. Subtraction needs a second description of one episode -- a +publisher's transcript or a second fetch -- and only 18 of 337 mirrored episodes have one. This +needs nothing but the episodes themselves. + +What it finds is "what this show says every week", which is advertising *and* the show's own +furniture: its welcome, its break cue, its credits. Telling those apart is [Recurring]'s +caller's problem, not its own -- see Furniture. +*/ + +/* +show builds a programme whose every line is unlike every other show's. + +Each word carries the prefix, so no run of six is ever shared by accident. The first version of +this helper gave every line the same tail -- "which continues for a while with its own +particular words" -- and the detector dutifully found it in all three episodes, which is correct +behaviour against an incorrect fixture. +*/ +func show(prefix string, n int) []string { + out := make([]string, 0, n) + for i := 0; i < n; i++ { + line := prefix + string(rune('a'+i)) + words := make([]string, 0, 12) + for w := 0; w < 12; w++ { + words = append(words, line+string(rune('0'+w))) + } + out = append(out, joinStrings(words)) + } + return out +} + +func joinStrings(parts []string) string { + out := "" + for i, p := range parts { + if i > 0 { + out += " " + } + out += p + } + return out +} + +var sponsorRead = []string{ + "This message comes from a mattress company who make mattresses for sleeping", + "Go to mattress dot com slash listen and use the promotional code today", + "Visit them now because terms and conditions will of course always apply", + "They have been making those mattresses for forty long years and counting", + "Every mattress is rolled up into a box and delivered to your door", +} + +func TestFindsWhatSeveralEpisodesShare(t *testing.T) { + target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3))...) + others := [][]Cue{ + words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3))...), + words(0, lines(show("gamma", 3), sponsorRead, show("gamma2", 3))...), + } + + got := Recurring(target, others) + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(got), got) + } + if got[0].StartMs != 3*cueMs || got[0].EndMs != 8*cueMs { + t.Errorf("got %d-%d, want the five shared lines", got[0].StartMs, got[0].EndMs) + } +} + +/* +One other episode is not a pattern. + +Two episodes of a show can share a turn of phrase, a guest's name, a running joke. A campaign +runs across the batch, so it turns up in several -- and requiring several is what keeps a +coincidence from becoming a skip. +*/ +func TestSomethingInOnlyOneOtherEpisodeIsNotRecurring(t *testing.T) { + target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3))...) + others := [][]Cue{ + words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3))...), + words(0, lines(show("gamma", 3), show("gamma2", 3))...), + } + + if got := Recurring(target, others); len(got) != 0 { + t.Errorf("got %+v, want nothing -- one other episode is a coincidence", got) + } +} + +// The programme is what differs, and none of it may be reported. +func TestTheProgrammeIsNotRecurring(t *testing.T) { + target := words(0, show("alpha", 8)...) + others := [][]Cue{ + words(0, show("beta", 8)...), + words(0, show("gamma", 8)...), + } + + if got := Recurring(target, others); len(got) != 0 { + t.Errorf("got %+v, want nothing", got) + } +} + +// A phrase both episodes happen to use is not a break, and skipping one would clip the show. +func TestAPassingPhraseIsTooShortToBeABreak(t *testing.T) { + shared := []string{"and we will be right back after this"} + target := words(0, lines(show("alpha", 4), shared, show("alpha2", 4))...) + others := [][]Cue{ + words(0, lines(show("beta", 4), shared, show("beta2", 4))...), + words(0, lines(show("gamma", 4), shared, show("gamma2", 4))...), + } + + if got := Recurring(target, others); len(got) != 0 { + t.Errorf("got %+v, want nothing -- one line is not a break", got) + } +} + +func TestTwoRecurringStretchesStaySeparate(t *testing.T) { + second := []string{ + "Support for this programme comes from a razor company with five sharp blades", + "Visit razors dot com slash listen today where terms and conditions apply", + "They will send you a handle and a year of replacement cartridges", + "And a further word from a company that sells boxes for other things", + "Those boxes are cardboard and they fold flat which is awfully convenient", + } + target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3), second)...) + others := [][]Cue{ + words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3), second)...), + words(0, lines(show("gamma", 3), sponsorRead, show("gamma2", 3), second)...), + } + + got := Recurring(target, others) + if len(got) != 2 { + t.Fatalf("got %d candidates, want 2: %+v", len(got), got) + } + if got[0].EndMs >= got[1].StartMs { + t.Errorf("the two overlap: %+v", got) + } +} + +func TestNothingToCompareAgainstFindsNothing(t *testing.T) { + target := words(0, show("alpha", 8)...) + if got := Recurring(target, nil); len(got) != 0 { + t.Errorf("got %+v", got) + } + if got := Recurring(nil, [][]Cue{words(0, show("beta", 8)...)}); len(got) != 0 { + t.Errorf("got %+v", got) + } +} + +// Same shape as every other candidate, so one episode's breaks can come from either detector +// and be read the same way. +func TestACandidateSaysItRecurs(t *testing.T) { + target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3))...) + others := [][]Cue{ + words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3))...), + words(0, lines(show("gamma", 3), sponsorRead, show("gamma2", 3))...), + } + + got := Recurring(target, others) + if len(got) != 1 || len(got[0].Signals) != 1 || got[0].Signals[0] != SignalRecurring { + t.Errorf("got %+v, want one marked %q", got, SignalRecurring) + } +} + +/* +Recurring text is published as the show's own audio, never as advertising. + +An honest limit, measured rather than assumed. A show's recurring text is both its sponsor reads +and its furniture -- the welcome, the break cue, the credits -- and telling those apart would +need a signal that separates them. None does. Scored against the 404 recurring stretches in +episodes that also hold a publisher's master, which says which is which: + + ad copy markers 5% of the advertising, 4% of the furniture + length mean 51s against 29s, overlapping throughout + position ads 62 open / 176 mid / 41 close + furniture 12 open / 61 mid / 52 close + +A first attempt using ad copy agreed with the master 33% of the time, and called 263 +advertisements furniture. + +So the detector claims only what it measured: this is text the show says every week. That is +exactly what KindCue means, and it leaves the listener's own setting in charge -- furniture is +skipped by preference, advertising unconditionally, and claiming the stronger of the two on a +coin flip would be the worse mistake. +*/ +func TestRecurringTextIsTheShowsOwnAudioAndNotAdvertising(t *testing.T) { + target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3))...) + others := [][]Cue{ + words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3))...), + words(0, lines(show("gamma", 3), sponsorRead, show("gamma2", 3))...), + } + + got := Recurring(target, others) + if len(got) != 1 { + t.Fatalf("got %+v", got) + } + if got[0].Signals[0] == SignalDifference { + t.Error("recurring text must not claim to be measured advertising") + } +} |
