Find what a show says every week, and publish it as the show's own audio

The text analogue of the audio scan, and it costs nothing: the server already
holds transcripts for eight episodes of each show, mirrored within a minute of
each other, carrying one campaign and eight different programmes. Text several
of them share is not the programme.

It reaches where subtraction cannot. Subtraction needs a second description of
one episode -- a publisher's transcript or a second fetch -- and 18 of 337
mirrored episodes have one. This needs only the episodes already held. Measured
against the length gap on the live library: Heavyweight 111%, Incels 105%, Ezra
Klein 92%, Tangle 83%, against refetching's 21% on the same show.

**It is published as the show's own audio, not as advertising, and that is a
measured limit rather than caution.** What recurs is a show's sponsor reads and
its furniture together -- its welcome, its break cue, its credits -- and I tried
to separate them and could not. Scored against the 404 recurring stretches in
episodes that also hold a publisher's master, which says which is which:

    ad copy markers   5% of the advertising, 4% of the furniture
    length            mean 51s against 29s, overlapping throughout
    position          ads 62 open / 176 mid / 41 close
                      furniture 12 open / 61 mid / 52 close

A first classifier built on ad copy agreed with the master 33% of the time and
called 263 advertisements furniture. None of those signals separates the two.

So the detector claims only what it measured: this is text the show says every
week. That is what KindCue means, and it leaves the listener's setting in charge
-- a show's own audio is skipped by preference, advertising unconditionally, and
claiming the stronger of the two on a coin flip would be the worse mistake.

Refused when more than 35% of an episode recurs, the same share the ad scan
refuses at: a re-aired episode or a formulaic segment would otherwise be eaten
whole. Guarded by share rather than by the length gap, because the episodes this
is worth most on are the ones with no gap at all -- a show that bakes its reads
into what it publishes declares the length it serves, so the gap is blind to
every advertisement in it.

Skipped entirely for an episode that already has measured advertising. The
better answer must not be overwritten by the vaguer one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B

1 files changed, 86 insertions, 1 deletions

diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go
index a98e01f..6cbfb65 100644
--- a/server/cmd/listend/main.go
+++ b/server/cmd/listend/main.go
@@ -40,6 +40,10 @@ import (
 	"github.com/terst/listen/server/internal/transcript"
 )
 
+// minRecurringCorpus is how many of a show's episodes must have words before what they share
+// means anything. Three: the target and the two others a stretch has to turn up in.
+const minRecurringCorpus = 3
+
 func main() {
 	var (
 		addr      = flag.String("addr", "127.0.0.1:38090", "address to listen on")
@@ -378,6 +382,7 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) {
 		log.Printf("%s: mirrored %d new episode(s)", channel.Title, fetched)
 	}
 	w.harvestTranscripts(ctx, show, kept)
+	w.findRecurring(show)
 	w.detect(ctx, show, kept)
 }
 
@@ -569,7 +574,13 @@ func (w *worker) rederiveAds() {
 			// Publishing is decided every time, even when the candidates have not moved: what
 			// is fit to act on is a separate judgement from what was found, and it changes on
 			// its own. Only the file is left alone when nothing moved.
-			w.publishWordBreaks(e, found)
+			//
+			// Only for episodes this pass owns. An episode with no master has its breaks from
+			// findRecurring, and re-deriving subtraction for it would clear them -- it has
+			// nothing to subtract.
+			if e.MasterCues > 0 {
+				w.publishWordBreaks(e, found)
+			}
 			if sameCandidates(found, held.Ads) {
 				continue
 			}
@@ -607,6 +618,80 @@ func sameCandidates(left, right []transcript.Candidate) bool {
 }
 
 /*
+findRecurring works out what each of a show's episodes says every week.
+
+Per show rather than per episode, because the comparison is against the show's other episodes.
+Eight of them are mirrored within a minute of each other, so they carry one advertising campaign
+and eight different programmes -- the text analogue of what makes the audio scan work, using
+transcripts already on disk and costing no download and no recognition.
+
+Published as the show's own audio rather than as advertising. What recurs is the sponsor reads
+and the furniture together, and nothing measurable separates them: ad copy turns up in 5% of the
+advertising and 4% of the furniture, and length and position overlap throughout. The listener's
+own setting governs a show's own audio, so claiming the weaker of the two is the safe mistake.
+
+Skipped for an episode that already has measured advertising. Subtraction knows which stretches
+are inserted; this only knows that they repeat, and the better answer should not be overwritten
+by the vaguer one.
+*/
+func (w *worker) findRecurring(show *store.Show) {
+	type held struct {
+		episode *store.Episode
+		cues    []transcript.Cue
+	}
+	var corpus []held
+	for _, e := range show.Episodes {
+		if !e.Mirrored() || !e.HasTranscript() {
+			continue
+		}
+		stored, err := w.transcripts.Get(e.ID)
+		if err != nil {
+			continue
+		}
+		corpus = append(corpus, held{e, stored.Cues})
+	}
+	if len(corpus) < minRecurringCorpus {
+		return
+	}
+
+	published, refused := 0, 0
+	for i, target := range corpus {
+		if target.episode.MasterCues > 0 {
+			continue
+		}
+		others := make([][]transcript.Cue, 0, len(corpus)-1)
+		for j, other := range corpus {
+			if i != j {
+				others = append(others, other.cues)
+			}
+		}
+		found := transcript.Recurring(target.cues, others)
+		fit, why := transcript.PublishableRecurring(found, target.episode.DurationMs)
+		target.episode.WordAdsRefused = why
+		target.episode.WordBreaks = nil
+		for _, c := range fit {
+			target.episode.WordBreaks = append(target.episode.WordBreaks, store.Break{
+				StartMs: c.StartMs,
+				EndMs:   c.EndMs,
+				// What recurs is the show's own recurring audio, which is what this kind
+				// means. The listener decides whether to skip it.
+				Kind: string(skips.KindCue),
+			})
+		}
+		if len(fit) > 0 {
+			published++
+		}
+		if why != "" {
+			refused++
+		}
+	}
+	if published > 0 || refused > 0 {
+		log.Printf("%s: recurring text -- %d episode(s) publish it, %d refused, from a corpus of %d",
+			show.Title, published, refused, len(corpus))
+	}
+}
+
+/*
 publishWordBreaks turns what the words found into stretches the phone will actually seek past.
 
 The only place anything read out of a transcript reaches playback. Everything else in this