6 files changed, 522 insertions, 1 deletions

diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go
index a98e01f..6cbfb65 100644
--- a/server/cmd/listend/main.go
+++ b/server/cmd/listend/main.go
@@ -40,6 +40,10 @@ import (
 	"github.com/terst/listen/server/internal/transcript"
 )
 
+// minRecurringCorpus is how many of a show's episodes must have words before what they share
+// means anything. Three: the target and the two others a stretch has to turn up in.
+const minRecurringCorpus = 3
+
 func main() {
 	var (
 		addr      = flag.String("addr", "127.0.0.1:38090", "address to listen on")
@@ -378,6 +382,7 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) {
 		log.Printf("%s: mirrored %d new episode(s)", channel.Title, fetched)
 	}
 	w.harvestTranscripts(ctx, show, kept)
+	w.findRecurring(show)
 	w.detect(ctx, show, kept)
 }
 
@@ -569,7 +574,13 @@ func (w *worker) rederiveAds() {
 			// Publishing is decided every time, even when the candidates have not moved: what
 			// is fit to act on is a separate judgement from what was found, and it changes on
 			// its own. Only the file is left alone when nothing moved.
-			w.publishWordBreaks(e, found)
+			//
+			// Only for episodes this pass owns. An episode with no master has its breaks from
+			// findRecurring, and re-deriving subtraction for it would clear them -- it has
+			// nothing to subtract.
+			if e.MasterCues > 0 {
+				w.publishWordBreaks(e, found)
+			}
 			if sameCandidates(found, held.Ads) {
 				continue
 			}
@@ -607,6 +618,80 @@ func sameCandidates(left, right []transcript.Candidate) bool {
 }
 
 /*
+findRecurring works out what each of a show's episodes says every week.
+
+Per show rather than per episode, because the comparison is against the show's other episodes.
+Eight of them are mirrored within a minute of each other, so they carry one advertising campaign
+and eight different programmes -- the text analogue of what makes the audio scan work, using
+transcripts already on disk and costing no download and no recognition.
+
+Published as the show's own audio rather than as advertising. What recurs is the sponsor reads
+and the furniture together, and nothing measurable separates them: ad copy turns up in 5% of the
+advertising and 4% of the furniture, and length and position overlap throughout. The listener's
+own setting governs a show's own audio, so claiming the weaker of the two is the safe mistake.
+
+Skipped for an episode that already has measured advertising. Subtraction knows which stretches
+are inserted; this only knows that they repeat, and the better answer should not be overwritten
+by the vaguer one.
+*/
+func (w *worker) findRecurring(show *store.Show) {
+	type held struct {
+		episode *store.Episode
+		cues    []transcript.Cue
+	}
+	var corpus []held
+	for _, e := range show.Episodes {
+		if !e.Mirrored() || !e.HasTranscript() {
+			continue
+		}
+		stored, err := w.transcripts.Get(e.ID)
+		if err != nil {
+			continue
+		}
+		corpus = append(corpus, held{e, stored.Cues})
+	}
+	if len(corpus) < minRecurringCorpus {
+		return
+	}
+
+	published, refused := 0, 0
+	for i, target := range corpus {
+		if target.episode.MasterCues > 0 {
+			continue
+		}
+		others := make([][]transcript.Cue, 0, len(corpus)-1)
+		for j, other := range corpus {
+			if i != j {
+				others = append(others, other.cues)
+			}
+		}
+		found := transcript.Recurring(target.cues, others)
+		fit, why := transcript.PublishableRecurring(found, target.episode.DurationMs)
+		target.episode.WordAdsRefused = why
+		target.episode.WordBreaks = nil
+		for _, c := range fit {
+			target.episode.WordBreaks = append(target.episode.WordBreaks, store.Break{
+				StartMs: c.StartMs,
+				EndMs:   c.EndMs,
+				// What recurs is the show's own recurring audio, which is what this kind
+				// means. The listener decides whether to skip it.
+				Kind: string(skips.KindCue),
+			})
+		}
+		if len(fit) > 0 {
+			published++
+		}
+		if why != "" {
+			refused++
+		}
+	}
+	if published > 0 || refused > 0 {
+		log.Printf("%s: recurring text -- %d episode(s) publish it, %d refused, from a corpus of %d",
+			show.Title, published, refused, len(corpus))
+	}
+}
+
+/*
 publishWordBreaks turns what the words found into stretches the phone will actually seek past.
 
 The only place anything read out of a transcript reaches playback. Everything else in this
diff --git a/server/furniture-probe b/server/furniture-probe
new file mode 100755
index 0000000..daf1701
--- /dev/null
+++ b/server/furniture-probe
diff --git a/server/internal/transcript/publish.go b/server/internal/transcript/publish.go
index 5d6958c..670a8d5 100644
--- a/server/internal/transcript/publish.go
+++ b/server/internal/transcript/publish.go
@@ -59,3 +59,49 @@ func Publishable(found []Candidate, servedMs, declaredMs int64) ([]Candidate, st
 	}
 	return measured, ""
 }
+
+// recurringShareLimit is how much of an episode may recur before the result is refused, as a
+// percentage. Thirty-five, the same share the ad scan refuses at and for the same reason: a
+// re-aired episode, a two-parter or a show built around a formulaic segment would otherwise be
+// eaten whole.
+const recurringShareLimit = 35
+
+/*
+PublishableRecurring returns the recurring text fit to skip as the show's own audio.
+
+Separate from [Publishable] because it publishes a different thing. Subtraction measures
+advertising and the phone skips advertising unconditionally; this measures only that a show says
+something every week, which is its sponsor reads and its furniture together, and nothing
+available separates them -- ad copy appears in 5% of the advertising and 4% of the furniture,
+and length and position overlap throughout. So it is published as the show's own audio, which
+the listener's own setting governs, and the stronger claim is left unmade.
+
+Guarded by share of the episode rather than by the length gap, because the episodes this is
+worth most on are the ones with no gap at all: a show that bakes its reads into what it
+publishes declares the length it serves, so the gap is blind to every advertisement in it.
+*/
+func PublishableRecurring(found []Candidate, episodeMs int64) ([]Candidate, string) {
+	recurring := make([]Candidate, 0, len(found))
+	for _, c := range found {
+		if len(c.Signals) == 1 && c.Signals[0] == SignalRecurring {
+			recurring = append(recurring, c)
+		}
+	}
+	if len(recurring) == 0 {
+		return nil, ""
+	}
+	if episodeMs <= 0 {
+		return nil, "the episode's length is unknown, so there is no share to judge this against"
+	}
+
+	var total int64
+	for _, c := range recurring {
+		total += c.EndMs - c.StartMs
+	}
+	if total*100 > episodeMs*recurringShareLimit {
+		return nil, fmt.Sprintf(
+			"%s of a %s episode recurs, which is too much of it to have been understood",
+			clock(total), clock(episodeMs))
+	}
+	return recurring, ""
+}
diff --git a/server/internal/transcript/publish_test.go b/server/internal/transcript/publish_test.go
index edcf9c9..8c98a8f 100644
--- a/server/internal/transcript/publish_test.go
+++ b/server/internal/transcript/publish_test.go
@@ -116,3 +116,72 @@ func TestNothingFoundPublishesNothingAndSaysNothing(t *testing.T) {
 		t.Errorf("refused %q, but there was nothing to refuse", why)
 	}
 }
+
+/*
+Recurring text is published as the show's own audio, governed by the listener's preference.
+
+It is not advertising as far as anything here can tell -- see the measurement in
+recurring_test.go -- so it is published as what it demonstrably is: text this show says every
+week. The phone skips a show's own audio only if asked, and advertising always, so claiming the
+weaker of the two is the safe mistake.
+
+Guarded by share of the episode rather than by the length gap, because the episodes this is
+worth most on are the ones with no gap at all: a show that bakes its reads into what it
+publishes declares the length it serves, and the gap is blind to every advertisement in it.
+*/
+
+func TestRecurringTextIsPublishableAsTheShowsOwn(t *testing.T) {
+	found := candidates(SignalRecurring, [2]int64{0, 60_000}, [2]int64{600_000, 700_000})
+
+	got, why := PublishableRecurring(found, 3_600_000)
+	if why != "" {
+		t.Fatalf("refused: %s", why)
+	}
+	if len(got) != 2 {
+		t.Errorf("got %d, want 2", len(got))
+	}
+}
+
+/*
+A show that recurs through a third of itself has not been understood.
+
+The same share the ad scan refuses at, and for the same reason: a re-aired episode, a two-parter
+or a show with a formulaic segment would otherwise be eaten whole.
+*/
+func TestTooMuchOfTheEpisodeRecurringIsRefused(t *testing.T) {
+	found := candidates(SignalRecurring, [2]int64{0, 1_500_000})
+
+	got, why := PublishableRecurring(found, 3_600_000)
+	if len(got) != 0 {
+		t.Errorf("published %d covering %d%% of the episode", len(got), 1_500_000*100/3_600_000)
+	}
+	if why == "" {
+		t.Error("said nothing about why")
+	}
+}
+
+// An episode of unknown length cannot be judged by share of itself.
+func TestRecurringTextNeedsTheEpisodeLength(t *testing.T) {
+	found := candidates(SignalRecurring, [2]int64{0, 60_000})
+
+	if got, why := PublishableRecurring(found, 0); len(got) != 0 || why == "" {
+		t.Errorf("got %d breaks, refused %q", len(got), why)
+	}
+}
+
+// Measured advertising is published by Publishable and must not come through here as the show's
+// own audio, which would weaken it from always-skipped to skipped-by-preference.
+func TestMeasuredAdvertisingDoesNotComeThroughAsFurniture(t *testing.T) {
+	found := candidates(SignalDifference, [2]int64{0, 60_000})
+
+	if got, _ := PublishableRecurring(found, 3_600_000); len(got) != 0 {
+		t.Error("measured advertising was republished as the show's own audio")
+	}
+}
+
+func TestNoRecurringTextPublishesNothing(t *testing.T) {
+	got, why := PublishableRecurring(nil, 3_600_000)
+	if len(got) != 0 || why != "" {
+		t.Errorf("got %d breaks, refused %q", len(got), why)
+	}
+}
diff --git a/server/internal/transcript/recurring.go b/server/internal/transcript/recurring.go
new file mode 100644
index 0000000..02fd68a
--- /dev/null
+++ b/server/internal/transcript/recurring.go
@@ -0,0 +1,114 @@
+package transcript
+
+import "strings"
+
+/*
+Finding what a show says in every episode.
+
+The text analogue of the audio scan, and it costs nothing: the server already holds transcripts
+for eight episodes of each show, mirrored within a minute of each other. They carry the same
+advertising campaign and different programmes, so text appearing across several of them is not
+the programme.
+
+It reaches where subtraction cannot. Subtraction needs a second description of one episode --
+the publisher's transcript, or a second fetch weeks later -- and only 18 of 337 mirrored
+episodes have one. This needs nothing but the episodes the server already holds, which is why
+it is worth more than either of the other two: measured against the length gap on the live
+library, Heavyweight 111%, Incels 105%, Ezra Klein 92%, Tangle 83%, against refetching's 21% on
+the same show.
+
+What it finds is "what this show says every week", which is advertising *and* the show's own
+furniture -- its welcome, its break cue, its credits. Those are not the same thing to a
+listener and are not treated as the same thing here; see Furniture.
+*/
+
+// SignalRecurring marks a candidate found because several of a show's episodes say it.
+const SignalRecurring = "recurring"
+
+const (
+	// recurringWords is how long a run must be to count as the same text in two episodes.
+	// Six: long enough that an ordinary sentence of programme does not collide with another
+	// episode's, short enough to survive recognition hearing one word differently.
+	recurringWords = 6
+	// minOtherEpisodes is how many other episodes must say it. Two, because two episodes of a
+	// show share turns of phrase, running jokes and guests' names, and one coincidence should
+	// not become a skip. A campaign runs across the whole batch.
+	minOtherEpisodes = 2
+)
+
+// Recurring reports the stretches of target that at least [minOtherEpisodes] of others also say.
+func Recurring(target []Cue, others [][]Cue) []Candidate {
+	targetWords := tokenise(target)
+	if len(targetWords) < recurringWords || len(others) < minOtherEpisodes {
+		return nil
+	}
+
+	elsewhere := make([]map[string]bool, 0, len(others))
+	for _, other := range others {
+		elsewhere = append(elsewhere, runsOf(tokenise(other), recurringWords))
+	}
+
+	// A word is shared when some run covering it is said by enough other episodes. Marking
+	// every word of a matching run, rather than its first, is what makes the stretches join up
+	// into one break instead of a chain of six-word fragments.
+	shared := make([]bool, len(targetWords))
+	for i := 0; i+recurringWords <= len(targetWords); i++ {
+		run := joinWords(targetWords[i : i+recurringWords])
+		said := 0
+		for _, other := range elsewhere {
+			if other[run] {
+				said++
+			}
+		}
+		if said < minOtherEpisodes {
+			continue
+		}
+		for j := i; j < i+recurringWords; j++ {
+			shared[j] = true
+		}
+	}
+
+	return candidatesFrom(targetWords, shared, SignalRecurring)
+}
+
+// candidatesFrom turns runs of marked words into candidates, dropping anything too brief to be
+// a break -- a line both episodes happen to use is not something to seek past.
+func candidatesFrom(words []token, marked []bool, signal string) []Candidate {
+	var found []Candidate
+	for i := 0; i < len(words); {
+		if !marked[i] {
+			i++
+			continue
+		}
+		start := i
+		for i < len(words) && marked[i] {
+			i++
+		}
+		candidate := Candidate{
+			StartMs: words[start].startMs,
+			EndMs:   words[i-1].endMs,
+			Signals: []string{signal},
+		}
+		if candidate.EndMs-candidate.StartMs >= minCandidateMs {
+			found = append(found, candidate)
+		}
+	}
+	return found
+}
+
+// runsOf indexes every run of n words.
+func runsOf(words []token, n int) map[string]bool {
+	runs := make(map[string]bool, len(words))
+	for i := 0; i+n <= len(words); i++ {
+		runs[joinWords(words[i:i+n])] = true
+	}
+	return runs
+}
+
+func joinWords(words []token) string {
+	parts := make([]string, len(words))
+	for i, w := range words {
+		parts[i] = w.word
+	}
+	return strings.Join(parts, " ")
+}
diff --git a/server/internal/transcript/recurring_test.go b/server/internal/transcript/recurring_test.go
new file mode 100644
index 0000000..0de5086
--- /dev/null
+++ b/server/internal/transcript/recurring_test.go
@@ -0,0 +1,207 @@
+package transcript
+
+import "testing"
+
+/*
+Finding what a show says in every episode.
+
+The text analogue of the audio scan, and it costs nothing: the server already holds transcripts
+for eight episodes of each show, mirrored within a minute of each other. They carry the same
+advertising campaign and different programmes, so text appearing across several of them is not
+the programme.
+
+It reaches where subtraction cannot. Subtraction needs a second description of one episode -- a
+publisher's transcript or a second fetch -- and only 18 of 337 mirrored episodes have one. This
+needs nothing but the episodes themselves.
+
+What it finds is "what this show says every week", which is advertising *and* the show's own
+furniture: its welcome, its break cue, its credits. Telling those apart is [Recurring]'s
+caller's problem, not its own -- see Furniture.
+*/
+
+/*
+show builds a programme whose every line is unlike every other show's.
+
+Each word carries the prefix, so no run of six is ever shared by accident. The first version of
+this helper gave every line the same tail -- "which continues for a while with its own
+particular words" -- and the detector dutifully found it in all three episodes, which is correct
+behaviour against an incorrect fixture.
+*/
+func show(prefix string, n int) []string {
+	out := make([]string, 0, n)
+	for i := 0; i < n; i++ {
+		line := prefix + string(rune('a'+i))
+		words := make([]string, 0, 12)
+		for w := 0; w < 12; w++ {
+			words = append(words, line+string(rune('0'+w)))
+		}
+		out = append(out, joinStrings(words))
+	}
+	return out
+}
+
+func joinStrings(parts []string) string {
+	out := ""
+	for i, p := range parts {
+		if i > 0 {
+			out += " "
+		}
+		out += p
+	}
+	return out
+}
+
+var sponsorRead = []string{
+	"This message comes from a mattress company who make mattresses for sleeping",
+	"Go to mattress dot com slash listen and use the promotional code today",
+	"Visit them now because terms and conditions will of course always apply",
+	"They have been making those mattresses for forty long years and counting",
+	"Every mattress is rolled up into a box and delivered to your door",
+}
+
+func TestFindsWhatSeveralEpisodesShare(t *testing.T) {
+	target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3))...)
+	others := [][]Cue{
+		words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3))...),
+		words(0, lines(show("gamma", 3), sponsorRead, show("gamma2", 3))...),
+	}
+
+	got := Recurring(target, others)
+	if len(got) != 1 {
+		t.Fatalf("got %d candidates, want 1: %+v", len(got), got)
+	}
+	if got[0].StartMs != 3*cueMs || got[0].EndMs != 8*cueMs {
+		t.Errorf("got %d-%d, want the five shared lines", got[0].StartMs, got[0].EndMs)
+	}
+}
+
+/*
+One other episode is not a pattern.
+
+Two episodes of a show can share a turn of phrase, a guest's name, a running joke. A campaign
+runs across the batch, so it turns up in several -- and requiring several is what keeps a
+coincidence from becoming a skip.
+*/
+func TestSomethingInOnlyOneOtherEpisodeIsNotRecurring(t *testing.T) {
+	target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3))...)
+	others := [][]Cue{
+		words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3))...),
+		words(0, lines(show("gamma", 3), show("gamma2", 3))...),
+	}
+
+	if got := Recurring(target, others); len(got) != 0 {
+		t.Errorf("got %+v, want nothing -- one other episode is a coincidence", got)
+	}
+}
+
+// The programme is what differs, and none of it may be reported.
+func TestTheProgrammeIsNotRecurring(t *testing.T) {
+	target := words(0, show("alpha", 8)...)
+	others := [][]Cue{
+		words(0, show("beta", 8)...),
+		words(0, show("gamma", 8)...),
+	}
+
+	if got := Recurring(target, others); len(got) != 0 {
+		t.Errorf("got %+v, want nothing", got)
+	}
+}
+
+// A phrase both episodes happen to use is not a break, and skipping one would clip the show.
+func TestAPassingPhraseIsTooShortToBeABreak(t *testing.T) {
+	shared := []string{"and we will be right back after this"}
+	target := words(0, lines(show("alpha", 4), shared, show("alpha2", 4))...)
+	others := [][]Cue{
+		words(0, lines(show("beta", 4), shared, show("beta2", 4))...),
+		words(0, lines(show("gamma", 4), shared, show("gamma2", 4))...),
+	}
+
+	if got := Recurring(target, others); len(got) != 0 {
+		t.Errorf("got %+v, want nothing -- one line is not a break", got)
+	}
+}
+
+func TestTwoRecurringStretchesStaySeparate(t *testing.T) {
+	second := []string{
+		"Support for this programme comes from a razor company with five sharp blades",
+		"Visit razors dot com slash listen today where terms and conditions apply",
+		"They will send you a handle and a year of replacement cartridges",
+		"And a further word from a company that sells boxes for other things",
+		"Those boxes are cardboard and they fold flat which is awfully convenient",
+	}
+	target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3), second)...)
+	others := [][]Cue{
+		words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3), second)...),
+		words(0, lines(show("gamma", 3), sponsorRead, show("gamma2", 3), second)...),
+	}
+
+	got := Recurring(target, others)
+	if len(got) != 2 {
+		t.Fatalf("got %d candidates, want 2: %+v", len(got), got)
+	}
+	if got[0].EndMs >= got[1].StartMs {
+		t.Errorf("the two overlap: %+v", got)
+	}
+}
+
+func TestNothingToCompareAgainstFindsNothing(t *testing.T) {
+	target := words(0, show("alpha", 8)...)
+	if got := Recurring(target, nil); len(got) != 0 {
+		t.Errorf("got %+v", got)
+	}
+	if got := Recurring(nil, [][]Cue{words(0, show("beta", 8)...)}); len(got) != 0 {
+		t.Errorf("got %+v", got)
+	}
+}
+
+// Same shape as every other candidate, so one episode's breaks can come from either detector
+// and be read the same way.
+func TestACandidateSaysItRecurs(t *testing.T) {
+	target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3))...)
+	others := [][]Cue{
+		words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3))...),
+		words(0, lines(show("gamma", 3), sponsorRead, show("gamma2", 3))...),
+	}
+
+	got := Recurring(target, others)
+	if len(got) != 1 || len(got[0].Signals) != 1 || got[0].Signals[0] != SignalRecurring {
+		t.Errorf("got %+v, want one marked %q", got, SignalRecurring)
+	}
+}
+
+/*
+Recurring text is published as the show's own audio, never as advertising.
+
+An honest limit, measured rather than assumed. A show's recurring text is both its sponsor reads
+and its furniture -- the welcome, the break cue, the credits -- and telling those apart would
+need a signal that separates them. None does. Scored against the 404 recurring stretches in
+episodes that also hold a publisher's master, which says which is which:
+
+	ad copy markers   5% of the advertising, 4% of the furniture
+	length            mean 51s against 29s, overlapping throughout
+	position          ads 62 open / 176 mid / 41 close
+	                  furniture 12 open / 61 mid / 52 close
+
+A first attempt using ad copy agreed with the master 33% of the time, and called 263
+advertisements furniture.
+
+So the detector claims only what it measured: this is text the show says every week. That is
+exactly what KindCue means, and it leaves the listener's own setting in charge -- furniture is
+skipped by preference, advertising unconditionally, and claiming the stronger of the two on a
+coin flip would be the worse mistake.
+*/
+func TestRecurringTextIsTheShowsOwnAudioAndNotAdvertising(t *testing.T) {
+	target := words(0, lines(show("alpha", 3), sponsorRead, show("alpha2", 3))...)
+	others := [][]Cue{
+		words(0, lines(show("beta", 3), sponsorRead, show("beta2", 3))...),
+		words(0, lines(show("gamma", 3), sponsorRead, show("gamma2", 3))...),
+	}
+
+	got := Recurring(target, others)
+	if len(got) != 1 {
+		t.Fatalf("got %+v", got)
+	}
+	if got[0].Signals[0] == SignalDifference {
+		t.Error("recurring text must not claim to be measured advertising")
+	}
+}