package transcript import "strings" /* Finding what a show says in every episode. The text analogue of the audio scan, and it costs nothing: the server already holds transcripts for eight episodes of each show, mirrored within a minute of each other. They carry the same advertising campaign and different programmes, so text appearing across several of them is not the programme. It reaches where subtraction cannot. Subtraction needs a second description of one episode -- the publisher's transcript, or a second fetch weeks later -- and only 18 of 337 mirrored episodes have one. This needs nothing but the episodes the server already holds, which is why it is worth more than either of the other two: measured against the length gap on the live library, Heavyweight 111%, Incels 105%, Ezra Klein 92%, Tangle 83%, against refetching's 21% on the same show. What it finds is "what this show says every week", which is advertising *and* the show's own furniture -- its welcome, its break cue, its credits. Those are not the same thing to a listener and are not treated as the same thing here; see Furniture. */ // SignalRecurring marks a candidate found because several of a show's episodes say it. const SignalRecurring = "recurring" const ( // recurringWords is how long a run must be to count as the same text in two episodes. // Six: long enough that an ordinary sentence of programme does not collide with another // episode's, short enough to survive recognition hearing one word differently. recurringWords = 6 // minOtherEpisodes is how many other episodes must say it. Two, because two episodes of a // show share turns of phrase, running jokes and guests' names, and one coincidence should // not become a skip. A campaign runs across the whole batch. minOtherEpisodes = 2 ) // Recurring reports the stretches of target that at least [minOtherEpisodes] of others also say. func Recurring(target []Cue, others [][]Cue) []Candidate { targetWords := tokenise(target) if len(targetWords) < recurringWords || len(others) < minOtherEpisodes { return nil } elsewhere := make([]map[string]bool, 0, len(others)) for _, other := range others { elsewhere = append(elsewhere, runsOf(tokenise(other), recurringWords)) } // A word is shared when some run covering it is said by enough other episodes. Marking // every word of a matching run, rather than its first, is what makes the stretches join up // into one break instead of a chain of six-word fragments. shared := make([]bool, len(targetWords)) for i := 0; i+recurringWords <= len(targetWords); i++ { run := joinWords(targetWords[i : i+recurringWords]) said := 0 for _, other := range elsewhere { if other[run] { said++ } } if said < minOtherEpisodes { continue } for j := i; j < i+recurringWords; j++ { shared[j] = true } } return candidatesFrom(targetWords, shared, SignalRecurring) } // candidatesFrom turns runs of marked words into candidates, dropping anything too brief to be // a break -- a line both episodes happen to use is not something to seek past. func candidatesFrom(words []token, marked []bool, signal string) []Candidate { var found []Candidate for i := 0; i < len(words); { if !marked[i] { i++ continue } start := i for i < len(words) && marked[i] { i++ } candidate := Candidate{ StartMs: words[start].startMs, EndMs: words[i-1].endMs, Signals: []string{signal}, } if candidate.EndMs-candidate.StartMs >= minCandidateMs { found = append(found, candidate) } } return found } // runsOf indexes every run of n words. func runsOf(words []token, n int) map[string]bool { runs := make(map[string]bool, len(words)) for i := 0; i+n <= len(words); i++ { runs[joinWords(words[i:i+n])] = true } return runs } func joinWords(words []token) string { parts := make([]string, len(words)) for i, w := range words { parts[i] = w.word } return strings.Join(parts, " ") }