diff options
| -rw-r--r-- | server/cmd/listend/main.go | 7 | ||||
| -rw-r--r-- | server/internal/transcript/trim.go | 125 | ||||
| -rw-r--r-- | server/internal/transcript/trim_test.go | 165 |
3 files changed, 296 insertions, 1 deletions
diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index d45f890..100eae5 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -544,7 +544,12 @@ advertisement. func (w *worker) adsIn(e *store.Episode, cues []transcript.Cue) []transcript.Candidate { if e.MasterCues > 0 { if master, err := w.transcripts.Master(e.ID); err == nil { - if found := transcript.Inserted(master.Cues, cues); len(found) > 0 { + // Trimmed as well as found: subtraction answers "not in the master", which takes + // in the last line of the programme whenever recognition dented it enough to stop + // it matching. See transcript.Trimmed. + if found := transcript.Trimmed( + transcript.Inserted(master.Cues, cues), master.Cues, cues, + ); len(found) > 0 { return found } } diff --git a/server/internal/transcript/trim.go b/server/internal/transcript/trim.go new file mode 100644 index 0000000..c8984aa --- /dev/null +++ b/server/internal/transcript/trim.go @@ -0,0 +1,125 @@ +package transcript + +import "strings" + +/* +Pulling a candidate's edges back off the programme. + +Subtraction answers "not in the master", which is a superset of "advertising". Measured across +the eighteen episodes holding a master, candidates come to a mean of exactly 100% of what the +length gap implies, and individually overshoot by up to 10%. Reading their edges shows two +causes. + +The first is the programme leaking in: Darknet Diaries' "NSL" opens a break on "I'm Jack +Resider." and closes one on "So Nick and ACLU, we're going to challenge", and Incels opens two +on "let's stop here for a...". Those lines failed to anchor only because recognition misheard a +word near them, and skipping them is the harm this whole thing exists to avoid. They are in the +master, so they can be found there and given back. + +The second is whisper's own non-speech markers and a title card -- "[MUSIC PLAYING]", "(upbeat +music)", "Today's episode, Tamara." Those are genuinely absent from a publisher's written +transcript, so nothing here can tell them from an advertisement. They are left alone: they sit +against the break anyway, and a rule invented to catch them would be a guess. +*/ + +const ( + // trimRunWords is how many consecutive words of a line must appear in the master before it + // is treated as the programme. An advertisement's words are not in the master at all -- + // that is what made it a candidate -- so what this length really decides is how badly a + // line may have been misheard and still be recognised as the programme. + // + // Measured at four, five and six across the eighteen episodes holding a master. Five and + // six give identical results, and four differs from them by one point on one episode, + // which is noise on a sample this size. Four is kept because it is the only one of the + // three that can still recognise a short line with a mistake in the middle of it -- an + // eight-word line with a dented fifth word has no clean five-word run anywhere, and "So + // Nick and ACLU, we're going to challenge" is eight words. Erring towards trimming also + // errs towards smaller candidates, which is the safe direction: a skip into the programme + // is worse than a missed advertisement. + trimRunWords = 4 + // trimMinWords is the shortest line this can judge. A two-word line gives nothing to look + // for, so it is kept rather than guessed at, which is what leaves the music markers alone. + trimMinWords = 3 +) + +// Trimmed pulls each candidate's edges in off any line that the master also contains. +// +// Only from the ends, and only while the lines keep matching: a line inside the break that +// happens to echo the programme is not a reason to split an advertisement in two. +func Trimmed(found []Candidate, master, served []Cue) []Candidate { + if len(found) == 0 { + return nil + } + runs := runsIn(master) + + out := make([]Candidate, 0, len(found)) + for _, candidate := range found { + inside := cuesWithin(served, candidate) + first, last := 0, len(inside)-1 + for first <= last && inMaster(inside[first], runs) { + first++ + } + for last > first && inMaster(inside[last], runs) { + last-- + } + if first > last { + // Every line of it is in the master, so none of it was an advertisement. + continue + } + trimmed := Candidate{ + StartMs: inside[first].StartMs, + EndMs: inside[last].EndMs, + Signals: candidate.Signals, + } + if trimmed.EndMs-trimmed.StartMs < minCandidateMs { + continue + } + out = append(out, trimmed) + } + return out +} + +// inMaster reports whether a line's words appear in the master. +func inMaster(cue Cue, runs map[string]bool) bool { + words := strings.FieldsFunc(strings.ToLower(cue.Text), notWord) + if len(words) < trimMinWords { + return false + } + size := trimRunWords + if len(words) < size { + size = len(words) + } + for i := 0; i+size <= len(words); i++ { + if runs[strings.Join(words[i:i+size], " ")] { + return true + } + } + return false +} + +// runsIn is every run of words in the master that a line could be recognised by, at both the +// full length and the shorter one a brief line is judged on. +func runsIn(master []Cue) map[string]bool { + words := tokenise(master) + runs := make(map[string]bool, len(words)*2) + for size := trimMinWords; size <= trimRunWords; size++ { + for i := 0; i+size <= len(words); i++ { + parts := make([]string, size) + for j := 0; j < size; j++ { + parts[j] = words[i+j].word + } + runs[strings.Join(parts, " ")] = true + } + } + return runs +} + +func cuesWithin(cues []Cue, candidate Candidate) []Cue { + var out []Cue + for _, cue := range cues { + if cue.StartMs >= candidate.StartMs && cue.EndMs <= candidate.EndMs { + out = append(out, cue) + } + } + return out +} diff --git a/server/internal/transcript/trim_test.go b/server/internal/transcript/trim_test.go new file mode 100644 index 0000000..af0ce0e --- /dev/null +++ b/server/internal/transcript/trim_test.go @@ -0,0 +1,165 @@ +package transcript + +import "testing" + +/* +Pulling a candidate's edges back off the programme. + +Subtraction answers "not in the master", which is a superset of "advertising". Measured across +the eighteen episodes holding a master, candidates run to a mean of exactly 100% of the length +gap but individually overshoot by up to 10%, and reading their edges shows two different causes: + + Darknet Diaries "NSL" opens a break on "I'm Jack Resider." and closes one on + "So Nick and ACLU, we're going to challenge" + Incels "Worship..." opens two breaks on "let's stop here for a..." + Heavyweight "#69" opens breaks on "[MUSIC PLAYING]" and "(upbeat music)" + +The first kind is the programme, and skipping it is the harm this exists to prevent. It failed +to anchor only because recognition misheard a word nearby -- the words themselves are in the +master and can be found there. + +The second kind is whisper's own non-speech markers and a title card. They are genuinely absent +from a publisher's written transcript, so nothing here can tell them from an advertisement, and +they are left alone: they sit against the break anyway, and inventing a rule to catch them would +be guessing. +*/ + +// advert is eight lines nothing in any master contains. +var advert = []string{ + "This message comes from a mattress company who make mattresses for sleeping", + "Go to mattress dot com slash listen and use the promotional code today", + "Visit them now because terms and conditions will of course always apply", + "They have been making those mattresses for forty long years and counting", + "Every mattress is rolled up into a box and delivered to your door", + "Try one for a hundred nights and send it back if you dislike it", + "That offer stands until the end of the month for new customers only", + "Again that address is mattress dot com slash listen for the full details", +} + +/* +The line that leaks in is the one recognition dented. + +Eight words with a mistake in the middle: every five-word window contains the mistake, so none +of them anchors and the line falls inside the break -- while "and now a word" is untouched and +is still in the master, which is what gives it back. That is the exact mechanism behind "I'm +Jack Resider." opening a Darknet Diaries break. +*/ +const ( + leakedAsWritten = "And now a word about the thing today" + leakedAsHeard = "And now a word abort the thing today" +) + +func lines(parts ...[]string) []string { + var out []string + for _, part := range parts { + out = append(out, part...) + } + return out +} + +func TestTrimsProgrammeFromTheStartOfABreak(t *testing.T) { + master := words(0, programme[0], leakedAsWritten, programme[1], programme[2]) + served := words(0, lines( + []string{programme[0], leakedAsHeard}, advert, []string{programme[1], programme[2]}, + )...) + + found := Inserted(master, served) + if len(found) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(found), found) + } + if found[0].StartMs != 2000 { + t.Fatalf("the misheard line did not fall inside the break: %+v", found[0]) + } + + trimmed := Trimmed(found, master, served) + if len(trimmed) != 1 { + t.Fatalf("trimming lost the candidate: %+v", trimmed) + } + if trimmed[0].StartMs != 4000 { + t.Errorf("start = %d, want 4000 -- the advertisement's own first line", trimmed[0].StartMs) + } + if trimmed[0].EndMs != found[0].EndMs { + t.Errorf("the end moved too: %d -> %d", found[0].EndMs, trimmed[0].EndMs) + } +} + +func TestTrimsProgrammeFromTheEndOfABreak(t *testing.T) { + master := words(0, programme[0], leakedAsWritten, programme[1], programme[2]) + served := words(0, lines( + []string{programme[0]}, advert, []string{leakedAsHeard, programme[1], programme[2]}, + )...) + + found := Inserted(master, served) + if len(found) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(found), found) + } + + trimmed := Trimmed(found, master, served) + if len(trimmed) != 1 { + t.Fatalf("trimming lost the candidate: %+v", trimmed) + } + if trimmed[0].EndMs >= found[0].EndMs { + t.Errorf("end did not move: %d -> %d", found[0].EndMs, trimmed[0].EndMs) + } + if trimmed[0].EndMs != 18000 { + t.Errorf("end = %d, want 18000 -- the advertisement's own last line", trimmed[0].EndMs) + } +} + +/* +The advertisement itself is never trimmed. + +Its words are not in the master -- that is what made it a candidate -- so every line survives, +including the first and the last. +*/ +func TestTheAdvertisementItselfIsLeftWhole(t *testing.T) { + master := words(0, programme[0], programme[1], programme[2]) + served := words(0, lines([]string{programme[0]}, advert, []string{programme[1], programme[2]})...) + + found := Inserted(master, served) + trimmed := Trimmed(found, master, served) + if len(trimmed) != 1 || + trimmed[0].StartMs != found[0].StartMs || trimmed[0].EndMs != found[0].EndMs { + t.Errorf("got %+v, want %+v unchanged", trimmed, found) + } +} + +/* +Non-speech markers and title cards are left alone. + +"[MUSIC PLAYING]" and "Today's episode, Tamara." are genuinely absent from a publisher's written +transcript, so nothing here can tell them from an advertisement. They sit against the break +anyway, and a rule invented to catch them would be a guess. +*/ +func TestNonSpeechMarkersAreNotTrimmed(t *testing.T) { + master := words(0, programme[0], programme[1], programme[2]) + served := words(0, lines( + []string{programme[0], "[MUSIC PLAYING]"}, advert, []string{programme[1], programme[2]}, + )...) + + found := Inserted(master, served) + trimmed := Trimmed(found, master, served) + if len(trimmed) != 1 { + t.Fatalf("got %+v", trimmed) + } + if trimmed[0].StartMs != found[0].StartMs { + t.Errorf("trimmed the music marker away: %d -> %d", found[0].StartMs, trimmed[0].StartMs) + } +} + +// Trimming a candidate down to nothing means it was never an advertisement. +func TestACandidateTrimmedBelowTheFloorIsDropped(t *testing.T) { + master := words(0, programme...) + served := words(0, programme...) + found := []Candidate{{StartMs: 0, EndMs: 6000, Signals: []string{SignalDifference}}} + + if got := Trimmed(found, master, served); len(got) != 0 { + t.Errorf("got %+v, want nothing left -- every line of it is the programme", got) + } +} + +func TestTrimmingNothingIsNothing(t *testing.T) { + if got := Trimmed(nil, words(0, programme...), words(0, programme...)); len(got) != 0 { + t.Errorf("got %+v", got) + } +} |