From 8c6343c9185bd3978e1e2f81374eef0571e1965d Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Thu, 29 Oct 2026 20:07:26 +0000 Subject: Decide where a break is line by line, not by how far the master drifted Three episodes were missing a whole advertisement, which was the largest error left in subtraction. Diagnosed rather than guessed at: Incels "From Victim to Violence" has a three-minute pre-roll in which the served copy runs 463 words against the master's 84, and it was refused because the master advanced more than the ten words of drift the rule allowed. That guard was there for a real reason -- a stretch where both texts advanced is one where you cannot say which part of it is the advertisement -- but it answered the question at the wrong resolution. The gap as a whole need not be all advertising; its individual lines are each either in the master or not, and that is answerable. So the drift guard is gone and every gap is partitioned line by line against the master instead, which is the same machinery the edge trim already used. The trim is no longer a separate step a caller has to remember, which is just as well: a caller who forgot it would publish a skip over the last sentence of the programme. Measured across all eighteen episodes holding a master: before 73% - 110%, mean 99%, three episodes missing a pod after 99% - 110%, mean 103%, every episode finding every pod #67 Jenna 81% -> 106% 3 pods -> 4 Behind the Scenes 73% -> 101% 3 pods -> 4 From Victim to Violence 78% -> 99% 3 pods -> 4 Nothing got worse. The fifteen episodes that were already right are unchanged to the percentage point. The fixtures spoke at 360 words a minute, which hid the fact that the word floor and the duration floor disagree about what counts as an advertisement. They now run at about 180, which is roughly how fast people talk, and the two floors agree. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- server/cmd/listend/main.go | 7 +--- server/internal/transcript/diff.go | 10 ++--- server/internal/transcript/diff_test.go | 29 +++++++++----- server/internal/transcript/trim.go | 7 +++- server/internal/transcript/trim_test.go | 69 +++++++++++++-------------------- 5 files changed, 55 insertions(+), 67 deletions(-) (limited to 'server') diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index dad5c30..9914284 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -598,12 +598,7 @@ advertisement. func (w *worker) adsIn(e *store.Episode, cues []transcript.Cue) []transcript.Candidate { if e.MasterCues > 0 { if master, err := w.transcripts.Master(e.ID); err == nil { - // Trimmed as well as found: subtraction answers "not in the master", which takes - // in the last line of the programme whenever recognition dented it enough to stop - // it matching. See transcript.Trimmed. - if found := transcript.Trimmed( - transcript.Inserted(master.Cues, cues), master.Cues, cues, - ); len(found) > 0 { + if found := transcript.Inserted(master.Cues, cues); len(found) > 0 { return found } } diff --git a/server/internal/transcript/diff.go b/server/internal/transcript/diff.go index 667949f..acf6dad 100644 --- a/server/internal/transcript/diff.go +++ b/server/internal/transcript/diff.go @@ -43,10 +43,6 @@ const ( // speech, the same floor the phrase spotter uses and the same ad unit the length-gap rule // uses. minInsertedWords = 40 - // masterDriftWords is how far the master may advance across a gap that is still read as an - // insertion. Inside a real ad pod the master does not advance at all; a few words of drift - // is an anchor lost to recognition error at the edge of the break. - masterDriftWords = 10 ) // token is one word of a transcript and where it was said. @@ -83,7 +79,7 @@ func Inserted(master, served []Cue) []Candidate { masterFrom := previous.master + anchorWords servedGap := next.served - servedFrom masterGap := next.master - masterFrom - if servedGap-masterGap >= minInsertedWords && masterGap <= masterDriftWords { + if servedGap-masterGap >= minInsertedWords && servedFrom < len(servedWords) { from := servedWords[servedFrom] to := servedWords[next.served-1] found = append(found, Candidate{ @@ -94,7 +90,9 @@ func Inserted(master, served []Cue) []Candidate { } previous = next } - return found + // Every gap is then partitioned line by line against the master, which is what decides + // where within it the advertising actually is. See [Trimmed]. + return trimmed(found, master, served) } // anchor ties a position in the master to the position in the served copy saying the same words. diff --git a/server/internal/transcript/diff_test.go b/server/internal/transcript/diff_test.go index e24be51..7c40e44 100644 --- a/server/internal/transcript/diff_test.go +++ b/server/internal/transcript/diff_test.go @@ -22,13 +22,18 @@ errors: a five-word window containing a mistake simply fails to anchor, and its still do. */ -// words builds cues of one sentence each, two seconds apart, so a test reads as text. +// cueMs is how long one line of these fixtures lasts. Four seconds for a twelve-word line is +// about 180 words a minute, which is roughly how fast people talk -- and it matters, because +// the word floor and the duration floor have to agree on what counts as an advertisement. +const cueMs = 4000 + +// words builds cues of one sentence each, so a test reads as text. func words(startMs int64, lines ...string) []Cue { cues := make([]Cue, 0, len(lines)) at := startMs for _, line := range lines { - cues = append(cues, Cue{StartMs: at, EndMs: at + 2000, Text: line}) - at += 2000 + cues = append(cues, Cue{StartMs: at, EndMs: at + cueMs, Text: line}) + at += cueMs } return cues } @@ -60,8 +65,8 @@ func TestFindsTheAdvertisingThatIsOnlyInTheServedCopy(t *testing.T) { t.Fatalf("got %d candidates, want 1: %+v", len(got), got) } // The four inserted lines begin at 6000 and the programme resumes at 14000. - if got[0].StartMs != 6000 || got[0].EndMs != 14000 { - t.Errorf("got %d-%d, want 6000-14000", got[0].StartMs, got[0].EndMs) + if got[0].StartMs != 3*cueMs || got[0].EndMs != 7*cueMs { + t.Errorf("got %d-%d, want the four inserted lines", got[0].StartMs, got[0].EndMs) } } @@ -97,6 +102,7 @@ func TestFindsAPreRoll(t *testing.T) { "Stay tuned for the following messages from our sponsors on this fine Tuesday", "This message comes from a mattress company who have been making mattresses for forty years", "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + "They have been making those mattresses for forty long years and are still counting", programme[0], programme[1], programme[2], @@ -108,26 +114,27 @@ func TestFindsAPreRoll(t *testing.T) { if len(got) != 1 { t.Fatalf("got %d candidates, want 1: %+v", len(got), got) } - if got[0].StartMs != 0 || got[0].EndMs != 6000 { - t.Errorf("got %d-%d, want 0-6000", got[0].StartMs, got[0].EndMs) + if got[0].StartMs != 0 || got[0].EndMs != 4*cueMs { + t.Errorf("got %d-%d, want the four inserted lines", got[0].StartMs, got[0].EndMs) } } // And at the end, after the programme has finished. func TestFindsAPostRoll(t *testing.T) { master := words(0, programme...) - served := append(words(0, programme...), words(10000, + served := append(words(0, programme...), words(5*cueMs, "Stay tuned for the following messages from our sponsors on this fine Tuesday", "This message comes from a mattress company who have been making mattresses for forty years", "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + "They have been making those mattresses for forty long years and are still counting", )...) got := Inserted(master, served) if len(got) != 1 { t.Fatalf("got %d candidates, want 1: %+v", len(got), got) } - if got[0].StartMs != 10000 { - t.Errorf("starts at %d, want 10000", got[0].StartMs) + if got[0].StartMs != 5*cueMs { + t.Errorf("starts at %d, want where the programme ended", got[0].StartMs) } } @@ -138,11 +145,13 @@ func TestFindsSeveralBreaksSeparately(t *testing.T) { "Stay tuned for the following messages from our sponsors on this fine Tuesday", "This message comes from a mattress company who have been making mattresses for forty years", "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + "They have been making those mattresses for forty long years and are still counting", programme[1], programme[2], "Support for this programme comes from a razor company with five blades on every razor", "Visit razors dot com slash listen today where terms and conditions will of course apply", "And a further word from a company that sells boxes for putting other things inside", + "Those boxes are cardboard and they fold flat which is awfully convenient indeed", programme[3], programme[4], ) diff --git a/server/internal/transcript/trim.go b/server/internal/transcript/trim.go index c8984aa..995bc85 100644 --- a/server/internal/transcript/trim.go +++ b/server/internal/transcript/trim.go @@ -42,11 +42,14 @@ const ( trimMinWords = 3 ) -// Trimmed pulls each candidate's edges in off any line that the master also contains. +// trimmed pulls each candidate's edges in off any line that the master also contains. // // Only from the ends, and only while the lines keep matching: a line inside the break that // happens to echo the programme is not a reason to split an advertisement in two. -func Trimmed(found []Candidate, master, served []Cue) []Candidate { +// +// Part of [Inserted] rather than a step a caller has to remember, because a caller that forgot +// it would publish a skip over the last sentence of the programme. +func trimmed(found []Candidate, master, served []Cue) []Candidate { if len(found) == 0 { return nil } diff --git a/server/internal/transcript/trim_test.go b/server/internal/transcript/trim_test.go index af0ce0e..51f3965 100644 --- a/server/internal/transcript/trim_test.go +++ b/server/internal/transcript/trim_test.go @@ -63,23 +63,13 @@ func TestTrimsProgrammeFromTheStartOfABreak(t *testing.T) { []string{programme[0], leakedAsHeard}, advert, []string{programme[1], programme[2]}, )...) - found := Inserted(master, served) - if len(found) != 1 { - t.Fatalf("got %d candidates, want 1: %+v", len(found), found) + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(got), got) } - if found[0].StartMs != 2000 { - t.Fatalf("the misheard line did not fall inside the break: %+v", found[0]) - } - - trimmed := Trimmed(found, master, served) - if len(trimmed) != 1 { - t.Fatalf("trimming lost the candidate: %+v", trimmed) - } - if trimmed[0].StartMs != 4000 { - t.Errorf("start = %d, want 4000 -- the advertisement's own first line", trimmed[0].StartMs) - } - if trimmed[0].EndMs != found[0].EndMs { - t.Errorf("the end moved too: %d -> %d", found[0].EndMs, trimmed[0].EndMs) + if got[0].StartMs != 2*cueMs { + t.Errorf("start = %d, want the advertisement's own first line, not the misheard one", + got[0].StartMs) } } @@ -89,20 +79,13 @@ func TestTrimsProgrammeFromTheEndOfABreak(t *testing.T) { []string{programme[0]}, advert, []string{leakedAsHeard, programme[1], programme[2]}, )...) - found := Inserted(master, served) - if len(found) != 1 { - t.Fatalf("got %d candidates, want 1: %+v", len(found), found) + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(got), got) } - - trimmed := Trimmed(found, master, served) - if len(trimmed) != 1 { - t.Fatalf("trimming lost the candidate: %+v", trimmed) - } - if trimmed[0].EndMs >= found[0].EndMs { - t.Errorf("end did not move: %d -> %d", found[0].EndMs, trimmed[0].EndMs) - } - if trimmed[0].EndMs != 18000 { - t.Errorf("end = %d, want 18000 -- the advertisement's own last line", trimmed[0].EndMs) + if got[0].EndMs != 9*cueMs { + t.Errorf("end = %d, want the advertisement's own last line, not the misheard one", + got[0].EndMs) } } @@ -116,11 +99,12 @@ func TestTheAdvertisementItselfIsLeftWhole(t *testing.T) { master := words(0, programme[0], programme[1], programme[2]) served := words(0, lines([]string{programme[0]}, advert, []string{programme[1], programme[2]})...) - found := Inserted(master, served) - trimmed := Trimmed(found, master, served) - if len(trimmed) != 1 || - trimmed[0].StartMs != found[0].StartMs || trimmed[0].EndMs != found[0].EndMs { - t.Errorf("got %+v, want %+v unchanged", trimmed, found) + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %+v, want one candidate", got) + } + if got[0].StartMs != cueMs || got[0].EndMs != 9*cueMs { + t.Errorf("got %d-%d, want the whole advertisement", got[0].StartMs, got[0].EndMs) } } @@ -137,13 +121,12 @@ func TestNonSpeechMarkersAreNotTrimmed(t *testing.T) { []string{programme[0], "[MUSIC PLAYING]"}, advert, []string{programme[1], programme[2]}, )...) - found := Inserted(master, served) - trimmed := Trimmed(found, master, served) - if len(trimmed) != 1 { - t.Fatalf("got %+v", trimmed) + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %+v, want one candidate", got) } - if trimmed[0].StartMs != found[0].StartMs { - t.Errorf("trimmed the music marker away: %d -> %d", found[0].StartMs, trimmed[0].StartMs) + if got[0].StartMs != cueMs { + t.Errorf("start = %d, want the music marker kept with the break", got[0].StartMs) } } @@ -151,15 +134,15 @@ func TestNonSpeechMarkersAreNotTrimmed(t *testing.T) { func TestACandidateTrimmedBelowTheFloorIsDropped(t *testing.T) { master := words(0, programme...) served := words(0, programme...) - found := []Candidate{{StartMs: 0, EndMs: 6000, Signals: []string{SignalDifference}}} + found := []Candidate{{StartMs: 0, EndMs: 3 * cueMs, Signals: []string{SignalDifference}}} - if got := Trimmed(found, master, served); len(got) != 0 { + if got := trimmed(found, master, served); len(got) != 0 { t.Errorf("got %+v, want nothing left -- every line of it is the programme", got) } } func TestTrimmingNothingIsNothing(t *testing.T) { - if got := Trimmed(nil, words(0, programme...), words(0, programme...)); len(got) != 0 { + if got := trimmed(nil, words(0, programme...), words(0, programme...)); len(got) != 0 { t.Errorf("got %+v", got) } } -- cgit v1.2.3