diff options
| -rw-r--r-- | server/cmd/listend/main.go | 83 | ||||
| -rw-r--r-- | server/internal/store/store.go | 17 | ||||
| -rw-r--r-- | server/internal/transcribe/whisper.go | 4 | ||||
| -rw-r--r-- | server/internal/transcript/diff.go | 200 | ||||
| -rw-r--r-- | server/internal/transcript/diff_test.go | 222 | ||||
| -rw-r--r-- | server/internal/transcript/fits.go | 29 | ||||
| -rw-r--r-- | server/internal/transcript/harvest.go | 16 | ||||
| -rw-r--r-- | server/internal/transcript/harvest_test.go | 95 | ||||
| -rw-r--r-- | server/internal/transcript/store.go | 43 |
9 files changed, 694 insertions, 15 deletions
diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index 1b746f1..d93325d 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -376,12 +376,18 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) { // every poll would be work done for ever to reach the same answer -- and the refusal is itself // the record of which episodes speech recognition would have to do. func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candidates []*store.Episode) { - taken, declined, spotted := 0, 0, 0 + taken, declined, spotted, masters := 0, 0, 0, 0 for _, e := range candidates { if ctx.Err() != nil { return } - if !e.Mirrored() || e.TranscriptExamined() || e.TranscriptURL == "" { + // A declined episode is examined and has no words, but its publisher's transcript is + // still the ad-free master and is worth fetching -- it is what Inserted subtracts. + // Episodes declined before masters were kept arrive here once. + if !e.Mirrored() || e.TranscriptURL == "" { + continue + } + if e.TranscriptExamined() && !e.NeedsMaster() { continue } result, err := w.harvester.Harvest(ctx, transcript.Offer{ @@ -396,6 +402,15 @@ func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candi log.Printf("%s: transcript for %q: %v", show.Title, e.Title, err) continue } + if result.Master > 0 { + e.MasterCues = result.Master + e.MasterNote = "" + masters++ + } else if e.NeedsMaster() { + // Offered, fetched, and not usable as a master either. Recorded so it is not + // fetched again on every poll for ever. + e.MasterNote = result.Note + } switch { case result.Stored: e.TranscriptSource = string(transcript.SourcePublished) @@ -410,11 +425,12 @@ func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candi declined++ } } - if taken > 0 || declined > 0 { + if taken > 0 || declined > 0 || masters > 0 { // The ad count is reported, not acted on. These are candidates read out of the words, // and nothing publishes them as ranges to skip -- see transcript.Spot. - log.Printf("%s: transcripts -- took %d, declined %d, %d ad candidate(s) in the words", - show.Title, taken, declined, spotted) + log.Printf("%s: transcripts -- took %d, declined %d (%d kept as ad-free masters), "+ + "%d ad candidate(s) in the words", + show.Title, taken, declined, masters, spotted) } } @@ -473,7 +489,7 @@ func (w *worker) transcribeSome(ctx context.Context) { Source: transcript.SourceASR, LanguageCode: result.LanguageCode, Cues: result.Cues, - Ads: transcript.Spot(result.Cues), + Ads: w.adsIn(e, result.Cues), } if err := w.transcripts.Put(stored); err != nil { log.Printf("transcribe %q: %v", e.Title, err) @@ -499,8 +515,9 @@ func (w *worker) transcribeSome(ctx context.Context) { done++ breaker.Success() failing = nil - log.Printf("transcribed %q: %d lines, %d ad candidate(s), took %s", - e.Title, len(stored.Cues), len(stored.Ads), time.Since(started).Round(time.Second)) + log.Printf("transcribed %q: %d lines, %s, took %s", + e.Title, len(stored.Cues), adAccounting(e, stored.Ads), + time.Since(started).Round(time.Second)) w.save() } @@ -510,6 +527,31 @@ func (w *worker) transcribeSome(ctx context.Context) { } } +/* +adsIn works out which stretches of a recognised episode are advertising. + +By subtraction where the publisher's ad-free master is held, and by reading the ad copy where it +is not. They are not close in quality and the choice is not a preference. Measured on Darknet +Diaries' "Conti": the feed declares 59:17 and the file served runs 63:33, so 4:15 of advertising +was inserted -- subtraction found 4:20 of it in three pods, each starting on the first word of +an ad read and ending on the last, while reading the copy found 0:16. 102% against 6%. + +Subtraction wins because it is a measurement. The master says what the programme is, the +recognised copy says what was played, and the difference is the advertising by definition -- +there is no phrase list to be wrong about, and no show whose hosts happen to talk like an +advertisement. +*/ +func (w *worker) adsIn(e *store.Episode, cues []transcript.Cue) []transcript.Candidate { + if e.MasterCues > 0 { + if master, err := w.transcripts.Master(e.ID); err == nil { + if found := transcript.Inserted(master.Cues, cues); len(found) > 0 { + return found + } + } + } + return transcript.Spot(cues) +} + // giveBackAttempts unmarks episodes that failed for a reason that was nothing to do with them, // so they are tried again rather than hidden for ever. func giveBackAttempts(episodes []*store.Episode) { @@ -518,6 +560,31 @@ func giveBackAttempts(episodes []*store.Episode) { } } +/* +adAccounting describes what was found against what the lengths say is there. + +The declared-versus-served gap comes from the feed and the file rather than from any detector, +which makes it the one number able to contradict one. Every other readout in this project that +reported only what it found turned out to be hiding something. +*/ +func adAccounting(e *store.Episode, found []transcript.Candidate) string { + var total int64 + for _, c := range found { + total += c.EndMs - c.StartMs + } + how := "read from the ad copy" + if e.MasterCues > 0 { + how = "by subtracting the publisher's master" + } + gap := e.DurationMs - e.DeclaredDurationMs + if gap <= 0 { + return fmt.Sprintf("%d ad candidate(s) %s, %s; nothing was inserted into this copy", + len(found), how, clock(total)) + } + return fmt.Sprintf("%d ad candidate(s) %s, %s of the %s inserted (%d%%)", + len(found), how, clock(total), clock(gap), total*100/gap) +} + // pendingRecognition is every mirrored episode still without words, newest first. func (w *worker) pendingRecognition() []*store.Episode { var pending []*store.Episode diff --git a/server/internal/store/store.go b/server/internal/store/store.go index 3c44060..7f81bd5 100644 --- a/server/internal/store/store.go +++ b/server/internal/store/store.go @@ -68,6 +68,12 @@ type Episode struct { TranscriptCues int `json:"transcriptCues,omitempty"` // TranscriptAtUnix is when the transcript landed. TranscriptAtUnix int64 `json:"transcriptAtUnix,omitempty"` + // MasterCues is how many cues the publisher's ad-free transcript holds, kept for episodes + // whose served copy carries inserted advertising. MasterNote is why there is none. Three + // states as everywhere else: words, a reason, or nobody has looked. + MasterCues int `json:"masterCues,omitempty"` + MasterNote string `json:"masterNote,omitempty"` + // TranscribeAttemptedAtUnix is when speech recognition last ran over this episode, whether // it worked or not, so a failure is not retried on every poll for ever. Recorded only when // a run actually finished: one interrupted by a restart is not an attempt, or a redeploy @@ -93,6 +99,17 @@ func (e *Episode) TranscriptExamined() bool { return e.HasTranscript() || e.TranscriptNote != "" } +// NeedsMaster reports whether the publisher's ad-free transcript should still be fetched. +// +// Only for an episode whose own transcript was declined: that is the one case where the +// publisher's words describe a different recording from the file being played, and so the one +// case where subtracting them locates the advertising. An episode that was never offered a +// transcript has nothing to fetch. +func (e *Episode) NeedsMaster() bool { + return e.Mirrored() && e.TranscriptNote != "" && e.TranscriptURL != "" && + e.MasterCues == 0 && e.MasterNote == "" +} + // NeedsRecognising reports whether speech recognition should be run over this episode. // // Deliberately not the negation of [Episode.TranscriptExamined]. An episode whose publisher's diff --git a/server/internal/transcribe/whisper.go b/server/internal/transcribe/whisper.go index c6e4f83..1789beb 100644 --- a/server/internal/transcribe/whisper.go +++ b/server/internal/transcribe/whisper.go @@ -169,6 +169,10 @@ type whisperOutput struct { } `json:"transcription"` } +// ParseOutput reads whisper-cli's output into cues. Exported for tools that work from a file +// whisper wrote earlier rather than running it. +func ParseOutput(raw []byte) (Result, error) { return parseWhisperJSON(raw) } + // parseWhisperJSON reads whisper-cli's output into cues. // // The offsets are milliseconds, which is what both the app and the ad scan want; the timestamps diff --git a/server/internal/transcript/diff.go b/server/internal/transcript/diff.go new file mode 100644 index 0000000..667949f --- /dev/null +++ b/server/internal/transcript/diff.go @@ -0,0 +1,200 @@ +package transcript + +import ( + "sort" + "strings" + "unicode" +) + +/* +Finding the advertising by subtraction. + +A publisher's transcript describes the ad-free master. Speech recognition over the mirrored file +describes the copy the phone will play. The copy is the master with advertising stitched into +it, so the text present in the served copy and absent from the master *is* the advertising, with +its own boundaries. + +This is a measurement rather than a guess, which makes it different in kind from everything else +in this project. The audio scan infers advertising from what several episodes share and checks +itself against the declared-versus-served length gap; the phrase spotter infers it from ad copy +and checks itself against nothing. This one derives the answer from two independent descriptions +of the same recording. + +It needs both halves, which is why a declined publisher transcript is kept rather than thrown +away. Declined means "this copy carries inserted advertising the master does not describe" -- +precisely the episode this works on. +*/ + +// SignalDifference marks a candidate found by subtracting the master from the served copy, +// rather than by reading ad copy. Carried so one episode's candidates can come from either +// detector and still be told apart. +const SignalDifference = "difference" + +const ( + // anchorWords is how long a matching run must be to anchor the two texts together. + // Five: long enough that an ordinary phrase does not match in several places, short enough + // that it still fits between two recognition errors. Recognition is approximate -- whisper + // writes "Bombus" for "Bombas" -- so a window containing a mistake simply fails to anchor + // and its neighbours carry the alignment instead. + anchorWords = 5 + // minInsertedWords is the shortest stretch worth calling an advertisement. A publisher who + // trims a sentence, or recognition that drops a clause, leaves a gap of a few words; + // skipping over one would clip the programme. Forty words is about fifteen seconds of + // speech, the same floor the phrase spotter uses and the same ad unit the length-gap rule + // uses. + minInsertedWords = 40 + // masterDriftWords is how far the master may advance across a gap that is still read as an + // insertion. Inside a real ad pod the master does not advance at all; a few words of drift + // is an anchor lost to recognition error at the edge of the break. + masterDriftWords = 10 +) + +// token is one word of a transcript and where it was said. +type token struct { + word string + startMs int64 + endMs int64 +} + +// Inserted reports the stretches of the served copy that the master does not contain. +// +// Returns nothing when either side is missing: there is nothing to subtract, which is a +// different answer from "no advertising" and the caller sees it as the absence of candidates +// either way. The episode record says which of the two it is. +func Inserted(master, served []Cue) []Candidate { + masterWords := tokenise(master) + servedWords := tokenise(served) + if len(masterWords) < anchorWords || len(servedWords) < anchorWords { + return nil + } + + anchors := alignment(masterWords, servedWords) + + // Walk the gaps between consecutive anchors, including the ones at each end. A gap where + // the served copy advanced a long way and the master did not is inserted content. + // An anchor names the start of a run of [anchorWords] matching words, so the content + // between two anchors begins after the first one's run ends, not after its first word. + // Getting this wrong puts the start of every break one anchor-length early, which on a real + // episode is a skip over the last sentence of the programme. + var found []Candidate + previous := anchor{master: -anchorWords, served: -anchorWords} + for _, next := range append(anchors, anchor{master: len(masterWords), served: len(servedWords)}) { + servedFrom := previous.served + anchorWords + masterFrom := previous.master + anchorWords + servedGap := next.served - servedFrom + masterGap := next.master - masterFrom + if servedGap-masterGap >= minInsertedWords && masterGap <= masterDriftWords { + from := servedWords[servedFrom] + to := servedWords[next.served-1] + found = append(found, Candidate{ + StartMs: from.startMs, + EndMs: to.endMs, + Signals: []string{SignalDifference}, + }) + } + previous = next + } + return found +} + +// anchor ties a position in the master to the position in the served copy saying the same words. +type anchor struct{ master, served int } + +/* +alignment finds where the two texts agree. + +Anchored on runs of [anchorWords] words that occur exactly once in each text. Uniqueness is what +makes an anchor trustworthy: a phrase appearing twice in the master cannot say which occurrence +a served phrase matches, and a wrong anchor drags the alignment across a break. Inside an +advertisement there are no matching runs at all, which is what makes a pod show up as a gap. + +The anchors are then reduced to the longest increasing run, because a handful will still be +spurious -- a stock phrase the publisher and an advertiser both used -- and those point +backwards. +*/ +func alignment(masterWords, servedWords []token) []anchor { + masterRuns := uniqueRuns(masterWords) + servedRuns := uniqueRuns(servedWords) + + var anchors []anchor + for run, servedAt := range servedRuns { + if masterAt, ok := masterRuns[run]; ok { + anchors = append(anchors, anchor{master: masterAt, served: servedAt}) + } + } + sort.Slice(anchors, func(i, j int) bool { return anchors[i].served < anchors[j].served }) + return longestIncreasing(anchors) +} + +// uniqueRuns maps every run of [anchorWords] words that occurs exactly once to where it starts. +func uniqueRuns(words []token) map[string]int { + seen := make(map[string]int, len(words)) + repeated := make(map[string]bool) + for i := 0; i+anchorWords <= len(words); i++ { + parts := make([]string, anchorWords) + for j := 0; j < anchorWords; j++ { + parts[j] = words[i+j].word + } + run := strings.Join(parts, " ") + if _, ok := seen[run]; ok { + repeated[run] = true + continue + } + seen[run] = i + } + for run := range repeated { + delete(seen, run) + } + return seen +} + +// longestIncreasing keeps the longest run of anchors whose master positions increase, dropping +// the ones that would have the alignment going backwards. +func longestIncreasing(anchors []anchor) []anchor { + if len(anchors) == 0 { + return nil + } + // Patience sorting: tails[k] is the index of the anchor ending the best run of length k+1. + var tails []int + previous := make([]int, len(anchors)) + for i, a := range anchors { + previous[i] = -1 + at := sort.Search(len(tails), func(k int) bool { + return anchors[tails[k]].master >= a.master + }) + if at > 0 { + previous[i] = tails[at-1] + } + if at == len(tails) { + tails = append(tails, i) + } else { + tails[at] = i + } + } + + out := make([]anchor, 0, len(tails)) + for i := tails[len(tails)-1]; i >= 0; i = previous[i] { + out = append(out, anchors[i]) + } + for left, right := 0, len(out)-1; left < right; left, right = left+1, right-1 { + out[left], out[right] = out[right], out[left] + } + return out +} + +// tokenise splits cues into comparable words, each carrying the time of the cue it came from. +// +// Case and punctuation are dropped because they are exactly what differs between a publisher's +// written transcript and a machine's reading of the same speech, and neither carries meaning +// for matching. +func tokenise(cues []Cue) []token { + var out []token + for _, cue := range cues { + for _, word := range strings.FieldsFunc(strings.ToLower(cue.Text), notWord) { + out = append(out, token{word: word, startMs: cue.StartMs, endMs: cue.EndMs}) + } + } + return out +} + +func notWord(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) } diff --git a/server/internal/transcript/diff_test.go b/server/internal/transcript/diff_test.go new file mode 100644 index 0000000..e24be51 --- /dev/null +++ b/server/internal/transcript/diff_test.go @@ -0,0 +1,222 @@ +package transcript + +import "testing" + +/* +Finding the advertising by subtraction. + +Peter's idea, and it is the strongest detector in this project because it is a measurement +rather than a guess. A publisher's transcript describes the ad-free master. Speech recognition +over the mirrored file describes the copy the phone will play. The copy is the master with +advertising stitched into it, so the text that is in the served copy and not in the master *is* +the advertising, with its own boundaries. + +It needs both halves, which is why a declined publisher transcript is now kept rather than +thrown away: declined means "this copy carries inserted advertising", which is exactly the +episode this works on. + +The matching has to survive recognition being approximate. Whisper writes "Bombus" where the +publisher wrote "Bombas" and spells numbers differently, so anything demanding exact agreement +finds nothing. Anchoring on runs of words that occur once in each text rides over scattered +errors: a five-word window containing a mistake simply fails to anchor, and its neighbours +still do. +*/ + +// words builds cues of one sentence each, two seconds apart, so a test reads as text. +func words(startMs int64, lines ...string) []Cue { + cues := make([]Cue, 0, len(lines)) + at := startMs + for _, line := range lines { + cues = append(cues, Cue{StartMs: at, EndMs: at + 2000, Text: line}) + at += 2000 + } + return cues +} + +var programme = []string{ + "Good morning and welcome to the programme on a Tuesday in October", + "Our first story today concerns the election and what the polling actually shows", + "We spoke to three analysts about turnout in the suburbs and what it means", + "After the break we will look at the markets and the weather across the country", + "And that is all we have time for this morning thank you for listening", +} + +func TestFindsTheAdvertisingThatIsOnlyInTheServedCopy(t *testing.T) { + master := words(0, programme...) + served := words(0, + programme[0], + programme[1], + programme[2], + "Stay tuned for the following messages from our sponsors", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + "This message comes from a razor company with five blades on every single razor they sell", + programme[3], + programme[4], + ) + + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(got), got) + } + // The four inserted lines begin at 6000 and the programme resumes at 14000. + if got[0].StartMs != 6000 || got[0].EndMs != 14000 { + t.Errorf("got %d-%d, want 6000-14000", got[0].StartMs, got[0].EndMs) + } +} + +/* +Recognition is approximate, and that must not matter. + +Every served line here differs from the master: a misheard brand, a spelled-out number, lost +punctuation. None of it is inserted content and none of it may be reported as advertising. +*/ +func TestRecognitionErrorsAreNotAdvertising(t *testing.T) { + master := words(0, + "Good morning and welcome to the programme on a Tuesday in October", + "Our first story today concerns the election and what the polling actually shows", + "We spoke to three analysts about turnout in the suburbs and what it means", + "After the break we will look at the markets and the weather across the country", + ) + served := words(0, + "Good morning, and welcome to the program on a Tuesday in October.", + "Our first story today concerns the election and what the polling actually showed", + "We spoke to 3 analysts about turnout in the suburbs and what it means", + "After the break, we will look at the markets and the weather across the country.", + ) + + if got := Inserted(master, served); len(got) != 0 { + t.Errorf("got %+v, want nothing -- these are the same words, recognised imperfectly", got) + } +} + +// Advertising at the very start, before any of the programme has been said. +func TestFindsAPreRoll(t *testing.T) { + master := words(0, programme...) + served := words(0, + "Stay tuned for the following messages from our sponsors on this fine Tuesday", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + programme[0], + programme[1], + programme[2], + programme[3], + programme[4], + ) + + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(got), got) + } + if got[0].StartMs != 0 || got[0].EndMs != 6000 { + t.Errorf("got %d-%d, want 0-6000", got[0].StartMs, got[0].EndMs) + } +} + +// And at the end, after the programme has finished. +func TestFindsAPostRoll(t *testing.T) { + master := words(0, programme...) + served := append(words(0, programme...), words(10000, + "Stay tuned for the following messages from our sponsors on this fine Tuesday", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + )...) + + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(got), got) + } + if got[0].StartMs != 10000 { + t.Errorf("starts at %d, want 10000", got[0].StartMs) + } +} + +func TestFindsSeveralBreaksSeparately(t *testing.T) { + master := words(0, programme...) + served := words(0, + programme[0], + "Stay tuned for the following messages from our sponsors on this fine Tuesday", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + programme[1], + programme[2], + "Support for this programme comes from a razor company with five blades on every razor", + "Visit razors dot com slash listen today where terms and conditions will of course apply", + "And a further word from a company that sells boxes for putting other things inside", + programme[3], + programme[4], + ) + + got := Inserted(master, served) + if len(got) != 2 { + t.Fatalf("got %d candidates, want 2: %+v", len(got), got) + } + if got[0].EndMs >= got[1].StartMs { + t.Errorf("the two overlap: %+v", got) + } +} + +/* +A few words' difference is not an advertisement. + +A publisher who trims a sentence, or recognition that drops a clause, leaves a small gap. Being +skipped over it would clip the programme, and advertising is not sold in two-second units. +*/ +func TestASmallDifferenceIsNotABreak(t *testing.T) { + master := words(0, programme...) + served := words(0, + programme[0], + "And one short extra sentence here", + programme[1], + programme[2], + programme[3], + programme[4], + ) + + if got := Inserted(master, served); len(got) != 0 { + t.Errorf("got %+v, want nothing -- one sentence is not an ad break", got) + } +} + +// Nothing to subtract from, and nothing to subtract. +func TestMissingEitherHalfFindsNothing(t *testing.T) { + if got := Inserted(nil, words(0, programme...)); len(got) != 0 { + t.Errorf("got %+v with no master", got) + } + if got := Inserted(words(0, programme...), nil); len(got) != 0 { + t.Errorf("got %+v with no served copy", got) + } +} + +// A served copy identical to the master carries no advertising, which is the case for every +// episode whose publisher transcript was accepted in the first place. +func TestAnIdenticalCopyCarriesNothing(t *testing.T) { + cues := words(0, programme...) + if got := Inserted(cues, cues); len(got) != 0 { + t.Errorf("got %+v, want nothing", got) + } +} + +/* +What was found, in the same shape the phrase spotter produces, so one episode's candidates can +come from either and be read the same way. +*/ +func TestACandidateSaysItCameFromTheDifference(t *testing.T) { + master := words(0, programme...) + served := words(0, + programme[0], programme[1], programme[2], + "Stay tuned for the following messages from our sponsors on this fine Tuesday", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + "This message comes from a razor company with five blades on every single razor they sell", + programme[3], programme[4], + ) + + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates", len(got)) + } + if len(got[0].Signals) != 1 || got[0].Signals[0] != SignalDifference { + t.Errorf("signals = %v, want [%s]", got[0].Signals, SignalDifference) + } +} diff --git a/server/internal/transcript/fits.go b/server/internal/transcript/fits.go index 7597039..b0495d0 100644 --- a/server/internal/transcript/fits.go +++ b/server/internal/transcript/fits.go @@ -21,10 +21,27 @@ const ( // different thing from one refused because it describes another recording: the first is an // episode that needs speech recognition, and the second is a publisher's mistake. type Verdict struct { - OK bool - Why string + OK bool + Why string + Reason Reason } +// Reason says which kind of refusal this is, so a caller can act on it. Read rather than +// matched against the wording of Why, which is for people. +type Reason string + +const ( + // ReasonInserted: the file carries advertising the publisher's master does not describe. + // The transcript is still the master, and subtracting it from a recognised reading of the + // served file is what locates that advertising. See Inserted. + ReasonInserted Reason = "inserted" + // ReasonOtherRecording: the transcript describes something that is not this file at all, + // so it is no use for anything. + ReasonOtherRecording Reason = "other-recording" + // ReasonEmpty: there were no cues to judge. + ReasonEmpty Reason = "empty" +) + // Fits reports whether a transcript ending at lastCueEndMs describes a mirrored file of // servedMs, given the declaredMs the feed claims. // @@ -35,18 +52,18 @@ type Verdict struct { // difference is nothing. func Fits(lastCueEndMs, servedMs, declaredMs int64) Verdict { if lastCueEndMs <= 0 { - return Verdict{Why: "the transcript has no cues"} + return Verdict{Why: "the transcript has no cues", Reason: ReasonEmpty} } if lastCueEndMs > servedMs { return Verdict{Why: fmt.Sprintf( "the transcript runs %s past the end of the file, so it describes another recording", - clock(lastCueEndMs-servedMs))} + clock(lastCueEndMs-servedMs)), Reason: ReasonOtherRecording} } if declaredMs > 0 { if gap := servedMs - declaredMs; gap > insertionMs { return Verdict{Why: fmt.Sprintf( "this copy carries %s of inserted advertising the transcript does not describe", - clock(gap))} + clock(gap)), Reason: ReasonInserted} } return Verdict{OK: true} } @@ -54,7 +71,7 @@ func Fits(lastCueEndMs, servedMs, declaredMs int64) Verdict { if gap := servedMs - lastCueEndMs; gap > tailMs { return Verdict{Why: fmt.Sprintf( "the transcript stops %s before the file does, and the feed declares no length", - clock(gap))} + clock(gap)), Reason: ReasonInserted} } return Verdict{OK: true} } diff --git a/server/internal/transcript/harvest.go b/server/internal/transcript/harvest.go index 64130fd..36befd8 100644 --- a/server/internal/transcript/harvest.go +++ b/server/internal/transcript/harvest.go @@ -32,6 +32,11 @@ type Result struct { Cues int // Ads is how many stretches of the stored transcript read as advertising. Candidates only. Ads int + // Master is how many cues were kept as the publisher's ad-free transcript. Non-zero only + // when the transcript was declined for this copy because advertising was inserted into it: + // the words are then the programme without the advertising, which is what [Inserted] + // subtracts from a recognised reading of the served file. + Master int // Note is why nothing was stored, and is empty when something was. It is a verdict about // this copy of the episode, so it is recorded rather than retried -- and for an episode // carrying inserted advertising it is also the signal that this is one recognition has to do. @@ -75,7 +80,16 @@ func (h Harvester) Harvest(ctx context.Context, offer Offer) (Result, error) { } if verdict := Fits(LastCueEndMs(cues), offer.ServedMs, offer.DeclaredMs); !verdict.OK { - return Result{Note: verdict.Why}, nil + if verdict.Reason != ReasonInserted { + // A transcript of some other recording is no use for anything, including + // subtraction. + return Result{Note: verdict.Why}, nil + } + master := Transcript{EpisodeID: offer.EpisodeID, Source: SourcePublished, Cues: cues} + if err := h.Store.PutMaster(master); err != nil { + return Result{}, err + } + return Result{Note: verdict.Why, Master: len(cues)}, nil } stored := Transcript{ diff --git a/server/internal/transcript/harvest_test.go b/server/internal/transcript/harvest_test.go index 25327ae..768d11f 100644 --- a/server/internal/transcript/harvest_test.go +++ b/server/internal/transcript/harvest_test.go @@ -200,3 +200,98 @@ Right, where were we? t.Errorf("stored ads = %+v", held.Ads) } } + +/* +A declined transcript is kept as the master. + +Declined means "this copy carries inserted advertising the publisher's master does not +describe", which is the one case where the publisher's words are worth more than usual: they +are the programme without the advertising, and subtracting them from a recognised reading of +the served file says exactly where that advertising is. + +It is still not the episode's transcript. Reading along with it would be minutes out of step. +*/ +func TestADeclinedTranscriptIsKeptAsTheMaster(t *testing.T) { + server := serving(t, 200, harvestSRT, "application/srt") + store := Store{Dir: t.TempDir()} + harvester := Harvester{Store: store, Client: loopbackClient()} + + got, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", URL: server.URL, Type: "application/srt", + ServedMs: 1_480_000, DeclaredMs: 1_216_000, + }) + if err != nil { + t.Fatalf("Harvest: %v", err) + } + if got.Stored { + t.Error("stored it as the episode's transcript") + } + if got.Master != 2 { + t.Errorf("Master = %d, want 2 cues kept", got.Master) + } + if !store.HasMaster("ep1") { + t.Fatal("the master was not kept") + } + master, err := store.Master("ep1") + if err != nil { + t.Fatalf("Master: %v", err) + } + if master.EpisodeID != "ep1" || len(master.Cues) != 2 { + t.Errorf("got %+v", master) + } + if store.Has("ep1") { + t.Error("the declined transcript was also stored as the episode's own") + } +} + +// A transcript of a different recording is no use for subtraction either. +func TestATranscriptOfAnotherRecordingIsNotKept(t *testing.T) { + server := serving(t, 200, harvestSRT, "application/srt") + store := Store{Dir: t.TempDir()} + harvester := Harvester{Store: store, Client: loopbackClient()} + + // The transcript outlasts the file, so it describes something else. + if _, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", URL: server.URL, Type: "application/srt", + ServedMs: 600_000, DeclaredMs: 600_000, + }); err != nil { + t.Fatalf("Harvest: %v", err) + } + + if store.HasMaster("ep1") { + t.Error("kept a transcript of a different recording as the master") + } +} + +// An accepted transcript needs no master: it is the same text, and the episode carries no +// inserted advertising to subtract anything from. +func TestAnAcceptedTranscriptIsNotAlsoKeptAsAMaster(t *testing.T) { + server := serving(t, 200, harvestSRT, "application/srt") + store := Store{Dir: t.TempDir()} + harvester := Harvester{Store: store, Client: loopbackClient()} + + if _, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", URL: server.URL, Type: "application/srt", + ServedMs: 1_217_000, DeclaredMs: 1_216_000, + }); err != nil { + t.Fatalf("Harvest: %v", err) + } + + if store.HasMaster("ep1") { + t.Error("kept a redundant master") + } +} + +// Pruning an episode takes the master with it: it is evidence about a file nobody holds. +func TestPruningTakesTheMasterToo(t *testing.T) { + store := Store{Dir: t.TempDir()} + if err := store.PutMaster(Transcript{EpisodeID: "ep1", Cues: []Cue{{Text: "x", EndMs: 1}}}); err != nil { + t.Fatalf("PutMaster: %v", err) + } + if err := store.Prune("ep1"); err != nil { + t.Fatalf("Prune: %v", err) + } + if store.HasMaster("ep1") { + t.Error("the master outlived the episode") + } +} diff --git a/server/internal/transcript/store.go b/server/internal/transcript/store.go index 8cf9419..9450b3a 100644 --- a/server/internal/transcript/store.go +++ b/server/internal/transcript/store.go @@ -46,6 +46,15 @@ func (t Transcript) LastEndMs() int64 { return LastCueEndMs(t.Cues) } // tap the listener makes. type Store struct{ Dir string } +// masterSuffix names the file holding the publisher's own transcript of the ad-free programme, +// kept beside the one the phone reads. +// +// Two files rather than two fields in one, because they answer different questions and have +// different lifetimes: one is what to show somebody reading along, the other is evidence used +// to work out where the advertising is. For an episode with no inserted advertising they would +// be the same text, and only the first is written. +const masterSuffix = ".master" + func (s Store) path(episodeID string) (string, error) { // The id decides a filename, and it arrives from the state file, so it is checked rather // than trusted: anything with a separator or a dot segment in it could write elsewhere. @@ -94,6 +103,34 @@ func (s Store) Put(t Transcript) error { return nil } +// PutMaster stores the publisher's transcript of the ad-free programme. +// +// Kept even when it cannot be used as the episode's transcript -- especially then. A transcript +// declined because this copy carries inserted advertising is the master that advertising was +// inserted into, and subtracting it from a recognised reading of the served file is what says +// where. See Inserted. +func (s Store) PutMaster(t Transcript) error { + return s.Put(Transcript{ + EpisodeID: t.EpisodeID + masterSuffix, + Source: SourcePublished, + LanguageCode: t.LanguageCode, + Cues: t.Cues, + }) +} + +// Master reads the publisher's transcript of the ad-free programme. +func (s Store) Master(episodeID string) (Transcript, error) { + held, err := s.Get(episodeID + masterSuffix) + if err != nil { + return Transcript{}, err + } + held.EpisodeID = episodeID + return held, nil +} + +// HasMaster reports whether the publisher's ad-free transcript is on disk. +func (s Store) HasMaster(episodeID string) bool { return s.Has(episodeID + masterSuffix) } + // Get reads an episode's transcript. func (s Store) Get(episodeID string) (Transcript, error) { name, err := s.path(episodeID) @@ -131,5 +168,11 @@ func (s Store) Prune(episodeID string) error { if err := os.Remove(name); err != nil && !os.IsNotExist(err) { return fmt.Errorf("transcript: %w", err) } + // The master goes with it: it is evidence about a file that is no longer held. + if master, err := s.path(episodeID + masterSuffix); err == nil { + if err := os.Remove(master); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("transcript: %w", err) + } + } return nil } |