From 044f5099137faabda40589441987c5ff3d713a0b Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Thu, 29 Oct 2026 08:16:58 +0000 Subject: Find the advertising by subtracting the publisher's master Peter's idea, and it is the best detector in this project because it is a measurement rather than a guess. A publisher's transcript describes the ad-free master. Recognition over the mirrored file describes the copy the phone will play. The copy is the master with advertising stitched into it, so the text present in the served copy and absent from the master *is* the advertising, with its own boundaries. No phrase list to be wrong about, and no show whose hosts happen to talk like an advert. Measured on Darknet Diaries' "Conti" before it was wired in. The feed declares 59:17, the file served runs 63:33, so 4:15 was inserted. Subtraction found **4:20 in three pods -- 102%** against a figure no detector produced. Reading the ad copy on the same words found 0:16, or 6%. Every pod starts on the first word of an ad read and ends on the last: a NetSuite pre-roll, a 2:20 midroll, and a Doppel read at 32:08. Nothing of the programme is clipped. The alignment has to survive recognition being approximate -- whisper writes "Bombus" for "Bombas" and spells numbers differently -- so it anchors on runs of five words occurring exactly once in each text. A window containing a mistake simply fails to anchor and its neighbours carry the alignment instead. Uniqueness is what makes an anchor safe: a phrase appearing twice in the master cannot say which occurrence it matches, and a wrong anchor drags the alignment across a break. Inside an advertisement nothing matches at all, which is exactly what makes a pod show up as a gap. An anchor names the *start* of a matching run, so the content between two anchors begins after the first run ends. Getting that wrong put every break one anchor-length early, which on a real episode is a skip over the last sentence of the programme. The tests pinned it. **This is why a declined transcript is now kept.** Declined means "this copy carries inserted advertising the master does not describe" -- precisely the episode subtraction works on. One declined for describing a different recording is still thrown away, since it is no use for anything, and Fits now carries a reason rather than making callers match on its wording. The nineteen episodes declined before this existed fetch their masters once on the next poll. Nothing is published as a skip yet. The candidates are stored with the words and dimmed on the transcript screen, and every episode now logs what was found against what the lengths imply. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- server/internal/store/store.go | 17 +++ server/internal/transcribe/whisper.go | 4 + server/internal/transcript/diff.go | 200 ++++++++++++++++++++++++++ server/internal/transcript/diff_test.go | 222 +++++++++++++++++++++++++++++ server/internal/transcript/fits.go | 29 +++- server/internal/transcript/harvest.go | 16 ++- server/internal/transcript/harvest_test.go | 95 ++++++++++++ server/internal/transcript/store.go | 43 ++++++ 8 files changed, 619 insertions(+), 7 deletions(-) create mode 100644 server/internal/transcript/diff.go create mode 100644 server/internal/transcript/diff_test.go (limited to 'server/internal') diff --git a/server/internal/store/store.go b/server/internal/store/store.go index 3c44060..7f81bd5 100644 --- a/server/internal/store/store.go +++ b/server/internal/store/store.go @@ -68,6 +68,12 @@ type Episode struct { TranscriptCues int `json:"transcriptCues,omitempty"` // TranscriptAtUnix is when the transcript landed. TranscriptAtUnix int64 `json:"transcriptAtUnix,omitempty"` + // MasterCues is how many cues the publisher's ad-free transcript holds, kept for episodes + // whose served copy carries inserted advertising. MasterNote is why there is none. Three + // states as everywhere else: words, a reason, or nobody has looked. + MasterCues int `json:"masterCues,omitempty"` + MasterNote string `json:"masterNote,omitempty"` + // TranscribeAttemptedAtUnix is when speech recognition last ran over this episode, whether // it worked or not, so a failure is not retried on every poll for ever. Recorded only when // a run actually finished: one interrupted by a restart is not an attempt, or a redeploy @@ -93,6 +99,17 @@ func (e *Episode) TranscriptExamined() bool { return e.HasTranscript() || e.TranscriptNote != "" } +// NeedsMaster reports whether the publisher's ad-free transcript should still be fetched. +// +// Only for an episode whose own transcript was declined: that is the one case where the +// publisher's words describe a different recording from the file being played, and so the one +// case where subtracting them locates the advertising. An episode that was never offered a +// transcript has nothing to fetch. +func (e *Episode) NeedsMaster() bool { + return e.Mirrored() && e.TranscriptNote != "" && e.TranscriptURL != "" && + e.MasterCues == 0 && e.MasterNote == "" +} + // NeedsRecognising reports whether speech recognition should be run over this episode. // // Deliberately not the negation of [Episode.TranscriptExamined]. An episode whose publisher's diff --git a/server/internal/transcribe/whisper.go b/server/internal/transcribe/whisper.go index c6e4f83..1789beb 100644 --- a/server/internal/transcribe/whisper.go +++ b/server/internal/transcribe/whisper.go @@ -169,6 +169,10 @@ type whisperOutput struct { } `json:"transcription"` } +// ParseOutput reads whisper-cli's output into cues. Exported for tools that work from a file +// whisper wrote earlier rather than running it. +func ParseOutput(raw []byte) (Result, error) { return parseWhisperJSON(raw) } + // parseWhisperJSON reads whisper-cli's output into cues. // // The offsets are milliseconds, which is what both the app and the ad scan want; the timestamps diff --git a/server/internal/transcript/diff.go b/server/internal/transcript/diff.go new file mode 100644 index 0000000..667949f --- /dev/null +++ b/server/internal/transcript/diff.go @@ -0,0 +1,200 @@ +package transcript + +import ( + "sort" + "strings" + "unicode" +) + +/* +Finding the advertising by subtraction. + +A publisher's transcript describes the ad-free master. Speech recognition over the mirrored file +describes the copy the phone will play. The copy is the master with advertising stitched into +it, so the text present in the served copy and absent from the master *is* the advertising, with +its own boundaries. + +This is a measurement rather than a guess, which makes it different in kind from everything else +in this project. The audio scan infers advertising from what several episodes share and checks +itself against the declared-versus-served length gap; the phrase spotter infers it from ad copy +and checks itself against nothing. This one derives the answer from two independent descriptions +of the same recording. + +It needs both halves, which is why a declined publisher transcript is kept rather than thrown +away. Declined means "this copy carries inserted advertising the master does not describe" -- +precisely the episode this works on. +*/ + +// SignalDifference marks a candidate found by subtracting the master from the served copy, +// rather than by reading ad copy. Carried so one episode's candidates can come from either +// detector and still be told apart. +const SignalDifference = "difference" + +const ( + // anchorWords is how long a matching run must be to anchor the two texts together. + // Five: long enough that an ordinary phrase does not match in several places, short enough + // that it still fits between two recognition errors. Recognition is approximate -- whisper + // writes "Bombus" for "Bombas" -- so a window containing a mistake simply fails to anchor + // and its neighbours carry the alignment instead. + anchorWords = 5 + // minInsertedWords is the shortest stretch worth calling an advertisement. A publisher who + // trims a sentence, or recognition that drops a clause, leaves a gap of a few words; + // skipping over one would clip the programme. Forty words is about fifteen seconds of + // speech, the same floor the phrase spotter uses and the same ad unit the length-gap rule + // uses. + minInsertedWords = 40 + // masterDriftWords is how far the master may advance across a gap that is still read as an + // insertion. Inside a real ad pod the master does not advance at all; a few words of drift + // is an anchor lost to recognition error at the edge of the break. + masterDriftWords = 10 +) + +// token is one word of a transcript and where it was said. +type token struct { + word string + startMs int64 + endMs int64 +} + +// Inserted reports the stretches of the served copy that the master does not contain. +// +// Returns nothing when either side is missing: there is nothing to subtract, which is a +// different answer from "no advertising" and the caller sees it as the absence of candidates +// either way. The episode record says which of the two it is. +func Inserted(master, served []Cue) []Candidate { + masterWords := tokenise(master) + servedWords := tokenise(served) + if len(masterWords) < anchorWords || len(servedWords) < anchorWords { + return nil + } + + anchors := alignment(masterWords, servedWords) + + // Walk the gaps between consecutive anchors, including the ones at each end. A gap where + // the served copy advanced a long way and the master did not is inserted content. + // An anchor names the start of a run of [anchorWords] matching words, so the content + // between two anchors begins after the first one's run ends, not after its first word. + // Getting this wrong puts the start of every break one anchor-length early, which on a real + // episode is a skip over the last sentence of the programme. + var found []Candidate + previous := anchor{master: -anchorWords, served: -anchorWords} + for _, next := range append(anchors, anchor{master: len(masterWords), served: len(servedWords)}) { + servedFrom := previous.served + anchorWords + masterFrom := previous.master + anchorWords + servedGap := next.served - servedFrom + masterGap := next.master - masterFrom + if servedGap-masterGap >= minInsertedWords && masterGap <= masterDriftWords { + from := servedWords[servedFrom] + to := servedWords[next.served-1] + found = append(found, Candidate{ + StartMs: from.startMs, + EndMs: to.endMs, + Signals: []string{SignalDifference}, + }) + } + previous = next + } + return found +} + +// anchor ties a position in the master to the position in the served copy saying the same words. +type anchor struct{ master, served int } + +/* +alignment finds where the two texts agree. + +Anchored on runs of [anchorWords] words that occur exactly once in each text. Uniqueness is what +makes an anchor trustworthy: a phrase appearing twice in the master cannot say which occurrence +a served phrase matches, and a wrong anchor drags the alignment across a break. Inside an +advertisement there are no matching runs at all, which is what makes a pod show up as a gap. + +The anchors are then reduced to the longest increasing run, because a handful will still be +spurious -- a stock phrase the publisher and an advertiser both used -- and those point +backwards. +*/ +func alignment(masterWords, servedWords []token) []anchor { + masterRuns := uniqueRuns(masterWords) + servedRuns := uniqueRuns(servedWords) + + var anchors []anchor + for run, servedAt := range servedRuns { + if masterAt, ok := masterRuns[run]; ok { + anchors = append(anchors, anchor{master: masterAt, served: servedAt}) + } + } + sort.Slice(anchors, func(i, j int) bool { return anchors[i].served < anchors[j].served }) + return longestIncreasing(anchors) +} + +// uniqueRuns maps every run of [anchorWords] words that occurs exactly once to where it starts. +func uniqueRuns(words []token) map[string]int { + seen := make(map[string]int, len(words)) + repeated := make(map[string]bool) + for i := 0; i+anchorWords <= len(words); i++ { + parts := make([]string, anchorWords) + for j := 0; j < anchorWords; j++ { + parts[j] = words[i+j].word + } + run := strings.Join(parts, " ") + if _, ok := seen[run]; ok { + repeated[run] = true + continue + } + seen[run] = i + } + for run := range repeated { + delete(seen, run) + } + return seen +} + +// longestIncreasing keeps the longest run of anchors whose master positions increase, dropping +// the ones that would have the alignment going backwards. +func longestIncreasing(anchors []anchor) []anchor { + if len(anchors) == 0 { + return nil + } + // Patience sorting: tails[k] is the index of the anchor ending the best run of length k+1. + var tails []int + previous := make([]int, len(anchors)) + for i, a := range anchors { + previous[i] = -1 + at := sort.Search(len(tails), func(k int) bool { + return anchors[tails[k]].master >= a.master + }) + if at > 0 { + previous[i] = tails[at-1] + } + if at == len(tails) { + tails = append(tails, i) + } else { + tails[at] = i + } + } + + out := make([]anchor, 0, len(tails)) + for i := tails[len(tails)-1]; i >= 0; i = previous[i] { + out = append(out, anchors[i]) + } + for left, right := 0, len(out)-1; left < right; left, right = left+1, right-1 { + out[left], out[right] = out[right], out[left] + } + return out +} + +// tokenise splits cues into comparable words, each carrying the time of the cue it came from. +// +// Case and punctuation are dropped because they are exactly what differs between a publisher's +// written transcript and a machine's reading of the same speech, and neither carries meaning +// for matching. +func tokenise(cues []Cue) []token { + var out []token + for _, cue := range cues { + for _, word := range strings.FieldsFunc(strings.ToLower(cue.Text), notWord) { + out = append(out, token{word: word, startMs: cue.StartMs, endMs: cue.EndMs}) + } + } + return out +} + +func notWord(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) } diff --git a/server/internal/transcript/diff_test.go b/server/internal/transcript/diff_test.go new file mode 100644 index 0000000..e24be51 --- /dev/null +++ b/server/internal/transcript/diff_test.go @@ -0,0 +1,222 @@ +package transcript + +import "testing" + +/* +Finding the advertising by subtraction. + +Peter's idea, and it is the strongest detector in this project because it is a measurement +rather than a guess. A publisher's transcript describes the ad-free master. Speech recognition +over the mirrored file describes the copy the phone will play. The copy is the master with +advertising stitched into it, so the text that is in the served copy and not in the master *is* +the advertising, with its own boundaries. + +It needs both halves, which is why a declined publisher transcript is now kept rather than +thrown away: declined means "this copy carries inserted advertising", which is exactly the +episode this works on. + +The matching has to survive recognition being approximate. Whisper writes "Bombus" where the +publisher wrote "Bombas" and spells numbers differently, so anything demanding exact agreement +finds nothing. Anchoring on runs of words that occur once in each text rides over scattered +errors: a five-word window containing a mistake simply fails to anchor, and its neighbours +still do. +*/ + +// words builds cues of one sentence each, two seconds apart, so a test reads as text. +func words(startMs int64, lines ...string) []Cue { + cues := make([]Cue, 0, len(lines)) + at := startMs + for _, line := range lines { + cues = append(cues, Cue{StartMs: at, EndMs: at + 2000, Text: line}) + at += 2000 + } + return cues +} + +var programme = []string{ + "Good morning and welcome to the programme on a Tuesday in October", + "Our first story today concerns the election and what the polling actually shows", + "We spoke to three analysts about turnout in the suburbs and what it means", + "After the break we will look at the markets and the weather across the country", + "And that is all we have time for this morning thank you for listening", +} + +func TestFindsTheAdvertisingThatIsOnlyInTheServedCopy(t *testing.T) { + master := words(0, programme...) + served := words(0, + programme[0], + programme[1], + programme[2], + "Stay tuned for the following messages from our sponsors", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + "This message comes from a razor company with five blades on every single razor they sell", + programme[3], + programme[4], + ) + + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(got), got) + } + // The four inserted lines begin at 6000 and the programme resumes at 14000. + if got[0].StartMs != 6000 || got[0].EndMs != 14000 { + t.Errorf("got %d-%d, want 6000-14000", got[0].StartMs, got[0].EndMs) + } +} + +/* +Recognition is approximate, and that must not matter. + +Every served line here differs from the master: a misheard brand, a spelled-out number, lost +punctuation. None of it is inserted content and none of it may be reported as advertising. +*/ +func TestRecognitionErrorsAreNotAdvertising(t *testing.T) { + master := words(0, + "Good morning and welcome to the programme on a Tuesday in October", + "Our first story today concerns the election and what the polling actually shows", + "We spoke to three analysts about turnout in the suburbs and what it means", + "After the break we will look at the markets and the weather across the country", + ) + served := words(0, + "Good morning, and welcome to the program on a Tuesday in October.", + "Our first story today concerns the election and what the polling actually showed", + "We spoke to 3 analysts about turnout in the suburbs and what it means", + "After the break, we will look at the markets and the weather across the country.", + ) + + if got := Inserted(master, served); len(got) != 0 { + t.Errorf("got %+v, want nothing -- these are the same words, recognised imperfectly", got) + } +} + +// Advertising at the very start, before any of the programme has been said. +func TestFindsAPreRoll(t *testing.T) { + master := words(0, programme...) + served := words(0, + "Stay tuned for the following messages from our sponsors on this fine Tuesday", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + programme[0], + programme[1], + programme[2], + programme[3], + programme[4], + ) + + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(got), got) + } + if got[0].StartMs != 0 || got[0].EndMs != 6000 { + t.Errorf("got %d-%d, want 0-6000", got[0].StartMs, got[0].EndMs) + } +} + +// And at the end, after the programme has finished. +func TestFindsAPostRoll(t *testing.T) { + master := words(0, programme...) + served := append(words(0, programme...), words(10000, + "Stay tuned for the following messages from our sponsors on this fine Tuesday", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + )...) + + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates, want 1: %+v", len(got), got) + } + if got[0].StartMs != 10000 { + t.Errorf("starts at %d, want 10000", got[0].StartMs) + } +} + +func TestFindsSeveralBreaksSeparately(t *testing.T) { + master := words(0, programme...) + served := words(0, + programme[0], + "Stay tuned for the following messages from our sponsors on this fine Tuesday", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + programme[1], + programme[2], + "Support for this programme comes from a razor company with five blades on every razor", + "Visit razors dot com slash listen today where terms and conditions will of course apply", + "And a further word from a company that sells boxes for putting other things inside", + programme[3], + programme[4], + ) + + got := Inserted(master, served) + if len(got) != 2 { + t.Fatalf("got %d candidates, want 2: %+v", len(got), got) + } + if got[0].EndMs >= got[1].StartMs { + t.Errorf("the two overlap: %+v", got) + } +} + +/* +A few words' difference is not an advertisement. + +A publisher who trims a sentence, or recognition that drops a clause, leaves a small gap. Being +skipped over it would clip the programme, and advertising is not sold in two-second units. +*/ +func TestASmallDifferenceIsNotABreak(t *testing.T) { + master := words(0, programme...) + served := words(0, + programme[0], + "And one short extra sentence here", + programme[1], + programme[2], + programme[3], + programme[4], + ) + + if got := Inserted(master, served); len(got) != 0 { + t.Errorf("got %+v, want nothing -- one sentence is not an ad break", got) + } +} + +// Nothing to subtract from, and nothing to subtract. +func TestMissingEitherHalfFindsNothing(t *testing.T) { + if got := Inserted(nil, words(0, programme...)); len(got) != 0 { + t.Errorf("got %+v with no master", got) + } + if got := Inserted(words(0, programme...), nil); len(got) != 0 { + t.Errorf("got %+v with no served copy", got) + } +} + +// A served copy identical to the master carries no advertising, which is the case for every +// episode whose publisher transcript was accepted in the first place. +func TestAnIdenticalCopyCarriesNothing(t *testing.T) { + cues := words(0, programme...) + if got := Inserted(cues, cues); len(got) != 0 { + t.Errorf("got %+v, want nothing", got) + } +} + +/* +What was found, in the same shape the phrase spotter produces, so one episode's candidates can +come from either and be read the same way. +*/ +func TestACandidateSaysItCameFromTheDifference(t *testing.T) { + master := words(0, programme...) + served := words(0, + programme[0], programme[1], programme[2], + "Stay tuned for the following messages from our sponsors on this fine Tuesday", + "This message comes from a mattress company who have been making mattresses for forty years", + "Go to mattress dot com slash listen and use the promotional code for twenty percent off", + "This message comes from a razor company with five blades on every single razor they sell", + programme[3], programme[4], + ) + + got := Inserted(master, served) + if len(got) != 1 { + t.Fatalf("got %d candidates", len(got)) + } + if len(got[0].Signals) != 1 || got[0].Signals[0] != SignalDifference { + t.Errorf("signals = %v, want [%s]", got[0].Signals, SignalDifference) + } +} diff --git a/server/internal/transcript/fits.go b/server/internal/transcript/fits.go index 7597039..b0495d0 100644 --- a/server/internal/transcript/fits.go +++ b/server/internal/transcript/fits.go @@ -21,10 +21,27 @@ const ( // different thing from one refused because it describes another recording: the first is an // episode that needs speech recognition, and the second is a publisher's mistake. type Verdict struct { - OK bool - Why string + OK bool + Why string + Reason Reason } +// Reason says which kind of refusal this is, so a caller can act on it. Read rather than +// matched against the wording of Why, which is for people. +type Reason string + +const ( + // ReasonInserted: the file carries advertising the publisher's master does not describe. + // The transcript is still the master, and subtracting it from a recognised reading of the + // served file is what locates that advertising. See Inserted. + ReasonInserted Reason = "inserted" + // ReasonOtherRecording: the transcript describes something that is not this file at all, + // so it is no use for anything. + ReasonOtherRecording Reason = "other-recording" + // ReasonEmpty: there were no cues to judge. + ReasonEmpty Reason = "empty" +) + // Fits reports whether a transcript ending at lastCueEndMs describes a mirrored file of // servedMs, given the declaredMs the feed claims. // @@ -35,18 +52,18 @@ type Verdict struct { // difference is nothing. func Fits(lastCueEndMs, servedMs, declaredMs int64) Verdict { if lastCueEndMs <= 0 { - return Verdict{Why: "the transcript has no cues"} + return Verdict{Why: "the transcript has no cues", Reason: ReasonEmpty} } if lastCueEndMs > servedMs { return Verdict{Why: fmt.Sprintf( "the transcript runs %s past the end of the file, so it describes another recording", - clock(lastCueEndMs-servedMs))} + clock(lastCueEndMs-servedMs)), Reason: ReasonOtherRecording} } if declaredMs > 0 { if gap := servedMs - declaredMs; gap > insertionMs { return Verdict{Why: fmt.Sprintf( "this copy carries %s of inserted advertising the transcript does not describe", - clock(gap))} + clock(gap)), Reason: ReasonInserted} } return Verdict{OK: true} } @@ -54,7 +71,7 @@ func Fits(lastCueEndMs, servedMs, declaredMs int64) Verdict { if gap := servedMs - lastCueEndMs; gap > tailMs { return Verdict{Why: fmt.Sprintf( "the transcript stops %s before the file does, and the feed declares no length", - clock(gap))} + clock(gap)), Reason: ReasonInserted} } return Verdict{OK: true} } diff --git a/server/internal/transcript/harvest.go b/server/internal/transcript/harvest.go index 64130fd..36befd8 100644 --- a/server/internal/transcript/harvest.go +++ b/server/internal/transcript/harvest.go @@ -32,6 +32,11 @@ type Result struct { Cues int // Ads is how many stretches of the stored transcript read as advertising. Candidates only. Ads int + // Master is how many cues were kept as the publisher's ad-free transcript. Non-zero only + // when the transcript was declined for this copy because advertising was inserted into it: + // the words are then the programme without the advertising, which is what [Inserted] + // subtracts from a recognised reading of the served file. + Master int // Note is why nothing was stored, and is empty when something was. It is a verdict about // this copy of the episode, so it is recorded rather than retried -- and for an episode // carrying inserted advertising it is also the signal that this is one recognition has to do. @@ -75,7 +80,16 @@ func (h Harvester) Harvest(ctx context.Context, offer Offer) (Result, error) { } if verdict := Fits(LastCueEndMs(cues), offer.ServedMs, offer.DeclaredMs); !verdict.OK { - return Result{Note: verdict.Why}, nil + if verdict.Reason != ReasonInserted { + // A transcript of some other recording is no use for anything, including + // subtraction. + return Result{Note: verdict.Why}, nil + } + master := Transcript{EpisodeID: offer.EpisodeID, Source: SourcePublished, Cues: cues} + if err := h.Store.PutMaster(master); err != nil { + return Result{}, err + } + return Result{Note: verdict.Why, Master: len(cues)}, nil } stored := Transcript{ diff --git a/server/internal/transcript/harvest_test.go b/server/internal/transcript/harvest_test.go index 25327ae..768d11f 100644 --- a/server/internal/transcript/harvest_test.go +++ b/server/internal/transcript/harvest_test.go @@ -200,3 +200,98 @@ Right, where were we? t.Errorf("stored ads = %+v", held.Ads) } } + +/* +A declined transcript is kept as the master. + +Declined means "this copy carries inserted advertising the publisher's master does not +describe", which is the one case where the publisher's words are worth more than usual: they +are the programme without the advertising, and subtracting them from a recognised reading of +the served file says exactly where that advertising is. + +It is still not the episode's transcript. Reading along with it would be minutes out of step. +*/ +func TestADeclinedTranscriptIsKeptAsTheMaster(t *testing.T) { + server := serving(t, 200, harvestSRT, "application/srt") + store := Store{Dir: t.TempDir()} + harvester := Harvester{Store: store, Client: loopbackClient()} + + got, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", URL: server.URL, Type: "application/srt", + ServedMs: 1_480_000, DeclaredMs: 1_216_000, + }) + if err != nil { + t.Fatalf("Harvest: %v", err) + } + if got.Stored { + t.Error("stored it as the episode's transcript") + } + if got.Master != 2 { + t.Errorf("Master = %d, want 2 cues kept", got.Master) + } + if !store.HasMaster("ep1") { + t.Fatal("the master was not kept") + } + master, err := store.Master("ep1") + if err != nil { + t.Fatalf("Master: %v", err) + } + if master.EpisodeID != "ep1" || len(master.Cues) != 2 { + t.Errorf("got %+v", master) + } + if store.Has("ep1") { + t.Error("the declined transcript was also stored as the episode's own") + } +} + +// A transcript of a different recording is no use for subtraction either. +func TestATranscriptOfAnotherRecordingIsNotKept(t *testing.T) { + server := serving(t, 200, harvestSRT, "application/srt") + store := Store{Dir: t.TempDir()} + harvester := Harvester{Store: store, Client: loopbackClient()} + + // The transcript outlasts the file, so it describes something else. + if _, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", URL: server.URL, Type: "application/srt", + ServedMs: 600_000, DeclaredMs: 600_000, + }); err != nil { + t.Fatalf("Harvest: %v", err) + } + + if store.HasMaster("ep1") { + t.Error("kept a transcript of a different recording as the master") + } +} + +// An accepted transcript needs no master: it is the same text, and the episode carries no +// inserted advertising to subtract anything from. +func TestAnAcceptedTranscriptIsNotAlsoKeptAsAMaster(t *testing.T) { + server := serving(t, 200, harvestSRT, "application/srt") + store := Store{Dir: t.TempDir()} + harvester := Harvester{Store: store, Client: loopbackClient()} + + if _, err := harvester.Harvest(context.Background(), Offer{ + EpisodeID: "ep1", URL: server.URL, Type: "application/srt", + ServedMs: 1_217_000, DeclaredMs: 1_216_000, + }); err != nil { + t.Fatalf("Harvest: %v", err) + } + + if store.HasMaster("ep1") { + t.Error("kept a redundant master") + } +} + +// Pruning an episode takes the master with it: it is evidence about a file nobody holds. +func TestPruningTakesTheMasterToo(t *testing.T) { + store := Store{Dir: t.TempDir()} + if err := store.PutMaster(Transcript{EpisodeID: "ep1", Cues: []Cue{{Text: "x", EndMs: 1}}}); err != nil { + t.Fatalf("PutMaster: %v", err) + } + if err := store.Prune("ep1"); err != nil { + t.Fatalf("Prune: %v", err) + } + if store.HasMaster("ep1") { + t.Error("the master outlived the episode") + } +} diff --git a/server/internal/transcript/store.go b/server/internal/transcript/store.go index 8cf9419..9450b3a 100644 --- a/server/internal/transcript/store.go +++ b/server/internal/transcript/store.go @@ -46,6 +46,15 @@ func (t Transcript) LastEndMs() int64 { return LastCueEndMs(t.Cues) } // tap the listener makes. type Store struct{ Dir string } +// masterSuffix names the file holding the publisher's own transcript of the ad-free programme, +// kept beside the one the phone reads. +// +// Two files rather than two fields in one, because they answer different questions and have +// different lifetimes: one is what to show somebody reading along, the other is evidence used +// to work out where the advertising is. For an episode with no inserted advertising they would +// be the same text, and only the first is written. +const masterSuffix = ".master" + func (s Store) path(episodeID string) (string, error) { // The id decides a filename, and it arrives from the state file, so it is checked rather // than trusted: anything with a separator or a dot segment in it could write elsewhere. @@ -94,6 +103,34 @@ func (s Store) Put(t Transcript) error { return nil } +// PutMaster stores the publisher's transcript of the ad-free programme. +// +// Kept even when it cannot be used as the episode's transcript -- especially then. A transcript +// declined because this copy carries inserted advertising is the master that advertising was +// inserted into, and subtracting it from a recognised reading of the served file is what says +// where. See Inserted. +func (s Store) PutMaster(t Transcript) error { + return s.Put(Transcript{ + EpisodeID: t.EpisodeID + masterSuffix, + Source: SourcePublished, + LanguageCode: t.LanguageCode, + Cues: t.Cues, + }) +} + +// Master reads the publisher's transcript of the ad-free programme. +func (s Store) Master(episodeID string) (Transcript, error) { + held, err := s.Get(episodeID + masterSuffix) + if err != nil { + return Transcript{}, err + } + held.EpisodeID = episodeID + return held, nil +} + +// HasMaster reports whether the publisher's ad-free transcript is on disk. +func (s Store) HasMaster(episodeID string) bool { return s.Has(episodeID + masterSuffix) } + // Get reads an episode's transcript. func (s Store) Get(episodeID string) (Transcript, error) { name, err := s.path(episodeID) @@ -131,5 +168,11 @@ func (s Store) Prune(episodeID string) error { if err := os.Remove(name); err != nil && !os.IsNotExist(err) { return fmt.Errorf("transcript: %w", err) } + // The master goes with it: it is evidence about a file that is no longer held. + if master, err := s.path(episodeID + masterSuffix); err == nil { + if err := os.Remove(master); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("transcript: %w", err) + } + } return nil } -- cgit v1.2.3