Find the advertising by subtracting the publisher's master

Peter's idea, and it is the best detector in this project because it is a
measurement rather than a guess.

A publisher's transcript describes the ad-free master. Recognition over the
mirrored file describes the copy the phone will play. The copy is the master
with advertising stitched into it, so the text present in the served copy and
absent from the master *is* the advertising, with its own boundaries. No phrase
list to be wrong about, and no show whose hosts happen to talk like an advert.

Measured on Darknet Diaries' "Conti" before it was wired in. The feed declares
59:17, the file served runs 63:33, so 4:15 was inserted. Subtraction found
**4:20 in three pods -- 102%** against a figure no detector produced. Reading the
ad copy on the same words found 0:16, or 6%. Every pod starts on the first word
of an ad read and ends on the last: a NetSuite pre-roll, a 2:20 midroll, and a
Doppel read at 32:08. Nothing of the programme is clipped.

The alignment has to survive recognition being approximate -- whisper writes
"Bombus" for "Bombas" and spells numbers differently -- so it anchors on runs of
five words occurring exactly once in each text. A window containing a mistake
simply fails to anchor and its neighbours carry the alignment instead.
Uniqueness is what makes an anchor safe: a phrase appearing twice in the master
cannot say which occurrence it matches, and a wrong anchor drags the alignment
across a break. Inside an advertisement nothing matches at all, which is exactly
what makes a pod show up as a gap.

An anchor names the *start* of a matching run, so the content between two
anchors begins after the first run ends. Getting that wrong put every break one
anchor-length early, which on a real episode is a skip over the last sentence of
the programme. The tests pinned it.

**This is why a declined transcript is now kept.** Declined means "this copy
carries inserted advertising the master does not describe" -- precisely the
episode subtraction works on. One declined for describing a different recording
is still thrown away, since it is no use for anything, and Fits now carries a
reason rather than making callers match on its wording. The nineteen episodes
declined before this existed fetch their masters once on the next poll.

Nothing is published as a skip yet. The candidates are stored with the words and
dimmed on the transcript screen, and every episode now logs what was found
against what the lengths imply.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B

1 files changed, 75 insertions, 8 deletions

diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go
index 1b746f1..d93325d 100644
--- a/server/cmd/listend/main.go
+++ b/server/cmd/listend/main.go
@@ -376,12 +376,18 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) {
 // every poll would be work done for ever to reach the same answer -- and the refusal is itself
 // the record of which episodes speech recognition would have to do.
 func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candidates []*store.Episode) {
-	taken, declined, spotted := 0, 0, 0
+	taken, declined, spotted, masters := 0, 0, 0, 0
 	for _, e := range candidates {
 		if ctx.Err() != nil {
 			return
 		}
-		if !e.Mirrored() || e.TranscriptExamined() || e.TranscriptURL == "" {
+		// A declined episode is examined and has no words, but its publisher's transcript is
+		// still the ad-free master and is worth fetching -- it is what Inserted subtracts.
+		// Episodes declined before masters were kept arrive here once.
+		if !e.Mirrored() || e.TranscriptURL == "" {
+			continue
+		}
+		if e.TranscriptExamined() && !e.NeedsMaster() {
 			continue
 		}
 		result, err := w.harvester.Harvest(ctx, transcript.Offer{
@@ -396,6 +402,15 @@ func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candi
 			log.Printf("%s: transcript for %q: %v", show.Title, e.Title, err)
 			continue
 		}
+		if result.Master > 0 {
+			e.MasterCues = result.Master
+			e.MasterNote = ""
+			masters++
+		} else if e.NeedsMaster() {
+			// Offered, fetched, and not usable as a master either. Recorded so it is not
+			// fetched again on every poll for ever.
+			e.MasterNote = result.Note
+		}
 		switch {
 		case result.Stored:
 			e.TranscriptSource = string(transcript.SourcePublished)
@@ -410,11 +425,12 @@ func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candi
 			declined++
 		}
 	}
-	if taken > 0 || declined > 0 {
+	if taken > 0 || declined > 0 || masters > 0 {
 		// The ad count is reported, not acted on. These are candidates read out of the words,
 		// and nothing publishes them as ranges to skip -- see transcript.Spot.
-		log.Printf("%s: transcripts -- took %d, declined %d, %d ad candidate(s) in the words",
-			show.Title, taken, declined, spotted)
+		log.Printf("%s: transcripts -- took %d, declined %d (%d kept as ad-free masters), "+
+			"%d ad candidate(s) in the words",
+			show.Title, taken, declined, masters, spotted)
 	}
 }
 
@@ -473,7 +489,7 @@ func (w *worker) transcribeSome(ctx context.Context) {
 			Source:       transcript.SourceASR,
 			LanguageCode: result.LanguageCode,
 			Cues:         result.Cues,
-			Ads:          transcript.Spot(result.Cues),
+			Ads:          w.adsIn(e, result.Cues),
 		}
 		if err := w.transcripts.Put(stored); err != nil {
 			log.Printf("transcribe %q: %v", e.Title, err)
@@ -499,8 +515,9 @@ func (w *worker) transcribeSome(ctx context.Context) {
 		done++
 		breaker.Success()
 		failing = nil
-		log.Printf("transcribed %q: %d lines, %d ad candidate(s), took %s",
-			e.Title, len(stored.Cues), len(stored.Ads), time.Since(started).Round(time.Second))
+		log.Printf("transcribed %q: %d lines, %s, took %s",
+			e.Title, len(stored.Cues), adAccounting(e, stored.Ads),
+			time.Since(started).Round(time.Second))
 		w.save()
 	}
 
@@ -510,6 +527,31 @@ func (w *worker) transcribeSome(ctx context.Context) {
 	}
 }
 
+/*
+adsIn works out which stretches of a recognised episode are advertising.
+
+By subtraction where the publisher's ad-free master is held, and by reading the ad copy where it
+is not. They are not close in quality and the choice is not a preference. Measured on Darknet
+Diaries' "Conti": the feed declares 59:17 and the file served runs 63:33, so 4:15 of advertising
+was inserted -- subtraction found 4:20 of it in three pods, each starting on the first word of
+an ad read and ending on the last, while reading the copy found 0:16. 102% against 6%.
+
+Subtraction wins because it is a measurement. The master says what the programme is, the
+recognised copy says what was played, and the difference is the advertising by definition --
+there is no phrase list to be wrong about, and no show whose hosts happen to talk like an
+advertisement.
+*/
+func (w *worker) adsIn(e *store.Episode, cues []transcript.Cue) []transcript.Candidate {
+	if e.MasterCues > 0 {
+		if master, err := w.transcripts.Master(e.ID); err == nil {
+			if found := transcript.Inserted(master.Cues, cues); len(found) > 0 {
+				return found
+			}
+		}
+	}
+	return transcript.Spot(cues)
+}
+
 // giveBackAttempts unmarks episodes that failed for a reason that was nothing to do with them,
 // so they are tried again rather than hidden for ever.
 func giveBackAttempts(episodes []*store.Episode) {
@@ -518,6 +560,31 @@ func giveBackAttempts(episodes []*store.Episode) {
 	}
 }
 
+/*
+adAccounting describes what was found against what the lengths say is there.
+
+The declared-versus-served gap comes from the feed and the file rather than from any detector,
+which makes it the one number able to contradict one. Every other readout in this project that
+reported only what it found turned out to be hiding something.
+*/
+func adAccounting(e *store.Episode, found []transcript.Candidate) string {
+	var total int64
+	for _, c := range found {
+		total += c.EndMs - c.StartMs
+	}
+	how := "read from the ad copy"
+	if e.MasterCues > 0 {
+		how = "by subtracting the publisher's master"
+	}
+	gap := e.DurationMs - e.DeclaredDurationMs
+	if gap <= 0 {
+		return fmt.Sprintf("%d ad candidate(s) %s, %s; nothing was inserted into this copy",
+			len(found), how, clock(total))
+	}
+	return fmt.Sprintf("%d ad candidate(s) %s, %s of the %s inserted (%d%%)",
+		len(found), how, clock(total), clock(gap), total*100/gap)
+}
+
 // pendingRecognition is every mirrored episode still without words, newest first.
 func (w *worker) pendingRecognition() []*store.Episode {
 	var pending []*store.Episode