Deep clean, part one: get the pipeline out of the command

cmd/listend/main.go had reached 1,249 lines carrying nine passes, and cmd has no
test file -- so every judgement in the transcript pipeline lived where nothing
could reach it. That is not hypothetical: a refetch master blocked the recurring
detector for a fortnight because the condition was wrong in a place no test
could see.

internal/words now holds the five passes with judgement in them -- Harvest,
Recognise, Rederive, Recurring, SecondCopy -- behind a Pipeline that names its
dependencies. main.go is 631 lines and does what a command should: read flags,
wire things together, serve.

**Moving it found a latent bug.** Pruning an episode's audio cleared its
transcript fields but not its master ones, so a re-mirrored episode would claim
a master whose file had been deleted -- a second description it could not
produce, and subtraction finding nothing for it for ever. Exactly the shape of
the bug that had just cost a fortnight. Pipeline.Forget now clears all of it,
including the attempt timestamps, so a re-mirrored episode is done again rather
than skipped for ever. Pinned by a test that asserts it needs recognising
afterwards.

Also in this part: Gradle's file-system watching is off. Chaquopy extracts a
3,200-file Python environment into app/build mid-build and the watcher tripped
over re-registering it, printing "Caught exception: Already watching path" in
front of every build error this project produced all week. It costs nothing to
turn off -- measured twice each on a no-op compileDebugKotlin, 1.50s and 1.59s
with, 1.46s and 1.46s without -- because that extraction invalidates the VFS
anyway. A watcher holding stale state about a tree another tool rewrites causes
the "clean and try again" class of problem rather than curing it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B

1 files changed, 39 insertions, 657 deletions

diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go
index 43921bd..e1329a9 100644
--- a/server/cmd/listend/main.go
+++ b/server/cmd/listend/main.go
@@ -23,7 +23,6 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
-	"sort"
 	"sync"
 	"syscall"
 	"time"
@@ -38,20 +37,9 @@ import (
 	"github.com/terst/listen/server/internal/store"
 	"github.com/terst/listen/server/internal/transcribe"
 	"github.com/terst/listen/server/internal/transcript"
+	"github.com/terst/listen/server/internal/words"
 )
 
-// tierRankOf orders the tiers for work: topical first, archive last.
-func tierRankOf(tier string) int {
-	switch tier {
-	case store.TierTopical:
-		return 0
-	case store.TierArchive:
-		return 2
-	default:
-		return 1
-	}
-}
-
 // minRecurringCorpus is how many of a show's episodes must have words before what they share
 // means anything. Three: the target and the two others a stretch has to turn up in.
 const minRecurringCorpus = 3
@@ -99,6 +87,7 @@ func main() {
 	decoder := decode.Decoder{}
 	transcriptDir := filepath.Join(*dataDir, "transcripts")
 	transcripts := transcript.Store{Dir: transcriptDir}
+	secondCopyDir := filepath.Join(*dataDir, "work", "second")
 	worker := &worker{
 		state:         state,
 		statePath:     statePath,
@@ -107,22 +96,26 @@ func main() {
 		keep:          *keep,
 		mirror:        &mirror.Mirror{AudioDir: audioDir, Probe: decoder},
 		decoder:       decoder,
-		transcripts:   transcripts,
-		harvester:     transcript.Harvester{Store: transcripts},
-		recogniser: transcribe.Whisper{
-			Binary:  *whisperBin,
-			Model:   *whisperModel,
-			Threads: *whisperThreads,
-			// Beside the audio rather than in /tmp: a decoded hour is 115MB of PCM, and the
-			// data directory is the volume with room for it.
-			WorkDir: filepath.Join(*dataDir, "work"),
+		words: &words.Pipeline{
+			State:       state,
+			AudioDir:    audioDir,
+			Transcripts: transcripts,
+			Harvester:   transcript.Harvester{Store: transcripts},
+			Recogniser: transcribe.Whisper{
+				Binary:  *whisperBin,
+				Model:   *whisperModel,
+				Threads: *whisperThreads,
+				// Beside the audio rather than in /tmp: a decoded hour is 115MB of PCM, and
+				// the data directory is the volume with room for it.
+				WorkDir: filepath.Join(*dataDir, "work"),
+			},
+			SecondCopies:     &mirror.Mirror{AudioDir: secondCopyDir, Probe: decoder},
+			SecondCopyDir:    secondCopyDir,
+			RecogniseBudget:  *transcribeBudget,
+			SecondCopyBudget: *refetchBudget,
 		},
-		transcribeBudget: *transcribeBudget,
-		refetchMirror:    &mirror.Mirror{AudioDir: filepath.Join(*dataDir, "work", "refetch"), Probe: decoder},
-		refetchDir:       filepath.Join(*dataDir, "work", "refetch"),
-		refetchBudget:    *refetchBudget,
-		requests:         make(chan request, 64),
-		wakeups:          make(chan struct{}, 1),
+		requests: make(chan request, 64),
+		wakeups:  make(chan struct{}, 1),
 	}
 
 	server := api.New(state, worker, api.Options{
@@ -150,12 +143,12 @@ func main() {
 	defer stop()
 
 	if *once {
-		worker.rederiveAds()
+		worker.words.Rederive()
 		worker.pollAll(ctx)
 		// Recognition too, so a deployment check exercises the expensive path rather than
 		// discovering it is misconfigured hours later in a log nobody is reading.
-		worker.transcribeSome(ctx)
-		worker.refetchSome(ctx)
+		worker.words.Recognise(ctx)
+		worker.words.SecondCopy(ctx)
 		if err := state.Save(statePath); err != nil {
 			log.Fatalf("save: %v", err)
 		}
@@ -208,21 +201,12 @@ type worker struct {
 	decoder       decode.Decoder
 	// transcripts holds each episode's words, and harvester takes the publishers' own where
 	// they describe the copy that was mirrored. See transcript.Fits for when they do not.
-	transcripts transcript.Store
-	harvester   transcript.Harvester
-	// recogniser turns an episode's audio into its words where no usable published transcript
-	// exists. Zero when recognition is not configured, which is a working state.
-	recogniser transcribe.Whisper
-	// transcribeBudget is how long one cycle may spend recognising. Bounded so the backlog is
-	// worked through steadily rather than monopolising the machine -- see transcribeSome.
-	transcribeBudget time.Duration
-	// refetchMirror downloads a second copy of an episode into its own directory, so the one
-	// the phone plays is never touched. Its files are deleted as soon as they are recognised.
-	refetchMirror *mirror.Mirror
-	refetchDir    string
-	refetchBudget time.Duration
-	requests      chan request
-	wakeups       chan struct{}
+	// words is the transcript side: taking publishers' transcripts, recognising the rest, and
+	// working out what to skip from what was said. It lives in its own package because none
+	// of it could be tested here.
+	words    *words.Pipeline
+	requests chan request
+	wakeups  chan struct{}
 }
 
 // wake asks for a poll now. Coalescing, so a burst of subscription updates is one poll.
@@ -244,12 +228,12 @@ func (w *worker) Request(feedURL, guid, enclosureURL, title string) {
 }
 
 func (w *worker) run(ctx context.Context, every time.Duration) {
-	w.rederiveAds()
+	w.words.Rederive()
 	w.save()
 	w.pollAll(ctx)
 	w.save()
-	w.transcribeSome(ctx)
-	w.refetchSome(ctx)
+	w.words.Recognise(ctx)
+	w.words.SecondCopy(ctx)
 
 	ticker := time.NewTicker(every)
 	defer ticker.Stop()
@@ -263,8 +247,8 @@ func (w *worker) run(ctx context.Context, every time.Duration) {
 			// Only on the timer. A wakeup is the phone pushing a subscription list and
 			// wanting that show polled now; following it with three quarters of an hour of
 			// recognition would make a new subscription feel broken.
-			w.transcribeSome(ctx)
-			w.refetchSome(ctx)
+			w.words.Recognise(ctx)
+			w.words.SecondCopy(ctx)
 		case <-w.wakeups:
 			w.pollAll(ctx)
 			w.save()
@@ -367,14 +351,8 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) {
 		if e.Mirrored() {
 			_ = w.mirror.Prune(e)
 		}
-		// The words go with the audio. A transcript of a file nobody holds any more is of no
-		// use to the phone, which plays the server's bytes, and the directory would otherwise
-		// grow for ever.
-		if e.TranscriptExamined() {
-			_ = w.transcripts.Prune(e.ID)
-			e.TranscriptSource, e.TranscriptCues, e.TranscriptNote = "", 0, ""
-			e.TranscriptAtUnix = 0
-		}
+		// The words go with the audio -- see words.Pipeline.Forget.
+		w.words.Forget(e)
 	}
 
 	fetched := 0
@@ -395,607 +373,11 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) {
 	if fetched > 0 {
 		log.Printf("%s: mirrored %d new episode(s)", channel.Title, fetched)
 	}
-	w.harvestTranscripts(ctx, show, kept)
-	w.findRecurring(show)
+	w.words.Harvest(ctx, show, kept)
+	w.words.Recurring(show)
 	w.detect(ctx, show, kept)
 }
 
-// harvestTranscripts takes the publishers' own transcripts for a show's mirrored episodes.
-//
-// Only where the publisher's master is what was mirrored, which transcript.Fits decides from the
-// declared-versus-served length gap rather than from the words. A show using dynamic ad
-// insertion publishes a transcript of the ad-free programme, and reading along with it would be
-// minutes out of step with the audio the phone plays.
-//
-// Each episode is examined once. A refusal is a judgement about this copy, so re-fetching it on
-// every poll would be work done for ever to reach the same answer -- and the refusal is itself
-// the record of which episodes speech recognition would have to do.
-func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candidates []*store.Episode) {
-	taken, declined, spotted, masters := 0, 0, 0, 0
-	for _, e := range candidates {
-		if ctx.Err() != nil {
-			return
-		}
-		// A declined episode is examined and has no words, but its publisher's transcript is
-		// still the ad-free master and is worth fetching -- it is what Inserted subtracts.
-		// Episodes declined before masters were kept arrive here once.
-		if !e.Mirrored() || e.TranscriptURL == "" {
-			continue
-		}
-		if e.TranscriptExamined() && !e.NeedsMaster() {
-			continue
-		}
-		result, err := w.harvester.Harvest(ctx, transcript.Offer{
-			EpisodeID:  e.ID,
-			URL:        e.TranscriptURL,
-			Type:       e.TranscriptType,
-			ServedMs:   e.DurationMs,
-			DeclaredMs: e.DeclaredDurationMs,
-		})
-		if err != nil {
-			// The download failed, so nothing has been decided and the next poll tries again.
-			log.Printf("%s: transcript for %q: %v", show.Title, e.Title, err)
-			continue
-		}
-		if result.Master > 0 {
-			e.MasterCues = result.Master
-			e.MasterNote = ""
-			masters++
-		} else if e.NeedsMaster() {
-			// Offered, fetched, and not usable as a master either. Recorded so it is not
-			// fetched again on every poll for ever.
-			e.MasterNote = result.Note
-		}
-		switch {
-		case result.Stored:
-			e.TranscriptSource = string(transcript.SourcePublished)
-			e.TranscriptCues = result.Cues
-			e.TranscriptAtUnix = time.Now().Unix()
-			e.TranscriptNote = ""
-			taken++
-			spotted += result.Ads
-		case result.Note != "":
-			e.TranscriptNote = result.Note
-			e.TranscriptAtUnix = time.Now().Unix()
-			declined++
-		}
-	}
-	if taken > 0 || declined > 0 || masters > 0 {
-		// The ad count is reported, not acted on. These are candidates read out of the words,
-		// and nothing publishes them as ranges to skip -- see transcript.Spot.
-		log.Printf("%s: transcripts -- took %d, declined %d (%d kept as ad-free masters), "+
-			"%d ad candidate(s) in the words",
-			show.Title, taken, declined, masters, spotted)
-	}
-}
-
-// transcribeSome recognises the words in episodes that have none, for as long as the budget
-// allows.
-//
-// Bounded rather than exhaustive, and run after polling rather than inside it. Recognition is
-// the most expensive thing this server does -- measured at about 11x realtime with base.en on
-// six threads, so an hour-long episode costs about five minutes -- and the backlog at the time
-// this was built was 304 episodes. Spending a fixed slice of each cycle works through that in a
-// few days without ever making a poll wait hours or taking the machine over.
-//
-// Newest first. An episode Peter is likely to play soon is worth having words for before one
-// from the back of the catalogue.
-func (w *worker) transcribeSome(ctx context.Context) {
-	if !w.recogniser.Available() || w.transcribeBudget <= 0 {
-		return
-	}
-	deadline := time.Now().Add(w.transcribeBudget)
-	done, failed := 0, 0
-	// Attempts made during the current run of failures, so they can be given back if the run
-	// turns out to be the engine rather than the episodes. See transcribe.Breaker.
-	var breaker transcribe.Breaker
-	var failing []*store.Episode
-
-	for _, e := range w.pendingRecognition() {
-		if ctx.Err() != nil || time.Now().After(deadline) {
-			break
-		}
-		started := time.Now()
-		result, err := w.recogniser.Transcribe(ctx, filepath.Join(w.mirror.AudioDir, e.Path))
-		if ctx.Err() != nil {
-			// Interrupted rather than attempted. Recording it would quietly skip whatever was
-			// in flight across a redeploy, and this runs for days at a time.
-			return
-		}
-		e.TranscribeAttemptedAtUnix = time.Now().Unix()
-		if err != nil {
-			log.Printf("transcribe %q: %v", e.Title, err)
-			failed++
-			failing = append(failing, e)
-			if breaker.Failure() {
-				giveBackAttempts(failing)
-				log.Printf("recognition abandoned after %d failures in a row -- "+
-					"giving those attempts back, since this is the engine and not the episodes",
-					breaker.Run())
-				w.save()
-				return
-			}
-			w.save()
-			continue
-		}
-
-		stored := transcript.Transcript{
-			EpisodeID:    e.ID,
-			Source:       transcript.SourceASR,
-			LanguageCode: result.LanguageCode,
-			Cues:         result.Cues,
-			Ads:          w.adsIn(e, result.Cues),
-		}
-		if err := w.transcripts.Put(stored); err != nil {
-			log.Printf("transcribe %q: %v", e.Title, err)
-			failed++
-			failing = append(failing, e)
-			if breaker.Failure() {
-				giveBackAttempts(failing)
-				log.Printf("recognition abandoned after %d failures in a row -- "+
-					"giving those attempts back, since this is the engine and not the episodes",
-					breaker.Run())
-				w.save()
-				return
-			}
-			w.save()
-			continue
-		}
-		e.TranscriptSource = string(transcript.SourceASR)
-		e.TranscriptCues = len(stored.Cues)
-		e.TranscriptAtUnix = time.Now().Unix()
-		// The note said why the publisher's transcript could not be used. There are words now,
-		// so it is answered rather than still true.
-		e.TranscriptNote = ""
-		done++
-		breaker.Success()
-		failing = nil
-		log.Printf("transcribed %q: %d lines, %s, took %s",
-			e.Title, len(stored.Cues), adAccounting(e, stored.Ads),
-			time.Since(started).Round(time.Second))
-		w.save()
-	}
-
-	if done > 0 || failed > 0 {
-		log.Printf("recognition -- %d done, %d failed, %d still waiting",
-			done, failed, len(w.pendingRecognition()))
-	}
-}
-
-/*
-rederiveAds works out every stored episode's advertising again, from the words it already has.
-
-Recognition is the expensive part and its output does not change; which stretches of it are
-advertising is a judgement that has. When subtraction learned to pull its edges back off the
-programme, the episodes already transcribed kept the old boundaries, and re-recognising them to
-pick up a better answer would cost hours of CPU for text that is already on disk.
-
-So it runs over the stored transcripts at startup instead -- eighteen alignments, a few
-milliseconds each -- and the next improvement costs nothing to apply either.
-*/
-func (w *worker) rederiveAds() {
-	changed := 0
-	for _, show := range w.state.Shows() {
-		for _, e := range show.Episodes {
-			if e.MasterCues == 0 || !e.HasTranscript() {
-				continue
-			}
-			held, err := w.transcripts.Get(e.ID)
-			if err != nil {
-				continue
-			}
-			found := w.adsIn(e, held.Cues)
-			// Publishing is decided every time, even when the candidates have not moved: what
-			// is fit to act on is a separate judgement from what was found, and it changes on
-			// its own. Only the file is left alone when nothing moved.
-			//
-			// Only for episodes this pass owns. An episode with no master has its breaks from
-			// findRecurring, and re-deriving subtraction for it would clear them -- it has
-			// nothing to subtract.
-			if e.MasterCues > 0 {
-				w.publishWordBreaks(e, found)
-			}
-			if sameCandidates(found, held.Ads) {
-				continue
-			}
-			held.Ads = found
-			if err := w.transcripts.Put(held); err != nil {
-				log.Printf("re-deriving %q: %v", e.Title, err)
-				continue
-			}
-			changed++
-		}
-	}
-	published, breaks := 0, 0
-	for _, show := range w.state.Shows() {
-		for _, e := range show.Episodes {
-			if len(e.WordBreaks) > 0 {
-				published++
-				breaks += len(e.WordBreaks)
-			}
-		}
-	}
-	log.Printf("re-derived the advertising in %d stored transcript(s); "+
-		"%d episode(s) publish %d break(s) found in their words", changed, published, breaks)
-}
-
-func sameCandidates(left, right []transcript.Candidate) bool {
-	if len(left) != len(right) {
-		return false
-	}
-	for i := range left {
-		if left[i].StartMs != right[i].StartMs || left[i].EndMs != right[i].EndMs {
-			return false
-		}
-	}
-	return true
-}
-
-/*
-findRecurring works out what each of a show's episodes says every week.
-
-Per show rather than per episode, because the comparison is against the show's other episodes.
-Eight of them are mirrored within a minute of each other, so they carry one advertising campaign
-and eight different programmes -- the text analogue of what makes the audio scan work, using
-transcripts already on disk and costing no download and no recognition.
-
-Published as the show's own audio rather than as advertising. What recurs is the sponsor reads
-and the furniture together, and nothing measurable separates them: ad copy turns up in 5% of the
-advertising and 4% of the furniture, and length and position overlap throughout. The listener's
-own setting governs a show's own audio, so claiming the weaker of the two is the safe mistake.
-
-Skipped for an episode that already has measured advertising. Subtraction knows which stretches
-are inserted; this only knows that they repeat, and the better answer should not be overwritten
-by the vaguer one.
-*/
-func (w *worker) findRecurring(show *store.Show) {
-	type held struct {
-		episode *store.Episode
-		cues    []transcript.Cue
-	}
-	var corpus []held
-	for _, e := range show.Episodes {
-		if !e.Mirrored() || !e.HasTranscript() {
-			continue
-		}
-		stored, err := w.transcripts.Get(e.ID)
-		if err != nil {
-			continue
-		}
-		corpus = append(corpus, held{e, stored.Cues})
-	}
-	if len(corpus) < minRecurringCorpus {
-		return
-	}
-
-	published, refused := 0, 0
-	for i, target := range corpus {
-		// Skipped only where subtraction actually produced something, not merely where a
-		// master exists. A refetch master that carried the same campaign yields no candidates
-		// at all, and treating "has a master" as "has a better answer" left those episodes
-		// with nothing -- Tangle's "The Israeli elections" has a 229-second gap, a refetch
-		// master, no subtraction candidates, and was excluded from this.
-		if len(target.episode.WordBreaks) > 0 {
-			continue
-		}
-		others := make([][]transcript.Cue, 0, len(corpus)-1)
-		for j, other := range corpus {
-			if i != j {
-				others = append(others, other.cues)
-			}
-		}
-		found := transcript.Recurring(target.cues, others)
-		fit, why := transcript.PublishableRecurring(found, target.episode.DurationMs)
-		target.episode.WordAdsRefused = why
-		target.episode.WordBreaks = nil
-		for _, c := range fit {
-			target.episode.WordBreaks = append(target.episode.WordBreaks, store.Break{
-				StartMs: c.StartMs,
-				EndMs:   c.EndMs,
-				// What recurs is the show's own recurring audio, which is what this kind
-				// means. The listener decides whether to skip it.
-				Kind: string(skips.KindCue),
-			})
-		}
-		if len(fit) > 0 {
-			published++
-		}
-		if why != "" {
-			refused++
-		}
-	}
-	if published > 0 || refused > 0 {
-		log.Printf("%s: recurring text -- %d episode(s) publish it, %d refused, from a corpus of %d",
-			show.Title, published, refused, len(corpus))
-	}
-}
-
-/*
-publishWordBreaks turns what the words found into stretches the phone will actually seek past.
-
-The only place anything read out of a transcript reaches playback. Everything else in this
-pipeline produces candidates to look at; transcript.Publishable decides which of them are fit to
-act on, and refuses the rest with a reason rather than silently dropping them.
-
-Kept in their own field so the ad scan, which rewrites its own list from scratch on every
-detection pass, cannot wipe them -- and so that which detector found a break stays answerable.
-*/
-func (w *worker) publishWordBreaks(e *store.Episode, found []transcript.Candidate) {
-	fit, why := transcript.Publishable(found, e.DurationMs, e.DeclaredDurationMs)
-	e.WordAdsRefused = why
-	e.WordBreaks = nil
-	for _, c := range fit {
-		e.WordBreaks = append(e.WordBreaks, store.Break{
-			StartMs: c.StartMs,
-			EndMs:   c.EndMs,
-			Kind:    string(skips.KindAd),
-		})
-	}
-}
-
-/*
-adsIn works out which stretches of a recognised episode are advertising.
-
-By subtraction where the publisher's ad-free master is held, and by reading the ad copy where it
-is not. They are not close in quality and the choice is not a preference. Measured on Darknet
-Diaries' "Conti": the feed declares 59:17 and the file served runs 63:33, so 4:15 of advertising
-was inserted -- subtraction found 4:20 of it in three pods, each starting on the first word of
-an ad read and ending on the last, while reading the copy found 0:16. 102% against 6%.
-
-Subtraction wins because it is a measurement. The master says what the programme is, the
-recognised copy says what was played, and the difference is the advertising by definition --
-there is no phrase list to be wrong about, and no show whose hosts happen to talk like an
-advertisement.
-*/
-func (w *worker) adsIn(e *store.Episode, cues []transcript.Cue) []transcript.Candidate {
-	if e.MasterCues > 0 {
-		if master, err := w.transcripts.Master(e.ID); err == nil {
-			if found := transcript.Inserted(master.Cues, cues); len(found) > 0 {
-				return found
-			}
-		}
-	}
-	return transcript.Spot(cues)
-}
-
-// giveBackAttempts unmarks episodes that failed for a reason that was nothing to do with them,
-// so they are tried again rather than hidden for ever.
-func giveBackAttempts(episodes []*store.Episode) {
-	for _, e := range episodes {
-		e.TranscribeAttemptedAtUnix = 0
-	}
-}
-
-/*
-adAccounting describes what was found against what the lengths say is there.
-
-The declared-versus-served gap comes from the feed and the file rather than from any detector,
-which makes it the one number able to contradict one.
-
-It bounds only the advertising a CDN *inserted* into this copy, which is not all the advertising
-in it. A show that bakes its sponsor reads into the file it publishes has no gap at all and can
-still be a third advertisements -- The 404 Media Podcast declares within 11 seconds of what it
-serves and carries Mack Weldon, Incogni and Warby Parker reads in the middle of it. So finding
-more than the gap implies is not over-reporting, and saying so in the readout matters: the first
-reading of a "728%" line was that the detector had gone wrong, and it had not.
-*/
-func adAccounting(e *store.Episode, found []transcript.Candidate) string {
-	var total int64
-	for _, c := range found {
-		total += c.EndMs - c.StartMs
-	}
-	how := "read from the ad copy"
-	switch {
-	case e.MasterSource == string(transcript.SourceRefetch):
-		how = "by subtracting a second fetch"
-	case e.MasterCues > 0:
-		how = "by subtracting the publisher's master"
-	}
-	gap := e.DurationMs - e.DeclaredDurationMs
-	switch {
-	case gap <= 0:
-		return fmt.Sprintf("%d ad candidate(s) %s, %s; nothing was inserted into this copy, "+
-			"so anything found is baked into what the publisher serves",
-			len(found), how, clock(total))
-	case total > gap:
-		// Deliberately does not say why it is over. Two causes look identical here and the
-		// first guess was the wrong one: advertising baked into what the publisher serves,
-		// which the gap cannot see, and candidate edges reaching past the advertisement.
-		// Subtraction answers "not in the master", which is a superset of "advertising" --
-		// a title card, a music sting and whisper's own [MUSIC] markers are all absent from a
-		// publisher's written transcript too. Checked on Heavyweight "#70 Cayden", where a
-		// 134-second candidate opens on `Today's episode, "Caiton."` and `[MUSIC]`.
-		return fmt.Sprintf("%d ad candidate(s) %s, %s against %s inserted -- over by %s, "+
-			"which is either baked-in advertising or candidate edges reaching too far",
-			len(found), how, clock(total), clock(gap), clock(total-gap))
-	default:
-		return fmt.Sprintf("%d ad candidate(s) %s, %s of the %s inserted (%d%%)",
-			len(found), how, clock(total), clock(gap), total*100/gap)
-	}
-}
-
-/*
-refetchSome fetches a second copy of episodes that have no master, and recognises it.
-
-The extension of subtraction to the rest of the library. Only 38 of 336 mirrored episodes are
-offered a publisher's transcript, and only 18 of those are the ad-carrying kind subtraction
-works on -- so the method that measures itself at 92-104% against the length gap covers about
-five per cent of what Peter listens to.
-
-A second fetch stands in for the master. Advertising is inserted per request, which this project
-already measured the hard way: the same Radiolab episode came back 7:29 of advertising on one
-fetch and 5:34 on another. So two copies agree on the programme and differ on the advertising,
-and subtracting one from the other says where the advertising in the other one is. What it
-cannot see is a campaign both copies happened to carry -- the exact opposite blind spot to the
-audio scan, whose whole method is finding what a batch shares.
-
-The second copy's audio is deleted the moment it has been recognised. The phone plays the
-mirrored bytes and nothing else may claim that slot; this is evidence, not a recording.
-*/
-func (w *worker) refetchSome(ctx context.Context) {
-	if !w.recogniser.Available() || w.refetchBudget <= 0 {
-		return
-	}
-	deadline := time.Now().Add(w.refetchBudget)
-	done, failed := 0, 0
-	var breaker transcribe.Breaker
-	var failing []*store.Episode
-
-	for _, e := range w.pendingRefetch() {
-		if ctx.Err() != nil || time.Now().After(deadline) {
-			break
-		}
-		found, secondMs, err := w.refetchOne(ctx, e)
-		if ctx.Err() != nil {
-			return
-		}
-		e.RefetchAttemptedAtUnix = time.Now().Unix()
-		if err != nil {
-			log.Printf("refetching %q: %v", e.Title, err)
-			failed++
-			failing = append(failing, e)
-			if breaker.Failure() {
-				for _, given := range failing {
-					given.RefetchAttemptedAtUnix = 0
-				}
-				log.Printf("refetching abandoned after %d failures in a row -- giving those "+
-					"attempts back, since this is the machine and not the episodes", breaker.Run())
-				w.save()
-				return
-			}
-			w.save()
-			continue
-		}
-		breaker.Success()
-		failing = nil
-		done++
-		// The two lengths are the first thing to look at when this finds nothing: two fetches
-		// that came back the same length probably carried the same campaign, and there is
-		// then nothing for subtraction to see.
-		log.Printf("second copy of %q: %s against the mirrored %s, %d lines, %s",
-			e.Title, clock(secondMs), clock(e.DurationMs), e.MasterCues, adAccounting(e, found))
-		w.save()
-	}
-	if done > 0 || failed > 0 {
-		log.Printf("second copies -- %d done, %d failed, %d still waiting",
-			done, failed, len(w.pendingRefetch()))
-	}
-}
-
-// refetchOne downloads one episode again, recognises it, keeps the words and throws the audio
-// away.
-func (w *worker) refetchOne(
-	ctx context.Context, e *store.Episode,
-) ([]transcript.Candidate, int64, error) {
-	// Its own id and its own directory, so nothing can mistake this for the copy the phone
-	// plays even for the moment it exists.
-	second := &store.Episode{ID: e.ID + "-second", EnclosureURL: e.EnclosureURL}
-	if err := w.refetchMirror.Fetch(ctx, second); err != nil {
-		return nil, 0, err
-	}
-	defer func() { _ = w.refetchMirror.Prune(second) }()
-
-	result, err := w.recogniser.Transcribe(ctx, filepath.Join(w.refetchDir, second.Path))
-	if err != nil {
-		return nil, second.DurationMs, err
-	}
-	if err := w.transcripts.PutMaster(transcript.Transcript{
-		EpisodeID:    e.ID,
-		Source:       transcript.SourceRefetch,
-		LanguageCode: result.LanguageCode,
-		Cues:         result.Cues,
-	}); err != nil {
-		return nil, second.DurationMs, err
-	}
-	e.MasterCues = len(result.Cues)
-	e.MasterSource = string(transcript.SourceRefetch)
-	e.MasterNote = ""
-
-	// Re-derive this episode's advertising now that there is something to subtract it from.
-	held, err := w.transcripts.Get(e.ID)
-	if err != nil {
-		return nil, second.DurationMs, err
-	}
-	held.Ads = w.adsIn(e, held.Cues)
-	if err := w.transcripts.Put(held); err != nil {
-		return nil, second.DurationMs, err
-	}
-	w.publishWordBreaks(e, held.Ads)
-	return held.Ads, second.DurationMs, nil
-}
-
-/*
-pendingRefetch is every episode with its own words, no master, and advertising to find.
-
-Newest first: an episode about to be played is worth the work before one from the back of a
-catalogue.
-*/
-func (w *worker) pendingRefetch() []*store.Episode {
-	now := time.Now().Unix()
-	var pending []*store.Episode
-	for _, show := range w.state.Shows() {
-		for _, e := range show.Episodes {
-			if !e.NeedsRefetch(now) {
-				continue
-			}
-			// Without a length gap nothing was inserted, so a second fetch would be the same
-			// recording and there would be nothing to subtract.
-			if !transcript.HasInsertedAdvertising(e.DurationMs, e.DeclaredDurationMs) {
-				continue
-			}
-			pending = append(pending, e)
-		}
-	}
-	sort.Slice(pending, func(i, j int) bool {
-		return pending[i].PublishedUnix > pending[j].PublishedUnix
-	})
-	return pending
-}
-
-/*
-pendingRecognition is every mirrored episode still without words, best first.
-
-An episode holding the publisher's ad-free master comes first, whatever its date. Recognising
-one of those produces an answer that checks itself: subtraction found 4:20 of the 4:15 the
-lengths imply on Darknet Diaries and 3:32 of 3:29 on Heavyweight. Recognising anything else
-leaves the ad copy to be read, which across the first nineteen real episodes found 0%, 0%, 15%
-and 29% of what the lengths say is there. Spending the budget where the result can be trusted
-first is worth more than spending it in date order.
-
-Newest first within each group, because an episode about to be played is worth having words for
-before one from the back of a catalogue.
-*/
-func (w *worker) pendingRecognition() []*store.Episode {
-	var pending []*store.Episode
-	for _, show := range w.state.Shows() {
-		for _, e := range show.Episodes {
-			if e.NeedsRecognising() {
-				pending = append(pending, e)
-			}
-		}
-	}
-	// Three things decide the order, and they are in the order they matter. A topical show is
-	// worthless by Thursday. An episode whose answer checks itself against the length gap is
-	// worth more than one that cannot. And within that, newest first.
-	rank := map[string]int{}
-	for _, show := range w.state.Shows() {
-		for _, e := range show.Episodes {
-			rank[e.ID] = tierRankOf(w.state.Tier(show.FeedURL))
-		}
-	}
-	sort.SliceStable(pending, func(i, j int) bool {
-		left, right := pending[i], pending[j]
-		if rank[left.ID] != rank[right.ID] {
-			return rank[left.ID] < rank[right.ID]
-		}
-		if (left.MasterCues > 0) != (right.MasterCues > 0) {
-			return left.MasterCues > 0
-		}
-		return left.PublishedUnix > right.PublishedUnix
-	})
-	return pending
-}
-
 // detect runs the intro/outro search over a show's mirrored episodes and records the result.
 //
 // Every mirrored episode at once, which is the whole reason this is a server: the phone could