diff options
Diffstat (limited to 'server')
| -rw-r--r-- | server/cmd/listend/main.go | 18 | ||||
| -rw-r--r-- | server/internal/store/store.go | 31 | ||||
| -rw-r--r-- | server/internal/store/store_test.go | 82 |
3 files changed, 127 insertions, 4 deletions
diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index 04c933f..3e5c65a 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -278,6 +278,7 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) { // over weeks and lost it whenever retention reclaimed the file it was holding. func (w *worker) detect(ctx context.Context, show *store.Show, candidates []*store.Episode) { var episodes []skips.Episode + var ids []string byID := map[string]*store.Episode{} for _, e := range candidates { if !e.Mirrored() { @@ -288,11 +289,15 @@ func (w *worker) detect(ctx context.Context, show *store.Show, candidates []*sto Path: filepath.Join(w.mirror.AudioDir, e.Path), DeclaredDurationMs: e.DeclaredDurationMs, }) + ids = append(ids, e.ID) byID[e.ID] = e } if len(episodes) < skips.DefaultConfirmThreshold || ctx.Err() != nil { return } + if !show.NeedsDetection(ids) { + return + } started := time.Now() result, err := skips.Detect(ctx, w.decoder, episodes, skips.DefaultConfig()) @@ -323,7 +328,9 @@ func (w *worker) detect(ctx context.Context, show *store.Show, candidates []*sto result.OutroConfirmedOn, len(episodes), checkNote(result.OutroCheck), time.Since(started).Round(time.Second)) - w.detectAds(ctx, show, episodes, byID) + if w.detectAds(ctx, show, episodes, byID) { + show.RecordDetected(ids) + } } // detectAds finds the advertising a batch of episodes shares, which is what simultaneous @@ -334,20 +341,22 @@ func (w *worker) detect(ctx context.Context, show *store.Show, candidates []*sto // is how it came to confirm an advertisement as an intro. This uses that dependence rather than // being defeated by it -- and reports what it found against the declared-versus-served length // gap, a figure it did not produce and which can therefore contradict it. +// Reports whether the scan completed, so an interrupted or failed pass is examined again rather +// than written off as done. func (w *worker) detectAds( ctx context.Context, show *store.Show, episodes []skips.Episode, byID map[string]*store.Episode, -) { +) bool { if ctx.Err() != nil { - return + return false } started := time.Now() ads, err := skips.DetectAds(ctx, w.decoder, episodes, skips.DefaultAdConfig()) if err != nil { log.Printf("%s: ad scan failed: %v", show.Title, err) - return + return false } var withBreaks, refused int @@ -381,6 +390,7 @@ func (w *worker) detectAds( log.Printf("%s: ads on %d/%d episode(s)%s, accounted %s of %s inserted, took %s", show.Title, withBreaks, len(episodes), refusedNote(refused), clock(foundTotal), clock(insertedTotal), time.Since(started).Round(time.Second)) + return true } func refusedNote(refused int) string { diff --git a/server/internal/store/store.go b/server/internal/store/store.go index a67e637..51e4ed7 100644 --- a/server/internal/store/store.go +++ b/server/internal/store/store.go @@ -75,6 +75,10 @@ type Show struct { // says so instead of silently having nothing. LastPolledUnix int64 `json:"lastPolledUnix,omitempty"` LastError string `json:"lastError,omitempty"` + + // DetectedBatch is the set of episodes the last detection pass examined. Stored as the + // plain ids rather than a hash so the state file says what was actually done. + DetectedBatch string `json:"detectedBatch,omitempty"` } // Partition splits a show's episodes into the newest keep and everything else, newest first. @@ -316,3 +320,30 @@ func matchIn(show *Show, guid, enclosureURL, title string) *Episode { } return nil } + +// NeedsDetection reports whether this batch of episodes still has to be examined. +// +// The batch is the unit, not the episode: every offset and every ad break is found by comparing +// an episode against the others fetched alongside it, so one arrival or one pruning legitimately +// changes all of them. An unchanged batch, though, produces results identical to the digit -- +// and re-deriving them cost four minutes a show against a two-hour poll interval, which for +// forty-two shows meant the machine never finished a cycle and never idled. +func (s *Show) NeedsDetection(episodeIDs []string) bool { + return s.DetectedBatch != batchKey(episodeIDs) +} + +// RecordDetected notes that this exact batch has been examined. +// +// Called only after a pass completes, so a cancelled or failed one is retried rather than +// written off. +func (s *Show) RecordDetected(episodeIDs []string) { + s.DetectedBatch = batchKey(episodeIDs) +} + +// batchKey identifies a set of episodes irrespective of order, which matters because feeds are +// not reliably sorted: the same eight episodes arriving shuffled must not read as a change. +func batchKey(episodeIDs []string) string { + ordered := append([]string(nil), episodeIDs...) + sort.Strings(ordered) + return strings.Join(ordered, " ") +} diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go index cd5cf6f..0540fbf 100644 --- a/server/internal/store/store_test.go +++ b/server/internal/store/store_test.go @@ -237,3 +237,85 @@ func TestSaveIsAtomic(t *testing.T) { } } } + +// Detection is the expensive thing this server does -- four minutes a show -- and it used to +// re-run on every poll over episodes that had not changed, producing results identical to the +// digit. Forty-two shows at ~6.5 minutes each is 4.5 hours of work against a two-hour poll +// interval, so the machine never finished a cycle and never idled. +// +// The batch is what has to match, not the individual episode: every answer depends on which +// other episodes it was compared against, so one new episode legitimately changes all of them. + +func TestAShowNotYetExaminedNeedsDetection(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)}) + + if !show.NeedsDetection([]string{"a", "b"}) { + t.Fatal("a show that has never been examined should need detection") + } +} + +func TestTheSameBatchIsNotExaminedTwice(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", nil) + + show.RecordDetected([]string{"a", "b", "c"}) + + if show.NeedsDetection([]string{"a", "b", "c"}) { + t.Fatal("re-examining an unchanged batch; this is the rerun that pegged the server") + } +} + +func TestANewEpisodeInTheBatchNeedsDetectionAgain(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", nil) + show.RecordDetected([]string{"a", "b", "c"}) + + if !show.NeedsDetection([]string{"a", "b", "c", "d"}) { + t.Fatal("a new episode changes what every other episode is compared against") + } +} + +func TestAPrunedEpisodeNeedsDetectionAgain(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", nil) + show.RecordDetected([]string{"a", "b", "c"}) + + if !show.NeedsDetection([]string{"b", "c"}) { + t.Fatal("losing an episode changes the comparison too") + } +} + +// Feeds are not reliably ordered, and Partition sorts by publish date, so the same eight +// episodes can arrive in a different order. Treating that as a change would never close +// the gate. +func TestTheOrderOfTheBatchDoesNotMatter(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", nil) + show.RecordDetected([]string{"a", "b", "c"}) + + if show.NeedsDetection([]string{"c", "a", "b"}) { + t.Fatal("the same batch in a different order is the same batch") + } +} + +// Otherwise every restart redoes every show -- which at 4.5 hours a cycle is most of a day. +func TestWhatHasBeenExaminedSurvivesARestart(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + + s := New() + s.Merge("https://f", "A Show", nil) + s.Show("https://f").RecordDetected([]string{"a", "b"}) + if err := s.Save(path); err != nil { + t.Fatalf("save: %v", err) + } + + loaded, err := Load(path) + if err != nil { + t.Fatalf("load: %v", err) + } + if loaded.Show("https://f").NeedsDetection([]string{"a", "b"}) { + t.Fatal("a restart would redo every show") + } +} |
