summaryrefslogtreecommitdiff
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/cmd/listend/main.go18
-rw-r--r--server/internal/store/store.go31
-rw-r--r--server/internal/store/store_test.go82
3 files changed, 127 insertions, 4 deletions
diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go
index 04c933f..3e5c65a 100644
--- a/server/cmd/listend/main.go
+++ b/server/cmd/listend/main.go
@@ -278,6 +278,7 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) {
// over weeks and lost it whenever retention reclaimed the file it was holding.
func (w *worker) detect(ctx context.Context, show *store.Show, candidates []*store.Episode) {
var episodes []skips.Episode
+ var ids []string
byID := map[string]*store.Episode{}
for _, e := range candidates {
if !e.Mirrored() {
@@ -288,11 +289,15 @@ func (w *worker) detect(ctx context.Context, show *store.Show, candidates []*sto
Path: filepath.Join(w.mirror.AudioDir, e.Path),
DeclaredDurationMs: e.DeclaredDurationMs,
})
+ ids = append(ids, e.ID)
byID[e.ID] = e
}
if len(episodes) < skips.DefaultConfirmThreshold || ctx.Err() != nil {
return
}
+ if !show.NeedsDetection(ids) {
+ return
+ }
started := time.Now()
result, err := skips.Detect(ctx, w.decoder, episodes, skips.DefaultConfig())
@@ -323,7 +328,9 @@ func (w *worker) detect(ctx context.Context, show *store.Show, candidates []*sto
result.OutroConfirmedOn, len(episodes), checkNote(result.OutroCheck),
time.Since(started).Round(time.Second))
- w.detectAds(ctx, show, episodes, byID)
+ if w.detectAds(ctx, show, episodes, byID) {
+ show.RecordDetected(ids)
+ }
}
// detectAds finds the advertising a batch of episodes shares, which is what simultaneous
@@ -334,20 +341,22 @@ func (w *worker) detect(ctx context.Context, show *store.Show, candidates []*sto
// is how it came to confirm an advertisement as an intro. This uses that dependence rather than
// being defeated by it -- and reports what it found against the declared-versus-served length
// gap, a figure it did not produce and which can therefore contradict it.
+// Reports whether the scan completed, so an interrupted or failed pass is examined again rather
+// than written off as done.
func (w *worker) detectAds(
ctx context.Context,
show *store.Show,
episodes []skips.Episode,
byID map[string]*store.Episode,
-) {
+) bool {
if ctx.Err() != nil {
- return
+ return false
}
started := time.Now()
ads, err := skips.DetectAds(ctx, w.decoder, episodes, skips.DefaultAdConfig())
if err != nil {
log.Printf("%s: ad scan failed: %v", show.Title, err)
- return
+ return false
}
var withBreaks, refused int
@@ -381,6 +390,7 @@ func (w *worker) detectAds(
log.Printf("%s: ads on %d/%d episode(s)%s, accounted %s of %s inserted, took %s",
show.Title, withBreaks, len(episodes), refusedNote(refused),
clock(foundTotal), clock(insertedTotal), time.Since(started).Round(time.Second))
+ return true
}
func refusedNote(refused int) string {
diff --git a/server/internal/store/store.go b/server/internal/store/store.go
index a67e637..51e4ed7 100644
--- a/server/internal/store/store.go
+++ b/server/internal/store/store.go
@@ -75,6 +75,10 @@ type Show struct {
// says so instead of silently having nothing.
LastPolledUnix int64 `json:"lastPolledUnix,omitempty"`
LastError string `json:"lastError,omitempty"`
+
+ // DetectedBatch is the set of episodes the last detection pass examined. Stored as the
+ // plain ids rather than a hash so the state file says what was actually done.
+ DetectedBatch string `json:"detectedBatch,omitempty"`
}
// Partition splits a show's episodes into the newest keep and everything else, newest first.
@@ -316,3 +320,30 @@ func matchIn(show *Show, guid, enclosureURL, title string) *Episode {
}
return nil
}
+
+// NeedsDetection reports whether this batch of episodes still has to be examined.
+//
+// The batch is the unit, not the episode: every offset and every ad break is found by comparing
+// an episode against the others fetched alongside it, so one arrival or one pruning legitimately
+// changes all of them. An unchanged batch, though, produces results identical to the digit --
+// and re-deriving them cost four minutes a show against a two-hour poll interval, which for
+// forty-two shows meant the machine never finished a cycle and never idled.
+func (s *Show) NeedsDetection(episodeIDs []string) bool {
+ return s.DetectedBatch != batchKey(episodeIDs)
+}
+
+// RecordDetected notes that this exact batch has been examined.
+//
+// Called only after a pass completes, so a cancelled or failed one is retried rather than
+// written off.
+func (s *Show) RecordDetected(episodeIDs []string) {
+ s.DetectedBatch = batchKey(episodeIDs)
+}
+
+// batchKey identifies a set of episodes irrespective of order, which matters because feeds are
+// not reliably sorted: the same eight episodes arriving shuffled must not read as a change.
+func batchKey(episodeIDs []string) string {
+ ordered := append([]string(nil), episodeIDs...)
+ sort.Strings(ordered)
+ return strings.Join(ordered, " ")
+}
diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go
index cd5cf6f..0540fbf 100644
--- a/server/internal/store/store_test.go
+++ b/server/internal/store/store_test.go
@@ -237,3 +237,85 @@ func TestSaveIsAtomic(t *testing.T) {
}
}
}
+
+// Detection is the expensive thing this server does -- four minutes a show -- and it used to
+// re-run on every poll over episodes that had not changed, producing results identical to the
+// digit. Forty-two shows at ~6.5 minutes each is 4.5 hours of work against a two-hour poll
+// interval, so the machine never finished a cycle and never idled.
+//
+// The batch is what has to match, not the individual episode: every answer depends on which
+// other episodes it was compared against, so one new episode legitimately changes all of them.
+
+func TestAShowNotYetExaminedNeedsDetection(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
+
+ if !show.NeedsDetection([]string{"a", "b"}) {
+ t.Fatal("a show that has never been examined should need detection")
+ }
+}
+
+func TestTheSameBatchIsNotExaminedTwice(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", nil)
+
+ show.RecordDetected([]string{"a", "b", "c"})
+
+ if show.NeedsDetection([]string{"a", "b", "c"}) {
+ t.Fatal("re-examining an unchanged batch; this is the rerun that pegged the server")
+ }
+}
+
+func TestANewEpisodeInTheBatchNeedsDetectionAgain(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", nil)
+ show.RecordDetected([]string{"a", "b", "c"})
+
+ if !show.NeedsDetection([]string{"a", "b", "c", "d"}) {
+ t.Fatal("a new episode changes what every other episode is compared against")
+ }
+}
+
+func TestAPrunedEpisodeNeedsDetectionAgain(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", nil)
+ show.RecordDetected([]string{"a", "b", "c"})
+
+ if !show.NeedsDetection([]string{"b", "c"}) {
+ t.Fatal("losing an episode changes the comparison too")
+ }
+}
+
+// Feeds are not reliably ordered, and Partition sorts by publish date, so the same eight
+// episodes can arrive in a different order. Treating that as a change would never close
+// the gate.
+func TestTheOrderOfTheBatchDoesNotMatter(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", nil)
+ show.RecordDetected([]string{"a", "b", "c"})
+
+ if show.NeedsDetection([]string{"c", "a", "b"}) {
+ t.Fatal("the same batch in a different order is the same batch")
+ }
+}
+
+// Otherwise every restart redoes every show -- which at 4.5 hours a cycle is most of a day.
+func TestWhatHasBeenExaminedSurvivesARestart(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "state.json")
+
+ s := New()
+ s.Merge("https://f", "A Show", nil)
+ s.Show("https://f").RecordDetected([]string{"a", "b"})
+ if err := s.Save(path); err != nil {
+ t.Fatalf("save: %v", err)
+ }
+
+ loaded, err := Load(path)
+ if err != nil {
+ t.Fatalf("load: %v", err)
+ }
+ if loaded.Show("https://f").NeedsDetection([]string{"a", "b"}) {
+ t.Fatal("a restart would redo every show")
+ }
+}