summaryrefslogtreecommitdiff
path: root/server/internal
diff options
context:
space:
mode:
Diffstat (limited to 'server/internal')
-rw-r--r--server/internal/store/store.go31
-rw-r--r--server/internal/store/store_test.go82
2 files changed, 113 insertions, 0 deletions
diff --git a/server/internal/store/store.go b/server/internal/store/store.go
index a67e637..51e4ed7 100644
--- a/server/internal/store/store.go
+++ b/server/internal/store/store.go
@@ -75,6 +75,10 @@ type Show struct {
// says so instead of silently having nothing.
LastPolledUnix int64 `json:"lastPolledUnix,omitempty"`
LastError string `json:"lastError,omitempty"`
+
+ // DetectedBatch is the set of episodes the last detection pass examined. Stored as the
+ // plain ids rather than a hash so the state file says what was actually done.
+ DetectedBatch string `json:"detectedBatch,omitempty"`
}
// Partition splits a show's episodes into the newest keep and everything else, newest first.
@@ -316,3 +320,30 @@ func matchIn(show *Show, guid, enclosureURL, title string) *Episode {
}
return nil
}
+
+// NeedsDetection reports whether this batch of episodes still has to be examined.
+//
+// The batch is the unit, not the episode: every offset and every ad break is found by comparing
+// an episode against the others fetched alongside it, so one arrival or one pruning legitimately
+// changes all of them. An unchanged batch, though, produces results identical to the digit --
+// and re-deriving them cost four minutes a show against a two-hour poll interval, which for
+// forty-two shows meant the machine never finished a cycle and never idled.
+func (s *Show) NeedsDetection(episodeIDs []string) bool {
+ return s.DetectedBatch != batchKey(episodeIDs)
+}
+
+// RecordDetected notes that this exact batch has been examined.
+//
+// Called only after a pass completes, so a cancelled or failed one is retried rather than
+// written off.
+func (s *Show) RecordDetected(episodeIDs []string) {
+ s.DetectedBatch = batchKey(episodeIDs)
+}
+
+// batchKey identifies a set of episodes irrespective of order, which matters because feeds are
+// not reliably sorted: the same eight episodes arriving shuffled must not read as a change.
+func batchKey(episodeIDs []string) string {
+ ordered := append([]string(nil), episodeIDs...)
+ sort.Strings(ordered)
+ return strings.Join(ordered, " ")
+}
diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go
index cd5cf6f..0540fbf 100644
--- a/server/internal/store/store_test.go
+++ b/server/internal/store/store_test.go
@@ -237,3 +237,85 @@ func TestSaveIsAtomic(t *testing.T) {
}
}
}
+
+// Detection is the expensive thing this server does -- four minutes a show -- and it used to
+// re-run on every poll over episodes that had not changed, producing results identical to the
+// digit. Forty-two shows at ~6.5 minutes each is 4.5 hours of work against a two-hour poll
+// interval, so the machine never finished a cycle and never idled.
+//
+// The batch is what has to match, not the individual episode: every answer depends on which
+// other episodes it was compared against, so one new episode legitimately changes all of them.
+
+func TestAShowNotYetExaminedNeedsDetection(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
+
+ if !show.NeedsDetection([]string{"a", "b"}) {
+ t.Fatal("a show that has never been examined should need detection")
+ }
+}
+
+func TestTheSameBatchIsNotExaminedTwice(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", nil)
+
+ show.RecordDetected([]string{"a", "b", "c"})
+
+ if show.NeedsDetection([]string{"a", "b", "c"}) {
+ t.Fatal("re-examining an unchanged batch; this is the rerun that pegged the server")
+ }
+}
+
+func TestANewEpisodeInTheBatchNeedsDetectionAgain(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", nil)
+ show.RecordDetected([]string{"a", "b", "c"})
+
+ if !show.NeedsDetection([]string{"a", "b", "c", "d"}) {
+ t.Fatal("a new episode changes what every other episode is compared against")
+ }
+}
+
+func TestAPrunedEpisodeNeedsDetectionAgain(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", nil)
+ show.RecordDetected([]string{"a", "b", "c"})
+
+ if !show.NeedsDetection([]string{"b", "c"}) {
+ t.Fatal("losing an episode changes the comparison too")
+ }
+}
+
+// Feeds are not reliably ordered, and Partition sorts by publish date, so the same eight
+// episodes can arrive in a different order. Treating that as a change would never close
+// the gate.
+func TestTheOrderOfTheBatchDoesNotMatter(t *testing.T) {
+ s := New()
+ show := s.Merge("https://f", "A Show", nil)
+ show.RecordDetected([]string{"a", "b", "c"})
+
+ if show.NeedsDetection([]string{"c", "a", "b"}) {
+ t.Fatal("the same batch in a different order is the same batch")
+ }
+}
+
+// Otherwise every restart redoes every show -- which at 4.5 hours a cycle is most of a day.
+func TestWhatHasBeenExaminedSurvivesARestart(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "state.json")
+
+ s := New()
+ s.Merge("https://f", "A Show", nil)
+ s.Show("https://f").RecordDetected([]string{"a", "b"})
+ if err := s.Save(path); err != nil {
+ t.Fatalf("save: %v", err)
+ }
+
+ loaded, err := Load(path)
+ if err != nil {
+ t.Fatalf("load: %v", err)
+ }
+ if loaded.Show("https://f").NeedsDetection([]string{"a", "b"}) {
+ t.Fatal("a restart would redo every show")
+ }
+}