diff options
Diffstat (limited to 'server/internal')
| -rw-r--r-- | server/internal/store/store.go | 31 | ||||
| -rw-r--r-- | server/internal/store/store_test.go | 82 |
2 files changed, 113 insertions, 0 deletions
diff --git a/server/internal/store/store.go b/server/internal/store/store.go index a67e637..51e4ed7 100644 --- a/server/internal/store/store.go +++ b/server/internal/store/store.go @@ -75,6 +75,10 @@ type Show struct { // says so instead of silently having nothing. LastPolledUnix int64 `json:"lastPolledUnix,omitempty"` LastError string `json:"lastError,omitempty"` + + // DetectedBatch is the set of episodes the last detection pass examined. Stored as the + // plain ids rather than a hash so the state file says what was actually done. + DetectedBatch string `json:"detectedBatch,omitempty"` } // Partition splits a show's episodes into the newest keep and everything else, newest first. @@ -316,3 +320,30 @@ func matchIn(show *Show, guid, enclosureURL, title string) *Episode { } return nil } + +// NeedsDetection reports whether this batch of episodes still has to be examined. +// +// The batch is the unit, not the episode: every offset and every ad break is found by comparing +// an episode against the others fetched alongside it, so one arrival or one pruning legitimately +// changes all of them. An unchanged batch, though, produces results identical to the digit -- +// and re-deriving them cost four minutes a show against a two-hour poll interval, which for +// forty-two shows meant the machine never finished a cycle and never idled. +func (s *Show) NeedsDetection(episodeIDs []string) bool { + return s.DetectedBatch != batchKey(episodeIDs) +} + +// RecordDetected notes that this exact batch has been examined. +// +// Called only after a pass completes, so a cancelled or failed one is retried rather than +// written off. +func (s *Show) RecordDetected(episodeIDs []string) { + s.DetectedBatch = batchKey(episodeIDs) +} + +// batchKey identifies a set of episodes irrespective of order, which matters because feeds are +// not reliably sorted: the same eight episodes arriving shuffled must not read as a change. +func batchKey(episodeIDs []string) string { + ordered := append([]string(nil), episodeIDs...) + sort.Strings(ordered) + return strings.Join(ordered, " ") +} diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go index cd5cf6f..0540fbf 100644 --- a/server/internal/store/store_test.go +++ b/server/internal/store/store_test.go @@ -237,3 +237,85 @@ func TestSaveIsAtomic(t *testing.T) { } } } + +// Detection is the expensive thing this server does -- four minutes a show -- and it used to +// re-run on every poll over episodes that had not changed, producing results identical to the +// digit. Forty-two shows at ~6.5 minutes each is 4.5 hours of work against a two-hour poll +// interval, so the machine never finished a cycle and never idled. +// +// The batch is what has to match, not the individual episode: every answer depends on which +// other episodes it was compared against, so one new episode legitimately changes all of them. + +func TestAShowNotYetExaminedNeedsDetection(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)}) + + if !show.NeedsDetection([]string{"a", "b"}) { + t.Fatal("a show that has never been examined should need detection") + } +} + +func TestTheSameBatchIsNotExaminedTwice(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", nil) + + show.RecordDetected([]string{"a", "b", "c"}) + + if show.NeedsDetection([]string{"a", "b", "c"}) { + t.Fatal("re-examining an unchanged batch; this is the rerun that pegged the server") + } +} + +func TestANewEpisodeInTheBatchNeedsDetectionAgain(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", nil) + show.RecordDetected([]string{"a", "b", "c"}) + + if !show.NeedsDetection([]string{"a", "b", "c", "d"}) { + t.Fatal("a new episode changes what every other episode is compared against") + } +} + +func TestAPrunedEpisodeNeedsDetectionAgain(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", nil) + show.RecordDetected([]string{"a", "b", "c"}) + + if !show.NeedsDetection([]string{"b", "c"}) { + t.Fatal("losing an episode changes the comparison too") + } +} + +// Feeds are not reliably ordered, and Partition sorts by publish date, so the same eight +// episodes can arrive in a different order. Treating that as a change would never close +// the gate. +func TestTheOrderOfTheBatchDoesNotMatter(t *testing.T) { + s := New() + show := s.Merge("https://f", "A Show", nil) + show.RecordDetected([]string{"a", "b", "c"}) + + if show.NeedsDetection([]string{"c", "a", "b"}) { + t.Fatal("the same batch in a different order is the same batch") + } +} + +// Otherwise every restart redoes every show -- which at 4.5 hours a cycle is most of a day. +func TestWhatHasBeenExaminedSurvivesARestart(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + + s := New() + s.Merge("https://f", "A Show", nil) + s.Show("https://f").RecordDetected([]string{"a", "b"}) + if err := s.Save(path); err != nil { + t.Fatalf("save: %v", err) + } + + loaded, err := Load(path) + if err != nil { + t.Fatalf("load: %v", err) + } + if loaded.Show("https://f").NeedsDetection([]string{"a", "b"}) { + t.Fatal("a restart would redo every show") + } +} |
