summaryrefslogtreecommitdiff
path: root/server/internal
diff options
context:
space:
mode:
Diffstat (limited to 'server/internal')
-rw-r--r--server/internal/api/api.go17
-rw-r--r--server/internal/api/api_test.go23
-rw-r--r--server/internal/store/store.go36
-rw-r--r--server/internal/store/store_test.go70
-rw-r--r--server/internal/transcript/publish.go22
-rw-r--r--server/internal/transcript/publish_test.go79
-rw-r--r--server/internal/words/pipeline.go29
-rw-r--r--server/internal/words/pipeline_test.go47
8 files changed, 310 insertions, 13 deletions
diff --git a/server/internal/api/api.go b/server/internal/api/api.go
index 6114c1b..d716bb4 100644
--- a/server/internal/api/api.go
+++ b/server/internal/api/api.go
@@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"strings"
+ "time"
"github.com/terst/listen/server/internal/store"
"github.com/terst/listen/server/internal/transcript"
@@ -46,6 +47,8 @@ type Server struct {
state *store.State
fetcher Fetcher
opts Options
+ // clock is the time source, left nil outside tests.
+ clock func() int64
}
// New builds the server.
@@ -53,6 +56,14 @@ func New(state *store.State, fetcher Fetcher, opts Options) *Server {
return &Server{state: state, fetcher: fetcher, opts: opts}
}
+// now is the clock, overridable in tests the same way the words pipeline's is.
+func (s *Server) now() int64 {
+ if s.clock != nil {
+ return s.clock()
+ }
+ return time.Now().Unix()
+}
+
type episodeResponse struct {
// Status is "ready" when the audio can be played now, or "pending" when the server has
// been asked to fetch it. There is deliberately no "unknown": an episode the phone knows
@@ -211,6 +222,12 @@ func (s *Server) episode(w http.ResponseWriter, r *http.Request) {
}
found := s.state.Match(feedURL, guid, enclosure, title)
+ if found != nil {
+ // The only moment the server learns anybody wants this episode. Recorded whether or not
+ // it is mirrored yet, and on every ask, so resuming something played last month keeps
+ // it from falling out of the mirror while it is being listened to.
+ found.Requested(s.now())
+ }
if found == nil || !found.Mirrored() {
// Ask for it rather than refusing. Everything Peter actually listens to should end up
// with offsets, including an episode older than the ones kept by default.
diff --git a/server/internal/api/api_test.go b/server/internal/api/api_test.go
index 94f7ef4..8a29b33 100644
--- a/server/internal/api/api_test.go
+++ b/server/internal/api/api_test.go
@@ -418,3 +418,26 @@ func TestATierForARemovedFeedIsDropped(t *testing.T) {
t.Errorf("got %q, want it dropped with the feed", got)
}
}
+
+/*
+Asking for an episode is what keeps it.
+
+Retention used to be publication date alone, so an episode outside the newest few was fetched on
+demand, served, and deleted on the next poll together with its transcript -- 23% of the server's
+whisper time over three weeks went on episodes it had already recognised and thrown away. The
+window still decides what is worth fetching; this is what decides what is worth keeping, and this
+request is the only moment the server learns that anybody wants it.
+
+Recorded for an episode already mirrored too, not only a missing one: resuming something played
+last month must not let it fall out while it is being listened to.
+*/
+func TestAskingForAnEpisodeMarksItWanted(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ seedMirrored(t, st, audioDir, "audio-bytes")
+
+ get(t, s, "/api/episode?feed=https://f&guid=g1")
+
+ if got := st.Match("https://f", "g1", "", "").RequestedAtUnix; got == 0 {
+ t.Error("asking for a mirrored episode left it unwanted, so a poll may still drop it")
+ }
+}
diff --git a/server/internal/store/store.go b/server/internal/store/store.go
index 2e20f2e..de763ed 100644
--- a/server/internal/store/store.go
+++ b/server/internal/store/store.go
@@ -78,6 +78,10 @@ type Episode struct {
TranscriptCues int `json:"transcriptCues,omitempty"`
// TranscriptAtUnix is when the transcript landed.
TranscriptAtUnix int64 `json:"transcriptAtUnix,omitempty"`
+ // RequestedAtUnix is when the phone last asked for this episode, which keeps it mirrored
+ // past the newest-N window.Zero for an episode nobody has reached for.
+ RequestedAtUnix int64 `json:"requestedAtUnix,omitempty"`
+
// MasterCues is how many cues the publisher's ad-free transcript holds, kept for episodes
// whose served copy carries inserted advertising. MasterNote is why there is none. Three
// states as everywhere else: words, a reason, or nobody has looked.
@@ -128,6 +132,23 @@ func (e *Episode) NeedsMaster() bool {
e.MasterCues == 0 && e.MasterNote == ""
}
+// keepRequestedSeconds is how long an episode stays mirrored after the phone last asked for it,
+// regardless of how far outside the newest-N window it has fallen.
+//
+// Four weeks, the same span refetchAfterSeconds uses, because it is the same kind of judgement
+// about how long something stays interesting. Long enough to work through a back catalogue
+// without re-fetching what was played last week; short enough that a binge does not pin a
+// show's whole archive to the disk for ever.
+const keepRequestedSeconds = 28 * 24 * 60 * 60
+
+// Requested records that the phone asked for this episode, which is what keeps it mirrored.
+func (e *Episode) Requested(now int64) { e.RequestedAtUnix = now }
+
+// wanted reports whether the phone has asked for this episode recently enough to keep it.
+func (e *Episode) wanted(now int64) bool {
+ return e.RequestedAtUnix > 0 && now-e.RequestedAtUnix < keepRequestedSeconds
+}
+
// refetchAfterSeconds is how old an episode must be before a second fetch is worth making.
// Four weeks: long enough that whatever campaign was running at mirroring has given way, and
// short enough that the episode is usually still being served.
@@ -256,7 +277,7 @@ type Show struct {
//
// By publish date rather than feed order, because feeds are not reliably sorted and the whole
// point is to mirror what is current.
-func (s *Show) Partition(keep int) (kept, pruned []*Episode) {
+func (s *Show) Partition(keep int, now int64) (kept, pruned []*Episode) {
ordered := make([]*Episode, len(s.Episodes))
copy(ordered, s.Episodes)
sort.SliceStable(ordered, func(i, j int) bool {
@@ -265,7 +286,18 @@ func (s *Show) Partition(keep int) (kept, pruned []*Episode) {
if keep >= len(ordered) {
return ordered, nil
}
- return ordered[:keep], ordered[keep:]
+ // The window decides what is worth fetching; being asked for decides what is worth keeping.
+ // Without this an episode outside the newest N was fetched on demand, served, and deleted
+ // again on the next poll -- with its transcript, which is the expensive half.
+ kept, pruned = ordered[:keep:keep], nil
+ for _, e := range ordered[keep:] {
+ if e.wanted(now) {
+ kept = append(kept, e)
+ continue
+ }
+ pruned = append(pruned, e)
+ }
+ return kept, pruned
}
// State is the server's whole persistent world.
diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go
index a88fdd4..657e7b1 100644
--- a/server/internal/store/store_test.go
+++ b/server/internal/store/store_test.go
@@ -136,7 +136,7 @@ func TestKeepNewestSelectsByPublishDateNotFeedOrder(t *testing.T) {
ep("g2", "Middle", "https://cdn/2.mp3", 200),
})
- keep, prune := s.Show("https://f").Partition(2)
+ keep, prune := s.Show("https://f").Partition(2, 0)
if len(keep) != 2 || len(prune) != 1 {
t.Fatalf("keep %d prune %d, want 2 and 1", len(keep), len(prune))
}
@@ -152,7 +152,7 @@ func TestKeepNewestKeepsEverythingWhenThereAreFewerThanTheLimit(t *testing.T) {
s := New()
s.Merge("https://f", "A Show", []feed.Episode{ep("g1", "One", "https://cdn/1.mp3", 100)})
- keep, prune := s.Show("https://f").Partition(8)
+ keep, prune := s.Show("https://f").Partition(8, 0)
if len(keep) != 1 || len(prune) != 0 {
t.Errorf("keep %d prune %d, want 1 and 0", len(keep), len(prune))
}
@@ -745,3 +745,69 @@ func TestTheGuardNamesWhatItRefused(t *testing.T) {
t.Errorf("error %q does not say how much it would have lost", err)
}
}
+
+/*
+An episode the phone asked for survives the keep window.
+
+Retention was publication date and nothing else, so anything outside the newest few was fetched
+on demand, served, and deleted again within one poll -- audio, transcript and all, because
+words.Forget removes the transcript file too. Playing a back catalogue therefore paid the full
+cost of recognition every time, and paid it again to resume.
+
+Measured on the live server over twenty-one days: 397 transcription runs covering 309 distinct
+episodes, 7.7 hours of the 33.4 hours of whisper time spent on episodes already transcribed once.
+23% of the budget, re-doing work that had been thrown away.
+
+So the window decides what to fetch, and being wanted decides what to keep.
+*/
+func TestAnEpisodeTheListenerAskedForIsNotPruned(t *testing.T) {
+ const now = 1_800_000_000
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{
+ ep("g1", "Oldest", "https://cdn/1.mp3", 100),
+ ep("g2", "Middle", "https://cdn/2.mp3", 200),
+ ep("g3", "Newest", "https://cdn/3.mp3", 300),
+ })
+ show := s.Show("https://f")
+ // The oldest, which the window would drop, asked for a moment ago.
+ for _, e := range show.Episodes {
+ if e.GUID == "g1" {
+ e.Requested(now)
+ }
+ }
+
+ keep, prune := show.Partition(1, now)
+
+ if len(prune) != 1 || prune[0].GUID != "g2" {
+ t.Fatalf("pruned %v, want only g2", guids(prune))
+ }
+ if len(keep) != 2 {
+ t.Fatalf("kept %v, want the newest and the one asked for", guids(keep))
+ }
+}
+
+// Wanted once is not wanted for ever, or a back-catalogue binge would keep every episode's audio.
+func TestAnEpisodeAskedForLongAgoIsPrunedAgain(t *testing.T) {
+ const now = 1_800_000_000
+ s := New()
+ s.Merge("https://f", "A Show", []feed.Episode{
+ ep("g1", "Oldest", "https://cdn/1.mp3", 100),
+ ep("g2", "Newest", "https://cdn/2.mp3", 200),
+ })
+ show := s.Show("https://f")
+ show.Episodes[0].Requested(now - keepRequestedSeconds - 1)
+
+ _, prune := show.Partition(1, now)
+
+ if len(prune) != 1 || prune[0].GUID != "g1" {
+ t.Errorf("pruned %v, want g1 back in the window's hands", guids(prune))
+ }
+}
+
+func guids(episodes []*Episode) []string {
+ out := make([]string, 0, len(episodes))
+ for _, e := range episodes {
+ out = append(out, e.GUID)
+ }
+ return out
+}
diff --git a/server/internal/transcript/publish.go b/server/internal/transcript/publish.go
index b9856c3..f79b61c 100644
--- a/server/internal/transcript/publish.go
+++ b/server/internal/transcript/publish.go
@@ -40,7 +40,7 @@ const overshootLimit = 150
//
// A refusal is a different state from having found nothing, and the caller keeps them apart --
// the same distinction the ad scan's own refusals have needed twice before.
-func Publishable(found []Candidate, servedMs, declaredMs int64) ([]Candidate, string) {
+func Publishable(found []Candidate, servedMs, declaredMs int64, master Source) ([]Candidate, string) {
if len(found) == 0 {
return nil, ""
}
@@ -55,6 +55,26 @@ func Publishable(found []Candidate, servedMs, declaredMs int64) ([]Candidate, st
return nil, "found by reading the ad copy, which has never agreed with the lengths"
}
+ // What was subtracted decides whether this is a measurement or a guess. A publisher's
+ // transcript describes the programme once; a refetch is another copy of the same assembled
+ // file, with its own advertising in its own places, so the difference between them is two ad
+ // loads rather than one. Measured on the live corpus: 94-104% of the length gap from a
+ // published master, a median of 33% from a refetch.
+ //
+ // An allowlist, not a "not refetch" test. Sixteen episodes carry a master whose label is
+ // blank -- the field postdates the first masters written -- and a denylist published them on
+ // the strength of not being the one named bad kind.
+ if master != SourcePublished {
+ kind := string(master)
+ if kind == "" {
+ kind = "a master that does not say what it is"
+ } else {
+ kind = "a " + kind
+ }
+ return nil, fmt.Sprintf("subtracted from %s rather than the publisher's transcript, "+
+ "which is the only master measured accurate enough to skip on", kind)
+ }
+
gap := servedMs - declaredMs
if !HasInsertedAdvertising(servedMs, declaredMs) {
return nil, "the lengths say nothing was inserted into this copy, so there is " +
diff --git a/server/internal/transcript/publish_test.go b/server/internal/transcript/publish_test.go
index 8c98a8f..92153f8 100644
--- a/server/internal/transcript/publish_test.go
+++ b/server/internal/transcript/publish_test.go
@@ -29,7 +29,7 @@ func TestSubtractedAdvertisingIsPublished(t *testing.T) {
found := candidates(SignalDifference, [2]int64{0, 57_000}, [2]int64{164_000, 304_000},
[2]int64{1_928_000, 1_992_000})
- got, why := Publishable(found, 3_813_000, 3_557_000)
+ got, why := Publishable(found, 3_813_000, 3_557_000, SourcePublished)
if why != "" {
t.Fatalf("refused: %s", why)
}
@@ -48,7 +48,7 @@ hint beside the words; not something to seek past.
func TestAdvertisingReadFromTheCopyIsNotPublished(t *testing.T) {
found := candidates("sponsor", [2]int64{60_000, 120_000})
- got, why := Publishable(found, 1_480_000, 1_216_000)
+ got, why := Publishable(found, 1_480_000, 1_216_000, SourcePublished)
if len(got) != 0 {
t.Errorf("published %d breaks found only by their wording", len(got))
}
@@ -67,7 +67,7 @@ finding it is what reading the copy is for. Acting on it is a different matter.
func TestNothingIsPublishedWithoutALengthGapToCheckAgainst(t *testing.T) {
found := candidates(SignalDifference, [2]int64{60_000, 120_000})
- got, why := Publishable(found, 1_216_500, 1_216_000)
+ got, why := Publishable(found, 1_216_500, 1_216_000, SourcePublished)
if len(got) != 0 {
t.Errorf("published %d breaks with nothing to check them against", len(got))
}
@@ -86,7 +86,7 @@ func TestClaimingHalfAgainAsMuchAsWasInsertedIsRefused(t *testing.T) {
// 4:15 inserted, but the candidates come to 9:00.
found := candidates(SignalDifference, [2]int64{0, 540_000})
- got, why := Publishable(found, 3_813_000, 3_557_000)
+ got, why := Publishable(found, 3_813_000, 3_557_000, SourcePublished)
if len(got) != 0 {
t.Errorf("published %d breaks claiming more than twice what was inserted", len(got))
}
@@ -101,14 +101,14 @@ func TestAModestOvershootIsStillPublished(t *testing.T) {
// 4:15 inserted, 4:40 found -- 110%.
found := candidates(SignalDifference, [2]int64{0, 280_000})
- got, why := Publishable(found, 3_813_000, 3_557_000)
+ got, why := Publishable(found, 3_813_000, 3_557_000, SourcePublished)
if len(got) != 1 || why != "" {
t.Errorf("got %d breaks, refused %q", len(got), why)
}
}
func TestNothingFoundPublishesNothingAndSaysNothing(t *testing.T) {
- got, why := Publishable(nil, 3_813_000, 3_557_000)
+ got, why := Publishable(nil, 3_813_000, 3_557_000, SourcePublished)
if len(got) != 0 {
t.Errorf("got %d breaks from nothing", len(got))
}
@@ -185,3 +185,70 @@ func TestNoRecurringTextPublishesNothing(t *testing.T) {
t.Errorf("got %d breaks, refused %q", len(got), why)
}
}
+
+/*
+Subtraction against a second fetch is not fit to skip.
+
+The master this subtracts from decides whether the result is a measurement or a guess, and until
+now nothing here asked which it was. A publisher's transcript is a second *description* of the
+programme, written once: subtracting it recovered 94% to 104% of the length gap across nine
+episodes in the live corpus. A refetch is a second *copy of the same dynamically-assembled file*,
+carrying its own advertising at its own positions, so the diff compares two ad loads instead of
+finding one -- a median of 33% across fifteen episodes, range 20% to 146%.
+
+That 33% is the same tier that measured 21% and 0% when the refetch detector was first tried and
+rejected, and it was reaching the phone anyway because this function was never told the
+difference. Measured 2026-11-02; see docs/detection-findings.md.
+*/
+func TestSubtractionFromARefetchIsNotPublished(t *testing.T) {
+ // Reply All #188: the published span claimed 146% of a length gap that is itself noise.
+ found := candidates(SignalDifference, [2]int64{1_154_200, 1_243_000})
+
+ got, why := Publishable(found, 2_616_000, 2_555_000, SourceRefetch)
+
+ if len(got) != 0 {
+ t.Errorf("published %d breaks from a refetch master, want none", len(got))
+ }
+ if why == "" {
+ t.Error("refused silently; a refusal has to say why, the same as every other one here")
+ }
+}
+
+/* The same candidates, from a publisher's master, still publish. */
+func TestSubtractionFromAPublishedMasterStillPublishes(t *testing.T) {
+ found := candidates(SignalDifference, [2]int64{0, 57_000}, [2]int64{164_000, 304_000})
+
+ got, why := Publishable(found, 3_813_000, 3_557_000, SourcePublished)
+ if why != "" {
+ t.Fatalf("refused: %s", why)
+ }
+ if len(got) != 2 {
+ t.Errorf("got %d breaks, want 2", len(got))
+ }
+}
+
+/*
+A master that will not say what it is does not publish either.
+
+The refetch check started as a denylist and that was the wrong shape. Sixteen episodes in the live
+corpus carry a master whose state-side label is blank -- the field postdates the first masters
+written, and Reclaim restored those files faithfully, empty label and all -- so they passed a
+"not refetch" test and published. They happened to be publisher transcripts, so the outcome was
+right by luck.
+
+An allowlist is the shape that does not depend on luck: subtraction publishes when the master is
+known to be the publisher's, and says so otherwise. Same reasoning as safefetch's, where the
+denylist missed a proxy and a whole family of address ranges.
+*/
+func TestSubtractionFromAnUnlabelledMasterIsNotPublished(t *testing.T) {
+ found := candidates(SignalDifference, [2]int64{0, 57_000}, [2]int64{164_000, 304_000})
+
+ got, why := Publishable(found, 3_813_000, 3_557_000, Source(""))
+
+ if len(got) != 0 {
+ t.Errorf("published %d breaks from an unlabelled master, want none", len(got))
+ }
+ if why == "" {
+ t.Error("refused silently")
+ }
+}
diff --git a/server/internal/words/pipeline.go b/server/internal/words/pipeline.go
index 2ae2c4e..8cd18ed 100644
--- a/server/internal/words/pipeline.go
+++ b/server/internal/words/pipeline.go
@@ -406,7 +406,13 @@ func (p *Pipeline) Recurring(show *store.Show) {
}
found := transcript.Recurring(target.cues, others)
fit, why := transcript.PublishableRecurring(found, target.episode.DurationMs)
- target.episode.WordAdsRefused = why
+ // Both passes answer "why is nothing published" through one field, and this one runs
+ // second. Overwriting unconditionally discarded whatever subtraction had just said --
+ // which, once subtraction began refusing every master that is not the publisher's, is
+ // the only explanation thirty-two episodes had.
+ if why != "" {
+ target.episode.WordAdsRefused = why
+ }
target.episode.WordBreaks = nil
for _, c := range fit {
target.episode.WordBreaks = append(target.episode.WordBreaks, store.Break{
@@ -441,7 +447,7 @@ Kept in their own field so the ad scan, which rewrites its own list from scratch
detection pass, cannot wipe them -- and so that which detector found a break stays answerable.
*/
func (p *Pipeline) publishWordBreaks(e *store.Episode, found []transcript.Candidate) {
- fit, why := transcript.Publishable(found, e.DurationMs, e.DeclaredDurationMs)
+ fit, why := transcript.Publishable(found, e.DurationMs, e.DeclaredDurationMs, p.masterSource(e))
e.WordAdsRefused = why
e.WordBreaks = nil
for _, c := range fit {
@@ -454,6 +460,25 @@ func (p *Pipeline) publishWordBreaks(e *store.Episode, found []transcript.Candid
}
/*
+masterSource is what kind of master an episode has, repaired from the master file when the state
+has no label for it.
+
+Publishing depends on this now, and the state's copy of it is not reliably present: the field
+postdates the first masters written, and Reclaim -- which rebuilt the words from the transcript
+files after the library was destroyed -- restored those files faithfully, blank label and all.
+Eighteen masters in the live corpus describe themselves correctly on disk and said nothing in the
+state. The file is the record; this copies it back.
+*/
+func (p *Pipeline) masterSource(e *store.Episode) transcript.Source {
+ if e.MasterSource == "" && e.MasterCues > 0 {
+ if master, err := p.Transcripts.Master(e.ID); err == nil && master.Source != "" {
+ e.MasterSource = string(master.Source)
+ }
+ }
+ return transcript.Source(e.MasterSource)
+}
+
+/*
adsIn works out which stretches of a recognised episode are advertising.
By subtraction where the publisher's ad-free master is held, and by reading the ad copy where it
diff --git a/server/internal/words/pipeline_test.go b/server/internal/words/pipeline_test.go
index c010c8e..9982a10 100644
--- a/server/internal/words/pipeline_test.go
+++ b/server/internal/words/pipeline_test.go
@@ -204,3 +204,50 @@ func TestReclaimLeavesWhatIsAlreadyThere(t *testing.T) {
t.Error("overwrote words that were already there")
}
}
+
+/*
+The recurring pass does not wipe the reason subtraction gave.
+
+Both passes write the same WordAdsRefused field, for two different judgements, and the recurring
+one runs second. It blanked the field on every episode it succeeded on, so an episode whose
+advertising was refused -- and then given the show's own cues instead -- arrived at the phone
+saying nothing about the advertising.
+
+That was survivable while refusals were rare. Making subtraction publish only from a publisher's
+master put a reason on thirty-two episodes in the live corpus, and measuring the change on a copy
+of that data is what showed all thirty-two being discarded a moment later.
+*/
+func TestRecurringKeepsSubtractionsRefusal(t *testing.T) {
+ dir := t.TempDir()
+ held := transcript.Store{Dir: dir}
+ // Three episodes with words, which clears the corpus floor, and nothing shared between them
+ // so the pass publishes no cues -- the case where it previously still blanked the field.
+ for _, id := range []string{"e1", "e2", "e3"} {
+ if err := held.Put(transcript.Transcript{
+ EpisodeID: id, Source: transcript.SourceASR,
+ Cues: []transcript.Cue{{StartMs: 0, EndMs: 1000, Text: "a line only " + id + " says"}},
+ }); err != nil {
+ t.Fatalf("Put %s: %v", id, err)
+ }
+ }
+
+ state := store.New()
+ state.Merge("https://f", "A Show", nil)
+ show := state.Show("https://f")
+ show.Episodes = []*store.Episode{
+ {ID: "e1", Path: "a.mp3", TranscriptCues: 1, DurationMs: 600_000, WordAdsRefused: "subtracted from a refetch"},
+ {ID: "e2", Path: "b.mp3", TranscriptCues: 1, DurationMs: 600_000, WordAdsRefused: "subtracted from a refetch"},
+ {ID: "e3", Path: "c.mp3", TranscriptCues: 1, DurationMs: 600_000, WordAdsRefused: "subtracted from a refetch"},
+ }
+
+ (&Pipeline{State: state, Transcripts: held}).Recurring(show)
+
+ for _, e := range show.Episodes {
+ if len(e.WordBreaks) != 0 {
+ t.Fatalf("%s published cues; the fixture meant to share nothing", e.ID)
+ }
+ if e.WordAdsRefused == "" {
+ t.Errorf("%s lost the reason no advertising was published", e.ID)
+ }
+ }
+}
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-14 22:39:33 +0000