summaryrefslogtreecommitdiff
path: root/server/internal
diff options
context:
space:
mode:
Diffstat (limited to 'server/internal')
-rw-r--r--server/internal/api/api.go5
-rw-r--r--server/internal/api/transcript_test.go37
-rw-r--r--server/internal/store/store.go66
-rw-r--r--server/internal/store/store_test.go129
-rw-r--r--server/internal/transcript/publish.go61
-rw-r--r--server/internal/transcript/publish_test.go118
6 files changed, 405 insertions, 11 deletions
diff --git a/server/internal/api/api.go b/server/internal/api/api.go
index 0d18c0f..367a1c6 100644
--- a/server/internal/api/api.go
+++ b/server/internal/api/api.go
@@ -207,8 +207,9 @@ func (s *Server) episode(w http.ResponseWriter, r *http.Request) {
return
}
- breaks := make([]breakRange, 0, len(found.Breaks))
- for _, br := range found.Breaks {
+ all := found.AllBreaks()
+ breaks := make([]breakRange, 0, len(all))
+ for _, br := range all {
breaks = append(breaks, breakRange{
StartMs: br.StartMs,
EndMs: br.EndMs,
diff --git a/server/internal/api/transcript_test.go b/server/internal/api/transcript_test.go
index 6d0ad7f..e6da3ff 100644
--- a/server/internal/api/transcript_test.go
+++ b/server/internal/api/transcript_test.go
@@ -301,3 +301,40 @@ func TestAnEpisodeWithNoAdvertisingInItsWordsSaysNothing(t *testing.T) {
t.Errorf("got %+v, want nothing claimed", *got.Transcript)
}
}
+
+/*
+Advertising read out of the words reaches the phone as a break like any other.
+
+It arrives in the same field the audio scan's breaks do, because the phone has skipped those
+since the on-device detector existed and nothing about playback needs to change. Which detector
+found it is kept on the server; what the phone needs to know is where not to listen.
+*/
+func TestBreaksFoundInTheWordsAreServedWithTheRest(t *testing.T) {
+ s, st, _, audioDir := newServer(t)
+ st.Merge("https://f", "A Show", []feed.Episode{{
+ GUID: "g1", Title: "One", EnclosureURL: "https://cdn/1.mp3", PublishedUnix: 100,
+ }})
+ e := st.Match("https://f", "g1", "", "")
+ e.Path = e.ID + ".mp3"
+ writeAudio(t, audioDir, e.Path)
+ e.Breaks = []store.Break{{StartMs: 60_000, EndMs: 90_000, Kind: "ad"}}
+ e.WordBreaks = []store.Break{{StartMs: 600_000, EndMs: 660_000, Kind: "ad"}}
+
+ rec := get(t, s, "/api/episode?feed=https://f&guid=g1")
+ var got struct {
+ Breaks []struct {
+ StartMs int64 `json:"startMs"`
+ EndMs int64 `json:"endMs"`
+ Kind string `json:"kind"`
+ } `json:"breaks"`
+ }
+ if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if len(got.Breaks) != 2 {
+ t.Fatalf("got %+v, want both", got.Breaks)
+ }
+ if got.Breaks[1].StartMs != 600_000 {
+ t.Errorf("got %+v, want the one from the words included", got.Breaks)
+ }
+}
diff --git a/server/internal/store/store.go b/server/internal/store/store.go
index 2a6992b..b86474d 100644
--- a/server/internal/store/store.go
+++ b/server/internal/store/store.go
@@ -38,6 +38,16 @@ type Episode struct {
// which for simultaneously-mirrored episodes is the advertising inserted into them.
Breaks []Break `json:"breaks,omitempty"`
+ // WordBreaks are the stretches read out of this episode's words and judged fit to skip --
+ // see transcript.Publishable. Their own field because the two passes each rewrite their own
+ // list wholesale: the ad scan sets Breaks from scratch every time it runs, and would wipe
+ // anything the transcript had put there.
+ WordBreaks []Break `json:"wordBreaks,omitempty"`
+ // WordAdsRefused records that the words turned up advertising which was not fit to publish.
+ // A different state from having found none, and kept apart for the same reason as
+ // AdsRefused.
+ WordAdsRefused string `json:"wordAdsRefused,omitempty"`
+
// FoundAdMs and InsertedAdMs are what the scan accounted for against what the feed's
// declared length implies is there. Kept because the second number does not come from the
// detector and so is the only thing able to contradict it.
@@ -118,14 +128,30 @@ func (e *Episode) NeedsMaster() bool {
e.MasterCues == 0 && e.MasterNote == ""
}
+// refetchAfterSeconds is how old an episode must be before a second fetch is worth making.
+// Four weeks: long enough that whatever campaign was running at mirroring has given way, and
+// short enough that the episode is usually still being served.
+const refetchAfterSeconds = 28 * 24 * 60 * 60
+
// NeedsRefetch reports whether a second copy of this episode is worth fetching.
//
// Only for an episode that has its own words but no master to subtract them from, and only
// where the lengths say advertising was inserted: without a gap the two fetches would be the
// same recording and there would be nothing to find.
-func (e *Episode) NeedsRefetch() bool {
- return e.Mirrored() && e.HasTranscript() && e.EnclosureURL != "" &&
- e.MasterCues == 0 && e.MasterNote == "" && e.RefetchAttemptedAtUnix == 0
+func (e *Episode) NeedsRefetch(now int64) bool {
+ if !e.Mirrored() || !e.HasTranscript() || e.EnclosureURL == "" ||
+ e.MasterCues != 0 || e.MasterNote != "" || e.RefetchAttemptedAtUnix != 0 {
+ return false
+ }
+ // Old enough that the campaign has had time to turn over. Measured: a second fetch of a
+ // Tangle episode minutes after the first came back 20 seconds different and subtraction
+ // found 21% of the advertising; another came back the same length to the second and found
+ // none at all. Insertion varies the amount per request, not the campaign.
+ //
+ // So this is for the shows whose episodes stay worth listening to -- a slow-drip archive, a
+ // leisure show -- and not for daily news, whose episodes are pruned long before any
+ // campaign rotates.
+ return e.PublishedUnix > 0 && now-e.PublishedUnix >= refetchAfterSeconds
}
// NeedsRecognising reports whether speech recognition should be run over this episode.
@@ -150,6 +176,40 @@ type Break struct {
Kind string `json:"kind,omitempty"`
}
+// AllBreaks is every stretch to skip, from both detectors, as one ordered list.
+//
+// Overlapping and touching ranges are merged, because playback seeks to a break's end: two
+// overlapping breaks would seek out of the first and straight into the second, and two touching
+// ones would resume for no time at all before skipping again.
+//
+// Advertising wins where two merged ranges disagree about what they are. The phone skips
+// advertising unconditionally and a show's own audio only by preference, and being wrong the
+// other way means sitting through an advertisement the server had already found.
+func (e *Episode) AllBreaks() []Break {
+ all := make([]Break, 0, len(e.Breaks)+len(e.WordBreaks))
+ all = append(all, e.Breaks...)
+ all = append(all, e.WordBreaks...)
+ sort.Slice(all, func(i, j int) bool { return all[i].StartMs < all[j].StartMs })
+
+ var merged []Break
+ for _, br := range all {
+ if last := len(merged) - 1; last >= 0 && br.StartMs <= merged[last].EndMs {
+ if br.EndMs > merged[last].EndMs {
+ merged[last].EndMs = br.EndMs
+ }
+ if br.Kind == "ad" {
+ merged[last].Kind = "ad"
+ }
+ if br.Partners > merged[last].Partners {
+ merged[last].Partners = br.Partners
+ }
+ continue
+ }
+ merged = append(merged, br)
+ }
+ return merged
+}
+
// Mirrored reports whether this episode's audio is on disk.
func (e *Episode) Mirrored() bool { return e.Path != "" }
diff --git a/server/internal/store/store_test.go b/server/internal/store/store_test.go
index 9020b17..c4eafdd 100644
--- a/server/internal/store/store_test.go
+++ b/server/internal/store/store_test.go
@@ -448,20 +448,29 @@ Only worth doing where there is advertising to find. An episode whose served len
the feed declares has none inserted, and the second fetch would be the same recording.
*/
+// now and longAgo frame the age rule: a second fetch is only worth making once the campaign
+// that was running at mirroring has had time to turn over.
+const (
+ now = int64(1_800_000_000)
+ longAgo = now - 60*24*60*60
+ lately = now - 2*24*60*60
+)
+
func TestAnEpisodeWithItsOwnWordsAndNoMasterIsWorthRefetching(t *testing.T) {
e := Episode{
Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3", TranscriptCues: 400,
+ PublishedUnix: longAgo,
}
- if !e.NeedsRefetch() {
+ if !e.NeedsRefetch(now) {
t.Error("an episode with words and nothing to subtract them from was passed over")
}
}
func TestAnEpisodeThatAlreadyHasAMasterIsNotRefetched(t *testing.T) {
e := Episode{
- Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3", TranscriptCues: 400, MasterCues: 300,
+ Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3", TranscriptCues: 400, MasterCues: 300, PublishedUnix: longAgo,
}
- if e.NeedsRefetch() {
+ if e.NeedsRefetch(now) {
t.Error("would fetch a second copy for an episode that already has a master")
}
}
@@ -469,7 +478,7 @@ func TestAnEpisodeThatAlreadyHasAMasterIsNotRefetched(t *testing.T) {
// Nothing to subtract from until its own words exist.
func TestAnEpisodeWithNoWordsYetIsNotRefetched(t *testing.T) {
e := Episode{Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3"}
- if e.NeedsRefetch() {
+ if e.NeedsRefetch(now) {
t.Error("would fetch a second copy before there was a first to compare it against")
}
}
@@ -479,9 +488,117 @@ func TestAnEpisodeWithNoWordsYetIsNotRefetched(t *testing.T) {
func TestAFinishedRefetchIsNotRepeated(t *testing.T) {
e := Episode{
Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3", TranscriptCues: 400,
- RefetchAttemptedAtUnix: 1_700_000_000,
+ RefetchAttemptedAtUnix: 1_700_000_000, PublishedUnix: longAgo,
}
- if e.NeedsRefetch() {
+ if e.NeedsRefetch(now) {
t.Error("would fetch a second copy again")
}
}
+
+/*
+Breaks from the words and breaks from the audio are kept apart and served together.
+
+They are found by different means, on different schedules, and each is rewritten wholesale by
+its own pass -- the ad scan sets Breaks from scratch on every detection, and would wipe anything
+the transcript put there. Keeping them in separate fields is what lets both exist.
+
+The phone wants one list. Overlapping ranges are merged rather than handed over as two, because
+playback seeks to a break's end: two overlapping breaks would seek out of the first and straight
+into the second.
+*/
+
+func TestBreaksFromBothSourcesAreServedAsOneList(t *testing.T) {
+ e := Episode{
+ Breaks: []Break{{StartMs: 60_000, EndMs: 90_000, Kind: "ad"}},
+ WordBreaks: []Break{{StartMs: 600_000, EndMs: 660_000, Kind: "ad"}},
+ }
+
+ got := e.AllBreaks()
+ if len(got) != 2 {
+ t.Fatalf("got %+v, want both", got)
+ }
+ if got[0].StartMs != 60_000 || got[1].StartMs != 600_000 {
+ t.Errorf("got %+v, want them in order", got)
+ }
+}
+
+func TestOverlappingBreaksBecomeOne(t *testing.T) {
+ e := Episode{
+ Breaks: []Break{{StartMs: 60_000, EndMs: 90_000, Kind: "ad"}},
+ WordBreaks: []Break{{StartMs: 80_000, EndMs: 120_000, Kind: "ad"}},
+ }
+
+ got := e.AllBreaks()
+ if len(got) != 1 {
+ t.Fatalf("got %+v, want one merged break", got)
+ }
+ if got[0].StartMs != 60_000 || got[0].EndMs != 120_000 {
+ t.Errorf("got %d-%d, want 60000-120000", got[0].StartMs, got[0].EndMs)
+ }
+}
+
+// Touching ranges are one stretch of audio, and leaving a gap of nothing between them would
+// have playback resume for no time at all and skip again.
+func TestTouchingBreaksBecomeOne(t *testing.T) {
+ e := Episode{
+ Breaks: []Break{{StartMs: 60_000, EndMs: 90_000, Kind: "ad"}},
+ WordBreaks: []Break{{StartMs: 90_000, EndMs: 120_000, Kind: "ad"}},
+ }
+
+ if got := e.AllBreaks(); len(got) != 1 {
+ t.Errorf("got %+v, want one", got)
+ }
+}
+
+/*
+Advertising wins when two merged breaks disagree about what they are.
+
+The phone skips advertising unconditionally and a show's own audio only by preference, so a
+stretch that either detector called advertising is advertising. Being wrong the other way means
+sitting through an advertisement the server already found.
+*/
+func TestAdvertisingWinsOverFurnitureWhenBreaksMerge(t *testing.T) {
+ e := Episode{
+ Breaks: []Break{{StartMs: 60_000, EndMs: 90_000, Kind: "intro"}},
+ WordBreaks: []Break{{StartMs: 80_000, EndMs: 120_000, Kind: "ad"}},
+ }
+
+ got := e.AllBreaks()
+ if len(got) != 1 || got[0].Kind != "ad" {
+ t.Errorf("got %+v, want one break called advertising", got)
+ }
+}
+
+func TestAnEpisodeWithNeitherKindOfBreakHasNone(t *testing.T) {
+ var e Episode
+ if got := e.AllBreaks(); len(got) != 0 {
+ t.Errorf("got %+v", got)
+ }
+}
+
+/*
+A fresh episode is not refetched.
+
+Measured on the live library: a second fetch made minutes after the first came back twenty
+seconds different and subtraction found 21% of the advertising; another came back the same
+length to the second and found none. Insertion varies the amount per request, not the campaign,
+so there is nothing to subtract until the campaign has turned over.
+*/
+func TestAFreshEpisodeIsNotRefetched(t *testing.T) {
+ e := Episode{
+ Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3", TranscriptCues: 400,
+ PublishedUnix: lately,
+ }
+ if e.NeedsRefetch(now) {
+ t.Error("would spend a download on a copy carrying the same campaign")
+ }
+}
+
+// Without a publish date there is no way to tell how old it is, and guessing wrong costs a
+// download and five minutes of recognition for nothing.
+func TestAnEpisodeOfUnknownAgeIsNotRefetched(t *testing.T) {
+ e := Episode{Path: "abc.mp3", EnclosureURL: "https://cdn/1.mp3", TranscriptCues: 400}
+ if e.NeedsRefetch(now) {
+ t.Error("would refetch an episode of unknown age")
+ }
+}
diff --git a/server/internal/transcript/publish.go b/server/internal/transcript/publish.go
new file mode 100644
index 0000000..5d6958c
--- /dev/null
+++ b/server/internal/transcript/publish.go
@@ -0,0 +1,61 @@
+package transcript
+
+import "fmt"
+
+/*
+Which of the advertising read out of an episode's words is fit to skip.
+
+Everything else in this package produces candidates to look at. This is the one function that
+decides what the phone will seek past, so it is deliberately narrow.
+
+Two detectors produce candidates and they are not close in quality. Subtraction -- comparing an
+episode's words against a second description of the same programme -- measured 92% to 104% of
+what the declared-versus-served length gap implies across eighteen episodes, finding every pod,
+with its edges pulled back off the programme. Reading the ad copy measured 0%, 0%, 15% and 29%
+on the same kind of episode. Only the first is acted on.
+
+The length gap is both the measure and the guard. It comes from the feed and the file rather
+than from any detector, which makes it the one number able to contradict one.
+*/
+
+// overshootLimit is how much more than the lengths imply a candidate set may claim and still be
+// published, as a percentage. The measured spread runs to 104%; 150 leaves room for a music
+// sting at the edge of a pod without leaving room for having found the wrong thing entirely.
+const overshootLimit = 150
+
+// Publishable returns the candidates fit to skip, or nothing and the reason.
+//
+// A refusal is a different state from having found nothing, and the caller keeps them apart --
+// the same distinction the ad scan's own refusals have needed twice before.
+func Publishable(found []Candidate, servedMs, declaredMs int64) ([]Candidate, string) {
+ if len(found) == 0 {
+ return nil, ""
+ }
+
+ measured := make([]Candidate, 0, len(found))
+ for _, c := range found {
+ if len(c.Signals) == 1 && c.Signals[0] == SignalDifference {
+ measured = append(measured, c)
+ }
+ }
+ if len(measured) == 0 {
+ return nil, "found by reading the ad copy, which has never agreed with the lengths"
+ }
+
+ gap := servedMs - declaredMs
+ if !HasInsertedAdvertising(servedMs, declaredMs) {
+ return nil, "the lengths say nothing was inserted into this copy, so there is " +
+ "nothing to check these against"
+ }
+
+ var total int64
+ for _, c := range measured {
+ total += c.EndMs - c.StartMs
+ }
+ if total*100 > gap*overshootLimit {
+ return nil, fmt.Sprintf(
+ "claims %s against the %s the lengths say was inserted, so it has found "+
+ "something other than advertising", clock(total), clock(gap))
+ }
+ return measured, ""
+}
diff --git a/server/internal/transcript/publish_test.go b/server/internal/transcript/publish_test.go
new file mode 100644
index 0000000..edcf9c9
--- /dev/null
+++ b/server/internal/transcript/publish_test.go
@@ -0,0 +1,118 @@
+package transcript
+
+import "testing"
+
+/*
+Which of the advertising read out of an episode's words is fit to skip.
+
+Two detectors produce candidates and they are not close in quality. Subtraction -- comparing the
+words against a second description of the same programme -- measures 92% to 104% of what the
+declared-versus-served length gap implies, across eighteen episodes, finding every pod. Reading
+the ad copy measures 0% to 29% on the same kind of episode. Only the first is fit to act on.
+
+The length gap is the guard as well as the measure. It comes from the feed and the file rather
+than from any detector, so it is the one number able to contradict one -- and a candidate set
+that claims half again as much advertising as the lengths say was inserted has found something
+other than advertising.
+*/
+
+func candidates(signal string, spans ...[2]int64) []Candidate {
+ out := make([]Candidate, 0, len(spans))
+ for _, s := range spans {
+ out = append(out, Candidate{StartMs: s[0], EndMs: s[1], Signals: []string{signal}})
+ }
+ return out
+}
+
+func TestSubtractedAdvertisingIsPublished(t *testing.T) {
+ // Darknet Diaries "Conti": 4:15 inserted, 4:20 found across three pods.
+ found := candidates(SignalDifference, [2]int64{0, 57_000}, [2]int64{164_000, 304_000},
+ [2]int64{1_928_000, 1_992_000})
+
+ got, why := Publishable(found, 3_813_000, 3_557_000)
+ if why != "" {
+ t.Fatalf("refused: %s", why)
+ }
+ if len(got) != 3 {
+ t.Errorf("got %d breaks, want 3", len(got))
+ }
+}
+
+/*
+Advertising recognised only by its copy is not published.
+
+It measured 0%, 0%, 15% and 29% of the length gap across the first real episodes, and on a show
+that bakes its reads in it found 2:13 where the lengths can see nothing at all. Useful as a
+hint beside the words; not something to seek past.
+*/
+func TestAdvertisingReadFromTheCopyIsNotPublished(t *testing.T) {
+ found := candidates("sponsor", [2]int64{60_000, 120_000})
+
+ got, why := Publishable(found, 1_480_000, 1_216_000)
+ if len(got) != 0 {
+ t.Errorf("published %d breaks found only by their wording", len(got))
+ }
+ if why == "" {
+ t.Error("said nothing about why")
+ }
+}
+
+/*
+Nothing is published for an episode the lengths say carries no inserted advertising.
+
+Without a gap there is no independent figure to check against, so a candidate set could claim
+anything. Those episodes do carry advertising -- baked into what the publisher serves -- and
+finding it is what reading the copy is for. Acting on it is a different matter.
+*/
+func TestNothingIsPublishedWithoutALengthGapToCheckAgainst(t *testing.T) {
+ found := candidates(SignalDifference, [2]int64{60_000, 120_000})
+
+ got, why := Publishable(found, 1_216_500, 1_216_000)
+ if len(got) != 0 {
+ t.Errorf("published %d breaks with nothing to check them against", len(got))
+ }
+ if why == "" {
+ t.Error("said nothing about why")
+ }
+}
+
+/*
+A candidate set claiming far more than was inserted has found something else.
+
+The one check the detector cannot talk its way out of: the feed says how long the programme is
+and the file says how long this copy is, and neither came from reading any words.
+*/
+func TestClaimingHalfAgainAsMuchAsWasInsertedIsRefused(t *testing.T) {
+ // 4:15 inserted, but the candidates come to 9:00.
+ found := candidates(SignalDifference, [2]int64{0, 540_000})
+
+ got, why := Publishable(found, 3_813_000, 3_557_000)
+ if len(got) != 0 {
+ t.Errorf("published %d breaks claiming more than twice what was inserted", len(got))
+ }
+ if why == "" {
+ t.Error("said nothing about why")
+ }
+}
+
+// The measured spread runs to 104%, and a little over is the music sting at the edge of a pod,
+// not a different kind of mistake.
+func TestAModestOvershootIsStillPublished(t *testing.T) {
+ // 4:15 inserted, 4:40 found -- 110%.
+ found := candidates(SignalDifference, [2]int64{0, 280_000})
+
+ got, why := Publishable(found, 3_813_000, 3_557_000)
+ if len(got) != 1 || why != "" {
+ t.Errorf("got %d breaks, refused %q", len(got), why)
+ }
+}
+
+func TestNothingFoundPublishesNothingAndSaysNothing(t *testing.T) {
+ got, why := Publishable(nil, 3_813_000, 3_557_000)
+ if len(got) != 0 {
+ t.Errorf("got %d breaks from nothing", len(got))
+ }
+ if why != "" {
+ t.Errorf("refused %q, but there was nothing to refuse", why)
+ }
+}
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-14 21:54:12 +0000