summaryrefslogtreecommitdiff
path: root/server/internal
diff options
context:
space:
mode:
Diffstat (limited to 'server/internal')
-rw-r--r--server/internal/detect/lags.go13
-rw-r--r--server/internal/detect/runs_test.go12
-rw-r--r--server/internal/skips/ads.go16
-rw-r--r--server/internal/skips/ads_test.go76
-rw-r--r--server/internal/skips/skips_test.go9
5 files changed, 102 insertions, 24 deletions
diff --git a/server/internal/detect/lags.go b/server/internal/detect/lags.go
index 55d9544..18df583 100644
--- a/server/internal/detect/lags.go
+++ b/server/internal/detect/lags.go
@@ -127,9 +127,18 @@ type RunOptions struct {
}
// DefaultRunOptions is sized for audio decimated to 250Hz: one-second windows, and nothing
-// shorter than four seconds taken seriously.
+// shorter than twelve seconds taken seriously.
+//
+// Advertising comes in units of roughly fifteen seconds, grouped into pods with real programme
+// between them. Below twelve the "agreement" is the detector matching itself by chance, not an
+// advertisement: a real 38-minute episode produced 37 breaks against the old 1000-sample (4s)
+// floor, 14 of them landing at *exactly* 4.0 seconds -- noise pinned at the floor, not findings.
+// Episodes the feed proves carry advertising (served measurably longer than declared) have a
+// median of 3 breaks each, never more than 7. Twelve rather than fifteen so a real 15-second ad
+// whose correlation lapses briefly at one end still survives; both floors were measured against
+// the declared-versus-served length gap and twelve scored better.
func DefaultRunOptions() RunOptions {
- return RunOptions{WindowSamples: 250, MinScore: 0.75, MinRunSamples: 1000}
+ return RunOptions{WindowSamples: 250, MinScore: 0.75, MinRunSamples: 3000}
}
// MatchedRuns reports the stretches over which a and b carry the same audio at the given lag.
diff --git a/server/internal/detect/runs_test.go b/server/internal/detect/runs_test.go
index adb47c0..06d98d1 100644
--- a/server/internal/detect/runs_test.go
+++ b/server/internal/detect/runs_test.go
@@ -5,6 +5,18 @@ import "testing"
// How far the agreement extends, which is what actually gets skipped. An advertisement has to
// be skipped in full: where it stops matters as much as that it is there.
+// Pinned against measured corpus behaviour, not picked. A real 38-minute episode produced 37
+// breaks, 14 of them exactly 4.0 seconds -- correlation noise pinned at the old floor, not
+// findings -- while episodes the feed proves carry advertising have a median of 3 breaks each.
+// Advertising comes in units of roughly fifteen seconds; below twelve the detector is agreeing
+// with itself by chance.
+func TestDefaultRunOptionsFloorIsTwelveSeconds(t *testing.T) {
+ opts := DefaultRunOptions()
+ if opts.MinRunSamples != 3000 {
+ t.Errorf("MinRunSamples = %d, want 3000 (12s at the 250Hz discovery rate)", opts.MinRunSamples)
+ }
+}
+
func TestFindsTheExtentOfASharedStretch(t *testing.T) {
shared := randomSignal(50, 5000)
a := place(randomSignal(11, 30000), shared, 4000)
diff --git a/server/internal/skips/ads.go b/server/internal/skips/ads.go
index 25cf572..2e44f3e 100644
--- a/server/internal/skips/ads.go
+++ b/server/internal/skips/ads.go
@@ -96,11 +96,15 @@ type AdConfig struct {
MaxBreakMs int64
// MergeGapMs bridges matches separated by less than this.
//
- // A pod of several advertisements is found as several matches, because agreement lapses
- // briefly between them. Published unbridged, 45% of the gaps between consecutive breaks came
- // in under ten seconds and 152 of 262 multi-break episodes had one -- which plays as a skip,
- // a short burst of advertisement, then another skip. Bridging also improves agreement with
- // the declared length gap, because what was bridged was advertising too.
+ // The model is the ad pod: a handful of advertisements back to back, with short gaps where
+ // agreement lapses between them, and long stretches of real programme between pods. 10s
+ // caught the lapses but not the pod -- fragments of one pod separated by more than ten
+ // seconds stayed as separate breaks instead of the single break a listener actually
+ // experiences. 60s bridges a whole pod's internal gaps while still leaving genuine programme
+ // between pods intact; combined with the twelve-second run floor this halved the breaks
+ // found corpus-wide (1091 to 539) while *improving* agreement with the declared-versus-served
+ // length gap (31% to 34% of episodes within 25%), which is the signal that what was removed
+ // was noise rather than real advertising.
MergeGapMs int64
// Runs is what counts as agreement within an alignment.
Runs detect.RunOptions
@@ -113,7 +117,7 @@ func DefaultAdConfig() AdConfig {
LagsPerPair: 8,
MaxSkipFraction: 0.35,
MaxBreakMs: 240_000,
- MergeGapMs: 10_000,
+ MergeGapMs: 60_000,
Runs: detect.DefaultRunOptions(),
}
}
diff --git a/server/internal/skips/ads_test.go b/server/internal/skips/ads_test.go
index 3a4ab5f..f558904 100644
--- a/server/internal/skips/ads_test.go
+++ b/server/internal/skips/ads_test.go
@@ -80,7 +80,7 @@ func adConfig() AdConfig {
}
func TestFindsAnAdSharedByTwoEpisodes(t *testing.T) {
- ad := adNoise(900, 10*adRate) // ten seconds
+ ad := adNoise(900, 16*adRate) // sixteen seconds: above the twelve-second run floor
stub := &adStub{
whole: map[string][]float64{
"a": put(adNoise(1, 300*adRate), ad, 20*adRate),
@@ -103,8 +103,8 @@ func TestFindsAnAdSharedByTwoEpisodes(t *testing.T) {
if br.StartMs < 19_000 || br.StartMs > 21_000 {
t.Errorf("e1 break starts at %dms, want about 20000", br.StartMs)
}
- if br.EndMs < 29_000 || br.EndMs > 31_000 {
- t.Errorf("e1 break ends at %dms, want about 30000", br.EndMs)
+ if br.EndMs < 35_000 || br.EndMs > 37_000 {
+ t.Errorf("e1 break ends at %dms, want about 36000", br.EndMs)
}
// And the same audio located in the other episode, at its own position.
if len(got.Breaks["e2"]) != 1 {
@@ -120,8 +120,8 @@ func TestFindsAnAdSharedByTwoEpisodes(t *testing.T) {
}
func TestFindsSeveralAdBreaksInOneEpisode(t *testing.T) {
- first := adNoise(901, 10*adRate)
- second := adNoise(902, 12*adRate)
+ first := adNoise(901, 16*adRate) // above the twelve-second run floor
+ second := adNoise(902, 16*adRate) // likewise
stub := &adStub{
whole: map[string][]float64{
"a": put(put(adNoise(11, 400*adRate), first, 15*adRate), second, 200*adRate),
@@ -146,12 +146,64 @@ func TestFindsSeveralAdBreaksInOneEpisode(t *testing.T) {
}
/*
+The defect this guards against, reproduced directly. A real 38-minute episode in the corpus
+produced 37 breaks, 14 of them landing at exactly the old four-second floor and scattered
+roughly one per minute through the whole thing: the detector agreeing with itself on a show's
+own reused texture, not advertising. Five short (sub-floor) stretches shared between two
+episodes, at a consistent offset, must not surface as breaks at all -- while a genuine
+thirty-second pod at the same offset still must.
+*/
+func TestScatteredSubFloorFragmentsProduceNoBreaksButARealPodStillDoes(t *testing.T) {
+ fragments := make([][]float64, 5)
+ for i := range fragments {
+ fragments[i] = adNoise(960+int64(i), 4*adRate) // 4s each: below the 12s run floor
+ }
+ realPod := adNoise(970, 30*adRate) // a genuine ad pod, above the floor
+
+ // Every fragment and the real pod sit at the same relative offsets in both episodes, just
+ // anchored at a different start -- the show's recurring texture plus one real ad, exactly
+ // as described.
+ schedule := []int{0, 60, 120, 180, 240}
+ withScatter := func(seed int64, start int) []float64 {
+ out := adNoise(seed, 450*adRate)
+ for i, rel := range schedule {
+ out = put(out, fragments[i], (start+rel)*adRate)
+ }
+ return put(out, realPod, (start+300)*adRate)
+ }
+ stub := &adStub{
+ whole: map[string][]float64{
+ "a": withScatter(85, 10),
+ "b": withScatter(86, 50),
+ },
+ durations: map[string]int64{"a": 450_000, "b": 450_000},
+ }
+ eps := []Episode{{ID: "e1", Path: "a"}, {ID: "e2", Path: "b"}}
+
+ got, err := DetectAds(context.Background(), stub, eps, adConfig())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(got.Breaks["e1"]) != 1 {
+ t.Fatalf("e1 breaks %+v, want exactly the one real pod and none of the five scattered fragments", got.Breaks["e1"])
+ }
+ br := got.Breaks["e1"][0]
+ if d := br.DurationMs(); d < 28_000 || d > 32_000 {
+ t.Errorf("break length %dms, want about 30000 for the real pod", d)
+ }
+ if br.StartMs < 300_000 || br.StartMs > 320_000 {
+ t.Errorf("break starts at %dms, want about 310000 (10s start + 300s schedule offset)", br.StartMs)
+ }
+}
+
+/*
The free ground truth, and the reason this is checkable where intro detection was not. The feed
declares the ad-free length and the server was served something longer; the difference is the
advertising. Whatever the scan finds should add up to roughly that much.
*/
func TestReportsFoundAgainstTheLengthGap(t *testing.T) {
- ad := adNoise(903, 10*adRate)
+ ad := adNoise(903, 16*adRate) // sixteen seconds: above the twelve-second run floor
stub := &adStub{
whole: map[string][]float64{
"a": put(adNoise(21, 300*adRate), ad, 20*adRate),
@@ -160,8 +212,8 @@ func TestReportsFoundAgainstTheLengthGap(t *testing.T) {
durations: map[string]int64{"a": 300_000, "b": 300_000},
}
eps := []Episode{
- {ID: "e1", Path: "a", DeclaredDurationMs: 290_000},
- {ID: "e2", Path: "b", DeclaredDurationMs: 290_000},
+ {ID: "e1", Path: "a", DeclaredDurationMs: 284_000},
+ {ID: "e2", Path: "b", DeclaredDurationMs: 284_000},
}
got, err := DetectAds(context.Background(), stub, eps, adConfig())
@@ -170,11 +222,11 @@ func TestReportsFoundAgainstTheLengthGap(t *testing.T) {
}
acc := got.Accounting["e1"]
- if acc.InsertedMs != 10_000 {
- t.Errorf("inserted %dms, want 10000 (300s served, 290s declared)", acc.InsertedMs)
+ if acc.InsertedMs != 16_000 {
+ t.Errorf("inserted %dms, want 16000 (300s served, 284s declared)", acc.InsertedMs)
}
- if acc.FoundMs < 9_000 || acc.FoundMs > 11_000 {
- t.Errorf("found %dms, want about 10000", acc.FoundMs)
+ if acc.FoundMs < 15_000 || acc.FoundMs > 17_000 {
+ t.Errorf("found %dms, want about 16000", acc.FoundMs)
}
}
diff --git a/server/internal/skips/skips_test.go b/server/internal/skips/skips_test.go
index 936723f..a691cff 100644
--- a/server/internal/skips/skips_test.go
+++ b/server/internal/skips/skips_test.go
@@ -331,8 +331,9 @@ func TestDefaultConfigUsesTheValidatedValues(t *testing.T) {
// Likewise for the ad scan, whose numbers were each chosen against measured corpus behaviour
// rather than picked: 250Hz is what makes a whole-episode correlation affordable, 0.35 and
-// 240s are the two refusals, and 10s is the bridging tolerance that removed the
-// skip-burst-skip stutter from 152 of 262 multi-break episodes.
+// 240s are the two refusals, and 60s is the pod-bridging tolerance -- a real pod is a few
+// advertisements back to back with short gaps between them, and 60s bridges those gaps while
+// still leaving genuine programme between pods intact.
func TestDefaultAdConfigUsesTheValidatedValues(t *testing.T) {
cfg := DefaultAdConfig()
if cfg.DiscoveryRate != 250 {
@@ -347,8 +348,8 @@ func TestDefaultAdConfigUsesTheValidatedValues(t *testing.T) {
if cfg.MaxBreakMs != 240_000 {
t.Errorf("MaxBreakMs = %d, want 240000", cfg.MaxBreakMs)
}
- if cfg.MergeGapMs != 10_000 {
- t.Errorf("MergeGapMs = %d, want 10000", cfg.MergeGapMs)
+ if cfg.MergeGapMs != 60_000 {
+ t.Errorf("MergeGapMs = %d, want 60000", cfg.MergeGapMs)
}
}