diff options
Diffstat (limited to 'server/internal')
| -rw-r--r-- | server/internal/detect/lags.go | 13 | ||||
| -rw-r--r-- | server/internal/detect/runs_test.go | 12 | ||||
| -rw-r--r-- | server/internal/skips/ads.go | 16 | ||||
| -rw-r--r-- | server/internal/skips/ads_test.go | 76 | ||||
| -rw-r--r-- | server/internal/skips/skips_test.go | 9 |
5 files changed, 102 insertions, 24 deletions
diff --git a/server/internal/detect/lags.go b/server/internal/detect/lags.go index 55d9544..18df583 100644 --- a/server/internal/detect/lags.go +++ b/server/internal/detect/lags.go @@ -127,9 +127,18 @@ type RunOptions struct { } // DefaultRunOptions is sized for audio decimated to 250Hz: one-second windows, and nothing -// shorter than four seconds taken seriously. +// shorter than twelve seconds taken seriously. +// +// Advertising comes in units of roughly fifteen seconds, grouped into pods with real programme +// between them. Below twelve the "agreement" is the detector matching itself by chance, not an +// advertisement: a real 38-minute episode produced 37 breaks against the old 1000-sample (4s) +// floor, 14 of them landing at *exactly* 4.0 seconds -- noise pinned at the floor, not findings. +// Episodes the feed proves carry advertising (served measurably longer than declared) have a +// median of 3 breaks each, never more than 7. Twelve rather than fifteen so a real 15-second ad +// whose correlation lapses briefly at one end still survives; both floors were measured against +// the declared-versus-served length gap and twelve scored better. func DefaultRunOptions() RunOptions { - return RunOptions{WindowSamples: 250, MinScore: 0.75, MinRunSamples: 1000} + return RunOptions{WindowSamples: 250, MinScore: 0.75, MinRunSamples: 3000} } // MatchedRuns reports the stretches over which a and b carry the same audio at the given lag. diff --git a/server/internal/detect/runs_test.go b/server/internal/detect/runs_test.go index adb47c0..06d98d1 100644 --- a/server/internal/detect/runs_test.go +++ b/server/internal/detect/runs_test.go @@ -5,6 +5,18 @@ import "testing" // How far the agreement extends, which is what actually gets skipped. An advertisement has to // be skipped in full: where it stops matters as much as that it is there. +// Pinned against measured corpus behaviour, not picked. A real 38-minute episode produced 37 +// breaks, 14 of them exactly 4.0 seconds -- correlation noise pinned at the old floor, not +// findings -- while episodes the feed proves carry advertising have a median of 3 breaks each. +// Advertising comes in units of roughly fifteen seconds; below twelve the detector is agreeing +// with itself by chance. +func TestDefaultRunOptionsFloorIsTwelveSeconds(t *testing.T) { + opts := DefaultRunOptions() + if opts.MinRunSamples != 3000 { + t.Errorf("MinRunSamples = %d, want 3000 (12s at the 250Hz discovery rate)", opts.MinRunSamples) + } +} + func TestFindsTheExtentOfASharedStretch(t *testing.T) { shared := randomSignal(50, 5000) a := place(randomSignal(11, 30000), shared, 4000) diff --git a/server/internal/skips/ads.go b/server/internal/skips/ads.go index 25cf572..2e44f3e 100644 --- a/server/internal/skips/ads.go +++ b/server/internal/skips/ads.go @@ -96,11 +96,15 @@ type AdConfig struct { MaxBreakMs int64 // MergeGapMs bridges matches separated by less than this. // - // A pod of several advertisements is found as several matches, because agreement lapses - // briefly between them. Published unbridged, 45% of the gaps between consecutive breaks came - // in under ten seconds and 152 of 262 multi-break episodes had one -- which plays as a skip, - // a short burst of advertisement, then another skip. Bridging also improves agreement with - // the declared length gap, because what was bridged was advertising too. + // The model is the ad pod: a handful of advertisements back to back, with short gaps where + // agreement lapses between them, and long stretches of real programme between pods. 10s + // caught the lapses but not the pod -- fragments of one pod separated by more than ten + // seconds stayed as separate breaks instead of the single break a listener actually + // experiences. 60s bridges a whole pod's internal gaps while still leaving genuine programme + // between pods intact; combined with the twelve-second run floor this halved the breaks + // found corpus-wide (1091 to 539) while *improving* agreement with the declared-versus-served + // length gap (31% to 34% of episodes within 25%), which is the signal that what was removed + // was noise rather than real advertising. MergeGapMs int64 // Runs is what counts as agreement within an alignment. Runs detect.RunOptions @@ -113,7 +117,7 @@ func DefaultAdConfig() AdConfig { LagsPerPair: 8, MaxSkipFraction: 0.35, MaxBreakMs: 240_000, - MergeGapMs: 10_000, + MergeGapMs: 60_000, Runs: detect.DefaultRunOptions(), } } diff --git a/server/internal/skips/ads_test.go b/server/internal/skips/ads_test.go index 3a4ab5f..f558904 100644 --- a/server/internal/skips/ads_test.go +++ b/server/internal/skips/ads_test.go @@ -80,7 +80,7 @@ func adConfig() AdConfig { } func TestFindsAnAdSharedByTwoEpisodes(t *testing.T) { - ad := adNoise(900, 10*adRate) // ten seconds + ad := adNoise(900, 16*adRate) // sixteen seconds: above the twelve-second run floor stub := &adStub{ whole: map[string][]float64{ "a": put(adNoise(1, 300*adRate), ad, 20*adRate), @@ -103,8 +103,8 @@ func TestFindsAnAdSharedByTwoEpisodes(t *testing.T) { if br.StartMs < 19_000 || br.StartMs > 21_000 { t.Errorf("e1 break starts at %dms, want about 20000", br.StartMs) } - if br.EndMs < 29_000 || br.EndMs > 31_000 { - t.Errorf("e1 break ends at %dms, want about 30000", br.EndMs) + if br.EndMs < 35_000 || br.EndMs > 37_000 { + t.Errorf("e1 break ends at %dms, want about 36000", br.EndMs) } // And the same audio located in the other episode, at its own position. if len(got.Breaks["e2"]) != 1 { @@ -120,8 +120,8 @@ func TestFindsAnAdSharedByTwoEpisodes(t *testing.T) { } func TestFindsSeveralAdBreaksInOneEpisode(t *testing.T) { - first := adNoise(901, 10*adRate) - second := adNoise(902, 12*adRate) + first := adNoise(901, 16*adRate) // above the twelve-second run floor + second := adNoise(902, 16*adRate) // likewise stub := &adStub{ whole: map[string][]float64{ "a": put(put(adNoise(11, 400*adRate), first, 15*adRate), second, 200*adRate), @@ -146,12 +146,64 @@ func TestFindsSeveralAdBreaksInOneEpisode(t *testing.T) { } /* +The defect this guards against, reproduced directly. A real 38-minute episode in the corpus +produced 37 breaks, 14 of them landing at exactly the old four-second floor and scattered +roughly one per minute through the whole thing: the detector agreeing with itself on a show's +own reused texture, not advertising. Five short (sub-floor) stretches shared between two +episodes, at a consistent offset, must not surface as breaks at all -- while a genuine +thirty-second pod at the same offset still must. +*/ +func TestScatteredSubFloorFragmentsProduceNoBreaksButARealPodStillDoes(t *testing.T) { + fragments := make([][]float64, 5) + for i := range fragments { + fragments[i] = adNoise(960+int64(i), 4*adRate) // 4s each: below the 12s run floor + } + realPod := adNoise(970, 30*adRate) // a genuine ad pod, above the floor + + // Every fragment and the real pod sit at the same relative offsets in both episodes, just + // anchored at a different start -- the show's recurring texture plus one real ad, exactly + // as described. + schedule := []int{0, 60, 120, 180, 240} + withScatter := func(seed int64, start int) []float64 { + out := adNoise(seed, 450*adRate) + for i, rel := range schedule { + out = put(out, fragments[i], (start+rel)*adRate) + } + return put(out, realPod, (start+300)*adRate) + } + stub := &adStub{ + whole: map[string][]float64{ + "a": withScatter(85, 10), + "b": withScatter(86, 50), + }, + durations: map[string]int64{"a": 450_000, "b": 450_000}, + } + eps := []Episode{{ID: "e1", Path: "a"}, {ID: "e2", Path: "b"}} + + got, err := DetectAds(context.Background(), stub, eps, adConfig()) + if err != nil { + t.Fatal(err) + } + + if len(got.Breaks["e1"]) != 1 { + t.Fatalf("e1 breaks %+v, want exactly the one real pod and none of the five scattered fragments", got.Breaks["e1"]) + } + br := got.Breaks["e1"][0] + if d := br.DurationMs(); d < 28_000 || d > 32_000 { + t.Errorf("break length %dms, want about 30000 for the real pod", d) + } + if br.StartMs < 300_000 || br.StartMs > 320_000 { + t.Errorf("break starts at %dms, want about 310000 (10s start + 300s schedule offset)", br.StartMs) + } +} + +/* The free ground truth, and the reason this is checkable where intro detection was not. The feed declares the ad-free length and the server was served something longer; the difference is the advertising. Whatever the scan finds should add up to roughly that much. */ func TestReportsFoundAgainstTheLengthGap(t *testing.T) { - ad := adNoise(903, 10*adRate) + ad := adNoise(903, 16*adRate) // sixteen seconds: above the twelve-second run floor stub := &adStub{ whole: map[string][]float64{ "a": put(adNoise(21, 300*adRate), ad, 20*adRate), @@ -160,8 +212,8 @@ func TestReportsFoundAgainstTheLengthGap(t *testing.T) { durations: map[string]int64{"a": 300_000, "b": 300_000}, } eps := []Episode{ - {ID: "e1", Path: "a", DeclaredDurationMs: 290_000}, - {ID: "e2", Path: "b", DeclaredDurationMs: 290_000}, + {ID: "e1", Path: "a", DeclaredDurationMs: 284_000}, + {ID: "e2", Path: "b", DeclaredDurationMs: 284_000}, } got, err := DetectAds(context.Background(), stub, eps, adConfig()) @@ -170,11 +222,11 @@ func TestReportsFoundAgainstTheLengthGap(t *testing.T) { } acc := got.Accounting["e1"] - if acc.InsertedMs != 10_000 { - t.Errorf("inserted %dms, want 10000 (300s served, 290s declared)", acc.InsertedMs) + if acc.InsertedMs != 16_000 { + t.Errorf("inserted %dms, want 16000 (300s served, 284s declared)", acc.InsertedMs) } - if acc.FoundMs < 9_000 || acc.FoundMs > 11_000 { - t.Errorf("found %dms, want about 10000", acc.FoundMs) + if acc.FoundMs < 15_000 || acc.FoundMs > 17_000 { + t.Errorf("found %dms, want about 16000", acc.FoundMs) } } diff --git a/server/internal/skips/skips_test.go b/server/internal/skips/skips_test.go index 936723f..a691cff 100644 --- a/server/internal/skips/skips_test.go +++ b/server/internal/skips/skips_test.go @@ -331,8 +331,9 @@ func TestDefaultConfigUsesTheValidatedValues(t *testing.T) { // Likewise for the ad scan, whose numbers were each chosen against measured corpus behaviour // rather than picked: 250Hz is what makes a whole-episode correlation affordable, 0.35 and -// 240s are the two refusals, and 10s is the bridging tolerance that removed the -// skip-burst-skip stutter from 152 of 262 multi-break episodes. +// 240s are the two refusals, and 60s is the pod-bridging tolerance -- a real pod is a few +// advertisements back to back with short gaps between them, and 60s bridges those gaps while +// still leaving genuine programme between pods intact. func TestDefaultAdConfigUsesTheValidatedValues(t *testing.T) { cfg := DefaultAdConfig() if cfg.DiscoveryRate != 250 { @@ -347,8 +348,8 @@ func TestDefaultAdConfigUsesTheValidatedValues(t *testing.T) { if cfg.MaxBreakMs != 240_000 { t.Errorf("MaxBreakMs = %d, want 240000", cfg.MaxBreakMs) } - if cfg.MergeGapMs != 10_000 { - t.Errorf("MergeGapMs = %d, want 10000", cfg.MergeGapMs) + if cfg.MergeGapMs != 60_000 { + t.Errorf("MergeGapMs = %d, want 60000", cfg.MergeGapMs) } } |
