diff options
| -rw-r--r-- | server/cmd/listen-score/main.go | 105 | ||||
| -rw-r--r-- | server/cmd/listend/main.go | 7 | ||||
| -rw-r--r-- | server/internal/score/score.go | 117 | ||||
| -rw-r--r-- | server/internal/score/score_test.go | 146 |
4 files changed, 374 insertions, 1 deletions
diff --git a/server/cmd/listen-score/main.go b/server/cmd/listen-score/main.go new file mode 100644 index 0000000..f4b0ba7 --- /dev/null +++ b/server/cmd/listen-score/main.go @@ -0,0 +1,105 @@ +/* +Command listen-score measures the detectors against what the listener actually heard. + +Every other number in this project comes from the declared-versus-served length gap, which says +how much advertising a copy carries and nothing about where. This is the only check on *where*: +each label is the listener pressing a button at a moment in an episode, saying "this is an +advertisement" where the detectors missed one or "this is not" where they cut something that +was not. + +Read-only. It opens the server's state file, scores what is published against the labels in it, +and prints. It changes nothing and is safe to run against a live server. +*/ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/terst/listen/server/internal/score" + "github.com/terst/listen/server/internal/store" +) + +func main() { + data := flag.String("data", "/site/listen.terst.org/data", "server data directory") + verbose := flag.Bool("v", false, "print every label and how it scored") + asJSON := flag.Bool("json", false, "print the summary as JSON") + flag.Parse() + + state, err := store.Load(filepath.Join(*data, "state.json")) + if err != nil { + fmt.Fprintf(os.Stderr, "listen-score: %v\n", err) + os.Exit(1) + } + + var total score.Summary + matched, unmatched := 0, 0 + + for _, label := range state.Labels() { + episode := state.Match(label.FeedURL, label.GUID, "", "") + if episode == nil { + // The mirror keeps eight episodes a show, so a label outlives the audio it + // describes. Counted rather than ignored: a corpus that has mostly aged out is a + // different thing from a corpus the detectors do badly on. + unmatched++ + continue + } + matched++ + + // Scored against this episode's own breaks. Pooling them would credit one episode's + // detection to another episode's label. + breaks := make([]score.Span, 0, 4) + for _, b := range episode.AllBreaks() { + breaks = append(breaks, score.Span{StartMs: b.StartMs, EndMs: b.EndMs}) + } + one := score.Score([]score.Label{{ + Span: score.Span{StartMs: label.StartMs, EndMs: label.EndMs}, + IsAd: label.Verdict == store.LabelAd, + EndMeasured: label.EndMeasured, + }}, breaks) + + total.AdsFound += one.AdsFound + total.AdsMissed += one.AdsMissed + total.ProgrammeCut += one.ProgrammeCut + total.ProgrammeKept += one.ProgrammeKept + + if *verbose { + fmt.Printf("%-6s %-9s %6.0fs-%-6.0fs %d break(s) %.44s\n", + outcome(one), label.Verdict, + float64(label.StartMs)/1000, float64(label.EndMs)/1000, + len(breaks), episode.Title) + } + } + + if *asJSON { + out := struct { + score.Summary + Labels int `json:"labels"` + Matched int `json:"matched"` + Unmatched int `json:"unmatched"` + }{total, matched + unmatched, matched, unmatched} + blob, _ := json.MarshalIndent(out, "", " ") + fmt.Println(string(blob)) + return + } + + fmt.Printf("\n%d label(s): %d against an episode still mirrored, %d outlived their audio\n", + matched+unmatched, matched, unmatched) + fmt.Println(total.String()) +} + +func outcome(s score.Summary) string { + switch { + case s.AdsFound > 0: + return "found" + case s.AdsMissed > 0: + return "MISSED" + case s.ProgrammeCut > 0: + return "CUT" + default: + return "kept" + } +} diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index 6cbfb65..377fedc 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -656,7 +656,12 @@ func (w *worker) findRecurring(show *store.Show) { published, refused := 0, 0 for i, target := range corpus { - if target.episode.MasterCues > 0 { + // Skipped only where subtraction actually produced something, not merely where a + // master exists. A refetch master that carried the same campaign yields no candidates + // at all, and treating "has a master" as "has a better answer" left those episodes + // with nothing -- Tangle's "The Israeli elections" has a 229-second gap, a refetch + // master, no subtraction candidates, and was excluded from this. + if len(target.episode.WordBreaks) > 0 { continue } others := make([][]transcript.Cue, 0, len(corpus)-1) diff --git a/server/internal/score/score.go b/server/internal/score/score.go new file mode 100644 index 0000000..34f3e0c --- /dev/null +++ b/server/internal/score/score.go @@ -0,0 +1,117 @@ +// Package score measures the detectors against what the listener actually heard. +// +// Everything else in this project is checked against the declared-versus-served length gap, +// which says how *much* advertising a copy carries and nothing about where. A detector could +// find the right total in the wrong places and score 100%. +// +// Labels are the only statement about *where* that no detector produced: each one is the +// listener pressing a button at a moment in an episode, saying "this is an advertisement" where +// the detectors missed one, or "this is not" where they cut something that was not. +package score + +import "fmt" + +// Span is a stretch of an episode, in milliseconds. +type Span struct { + StartMs int64 + EndMs int64 +} + +// Label is what the listener said about one span. +type Label struct { + Span + // IsAd is true for "this is an advertisement", false for "this is the programme". + IsAd bool + // EndMeasured says whether the end of the span is something the listener heard or where a + // jump happened to land. A report starts as a guess at the end and only becomes measured + // if he comes back and trims it -- the store documents this, and scoring that ignores it + // reads a break ending one second early as a complete miss. + EndMeasured bool +} + +// Summary is how a set of breaks did against a set of labels. +// +// The two kinds of mistake are counted separately and never netted off, because they are not +// equally bad: an advertisement nobody found means sitting through it, and a break over the +// programme means losing part of the show with no undo. +type Summary struct { + // AdsFound is advertising the listener pointed at that a break now covers entirely. + AdsFound int + // AdsMissed is advertising still not covered, or covered only in part -- the listener + // hears whatever is left either way. + AdsMissed int + // ProgrammeCut is programme the listener said was programme that a break runs into. + ProgrammeCut int + // ProgrammeKept is programme no break touches. + ProgrammeKept int +} + +// Score measures breaks against labels. +func Score(labels []Label, breaks []Span) Summary { + var out Summary + for _, label := range labels { + if label.IsAd { + if coveredBy(label, breaks) { + out.AdsFound++ + } else { + out.AdsMissed++ + } + continue + } + if overlapsAny(label.Span, breaks) { + out.ProgrammeCut++ + } else { + out.ProgrammeKept++ + } + } + return out +} + +/* +coveredBy reports whether a break covers an advertisement the listener pointed at. + +What "covers" means depends on whether the label's end was measured. The start always is: it is +where he pressed the button, so an advertisement was certainly playing there. The end often is +not -- it is where the jump happened to land -- and requiring a break to reach a guessed end +reads a one-second shortfall as a complete miss. On a real episode that turned four correct +detections into one. + +So a measured label must be covered end to end, and a guessed one must cover the moment he +actually heard it. +*/ +func coveredBy(l Label, breaks []Span) bool { + for _, b := range breaks { + if b.StartMs > l.StartMs { + continue + } + if l.EndMeasured && b.EndMs < l.EndMs { + continue + } + if b.EndMs <= l.StartMs { + continue + } + return true + } + return false +} + +// overlapsAny reports whether any break runs into a span. Touching at an edge does not count: a +// break that ends exactly where the programme resumes is correct, and is the common case once +// its edges have been trimmed. +func overlapsAny(s Span, breaks []Span) bool { + for _, b := range breaks { + if b.StartMs < s.EndMs && b.EndMs > s.StartMs { + return true + } + } + return false +} + +// String reads the score as a sentence, so a run can be understood without counting. +func (s Summary) String() string { + ads := s.AdsFound + s.AdsMissed + programme := s.ProgrammeCut + s.ProgrammeKept + return fmt.Sprintf( + "%d of %d reported advertisements are now covered; %d of %d stretches called programme "+ + "are cut into", s.AdsFound, ads, s.ProgrammeCut, programme) +} diff --git a/server/internal/score/score_test.go b/server/internal/score/score_test.go new file mode 100644 index 0000000..afa0039 --- /dev/null +++ b/server/internal/score/score_test.go @@ -0,0 +1,146 @@ +package score + +import "testing" + +/* +Scoring the detectors against what the listener actually heard. + +Everything else in this project is checked against the declared-versus-served length gap, which +says how *much* advertising a copy carries and nothing about where. A detector could find the +right total in the wrong places and score 100%. The position check I wrote earlier used the same +signal that builds the candidates, so it was partly circular and structurally blind to the one +failure it most needed to see. + +Labels are not. Each one is the listener pressing a button at a moment in an episode: "this is +an advertisement" when the detectors missed one, or "this is not" when they cut something that +was not. They are the only statement about *where* that no detector produced. + +The arithmetic is deliberately asymmetric, because the two mistakes are not equally bad. An +advertisement nobody found means sitting through it. A break over the programme means losing +part of the show, and there is no undo. +*/ + +func span(startMs, endMs int64) Span { return Span{StartMs: startMs, EndMs: endMs} } + +func TestAnAdvertisingLabelCoveredByABreakIsFound(t *testing.T) { + got := Score([]Label{{Span: span(60_000, 90_000), IsAd: true, EndMeasured: true}}, + []Span{span(55_000, 95_000)}) + + if got.AdsFound != 1 || got.AdsMissed != 0 { + t.Errorf("got %+v, want the advertisement counted as found", got) + } +} + +// A measured end is one the listener heard, so a break stopping short of it leaves him hearing +// the rest. +func TestAMeasuredAdvertisingLabelOnlyPartlyCoveredIsMissed(t *testing.T) { + got := Score([]Label{{Span: span(60_000, 90_000), IsAd: true, EndMeasured: true}}, + []Span{span(55_000, 70_000)}) + + if got.AdsFound != 0 || got.AdsMissed != 1 { + t.Errorf("got %+v, want it missed -- twenty seconds of it still play", got) + } +} + +/* +A guessed end is not something to score against. + +Twelve of the nineteen real labels have one: the listener pressed "this is an advertisement", +the player jumped, and the end of the label is wherever it landed. Requiring a break to reach +that point read a break ending one second early as a complete miss -- on Heavyweight's "#70 +Cayden", breaks at 829-999 and 1360-1519 against labels at 853-1000 and 1489-1540, three +correct detections scored as failures. + +What the start says is solid: an advertisement was playing at that moment. +*/ +func TestAGuessedEndIsNotRequiredToBeCovered(t *testing.T) { + got := Score([]Label{{Span: span(853_000, 1_000_000), IsAd: true}}, + []Span{span(829_000, 999_000)}) + + if got.AdsFound != 1 { + t.Errorf("got %+v, want it found -- the break covers where he heard it", got) + } +} + +// The start still has to be covered. A break that ends before the listener heard anything has +// not found what he pointed at. +func TestABreakEndingBeforeTheListenerHeardItIsStillAMiss(t *testing.T) { + got := Score([]Label{{Span: span(853_000, 1_000_000), IsAd: true}}, + []Span{span(600_000, 700_000)}) + + if got.AdsMissed != 1 { + t.Errorf("got %+v, want it missed", got) + } +} + +func TestAnAdvertisingLabelWithNoBreakNearItIsMissed(t *testing.T) { + got := Score([]Label{{Span: span(60_000, 90_000), IsAd: true, EndMeasured: true}}, nil) + + if got.AdsMissed != 1 { + t.Errorf("got %+v", got) + } +} + +/* +A break over something the listener said was the programme is the mistake that matters. + +Counted separately and never netted off against advertising found. A detector that skips ten +advertisements and one minute of the show has not scored ten out of eleven. +*/ +func TestABreakOverTheProgrammeIsCountedAgainstTheDetector(t *testing.T) { + got := Score([]Label{{Span: span(600_000, 660_000), IsAd: false}}, + []Span{span(590_000, 700_000)}) + + if got.ProgrammeCut != 1 { + t.Errorf("got %+v, want the programme cut counted", got) + } +} + +// Touching a programme label at its very edge is not cutting into it. A break that ends exactly +// where the programme begins is correct, and the common case after trimming. +func TestABreakThatOnlyTouchesTheProgrammeIsNotCountingAgainstIt(t *testing.T) { + got := Score([]Label{{Span: span(600_000, 660_000), IsAd: false}}, + []Span{span(540_000, 600_000)}) + + if got.ProgrammeCut != 0 { + t.Errorf("got %+v, want no cut -- the break ends where the programme starts", got) + } +} + +func TestAProgrammeLabelNoBreakTouchesIsLeftAlone(t *testing.T) { + got := Score([]Label{{Span: span(600_000, 660_000), IsAd: false}}, + []Span{span(60_000, 90_000)}) + + if got.ProgrammeCut != 0 || got.ProgrammeKept != 1 { + t.Errorf("got %+v", got) + } +} + +func TestScoringNothingSaysNothing(t *testing.T) { + got := Score(nil, []Span{span(0, 1000)}) + if got.AdsFound+got.AdsMissed+got.ProgrammeCut+got.ProgrammeKept != 0 { + t.Errorf("got %+v, want an empty score", got) + } +} + +// What the summary line says, so a run can be read without counting. +func TestTheScoreReadsAsASentence(t *testing.T) { + got := Summary{AdsFound: 3, AdsMissed: 1, ProgrammeCut: 0, ProgrammeKept: 2}.String() + if got == "" { + t.Fatal("no summary") + } + for _, want := range []string{"3", "4", "advertis"} { + if !contains(got, want) { + t.Errorf("summary %q does not mention %q", got, want) + } + } +} + +func contains(haystack, needle string) bool { + for i := 0; i+len(needle) <= len(haystack); i++ { + if haystack[i:i+len(needle)] == needle { + return true + } + } + return false +} |