From 43a78fba9c46704fed7db3fb5f6b138db2fb0820 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Sat, 31 Oct 2026 06:36:34 +0000 Subject: Score the detectors against what the listener heard, and fix two things it found Every other number in this project comes from the declared-versus-served length gap, which says how much advertising a copy carries and nothing about where. A detector could find the right total in the wrong places and score 100%. The position check I wrote earlier used the same signal that builds the candidates, so it was partly circular and structurally blind to the failure it most needed to see. The 19 labels are the only statement about *where* that no detector produced: each is Peter pressing a button at a moment in an episode. listen-score is read-only, takes the state file, and prints how the published breaks did. **Its first answer was wrong, and checking rather than reporting it caught that.** It said 2 of 18 advertisements were covered, which would have meant the detectors were far worse at position than at totals. They are not: breaks at 829-999 and 1360-1519 against labels at 853-1000 and 1489-1540 -- correct detections, scored as complete misses because the break stopped a second short of the label's end. Twelve of the nineteen labels have an end the store itself documents as a guess, "where a jump happened to land". Scoring against a guessed end is scoring against noise. A label's start is solid; its end counts only when measured. 5 of 18 after that. **A refetch master was blocking the better detector.** Tangle's "The Israeli elections" has a 229-second gap, a master from a second fetch that carried the same campaign, and therefore no subtraction candidates at all -- and findRecurring skipped it because it had a master. Treating "has a master" as "has a better answer" is wrong when the master is useless. It now skips only episodes that actually published something. **The one programme cut is the audio scan's, not the words'.** Its break at 696-708 sits exactly on the stretch Peter said was programme. Worth knowing before anything else gets blamed for it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- server/cmd/listen-score/main.go | 105 ++++++++++++++++++++++++++ server/cmd/listend/main.go | 7 +- server/internal/score/score.go | 117 +++++++++++++++++++++++++++++ server/internal/score/score_test.go | 146 ++++++++++++++++++++++++++++++++++++ 4 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 server/cmd/listen-score/main.go create mode 100644 server/internal/score/score.go create mode 100644 server/internal/score/score_test.go diff --git a/server/cmd/listen-score/main.go b/server/cmd/listen-score/main.go new file mode 100644 index 0000000..f4b0ba7 --- /dev/null +++ b/server/cmd/listen-score/main.go @@ -0,0 +1,105 @@ +/* +Command listen-score measures the detectors against what the listener actually heard. + +Every other number in this project comes from the declared-versus-served length gap, which says +how much advertising a copy carries and nothing about where. This is the only check on *where*: +each label is the listener pressing a button at a moment in an episode, saying "this is an +advertisement" where the detectors missed one or "this is not" where they cut something that +was not. + +Read-only. It opens the server's state file, scores what is published against the labels in it, +and prints. It changes nothing and is safe to run against a live server. +*/ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/terst/listen/server/internal/score" + "github.com/terst/listen/server/internal/store" +) + +func main() { + data := flag.String("data", "/site/listen.terst.org/data", "server data directory") + verbose := flag.Bool("v", false, "print every label and how it scored") + asJSON := flag.Bool("json", false, "print the summary as JSON") + flag.Parse() + + state, err := store.Load(filepath.Join(*data, "state.json")) + if err != nil { + fmt.Fprintf(os.Stderr, "listen-score: %v\n", err) + os.Exit(1) + } + + var total score.Summary + matched, unmatched := 0, 0 + + for _, label := range state.Labels() { + episode := state.Match(label.FeedURL, label.GUID, "", "") + if episode == nil { + // The mirror keeps eight episodes a show, so a label outlives the audio it + // describes. Counted rather than ignored: a corpus that has mostly aged out is a + // different thing from a corpus the detectors do badly on. + unmatched++ + continue + } + matched++ + + // Scored against this episode's own breaks. Pooling them would credit one episode's + // detection to another episode's label. + breaks := make([]score.Span, 0, 4) + for _, b := range episode.AllBreaks() { + breaks = append(breaks, score.Span{StartMs: b.StartMs, EndMs: b.EndMs}) + } + one := score.Score([]score.Label{{ + Span: score.Span{StartMs: label.StartMs, EndMs: label.EndMs}, + IsAd: label.Verdict == store.LabelAd, + EndMeasured: label.EndMeasured, + }}, breaks) + + total.AdsFound += one.AdsFound + total.AdsMissed += one.AdsMissed + total.ProgrammeCut += one.ProgrammeCut + total.ProgrammeKept += one.ProgrammeKept + + if *verbose { + fmt.Printf("%-6s %-9s %6.0fs-%-6.0fs %d break(s) %.44s\n", + outcome(one), label.Verdict, + float64(label.StartMs)/1000, float64(label.EndMs)/1000, + len(breaks), episode.Title) + } + } + + if *asJSON { + out := struct { + score.Summary + Labels int `json:"labels"` + Matched int `json:"matched"` + Unmatched int `json:"unmatched"` + }{total, matched + unmatched, matched, unmatched} + blob, _ := json.MarshalIndent(out, "", " ") + fmt.Println(string(blob)) + return + } + + fmt.Printf("\n%d label(s): %d against an episode still mirrored, %d outlived their audio\n", + matched+unmatched, matched, unmatched) + fmt.Println(total.String()) +} + +func outcome(s score.Summary) string { + switch { + case s.AdsFound > 0: + return "found" + case s.AdsMissed > 0: + return "MISSED" + case s.ProgrammeCut > 0: + return "CUT" + default: + return "kept" + } +} diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index 6cbfb65..377fedc 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -656,7 +656,12 @@ func (w *worker) findRecurring(show *store.Show) { published, refused := 0, 0 for i, target := range corpus { - if target.episode.MasterCues > 0 { + // Skipped only where subtraction actually produced something, not merely where a + // master exists. A refetch master that carried the same campaign yields no candidates + // at all, and treating "has a master" as "has a better answer" left those episodes + // with nothing -- Tangle's "The Israeli elections" has a 229-second gap, a refetch + // master, no subtraction candidates, and was excluded from this. + if len(target.episode.WordBreaks) > 0 { continue } others := make([][]transcript.Cue, 0, len(corpus)-1) diff --git a/server/internal/score/score.go b/server/internal/score/score.go new file mode 100644 index 0000000..34f3e0c --- /dev/null +++ b/server/internal/score/score.go @@ -0,0 +1,117 @@ +// Package score measures the detectors against what the listener actually heard. +// +// Everything else in this project is checked against the declared-versus-served length gap, +// which says how *much* advertising a copy carries and nothing about where. A detector could +// find the right total in the wrong places and score 100%. +// +// Labels are the only statement about *where* that no detector produced: each one is the +// listener pressing a button at a moment in an episode, saying "this is an advertisement" where +// the detectors missed one, or "this is not" where they cut something that was not. +package score + +import "fmt" + +// Span is a stretch of an episode, in milliseconds. +type Span struct { + StartMs int64 + EndMs int64 +} + +// Label is what the listener said about one span. +type Label struct { + Span + // IsAd is true for "this is an advertisement", false for "this is the programme". + IsAd bool + // EndMeasured says whether the end of the span is something the listener heard or where a + // jump happened to land. A report starts as a guess at the end and only becomes measured + // if he comes back and trims it -- the store documents this, and scoring that ignores it + // reads a break ending one second early as a complete miss. + EndMeasured bool +} + +// Summary is how a set of breaks did against a set of labels. +// +// The two kinds of mistake are counted separately and never netted off, because they are not +// equally bad: an advertisement nobody found means sitting through it, and a break over the +// programme means losing part of the show with no undo. +type Summary struct { + // AdsFound is advertising the listener pointed at that a break now covers entirely. + AdsFound int + // AdsMissed is advertising still not covered, or covered only in part -- the listener + // hears whatever is left either way. + AdsMissed int + // ProgrammeCut is programme the listener said was programme that a break runs into. + ProgrammeCut int + // ProgrammeKept is programme no break touches. + ProgrammeKept int +} + +// Score measures breaks against labels. +func Score(labels []Label, breaks []Span) Summary { + var out Summary + for _, label := range labels { + if label.IsAd { + if coveredBy(label, breaks) { + out.AdsFound++ + } else { + out.AdsMissed++ + } + continue + } + if overlapsAny(label.Span, breaks) { + out.ProgrammeCut++ + } else { + out.ProgrammeKept++ + } + } + return out +} + +/* +coveredBy reports whether a break covers an advertisement the listener pointed at. + +What "covers" means depends on whether the label's end was measured. The start always is: it is +where he pressed the button, so an advertisement was certainly playing there. The end often is +not -- it is where the jump happened to land -- and requiring a break to reach a guessed end +reads a one-second shortfall as a complete miss. On a real episode that turned four correct +detections into one. + +So a measured label must be covered end to end, and a guessed one must cover the moment he +actually heard it. +*/ +func coveredBy(l Label, breaks []Span) bool { + for _, b := range breaks { + if b.StartMs > l.StartMs { + continue + } + if l.EndMeasured && b.EndMs < l.EndMs { + continue + } + if b.EndMs <= l.StartMs { + continue + } + return true + } + return false +} + +// overlapsAny reports whether any break runs into a span. Touching at an edge does not count: a +// break that ends exactly where the programme resumes is correct, and is the common case once +// its edges have been trimmed. +func overlapsAny(s Span, breaks []Span) bool { + for _, b := range breaks { + if b.StartMs < s.EndMs && b.EndMs > s.StartMs { + return true + } + } + return false +} + +// String reads the score as a sentence, so a run can be understood without counting. +func (s Summary) String() string { + ads := s.AdsFound + s.AdsMissed + programme := s.ProgrammeCut + s.ProgrammeKept + return fmt.Sprintf( + "%d of %d reported advertisements are now covered; %d of %d stretches called programme "+ + "are cut into", s.AdsFound, ads, s.ProgrammeCut, programme) +} diff --git a/server/internal/score/score_test.go b/server/internal/score/score_test.go new file mode 100644 index 0000000..afa0039 --- /dev/null +++ b/server/internal/score/score_test.go @@ -0,0 +1,146 @@ +package score + +import "testing" + +/* +Scoring the detectors against what the listener actually heard. + +Everything else in this project is checked against the declared-versus-served length gap, which +says how *much* advertising a copy carries and nothing about where. A detector could find the +right total in the wrong places and score 100%. The position check I wrote earlier used the same +signal that builds the candidates, so it was partly circular and structurally blind to the one +failure it most needed to see. + +Labels are not. Each one is the listener pressing a button at a moment in an episode: "this is +an advertisement" when the detectors missed one, or "this is not" when they cut something that +was not. They are the only statement about *where* that no detector produced. + +The arithmetic is deliberately asymmetric, because the two mistakes are not equally bad. An +advertisement nobody found means sitting through it. A break over the programme means losing +part of the show, and there is no undo. +*/ + +func span(startMs, endMs int64) Span { return Span{StartMs: startMs, EndMs: endMs} } + +func TestAnAdvertisingLabelCoveredByABreakIsFound(t *testing.T) { + got := Score([]Label{{Span: span(60_000, 90_000), IsAd: true, EndMeasured: true}}, + []Span{span(55_000, 95_000)}) + + if got.AdsFound != 1 || got.AdsMissed != 0 { + t.Errorf("got %+v, want the advertisement counted as found", got) + } +} + +// A measured end is one the listener heard, so a break stopping short of it leaves him hearing +// the rest. +func TestAMeasuredAdvertisingLabelOnlyPartlyCoveredIsMissed(t *testing.T) { + got := Score([]Label{{Span: span(60_000, 90_000), IsAd: true, EndMeasured: true}}, + []Span{span(55_000, 70_000)}) + + if got.AdsFound != 0 || got.AdsMissed != 1 { + t.Errorf("got %+v, want it missed -- twenty seconds of it still play", got) + } +} + +/* +A guessed end is not something to score against. + +Twelve of the nineteen real labels have one: the listener pressed "this is an advertisement", +the player jumped, and the end of the label is wherever it landed. Requiring a break to reach +that point read a break ending one second early as a complete miss -- on Heavyweight's "#70 +Cayden", breaks at 829-999 and 1360-1519 against labels at 853-1000 and 1489-1540, three +correct detections scored as failures. + +What the start says is solid: an advertisement was playing at that moment. +*/ +func TestAGuessedEndIsNotRequiredToBeCovered(t *testing.T) { + got := Score([]Label{{Span: span(853_000, 1_000_000), IsAd: true}}, + []Span{span(829_000, 999_000)}) + + if got.AdsFound != 1 { + t.Errorf("got %+v, want it found -- the break covers where he heard it", got) + } +} + +// The start still has to be covered. A break that ends before the listener heard anything has +// not found what he pointed at. +func TestABreakEndingBeforeTheListenerHeardItIsStillAMiss(t *testing.T) { + got := Score([]Label{{Span: span(853_000, 1_000_000), IsAd: true}}, + []Span{span(600_000, 700_000)}) + + if got.AdsMissed != 1 { + t.Errorf("got %+v, want it missed", got) + } +} + +func TestAnAdvertisingLabelWithNoBreakNearItIsMissed(t *testing.T) { + got := Score([]Label{{Span: span(60_000, 90_000), IsAd: true, EndMeasured: true}}, nil) + + if got.AdsMissed != 1 { + t.Errorf("got %+v", got) + } +} + +/* +A break over something the listener said was the programme is the mistake that matters. + +Counted separately and never netted off against advertising found. A detector that skips ten +advertisements and one minute of the show has not scored ten out of eleven. +*/ +func TestABreakOverTheProgrammeIsCountedAgainstTheDetector(t *testing.T) { + got := Score([]Label{{Span: span(600_000, 660_000), IsAd: false}}, + []Span{span(590_000, 700_000)}) + + if got.ProgrammeCut != 1 { + t.Errorf("got %+v, want the programme cut counted", got) + } +} + +// Touching a programme label at its very edge is not cutting into it. A break that ends exactly +// where the programme begins is correct, and the common case after trimming. +func TestABreakThatOnlyTouchesTheProgrammeIsNotCountingAgainstIt(t *testing.T) { + got := Score([]Label{{Span: span(600_000, 660_000), IsAd: false}}, + []Span{span(540_000, 600_000)}) + + if got.ProgrammeCut != 0 { + t.Errorf("got %+v, want no cut -- the break ends where the programme starts", got) + } +} + +func TestAProgrammeLabelNoBreakTouchesIsLeftAlone(t *testing.T) { + got := Score([]Label{{Span: span(600_000, 660_000), IsAd: false}}, + []Span{span(60_000, 90_000)}) + + if got.ProgrammeCut != 0 || got.ProgrammeKept != 1 { + t.Errorf("got %+v", got) + } +} + +func TestScoringNothingSaysNothing(t *testing.T) { + got := Score(nil, []Span{span(0, 1000)}) + if got.AdsFound+got.AdsMissed+got.ProgrammeCut+got.ProgrammeKept != 0 { + t.Errorf("got %+v, want an empty score", got) + } +} + +// What the summary line says, so a run can be read without counting. +func TestTheScoreReadsAsASentence(t *testing.T) { + got := Summary{AdsFound: 3, AdsMissed: 1, ProgrammeCut: 0, ProgrammeKept: 2}.String() + if got == "" { + t.Fatal("no summary") + } + for _, want := range []string{"3", "4", "advertis"} { + if !contains(got, want) { + t.Errorf("summary %q does not mention %q", got, want) + } + } +} + +func contains(haystack, needle string) bool { + for i := 0; i+len(needle) <= len(haystack); i++ { + if haystack[i:i+len(needle)] == needle { + return true + } + } + return false +} -- cgit v1.2.3