From 43a78fba9c46704fed7db3fb5f6b138db2fb0820 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Sat, 31 Oct 2026 06:36:34 +0000 Subject: Score the detectors against what the listener heard, and fix two things it found Every other number in this project comes from the declared-versus-served length gap, which says how much advertising a copy carries and nothing about where. A detector could find the right total in the wrong places and score 100%. The position check I wrote earlier used the same signal that builds the candidates, so it was partly circular and structurally blind to the failure it most needed to see. The 19 labels are the only statement about *where* that no detector produced: each is Peter pressing a button at a moment in an episode. listen-score is read-only, takes the state file, and prints how the published breaks did. **Its first answer was wrong, and checking rather than reporting it caught that.** It said 2 of 18 advertisements were covered, which would have meant the detectors were far worse at position than at totals. They are not: breaks at 829-999 and 1360-1519 against labels at 853-1000 and 1489-1540 -- correct detections, scored as complete misses because the break stopped a second short of the label's end. Twelve of the nineteen labels have an end the store itself documents as a guess, "where a jump happened to land". Scoring against a guessed end is scoring against noise. A label's start is solid; its end counts only when measured. 5 of 18 after that. **A refetch master was blocking the better detector.** Tangle's "The Israeli elections" has a 229-second gap, a master from a second fetch that carried the same campaign, and therefore no subtraction candidates at all -- and findRecurring skipped it because it had a master. Treating "has a master" as "has a better answer" is wrong when the master is useless. It now skips only episodes that actually published something. **The one programme cut is the audio scan's, not the words'.** Its break at 696-708 sits exactly on the stretch Peter said was programme. Worth knowing before anything else gets blamed for it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- server/cmd/listen-score/main.go | 105 ++++++++++++++++++++++++++++++++++++++++ server/cmd/listend/main.go | 7 ++- 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 server/cmd/listen-score/main.go (limited to 'server/cmd') diff --git a/server/cmd/listen-score/main.go b/server/cmd/listen-score/main.go new file mode 100644 index 0000000..f4b0ba7 --- /dev/null +++ b/server/cmd/listen-score/main.go @@ -0,0 +1,105 @@ +/* +Command listen-score measures the detectors against what the listener actually heard. + +Every other number in this project comes from the declared-versus-served length gap, which says +how much advertising a copy carries and nothing about where. This is the only check on *where*: +each label is the listener pressing a button at a moment in an episode, saying "this is an +advertisement" where the detectors missed one or "this is not" where they cut something that +was not. + +Read-only. It opens the server's state file, scores what is published against the labels in it, +and prints. It changes nothing and is safe to run against a live server. +*/ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/terst/listen/server/internal/score" + "github.com/terst/listen/server/internal/store" +) + +func main() { + data := flag.String("data", "/site/listen.terst.org/data", "server data directory") + verbose := flag.Bool("v", false, "print every label and how it scored") + asJSON := flag.Bool("json", false, "print the summary as JSON") + flag.Parse() + + state, err := store.Load(filepath.Join(*data, "state.json")) + if err != nil { + fmt.Fprintf(os.Stderr, "listen-score: %v\n", err) + os.Exit(1) + } + + var total score.Summary + matched, unmatched := 0, 0 + + for _, label := range state.Labels() { + episode := state.Match(label.FeedURL, label.GUID, "", "") + if episode == nil { + // The mirror keeps eight episodes a show, so a label outlives the audio it + // describes. Counted rather than ignored: a corpus that has mostly aged out is a + // different thing from a corpus the detectors do badly on. + unmatched++ + continue + } + matched++ + + // Scored against this episode's own breaks. Pooling them would credit one episode's + // detection to another episode's label. + breaks := make([]score.Span, 0, 4) + for _, b := range episode.AllBreaks() { + breaks = append(breaks, score.Span{StartMs: b.StartMs, EndMs: b.EndMs}) + } + one := score.Score([]score.Label{{ + Span: score.Span{StartMs: label.StartMs, EndMs: label.EndMs}, + IsAd: label.Verdict == store.LabelAd, + EndMeasured: label.EndMeasured, + }}, breaks) + + total.AdsFound += one.AdsFound + total.AdsMissed += one.AdsMissed + total.ProgrammeCut += one.ProgrammeCut + total.ProgrammeKept += one.ProgrammeKept + + if *verbose { + fmt.Printf("%-6s %-9s %6.0fs-%-6.0fs %d break(s) %.44s\n", + outcome(one), label.Verdict, + float64(label.StartMs)/1000, float64(label.EndMs)/1000, + len(breaks), episode.Title) + } + } + + if *asJSON { + out := struct { + score.Summary + Labels int `json:"labels"` + Matched int `json:"matched"` + Unmatched int `json:"unmatched"` + }{total, matched + unmatched, matched, unmatched} + blob, _ := json.MarshalIndent(out, "", " ") + fmt.Println(string(blob)) + return + } + + fmt.Printf("\n%d label(s): %d against an episode still mirrored, %d outlived their audio\n", + matched+unmatched, matched, unmatched) + fmt.Println(total.String()) +} + +func outcome(s score.Summary) string { + switch { + case s.AdsFound > 0: + return "found" + case s.AdsMissed > 0: + return "MISSED" + case s.ProgrammeCut > 0: + return "CUT" + default: + return "kept" + } +} diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index 6cbfb65..377fedc 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -656,7 +656,12 @@ func (w *worker) findRecurring(show *store.Show) { published, refused := 0, 0 for i, target := range corpus { - if target.episode.MasterCues > 0 { + // Skipped only where subtraction actually produced something, not merely where a + // master exists. A refetch master that carried the same campaign yields no candidates + // at all, and treating "has a master" as "has a better answer" left those episodes + // with nothing -- Tangle's "The Israeli elections" has a 229-second gap, a refetch + // master, no subtraction candidates, and was excluded from this. + if len(target.episode.WordBreaks) > 0 { continue } others := make([][]transcript.Cue, 0, len(corpus)-1) -- cgit v1.2.3