From 34e50c04b18313d92e3b3a565965675fd8b50b8a Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Sat, 24 Oct 2026 09:01:05 +0000 Subject: Send the listener's labels to the server and keep them The phone has had two kinds of label and nowhere to put them: a mark, which is a judgement on a break detection found, and a report, which is the notification button pointing at advertising it missed. They catch opposite errors, and the signature library needs both. POST /api/labels now takes them and state.json keeps them. Episodes are named by feed and guid rather than by either side's own id -- the phone never sees the server's, and the server's belongs to a mirror that forgets an episode after eight. The labels sit beside the shows rather than on them for the same reason: the corpus has to outlive the audio it was given on. A report's end is sent as guessed or measured. It is only measured once the listener came back and trimmed it; otherwise it is wherever the jump landed, and a signature cut to that would carry programme. The phone re-sends everything each sweep and keeps no record of what arrived. The server replaces rather than accumulating, so a re-send is free, there is no sync state to get wrong, and a label lost to a crash comes back on the next refresh. Gated on audioFromServer, like every other server re-query: a label names a position in a particular file, and dynamic ad insertion means the publisher's copy is a different one. Accepting a label writes the state file immediately. Everything else is written at the end of a poll cycle, which for 61 feeds is hours, and a tap made once should not wait that long to become durable. Deployed and checked against the live server: a label is accepted, a re-send leaves one, an unauthenticated POST gets 401, and the label is on disk within seconds. The probe label used for that check was removed afterwards. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- server/cmd/listend/main.go | 4 + server/internal/api/api.go | 39 ++++++++ server/internal/api/labels_test.go | 167 +++++++++++++++++++++++++++++++++++ server/internal/store/labels.go | 109 +++++++++++++++++++++++ server/internal/store/labels_test.go | 135 ++++++++++++++++++++++++++++ server/internal/store/store.go | 11 ++- 6 files changed, 462 insertions(+), 3 deletions(-) create mode 100644 server/internal/api/labels_test.go create mode 100644 server/internal/store/labels.go create mode 100644 server/internal/store/labels_test.go (limited to 'server') diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index 524f438..c500a63 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -83,6 +83,10 @@ func main() { worker.save() worker.wake() }, + // Same reasoning, one step smaller: a label is a tap the listener made once, and it + // should not sit in memory until the end of a poll cycle. No wake -- nothing acts on a + // label yet, it is corpus for the signature library. + OnLabels: worker.save, }) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) diff --git a/server/internal/api/api.go b/server/internal/api/api.go index 0aeef8a..969e69d 100644 --- a/server/internal/api/api.go +++ b/server/internal/api/api.go @@ -30,6 +30,9 @@ type Options struct { // OnSubscriptions is called when the phone sends its list, so a newly-subscribed show is // polled at once rather than at the next interval. Optional. OnSubscriptions func(feeds []string) + // OnLabels is called when labels were accepted, so they reach disk now rather than at the + // end of the next poll cycle. Optional. + OnLabels func() } // Server is the HTTP surface. @@ -86,6 +89,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("/api/health", s.health) mux.HandleFunc("/api/subscriptions", s.authed(s.subscriptions)) mux.HandleFunc("/api/episode", s.authed(s.episode)) + mux.HandleFunc("/api/labels", s.authed(s.labels)) mux.HandleFunc("/audio/", s.authed(s.audio)) return mux } @@ -230,3 +234,38 @@ func writeJSON(w http.ResponseWriter, status int, body any) { w.WriteHeader(status) _ = json.NewEncoder(w).Encode(body) } + +// labels takes what the listener said about stretches of audio: a judgement on a break +// detection found, or a report of advertising it missed. +// +// Stored rather than acted on. These are the corpus the signature library will be built from, +// and the server has no business changing what it publishes because of one listener's tap -- +// that is what the shadow run is for. +func (s *Server) labels(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "post only"}) + return + } + var body struct { + Labels []store.Label `json:"labels"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "expected {\"labels\":[...]}"}) + return + } + // Counted rather than refused as a batch: a label is a tap the listener made once and + // cannot be asked to make again, so one malformed entry must not discard the rest. + accepted := s.state.AddLabels(body.Labels) + // Only when something usable arrived, which includes a re-send of what is already held: + // the phone sends everything every sweep and has no way to know what got there. That makes + // this a whole-state write once per refresh, which is immaterial next to a label waiting + // hours on the wrong side of a kill -9. + if accepted > 0 && s.opts.OnLabels != nil { + s.opts.OnLabels() + } + writeJSON(w, http.StatusOK, map[string]any{ + "accepted": accepted, + "rejected": len(body.Labels) - accepted, + "held": len(s.state.Labels()), + }) +} diff --git a/server/internal/api/labels_test.go b/server/internal/api/labels_test.go new file mode 100644 index 0000000..7242720 --- /dev/null +++ b/server/internal/api/labels_test.go @@ -0,0 +1,167 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/terst/listen/server/internal/store" +) + +func postLabels(t *testing.T, s *Server, body string, withToken bool) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/api/labels", strings.NewReader(body)) + if withToken { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, req) + return rec +} + +const oneAdLabel = `{"labels":[{"feedUrl":"https://example.com/feed.xml","guid":"g1", + "startMs":600000,"endMs":645000,"verdict":"ad","source":"report","endMeasured":true, + "givenAtUnix":1700000000}]}` + +func TestAcceptsALabelAndKeepsIt(t *testing.T) { + s, st, _, _ := newServer(t) + + rec := postLabels(t, s, oneAdLabel, true) + + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + var got struct { + Accepted int `json:"accepted"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.Accepted != 1 { + t.Fatalf("accepted %d, want 1", got.Accepted) + } + kept := st.Labels() + if len(kept) != 1 { + t.Fatalf("stored %d labels", len(kept)) + } + if kept[0].Verdict != store.LabelAd || kept[0].Source != store.LabelFromReport { + t.Fatalf("got %+v", kept[0]) + } + if !kept[0].EndMeasured { + t.Fatal("endMeasured was dropped; a guessed end and a heard one are not the same thing") + } +} + +func TestLabelsNeedTheToken(t *testing.T) { + s, st, _, _ := newServer(t) + + rec := postLabels(t, s, oneAdLabel, false) + + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status %d, want 401", rec.Code) + } + if len(st.Labels()) != 0 { + t.Fatal("stored a label from an unauthenticated request") + } +} + +func TestLabelsRefuseAnythingButPost(t *testing.T) { + s, _, _, _ := newServer(t) + + if rec := get(t, s, "/api/labels"); rec.Code != http.StatusMethodNotAllowed { + t.Fatalf("status %d, want 405", rec.Code) + } +} + +func TestRejectsBodyThatIsNotLabels(t *testing.T) { + s, st, _, _ := newServer(t) + + rec := postLabels(t, s, `not json at all`, true) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status %d, want 400", rec.Code) + } + if len(st.Labels()) != 0 { + t.Fatal("stored something from an unparseable body") + } +} + +// The phone sends what it has; the server decides what it can use. A batch with one bad entry +// must not cost the listener the good ones -- those labels are not reproducible. +func TestKeepsTheUsableLabelsFromAMixedBatch(t *testing.T) { + s, st, _, _ := newServer(t) + + rec := postLabels(t, s, `{"labels":[ + {"feedUrl":"f","guid":"g1","startMs":0,"endMs":1000,"verdict":"ad","source":"mark"}, + {"feedUrl":"f","guid":"g2","startMs":0,"endMs":0,"verdict":"ad","source":"mark"}, + {"feedUrl":"f","guid":"g3","startMs":0,"endMs":1000,"verdict":"programme","source":"mark"} + ]}`, true) + + if rec.Code != http.StatusOK { + t.Fatalf("status %d", rec.Code) + } + var got struct { + Accepted int `json:"accepted"` + Rejected int `json:"rejected"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &got) + if got.Accepted != 2 || got.Rejected != 1 { + t.Fatalf("accepted %d rejected %d, want 2 and 1", got.Accepted, got.Rejected) + } + if len(st.Labels()) != 2 { + t.Fatalf("stored %d", len(st.Labels())) + } +} + +// Sending the same labels again is the normal case, not an error: the phone re-sends whatever +// it holds and has no way to know what arrived last time. +func TestResendingTheSameLabelsChangesNothing(t *testing.T) { + s, st, _, _ := newServer(t) + postLabels(t, s, oneAdLabel, true) + + postLabels(t, s, oneAdLabel, true) + + if len(st.Labels()) != 1 { + t.Fatalf("stored %d labels after a resend", len(st.Labels())) + } +} + +func TestAnEmptyBatchIsAccepted(t *testing.T) { + s, _, _, _ := newServer(t) + + if rec := postLabels(t, s, `{"labels":[]}`, true); rec.Code != http.StatusOK { + t.Fatalf("status %d, want 200", rec.Code) + } +} + +// Labels arrive between poll cycles, and state is otherwise only written at the end of one. A +// corpus that survives only until the next kill -9 is not a corpus, so accepting labels asks +// the caller to persist. The phone re-sends everything each sweep, so nothing is lost for good +// either way -- but a label is a tap the listener made once and should not wait hours on disk. +func TestAcceptingLabelsAsksForThemToBeSaved(t *testing.T) { + audioDir := t.TempDir() + st := store.New() + saved := 0 + s := New(st, &fakeFetcher{}, Options{Token: token, AudioDir: audioDir, OnLabels: func() { saved++ }}) + + postLabels(t, s, oneAdLabel, true) + + if saved != 1 { + t.Fatalf("asked to save %d times, want 1", saved) + } +} + +func TestNothingUsableMeansNothingToSave(t *testing.T) { + audioDir := t.TempDir() + st := store.New() + saved := 0 + s := New(st, &fakeFetcher{}, Options{Token: token, AudioDir: audioDir, OnLabels: func() { saved++ }}) + + postLabels(t, s, `{"labels":[{"feedUrl":"f","guid":"g","startMs":0,"endMs":0,"verdict":"ad","source":"mark"}]}`, true) + + if saved != 0 { + t.Fatal("wrote the state file for a batch that changed nothing") + } +} diff --git a/server/internal/store/labels.go b/server/internal/store/labels.go new file mode 100644 index 0000000..282196d --- /dev/null +++ b/server/internal/store/labels.go @@ -0,0 +1,109 @@ +package store + +// Label is a listener's statement about one stretch of one episode's audio. +// +// Two things produce these, and they catch opposite errors. A *mark* is a judgement on a break +// detection already found -- it catches false positives. A *report* points at advertising +// detection missed entirely, which is the bigger problem: the scan accounts for roughly 38% of +// the advertising the feeds imply is there, so most of what gets heard was never found. +// +// Identified by feed and guid rather than by the server's own episode id, because that is what +// the phone has: it asks about an episode the same way. The server's id is an implementation +// detail of the mirror, and the mirror forgets an episode after eight. +type Label struct { + FeedURL string `json:"feedUrl"` + GUID string `json:"guid"` + + StartMs int64 `json:"startMs"` + EndMs int64 `json:"endMs"` + + // Verdict is [LabelAd] or [LabelProgramme]. + Verdict string `json:"verdict"` + // Source is [LabelFromMark] or [LabelFromReport]. + Source string `json:"source"` + + // EndMeasured distinguishes an end the listener heard from one that is where a jump + // happened to land. A report starts as a guess at the end and only becomes measured if he + // comes back and trims it, and a signature cut to a guessed end would carry programme. + EndMeasured bool `json:"endMeasured,omitempty"` + + GivenAtUnix int64 `json:"givenAtUnix"` +} + +// What a label can say. +const ( + LabelAd = "ad" + LabelProgramme = "programme" +) + +// Where a label came from. +const ( + LabelFromMark = "mark" + LabelFromReport = "report" +) + +// identity is what makes two labels the same statement: the same stretch of the same episode, +// arrived at the same way. Source is part of it deliberately -- judging a detected break and +// pointing at an undetected advertisement are different acts even when the numbers coincide. +type labelKey struct { + feedURL string + guid string + startMs int64 + source string +} + +func (l Label) key() labelKey { + return labelKey{feedURL: l.FeedURL, guid: l.GUID, startMs: l.StartMs, source: l.Source} +} + +// usable rejects a label that cannot be acted on: one that names no episode, covers no audio, +// or says something this server does not understand. A newer phone inventing a verdict is not a +// reason to store a value nothing can interpret. +func (l Label) usable() bool { + if l.FeedURL == "" || l.GUID == "" { + return false + } + if l.EndMs <= l.StartMs { + return false + } + if l.Verdict != LabelAd && l.Verdict != LabelProgramme { + return false + } + return l.Source == LabelFromMark || l.Source == LabelFromReport +} + +// AddLabels stores what the listener said, replacing any earlier statement about the same +// stretch from the same source, and returns how many were usable. +// +// Replacing rather than appending because he is allowed to change his mind, and two rows +// disagreeing about one stretch of audio would leave the library to guess which he meant. +func (s *State) AddLabels(labels []Label) int { + s.mu.Lock() + defer s.mu.Unlock() + added := 0 + for _, l := range labels { + if !l.usable() { + continue + } + replaced := false + for i, existing := range s.labels { + if existing.key() == l.key() { + s.labels[i] = l + replaced = true + break + } + } + if !replaced { + s.labels = append(s.labels, l) + } + added++ + } + return added +} + +// Labels returns every label the listener has given, oldest first. +func (s *State) Labels() []Label { + s.mu.RLock() + defer s.mu.RUnlock() + return append([]Label(nil), s.labels...) +} diff --git a/server/internal/store/labels_test.go b/server/internal/store/labels_test.go new file mode 100644 index 0000000..ce6a06d --- /dev/null +++ b/server/internal/store/labels_test.go @@ -0,0 +1,135 @@ +package store_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/terst/listen/server/internal/store" +) + +func label(guid string, startMs, endMs int64, verdict, source string) store.Label { + return store.Label{ + FeedURL: "https://example.com/feed.xml", GUID: guid, + StartMs: startMs, EndMs: endMs, Verdict: verdict, Source: source, GivenAtUnix: 100, + } +} + +func TestKeepsALabelTheListenerGave(t *testing.T) { + s := store.New() + + added := s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark)}) + + if added != 1 { + t.Fatalf("added %d, want 1", added) + } + got := s.Labels() + if len(got) != 1 || got[0].GUID != "g1" || got[0].Verdict != store.LabelAd { + t.Fatalf("got %+v", got) + } +} + +func TestChangingYourMindReplacesTheLabelRatherThanAddingASecond(t *testing.T) { + s := store.New() + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark)}) + + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelProgramme, store.LabelFromMark)}) + + got := s.Labels() + if len(got) != 1 { + t.Fatalf("one stretch of audio, one verdict; got %d: %+v", len(got), got) + } + if got[0].Verdict != store.LabelProgramme { + t.Fatalf("verdict %q, want the later one", got[0].Verdict) + } +} + +func TestTheSameStretchReportedAndMarkedAreSeparateStatements(t *testing.T) { + s := store.New() + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark)}) + + // Pointing at advertising the detector missed is a different act from judging a break it + // found, even when the numbers coincide, so one must not silently overwrite the other. + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromReport)}) + + if got := s.Labels(); len(got) != 2 { + t.Fatalf("got %d labels, want 2: %+v", len(got), got) + } +} + +func TestLabelsOnDifferentEpisodesDoNotCollide(t *testing.T) { + s := store.New() + + s.AddLabels([]store.Label{ + label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark), + label("g2", 60_000, 90_000, store.LabelAd, store.LabelFromMark), + }) + + if got := s.Labels(); len(got) != 2 { + t.Fatalf("got %d, want 2", len(got)) + } +} + +func TestRejectsALabelWithNothingToIdentifyItBy(t *testing.T) { + s := store.New() + + added := s.AddLabels([]store.Label{ + {FeedURL: "", GUID: "g1", EndMs: 10, Verdict: store.LabelAd}, + {FeedURL: "f", GUID: "", EndMs: 10, Verdict: store.LabelAd}, + {FeedURL: "f", GUID: "g", EndMs: 0, Verdict: store.LabelAd}, + {FeedURL: "f", GUID: "g", EndMs: 10, Verdict: "shrug"}, + }) + + if added != 0 { + t.Fatalf("accepted %d unusable labels", added) + } + if got := s.Labels(); len(got) != 0 { + t.Fatalf("stored %+v", got) + } +} + +func TestLabelsSurviveASaveAndLoad(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + s := store.New() + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromReport)}) + if err := s.Save(path); err != nil { + t.Fatalf("save: %v", err) + } + + again, err := store.Load(path) + if err != nil { + t.Fatalf("load: %v", err) + } + + got := again.Labels() + if len(got) != 1 || got[0].Source != store.LabelFromReport { + t.Fatalf("got %+v", got) + } +} + +// A label is evidence about a stretch of audio. The episode it was given on rotates out of the +// mirror after eight, and the label has to outlive that -- it is the corpus the signature +// library is built from, and throwing it away with the audio would be throwing away the point. +func TestLabelsOutliveTheEpisodeTheyWereGivenOn(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + s := store.New() + s.AddLabels([]store.Label{label("long-gone", 60_000, 90_000, store.LabelAd, store.LabelFromMark)}) + if err := s.Save(path); err != nil { + t.Fatalf("save: %v", err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(raw) == 0 { + t.Fatal("empty state file") + } + + again, _ := store.Load(path) + // Nothing in the state knows this episode any more; the label is still there. + if got := again.Labels(); len(got) != 1 || got[0].GUID != "long-gone" { + t.Fatalf("got %+v", got) + } +} diff --git a/server/internal/store/store.go b/server/internal/store/store.go index 4734009..7f04fa0 100644 --- a/server/internal/store/store.go +++ b/server/internal/store/store.go @@ -105,11 +105,15 @@ type State struct { mu sync.RWMutex feeds []string shows map[string]*Show + // labels are the listener's own statements about stretches of audio. Not held on the + // episodes: an episode leaves the mirror after eight and a label has to outlive it. + labels []Label } type persisted struct { - Feeds []string `json:"feeds"` - Shows map[string]*Show `json:"shows"` + Feeds []string `json:"feeds"` + Shows map[string]*Show `json:"shows"` + Labels []Label `json:"labels,omitempty"` } // New returns an empty state. @@ -132,6 +136,7 @@ func Load(path string) (*State, error) { } s := New() s.feeds = p.Feeds + s.labels = p.Labels if p.Shows != nil { s.shows = p.Shows } @@ -145,7 +150,7 @@ func Load(path string) (*State, error) { // away hours of detection. func (s *State) Save(path string) error { s.mu.RLock() - raw, err := json.MarshalIndent(persisted{Feeds: s.feeds, Shows: s.shows}, "", " ") + raw, err := json.MarshalIndent(persisted{Feeds: s.feeds, Shows: s.shows, Labels: s.labels}, "", " ") s.mu.RUnlock() if err != nil { return fmt.Errorf("save state: %w", err) -- cgit v1.2.3