summaryrefslogtreecommitdiff
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/cmd/listend/main.go4
-rw-r--r--server/internal/api/api.go39
-rw-r--r--server/internal/api/labels_test.go167
-rw-r--r--server/internal/store/labels.go109
-rw-r--r--server/internal/store/labels_test.go135
-rw-r--r--server/internal/store/store.go11
6 files changed, 462 insertions, 3 deletions
diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go
index 524f438..c500a63 100644
--- a/server/cmd/listend/main.go
+++ b/server/cmd/listend/main.go
@@ -83,6 +83,10 @@ func main() {
worker.save()
worker.wake()
},
+ // Same reasoning, one step smaller: a label is a tap the listener made once, and it
+ // should not sit in memory until the end of a poll cycle. No wake -- nothing acts on a
+ // label yet, it is corpus for the signature library.
+ OnLabels: worker.save,
})
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
diff --git a/server/internal/api/api.go b/server/internal/api/api.go
index 0aeef8a..969e69d 100644
--- a/server/internal/api/api.go
+++ b/server/internal/api/api.go
@@ -30,6 +30,9 @@ type Options struct {
// OnSubscriptions is called when the phone sends its list, so a newly-subscribed show is
// polled at once rather than at the next interval. Optional.
OnSubscriptions func(feeds []string)
+ // OnLabels is called when labels were accepted, so they reach disk now rather than at the
+ // end of the next poll cycle. Optional.
+ OnLabels func()
}
// Server is the HTTP surface.
@@ -86,6 +89,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("/api/health", s.health)
mux.HandleFunc("/api/subscriptions", s.authed(s.subscriptions))
mux.HandleFunc("/api/episode", s.authed(s.episode))
+ mux.HandleFunc("/api/labels", s.authed(s.labels))
mux.HandleFunc("/audio/", s.authed(s.audio))
return mux
}
@@ -230,3 +234,38 @@ func writeJSON(w http.ResponseWriter, status int, body any) {
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(body)
}
+
+// labels takes what the listener said about stretches of audio: a judgement on a break
+// detection found, or a report of advertising it missed.
+//
+// Stored rather than acted on. These are the corpus the signature library will be built from,
+// and the server has no business changing what it publishes because of one listener's tap --
+// that is what the shadow run is for.
+func (s *Server) labels(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "post only"})
+ return
+ }
+ var body struct {
+ Labels []store.Label `json:"labels"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ writeJSON(w, http.StatusBadRequest, map[string]string{"error": "expected {\"labels\":[...]}"})
+ return
+ }
+ // Counted rather than refused as a batch: a label is a tap the listener made once and
+ // cannot be asked to make again, so one malformed entry must not discard the rest.
+ accepted := s.state.AddLabels(body.Labels)
+ // Only when something usable arrived, which includes a re-send of what is already held:
+ // the phone sends everything every sweep and has no way to know what got there. That makes
+ // this a whole-state write once per refresh, which is immaterial next to a label waiting
+ // hours on the wrong side of a kill -9.
+ if accepted > 0 && s.opts.OnLabels != nil {
+ s.opts.OnLabels()
+ }
+ writeJSON(w, http.StatusOK, map[string]any{
+ "accepted": accepted,
+ "rejected": len(body.Labels) - accepted,
+ "held": len(s.state.Labels()),
+ })
+}
diff --git a/server/internal/api/labels_test.go b/server/internal/api/labels_test.go
new file mode 100644
index 0000000..7242720
--- /dev/null
+++ b/server/internal/api/labels_test.go
@@ -0,0 +1,167 @@
+package api
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/terst/listen/server/internal/store"
+)
+
+func postLabels(t *testing.T, s *Server, body string, withToken bool) *httptest.ResponseRecorder {
+ t.Helper()
+ req := httptest.NewRequest(http.MethodPost, "/api/labels", strings.NewReader(body))
+ if withToken {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, req)
+ return rec
+}
+
+const oneAdLabel = `{"labels":[{"feedUrl":"https://example.com/feed.xml","guid":"g1",
+ "startMs":600000,"endMs":645000,"verdict":"ad","source":"report","endMeasured":true,
+ "givenAtUnix":1700000000}]}`
+
+func TestAcceptsALabelAndKeepsIt(t *testing.T) {
+ s, st, _, _ := newServer(t)
+
+ rec := postLabels(t, s, oneAdLabel, true)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
+ }
+ var got struct {
+ Accepted int `json:"accepted"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if got.Accepted != 1 {
+ t.Fatalf("accepted %d, want 1", got.Accepted)
+ }
+ kept := st.Labels()
+ if len(kept) != 1 {
+ t.Fatalf("stored %d labels", len(kept))
+ }
+ if kept[0].Verdict != store.LabelAd || kept[0].Source != store.LabelFromReport {
+ t.Fatalf("got %+v", kept[0])
+ }
+ if !kept[0].EndMeasured {
+ t.Fatal("endMeasured was dropped; a guessed end and a heard one are not the same thing")
+ }
+}
+
+func TestLabelsNeedTheToken(t *testing.T) {
+ s, st, _, _ := newServer(t)
+
+ rec := postLabels(t, s, oneAdLabel, false)
+
+ if rec.Code != http.StatusUnauthorized {
+ t.Fatalf("status %d, want 401", rec.Code)
+ }
+ if len(st.Labels()) != 0 {
+ t.Fatal("stored a label from an unauthenticated request")
+ }
+}
+
+func TestLabelsRefuseAnythingButPost(t *testing.T) {
+ s, _, _, _ := newServer(t)
+
+ if rec := get(t, s, "/api/labels"); rec.Code != http.StatusMethodNotAllowed {
+ t.Fatalf("status %d, want 405", rec.Code)
+ }
+}
+
+func TestRejectsBodyThatIsNotLabels(t *testing.T) {
+ s, st, _, _ := newServer(t)
+
+ rec := postLabels(t, s, `not json at all`, true)
+
+ if rec.Code != http.StatusBadRequest {
+ t.Fatalf("status %d, want 400", rec.Code)
+ }
+ if len(st.Labels()) != 0 {
+ t.Fatal("stored something from an unparseable body")
+ }
+}
+
+// The phone sends what it has; the server decides what it can use. A batch with one bad entry
+// must not cost the listener the good ones -- those labels are not reproducible.
+func TestKeepsTheUsableLabelsFromAMixedBatch(t *testing.T) {
+ s, st, _, _ := newServer(t)
+
+ rec := postLabels(t, s, `{"labels":[
+ {"feedUrl":"f","guid":"g1","startMs":0,"endMs":1000,"verdict":"ad","source":"mark"},
+ {"feedUrl":"f","guid":"g2","startMs":0,"endMs":0,"verdict":"ad","source":"mark"},
+ {"feedUrl":"f","guid":"g3","startMs":0,"endMs":1000,"verdict":"programme","source":"mark"}
+ ]}`, true)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status %d", rec.Code)
+ }
+ var got struct {
+ Accepted int `json:"accepted"`
+ Rejected int `json:"rejected"`
+ }
+ _ = json.Unmarshal(rec.Body.Bytes(), &got)
+ if got.Accepted != 2 || got.Rejected != 1 {
+ t.Fatalf("accepted %d rejected %d, want 2 and 1", got.Accepted, got.Rejected)
+ }
+ if len(st.Labels()) != 2 {
+ t.Fatalf("stored %d", len(st.Labels()))
+ }
+}
+
+// Sending the same labels again is the normal case, not an error: the phone re-sends whatever
+// it holds and has no way to know what arrived last time.
+func TestResendingTheSameLabelsChangesNothing(t *testing.T) {
+ s, st, _, _ := newServer(t)
+ postLabels(t, s, oneAdLabel, true)
+
+ postLabels(t, s, oneAdLabel, true)
+
+ if len(st.Labels()) != 1 {
+ t.Fatalf("stored %d labels after a resend", len(st.Labels()))
+ }
+}
+
+func TestAnEmptyBatchIsAccepted(t *testing.T) {
+ s, _, _, _ := newServer(t)
+
+ if rec := postLabels(t, s, `{"labels":[]}`, true); rec.Code != http.StatusOK {
+ t.Fatalf("status %d, want 200", rec.Code)
+ }
+}
+
+// Labels arrive between poll cycles, and state is otherwise only written at the end of one. A
+// corpus that survives only until the next kill -9 is not a corpus, so accepting labels asks
+// the caller to persist. The phone re-sends everything each sweep, so nothing is lost for good
+// either way -- but a label is a tap the listener made once and should not wait hours on disk.
+func TestAcceptingLabelsAsksForThemToBeSaved(t *testing.T) {
+ audioDir := t.TempDir()
+ st := store.New()
+ saved := 0
+ s := New(st, &fakeFetcher{}, Options{Token: token, AudioDir: audioDir, OnLabels: func() { saved++ }})
+
+ postLabels(t, s, oneAdLabel, true)
+
+ if saved != 1 {
+ t.Fatalf("asked to save %d times, want 1", saved)
+ }
+}
+
+func TestNothingUsableMeansNothingToSave(t *testing.T) {
+ audioDir := t.TempDir()
+ st := store.New()
+ saved := 0
+ s := New(st, &fakeFetcher{}, Options{Token: token, AudioDir: audioDir, OnLabels: func() { saved++ }})
+
+ postLabels(t, s, `{"labels":[{"feedUrl":"f","guid":"g","startMs":0,"endMs":0,"verdict":"ad","source":"mark"}]}`, true)
+
+ if saved != 0 {
+ t.Fatal("wrote the state file for a batch that changed nothing")
+ }
+}
diff --git a/server/internal/store/labels.go b/server/internal/store/labels.go
new file mode 100644
index 0000000..282196d
--- /dev/null
+++ b/server/internal/store/labels.go
@@ -0,0 +1,109 @@
+package store
+
+// Label is a listener's statement about one stretch of one episode's audio.
+//
+// Two things produce these, and they catch opposite errors. A *mark* is a judgement on a break
+// detection already found -- it catches false positives. A *report* points at advertising
+// detection missed entirely, which is the bigger problem: the scan accounts for roughly 38% of
+// the advertising the feeds imply is there, so most of what gets heard was never found.
+//
+// Identified by feed and guid rather than by the server's own episode id, because that is what
+// the phone has: it asks about an episode the same way. The server's id is an implementation
+// detail of the mirror, and the mirror forgets an episode after eight.
+type Label struct {
+ FeedURL string `json:"feedUrl"`
+ GUID string `json:"guid"`
+
+ StartMs int64 `json:"startMs"`
+ EndMs int64 `json:"endMs"`
+
+ // Verdict is [LabelAd] or [LabelProgramme].
+ Verdict string `json:"verdict"`
+ // Source is [LabelFromMark] or [LabelFromReport].
+ Source string `json:"source"`
+
+ // EndMeasured distinguishes an end the listener heard from one that is where a jump
+ // happened to land. A report starts as a guess at the end and only becomes measured if he
+ // comes back and trims it, and a signature cut to a guessed end would carry programme.
+ EndMeasured bool `json:"endMeasured,omitempty"`
+
+ GivenAtUnix int64 `json:"givenAtUnix"`
+}
+
+// What a label can say.
+const (
+ LabelAd = "ad"
+ LabelProgramme = "programme"
+)
+
+// Where a label came from.
+const (
+ LabelFromMark = "mark"
+ LabelFromReport = "report"
+)
+
+// identity is what makes two labels the same statement: the same stretch of the same episode,
+// arrived at the same way. Source is part of it deliberately -- judging a detected break and
+// pointing at an undetected advertisement are different acts even when the numbers coincide.
+type labelKey struct {
+ feedURL string
+ guid string
+ startMs int64
+ source string
+}
+
+func (l Label) key() labelKey {
+ return labelKey{feedURL: l.FeedURL, guid: l.GUID, startMs: l.StartMs, source: l.Source}
+}
+
+// usable rejects a label that cannot be acted on: one that names no episode, covers no audio,
+// or says something this server does not understand. A newer phone inventing a verdict is not a
+// reason to store a value nothing can interpret.
+func (l Label) usable() bool {
+ if l.FeedURL == "" || l.GUID == "" {
+ return false
+ }
+ if l.EndMs <= l.StartMs {
+ return false
+ }
+ if l.Verdict != LabelAd && l.Verdict != LabelProgramme {
+ return false
+ }
+ return l.Source == LabelFromMark || l.Source == LabelFromReport
+}
+
+// AddLabels stores what the listener said, replacing any earlier statement about the same
+// stretch from the same source, and returns how many were usable.
+//
+// Replacing rather than appending because he is allowed to change his mind, and two rows
+// disagreeing about one stretch of audio would leave the library to guess which he meant.
+func (s *State) AddLabels(labels []Label) int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ added := 0
+ for _, l := range labels {
+ if !l.usable() {
+ continue
+ }
+ replaced := false
+ for i, existing := range s.labels {
+ if existing.key() == l.key() {
+ s.labels[i] = l
+ replaced = true
+ break
+ }
+ }
+ if !replaced {
+ s.labels = append(s.labels, l)
+ }
+ added++
+ }
+ return added
+}
+
+// Labels returns every label the listener has given, oldest first.
+func (s *State) Labels() []Label {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ return append([]Label(nil), s.labels...)
+}
diff --git a/server/internal/store/labels_test.go b/server/internal/store/labels_test.go
new file mode 100644
index 0000000..ce6a06d
--- /dev/null
+++ b/server/internal/store/labels_test.go
@@ -0,0 +1,135 @@
+package store_test
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/terst/listen/server/internal/store"
+)
+
+func label(guid string, startMs, endMs int64, verdict, source string) store.Label {
+ return store.Label{
+ FeedURL: "https://example.com/feed.xml", GUID: guid,
+ StartMs: startMs, EndMs: endMs, Verdict: verdict, Source: source, GivenAtUnix: 100,
+ }
+}
+
+func TestKeepsALabelTheListenerGave(t *testing.T) {
+ s := store.New()
+
+ added := s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark)})
+
+ if added != 1 {
+ t.Fatalf("added %d, want 1", added)
+ }
+ got := s.Labels()
+ if len(got) != 1 || got[0].GUID != "g1" || got[0].Verdict != store.LabelAd {
+ t.Fatalf("got %+v", got)
+ }
+}
+
+func TestChangingYourMindReplacesTheLabelRatherThanAddingASecond(t *testing.T) {
+ s := store.New()
+ s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark)})
+
+ s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelProgramme, store.LabelFromMark)})
+
+ got := s.Labels()
+ if len(got) != 1 {
+ t.Fatalf("one stretch of audio, one verdict; got %d: %+v", len(got), got)
+ }
+ if got[0].Verdict != store.LabelProgramme {
+ t.Fatalf("verdict %q, want the later one", got[0].Verdict)
+ }
+}
+
+func TestTheSameStretchReportedAndMarkedAreSeparateStatements(t *testing.T) {
+ s := store.New()
+ s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark)})
+
+ // Pointing at advertising the detector missed is a different act from judging a break it
+ // found, even when the numbers coincide, so one must not silently overwrite the other.
+ s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromReport)})
+
+ if got := s.Labels(); len(got) != 2 {
+ t.Fatalf("got %d labels, want 2: %+v", len(got), got)
+ }
+}
+
+func TestLabelsOnDifferentEpisodesDoNotCollide(t *testing.T) {
+ s := store.New()
+
+ s.AddLabels([]store.Label{
+ label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark),
+ label("g2", 60_000, 90_000, store.LabelAd, store.LabelFromMark),
+ })
+
+ if got := s.Labels(); len(got) != 2 {
+ t.Fatalf("got %d, want 2", len(got))
+ }
+}
+
+func TestRejectsALabelWithNothingToIdentifyItBy(t *testing.T) {
+ s := store.New()
+
+ added := s.AddLabels([]store.Label{
+ {FeedURL: "", GUID: "g1", EndMs: 10, Verdict: store.LabelAd},
+ {FeedURL: "f", GUID: "", EndMs: 10, Verdict: store.LabelAd},
+ {FeedURL: "f", GUID: "g", EndMs: 0, Verdict: store.LabelAd},
+ {FeedURL: "f", GUID: "g", EndMs: 10, Verdict: "shrug"},
+ })
+
+ if added != 0 {
+ t.Fatalf("accepted %d unusable labels", added)
+ }
+ if got := s.Labels(); len(got) != 0 {
+ t.Fatalf("stored %+v", got)
+ }
+}
+
+func TestLabelsSurviveASaveAndLoad(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "state.json")
+ s := store.New()
+ s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromReport)})
+ if err := s.Save(path); err != nil {
+ t.Fatalf("save: %v", err)
+ }
+
+ again, err := store.Load(path)
+ if err != nil {
+ t.Fatalf("load: %v", err)
+ }
+
+ got := again.Labels()
+ if len(got) != 1 || got[0].Source != store.LabelFromReport {
+ t.Fatalf("got %+v", got)
+ }
+}
+
+// A label is evidence about a stretch of audio. The episode it was given on rotates out of the
+// mirror after eight, and the label has to outlive that -- it is the corpus the signature
+// library is built from, and throwing it away with the audio would be throwing away the point.
+func TestLabelsOutliveTheEpisodeTheyWereGivenOn(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "state.json")
+ s := store.New()
+ s.AddLabels([]store.Label{label("long-gone", 60_000, 90_000, store.LabelAd, store.LabelFromMark)})
+ if err := s.Save(path); err != nil {
+ t.Fatalf("save: %v", err)
+ }
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read: %v", err)
+ }
+ if len(raw) == 0 {
+ t.Fatal("empty state file")
+ }
+
+ again, _ := store.Load(path)
+ // Nothing in the state knows this episode any more; the label is still there.
+ if got := again.Labels(); len(got) != 1 || got[0].GUID != "long-gone" {
+ t.Fatalf("got %+v", got)
+ }
+}
diff --git a/server/internal/store/store.go b/server/internal/store/store.go
index 4734009..7f04fa0 100644
--- a/server/internal/store/store.go
+++ b/server/internal/store/store.go
@@ -105,11 +105,15 @@ type State struct {
mu sync.RWMutex
feeds []string
shows map[string]*Show
+ // labels are the listener's own statements about stretches of audio. Not held on the
+ // episodes: an episode leaves the mirror after eight and a label has to outlive it.
+ labels []Label
}
type persisted struct {
- Feeds []string `json:"feeds"`
- Shows map[string]*Show `json:"shows"`
+ Feeds []string `json:"feeds"`
+ Shows map[string]*Show `json:"shows"`
+ Labels []Label `json:"labels,omitempty"`
}
// New returns an empty state.
@@ -132,6 +136,7 @@ func Load(path string) (*State, error) {
}
s := New()
s.feeds = p.Feeds
+ s.labels = p.Labels
if p.Shows != nil {
s.shows = p.Shows
}
@@ -145,7 +150,7 @@ func Load(path string) (*State, error) {
// away hours of detection.
func (s *State) Save(path string) error {
s.mu.RLock()
- raw, err := json.MarshalIndent(persisted{Feeds: s.feeds, Shows: s.shows}, "", " ")
+ raw, err := json.MarshalIndent(persisted{Feeds: s.feeds, Shows: s.shows, Labels: s.labels}, "", " ")
s.mu.RUnlock()
if err != nil {
return fmt.Errorf("save state: %w", err)