// Package api serves the phone: what to skip, and the audio those offsets belong to. package api import ( "encoding/json" "log" "net/http" "net/url" "os" "path/filepath" "strings" "time" "github.com/terst/listen/server/internal/store" "github.com/terst/listen/server/internal/transcript" ) // Fetcher is asked to mirror an episode the phone wants and the server does not have. // // Asynchronous on purpose: the phone gets an immediate "pending" rather than holding a request // open for the length of a download. type Fetcher interface { Request(feedURL, guid, enclosureURL, title string) } // Options configures the server. type Options struct { // Token guards everything but health. This serves a private library over the open // internet. Token string // AudioDir is where mirrored episodes live. AudioDir string // TranscriptDir is where episodes' words live, one file per episode. Separate from the // state file because a transcript is a thousand cues and state.json is rewritten whole // every time anything at all changes. TranscriptDir string // OnSubscriptions is called when the phone sends its list, so a newly-subscribed show is // polled at once rather than at the next interval. Optional. OnSubscriptions func(feeds []string) // OnLabels is called when labels were accepted, so they reach disk now rather than at the // end of the next poll cycle. Optional. OnLabels func() } // Server is the HTTP surface. type Server struct { state *store.State fetcher Fetcher opts Options // clock is the time source, left nil outside tests. clock func() int64 } // New builds the server. func New(state *store.State, fetcher Fetcher, opts Options) *Server { return &Server{state: state, fetcher: fetcher, opts: opts} } // now is the clock, overridable in tests the same way the words pipeline's is. func (s *Server) now() int64 { if s.clock != nil { return s.clock() } return time.Now().Unix() } type episodeResponse struct { // Status is "ready" when the audio can be played now, or "pending" when the server has // been asked to fetch it. There is deliberately no "unknown": an episode the phone knows // about is one the server can be told to go and get. Status string `json:"status"` AudioURL string `json:"audioUrl,omitempty"` DurationMs int64 `json:"durationMs,omitempty"` IntroMs *int64 `json:"introMs,omitempty"` OutroMs *int64 `json:"outroMs,omitempty"` // Detected says whether detection has run over this episode's show, so the phone can tell // "looked and found nothing" apart from "not looked at yet" -- a distinction the app got // wrong when the readout said "None" for both. Detected bool `json:"detected"` // Breaks are the stretches to skip: the advertising this episode shares with the others // mirrored alongside it. Breaks []breakRange `json:"breaks,omitempty"` // FoundAdMs and InsertedAdMs let the app show what was accounted for against what the // feed's declared length implies is there. FoundAdMs int64 `json:"foundAdMs,omitempty"` InsertedAdMs int64 `json:"insertedAdMs,omitempty"` // AdsRefused: matches were found but were too extensive to be advertising, so none were // published. Distinct from nothing having matched. AdsRefused bool `json:"adsRefused,omitempty"` // Transcript says whether this episode has words and where they came from, or why it has // none. Absent entirely when nobody has looked yet, which is a third state again. Transcript *transcriptNote `json:"transcript,omitempty"` } // transcriptNote is the line the phone needs on every episode: is there a transcript, whose // words are they, and if there is none, why not. // // The source travels with it because the two kinds are not the same thing to read along with. // A published transcript is the publisher's exact words; a recognised one is a machine's guess // at the same audio, and somebody reading it should know which they are looking at. type transcriptNote struct { Source string `json:"source,omitempty"` Cues int `json:"cues,omitempty"` Note string `json:"note,omitempty"` // URL is where the cues themselves are, absolute and token-carrying, usable as given -- // the same way the audio URL works. The phone keys episodes by feed and guid and the // server keys them by its own id, and handing over a URL is what keeps that from having // to change. Empty when there is nothing to fetch. URL string `json:"url,omitempty"` // Ads and AdMs are what reading the words concluded about the advertising in this episode: // how many stretches and how long they come to. Candidates, not skips -- nothing in the // playback path reads them. Ads int `json:"ads,omitempty"` AdMs int64 `json:"adMs,omitempty"` // Method is how they were found: "difference" where a second description of the programme // was available to subtract, or the kinds of ad copy that were read otherwise. The two are // not close in quality and the screen should not present them as if they were. Method string `json:"method,omitempty"` } type breakRange struct { StartMs int64 `json:"startMs"` EndMs int64 `json:"endMs"` Kind string `json:"kind,omitempty"` // Partners is how many other episodes were found to carry this audio. The phone shows it // when a break is tapped: it is the only evidence a listener can weigh before deciding // whether the detector got this one right. Partners int `json:"partners,omitempty"` // Origin is which detector found it: "audio", "words", or both. Same reasoning as // Partners -- a listener judging a skip wants to know what decided it. Origin string `json:"origin,omitempty"` } // Handler returns the routed, authenticated handler. func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("/api/health", s.health) mux.HandleFunc("/api/subscriptions", s.authed(s.subscriptions)) mux.HandleFunc("/api/episode", s.authed(s.episode)) mux.HandleFunc("/api/labels", s.authed(s.labels)) mux.HandleFunc("/api/transcript", s.authed(s.transcript)) mux.HandleFunc("/audio/", s.authed(s.audio)) return mux } func (s *Server) health(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) } // authed accepts the token in the Authorization header or the query string. // // The query string is not laziness: a media player fetching audio cannot always be made to set // a header, and an audio URL the phone cannot use as given would be useless. func (s *Server) authed(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if s.opts.Token == "" || !s.tokenOK(r) { w.Header().Set("WWW-Authenticate", `Bearer realm="listen"`) writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) return } next(w, r) } } func (s *Server) tokenOK(r *http.Request) bool { if header := r.Header.Get("Authorization"); strings.HasPrefix(header, "Bearer ") { if strings.TrimPrefix(header, "Bearer ") == s.opts.Token { return true } } return r.URL.Query().Get("token") == s.opts.Token } func (s *Server) subscriptions(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "post only"}) return } var body struct { Feeds []string `json:"feeds"` // Categories says what sort of listening each feed is, keyed by URL, which is also // separate from the list rather than folded into it, so a phone that knows nothing // about tiers keeps working unchanged. Categories map[string]string `json:"categories,omitempty"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "expected {\"feeds\":[...]}"}) return } removed, err := s.state.SetFeedsChecked(body.Feeds) if err != nil { // Refused rather than applied: losing most of a library in one push is another device // or a bug. Said out loud, because a silent refusal looks exactly like a silent // acceptance from the phone's end. log.Printf("subscriptions: %v", err) writeJSON(w, http.StatusConflict, map[string]string{"error": err.Error()}) return } // After the list, so a tier for a feed that was just removed is dropped with it. s.state.SetCategories(body.Categories) if s.opts.OnSubscriptions != nil { s.opts.OnSubscriptions(s.state.Feeds()) } writeJSON(w, http.StatusOK, map[string]any{ "feeds": len(s.state.Feeds()), "removed": removed, }) } func (s *Server) episode(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() feedURL, guid := q.Get("feed"), q.Get("guid") enclosure, title := q.Get("enclosure"), q.Get("title") if feedURL == "" || (guid == "" && enclosure == "" && title == "") { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "feed and at least one of guid, enclosure or title are required"}) return } found := s.state.Match(feedURL, guid, enclosure, title) if found != nil { // The only moment the server learns anybody wants this episode. Recorded whether or not // it is mirrored yet, and on every ask, so resuming something played last month keeps // it from falling out of the mirror while it is being listened to. found.Requested(s.now()) } if found == nil || !found.Mirrored() { // Ask for it rather than refusing. Everything Peter actually listens to should end up // with offsets, including an episode older than the ones kept by default. s.fetcher.Request(feedURL, guid, enclosure, title) writeJSON(w, http.StatusOK, episodeResponse{Status: "pending"}) return } if _, err := os.Stat(filepath.Join(s.opts.AudioDir, found.Path)); err != nil { // The record claims a file that has gone. Treat it as not mirrored and fetch again. s.fetcher.Request(feedURL, found.GUID, found.EnclosureURL, found.Title) writeJSON(w, http.StatusOK, episodeResponse{Status: "pending"}) return } all := found.AllBreaks() breaks := make([]breakRange, 0, len(all)) for _, br := range all { breaks = append(breaks, breakRange{ StartMs: br.StartMs, EndMs: br.EndMs, Kind: br.Kind, Partners: br.Partners, Origin: br.Origin, }) } writeJSON(w, http.StatusOK, episodeResponse{ Status: "ready", // No token. Apache logs the request line, so one in the query put a copy of the secret // in every line of a file logrotate keeps fourteen of. The phone sends // Authorization: Bearer for this host -- the downloader always did, and the player does // too since it got a ResolvingDataSource -- and tokenOK takes either, so the query // parameter cost a leak and bought nothing. Query tokens are still accepted, because // withdrawing that as well would lock out an older build with no way back in. AudioURL: "/audio/" + found.ID, DurationMs: found.DurationMs, IntroMs: found.IntroMs, OutroMs: found.OutroMs, Detected: found.DetectedAtUnix != 0, Breaks: breaks, FoundAdMs: found.FoundAdMs, InsertedAdMs: found.InsertedAdMs, AdsRefused: found.AdsRefused, Transcript: s.transcriptNoteFor(found), }) } func (s *Server) transcriptNoteFor(e *store.Episode) *transcriptNote { if !e.TranscriptExamined() { return nil } note := &transcriptNote{ Source: e.TranscriptSource, Cues: e.TranscriptCues, Note: e.TranscriptNote, } if e.HasTranscript() { // No token. The phone sends Authorization: Bearer on this request anyway and tokenOK // takes either, so putting it in the URL bought nothing and cost a copy of the token in // every line of Apache's access log -- which keeps fourteen rotations. note.URL = "/api/transcript?episode=" + url.QueryEscape(e.ID) if held, err := (transcript.Store{Dir: s.opts.TranscriptDir}).Get(e.ID); err == nil { note.Ads = len(held.Ads) for _, ad := range held.Ads { note.AdMs += ad.EndMs - ad.StartMs if note.Method == "" && len(ad.Signals) > 0 { note.Method = ad.Signals[0] } } } } return note } // transcript serves one episode's words, which are fetched only when somebody opens them -- // a thousand cues is not something to attach to every episode lookup. func (s *Server) transcript(w http.ResponseWriter, r *http.Request) { id := r.URL.Query().Get("episode") // Same rule as the audio route: the id decides a path, so anything with a separator or a // dot segment in it is an attempt to read elsewhere. if id == "" || strings.ContainsAny(id, "/\\") || strings.Contains(id, "..") { http.NotFound(w, r) return } held, err := (transcript.Store{Dir: s.opts.TranscriptDir}).Get(id) if err != nil { http.NotFound(w, r) return } writeJSON(w, http.StatusOK, held) } func (s *Server) audio(w http.ResponseWriter, r *http.Request) { id := strings.TrimPrefix(r.URL.Path, "/audio/") // No separators: an id is a flat hex string, and anything else is an attempt to read // outside the audio directory. if id == "" || strings.ContainsAny(id, "/\\") || strings.Contains(id, "..") { http.NotFound(w, r) return } found := s.episodeByID(id) if found == nil { http.NotFound(w, r) return } path := filepath.Join(s.opts.AudioDir, found.Path) info, err := os.Stat(path) if err != nil || info.IsDir() { http.NotFound(w, r) return } // ServeFile handles Range, If-Modified-Since and the content type, which a podcast player // needs for seeking. http.ServeFile(w, r, path) } func (s *Server) episodeByID(id string) *store.Episode { for _, show := range s.state.Shows() { for _, e := range show.Episodes { if e.ID == id && e.Mirrored() { return e } } } return nil } func writeJSON(w http.ResponseWriter, status int, body any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) _ = json.NewEncoder(w).Encode(body) } // labels takes what the listener said about stretches of audio: a judgement on a break // detection found, or a report of advertising it missed. // // Stored rather than acted on. These are the corpus the signature library will be built from, // and the server has no business changing what it publishes because of one listener's tap -- // that is what the shadow run is for. func (s *Server) labels(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "post only"}) return } var body struct { Labels []store.Label `json:"labels"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "expected {\"labels\":[...]}"}) return } // Counted rather than refused as a batch: a label is a tap the listener made once and // cannot be asked to make again, so one malformed entry must not discard the rest. accepted := s.state.AddLabels(body.Labels) // Only when something usable arrived, which includes a re-send of what is already held: // the phone sends everything every sweep and has no way to know what got there. That makes // this a whole-state write once per refresh, which is immaterial next to a label waiting // hours on the wrong side of a kill -9. if accepted > 0 && s.opts.OnLabels != nil { s.opts.OnLabels() } writeJSON(w, http.StatusOK, map[string]any{ "accepted": accepted, "rejected": len(body.Labels) - accepted, "held": len(s.state.Labels()), }) }