package transcript import ( "context" "fmt" "io" "net/http" "time" "github.com/terst/listen/server/internal/safefetch" ) // maxTranscriptBytes caps a download. The largest in this corpus is Darknet Diaries' 1784-cue // SRT at well under a megabyte; eight is room for an outlier and a refusal for a mistake. const maxTranscriptBytes = 8 << 20 // Offer is a publisher's transcript and the file it would have to describe. type Offer struct { EpisodeID string URL string Type string // ServedMs is the mirrored file's measured length, DeclaredMs the feed's claim. Their // difference is the advertising inserted into this copy, which is what decides whether the // publisher's master is what was mirrored. ServedMs int64 DeclaredMs int64 } // Result is what came of an offer: a transcript stored, or a recorded reason it was not. type Result struct { Stored bool Cues int // Ads is how many stretches of the stored transcript read as advertising. Candidates only. Ads int // Master is how many cues were kept as the publisher's ad-free transcript. Non-zero only // when the transcript was declined for this copy because advertising was inserted into it: // the words are then the programme without the advertising, which is what [Inserted] // subtracts from a recognised reading of the served file. Master int // Note is why nothing was stored, and is empty when something was. It is a verdict about // this copy of the episode, so it is recorded rather than retried -- and for an episode // carrying inserted advertising it is also the signal that this is one recognition has to do. Note string } // Harvester takes publishers' transcripts, when they describe the file the server mirrored. type Harvester struct { Store Store Client *http.Client } func (h Harvester) client() *http.Client { if h.Client != nil { return h.Client } // A transcript URL comes from a publisher's feed, so it names an address this server did // not choose. safefetch refuses one that resolves to somewhere only this host can reach. return safefetch.Client(60 * time.Second) } // Harvest fetches an offered transcript and stores it if it describes the mirrored file. // // An error means try again -- the download failed, so nothing has been decided. A Result with a // Note means do not try again: the transcript arrived and was judged. func (h Harvester) Harvest(ctx context.Context, offer Offer) (Result, error) { if offer.URL == "" { return Result{}, nil } body, err := h.download(ctx, offer.URL) if err != nil { return Result{}, err } cues, err := Parse(offer.Type, body) if err != nil { // The feed said this URL is the transcript and it is not one. Nothing about fetching it // again changes that, so it is a verdict. return Result{Note: err.Error()}, nil } if verdict := Fits(LastCueEndMs(cues), offer.ServedMs, offer.DeclaredMs); !verdict.OK { if verdict.Reason != ReasonInserted { // A transcript of some other recording is no use for anything, including // subtraction. return Result{Note: verdict.Why}, nil } master := Transcript{EpisodeID: offer.EpisodeID, Source: SourcePublished, Cues: cues} if err := h.Store.PutMaster(master); err != nil { return Result{}, err } return Result{Note: verdict.Why, Master: len(cues)}, nil } stored := Transcript{ EpisodeID: offer.EpisodeID, Source: SourcePublished, Cues: cues, // Read for advertising now rather than on demand: it is cheap against text, and a // candidate computed at harvest is a candidate that exists offline on the phone with // the words it belongs to. Ads: Spot(cues), } if err := h.Store.Put(stored); err != nil { return Result{}, err } return Result{Stored: true, Cues: len(cues), Ads: len(stored.Ads)}, nil } func (h Harvester) download(ctx context.Context, url string) (string, error) { if err := safefetch.CheckURL(url); err != nil { return "", err } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return "", fmt.Errorf("transcript: %w", err) } req.Header.Set("User-Agent", "listend/1.0") resp, err := h.client().Do(req) if err != nil { return "", fmt.Errorf("transcript: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("transcript: %s returned %s", url, resp.Status) } raw, err := io.ReadAll(io.LimitReader(resp.Body, maxTranscriptBytes)) if err != nil { return "", fmt.Errorf("transcript: %w", err) } return string(raw), nil }