1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
package transcript
import (
"context"
"fmt"
"io"
"net/http"
"time"
"github.com/terst/listen/server/internal/safefetch"
)
// maxTranscriptBytes caps a download. The largest in this corpus is Darknet Diaries' 1784-cue
// SRT at well under a megabyte; eight is room for an outlier and a refusal for a mistake.
const maxTranscriptBytes = 8 << 20
// Offer is a publisher's transcript and the file it would have to describe.
type Offer struct {
EpisodeID string
URL string
Type string
// ServedMs is the mirrored file's measured length, DeclaredMs the feed's claim. Their
// difference is the advertising inserted into this copy, which is what decides whether the
// publisher's master is what was mirrored.
ServedMs int64
DeclaredMs int64
}
// Result is what came of an offer: a transcript stored, or a recorded reason it was not.
type Result struct {
Stored bool
Cues int
// Ads is how many stretches of the stored transcript read as advertising. Candidates only.
Ads int
// Master is how many cues were kept as the publisher's ad-free transcript. Non-zero only
// when the transcript was declined for this copy because advertising was inserted into it:
// the words are then the programme without the advertising, which is what [Inserted]
// subtracts from a recognised reading of the served file.
Master int
// Note is why nothing was stored, and is empty when something was. It is a verdict about
// this copy of the episode, so it is recorded rather than retried -- and for an episode
// carrying inserted advertising it is also the signal that this is one recognition has to do.
Note string
}
// Harvester takes publishers' transcripts, when they describe the file the server mirrored.
type Harvester struct {
Store Store
Client *http.Client
}
func (h Harvester) client() *http.Client {
if h.Client != nil {
return h.Client
}
// A transcript URL comes from a publisher's feed, so it names an address this server did
// not choose. safefetch refuses one that resolves to somewhere only this host can reach.
return safefetch.Client(60 * time.Second)
}
// Harvest fetches an offered transcript and stores it if it describes the mirrored file.
//
// An error means try again -- the download failed, so nothing has been decided. A Result with a
// Note means do not try again: the transcript arrived and was judged.
func (h Harvester) Harvest(ctx context.Context, offer Offer) (Result, error) {
if offer.URL == "" {
return Result{}, nil
}
body, err := h.download(ctx, offer.URL)
if err != nil {
return Result{}, err
}
cues, err := Parse(offer.Type, body)
if err != nil {
// The feed said this URL is the transcript and it is not one. Nothing about fetching it
// again changes that, so it is a verdict.
return Result{Note: err.Error()}, nil
}
if verdict := Fits(LastCueEndMs(cues), offer.ServedMs, offer.DeclaredMs); !verdict.OK {
if verdict.Reason != ReasonInserted {
// A transcript of some other recording is no use for anything, including
// subtraction.
return Result{Note: verdict.Why}, nil
}
master := Transcript{EpisodeID: offer.EpisodeID, Source: SourcePublished, Cues: cues}
if err := h.Store.PutMaster(master); err != nil {
return Result{}, err
}
return Result{Note: verdict.Why, Master: len(cues)}, nil
}
stored := Transcript{
EpisodeID: offer.EpisodeID,
Source: SourcePublished,
Cues: cues,
// Read for advertising now rather than on demand: it is cheap against text, and a
// candidate computed at harvest is a candidate that exists offline on the phone with
// the words it belongs to.
Ads: Spot(cues),
}
if err := h.Store.Put(stored); err != nil {
return Result{}, err
}
return Result{Stored: true, Cues: len(cues), Ads: len(stored.Ads)}, nil
}
func (h Harvester) download(ctx context.Context, url string) (string, error) {
if err := safefetch.CheckURL(url); err != nil {
return "", err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return "", fmt.Errorf("transcript: %w", err)
}
req.Header.Set("User-Agent", "listend/1.0")
resp, err := h.client().Do(req)
if err != nil {
return "", fmt.Errorf("transcript: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("transcript: %s returned %s", url, resp.Status)
}
raw, err := io.ReadAll(io.LimitReader(resp.Body, maxTranscriptBytes))
if err != nil {
return "", fmt.Errorf("transcript: %w", err)
}
return string(raw), nil
}
|
