1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
package main
import (
"context"
"fmt"
"log"
"path/filepath"
"sort"
"time"
"github.com/terst/listen/server/internal/decode"
"github.com/terst/listen/server/internal/signature"
"github.com/terst/listen/server/internal/store"
)
// episodeAudio decodes episodes for fingerprinting, at the rate the signature package wants.
type episodeAudio struct {
decoder decode.Decoder
pathOf func(id string) string
}
func (a episodeAudio) Episode(ctx context.Context, id string) ([]float64, error) {
return a.decode(ctx, id, 0, 0)
}
func (a episodeAudio) Range(ctx context.Context, id string, startMs, durationMs int64) ([]float64, error) {
return a.decode(ctx, id, startMs, durationMs)
}
func (a episodeAudio) decode(ctx context.Context, id string, startMs, durationMs int64) ([]float64, error) {
path := a.pathOf(id)
if path == "" {
return nil, nil
}
clip, err := a.decoder.RangeAt(ctx, path, startMs, durationMs, signature.SampleRate)
if err != nil {
return nil, err
}
return clip.Samples, nil
}
// runShadow learns from a finished detection pass and logs what the library would have cut.
//
// It publishes nothing. The library is new and its proposals have never been checked against
// what a listener actually hears, so for now it watches a few weeks of real batches and says
// what it would have done. The log is the deliverable.
func (w *worker) runShadow(ctx context.Context, show *store.Show, episodes []*store.Episode) {
if w.signatures == nil || len(episodes) == 0 {
return
}
started := time.Now()
batch := signature.ShadowBatch{
// The detection batch is the occasion: these episodes were mirrored within minutes of
// each other and carry whatever campaign was running then, so they are one observation.
Occasion: show.FeedURL + "@" + show.DetectedBatch,
AtUnix: time.Now().Unix(),
}
// What the listener has said, indexed so each episode can pick up its own. Labels outlive
// the episodes they were given on, so most of these belong to batches long since rotated
// out; only the ones matching this batch's guids are of any use here.
labels := map[string][]signature.ShadowLabel{}
for _, l := range w.state.Labels() {
if l.FeedURL != show.FeedURL {
continue
}
labels[l.GUID] = append(labels[l.GUID], signature.ShadowLabel{
StartMs: l.StartMs,
EndMs: l.EndMs,
Advertising: l.Verdict == store.LabelAd,
EndMeasured: l.EndMeasured,
})
}
for _, e := range episodes {
ep := signature.ShadowEpisode{ID: e.ID, FeedURL: show.FeedURL, Labels: labels[e.GUID]}
for _, br := range e.Breaks {
ep.Breaks = append(ep.Breaks, signature.ShadowBreak{
StartMs: br.StartMs,
EndMs: br.EndMs,
Kind: br.Kind,
Partners: br.Partners,
})
}
batch.Episodes = append(batch.Episodes, ep)
}
report, err := signature.NewShadow(w.signatures, audioFor(w.decoder, w.mirror.AudioDir, episodes)).Run(ctx, batch)
if err != nil {
log.Printf("%s: shadow run failed: %v", show.Title, err)
return
}
novel := 0
for _, p := range report.WouldCut {
if p.Novel {
novel++
}
}
log.Printf("%s: shadow -- learned %d, reinforced %d%s, library %d (%d worth matching); "+
"would cut %d, %d of them new%s, took %s",
show.Title, report.Harvested, report.Reinforced, disownedNote(report.Disowned),
w.signatures.Len(), len(w.signatures.Trusted(shadowMinOccasions)),
len(report.WouldCut), novel, skippedNote(report.Skipped),
time.Since(started).Round(time.Second))
// The novel ones are the whole point: what the library would add to what detection already
// publishes. Logged individually so they can be listened to and judged.
sort.Slice(report.WouldCut, func(i, j int) bool { return report.WouldCut[i].StartMs < report.WouldCut[j].StartMs })
for _, p := range report.WouldCut {
if !p.Novel {
continue
}
log.Printf(" would cut %s %s-%s (%s, %d votes, seen on %d occasion(s), %d partners)",
p.EpisodeID, clock(p.StartMs), clock(p.EndMs), p.Entry.Kind, p.Votes,
p.Entry.Confidence(), p.Entry.Partners)
}
if err := w.signatures.Save(w.signaturePath); err != nil {
log.Printf("save signatures: %v", err)
}
}
// disownedNote reports signatures taken back because the listener said that audio is the
// programme. Always logged when it happens: it is the library being told it was wrong.
func disownedNote(n int) string {
if n == 0 {
return ""
}
return fmt.Sprintf(", disowned %d", n)
}
func skippedNote(n int) string {
if n == 0 {
return ""
}
return fmt.Sprintf(", %d unreadable", n)
}
// shadowMinOccasions is how many separate weeks a signature needs before the library would act
// on it, except for advertising detection has already corroborated. See signature.Entry.Trusted.
const shadowMinOccasions = 2
// audioFor supplies decoded episodes to the shadow run.
//
// The paths come from the batch rather than from a lookup: these are the episodes detection
// just finished with, so their files are the ones it read and are certainly still there.
//
// store.Episode.Path is relative to the audio directory, which is what detection joins it with
// before handing it to the decoder. Passing it raw sent ffmpeg looking in the working directory,
// and the first real shadow run died on it.
func audioFor(decoder decode.Decoder, audioDir string, episodes []*store.Episode) signature.Audio {
paths := make(map[string]string, len(episodes))
for _, e := range episodes {
if e.Path != "" {
paths[e.ID] = filepath.Join(audioDir, e.Path)
}
}
return episodeAudio{decoder: decoder, pathOf: func(id string) string { return paths[id] }}
}
|
