summaryrefslogtreecommitdiff
path: root/server/internal
diff options
context:
space:
mode:
Diffstat (limited to 'server/internal')
-rw-r--r--server/internal/safefetch/live_check_test.go2
-rw-r--r--server/internal/safefetch/safefetch_test.go16
-rw-r--r--server/internal/transcript/ads.go145
-rw-r--r--server/internal/transcript/ads_test.go211
-rw-r--r--server/internal/transcript/harvest.go14
-rw-r--r--server/internal/transcript/harvest_test.go48
-rw-r--r--server/internal/transcript/store.go4
7 files changed, 429 insertions, 11 deletions
diff --git a/server/internal/safefetch/live_check_test.go b/server/internal/safefetch/live_check_test.go
index a10d162..cc62c43 100644
--- a/server/internal/safefetch/live_check_test.go
+++ b/server/internal/safefetch/live_check_test.go
@@ -20,7 +20,7 @@ func TestTheLiveSubscriptionsAreAllReachable(t *testing.T) {
Feeds []string `json:"feeds"`
Shows map[string]struct {
Episodes []struct {
- EnclosureURL string `json:"enclosureUrl"`
+ EnclosureURL string `json:"enclosureUrl"`
} `json:"episodes"`
} `json:"shows"`
}
diff --git a/server/internal/safefetch/safefetch_test.go b/server/internal/safefetch/safefetch_test.go
index bcb7e20..ec3c84d 100644
--- a/server/internal/safefetch/safefetch_test.go
+++ b/server/internal/safefetch/safefetch_test.go
@@ -32,14 +32,14 @@ func TestLoopbackAndPrivateAddressesAreRefused(t *testing.T) {
"0.0.0.0", "::",
// Special-purpose ranges that are not "private" but are not the open internet
// either, and several of which route somewhere on a real network.
- "0.1.2.3", // "this network"
- "100.64.0.1", // carrier-grade NAT
- "192.0.0.1", // IETF protocol assignments
- "192.0.2.1", // documentation
- "198.18.0.1", // benchmarking
- "198.51.100.1", // documentation
- "203.0.113.1", // documentation
- "240.0.0.1", // reserved
+ "0.1.2.3", // "this network"
+ "100.64.0.1", // carrier-grade NAT
+ "192.0.0.1", // IETF protocol assignments
+ "192.0.2.1", // documentation
+ "198.18.0.1", // benchmarking
+ "198.51.100.1", // documentation
+ "203.0.113.1", // documentation
+ "240.0.0.1", // reserved
"255.255.255.255", // broadcast
"64:ff9b::7f00:1", // NAT64, carrying 127.0.0.1
"2001:db8::1", // documentation
diff --git a/server/internal/transcript/ads.go b/server/internal/transcript/ads.go
new file mode 100644
index 0000000..f9f9a12
--- /dev/null
+++ b/server/internal/transcript/ads.go
@@ -0,0 +1,145 @@
+package transcript
+
+import (
+ "regexp"
+ "strings"
+)
+
+/*
+Advertising read out of an episode's words.
+
+A second detector, and it finds a class the audio one structurally cannot. The audio scan
+compares episodes mirrored within a minute of each other, so what it finds is the advertising a
+CDN stitched into all of them -- validated against the declared-versus-served length gap. A show
+that inserts nothing has no gap, which by that measure means no advertising, and the host-read
+sponsor block in the middle of it is invisible.
+
+Every published transcript this server can use belongs to exactly such a show, because having no
+inserted advertising is the condition for using one. So the two detectors do not merely fail
+differently; they look at disjoint sets of episodes.
+
+Nothing here is published as a skip. These are candidates, recorded and reported, and the
+listener's labels are what can confirm them -- the same posture the signature library's shadow
+run takes, and for the same reason: a detector that acts on its own first guesses makes
+untrustworthy skips, and a skip into the middle of the programme is worse than no skip.
+*/
+
+// Candidate is a stretch of an episode whose words read as advertising.
+type Candidate struct {
+ StartMs int64 `json:"startMs"`
+ EndMs int64 `json:"endMs"`
+ // Signals are the kinds of marker that qualified it, so the candidate can be weighed
+ // rather than taken on faith -- the same reason a published break carries its partner count.
+ Signals []string `json:"signals"`
+}
+
+const (
+ // bridgeMs is how far apart two markers may be and still belong to one read. Measured on
+ // the Seahawks Rewind block, where "presented by" and "12% off" are 17.7 seconds apart
+ // with the product description between them; 25 gives that room without reaching the
+ // programme on either side.
+ bridgeMs = 25_000
+ // minCandidateMs is the shortest thing worth calling an advertisement. Advertising is sold
+ // in units of roughly fifteen seconds, the same figure the length-gap rule uses, and
+ // anything briefer is a mention rather than a break -- skipping one would clip the show.
+ minCandidateMs = 15_000
+ // minKinds is how many different kinds of marker must appear. Two, because one kind
+ // repeated says one thing: a host saying "sponsor" four times while describing a
+ // sponsorship deal is the programme, which is the same trap the toc=false chapter rule
+ // documents. Real examples of both are in the tests.
+ minKinds = 2
+)
+
+// marker is one kind of advertising language. Kind rather than phrase, so several wordings of
+// the same thing -- "sponsored by", "brought to you by", "presented by" -- count once.
+type marker struct {
+ kind string
+ pattern *regexp.Regexp
+}
+
+var markers = []marker{
+ {"sponsor", regexp.MustCompile(
+ `sponsored by|brought to you by|presented by|our sponsor|today's sponsor|this episode is sponsored`)},
+ {"code", regexp.MustCompile(
+ `promo code|promotional code|use code|discount code|code [a-z0-9]+ at checkout|at checkout`)},
+ // "dot com" rather than ".com" because this is spoken text: a transcript writes the URL
+ // the way it was said. The campaign path has to follow a domain -- a bare "slash" matches
+ // ordinary speech, and the corpus has "a children's movie slash book for adults" to prove
+ // it.
+ {"url", regexp.MustCompile(
+ `dot com|dot co dot uk|\.com[/ ]|(dot com|\.com) slash |visit [a-z0-9]+ dot `)},
+ {"offer", regexp.MustCompile(
+ `\d+% off|\d+ percent off|free trial|first month free|save up to|limited time|` +
+ `for a limited|your first order|sign up today|enroll in`)},
+ {"legal", regexp.MustCompile(
+ `terms and conditions|terms apply|restrictions apply|see site for details|while supplies last`)},
+}
+
+// Spot reads an episode's words and reports the stretches that sound like advertising.
+//
+// Density rather than any single phrase. One marker is a mention -- a show is routinely
+// "brought to you by" somebody in a single line of play-by-play, and a guest describing a
+// sponsorship deal says "sponsor" repeatedly -- and both of those are the programme. A real
+// read stacks different kinds of marker within a few seconds: who it is for, where to go, what
+// the offer is, and what the terms are.
+func Spot(cues []Cue) []Candidate {
+ type hit struct {
+ cue Cue
+ kinds []string
+ }
+ var hits []hit
+ for _, cue := range cues {
+ if kinds := kindsIn(cue.Text); len(kinds) > 0 {
+ hits = append(hits, hit{cue: cue, kinds: kinds})
+ }
+ }
+
+ var found []Candidate
+ for start := 0; start < len(hits); {
+ end := start
+ // Grow the run while the next marker is near enough to belong to the same read.
+ for end+1 < len(hits) && hits[end+1].cue.StartMs-hits[end].cue.EndMs <= bridgeMs {
+ end++
+ }
+
+ kinds := map[string]bool{}
+ for _, h := range hits[start : end+1] {
+ for _, k := range h.kinds {
+ kinds[k] = true
+ }
+ }
+ candidate := Candidate{
+ StartMs: hits[start].cue.StartMs,
+ EndMs: hits[end].cue.EndMs,
+ Signals: sortedKinds(kinds),
+ }
+ if len(kinds) >= minKinds && candidate.EndMs-candidate.StartMs >= minCandidateMs {
+ found = append(found, candidate)
+ }
+ start = end + 1
+ }
+ return found
+}
+
+func kindsIn(text string) []string {
+ lower := strings.ToLower(text)
+ var kinds []string
+ for _, m := range markers {
+ if m.pattern.MatchString(lower) {
+ kinds = append(kinds, m.kind)
+ }
+ }
+ return kinds
+}
+
+// sortedKinds returns the kinds in the fixed order markers are declared in, so a candidate
+// reads the same way every time it is reported.
+func sortedKinds(present map[string]bool) []string {
+ var out []string
+ for _, m := range markers {
+ if present[m.kind] {
+ out = append(out, m.kind)
+ }
+ }
+ return out
+}
diff --git a/server/internal/transcript/ads_test.go b/server/internal/transcript/ads_test.go
new file mode 100644
index 0000000..5512ff9
--- /dev/null
+++ b/server/internal/transcript/ads_test.go
@@ -0,0 +1,211 @@
+package transcript
+
+import "testing"
+
+/*
+Advertising read out of an episode's words.
+
+A second detector, finding a class the audio one structurally cannot. The audio scan works by
+comparing episodes mirrored within a minute of each other, which finds the advertising a CDN
+stitched into all of them -- and the declared-versus-served length gap validates it. A show that
+inserts nothing has no gap, so by that measure it carries no advertising at all, and the
+host-read sponsor block in the middle of it is invisible. Every published transcript the server
+can use belongs to exactly such a show, because that is the condition for using one.
+
+The cues below are real, from the live corpus harvested 2026-10-28, trimmed.
+*/
+
+// A 25-second host-read block from Seahawks Rewind, which the length gap cannot see because
+// that show inserts nothing: served 4938s against 4938s declared.
+var deltaBlock = []Cue{
+ {2461200, 2463800, "you know, you give away too many turnovers, you get little chance to win."},
+ {2467500, 2470200, "All right, today's game is presented by Delta."},
+ {2470200, 2473400, "12 status is back for its 10th anniversary, rewarding Seahawks"},
+ {2473400, 2476600, "fans all season long. Enroll in 12 status and earn"},
+ {2476600, 2481500, "one SkyMile for every Seahawk passing yard this season. 12"},
+ {2481500, 2485200, "status members also receive priority boarding on Delta flights out"},
+ {2485200, 2488600, "of Seattle-Tacoma International Airport and 12% off the Seahawks Pro"},
+ {2488600, 2492800, "Shop on the 12th of every month. Sign up today, 12status. com."},
+ {2492800, 2495300, "We got B. Walt back from TV. Did you guys"},
+}
+
+func TestFindsAHostReadAdBlock(t *testing.T) {
+ got := Spot(deltaBlock)
+
+ if len(got) != 1 {
+ t.Fatalf("got %d candidates, want 1: %+v", len(got), got)
+ }
+ if got[0].StartMs != 2467500 {
+ t.Errorf("starts at %d, want 2467500 -- the line that opens the read", got[0].StartMs)
+ }
+ if got[0].EndMs != 2492800 {
+ t.Errorf("ends at %d, want 2492800 -- the line that closes it", got[0].EndMs)
+ }
+}
+
+/*
+One mention is not a break.
+
+"The call of the game is brought to you by Beacon Plumbing" is a single line inside play-by-play,
+and the programme carries straight on. Skipping it would cut the game. Real, from the same show.
+*/
+func TestASingleSponsorMentionIsNotABreak(t *testing.T) {
+ cues := []Cue{
+ {3288000, 3290900, "First down."},
+ {3290900, 3292500, "The call of the game is brought to you by"},
+ {3292500, 3296700, "Beacon Plumbing. Stop freaking call Beacon. Calm yourself down, Michael."},
+ {3296700, 3298600, "Let's pause 10 seconds to give our network stations a"},
+ {3298600, 3302400, "chance to identify themselves here on the Columbia Bank Radio Network."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- one tag line is not an ad break", got)
+ }
+}
+
+/*
+A show talking about sponsorship is the programme.
+
+Real, from Wards Way Podcast: a guest describing how a sponsor came forward for his yacht event.
+The same trap the toc=false chapter rule already documents -- a show discussing its own
+sponsorship has a section called "Sponsors" which is the actual content.
+*/
+func TestTalkingAboutASponsorIsNotAnAdvertisement(t *testing.T) {
+ cues := []Cue{
+ {607500, 611700, "We're just gonna run a little conference in a hotel and look from across the water."},
+ {611700, 612100, "Sure."},
+ {612100, 618300, "And he kindly came forward and said, I'd love to get involved, and I'd like to sponsor it."},
+ {618300, 621900, "Why don't you host an event on my super mega yacht?"},
+ {621900, 626200, "To which I said, I'm quite happy to have 150 people on anybody's yacht for a party."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- this is two people talking", got)
+ }
+}
+
+// Two different kinds of marker, not two instances of one: a host saying "sponsor" four times
+// while discussing a sponsorship deal is still the programme.
+func TestOneKindOfMarkerRepeatedIsNotEnough(t *testing.T) {
+ cues := []Cue{
+ {0, 5000, "Our sponsor had a lot to say about that."},
+ {5000, 10000, "The sponsor, as you know, sponsors the whole series."},
+ {10000, 15000, "And that sponsor has been with us a long time."},
+ {15000, 20000, "Anyway, a sponsor is a sponsor."},
+ {20000, 25000, "Let's get back to it."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- one marker repeated says one thing", got)
+ }
+}
+
+// A promo code, a URL and an offer inside half a minute is an advertisement.
+func TestFindsACodeAndOfferRead(t *testing.T) {
+ cues := []Cue{
+ {600000, 604000, "Back in a moment."},
+ {610000, 615000, "This episode is sponsored by a mattress company."},
+ {615000, 620000, "Go to mattress dot com slash listen and use promo code LISTEN"},
+ {620000, 626000, "for 20% off your first order. Terms and conditions apply."},
+ {630000, 634000, "Right, where were we?"},
+ }
+
+ got := Spot(cues)
+ if len(got) != 1 {
+ t.Fatalf("got %d candidates, want 1: %+v", len(got), got)
+ }
+ if got[0].StartMs != 610000 || got[0].EndMs != 626000 {
+ t.Errorf("got %d-%d, want 610000-626000", got[0].StartMs, got[0].EndMs)
+ }
+}
+
+// What was found, so a candidate can be weighed rather than taken on faith -- the same reason a
+// published break carries its partner count.
+func TestACandidateSaysWhatItFound(t *testing.T) {
+ got := Spot(deltaBlock)
+
+ if len(got) != 1 {
+ t.Fatalf("got %d candidates, want 1", len(got))
+ }
+ if len(got[0].Signals) < 2 {
+ t.Errorf("signals = %v, want at least the two kinds that qualified it", got[0].Signals)
+ }
+}
+
+/*
+Two breaks in one episode are two candidates.
+
+They must not be bridged into one. The span between them is the programme, and a single
+candidate covering both would be a skip over the middle of the show.
+*/
+func TestTwoSeparateBreaksStaySeparate(t *testing.T) {
+ cues := []Cue{
+ {60000, 65000, "This episode is sponsored by a mattress company."},
+ {65000, 72000, "They have been making mattresses for forty years and they are very good."},
+ {72000, 78000, "Go to mattress dot com and use promo code LISTEN for 20% off."},
+ {300000, 305000, "The programme carries on for several minutes here."},
+ {900000, 905000, "Today's show is brought to you by a razor company."},
+ {905000, 912000, "Five blades, which is more blades than anyone has ever had."},
+ {912000, 918000, "Visit razors dot com slash listen, terms and conditions apply."},
+ }
+
+ got := Spot(cues)
+ if len(got) != 2 {
+ t.Fatalf("got %d candidates, want 2: %+v", len(got), got)
+ }
+ if got[0].EndMs >= got[1].StartMs {
+ t.Errorf("the two candidates overlap: %+v", got)
+ }
+}
+
+// A blip is not a break. Nothing shorter than an advertisement is worth skipping, and skipping
+// it would clip the programme on either side.
+func TestSomethingTooShortToBeAnAdvertisementIsNotOne(t *testing.T) {
+ cues := []Cue{
+ {0, 2000, "Sponsored by a mattress company, mattress dot com."},
+ {2000, 6000, "Anyway."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- two seconds is not an ad read", got)
+ }
+}
+
+func TestAnEmptyTranscriptHasNoAdvertising(t *testing.T) {
+ if got := Spot(nil); len(got) != 0 {
+ t.Errorf("got %+v", got)
+ }
+}
+
+/*
+"Slash" is an ordinary English word.
+
+Real, from the corpus: "is a children's movie slash book for four or adults". A campaign path is
+read aloud after a domain, so that is what the pattern requires -- a bare slash matches people
+talking.
+*/
+func TestSlashInOrdinarySpeechIsNotAURL(t *testing.T) {
+ cues := []Cue{
+ {0, 6000, "It is a children's movie slash book for adults, really."},
+ {6000, 12000, "Our sponsor had opinions about that, as you can imagine."},
+ {12000, 20000, "Anyway, the sponsor of this whole discussion is beside the point."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- \"movie slash book\" is not a web address", got)
+ }
+}
+
+// The real form, which does follow a domain.
+func TestACampaignPathAfterADomainIsAURL(t *testing.T) {
+ cues := []Cue{
+ {0, 5000, "This week we are brought to you by a mattress company."},
+ {5000, 11000, "They have been making mattresses for forty years and they are very good."},
+ {11000, 18000, "Go to mattress dot com slash listen for your first order."},
+ }
+
+ got := Spot(cues)
+ if len(got) != 1 {
+ t.Fatalf("got %d candidates, want 1: %+v", len(got), got)
+ }
+}
diff --git a/server/internal/transcript/harvest.go b/server/internal/transcript/harvest.go
index 6369ea3..64130fd 100644
--- a/server/internal/transcript/harvest.go
+++ b/server/internal/transcript/harvest.go
@@ -30,6 +30,8 @@ type Offer struct {
type Result struct {
Stored bool
Cues int
+ // Ads is how many stretches of the stored transcript read as advertising. Candidates only.
+ Ads int
// Note is why nothing was stored, and is empty when something was. It is a verdict about
// this copy of the episode, so it is recorded rather than retried -- and for an episode
// carrying inserted advertising it is also the signal that this is one recognition has to do.
@@ -76,11 +78,19 @@ func (h Harvester) Harvest(ctx context.Context, offer Offer) (Result, error) {
return Result{Note: verdict.Why}, nil
}
- stored := Transcript{EpisodeID: offer.EpisodeID, Source: SourcePublished, Cues: cues}
+ stored := Transcript{
+ EpisodeID: offer.EpisodeID,
+ Source: SourcePublished,
+ Cues: cues,
+ // Read for advertising now rather than on demand: it is cheap against text, and a
+ // candidate computed at harvest is a candidate that exists offline on the phone with
+ // the words it belongs to.
+ Ads: Spot(cues),
+ }
if err := h.Store.Put(stored); err != nil {
return Result{}, err
}
- return Result{Stored: true, Cues: len(cues)}, nil
+ return Result{Stored: true, Cues: len(cues), Ads: len(stored.Ads)}, nil
}
func (h Harvester) download(ctx context.Context, url string) (string, error) {
diff --git a/server/internal/transcript/harvest_test.go b/server/internal/transcript/harvest_test.go
index 23a192a..25327ae 100644
--- a/server/internal/transcript/harvest_test.go
+++ b/server/internal/transcript/harvest_test.go
@@ -152,3 +152,51 @@ func TestAnEpisodeOfferingNoTranscriptIsNotFetched(t *testing.T) {
// something only it can reach and hand the result to the phone as an episode's words. A
// httptest server is on loopback by definition, so the tests supply the Client seam.
func loopbackClient() *http.Client { return &http.Client{Timeout: 30 * time.Second} }
+
+/*
+A stored transcript carries what its words say about advertising.
+
+Computed at harvest rather than on demand: it is cheap against text, and a candidate worked out
+here is one that exists on the phone, offline, beside the words it belongs to.
+
+Candidates, never skips. See Spot.
+*/
+func TestAHarvestedTranscriptCarriesItsAdCandidates(t *testing.T) {
+ const withAnAd = `1
+00:10:00,000 --> 00:10:05,000
+This episode is sponsored by a mattress company.
+
+2
+00:10:05,000 --> 00:10:11,000
+They have been making mattresses for forty years and they are good.
+
+3
+00:10:11,000 --> 00:10:18,000
+Go to mattress dot com slash listen and use promo code LISTEN for 20% off.
+
+4
+00:20:00,000 --> 00:20:04,000
+Right, where were we?
+`
+ server := serving(t, 200, withAnAd, "application/srt")
+ store := Store{Dir: t.TempDir()}
+ harvester := Harvester{Store: store, Client: loopbackClient()}
+
+ got, err := harvester.Harvest(context.Background(), Offer{
+ EpisodeID: "ep1", URL: server.URL, Type: "application/srt",
+ ServedMs: 1_217_000, DeclaredMs: 1_216_000,
+ })
+ if err != nil {
+ t.Fatalf("Harvest: %v", err)
+ }
+ if got.Ads != 1 {
+ t.Fatalf("Ads = %d, want 1", got.Ads)
+ }
+ held, err := store.Get("ep1")
+ if err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+ if len(held.Ads) != 1 || held.Ads[0].StartMs != 600_000 {
+ t.Errorf("stored ads = %+v", held.Ads)
+ }
+}
diff --git a/server/internal/transcript/store.go b/server/internal/transcript/store.go
index d7e86a4..8cf9419 100644
--- a/server/internal/transcript/store.go
+++ b/server/internal/transcript/store.go
@@ -28,6 +28,10 @@ type Transcript struct {
// LanguageCode is empty when nobody said. Recognition reports what it detected.
LanguageCode string `json:"languageCode,omitempty"`
Cues []Cue `json:"cues"`
+ // Ads are the stretches whose words read as advertising. Candidates, not skips: they are
+ // stored with the transcript and shown beside it, and nothing publishes them as ranges to
+ // jump over. See Spot for why, and for what the listener's labels are for.
+ Ads []Candidate `json:"ads,omitempty"`
}
// LastEndMs is where the words stop.
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-14 22:28:29 +0000