summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.coverage-baseline2
-rw-r--r--app/schemas/org.terst.listen.data.AppDatabase/18.json1026
-rw-r--r--app/src/androidTest/java/org/terst/listen/data/MigrationTest.kt53
-rw-r--r--app/src/main/java/org/terst/listen/data/AppDatabase.kt18
-rw-r--r--app/src/main/java/org/terst/listen/data/Entities.kt10
-rw-r--r--app/src/main/java/org/terst/listen/server/ServerSkips.kt15
-rw-r--r--app/src/main/java/org/terst/listen/server/SkipServer.kt18
-rw-r--r--app/src/main/java/org/terst/listen/ui/TranscriptScreen.kt16
-rw-r--r--app/src/test/java/org/terst/listen/server/ServerSkipsTest.kt63
-rw-r--r--server/cmd/listend/main.go8
-rw-r--r--server/internal/safefetch/live_check_test.go2
-rw-r--r--server/internal/safefetch/safefetch_test.go16
-rw-r--r--server/internal/transcript/ads.go145
-rw-r--r--server/internal/transcript/ads_test.go211
-rw-r--r--server/internal/transcript/harvest.go14
-rw-r--r--server/internal/transcript/harvest_test.go48
-rw-r--r--server/internal/transcript/store.go4
17 files changed, 1646 insertions, 23 deletions
diff --git a/.coverage-baseline b/.coverage-baseline
index 1fbd300..9bfb6a0 100644
--- a/.coverage-baseline
+++ b/.coverage-baseline
@@ -1 +1 @@
-94.45
+94.36
diff --git a/app/schemas/org.terst.listen.data.AppDatabase/18.json b/app/schemas/org.terst.listen.data.AppDatabase/18.json
new file mode 100644
index 0000000..f9f7a2c
--- /dev/null
+++ b/app/schemas/org.terst.listen.data.AppDatabase/18.json
@@ -0,0 +1,1026 @@
+{
+ "formatVersion": 1,
+ "database": {
+ "version": 18,
+ "identityHash": "b832d4c76ecdc3df30db9edc8777c1c9",
+ "entities": [
+ {
+ "tableName": "subscriptions",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `feedUrl` TEXT NOT NULL, `kind` TEXT NOT NULL, `title` TEXT NOT NULL, `description` TEXT, `author` TEXT, `imageUrl` TEXT, `link` TEXT, `addedAtMs` INTEGER NOT NULL, `etag` TEXT, `lastModified` TEXT, `lastFetchedAtMs` INTEGER, `lastError` TEXT, `playbackSpeed` REAL, `skipIntroMs` INTEGER NOT NULL, `skipOutroMs` INTEGER NOT NULL, `dripEnabled` INTEGER NOT NULL DEFAULT 0, `dripIntervalMs` INTEGER NOT NULL DEFAULT 86400000, `dripCount` INTEGER NOT NULL DEFAULT 1, `dripOldestFirst` INTEGER NOT NULL DEFAULT 1, `dripLastReleaseAtMs` INTEGER, `skipAdChapters` INTEGER NOT NULL DEFAULT 0, `midrollPendingEpisodeId` TEXT, `bedtime` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`id`))",
+ "fields": [
+ {
+ "fieldPath": "id",
+ "columnName": "id",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "feedUrl",
+ "columnName": "feedUrl",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "kind",
+ "columnName": "kind",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "title",
+ "columnName": "title",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "description",
+ "columnName": "description",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "author",
+ "columnName": "author",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "imageUrl",
+ "columnName": "imageUrl",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "link",
+ "columnName": "link",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "addedAtMs",
+ "columnName": "addedAtMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "etag",
+ "columnName": "etag",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "lastModified",
+ "columnName": "lastModified",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "lastFetchedAtMs",
+ "columnName": "lastFetchedAtMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "lastError",
+ "columnName": "lastError",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "playbackSpeed",
+ "columnName": "playbackSpeed",
+ "affinity": "REAL"
+ },
+ {
+ "fieldPath": "skipIntroMs",
+ "columnName": "skipIntroMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "skipOutroMs",
+ "columnName": "skipOutroMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "dripEnabled",
+ "columnName": "dripEnabled",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "0"
+ },
+ {
+ "fieldPath": "dripIntervalMs",
+ "columnName": "dripIntervalMs",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "86400000"
+ },
+ {
+ "fieldPath": "dripCount",
+ "columnName": "dripCount",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "1"
+ },
+ {
+ "fieldPath": "dripOldestFirst",
+ "columnName": "dripOldestFirst",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "1"
+ },
+ {
+ "fieldPath": "dripLastReleaseAtMs",
+ "columnName": "dripLastReleaseAtMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "skipAdChapters",
+ "columnName": "skipAdChapters",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "0"
+ },
+ {
+ "fieldPath": "midrollPendingEpisodeId",
+ "columnName": "midrollPendingEpisodeId",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "bedtime",
+ "columnName": "bedtime",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "0"
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "id"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_subscriptions_feedUrl",
+ "unique": true,
+ "columnNames": [
+ "feedUrl"
+ ],
+ "orders": [],
+ "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_subscriptions_feedUrl` ON `${TABLE_NAME}` (`feedUrl`)"
+ }
+ ]
+ },
+ {
+ "tableName": "episodes",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `subscriptionId` TEXT NOT NULL, `guid` TEXT NOT NULL, `title` TEXT NOT NULL, `description` TEXT, `mediaUrl` TEXT NOT NULL, `mediaType` TEXT NOT NULL, `mediaLengthBytes` INTEGER, `durationMs` INTEGER, `publishedAtMs` INTEGER, `seasonNumber` INTEGER, `episodeNumber` INTEGER, `imageUrl` TEXT, `chaptersUrl` TEXT, `discoveredAtMs` INTEGER NOT NULL, `positionMs` INTEGER NOT NULL, `playedAtMs` INTEGER, `actualDurationMs` INTEGER, `downloadState` TEXT NOT NULL, `localPath` TEXT, `downloadedAtMs` INTEGER, `audioFromServer` INTEGER NOT NULL DEFAULT 0, `dripReleasedAtMs` INTEGER, `appliedRuleId` TEXT, `appliedRuleReason` TEXT, `speedOverride` REAL, `midrollCheckedAtMs` INTEGER, `chaptersJson` TEXT, `detectedIntroMs` INTEGER, `detectedOutroMs` INTEGER, PRIMARY KEY(`id`), FOREIGN KEY(`subscriptionId`) REFERENCES `subscriptions`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+ "fields": [
+ {
+ "fieldPath": "id",
+ "columnName": "id",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "subscriptionId",
+ "columnName": "subscriptionId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "guid",
+ "columnName": "guid",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "title",
+ "columnName": "title",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "description",
+ "columnName": "description",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "mediaUrl",
+ "columnName": "mediaUrl",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "mediaType",
+ "columnName": "mediaType",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "mediaLengthBytes",
+ "columnName": "mediaLengthBytes",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "durationMs",
+ "columnName": "durationMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "publishedAtMs",
+ "columnName": "publishedAtMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "seasonNumber",
+ "columnName": "seasonNumber",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "episodeNumber",
+ "columnName": "episodeNumber",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "imageUrl",
+ "columnName": "imageUrl",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "chaptersUrl",
+ "columnName": "chaptersUrl",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "discoveredAtMs",
+ "columnName": "discoveredAtMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "positionMs",
+ "columnName": "positionMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "playedAtMs",
+ "columnName": "playedAtMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "actualDurationMs",
+ "columnName": "actualDurationMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "downloadState",
+ "columnName": "downloadState",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "localPath",
+ "columnName": "localPath",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "downloadedAtMs",
+ "columnName": "downloadedAtMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "audioFromServer",
+ "columnName": "audioFromServer",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "0"
+ },
+ {
+ "fieldPath": "dripReleasedAtMs",
+ "columnName": "dripReleasedAtMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "appliedRuleId",
+ "columnName": "appliedRuleId",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "appliedRuleReason",
+ "columnName": "appliedRuleReason",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "speedOverride",
+ "columnName": "speedOverride",
+ "affinity": "REAL"
+ },
+ {
+ "fieldPath": "midrollCheckedAtMs",
+ "columnName": "midrollCheckedAtMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "chaptersJson",
+ "columnName": "chaptersJson",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "detectedIntroMs",
+ "columnName": "detectedIntroMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "detectedOutroMs",
+ "columnName": "detectedOutroMs",
+ "affinity": "INTEGER"
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "id"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_episodes_subscriptionId",
+ "unique": false,
+ "columnNames": [
+ "subscriptionId"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_episodes_subscriptionId` ON `${TABLE_NAME}` (`subscriptionId`)"
+ },
+ {
+ "name": "index_episodes_subscriptionId_guid",
+ "unique": true,
+ "columnNames": [
+ "subscriptionId",
+ "guid"
+ ],
+ "orders": [],
+ "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_episodes_subscriptionId_guid` ON `${TABLE_NAME}` (`subscriptionId`, `guid`)"
+ },
+ {
+ "name": "index_episodes_publishedAtMs",
+ "unique": false,
+ "columnNames": [
+ "publishedAtMs"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_episodes_publishedAtMs` ON `${TABLE_NAME}` (`publishedAtMs`)"
+ }
+ ],
+ "foreignKeys": [
+ {
+ "table": "subscriptions",
+ "onDelete": "CASCADE",
+ "onUpdate": "NO ACTION",
+ "columns": [
+ "subscriptionId"
+ ],
+ "referencedColumns": [
+ "id"
+ ]
+ }
+ ]
+ },
+ {
+ "tableName": "queue",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`episodeId` TEXT NOT NULL, `position` INTEGER NOT NULL, PRIMARY KEY(`episodeId`), FOREIGN KEY(`episodeId`) REFERENCES `episodes`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+ "fields": [
+ {
+ "fieldPath": "episodeId",
+ "columnName": "episodeId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "position",
+ "columnName": "position",
+ "affinity": "INTEGER",
+ "notNull": true
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "episodeId"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_queue_position",
+ "unique": false,
+ "columnNames": [
+ "position"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_queue_position` ON `${TABLE_NAME}` (`position`)"
+ }
+ ],
+ "foreignKeys": [
+ {
+ "table": "episodes",
+ "onDelete": "CASCADE",
+ "onUpdate": "NO ACTION",
+ "columns": [
+ "episodeId"
+ ],
+ "referencedColumns": [
+ "id"
+ ]
+ }
+ ]
+ },
+ {
+ "tableName": "rules",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `position` INTEGER NOT NULL, `enabled` INTEGER NOT NULL, `subscriptionId` TEXT, `titleContains` TEXT, `titleExcludes` TEXT, `minDurationMs` INTEGER, `maxDurationMs` INTEGER, `maxAgeMs` INTEGER, `seasonNumber` INTEGER, `action` TEXT NOT NULL, `autoDownload` INTEGER NOT NULL, `speed` REAL, PRIMARY KEY(`id`))",
+ "fields": [
+ {
+ "fieldPath": "id",
+ "columnName": "id",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "name",
+ "columnName": "name",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "position",
+ "columnName": "position",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "enabled",
+ "columnName": "enabled",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "subscriptionId",
+ "columnName": "subscriptionId",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "titleContains",
+ "columnName": "titleContains",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "titleExcludes",
+ "columnName": "titleExcludes",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "minDurationMs",
+ "columnName": "minDurationMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "maxDurationMs",
+ "columnName": "maxDurationMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "maxAgeMs",
+ "columnName": "maxAgeMs",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "seasonNumber",
+ "columnName": "seasonNumber",
+ "affinity": "INTEGER"
+ },
+ {
+ "fieldPath": "action",
+ "columnName": "action",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "autoDownload",
+ "columnName": "autoDownload",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "speed",
+ "columnName": "speed",
+ "affinity": "REAL"
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "id"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_rules_position",
+ "unique": false,
+ "columnNames": [
+ "position"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_rules_position` ON `${TABLE_NAME}` (`position`)"
+ }
+ ]
+ },
+ {
+ "tableName": "show_fingerprints",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`subscriptionId` TEXT NOT NULL, `kind` TEXT NOT NULL, `referenceClipPcm` BLOB, `pendingEpisodeId` TEXT, `sampleRate` INTEGER NOT NULL, `confirmedCount` INTEGER NOT NULL, `lastConfirmedEpisodeId` TEXT, `updatedAtMs` INTEGER NOT NULL, PRIMARY KEY(`subscriptionId`, `kind`), FOREIGN KEY(`subscriptionId`) REFERENCES `subscriptions`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+ "fields": [
+ {
+ "fieldPath": "subscriptionId",
+ "columnName": "subscriptionId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "kind",
+ "columnName": "kind",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "referenceClipPcm",
+ "columnName": "referenceClipPcm",
+ "affinity": "BLOB"
+ },
+ {
+ "fieldPath": "pendingEpisodeId",
+ "columnName": "pendingEpisodeId",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "sampleRate",
+ "columnName": "sampleRate",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "confirmedCount",
+ "columnName": "confirmedCount",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "lastConfirmedEpisodeId",
+ "columnName": "lastConfirmedEpisodeId",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "updatedAtMs",
+ "columnName": "updatedAtMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "subscriptionId",
+ "kind"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_show_fingerprints_subscriptionId",
+ "unique": false,
+ "columnNames": [
+ "subscriptionId"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_show_fingerprints_subscriptionId` ON `${TABLE_NAME}` (`subscriptionId`)"
+ }
+ ],
+ "foreignKeys": [
+ {
+ "table": "subscriptions",
+ "onDelete": "CASCADE",
+ "onUpdate": "NO ACTION",
+ "columns": [
+ "subscriptionId"
+ ],
+ "referencedColumns": [
+ "id"
+ ]
+ }
+ ]
+ },
+ {
+ "tableName": "show_midrolls",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`subscriptionId` TEXT NOT NULL, `id` TEXT NOT NULL, `referenceClipPath` TEXT NOT NULL, `sampleRate` INTEGER NOT NULL, `confirmedCount` INTEGER NOT NULL, `lastConfirmedEpisodeId` TEXT, `updatedAtMs` INTEGER NOT NULL, PRIMARY KEY(`subscriptionId`, `id`), FOREIGN KEY(`subscriptionId`) REFERENCES `subscriptions`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+ "fields": [
+ {
+ "fieldPath": "subscriptionId",
+ "columnName": "subscriptionId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "id",
+ "columnName": "id",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "referenceClipPath",
+ "columnName": "referenceClipPath",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "sampleRate",
+ "columnName": "sampleRate",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "confirmedCount",
+ "columnName": "confirmedCount",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "lastConfirmedEpisodeId",
+ "columnName": "lastConfirmedEpisodeId",
+ "affinity": "TEXT"
+ },
+ {
+ "fieldPath": "updatedAtMs",
+ "columnName": "updatedAtMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "subscriptionId",
+ "id"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_show_midrolls_subscriptionId",
+ "unique": false,
+ "columnNames": [
+ "subscriptionId"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_show_midrolls_subscriptionId` ON `${TABLE_NAME}` (`subscriptionId`)"
+ }
+ ],
+ "foreignKeys": [
+ {
+ "table": "subscriptions",
+ "onDelete": "CASCADE",
+ "onUpdate": "NO ACTION",
+ "columns": [
+ "subscriptionId"
+ ],
+ "referencedColumns": [
+ "id"
+ ]
+ }
+ ]
+ },
+ {
+ "tableName": "episode_midrolls",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`episodeId` TEXT NOT NULL, `candidateId` TEXT NOT NULL, `startMs` INTEGER NOT NULL, `endMs` INTEGER NOT NULL, `kind` TEXT NOT NULL DEFAULT 'ad', `partners` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`episodeId`, `candidateId`), FOREIGN KEY(`episodeId`) REFERENCES `episodes`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+ "fields": [
+ {
+ "fieldPath": "episodeId",
+ "columnName": "episodeId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "candidateId",
+ "columnName": "candidateId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "startMs",
+ "columnName": "startMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "endMs",
+ "columnName": "endMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "kind",
+ "columnName": "kind",
+ "affinity": "TEXT",
+ "notNull": true,
+ "defaultValue": "'ad'"
+ },
+ {
+ "fieldPath": "partners",
+ "columnName": "partners",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "0"
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "episodeId",
+ "candidateId"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_episode_midrolls_episodeId",
+ "unique": false,
+ "columnNames": [
+ "episodeId"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_episode_midrolls_episodeId` ON `${TABLE_NAME}` (`episodeId`)"
+ }
+ ],
+ "foreignKeys": [
+ {
+ "table": "episodes",
+ "onDelete": "CASCADE",
+ "onUpdate": "NO ACTION",
+ "columns": [
+ "episodeId"
+ ],
+ "referencedColumns": [
+ "id"
+ ]
+ }
+ ]
+ },
+ {
+ "tableName": "break_verdicts",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`episodeId` TEXT NOT NULL, `startMs` INTEGER NOT NULL, `endMs` INTEGER NOT NULL, `verdict` TEXT NOT NULL, `decidedAtMs` INTEGER NOT NULL, `kind` TEXT NOT NULL DEFAULT 'ad', `partners` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`episodeId`, `startMs`, `endMs`))",
+ "fields": [
+ {
+ "fieldPath": "episodeId",
+ "columnName": "episodeId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "startMs",
+ "columnName": "startMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "endMs",
+ "columnName": "endMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "verdict",
+ "columnName": "verdict",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "decidedAtMs",
+ "columnName": "decidedAtMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "kind",
+ "columnName": "kind",
+ "affinity": "TEXT",
+ "notNull": true,
+ "defaultValue": "'ad'"
+ },
+ {
+ "fieldPath": "partners",
+ "columnName": "partners",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "0"
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "episodeId",
+ "startMs",
+ "endMs"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_break_verdicts_episodeId",
+ "unique": false,
+ "columnNames": [
+ "episodeId"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_break_verdicts_episodeId` ON `${TABLE_NAME}` (`episodeId`)"
+ }
+ ]
+ },
+ {
+ "tableName": "ad_reports",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`episodeId` TEXT NOT NULL, `startMs` INTEGER NOT NULL, `endMs` INTEGER NOT NULL, `reportedAtMs` INTEGER NOT NULL, `correctionWindowMs` INTEGER NOT NULL, `endCorrected` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`episodeId`, `startMs`))",
+ "fields": [
+ {
+ "fieldPath": "episodeId",
+ "columnName": "episodeId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "startMs",
+ "columnName": "startMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "endMs",
+ "columnName": "endMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "reportedAtMs",
+ "columnName": "reportedAtMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "correctionWindowMs",
+ "columnName": "correctionWindowMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "endCorrected",
+ "columnName": "endCorrected",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "0"
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "episodeId",
+ "startMs"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_ad_reports_episodeId",
+ "unique": false,
+ "columnNames": [
+ "episodeId"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_ad_reports_episodeId` ON `${TABLE_NAME}` (`episodeId`)"
+ }
+ ]
+ },
+ {
+ "tableName": "episode_transcripts",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`episodeId` TEXT NOT NULL, `source` TEXT NOT NULL, `cues` INTEGER NOT NULL, `note` TEXT NOT NULL, `fetchedAtMs` INTEGER NOT NULL, PRIMARY KEY(`episodeId`), FOREIGN KEY(`episodeId`) REFERENCES `episodes`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+ "fields": [
+ {
+ "fieldPath": "episodeId",
+ "columnName": "episodeId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "source",
+ "columnName": "source",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "cues",
+ "columnName": "cues",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "note",
+ "columnName": "note",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "fetchedAtMs",
+ "columnName": "fetchedAtMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "episodeId"
+ ]
+ },
+ "foreignKeys": [
+ {
+ "table": "episodes",
+ "onDelete": "CASCADE",
+ "onUpdate": "NO ACTION",
+ "columns": [
+ "episodeId"
+ ],
+ "referencedColumns": [
+ "id"
+ ]
+ }
+ ]
+ },
+ {
+ "tableName": "transcript_cues",
+ "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`episodeId` TEXT NOT NULL, `startMs` INTEGER NOT NULL, `endMs` INTEGER NOT NULL, `text` TEXT NOT NULL, `advertising` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`episodeId`, `startMs`), FOREIGN KEY(`episodeId`) REFERENCES `episodes`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+ "fields": [
+ {
+ "fieldPath": "episodeId",
+ "columnName": "episodeId",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "startMs",
+ "columnName": "startMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "endMs",
+ "columnName": "endMs",
+ "affinity": "INTEGER",
+ "notNull": true
+ },
+ {
+ "fieldPath": "text",
+ "columnName": "text",
+ "affinity": "TEXT",
+ "notNull": true
+ },
+ {
+ "fieldPath": "advertising",
+ "columnName": "advertising",
+ "affinity": "INTEGER",
+ "notNull": true,
+ "defaultValue": "0"
+ }
+ ],
+ "primaryKey": {
+ "autoGenerate": false,
+ "columnNames": [
+ "episodeId",
+ "startMs"
+ ]
+ },
+ "indices": [
+ {
+ "name": "index_transcript_cues_episodeId",
+ "unique": false,
+ "columnNames": [
+ "episodeId"
+ ],
+ "orders": [],
+ "createSql": "CREATE INDEX IF NOT EXISTS `index_transcript_cues_episodeId` ON `${TABLE_NAME}` (`episodeId`)"
+ }
+ ],
+ "foreignKeys": [
+ {
+ "table": "episodes",
+ "onDelete": "CASCADE",
+ "onUpdate": "NO ACTION",
+ "columns": [
+ "episodeId"
+ ],
+ "referencedColumns": [
+ "id"
+ ]
+ }
+ ]
+ }
+ ],
+ "setupQueries": [
+ "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
+ "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'b832d4c76ecdc3df30db9edc8777c1c9')"
+ ]
+ }
+} \ No newline at end of file
diff --git a/app/src/androidTest/java/org/terst/listen/data/MigrationTest.kt b/app/src/androidTest/java/org/terst/listen/data/MigrationTest.kt
index 52f07eb..49d3406 100644
--- a/app/src/androidTest/java/org/terst/listen/data/MigrationTest.kt
+++ b/app/src/androidTest/java/org/terst/listen/data/MigrationTest.kt
@@ -803,4 +803,57 @@ class MigrationTest {
assertEquals("the transcript row outlived its episode", 0, it.getInt(0))
}
}
+
+ /**
+ * Cues gain a mark saying the server read them as advertising, and existing ones default to
+ * not. A mark on the words, never a range to skip -- nothing in the playback path reads it.
+ */
+ @Test
+ fun migrate17To18MarksAdvertisingWithoutDisturbingExistingCues() {
+ helper.createDatabase(name, 17).apply {
+ execSQL(
+ """INSERT INTO subscriptions
+ (id, feedUrl, kind, title, addedAtMs, skipIntroMs, skipOutroMs, dripEnabled,
+ dripIntervalMs, dripCount, dripOldestFirst, skipAdChapters, bedtime)
+ VALUES ('s1', 'https://example.com/feed.xml', 'PODCAST', 'A Show', 1000, 0, 0,
+ 0, 86400000, 1, 1, 1, 0)"""
+ )
+ execSQL(
+ """INSERT INTO episodes
+ (id, subscriptionId, guid, title, mediaUrl, mediaType, discoveredAtMs,
+ positionMs, downloadState)
+ VALUES ('e1', 's1', 'g1', 'One', 'https://cdn/1.mp3', 'audio/mpeg', 1000,
+ 161803, 'NONE')"""
+ )
+ execSQL(
+ "INSERT INTO episode_transcripts (episodeId, source, cues, note, fetchedAtMs) " +
+ "VALUES ('e1', 'published', 1, '', 99)"
+ )
+ execSQL(
+ "INSERT INTO transcript_cues (episodeId, startMs, endMs, text) " +
+ "VALUES ('e1', 0, 3120, 'Welcome back.')"
+ )
+ close()
+ }
+
+ val db = helper.runMigrationsAndValidate(name, 18, true, AppDatabase.MIGRATION_17_18)
+
+ db.query("SELECT text, advertising FROM transcript_cues WHERE episodeId = 'e1'").use {
+ it.moveToFirst()
+ assertEquals("Welcome back.", it.getString(0))
+ assertEquals("a line nobody judged is not an advertisement", 0, it.getInt(1))
+ }
+ db.execSQL(
+ "INSERT INTO transcript_cues (episodeId, startMs, endMs, text, advertising) " +
+ "VALUES ('e1', 4000, 9000, 'Our sponsor this week.', 1)"
+ )
+ db.query("SELECT advertising FROM transcript_cues WHERE startMs = 4000").use {
+ it.moveToFirst()
+ assertEquals(1, it.getInt(0))
+ }
+ db.query("SELECT positionMs FROM episodes WHERE id = 'e1'").use {
+ it.moveToFirst()
+ assertEquals("the crown jewel must survive", 161803L, it.getLong(0))
+ }
+ }
}
diff --git a/app/src/main/java/org/terst/listen/data/AppDatabase.kt b/app/src/main/java/org/terst/listen/data/AppDatabase.kt
index 96a25b9..5b9219d 100644
--- a/app/src/main/java/org/terst/listen/data/AppDatabase.kt
+++ b/app/src/main/java/org/terst/listen/data/AppDatabase.kt
@@ -34,7 +34,7 @@ class Converters {
EpisodeTranscriptEntity::class,
TranscriptCueEntity::class,
],
- version = 17,
+ version = 18,
exportSchema = true,
)
@TypeConverters(Converters::class)
@@ -469,6 +469,21 @@ abstract class AppDatabase : RoomDatabase() {
}
}
+ /**
+ * A cue gains a mark saying the server read it as advertising.
+ *
+ * A mark on the words, not a range to skip. Existing cues default to not: a line
+ * nobody has judged is not an advertisement, which is the same three-state care the
+ * transcript verdict itself takes.
+ */
+ val MIGRATION_17_18 = object : Migration(17, 18) {
+ override fun migrate(db: SupportSQLiteDatabase) {
+ db.execSQL(
+ "ALTER TABLE `transcript_cues` ADD COLUMN `advertising` INTEGER NOT NULL DEFAULT 0"
+ )
+ }
+ }
+
fun get(context: Context): AppDatabase = instance ?: synchronized(this) {
instance ?: build(context.applicationContext).also { instance = it }
}
@@ -482,6 +497,7 @@ abstract class AppDatabase : RoomDatabase() {
MIGRATION_6_7, MIGRATION_7_8, MIGRATION_8_9, MIGRATION_9_10,
MIGRATION_10_11, MIGRATION_11_12, MIGRATION_12_13, MIGRATION_13_14,
MIGRATION_14_15, MIGRATION_15_16, MIGRATION_16_17,
+ MIGRATION_17_18,
)
.addCallback(SEED_DEFAULT_FILTERS)
// No fallbackToDestructiveMigration. A listener's playback positions and play
diff --git a/app/src/main/java/org/terst/listen/data/Entities.kt b/app/src/main/java/org/terst/listen/data/Entities.kt
index 2cfd15f..5a81e5d 100644
--- a/app/src/main/java/org/terst/listen/data/Entities.kt
+++ b/app/src/main/java/org/terst/listen/data/Entities.kt
@@ -629,4 +629,14 @@ data class TranscriptCueEntity(
val startMs: Long,
val endMs: Long,
val text: String,
+ /**
+ * Whether the server read this line as part of an advertisement.
+ *
+ * A mark on the words, not a range to skip. It is stored on the cue rather than as its own
+ * table because it is a property of the line as displayed, and the only thing that reads it
+ * is the screen drawing that line. Nothing in the playback path sees it, deliberately: the
+ * detector that produced it has never been checked against anything, and a skip into the
+ * middle of the programme is worse than no skip.
+ */
+ @ColumnInfo(defaultValue = "0") val advertising: Boolean = false,
)
diff --git a/app/src/main/java/org/terst/listen/server/ServerSkips.kt b/app/src/main/java/org/terst/listen/server/ServerSkips.kt
index d7fb4f5..8dcd115 100644
--- a/app/src/main/java/org/terst/listen/server/ServerSkips.kt
+++ b/app/src/main/java/org/terst/listen/server/ServerSkips.kt
@@ -107,12 +107,19 @@ class ServerSkips(
note = "",
fetchedAtMs = System.currentTimeMillis(),
),
- fetched.cues.map {
+ fetched.cues.map { cue ->
TranscriptCueEntity(
episodeId = episode.id,
- startMs = it.startMs,
- endMs = it.endMs,
- text = it.text,
+ startMs = cue.startMs,
+ endMs = cue.endMs,
+ text = cue.text,
+ // Flattened onto the cue here rather than kept as ranges. The only thing
+ // that reads it is the screen drawing that line, and a range the playback
+ // path could reach is a skip waiting to be made out of a guess that has
+ // never been checked against anything.
+ advertising = fetched.ads.any { ad ->
+ cue.startMs >= ad.startMs && cue.endMs <= ad.endMs
+ },
)
},
)
diff --git a/app/src/main/java/org/terst/listen/server/SkipServer.kt b/app/src/main/java/org/terst/listen/server/SkipServer.kt
index 0980aff..a05eaf7 100644
--- a/app/src/main/java/org/terst/listen/server/SkipServer.kt
+++ b/app/src/main/java/org/terst/listen/server/SkipServer.kt
@@ -97,12 +97,29 @@ data class ServerCue(
val text: String = "",
)
+/**
+ * A stretch of an episode whose words read as advertising.
+ *
+ * A candidate, never a skip. It is shown beside the transcript so the reader can see what the
+ * server thinks is an advertisement, and nothing in the playback path reads it -- the same
+ * posture the signature library's shadow run takes, because a detector acting on its own first
+ * guesses makes skips into the middle of the programme.
+ */
+@Serializable
+data class ServerAdCandidate(
+ @SerialName("startMs") val startMs: Long = 0,
+ @SerialName("endMs") val endMs: Long = 0,
+ /** Which kinds of marker qualified it: sponsor, code, url, offer, legal. */
+ val signals: List<String> = emptyList(),
+)
+
/** An episode's words as the server holds them. */
@Serializable
data class ServerTranscript(
val source: String = "",
@SerialName("languageCode") val languageCode: String? = null,
val cues: List<ServerCue> = emptyList(),
+ val ads: List<ServerAdCandidate> = emptyList(),
)
/** What the server knows about one episode. */
@@ -322,6 +339,7 @@ class SkipServer(
// to seek to. Same filtering the breaks get, because the player acts on these
// numbers too -- a tap on a cue seeks to its start.
cues = parsed.cues.filter { it.endMs > it.startMs && it.text.isNotBlank() },
+ ads = parsed.ads.filter { it.endMs > it.startMs },
)
}
diff --git a/app/src/main/java/org/terst/listen/ui/TranscriptScreen.kt b/app/src/main/java/org/terst/listen/ui/TranscriptScreen.kt
index c6d6e52..81c1b84 100644
--- a/app/src/main/java/org/terst/listen/ui/TranscriptScreen.kt
+++ b/app/src/main/java/org/terst/listen/ui/TranscriptScreen.kt
@@ -100,6 +100,9 @@ fun TranscriptScreen(
}
}
+/** Advertising is dimmed, not hidden: a wrong call has to stay visible to be disagreed with. */
+private const val ADVERTISING_ALPHA = 0.45f
+
@Composable
internal fun CueList(
cues: List<TranscriptCueEntity>,
@@ -119,10 +122,15 @@ internal fun CueList(
cue.text,
style = MaterialTheme.typography.bodyMedium,
fontWeight = if (speaking) FontWeight.Bold else FontWeight.Normal,
- color = if (speaking) {
- MaterialTheme.colorScheme.onSurface
- } else {
- MaterialTheme.colorScheme.onSurfaceVariant
+ // Advertising is dimmed rather than hidden or struck through. The reader can
+ // see what the server thinks is an advertisement and disagree with it, which
+ // matters because nothing has ever checked this detector against anything --
+ // and hiding the lines would make a wrong call invisible.
+ color = when {
+ cue.advertising -> MaterialTheme.colorScheme.onSurfaceVariant
+ .copy(alpha = ADVERTISING_ALPHA)
+ speaking -> MaterialTheme.colorScheme.onSurface
+ else -> MaterialTheme.colorScheme.onSurfaceVariant
},
modifier = Modifier
.fillMaxWidth()
diff --git a/app/src/test/java/org/terst/listen/server/ServerSkipsTest.kt b/app/src/test/java/org/terst/listen/server/ServerSkipsTest.kt
index ae6a23b..3b0d671 100644
--- a/app/src/test/java/org/terst/listen/server/ServerSkipsTest.kt
+++ b/app/src/test/java/org/terst/listen/server/ServerSkipsTest.kt
@@ -453,4 +453,67 @@ class ServerSkipsTest {
assertEquals(1, transcripts.cueRows.size)
assertEquals(1, transcripts.rows.size)
}
+
+ /**
+ * Advertising the server read in the words is marked on the lines it covers.
+ *
+ * Flattened onto the cue rather than stored as ranges, deliberately. The only thing that
+ * reads it is the screen drawing that line; a range the playback path could reach would be
+ * a skip made out of a guess nothing has ever checked.
+ */
+ @Test
+ fun `marks the lines the server read as advertising`() = runTest {
+ val transcripts = FakeTranscriptDao()
+ val transport = StubTransport(
+ null,
+ alsoServing = mapOf("api/transcript" to """
+ {"source":"published","cues":[
+ {"startMs":0,"endMs":3000,"text":"Back in a moment."},
+ {"startMs":10000,"endMs":15000,"text":"Sponsored by a mattress company."},
+ {"startMs":15000,"endMs":26000,"text":"Use promo code LISTEN for 20% off."},
+ {"startMs":30000,"endMs":34000,"text":"Right, where were we?"}],
+ "ads":[{"startMs":10000,"endMs":26000,"signals":["sponsor","code"]}]}
+ """.trimIndent()),
+ )
+
+ subject(transport, Fakes(transcripts = transcripts)).record(
+ episode,
+ ServerEpisode(
+ ready = true, audioUrl = "x", durationMs = null, introMs = null, outroMs = null,
+ detected = true,
+ transcript = ServerTranscriptNote(
+ source = "published", cues = 4,
+ url = "https://listen.terst.org/api/transcript?episode=abc",
+ ),
+ ),
+ )
+
+ val marked = transcripts.cueRows.filter { it.advertising }.map { it.startMs }
+ assertEquals(listOf(10_000L, 15_000L), marked.sorted())
+ }
+
+ /** No candidates means no marks, rather than everything or nothing. */
+ @Test
+ fun `leaves every line unmarked when the server found no advertising`() = runTest {
+ val transcripts = FakeTranscriptDao()
+ val transport = StubTransport(
+ null,
+ alsoServing = mapOf("api/transcript" to
+ """{"source":"asr","cues":[{"startMs":0,"endMs":3000,"text":"Only this."}]}"""),
+ )
+
+ subject(transport, Fakes(transcripts = transcripts)).record(
+ episode,
+ ServerEpisode(
+ ready = true, audioUrl = "x", durationMs = null, introMs = null, outroMs = null,
+ detected = true,
+ transcript = ServerTranscriptNote(
+ source = "asr", cues = 1,
+ url = "https://listen.terst.org/api/transcript?episode=abc",
+ ),
+ ),
+ )
+
+ assertEquals(emptyList<Long>(), transcripts.cueRows.filter { it.advertising }.map { it.startMs })
+ }
}
diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go
index 0e787e4..495ed03 100644
--- a/server/cmd/listend/main.go
+++ b/server/cmd/listend/main.go
@@ -345,7 +345,7 @@ func (w *worker) pollOne(ctx context.Context, feedURL string) {
// every poll would be work done for ever to reach the same answer -- and the refusal is itself
// the record of which episodes speech recognition would have to do.
func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candidates []*store.Episode) {
- taken, declined := 0, 0
+ taken, declined, spotted := 0, 0, 0
for _, e := range candidates {
if ctx.Err() != nil {
return
@@ -372,6 +372,7 @@ func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candi
e.TranscriptAtUnix = time.Now().Unix()
e.TranscriptNote = ""
taken++
+ spotted += result.Ads
case result.Note != "":
e.TranscriptNote = result.Note
e.TranscriptAtUnix = time.Now().Unix()
@@ -379,7 +380,10 @@ func (w *worker) harvestTranscripts(ctx context.Context, show *store.Show, candi
}
}
if taken > 0 || declined > 0 {
- log.Printf("%s: transcripts -- took %d, declined %d", show.Title, taken, declined)
+ // The ad count is reported, not acted on. These are candidates read out of the words,
+ // and nothing publishes them as ranges to skip -- see transcript.Spot.
+ log.Printf("%s: transcripts -- took %d, declined %d, %d ad candidate(s) in the words",
+ show.Title, taken, declined, spotted)
}
}
diff --git a/server/internal/safefetch/live_check_test.go b/server/internal/safefetch/live_check_test.go
index a10d162..cc62c43 100644
--- a/server/internal/safefetch/live_check_test.go
+++ b/server/internal/safefetch/live_check_test.go
@@ -20,7 +20,7 @@ func TestTheLiveSubscriptionsAreAllReachable(t *testing.T) {
Feeds []string `json:"feeds"`
Shows map[string]struct {
Episodes []struct {
- EnclosureURL string `json:"enclosureUrl"`
+ EnclosureURL string `json:"enclosureUrl"`
} `json:"episodes"`
} `json:"shows"`
}
diff --git a/server/internal/safefetch/safefetch_test.go b/server/internal/safefetch/safefetch_test.go
index bcb7e20..ec3c84d 100644
--- a/server/internal/safefetch/safefetch_test.go
+++ b/server/internal/safefetch/safefetch_test.go
@@ -32,14 +32,14 @@ func TestLoopbackAndPrivateAddressesAreRefused(t *testing.T) {
"0.0.0.0", "::",
// Special-purpose ranges that are not "private" but are not the open internet
// either, and several of which route somewhere on a real network.
- "0.1.2.3", // "this network"
- "100.64.0.1", // carrier-grade NAT
- "192.0.0.1", // IETF protocol assignments
- "192.0.2.1", // documentation
- "198.18.0.1", // benchmarking
- "198.51.100.1", // documentation
- "203.0.113.1", // documentation
- "240.0.0.1", // reserved
+ "0.1.2.3", // "this network"
+ "100.64.0.1", // carrier-grade NAT
+ "192.0.0.1", // IETF protocol assignments
+ "192.0.2.1", // documentation
+ "198.18.0.1", // benchmarking
+ "198.51.100.1", // documentation
+ "203.0.113.1", // documentation
+ "240.0.0.1", // reserved
"255.255.255.255", // broadcast
"64:ff9b::7f00:1", // NAT64, carrying 127.0.0.1
"2001:db8::1", // documentation
diff --git a/server/internal/transcript/ads.go b/server/internal/transcript/ads.go
new file mode 100644
index 0000000..f9f9a12
--- /dev/null
+++ b/server/internal/transcript/ads.go
@@ -0,0 +1,145 @@
+package transcript
+
+import (
+ "regexp"
+ "strings"
+)
+
+/*
+Advertising read out of an episode's words.
+
+A second detector, and it finds a class the audio one structurally cannot. The audio scan
+compares episodes mirrored within a minute of each other, so what it finds is the advertising a
+CDN stitched into all of them -- validated against the declared-versus-served length gap. A show
+that inserts nothing has no gap, which by that measure means no advertising, and the host-read
+sponsor block in the middle of it is invisible.
+
+Every published transcript this server can use belongs to exactly such a show, because having no
+inserted advertising is the condition for using one. So the two detectors do not merely fail
+differently; they look at disjoint sets of episodes.
+
+Nothing here is published as a skip. These are candidates, recorded and reported, and the
+listener's labels are what can confirm them -- the same posture the signature library's shadow
+run takes, and for the same reason: a detector that acts on its own first guesses makes
+untrustworthy skips, and a skip into the middle of the programme is worse than no skip.
+*/
+
+// Candidate is a stretch of an episode whose words read as advertising.
+type Candidate struct {
+ StartMs int64 `json:"startMs"`
+ EndMs int64 `json:"endMs"`
+ // Signals are the kinds of marker that qualified it, so the candidate can be weighed
+ // rather than taken on faith -- the same reason a published break carries its partner count.
+ Signals []string `json:"signals"`
+}
+
+const (
+ // bridgeMs is how far apart two markers may be and still belong to one read. Measured on
+ // the Seahawks Rewind block, where "presented by" and "12% off" are 17.7 seconds apart
+ // with the product description between them; 25 gives that room without reaching the
+ // programme on either side.
+ bridgeMs = 25_000
+ // minCandidateMs is the shortest thing worth calling an advertisement. Advertising is sold
+ // in units of roughly fifteen seconds, the same figure the length-gap rule uses, and
+ // anything briefer is a mention rather than a break -- skipping one would clip the show.
+ minCandidateMs = 15_000
+ // minKinds is how many different kinds of marker must appear. Two, because one kind
+ // repeated says one thing: a host saying "sponsor" four times while describing a
+ // sponsorship deal is the programme, which is the same trap the toc=false chapter rule
+ // documents. Real examples of both are in the tests.
+ minKinds = 2
+)
+
+// marker is one kind of advertising language. Kind rather than phrase, so several wordings of
+// the same thing -- "sponsored by", "brought to you by", "presented by" -- count once.
+type marker struct {
+ kind string
+ pattern *regexp.Regexp
+}
+
+var markers = []marker{
+ {"sponsor", regexp.MustCompile(
+ `sponsored by|brought to you by|presented by|our sponsor|today's sponsor|this episode is sponsored`)},
+ {"code", regexp.MustCompile(
+ `promo code|promotional code|use code|discount code|code [a-z0-9]+ at checkout|at checkout`)},
+ // "dot com" rather than ".com" because this is spoken text: a transcript writes the URL
+ // the way it was said. The campaign path has to follow a domain -- a bare "slash" matches
+ // ordinary speech, and the corpus has "a children's movie slash book for adults" to prove
+ // it.
+ {"url", regexp.MustCompile(
+ `dot com|dot co dot uk|\.com[/ ]|(dot com|\.com) slash |visit [a-z0-9]+ dot `)},
+ {"offer", regexp.MustCompile(
+ `\d+% off|\d+ percent off|free trial|first month free|save up to|limited time|` +
+ `for a limited|your first order|sign up today|enroll in`)},
+ {"legal", regexp.MustCompile(
+ `terms and conditions|terms apply|restrictions apply|see site for details|while supplies last`)},
+}
+
+// Spot reads an episode's words and reports the stretches that sound like advertising.
+//
+// Density rather than any single phrase. One marker is a mention -- a show is routinely
+// "brought to you by" somebody in a single line of play-by-play, and a guest describing a
+// sponsorship deal says "sponsor" repeatedly -- and both of those are the programme. A real
+// read stacks different kinds of marker within a few seconds: who it is for, where to go, what
+// the offer is, and what the terms are.
+func Spot(cues []Cue) []Candidate {
+ type hit struct {
+ cue Cue
+ kinds []string
+ }
+ var hits []hit
+ for _, cue := range cues {
+ if kinds := kindsIn(cue.Text); len(kinds) > 0 {
+ hits = append(hits, hit{cue: cue, kinds: kinds})
+ }
+ }
+
+ var found []Candidate
+ for start := 0; start < len(hits); {
+ end := start
+ // Grow the run while the next marker is near enough to belong to the same read.
+ for end+1 < len(hits) && hits[end+1].cue.StartMs-hits[end].cue.EndMs <= bridgeMs {
+ end++
+ }
+
+ kinds := map[string]bool{}
+ for _, h := range hits[start : end+1] {
+ for _, k := range h.kinds {
+ kinds[k] = true
+ }
+ }
+ candidate := Candidate{
+ StartMs: hits[start].cue.StartMs,
+ EndMs: hits[end].cue.EndMs,
+ Signals: sortedKinds(kinds),
+ }
+ if len(kinds) >= minKinds && candidate.EndMs-candidate.StartMs >= minCandidateMs {
+ found = append(found, candidate)
+ }
+ start = end + 1
+ }
+ return found
+}
+
+func kindsIn(text string) []string {
+ lower := strings.ToLower(text)
+ var kinds []string
+ for _, m := range markers {
+ if m.pattern.MatchString(lower) {
+ kinds = append(kinds, m.kind)
+ }
+ }
+ return kinds
+}
+
+// sortedKinds returns the kinds in the fixed order markers are declared in, so a candidate
+// reads the same way every time it is reported.
+func sortedKinds(present map[string]bool) []string {
+ var out []string
+ for _, m := range markers {
+ if present[m.kind] {
+ out = append(out, m.kind)
+ }
+ }
+ return out
+}
diff --git a/server/internal/transcript/ads_test.go b/server/internal/transcript/ads_test.go
new file mode 100644
index 0000000..5512ff9
--- /dev/null
+++ b/server/internal/transcript/ads_test.go
@@ -0,0 +1,211 @@
+package transcript
+
+import "testing"
+
+/*
+Advertising read out of an episode's words.
+
+A second detector, finding a class the audio one structurally cannot. The audio scan works by
+comparing episodes mirrored within a minute of each other, which finds the advertising a CDN
+stitched into all of them -- and the declared-versus-served length gap validates it. A show that
+inserts nothing has no gap, so by that measure it carries no advertising at all, and the
+host-read sponsor block in the middle of it is invisible. Every published transcript the server
+can use belongs to exactly such a show, because that is the condition for using one.
+
+The cues below are real, from the live corpus harvested 2026-10-28, trimmed.
+*/
+
+// A 25-second host-read block from Seahawks Rewind, which the length gap cannot see because
+// that show inserts nothing: served 4938s against 4938s declared.
+var deltaBlock = []Cue{
+ {2461200, 2463800, "you know, you give away too many turnovers, you get little chance to win."},
+ {2467500, 2470200, "All right, today's game is presented by Delta."},
+ {2470200, 2473400, "12 status is back for its 10th anniversary, rewarding Seahawks"},
+ {2473400, 2476600, "fans all season long. Enroll in 12 status and earn"},
+ {2476600, 2481500, "one SkyMile for every Seahawk passing yard this season. 12"},
+ {2481500, 2485200, "status members also receive priority boarding on Delta flights out"},
+ {2485200, 2488600, "of Seattle-Tacoma International Airport and 12% off the Seahawks Pro"},
+ {2488600, 2492800, "Shop on the 12th of every month. Sign up today, 12status. com."},
+ {2492800, 2495300, "We got B. Walt back from TV. Did you guys"},
+}
+
+func TestFindsAHostReadAdBlock(t *testing.T) {
+ got := Spot(deltaBlock)
+
+ if len(got) != 1 {
+ t.Fatalf("got %d candidates, want 1: %+v", len(got), got)
+ }
+ if got[0].StartMs != 2467500 {
+ t.Errorf("starts at %d, want 2467500 -- the line that opens the read", got[0].StartMs)
+ }
+ if got[0].EndMs != 2492800 {
+ t.Errorf("ends at %d, want 2492800 -- the line that closes it", got[0].EndMs)
+ }
+}
+
+/*
+One mention is not a break.
+
+"The call of the game is brought to you by Beacon Plumbing" is a single line inside play-by-play,
+and the programme carries straight on. Skipping it would cut the game. Real, from the same show.
+*/
+func TestASingleSponsorMentionIsNotABreak(t *testing.T) {
+ cues := []Cue{
+ {3288000, 3290900, "First down."},
+ {3290900, 3292500, "The call of the game is brought to you by"},
+ {3292500, 3296700, "Beacon Plumbing. Stop freaking call Beacon. Calm yourself down, Michael."},
+ {3296700, 3298600, "Let's pause 10 seconds to give our network stations a"},
+ {3298600, 3302400, "chance to identify themselves here on the Columbia Bank Radio Network."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- one tag line is not an ad break", got)
+ }
+}
+
+/*
+A show talking about sponsorship is the programme.
+
+Real, from Wards Way Podcast: a guest describing how a sponsor came forward for his yacht event.
+The same trap the toc=false chapter rule already documents -- a show discussing its own
+sponsorship has a section called "Sponsors" which is the actual content.
+*/
+func TestTalkingAboutASponsorIsNotAnAdvertisement(t *testing.T) {
+ cues := []Cue{
+ {607500, 611700, "We're just gonna run a little conference in a hotel and look from across the water."},
+ {611700, 612100, "Sure."},
+ {612100, 618300, "And he kindly came forward and said, I'd love to get involved, and I'd like to sponsor it."},
+ {618300, 621900, "Why don't you host an event on my super mega yacht?"},
+ {621900, 626200, "To which I said, I'm quite happy to have 150 people on anybody's yacht for a party."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- this is two people talking", got)
+ }
+}
+
+// Two different kinds of marker, not two instances of one: a host saying "sponsor" four times
+// while discussing a sponsorship deal is still the programme.
+func TestOneKindOfMarkerRepeatedIsNotEnough(t *testing.T) {
+ cues := []Cue{
+ {0, 5000, "Our sponsor had a lot to say about that."},
+ {5000, 10000, "The sponsor, as you know, sponsors the whole series."},
+ {10000, 15000, "And that sponsor has been with us a long time."},
+ {15000, 20000, "Anyway, a sponsor is a sponsor."},
+ {20000, 25000, "Let's get back to it."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- one marker repeated says one thing", got)
+ }
+}
+
+// A promo code, a URL and an offer inside half a minute is an advertisement.
+func TestFindsACodeAndOfferRead(t *testing.T) {
+ cues := []Cue{
+ {600000, 604000, "Back in a moment."},
+ {610000, 615000, "This episode is sponsored by a mattress company."},
+ {615000, 620000, "Go to mattress dot com slash listen and use promo code LISTEN"},
+ {620000, 626000, "for 20% off your first order. Terms and conditions apply."},
+ {630000, 634000, "Right, where were we?"},
+ }
+
+ got := Spot(cues)
+ if len(got) != 1 {
+ t.Fatalf("got %d candidates, want 1: %+v", len(got), got)
+ }
+ if got[0].StartMs != 610000 || got[0].EndMs != 626000 {
+ t.Errorf("got %d-%d, want 610000-626000", got[0].StartMs, got[0].EndMs)
+ }
+}
+
+// What was found, so a candidate can be weighed rather than taken on faith -- the same reason a
+// published break carries its partner count.
+func TestACandidateSaysWhatItFound(t *testing.T) {
+ got := Spot(deltaBlock)
+
+ if len(got) != 1 {
+ t.Fatalf("got %d candidates, want 1", len(got))
+ }
+ if len(got[0].Signals) < 2 {
+ t.Errorf("signals = %v, want at least the two kinds that qualified it", got[0].Signals)
+ }
+}
+
+/*
+Two breaks in one episode are two candidates.
+
+They must not be bridged into one. The span between them is the programme, and a single
+candidate covering both would be a skip over the middle of the show.
+*/
+func TestTwoSeparateBreaksStaySeparate(t *testing.T) {
+ cues := []Cue{
+ {60000, 65000, "This episode is sponsored by a mattress company."},
+ {65000, 72000, "They have been making mattresses for forty years and they are very good."},
+ {72000, 78000, "Go to mattress dot com and use promo code LISTEN for 20% off."},
+ {300000, 305000, "The programme carries on for several minutes here."},
+ {900000, 905000, "Today's show is brought to you by a razor company."},
+ {905000, 912000, "Five blades, which is more blades than anyone has ever had."},
+ {912000, 918000, "Visit razors dot com slash listen, terms and conditions apply."},
+ }
+
+ got := Spot(cues)
+ if len(got) != 2 {
+ t.Fatalf("got %d candidates, want 2: %+v", len(got), got)
+ }
+ if got[0].EndMs >= got[1].StartMs {
+ t.Errorf("the two candidates overlap: %+v", got)
+ }
+}
+
+// A blip is not a break. Nothing shorter than an advertisement is worth skipping, and skipping
+// it would clip the programme on either side.
+func TestSomethingTooShortToBeAnAdvertisementIsNotOne(t *testing.T) {
+ cues := []Cue{
+ {0, 2000, "Sponsored by a mattress company, mattress dot com."},
+ {2000, 6000, "Anyway."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- two seconds is not an ad read", got)
+ }
+}
+
+func TestAnEmptyTranscriptHasNoAdvertising(t *testing.T) {
+ if got := Spot(nil); len(got) != 0 {
+ t.Errorf("got %+v", got)
+ }
+}
+
+/*
+"Slash" is an ordinary English word.
+
+Real, from the corpus: "is a children's movie slash book for four or adults". A campaign path is
+read aloud after a domain, so that is what the pattern requires -- a bare slash matches people
+talking.
+*/
+func TestSlashInOrdinarySpeechIsNotAURL(t *testing.T) {
+ cues := []Cue{
+ {0, 6000, "It is a children's movie slash book for adults, really."},
+ {6000, 12000, "Our sponsor had opinions about that, as you can imagine."},
+ {12000, 20000, "Anyway, the sponsor of this whole discussion is beside the point."},
+ }
+
+ if got := Spot(cues); len(got) != 0 {
+ t.Errorf("got %+v, want nothing -- \"movie slash book\" is not a web address", got)
+ }
+}
+
+// The real form, which does follow a domain.
+func TestACampaignPathAfterADomainIsAURL(t *testing.T) {
+ cues := []Cue{
+ {0, 5000, "This week we are brought to you by a mattress company."},
+ {5000, 11000, "They have been making mattresses for forty years and they are very good."},
+ {11000, 18000, "Go to mattress dot com slash listen for your first order."},
+ }
+
+ got := Spot(cues)
+ if len(got) != 1 {
+ t.Fatalf("got %d candidates, want 1: %+v", len(got), got)
+ }
+}
diff --git a/server/internal/transcript/harvest.go b/server/internal/transcript/harvest.go
index 6369ea3..64130fd 100644
--- a/server/internal/transcript/harvest.go
+++ b/server/internal/transcript/harvest.go
@@ -30,6 +30,8 @@ type Offer struct {
type Result struct {
Stored bool
Cues int
+ // Ads is how many stretches of the stored transcript read as advertising. Candidates only.
+ Ads int
// Note is why nothing was stored, and is empty when something was. It is a verdict about
// this copy of the episode, so it is recorded rather than retried -- and for an episode
// carrying inserted advertising it is also the signal that this is one recognition has to do.
@@ -76,11 +78,19 @@ func (h Harvester) Harvest(ctx context.Context, offer Offer) (Result, error) {
return Result{Note: verdict.Why}, nil
}
- stored := Transcript{EpisodeID: offer.EpisodeID, Source: SourcePublished, Cues: cues}
+ stored := Transcript{
+ EpisodeID: offer.EpisodeID,
+ Source: SourcePublished,
+ Cues: cues,
+ // Read for advertising now rather than on demand: it is cheap against text, and a
+ // candidate computed at harvest is a candidate that exists offline on the phone with
+ // the words it belongs to.
+ Ads: Spot(cues),
+ }
if err := h.Store.Put(stored); err != nil {
return Result{}, err
}
- return Result{Stored: true, Cues: len(cues)}, nil
+ return Result{Stored: true, Cues: len(cues), Ads: len(stored.Ads)}, nil
}
func (h Harvester) download(ctx context.Context, url string) (string, error) {
diff --git a/server/internal/transcript/harvest_test.go b/server/internal/transcript/harvest_test.go
index 23a192a..25327ae 100644
--- a/server/internal/transcript/harvest_test.go
+++ b/server/internal/transcript/harvest_test.go
@@ -152,3 +152,51 @@ func TestAnEpisodeOfferingNoTranscriptIsNotFetched(t *testing.T) {
// something only it can reach and hand the result to the phone as an episode's words. A
// httptest server is on loopback by definition, so the tests supply the Client seam.
func loopbackClient() *http.Client { return &http.Client{Timeout: 30 * time.Second} }
+
+/*
+A stored transcript carries what its words say about advertising.
+
+Computed at harvest rather than on demand: it is cheap against text, and a candidate worked out
+here is one that exists on the phone, offline, beside the words it belongs to.
+
+Candidates, never skips. See Spot.
+*/
+func TestAHarvestedTranscriptCarriesItsAdCandidates(t *testing.T) {
+ const withAnAd = `1
+00:10:00,000 --> 00:10:05,000
+This episode is sponsored by a mattress company.
+
+2
+00:10:05,000 --> 00:10:11,000
+They have been making mattresses for forty years and they are good.
+
+3
+00:10:11,000 --> 00:10:18,000
+Go to mattress dot com slash listen and use promo code LISTEN for 20% off.
+
+4
+00:20:00,000 --> 00:20:04,000
+Right, where were we?
+`
+ server := serving(t, 200, withAnAd, "application/srt")
+ store := Store{Dir: t.TempDir()}
+ harvester := Harvester{Store: store, Client: loopbackClient()}
+
+ got, err := harvester.Harvest(context.Background(), Offer{
+ EpisodeID: "ep1", URL: server.URL, Type: "application/srt",
+ ServedMs: 1_217_000, DeclaredMs: 1_216_000,
+ })
+ if err != nil {
+ t.Fatalf("Harvest: %v", err)
+ }
+ if got.Ads != 1 {
+ t.Fatalf("Ads = %d, want 1", got.Ads)
+ }
+ held, err := store.Get("ep1")
+ if err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+ if len(held.Ads) != 1 || held.Ads[0].StartMs != 600_000 {
+ t.Errorf("stored ads = %+v", held.Ads)
+ }
+}
diff --git a/server/internal/transcript/store.go b/server/internal/transcript/store.go
index d7e86a4..8cf9419 100644
--- a/server/internal/transcript/store.go
+++ b/server/internal/transcript/store.go
@@ -28,6 +28,10 @@ type Transcript struct {
// LanguageCode is empty when nobody said. Recognition reports what it detected.
LanguageCode string `json:"languageCode,omitempty"`
Cues []Cue `json:"cues"`
+ // Ads are the stretches whose words read as advertising. Candidates, not skips: they are
+ // stored with the transcript and shown beside it, and nothing publishes them as ranges to
+ // jump over. See Spot for why, and for what the listener's labels are for.
+ Ads []Candidate `json:"ads,omitempty"`
}
// LastEndMs is where the words stop.
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-14 21:35:29 +0000