diff options
| author | Peter Stone <thepeterstone@gmail.com> | 2026-10-24 09:01:05 +0000 |
|---|---|---|
| committer | Peter Stone <thepeterstone@gmail.com> | 2026-10-24 09:01:05 +0000 |
| commit | 34e50c04b18313d92e3b3a565965675fd8b50b8a (patch) | |
| tree | ba408851ac65071383d21060f6c133edd09cfb30 /app/src | |
| parent | bc33a9bdee12e41ed8b017c3f751a5dcdf6d1c97 (diff) | |
Send the listener's labels to the server and keep them
The phone has had two kinds of label and nowhere to put them: a mark,
which is a judgement on a break detection found, and a report, which
is the notification button pointing at advertising it missed. They
catch opposite errors, and the signature library needs both.
POST /api/labels now takes them and state.json keeps them. Episodes
are named by feed and guid rather than by either side's own id -- the
phone never sees the server's, and the server's belongs to a mirror
that forgets an episode after eight. The labels sit beside the shows
rather than on them for the same reason: the corpus has to outlive the
audio it was given on.
A report's end is sent as guessed or measured. It is only measured
once the listener came back and trimmed it; otherwise it is wherever
the jump landed, and a signature cut to that would carry programme.
The phone re-sends everything each sweep and keeps no record of what
arrived. The server replaces rather than accumulating, so a re-send is
free, there is no sync state to get wrong, and a label lost to a crash
comes back on the next refresh. Gated on audioFromServer, like every
other server re-query: a label names a position in a particular file,
and dynamic ad insertion means the publisher's copy is a different one.
Accepting a label writes the state file immediately. Everything else
is written at the end of a poll cycle, which for 61 feeds is hours,
and a tap made once should not wait that long to become durable.
Deployed and checked against the live server: a label is accepted, a
re-send leaves one, an unauthenticated POST gets 401, and the label is
on disk within seconds. The probe label used for that check was
removed afterwards.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B
Diffstat (limited to 'app/src')
8 files changed, 474 insertions, 2 deletions
diff --git a/app/src/main/java/org/terst/listen/data/Repositories.kt b/app/src/main/java/org/terst/listen/data/Repositories.kt index 2b972b8..35d96c0 100644 --- a/app/src/main/java/org/terst/listen/data/Repositories.kt +++ b/app/src/main/java/org/terst/listen/data/Repositories.kt @@ -7,8 +7,9 @@ import kotlinx.coroutines.CancellationException import kotlinx.coroutines.flow.first import okhttp3.OkHttpClient import org.terst.listen.server.OkHttpSkipTransport -import org.terst.listen.server.ServerSkips import org.terst.listen.download.WaitingDownloads +import org.terst.listen.server.LabelPusher +import org.terst.listen.server.ServerSkips import org.terst.listen.server.SkipRefresher import org.terst.listen.server.SkipServer import org.terst.listen.server.SkipServerConfig @@ -198,7 +199,11 @@ class AppGraph(private val context: Context) { * known to be the server's own copy of the audio. See [SkipRefresher]'s own documentation * for why that second condition is load-bearing, not an optimisation. */ - val skipRefresher: SkipRefresher by lazy { SkipRefresher(episodes, serverSkips) } + val labelPusher: LabelPusher by lazy { + LabelPusher(episodes, subscriptions, breakVerdicts, adReports) { body -> skipServer.pushLabels(body) } + } + + val skipRefresher: SkipRefresher by lazy { SkipRefresher(episodes, serverSkips, labelPusher) } val downloader: EpisodeDownloader by lazy { EpisodeDownloader( diff --git a/app/src/main/java/org/terst/listen/server/LabelPusher.kt b/app/src/main/java/org/terst/listen/server/LabelPusher.kt new file mode 100644 index 0000000..2fde694 --- /dev/null +++ b/app/src/main/java/org/terst/listen/server/LabelPusher.kt @@ -0,0 +1,55 @@ +package org.terst.listen.server + +import org.terst.listen.data.AdReportDao +import org.terst.listen.data.BreakVerdictDao +import org.terst.listen.data.EpisodeDao +import org.terst.listen.data.SubscriptionDao + +/** Where a batch of labels goes. Returns whether the server took them. */ +fun interface LabelSink { + suspend fun send(body: String): Boolean +} + +/** + * Sends the listener's own labels to the server. + * + * Everything he has, every sweep. There is no record of what arrived last time and deliberately + * none: the server replaces a statement about the same stretch rather than collecting duplicates, + * so re-sending is free and has no state to get wrong. A handful of taps is not worth a sync + * protocol. + * + * Gated on [org.terst.listen.data.EpisodeEntity.audioFromServer], the same invariant the refresh + * sweep uses and for the same reason: a label names a position in a particular file, and a show + * using dynamic ad insertion serves a different file on every request. A position measured + * against the publisher's copy points at the wrong audio in the server's. + */ +class LabelPusher( + private val episodes: EpisodeDao, + private val subscriptions: SubscriptionDao, + private val verdicts: BreakVerdictDao, + private val reports: AdReportDao, + private val sink: LabelSink, +) { + + /** Sends everything there is to send, and returns how many labels went. */ + suspend fun pushAll(): Int { + val labels = gather() + val body = OutgoingLabels.body(labels) ?: return 0 + return if (sink.send(body)) labels.size else 0 + } + + private suspend fun gather(): List<OutgoingLabel> { + val feedUrls = mutableMapOf<String, String?>() + val out = mutableListOf<OutgoingLabel>() + for (episode in episodes.downloadedFromServer()) { + // An episode whose show has been unsubscribed has no feed to name it by, and a + // label the server cannot tie to an episode is not worth sending. + val feedUrl = feedUrls.getOrPut(episode.subscriptionId) { + subscriptions.getById(episode.subscriptionId)?.feedUrl + } ?: continue + out += OutgoingLabels.fromVerdicts(feedUrl, episode.guid, verdicts.getForEpisode(episode.id)) + out += OutgoingLabels.fromReports(feedUrl, episode.guid, reports.getForEpisode(episode.id)) + } + return out + } +} diff --git a/app/src/main/java/org/terst/listen/server/OutgoingLabels.kt b/app/src/main/java/org/terst/listen/server/OutgoingLabels.kt new file mode 100644 index 0000000..7ae6540 --- /dev/null +++ b/app/src/main/java/org/terst/listen/server/OutgoingLabels.kt @@ -0,0 +1,102 @@ +package org.terst.listen.server + +import kotlinx.serialization.json.JsonPrimitive +import org.terst.listen.data.AdReportEntity +import org.terst.listen.data.BreakVerdictEntity +import org.terst.listen.playback.BreakVerdicts + +/** One statement about a stretch of audio, in the shape the server stores. */ +data class OutgoingLabel( + val feedUrl: String, + val guid: String, + val startMs: Long, + val endMs: Long, + val verdict: String, + val source: String, + val endMeasured: Boolean, + val givenAtUnix: Long, +) + +/** + * Turns what the listener said into what the server keeps. + * + * Two tables feed this and they catch opposite errors. A *mark* is a judgement on a break + * detection found, from tapping it on the episode screen -- it catches the detector claiming + * something that is not advertising. A *report* is the notification button, pointing at + * advertising detection missed entirely, which is the larger problem: the scan accounts for + * roughly 38% of what the feeds imply is there. + * + * Identified by feed and guid, not by the app's own episode id, because that id means nothing + * to the server -- and not by the server's either, which the app never sees. + */ +object OutgoingLabels { + + const val AD = "ad" + const val PROGRAMME = "programme" + const val FROM_MARK = "mark" + const val FROM_REPORT = "report" + + /** Judgements on breaks the detector found. */ + fun fromVerdicts(feedUrl: String, guid: String, verdicts: List<BreakVerdictEntity>): List<OutgoingLabel> = + verdicts.mapNotNull { row -> + // A verdict this build does not recognise is dropped rather than guessed at. There + // is no honest translation of a value whose meaning this app does not know. + val verdict = when (row.verdict) { + BreakVerdicts.VALID -> AD + BreakVerdicts.INVALID -> PROGRAMME + else -> return@mapNotNull null + } + OutgoingLabel( + feedUrl = feedUrl, + guid = guid, + startMs = row.startMs, + endMs = row.endMs, + verdict = verdict, + source = FROM_MARK, + // A mark is about a range the detector drew, so both its edges are as measured + // as anything here gets. + endMeasured = true, + givenAtUnix = row.decidedAtMs / MILLIS_PER_SECOND, + ) + } + + /** Advertising the listener pointed at himself. */ + fun fromReports(feedUrl: String, guid: String, reports: List<AdReportEntity>): List<OutgoingLabel> = + reports.map { row -> + OutgoingLabel( + feedUrl = feedUrl, + guid = guid, + startMs = row.startMs, + endMs = row.endMs, + verdict = AD, + source = FROM_REPORT, + // Only true once he came back and trimmed it. Otherwise the end is wherever the + // jump happened to land, and a signature cut to that would carry programme. + endMeasured = row.endCorrected, + givenAtUnix = row.reportedAtMs / MILLIS_PER_SECOND, + ) + } + + /** The request body, or null when there is nothing worth a round trip. */ + fun body(labels: List<OutgoingLabel>): String? { + if (labels.isEmpty()) return null + return buildString { + append("{\"labels\":[") + labels.forEachIndexed { index, l -> + if (index > 0) append(",") + append("{\"feedUrl\":").append(JsonPrimitive(l.feedUrl)) + append(",\"guid\":").append(JsonPrimitive(l.guid)) + append(",\"startMs\":").append(l.startMs) + append(",\"endMs\":").append(l.endMs) + append(",\"verdict\":").append(JsonPrimitive(l.verdict)) + append(",\"source\":").append(JsonPrimitive(l.source)) + append(",\"endMeasured\":").append(l.endMeasured) + append(",\"givenAtUnix\":").append(l.givenAtUnix) + append("}") + } + append("]}") + } + } + + private const val MILLIS_PER_SECOND = 1000L +} diff --git a/app/src/main/java/org/terst/listen/server/SkipRefresher.kt b/app/src/main/java/org/terst/listen/server/SkipRefresher.kt index 12863f7..620f44a 100644 --- a/app/src/main/java/org/terst/listen/server/SkipRefresher.kt +++ b/app/src/main/java/org/terst/listen/server/SkipRefresher.kt @@ -38,6 +38,11 @@ private const val CONCURRENT_QUERIES = 6 class SkipRefresher( private val episodes: EpisodeDao, private val serverAudio: ServerAudio, + /** + * Sends the listener's own labels on the same sweep. Optional so a caller that only wants + * the re-query -- every existing test -- does not have to build one. + */ + private val labels: LabelPusher? = null, ) { /** @@ -76,5 +81,18 @@ class SkipRefresher( } } }.awaitAll() + + // After the queries, not before: the sweep's job is to pick up the server's improved + // answers, and pushing first would only delay that behind a round trip that nothing + // waits on. A failed push is not a failed refresh -- the labels are still on the phone + // and go again next sweep, which is the whole reason this re-sends everything. + @Suppress("TooGenericExceptionCaught") + try { + labels?.pushAll() + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + android.util.Log.w("SkipRefresher", "label push failed", e) + } } } diff --git a/app/src/main/java/org/terst/listen/server/SkipServer.kt b/app/src/main/java/org/terst/listen/server/SkipServer.kt index 4872ab9..69aaef0 100644 --- a/app/src/main/java/org/terst/listen/server/SkipServer.kt +++ b/app/src/main/java/org/terst/listen/server/SkipServer.kt @@ -170,6 +170,18 @@ class SkipServer( } /** + * Sends the listener's labels -- what he said about stretches of audio the detector found, + * and the advertising he pointed at that it missed. + * + * Returns false when there is no server configured, so the caller can tell "nowhere to send + * it" apart from "sent and refused" without inspecting the reply. + */ + suspend fun pushLabels(body: String): Boolean { + val target = config() ?: return false + return observe(target, transport.post(base(target) + "/api/labels", target.token, body)) != null + } + + /** * Asks what to skip in one episode, and where its audio is. * * Null whenever there is no answer -- not configured, unreachable, or a response that is not diff --git a/app/src/test/java/org/terst/listen/data/FakeDaos.kt b/app/src/test/java/org/terst/listen/data/FakeDaos.kt index 8f28795..8288e16 100644 --- a/app/src/test/java/org/terst/listen/data/FakeDaos.kt +++ b/app/src/test/java/org/terst/listen/data/FakeDaos.kt @@ -423,3 +423,21 @@ class FakeBreakVerdictDao : BreakVerdictDao { rows.removeAll { it.episodeId == episodeId && it.startMs == startMs && it.endMs == endMs } } } + +class FakeAdReportDao : AdReportDao { + val rows = mutableListOf<AdReportEntity>() + + override suspend fun getForEpisode(episodeId: String) = + rows.filter { it.episodeId == episodeId }.sortedBy { it.startMs } + + override suspend fun getAll() = rows.sortedBy { it.reportedAtMs } + + override suspend fun upsert(report: AdReportEntity) { + rows.removeAll { it.episodeId == report.episodeId && it.startMs == report.startMs } + rows += report + } + + override suspend fun delete(episodeId: String, startMs: Long) { + rows.removeAll { it.episodeId == episodeId && it.startMs == startMs } + } +} diff --git a/app/src/test/java/org/terst/listen/server/LabelPusherTest.kt b/app/src/test/java/org/terst/listen/server/LabelPusherTest.kt new file mode 100644 index 0000000..c71d8a6 --- /dev/null +++ b/app/src/test/java/org/terst/listen/server/LabelPusherTest.kt @@ -0,0 +1,141 @@ +package org.terst.listen.server + +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.terst.listen.data.AdReportEntity +import org.terst.listen.data.BreakVerdictEntity +import org.terst.listen.data.DownloadState +import org.terst.listen.data.EpisodeEntity +import org.terst.listen.data.FakeAdReportDao +import org.terst.listen.data.FakeBreakVerdictDao +import org.terst.listen.data.FakeEpisodeDao +import org.terst.listen.data.FakeSubscriptionDao +import org.terst.listen.data.SubscriptionEntity +import org.terst.listen.playback.BreakVerdicts + +/** + * Sending the listener's own labels to the server. + * + * Gated on the audio having come from the server, for the same reason every other server + * re-query is: a label describes a position in a specific file, and a show using dynamic ad + * insertion serves a different file every time. A label taken against the publisher's copy + * would point at the wrong audio in the server's. + */ +class LabelPusherTest { + + private val episodes = FakeEpisodeDao() + private val subscriptions = FakeSubscriptionDao() + private val verdicts = FakeBreakVerdictDao() + private val reports = FakeAdReportDao() + private var sent: String? = null + private var sends = 0 + + private val pusher = LabelPusher(episodes, subscriptions, verdicts, reports) { body -> + sends++ + sent = body + true + } + + private suspend fun show(id: String = "s1", feedUrl: String = "https://x/f.xml") { + subscriptions.insert(SubscriptionEntity(id = id, feedUrl = feedUrl, title = "S", addedAtMs = 0)) + } + + private suspend fun episode(id: String, fromServer: Boolean = true, subscriptionId: String = "s1") { + episodes.insertAll( + listOf( + EpisodeEntity( + id = id, subscriptionId = subscriptionId, guid = "guid-$id", title = id, + mediaUrl = "https://cdn/$id.mp3", mediaType = "audio/mpeg", discoveredAtMs = 0, + downloadState = DownloadState.DOWNLOADED, audioFromServer = fromServer, + ) + ) + ) + } + + @Test + fun `sends a mark and a report together`() = runTest { + show() + episode("e1") + verdicts.upsert(BreakVerdictEntity("e1", 60_000, 90_000, BreakVerdicts.VALID, 1_700_000_000_000)) + reports.upsert(AdReportEntity("e1", 600_000, 625_000, 1_700_000_000_000, 20_000)) + + val n = pusher.pushAll() + + assertEquals(2, n) + assertEquals(1, sends) + assertTrue(sent!!, sent!!.contains("\"guid\":\"guid-e1\"")) + assertTrue(sent!!, sent!!.contains("\"source\":\"mark\"")) + assertTrue(sent!!, sent!!.contains("\"source\":\"report\"")) + } + + @Test + fun `an episode whose audio came from the publisher is not labelled`() = runTest { + show() + episode("e1", fromServer = false) + verdicts.upsert(BreakVerdictEntity("e1", 60_000, 90_000, BreakVerdicts.VALID, 1)) + + val n = pusher.pushAll() + + assertEquals("a position in the publisher's file means nothing in the server's", 0, n) + assertEquals(0, sends) + } + + @Test + fun `nothing to say means no request at all`() = runTest { + show() + episode("e1") + + assertEquals(0, pusher.pushAll()) + assertEquals(0, sends) + assertNull(sent) + } + + @Test + fun `an episode whose show has gone is skipped rather than sent without a feed`() = runTest { + episode("e1", subscriptionId = "vanished") + verdicts.upsert(BreakVerdictEntity("e1", 60_000, 90_000, BreakVerdicts.VALID, 1)) + + assertEquals(0, pusher.pushAll()) + assertEquals(0, sends) + } + + @Test + fun `every episode's labels go in one request`() = runTest { + show() + episode("e1") + episode("e2") + verdicts.upsert(BreakVerdictEntity("e1", 1, 2, BreakVerdicts.VALID, 1)) + verdicts.upsert(BreakVerdictEntity("e2", 3, 4, BreakVerdicts.INVALID, 1)) + + assertEquals(2, pusher.pushAll()) + assertEquals("one round trip, not one per episode", 1, sends) + } + + @Test + fun `a refused send is reported as nothing sent`() = runTest { + show() + episode("e1") + verdicts.upsert(BreakVerdictEntity("e1", 1, 2, BreakVerdicts.VALID, 1)) + val refusing = LabelPusher(episodes, subscriptions, verdicts, reports) { false } + + assertEquals(0, refusing.pushAll()) + } + + @Test + fun `each episode is labelled against its own show's feed`() = runTest { + show("s1", "https://one/f.xml") + show("s2", "https://two/f.xml") + episode("e1", subscriptionId = "s1") + episode("e2", subscriptionId = "s2") + verdicts.upsert(BreakVerdictEntity("e1", 1, 2, BreakVerdicts.VALID, 1)) + verdicts.upsert(BreakVerdictEntity("e2", 3, 4, BreakVerdicts.VALID, 1)) + + pusher.pushAll() + + assertTrue(sent!!, sent!!.contains("https://one/f.xml")) + assertTrue(sent!!, sent!!.contains("https://two/f.xml")) + } +} diff --git a/app/src/test/java/org/terst/listen/server/OutgoingLabelsTest.kt b/app/src/test/java/org/terst/listen/server/OutgoingLabelsTest.kt new file mode 100644 index 0000000..7e6bc48 --- /dev/null +++ b/app/src/test/java/org/terst/listen/server/OutgoingLabelsTest.kt @@ -0,0 +1,121 @@ +package org.terst.listen.server + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.terst.listen.data.AdReportEntity +import org.terst.listen.data.BreakVerdictEntity +import org.terst.listen.playback.BreakVerdicts + +/** + * What the phone sends the server about stretches of audio the listener judged or pointed at. + */ +class OutgoingLabelsTest { + + private fun verdict(startMs: Long, endMs: Long, verdict: String) = BreakVerdictEntity( + episodeId = "e1", startMs = startMs, endMs = endMs, + verdict = verdict, decidedAtMs = 1_700_000_000_000, kind = "ad", partners = 5, + ) + + private fun report(startMs: Long, endMs: Long, corrected: Boolean) = AdReportEntity( + episodeId = "e1", startMs = startMs, endMs = endMs, + reportedAtMs = 1_700_000_000_000, correctionWindowMs = 20_000, endCorrected = corrected, + ) + + @Test + fun `marking a break right says it is advertising`() { + val out = OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(60_000, 90_000, BreakVerdicts.VALID))) + + assertEquals(1, out.size) + assertEquals(OutgoingLabels.AD, out[0].verdict) + assertEquals(OutgoingLabels.FROM_MARK, out[0].source) + assertEquals("f", out[0].feedUrl) + assertEquals("g", out[0].guid) + } + + @Test + fun `marking a break wrong says it is the programme`() { + val out = OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(60_000, 90_000, BreakVerdicts.INVALID))) + + assertEquals(OutgoingLabels.PROGRAMME, out[0].verdict) + } + + @Test + fun `a report is advertising the detector missed`() { + val out = OutgoingLabels.fromReports("f", "g", listOf(report(600_000, 625_000, corrected = false))) + + assertEquals(OutgoingLabels.AD, out[0].verdict) + assertEquals(OutgoingLabels.FROM_REPORT, out[0].source) + } + + @Test + fun `an uncorrected end is sent as the guess it is`() { + val out = OutgoingLabels.fromReports("f", "g", listOf(report(600_000, 625_000, corrected = false))) + + assertFalse("where the jump landed is not where the advertising stopped", out[0].endMeasured) + } + + @Test + fun `an end the listener trimmed is sent as measured`() { + val out = OutgoingLabels.fromReports("f", "g", listOf(report(600_000, 622_000, corrected = true))) + + assertTrue(out[0].endMeasured) + } + + @Test + fun `times go out in seconds, which is what the server speaks`() { + val out = OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(60_000, 90_000, BreakVerdicts.VALID))) + + assertEquals(1_700_000_000L, out[0].givenAtUnix) + } + + @Test + fun `a verdict this app does not recognise is not invented into one`() { + val out = OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(60_000, 90_000, "maybe"))) + + assertEquals("an unknown verdict has no honest translation", 0, out.size) + } + + // --- the wire --------------------------------------------------------------------------- + + @Test + fun `an empty push is not worth making`() { + assertEquals(null, OutgoingLabels.body(emptyList())) + } + + @Test + fun `the body carries every field the server stores`() { + val body = OutgoingLabels.body( + OutgoingLabels.fromReports("https://x/f.xml", "g1", listOf(report(600_000, 622_000, true))) + )!! + + assertTrue(body, body.contains("\"feedUrl\":\"https://x/f.xml\"")) + assertTrue(body, body.contains("\"guid\":\"g1\"")) + assertTrue(body, body.contains("\"startMs\":600000")) + assertTrue(body, body.contains("\"endMs\":622000")) + assertTrue(body, body.contains("\"verdict\":\"ad\"")) + assertTrue(body, body.contains("\"source\":\"report\"")) + assertTrue(body, body.contains("\"endMeasured\":true")) + } + + @Test + fun `a feed url with a quote in it cannot break the body`() { + val body = OutgoingLabels.body( + OutgoingLabels.fromVerdicts("https://x/\"odd\".xml", "g", listOf(verdict(1, 2, BreakVerdicts.VALID))) + )!! + + assertTrue(body, body.contains("\\\"odd\\\"")) + } + + @Test + fun `several labels go in one request`() { + val body = OutgoingLabels.body( + OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(1, 2, BreakVerdicts.VALID))) + + OutgoingLabels.fromReports("f", "g", listOf(report(3, 4, false))) + )!! + + assertEquals(2, Regex("\"feedUrl\"").findAll(body).count()) + assertTrue(body.startsWith("{\"labels\":[")) + } +} |
