diff options
| author | Peter Stone <thepeterstone@gmail.com> | 2026-10-24 09:01:05 +0000 |
|---|---|---|
| committer | Peter Stone <thepeterstone@gmail.com> | 2026-10-24 09:01:05 +0000 |
| commit | 34e50c04b18313d92e3b3a565965675fd8b50b8a (patch) | |
| tree | ba408851ac65071383d21060f6c133edd09cfb30 | |
| parent | bc33a9bdee12e41ed8b017c3f751a5dcdf6d1c97 (diff) | |
Send the listener's labels to the server and keep them
The phone has had two kinds of label and nowhere to put them: a mark,
which is a judgement on a break detection found, and a report, which
is the notification button pointing at advertising it missed. They
catch opposite errors, and the signature library needs both.
POST /api/labels now takes them and state.json keeps them. Episodes
are named by feed and guid rather than by either side's own id -- the
phone never sees the server's, and the server's belongs to a mirror
that forgets an episode after eight. The labels sit beside the shows
rather than on them for the same reason: the corpus has to outlive the
audio it was given on.
A report's end is sent as guessed or measured. It is only measured
once the listener came back and trimmed it; otherwise it is wherever
the jump landed, and a signature cut to that would carry programme.
The phone re-sends everything each sweep and keeps no record of what
arrived. The server replaces rather than accumulating, so a re-send is
free, there is no sync state to get wrong, and a label lost to a crash
comes back on the next refresh. Gated on audioFromServer, like every
other server re-query: a label names a position in a particular file,
and dynamic ad insertion means the publisher's copy is a different one.
Accepting a label writes the state file immediately. Everything else
is written at the end of a poll cycle, which for 61 feeds is hours,
and a tap made once should not wait that long to become durable.
Deployed and checked against the live server: a label is accepted, a
re-send leaves one, an unauthenticated POST gets 401, and the label is
on disk within seconds. The probe label used for that check was
removed afterwards.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B
| -rw-r--r-- | .coverage-baseline | 2 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/listen/data/Repositories.kt | 9 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/listen/server/LabelPusher.kt | 55 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/listen/server/OutgoingLabels.kt | 102 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/listen/server/SkipRefresher.kt | 18 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/listen/server/SkipServer.kt | 12 | ||||
| -rw-r--r-- | app/src/test/java/org/terst/listen/data/FakeDaos.kt | 18 | ||||
| -rw-r--r-- | app/src/test/java/org/terst/listen/server/LabelPusherTest.kt | 141 | ||||
| -rw-r--r-- | app/src/test/java/org/terst/listen/server/OutgoingLabelsTest.kt | 121 | ||||
| -rw-r--r-- | docs/superpowers/plans/2026-10-20-corrections-and-signature-library.md | 34 | ||||
| -rw-r--r-- | server/cmd/listend/main.go | 4 | ||||
| -rw-r--r-- | server/internal/api/api.go | 39 | ||||
| -rw-r--r-- | server/internal/api/labels_test.go | 167 | ||||
| -rw-r--r-- | server/internal/store/labels.go | 109 | ||||
| -rw-r--r-- | server/internal/store/labels_test.go | 135 | ||||
| -rw-r--r-- | server/internal/store/store.go | 11 |
16 files changed, 967 insertions, 10 deletions
diff --git a/.coverage-baseline b/.coverage-baseline index db882fc..1fbd300 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -94.43 +94.45 diff --git a/app/src/main/java/org/terst/listen/data/Repositories.kt b/app/src/main/java/org/terst/listen/data/Repositories.kt index 2b972b8..35d96c0 100644 --- a/app/src/main/java/org/terst/listen/data/Repositories.kt +++ b/app/src/main/java/org/terst/listen/data/Repositories.kt @@ -7,8 +7,9 @@ import kotlinx.coroutines.CancellationException import kotlinx.coroutines.flow.first import okhttp3.OkHttpClient import org.terst.listen.server.OkHttpSkipTransport -import org.terst.listen.server.ServerSkips import org.terst.listen.download.WaitingDownloads +import org.terst.listen.server.LabelPusher +import org.terst.listen.server.ServerSkips import org.terst.listen.server.SkipRefresher import org.terst.listen.server.SkipServer import org.terst.listen.server.SkipServerConfig @@ -198,7 +199,11 @@ class AppGraph(private val context: Context) { * known to be the server's own copy of the audio. See [SkipRefresher]'s own documentation * for why that second condition is load-bearing, not an optimisation. */ - val skipRefresher: SkipRefresher by lazy { SkipRefresher(episodes, serverSkips) } + val labelPusher: LabelPusher by lazy { + LabelPusher(episodes, subscriptions, breakVerdicts, adReports) { body -> skipServer.pushLabels(body) } + } + + val skipRefresher: SkipRefresher by lazy { SkipRefresher(episodes, serverSkips, labelPusher) } val downloader: EpisodeDownloader by lazy { EpisodeDownloader( diff --git a/app/src/main/java/org/terst/listen/server/LabelPusher.kt b/app/src/main/java/org/terst/listen/server/LabelPusher.kt new file mode 100644 index 0000000..2fde694 --- /dev/null +++ b/app/src/main/java/org/terst/listen/server/LabelPusher.kt @@ -0,0 +1,55 @@ +package org.terst.listen.server + +import org.terst.listen.data.AdReportDao +import org.terst.listen.data.BreakVerdictDao +import org.terst.listen.data.EpisodeDao +import org.terst.listen.data.SubscriptionDao + +/** Where a batch of labels goes. Returns whether the server took them. */ +fun interface LabelSink { + suspend fun send(body: String): Boolean +} + +/** + * Sends the listener's own labels to the server. + * + * Everything he has, every sweep. There is no record of what arrived last time and deliberately + * none: the server replaces a statement about the same stretch rather than collecting duplicates, + * so re-sending is free and has no state to get wrong. A handful of taps is not worth a sync + * protocol. + * + * Gated on [org.terst.listen.data.EpisodeEntity.audioFromServer], the same invariant the refresh + * sweep uses and for the same reason: a label names a position in a particular file, and a show + * using dynamic ad insertion serves a different file on every request. A position measured + * against the publisher's copy points at the wrong audio in the server's. + */ +class LabelPusher( + private val episodes: EpisodeDao, + private val subscriptions: SubscriptionDao, + private val verdicts: BreakVerdictDao, + private val reports: AdReportDao, + private val sink: LabelSink, +) { + + /** Sends everything there is to send, and returns how many labels went. */ + suspend fun pushAll(): Int { + val labels = gather() + val body = OutgoingLabels.body(labels) ?: return 0 + return if (sink.send(body)) labels.size else 0 + } + + private suspend fun gather(): List<OutgoingLabel> { + val feedUrls = mutableMapOf<String, String?>() + val out = mutableListOf<OutgoingLabel>() + for (episode in episodes.downloadedFromServer()) { + // An episode whose show has been unsubscribed has no feed to name it by, and a + // label the server cannot tie to an episode is not worth sending. + val feedUrl = feedUrls.getOrPut(episode.subscriptionId) { + subscriptions.getById(episode.subscriptionId)?.feedUrl + } ?: continue + out += OutgoingLabels.fromVerdicts(feedUrl, episode.guid, verdicts.getForEpisode(episode.id)) + out += OutgoingLabels.fromReports(feedUrl, episode.guid, reports.getForEpisode(episode.id)) + } + return out + } +} diff --git a/app/src/main/java/org/terst/listen/server/OutgoingLabels.kt b/app/src/main/java/org/terst/listen/server/OutgoingLabels.kt new file mode 100644 index 0000000..7ae6540 --- /dev/null +++ b/app/src/main/java/org/terst/listen/server/OutgoingLabels.kt @@ -0,0 +1,102 @@ +package org.terst.listen.server + +import kotlinx.serialization.json.JsonPrimitive +import org.terst.listen.data.AdReportEntity +import org.terst.listen.data.BreakVerdictEntity +import org.terst.listen.playback.BreakVerdicts + +/** One statement about a stretch of audio, in the shape the server stores. */ +data class OutgoingLabel( + val feedUrl: String, + val guid: String, + val startMs: Long, + val endMs: Long, + val verdict: String, + val source: String, + val endMeasured: Boolean, + val givenAtUnix: Long, +) + +/** + * Turns what the listener said into what the server keeps. + * + * Two tables feed this and they catch opposite errors. A *mark* is a judgement on a break + * detection found, from tapping it on the episode screen -- it catches the detector claiming + * something that is not advertising. A *report* is the notification button, pointing at + * advertising detection missed entirely, which is the larger problem: the scan accounts for + * roughly 38% of what the feeds imply is there. + * + * Identified by feed and guid, not by the app's own episode id, because that id means nothing + * to the server -- and not by the server's either, which the app never sees. + */ +object OutgoingLabels { + + const val AD = "ad" + const val PROGRAMME = "programme" + const val FROM_MARK = "mark" + const val FROM_REPORT = "report" + + /** Judgements on breaks the detector found. */ + fun fromVerdicts(feedUrl: String, guid: String, verdicts: List<BreakVerdictEntity>): List<OutgoingLabel> = + verdicts.mapNotNull { row -> + // A verdict this build does not recognise is dropped rather than guessed at. There + // is no honest translation of a value whose meaning this app does not know. + val verdict = when (row.verdict) { + BreakVerdicts.VALID -> AD + BreakVerdicts.INVALID -> PROGRAMME + else -> return@mapNotNull null + } + OutgoingLabel( + feedUrl = feedUrl, + guid = guid, + startMs = row.startMs, + endMs = row.endMs, + verdict = verdict, + source = FROM_MARK, + // A mark is about a range the detector drew, so both its edges are as measured + // as anything here gets. + endMeasured = true, + givenAtUnix = row.decidedAtMs / MILLIS_PER_SECOND, + ) + } + + /** Advertising the listener pointed at himself. */ + fun fromReports(feedUrl: String, guid: String, reports: List<AdReportEntity>): List<OutgoingLabel> = + reports.map { row -> + OutgoingLabel( + feedUrl = feedUrl, + guid = guid, + startMs = row.startMs, + endMs = row.endMs, + verdict = AD, + source = FROM_REPORT, + // Only true once he came back and trimmed it. Otherwise the end is wherever the + // jump happened to land, and a signature cut to that would carry programme. + endMeasured = row.endCorrected, + givenAtUnix = row.reportedAtMs / MILLIS_PER_SECOND, + ) + } + + /** The request body, or null when there is nothing worth a round trip. */ + fun body(labels: List<OutgoingLabel>): String? { + if (labels.isEmpty()) return null + return buildString { + append("{\"labels\":[") + labels.forEachIndexed { index, l -> + if (index > 0) append(",") + append("{\"feedUrl\":").append(JsonPrimitive(l.feedUrl)) + append(",\"guid\":").append(JsonPrimitive(l.guid)) + append(",\"startMs\":").append(l.startMs) + append(",\"endMs\":").append(l.endMs) + append(",\"verdict\":").append(JsonPrimitive(l.verdict)) + append(",\"source\":").append(JsonPrimitive(l.source)) + append(",\"endMeasured\":").append(l.endMeasured) + append(",\"givenAtUnix\":").append(l.givenAtUnix) + append("}") + } + append("]}") + } + } + + private const val MILLIS_PER_SECOND = 1000L +} diff --git a/app/src/main/java/org/terst/listen/server/SkipRefresher.kt b/app/src/main/java/org/terst/listen/server/SkipRefresher.kt index 12863f7..620f44a 100644 --- a/app/src/main/java/org/terst/listen/server/SkipRefresher.kt +++ b/app/src/main/java/org/terst/listen/server/SkipRefresher.kt @@ -38,6 +38,11 @@ private const val CONCURRENT_QUERIES = 6 class SkipRefresher( private val episodes: EpisodeDao, private val serverAudio: ServerAudio, + /** + * Sends the listener's own labels on the same sweep. Optional so a caller that only wants + * the re-query -- every existing test -- does not have to build one. + */ + private val labels: LabelPusher? = null, ) { /** @@ -76,5 +81,18 @@ class SkipRefresher( } } }.awaitAll() + + // After the queries, not before: the sweep's job is to pick up the server's improved + // answers, and pushing first would only delay that behind a round trip that nothing + // waits on. A failed push is not a failed refresh -- the labels are still on the phone + // and go again next sweep, which is the whole reason this re-sends everything. + @Suppress("TooGenericExceptionCaught") + try { + labels?.pushAll() + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + android.util.Log.w("SkipRefresher", "label push failed", e) + } } } diff --git a/app/src/main/java/org/terst/listen/server/SkipServer.kt b/app/src/main/java/org/terst/listen/server/SkipServer.kt index 4872ab9..69aaef0 100644 --- a/app/src/main/java/org/terst/listen/server/SkipServer.kt +++ b/app/src/main/java/org/terst/listen/server/SkipServer.kt @@ -170,6 +170,18 @@ class SkipServer( } /** + * Sends the listener's labels -- what he said about stretches of audio the detector found, + * and the advertising he pointed at that it missed. + * + * Returns false when there is no server configured, so the caller can tell "nowhere to send + * it" apart from "sent and refused" without inspecting the reply. + */ + suspend fun pushLabels(body: String): Boolean { + val target = config() ?: return false + return observe(target, transport.post(base(target) + "/api/labels", target.token, body)) != null + } + + /** * Asks what to skip in one episode, and where its audio is. * * Null whenever there is no answer -- not configured, unreachable, or a response that is not diff --git a/app/src/test/java/org/terst/listen/data/FakeDaos.kt b/app/src/test/java/org/terst/listen/data/FakeDaos.kt index 8f28795..8288e16 100644 --- a/app/src/test/java/org/terst/listen/data/FakeDaos.kt +++ b/app/src/test/java/org/terst/listen/data/FakeDaos.kt @@ -423,3 +423,21 @@ class FakeBreakVerdictDao : BreakVerdictDao { rows.removeAll { it.episodeId == episodeId && it.startMs == startMs && it.endMs == endMs } } } + +class FakeAdReportDao : AdReportDao { + val rows = mutableListOf<AdReportEntity>() + + override suspend fun getForEpisode(episodeId: String) = + rows.filter { it.episodeId == episodeId }.sortedBy { it.startMs } + + override suspend fun getAll() = rows.sortedBy { it.reportedAtMs } + + override suspend fun upsert(report: AdReportEntity) { + rows.removeAll { it.episodeId == report.episodeId && it.startMs == report.startMs } + rows += report + } + + override suspend fun delete(episodeId: String, startMs: Long) { + rows.removeAll { it.episodeId == episodeId && it.startMs == startMs } + } +} diff --git a/app/src/test/java/org/terst/listen/server/LabelPusherTest.kt b/app/src/test/java/org/terst/listen/server/LabelPusherTest.kt new file mode 100644 index 0000000..c71d8a6 --- /dev/null +++ b/app/src/test/java/org/terst/listen/server/LabelPusherTest.kt @@ -0,0 +1,141 @@ +package org.terst.listen.server + +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.terst.listen.data.AdReportEntity +import org.terst.listen.data.BreakVerdictEntity +import org.terst.listen.data.DownloadState +import org.terst.listen.data.EpisodeEntity +import org.terst.listen.data.FakeAdReportDao +import org.terst.listen.data.FakeBreakVerdictDao +import org.terst.listen.data.FakeEpisodeDao +import org.terst.listen.data.FakeSubscriptionDao +import org.terst.listen.data.SubscriptionEntity +import org.terst.listen.playback.BreakVerdicts + +/** + * Sending the listener's own labels to the server. + * + * Gated on the audio having come from the server, for the same reason every other server + * re-query is: a label describes a position in a specific file, and a show using dynamic ad + * insertion serves a different file every time. A label taken against the publisher's copy + * would point at the wrong audio in the server's. + */ +class LabelPusherTest { + + private val episodes = FakeEpisodeDao() + private val subscriptions = FakeSubscriptionDao() + private val verdicts = FakeBreakVerdictDao() + private val reports = FakeAdReportDao() + private var sent: String? = null + private var sends = 0 + + private val pusher = LabelPusher(episodes, subscriptions, verdicts, reports) { body -> + sends++ + sent = body + true + } + + private suspend fun show(id: String = "s1", feedUrl: String = "https://x/f.xml") { + subscriptions.insert(SubscriptionEntity(id = id, feedUrl = feedUrl, title = "S", addedAtMs = 0)) + } + + private suspend fun episode(id: String, fromServer: Boolean = true, subscriptionId: String = "s1") { + episodes.insertAll( + listOf( + EpisodeEntity( + id = id, subscriptionId = subscriptionId, guid = "guid-$id", title = id, + mediaUrl = "https://cdn/$id.mp3", mediaType = "audio/mpeg", discoveredAtMs = 0, + downloadState = DownloadState.DOWNLOADED, audioFromServer = fromServer, + ) + ) + ) + } + + @Test + fun `sends a mark and a report together`() = runTest { + show() + episode("e1") + verdicts.upsert(BreakVerdictEntity("e1", 60_000, 90_000, BreakVerdicts.VALID, 1_700_000_000_000)) + reports.upsert(AdReportEntity("e1", 600_000, 625_000, 1_700_000_000_000, 20_000)) + + val n = pusher.pushAll() + + assertEquals(2, n) + assertEquals(1, sends) + assertTrue(sent!!, sent!!.contains("\"guid\":\"guid-e1\"")) + assertTrue(sent!!, sent!!.contains("\"source\":\"mark\"")) + assertTrue(sent!!, sent!!.contains("\"source\":\"report\"")) + } + + @Test + fun `an episode whose audio came from the publisher is not labelled`() = runTest { + show() + episode("e1", fromServer = false) + verdicts.upsert(BreakVerdictEntity("e1", 60_000, 90_000, BreakVerdicts.VALID, 1)) + + val n = pusher.pushAll() + + assertEquals("a position in the publisher's file means nothing in the server's", 0, n) + assertEquals(0, sends) + } + + @Test + fun `nothing to say means no request at all`() = runTest { + show() + episode("e1") + + assertEquals(0, pusher.pushAll()) + assertEquals(0, sends) + assertNull(sent) + } + + @Test + fun `an episode whose show has gone is skipped rather than sent without a feed`() = runTest { + episode("e1", subscriptionId = "vanished") + verdicts.upsert(BreakVerdictEntity("e1", 60_000, 90_000, BreakVerdicts.VALID, 1)) + + assertEquals(0, pusher.pushAll()) + assertEquals(0, sends) + } + + @Test + fun `every episode's labels go in one request`() = runTest { + show() + episode("e1") + episode("e2") + verdicts.upsert(BreakVerdictEntity("e1", 1, 2, BreakVerdicts.VALID, 1)) + verdicts.upsert(BreakVerdictEntity("e2", 3, 4, BreakVerdicts.INVALID, 1)) + + assertEquals(2, pusher.pushAll()) + assertEquals("one round trip, not one per episode", 1, sends) + } + + @Test + fun `a refused send is reported as nothing sent`() = runTest { + show() + episode("e1") + verdicts.upsert(BreakVerdictEntity("e1", 1, 2, BreakVerdicts.VALID, 1)) + val refusing = LabelPusher(episodes, subscriptions, verdicts, reports) { false } + + assertEquals(0, refusing.pushAll()) + } + + @Test + fun `each episode is labelled against its own show's feed`() = runTest { + show("s1", "https://one/f.xml") + show("s2", "https://two/f.xml") + episode("e1", subscriptionId = "s1") + episode("e2", subscriptionId = "s2") + verdicts.upsert(BreakVerdictEntity("e1", 1, 2, BreakVerdicts.VALID, 1)) + verdicts.upsert(BreakVerdictEntity("e2", 3, 4, BreakVerdicts.VALID, 1)) + + pusher.pushAll() + + assertTrue(sent!!, sent!!.contains("https://one/f.xml")) + assertTrue(sent!!, sent!!.contains("https://two/f.xml")) + } +} diff --git a/app/src/test/java/org/terst/listen/server/OutgoingLabelsTest.kt b/app/src/test/java/org/terst/listen/server/OutgoingLabelsTest.kt new file mode 100644 index 0000000..7e6bc48 --- /dev/null +++ b/app/src/test/java/org/terst/listen/server/OutgoingLabelsTest.kt @@ -0,0 +1,121 @@ +package org.terst.listen.server + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.terst.listen.data.AdReportEntity +import org.terst.listen.data.BreakVerdictEntity +import org.terst.listen.playback.BreakVerdicts + +/** + * What the phone sends the server about stretches of audio the listener judged or pointed at. + */ +class OutgoingLabelsTest { + + private fun verdict(startMs: Long, endMs: Long, verdict: String) = BreakVerdictEntity( + episodeId = "e1", startMs = startMs, endMs = endMs, + verdict = verdict, decidedAtMs = 1_700_000_000_000, kind = "ad", partners = 5, + ) + + private fun report(startMs: Long, endMs: Long, corrected: Boolean) = AdReportEntity( + episodeId = "e1", startMs = startMs, endMs = endMs, + reportedAtMs = 1_700_000_000_000, correctionWindowMs = 20_000, endCorrected = corrected, + ) + + @Test + fun `marking a break right says it is advertising`() { + val out = OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(60_000, 90_000, BreakVerdicts.VALID))) + + assertEquals(1, out.size) + assertEquals(OutgoingLabels.AD, out[0].verdict) + assertEquals(OutgoingLabels.FROM_MARK, out[0].source) + assertEquals("f", out[0].feedUrl) + assertEquals("g", out[0].guid) + } + + @Test + fun `marking a break wrong says it is the programme`() { + val out = OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(60_000, 90_000, BreakVerdicts.INVALID))) + + assertEquals(OutgoingLabels.PROGRAMME, out[0].verdict) + } + + @Test + fun `a report is advertising the detector missed`() { + val out = OutgoingLabels.fromReports("f", "g", listOf(report(600_000, 625_000, corrected = false))) + + assertEquals(OutgoingLabels.AD, out[0].verdict) + assertEquals(OutgoingLabels.FROM_REPORT, out[0].source) + } + + @Test + fun `an uncorrected end is sent as the guess it is`() { + val out = OutgoingLabels.fromReports("f", "g", listOf(report(600_000, 625_000, corrected = false))) + + assertFalse("where the jump landed is not where the advertising stopped", out[0].endMeasured) + } + + @Test + fun `an end the listener trimmed is sent as measured`() { + val out = OutgoingLabels.fromReports("f", "g", listOf(report(600_000, 622_000, corrected = true))) + + assertTrue(out[0].endMeasured) + } + + @Test + fun `times go out in seconds, which is what the server speaks`() { + val out = OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(60_000, 90_000, BreakVerdicts.VALID))) + + assertEquals(1_700_000_000L, out[0].givenAtUnix) + } + + @Test + fun `a verdict this app does not recognise is not invented into one`() { + val out = OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(60_000, 90_000, "maybe"))) + + assertEquals("an unknown verdict has no honest translation", 0, out.size) + } + + // --- the wire --------------------------------------------------------------------------- + + @Test + fun `an empty push is not worth making`() { + assertEquals(null, OutgoingLabels.body(emptyList())) + } + + @Test + fun `the body carries every field the server stores`() { + val body = OutgoingLabels.body( + OutgoingLabels.fromReports("https://x/f.xml", "g1", listOf(report(600_000, 622_000, true))) + )!! + + assertTrue(body, body.contains("\"feedUrl\":\"https://x/f.xml\"")) + assertTrue(body, body.contains("\"guid\":\"g1\"")) + assertTrue(body, body.contains("\"startMs\":600000")) + assertTrue(body, body.contains("\"endMs\":622000")) + assertTrue(body, body.contains("\"verdict\":\"ad\"")) + assertTrue(body, body.contains("\"source\":\"report\"")) + assertTrue(body, body.contains("\"endMeasured\":true")) + } + + @Test + fun `a feed url with a quote in it cannot break the body`() { + val body = OutgoingLabels.body( + OutgoingLabels.fromVerdicts("https://x/\"odd\".xml", "g", listOf(verdict(1, 2, BreakVerdicts.VALID))) + )!! + + assertTrue(body, body.contains("\\\"odd\\\"")) + } + + @Test + fun `several labels go in one request`() { + val body = OutgoingLabels.body( + OutgoingLabels.fromVerdicts("f", "g", listOf(verdict(1, 2, BreakVerdicts.VALID))) + + OutgoingLabels.fromReports("f", "g", listOf(report(3, 4, false))) + )!! + + assertEquals(2, Regex("\"feedUrl\"").findAll(body).count()) + assertTrue(body.startsWith("{\"labels\":[")) + } +} diff --git a/docs/superpowers/plans/2026-10-20-corrections-and-signature-library.md b/docs/superpowers/plans/2026-10-20-corrections-and-signature-library.md index 2572cc3..fb4ad2c 100644 --- a/docs/superpowers/plans/2026-10-20-corrections-and-signature-library.md +++ b/docs/superpowers/plans/2026-10-20-corrections-and-signature-library.md @@ -174,11 +174,37 @@ between them, sometimes followed by one or more skips back, is a pretty strong s block."* A press and a skip-burst are two observations of the same event, and a skip *back* after over-shooting brackets the end precisely. Treat them as one input stream. -### 2. Push verdicts to the server — NOT STARTED +### 2. Push verdicts to the server — DONE 2026-10-24 -Phone to server. Needs an endpoint and server-side storage. Design note: a verdict is about a -*range of a specific episode's audio*, so it is only meaningful for episodes whose bytes the -server itself served — gate on `audioFromServer`, the same invariant the refresh sweep uses. +`POST /api/labels`, stored in `state.json` under a top-level `labels` list. Deployed and +exercised live: accepts, de-duplicates a re-send, refuses without the token. + +**Both kinds of label go, not just verdicts.** A *mark* is a judgement on a break detection +found; a *report* is the notification button pointing at advertising it missed. They catch +opposite errors and the library needs both, so the wire format carries `source` to say which. + +**Identified by feed and guid**, not by either side's episode id. The phone never sees the +server's id, and the server's id belongs to a mirror that forgets an episode after eight. + +**Labels outlive the episode.** They sit beside the shows rather than on them, so an episode +leaving the eight-episode window does not take the corpus with it. That is the same reasoning as +the app's own no-foreign-key decision for `break_verdicts`. + +**A report's end is marked as guessed or measured.** `endMeasured` is only true once the listener +came back and trimmed it; otherwise the end is wherever the 25-second jump happened to land. A +signature cut to a guessed end would carry programme audio, so the library has to know which it +is holding. + +**The phone re-sends everything, every sweep.** No record of what arrived, deliberately: the +server replaces rather than accumulating, so a re-send is free and there is no sync state to get +wrong. A handful of taps does not need a protocol. It also means a label lost to a crash comes +back on the next refresh. + +**Accepting a label writes the state file at once.** Everything else is written at the end of a +poll cycle, which for 61 feeds is hours. A label is a tap made once and should not wait that long +on the wrong side of a kill. + +Gated on `audioFromServer`, as the design note required. ### 3. The index spike — DONE 2026-10-24. **It passes.** diff --git a/server/cmd/listend/main.go b/server/cmd/listend/main.go index 524f438..c500a63 100644 --- a/server/cmd/listend/main.go +++ b/server/cmd/listend/main.go @@ -83,6 +83,10 @@ func main() { worker.save() worker.wake() }, + // Same reasoning, one step smaller: a label is a tap the listener made once, and it + // should not sit in memory until the end of a poll cycle. No wake -- nothing acts on a + // label yet, it is corpus for the signature library. + OnLabels: worker.save, }) ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) diff --git a/server/internal/api/api.go b/server/internal/api/api.go index 0aeef8a..969e69d 100644 --- a/server/internal/api/api.go +++ b/server/internal/api/api.go @@ -30,6 +30,9 @@ type Options struct { // OnSubscriptions is called when the phone sends its list, so a newly-subscribed show is // polled at once rather than at the next interval. Optional. OnSubscriptions func(feeds []string) + // OnLabels is called when labels were accepted, so they reach disk now rather than at the + // end of the next poll cycle. Optional. + OnLabels func() } // Server is the HTTP surface. @@ -86,6 +89,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("/api/health", s.health) mux.HandleFunc("/api/subscriptions", s.authed(s.subscriptions)) mux.HandleFunc("/api/episode", s.authed(s.episode)) + mux.HandleFunc("/api/labels", s.authed(s.labels)) mux.HandleFunc("/audio/", s.authed(s.audio)) return mux } @@ -230,3 +234,38 @@ func writeJSON(w http.ResponseWriter, status int, body any) { w.WriteHeader(status) _ = json.NewEncoder(w).Encode(body) } + +// labels takes what the listener said about stretches of audio: a judgement on a break +// detection found, or a report of advertising it missed. +// +// Stored rather than acted on. These are the corpus the signature library will be built from, +// and the server has no business changing what it publishes because of one listener's tap -- +// that is what the shadow run is for. +func (s *Server) labels(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "post only"}) + return + } + var body struct { + Labels []store.Label `json:"labels"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "expected {\"labels\":[...]}"}) + return + } + // Counted rather than refused as a batch: a label is a tap the listener made once and + // cannot be asked to make again, so one malformed entry must not discard the rest. + accepted := s.state.AddLabels(body.Labels) + // Only when something usable arrived, which includes a re-send of what is already held: + // the phone sends everything every sweep and has no way to know what got there. That makes + // this a whole-state write once per refresh, which is immaterial next to a label waiting + // hours on the wrong side of a kill -9. + if accepted > 0 && s.opts.OnLabels != nil { + s.opts.OnLabels() + } + writeJSON(w, http.StatusOK, map[string]any{ + "accepted": accepted, + "rejected": len(body.Labels) - accepted, + "held": len(s.state.Labels()), + }) +} diff --git a/server/internal/api/labels_test.go b/server/internal/api/labels_test.go new file mode 100644 index 0000000..7242720 --- /dev/null +++ b/server/internal/api/labels_test.go @@ -0,0 +1,167 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/terst/listen/server/internal/store" +) + +func postLabels(t *testing.T, s *Server, body string, withToken bool) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/api/labels", strings.NewReader(body)) + if withToken { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, req) + return rec +} + +const oneAdLabel = `{"labels":[{"feedUrl":"https://example.com/feed.xml","guid":"g1", + "startMs":600000,"endMs":645000,"verdict":"ad","source":"report","endMeasured":true, + "givenAtUnix":1700000000}]}` + +func TestAcceptsALabelAndKeepsIt(t *testing.T) { + s, st, _, _ := newServer(t) + + rec := postLabels(t, s, oneAdLabel, true) + + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + var got struct { + Accepted int `json:"accepted"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.Accepted != 1 { + t.Fatalf("accepted %d, want 1", got.Accepted) + } + kept := st.Labels() + if len(kept) != 1 { + t.Fatalf("stored %d labels", len(kept)) + } + if kept[0].Verdict != store.LabelAd || kept[0].Source != store.LabelFromReport { + t.Fatalf("got %+v", kept[0]) + } + if !kept[0].EndMeasured { + t.Fatal("endMeasured was dropped; a guessed end and a heard one are not the same thing") + } +} + +func TestLabelsNeedTheToken(t *testing.T) { + s, st, _, _ := newServer(t) + + rec := postLabels(t, s, oneAdLabel, false) + + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status %d, want 401", rec.Code) + } + if len(st.Labels()) != 0 { + t.Fatal("stored a label from an unauthenticated request") + } +} + +func TestLabelsRefuseAnythingButPost(t *testing.T) { + s, _, _, _ := newServer(t) + + if rec := get(t, s, "/api/labels"); rec.Code != http.StatusMethodNotAllowed { + t.Fatalf("status %d, want 405", rec.Code) + } +} + +func TestRejectsBodyThatIsNotLabels(t *testing.T) { + s, st, _, _ := newServer(t) + + rec := postLabels(t, s, `not json at all`, true) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status %d, want 400", rec.Code) + } + if len(st.Labels()) != 0 { + t.Fatal("stored something from an unparseable body") + } +} + +// The phone sends what it has; the server decides what it can use. A batch with one bad entry +// must not cost the listener the good ones -- those labels are not reproducible. +func TestKeepsTheUsableLabelsFromAMixedBatch(t *testing.T) { + s, st, _, _ := newServer(t) + + rec := postLabels(t, s, `{"labels":[ + {"feedUrl":"f","guid":"g1","startMs":0,"endMs":1000,"verdict":"ad","source":"mark"}, + {"feedUrl":"f","guid":"g2","startMs":0,"endMs":0,"verdict":"ad","source":"mark"}, + {"feedUrl":"f","guid":"g3","startMs":0,"endMs":1000,"verdict":"programme","source":"mark"} + ]}`, true) + + if rec.Code != http.StatusOK { + t.Fatalf("status %d", rec.Code) + } + var got struct { + Accepted int `json:"accepted"` + Rejected int `json:"rejected"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &got) + if got.Accepted != 2 || got.Rejected != 1 { + t.Fatalf("accepted %d rejected %d, want 2 and 1", got.Accepted, got.Rejected) + } + if len(st.Labels()) != 2 { + t.Fatalf("stored %d", len(st.Labels())) + } +} + +// Sending the same labels again is the normal case, not an error: the phone re-sends whatever +// it holds and has no way to know what arrived last time. +func TestResendingTheSameLabelsChangesNothing(t *testing.T) { + s, st, _, _ := newServer(t) + postLabels(t, s, oneAdLabel, true) + + postLabels(t, s, oneAdLabel, true) + + if len(st.Labels()) != 1 { + t.Fatalf("stored %d labels after a resend", len(st.Labels())) + } +} + +func TestAnEmptyBatchIsAccepted(t *testing.T) { + s, _, _, _ := newServer(t) + + if rec := postLabels(t, s, `{"labels":[]}`, true); rec.Code != http.StatusOK { + t.Fatalf("status %d, want 200", rec.Code) + } +} + +// Labels arrive between poll cycles, and state is otherwise only written at the end of one. A +// corpus that survives only until the next kill -9 is not a corpus, so accepting labels asks +// the caller to persist. The phone re-sends everything each sweep, so nothing is lost for good +// either way -- but a label is a tap the listener made once and should not wait hours on disk. +func TestAcceptingLabelsAsksForThemToBeSaved(t *testing.T) { + audioDir := t.TempDir() + st := store.New() + saved := 0 + s := New(st, &fakeFetcher{}, Options{Token: token, AudioDir: audioDir, OnLabels: func() { saved++ }}) + + postLabels(t, s, oneAdLabel, true) + + if saved != 1 { + t.Fatalf("asked to save %d times, want 1", saved) + } +} + +func TestNothingUsableMeansNothingToSave(t *testing.T) { + audioDir := t.TempDir() + st := store.New() + saved := 0 + s := New(st, &fakeFetcher{}, Options{Token: token, AudioDir: audioDir, OnLabels: func() { saved++ }}) + + postLabels(t, s, `{"labels":[{"feedUrl":"f","guid":"g","startMs":0,"endMs":0,"verdict":"ad","source":"mark"}]}`, true) + + if saved != 0 { + t.Fatal("wrote the state file for a batch that changed nothing") + } +} diff --git a/server/internal/store/labels.go b/server/internal/store/labels.go new file mode 100644 index 0000000..282196d --- /dev/null +++ b/server/internal/store/labels.go @@ -0,0 +1,109 @@ +package store + +// Label is a listener's statement about one stretch of one episode's audio. +// +// Two things produce these, and they catch opposite errors. A *mark* is a judgement on a break +// detection already found -- it catches false positives. A *report* points at advertising +// detection missed entirely, which is the bigger problem: the scan accounts for roughly 38% of +// the advertising the feeds imply is there, so most of what gets heard was never found. +// +// Identified by feed and guid rather than by the server's own episode id, because that is what +// the phone has: it asks about an episode the same way. The server's id is an implementation +// detail of the mirror, and the mirror forgets an episode after eight. +type Label struct { + FeedURL string `json:"feedUrl"` + GUID string `json:"guid"` + + StartMs int64 `json:"startMs"` + EndMs int64 `json:"endMs"` + + // Verdict is [LabelAd] or [LabelProgramme]. + Verdict string `json:"verdict"` + // Source is [LabelFromMark] or [LabelFromReport]. + Source string `json:"source"` + + // EndMeasured distinguishes an end the listener heard from one that is where a jump + // happened to land. A report starts as a guess at the end and only becomes measured if he + // comes back and trims it, and a signature cut to a guessed end would carry programme. + EndMeasured bool `json:"endMeasured,omitempty"` + + GivenAtUnix int64 `json:"givenAtUnix"` +} + +// What a label can say. +const ( + LabelAd = "ad" + LabelProgramme = "programme" +) + +// Where a label came from. +const ( + LabelFromMark = "mark" + LabelFromReport = "report" +) + +// identity is what makes two labels the same statement: the same stretch of the same episode, +// arrived at the same way. Source is part of it deliberately -- judging a detected break and +// pointing at an undetected advertisement are different acts even when the numbers coincide. +type labelKey struct { + feedURL string + guid string + startMs int64 + source string +} + +func (l Label) key() labelKey { + return labelKey{feedURL: l.FeedURL, guid: l.GUID, startMs: l.StartMs, source: l.Source} +} + +// usable rejects a label that cannot be acted on: one that names no episode, covers no audio, +// or says something this server does not understand. A newer phone inventing a verdict is not a +// reason to store a value nothing can interpret. +func (l Label) usable() bool { + if l.FeedURL == "" || l.GUID == "" { + return false + } + if l.EndMs <= l.StartMs { + return false + } + if l.Verdict != LabelAd && l.Verdict != LabelProgramme { + return false + } + return l.Source == LabelFromMark || l.Source == LabelFromReport +} + +// AddLabels stores what the listener said, replacing any earlier statement about the same +// stretch from the same source, and returns how many were usable. +// +// Replacing rather than appending because he is allowed to change his mind, and two rows +// disagreeing about one stretch of audio would leave the library to guess which he meant. +func (s *State) AddLabels(labels []Label) int { + s.mu.Lock() + defer s.mu.Unlock() + added := 0 + for _, l := range labels { + if !l.usable() { + continue + } + replaced := false + for i, existing := range s.labels { + if existing.key() == l.key() { + s.labels[i] = l + replaced = true + break + } + } + if !replaced { + s.labels = append(s.labels, l) + } + added++ + } + return added +} + +// Labels returns every label the listener has given, oldest first. +func (s *State) Labels() []Label { + s.mu.RLock() + defer s.mu.RUnlock() + return append([]Label(nil), s.labels...) +} diff --git a/server/internal/store/labels_test.go b/server/internal/store/labels_test.go new file mode 100644 index 0000000..ce6a06d --- /dev/null +++ b/server/internal/store/labels_test.go @@ -0,0 +1,135 @@ +package store_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/terst/listen/server/internal/store" +) + +func label(guid string, startMs, endMs int64, verdict, source string) store.Label { + return store.Label{ + FeedURL: "https://example.com/feed.xml", GUID: guid, + StartMs: startMs, EndMs: endMs, Verdict: verdict, Source: source, GivenAtUnix: 100, + } +} + +func TestKeepsALabelTheListenerGave(t *testing.T) { + s := store.New() + + added := s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark)}) + + if added != 1 { + t.Fatalf("added %d, want 1", added) + } + got := s.Labels() + if len(got) != 1 || got[0].GUID != "g1" || got[0].Verdict != store.LabelAd { + t.Fatalf("got %+v", got) + } +} + +func TestChangingYourMindReplacesTheLabelRatherThanAddingASecond(t *testing.T) { + s := store.New() + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark)}) + + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelProgramme, store.LabelFromMark)}) + + got := s.Labels() + if len(got) != 1 { + t.Fatalf("one stretch of audio, one verdict; got %d: %+v", len(got), got) + } + if got[0].Verdict != store.LabelProgramme { + t.Fatalf("verdict %q, want the later one", got[0].Verdict) + } +} + +func TestTheSameStretchReportedAndMarkedAreSeparateStatements(t *testing.T) { + s := store.New() + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark)}) + + // Pointing at advertising the detector missed is a different act from judging a break it + // found, even when the numbers coincide, so one must not silently overwrite the other. + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromReport)}) + + if got := s.Labels(); len(got) != 2 { + t.Fatalf("got %d labels, want 2: %+v", len(got), got) + } +} + +func TestLabelsOnDifferentEpisodesDoNotCollide(t *testing.T) { + s := store.New() + + s.AddLabels([]store.Label{ + label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromMark), + label("g2", 60_000, 90_000, store.LabelAd, store.LabelFromMark), + }) + + if got := s.Labels(); len(got) != 2 { + t.Fatalf("got %d, want 2", len(got)) + } +} + +func TestRejectsALabelWithNothingToIdentifyItBy(t *testing.T) { + s := store.New() + + added := s.AddLabels([]store.Label{ + {FeedURL: "", GUID: "g1", EndMs: 10, Verdict: store.LabelAd}, + {FeedURL: "f", GUID: "", EndMs: 10, Verdict: store.LabelAd}, + {FeedURL: "f", GUID: "g", EndMs: 0, Verdict: store.LabelAd}, + {FeedURL: "f", GUID: "g", EndMs: 10, Verdict: "shrug"}, + }) + + if added != 0 { + t.Fatalf("accepted %d unusable labels", added) + } + if got := s.Labels(); len(got) != 0 { + t.Fatalf("stored %+v", got) + } +} + +func TestLabelsSurviveASaveAndLoad(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + s := store.New() + s.AddLabels([]store.Label{label("g1", 60_000, 90_000, store.LabelAd, store.LabelFromReport)}) + if err := s.Save(path); err != nil { + t.Fatalf("save: %v", err) + } + + again, err := store.Load(path) + if err != nil { + t.Fatalf("load: %v", err) + } + + got := again.Labels() + if len(got) != 1 || got[0].Source != store.LabelFromReport { + t.Fatalf("got %+v", got) + } +} + +// A label is evidence about a stretch of audio. The episode it was given on rotates out of the +// mirror after eight, and the label has to outlive that -- it is the corpus the signature +// library is built from, and throwing it away with the audio would be throwing away the point. +func TestLabelsOutliveTheEpisodeTheyWereGivenOn(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "state.json") + s := store.New() + s.AddLabels([]store.Label{label("long-gone", 60_000, 90_000, store.LabelAd, store.LabelFromMark)}) + if err := s.Save(path); err != nil { + t.Fatalf("save: %v", err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(raw) == 0 { + t.Fatal("empty state file") + } + + again, _ := store.Load(path) + // Nothing in the state knows this episode any more; the label is still there. + if got := again.Labels(); len(got) != 1 || got[0].GUID != "long-gone" { + t.Fatalf("got %+v", got) + } +} diff --git a/server/internal/store/store.go b/server/internal/store/store.go index 4734009..7f04fa0 100644 --- a/server/internal/store/store.go +++ b/server/internal/store/store.go @@ -105,11 +105,15 @@ type State struct { mu sync.RWMutex feeds []string shows map[string]*Show + // labels are the listener's own statements about stretches of audio. Not held on the + // episodes: an episode leaves the mirror after eight and a label has to outlive it. + labels []Label } type persisted struct { - Feeds []string `json:"feeds"` - Shows map[string]*Show `json:"shows"` + Feeds []string `json:"feeds"` + Shows map[string]*Show `json:"shows"` + Labels []Label `json:"labels,omitempty"` } // New returns an empty state. @@ -132,6 +136,7 @@ func Load(path string) (*State, error) { } s := New() s.feeds = p.Feeds + s.labels = p.Labels if p.Shows != nil { s.shows = p.Shows } @@ -145,7 +150,7 @@ func Load(path string) (*State, error) { // away hours of detection. func (s *State) Save(path string) error { s.mu.RLock() - raw, err := json.MarshalIndent(persisted{Feeds: s.feeds, Shows: s.shows}, "", " ") + raw, err := json.MarshalIndent(persisted{Feeds: s.feeds, Shows: s.shows, Labels: s.labels}, "", " ") s.mu.RUnlock() if err != nil { return fmt.Errorf("save state: %w", err) |
