diff options
Diffstat (limited to 'app/src')
20 files changed, 1517 insertions, 151 deletions
diff --git a/app/src/main/java/org/terst/listen/fingerprint/AudioCorrelation.kt b/app/src/main/java/org/terst/listen/fingerprint/AudioCorrelation.kt index 0b477a1..1576bc8 100644 --- a/app/src/main/java/org/terst/listen/fingerprint/AudioCorrelation.kt +++ b/app/src/main/java/org/terst/listen/fingerprint/AudioCorrelation.kt @@ -29,11 +29,68 @@ object AudioCorrelation { private const val MIN_QUERY_ENERGY = 1e-6 /** + * A target with its expensive, query-independent half already computed. + * + * The transform of the target and its rolling sums depend only on the target and on how long + * a query will be -- not on the query's contents. A window search correlates many queries + * against one unchanged target ([IntroOutroDetector]'s candidate search does it 117 times), + * and recomputing this for each of them was most of why fingerprinting could not finish + * inside a WorkManager worker. + * + * Opaque on purpose: [prepareTarget] is the only way to build one, so the invariant that the + * transform matches [querySize] cannot be broken from outside. + */ + class PreparedTarget internal constructor( + internal val targetSize: Int, + internal val querySize: Int, + internal val transformRe: DoubleArray, + internal val transformIm: DoubleArray, + internal val localSum: DoubleArray, + internal val localSumSq: DoubleArray, + ) + + /** + * [target] with everything that does not depend on the query's contents computed once, for + * queries of exactly [querySize] samples. + * + * Returns a prepared target even when [querySize] does not fit, so the caller does not have + * to duplicate that check; [bestMatch] reports the same null it always did. + */ + fun prepareTarget(target: DoubleArray, querySize: Int): PreparedTarget { + if (querySize <= 0 || querySize > target.size) { + return PreparedTarget(target.size, querySize, DoubleArray(0), DoubleArray(0), DoubleArray(0), DoubleArray(0)) + } + val n = Fft.nextPowerOfTwo(target.size + querySize - 1) + val re = target.copyOf(n) + val im = DoubleArray(n) + Fft.transform(re, im, inverse = false) + return PreparedTarget( + targetSize = target.size, + querySize = querySize, + transformRe = re, + transformIm = im, + localSum = rollingSum(target, querySize), + localSumSq = rollingSum(DoubleArray(target.size) { target[it] * target[it] }, querySize), + ) + } + + /** * The best-aligned position of [query] inside [target], or null if there is nowhere valid * to place it (the query is longer than the target) or the query itself carries no signal * to match against (silence has an undefined correlation with anything). */ - fun bestMatch(query: DoubleArray, target: DoubleArray): Match? { + fun bestMatch(query: DoubleArray, target: DoubleArray): Match? = + bestMatch(query, prepareTarget(target, query.size)) + + /** + * The same answer as [bestMatch], against a target prepared once by [prepareTarget]. + * + * Null for a query whose length is not the one the target was prepared for: the prepared + * transform is sized for that length and silently reusing it at another would not be a + * slower correct answer, it would be a wrong one. + */ + fun bestMatch(query: DoubleArray, target: PreparedTarget): Match? { + if (query.size != target.querySize) return null val scores = normalizedCrossCorrelation(query, target) ?: return null if (scores.isEmpty()) return null var bestIndex = 0 @@ -59,9 +116,9 @@ object AudioCorrelation { * (each window's own standard deviation times the query's) comes from a rolling sum and * sum-of-squares over `target`, needing no FFT. */ - private fun normalizedCrossCorrelation(query: DoubleArray, target: DoubleArray): DoubleArray? { + private fun normalizedCrossCorrelation(query: DoubleArray, target: PreparedTarget): DoubleArray? { val win = query.size - if (win == 0 || win > target.size) return null + if (win == 0 || win > target.targetSize) return null val queryMean = query.average() val centeredQuery = DoubleArray(win) { query[it] - queryMean } @@ -70,13 +127,10 @@ object AudioCorrelation { val dot = validConvolution(target, reversed(centeredQuery)) - val localSum = rollingSum(target, win) - val localSumSq = rollingSum(DoubleArray(target.size) { target[it] * target[it] }, win) - val scores = DoubleArray(dot.size) for (i in dot.indices) { - val mean = localSum[i] / win - val variance = max(localSumSq[i] / win - mean * mean, 0.0) + val mean = target.localSum[i] / win + val variance = max(target.localSumSq[i] / win - mean * mean, 0.0) val localNorm = sqrt(variance * win) val denom = localNorm * queryNorm scores[i] = if (denom < MIN_QUERY_ENERGY) 0.0 else dot[i] / denom @@ -100,31 +154,28 @@ object AudioCorrelation { } /** - * The "valid" slice of convolving [target] with [kernel]: length `target.size - kernel.size - * + 1`, computed via zero-padded FFT rather than the O(n*m) direct sum. + * The "valid" slice of convolving [target] with [kernel]: length `target.targetSize - + * kernel.size + 1`, computed via zero-padded FFT rather than the O(n*m) direct sum. + * + * Two transforms, not three: the target's was already done by [prepareTarget]. */ - private fun validConvolution(target: DoubleArray, kernel: DoubleArray): DoubleArray { - val fullLength = target.size + kernel.size - 1 - val n = Fft.nextPowerOfTwo(fullLength) + private fun validConvolution(target: PreparedTarget, kernel: DoubleArray): DoubleArray { + val n = target.transformRe.size - val targetRe = target.copyOf(n) - val targetIm = DoubleArray(n) val kernelRe = kernel.copyOf(n) val kernelIm = DoubleArray(n) - - Fft.transform(targetRe, targetIm, inverse = false) Fft.transform(kernelRe, kernelIm, inverse = false) val productRe = DoubleArray(n) val productIm = DoubleArray(n) for (i in 0 until n) { - productRe[i] = targetRe[i] * kernelRe[i] - targetIm[i] * kernelIm[i] - productIm[i] = targetRe[i] * kernelIm[i] + targetIm[i] * kernelRe[i] + productRe[i] = target.transformRe[i] * kernelRe[i] - target.transformIm[i] * kernelIm[i] + productIm[i] = target.transformRe[i] * kernelIm[i] + target.transformIm[i] * kernelRe[i] } Fft.transform(productRe, productIm, inverse = true) // The valid region starts once the kernel is fully overlapped with the target. - val validLength = target.size - kernel.size + 1 + val validLength = target.targetSize - kernel.size + 1 return DoubleArray(validLength) { productRe[it + kernel.size - 1] } } } diff --git a/app/src/main/java/org/terst/listen/fingerprint/IntroOutroDetector.kt b/app/src/main/java/org/terst/listen/fingerprint/IntroOutroDetector.kt index 9050466..ebbe4a6 100644 --- a/app/src/main/java/org/terst/listen/fingerprint/IntroOutroDetector.kt +++ b/app/src/main/java/org/terst/listen/fingerprint/IntroOutroDetector.kt @@ -60,22 +60,65 @@ object IntroOutroDetector { private const val MIN_SCORE = 0.6 private const val CANDIDATE_STEP_DIVISOR = 4 + /** Matches [MidrollUpdater]'s own coarse factor, which was validated against real shows. */ + private const val DEFAULT_COARSE_FACTOR = 8 + private const val DEFAULT_FINE_CANDIDATES = 3 + + /** + * Below this many samples a coarse window stops carrying enough signal to rank positions by, + * so the search runs at full resolution instead. Also what keeps the unit tests' 300-sample + * clips on the path they were written against. + */ + private const val MIN_COARSE_WINDOW_SAMPLES = 512 + + /** + * What the candidate search is allowed to cost. Mirrors [MidrollUpdater.Tuning], including + * the reason it exists: the search's resolution is a performance choice, and the only honest + * way to show that is to make it a value a test can set. + * + * Nothing here is a detection threshold. `MIN_SCORE`, `CONFIRM_THRESHOLD` and the reference + * length are deliberately *not* reachable from the app or from tuning -- see CLAUDE.md. + */ + data class Tuning( + /** + * How much to shrink both clips before searching for a candidate position. 1 searches at + * full resolution, which is what this did before and is kept reachable so the fast path + * can be compared against it. + */ + val coarseFactor: Int = DEFAULT_COARSE_FACTOR, + /** + * How many coarse hits get verified at full resolution. More than one because + * decimation can reorder near-equal peaks, and the real intro losing a coarse tie to a + * spurious one would otherwise be unrecoverable. + */ + val fineCandidates: Int = DEFAULT_FINE_CANDIDATES, + /** + * How many independently-fetched episodes must agree before a candidate is trusted. + * + * Here with the cost knobs rather than as its own argument because [MidrollUpdater.Tuning] + * already carries the same value the same way -- but unlike them it is a *safety* + * threshold, and the reason it is three is in this class's own documentation. It is + * reachable for tests, not for the app; see CLAUDE.md. + */ + val confirmThreshold: Int = CONFIRM_THRESHOLD, + ) + fun check( state: FingerprintState, episodeId: String, clip: DoubleArray, referenceLengthSamples: Int, - confirmThreshold: Int = CONFIRM_THRESHOLD, + tuning: Tuning = Tuning(), ): FingerprintUpdate { if (episodeId == state.lastConfirmedEpisodeId || episodeId == state.pendingEpisodeId) { - return toUpdate(state, confirmThreshold) + return toUpdate(state, tuning.confirmThreshold) } val reference = state.referenceClip if (reference == null) { - return checkAgainstPending(state, episodeId, clip, referenceLengthSamples) + return checkAgainstPending(state, episodeId, clip, referenceLengthSamples, tuning) } - return checkAgainstReference(state, episodeId, clip, reference, confirmThreshold) + return checkAgainstReference(state, episodeId, clip, reference, tuning.confirmThreshold) } private fun checkAgainstPending( @@ -83,6 +126,7 @@ object IntroOutroDetector { episodeId: String, clip: DoubleArray, referenceLengthSamples: Int, + tuning: Tuning, ): FingerprintUpdate { val pending = state.pendingClip if (pending == null) { @@ -91,7 +135,7 @@ object IntroOutroDetector { ) } - val candidate = findCandidate(pending, clip, referenceLengthSamples) + val candidate = findCandidate(pending, clip, referenceLengthSamples, tuning) if (candidate == null) { // No shared segment between the only two episodes seen so far -- this one becomes // the new basis, same as if nothing had ever been seen. @@ -139,32 +183,96 @@ object IntroOutroDetector { } /** - * Slides a [referenceLengthSamples]-long window across [a] a quarter-window at a time, - * keeping whichever position matches best somewhere in [b] -- or null if nothing matches - * strongly enough to trust. + * Finds a [referenceLengthSamples]-long segment of [a] that also occurs in [b], or null if + * nothing matches strongly enough to trust. + * + * Coarse-to-fine, the composition [MidrollUpdater] already uses and for the same reason: the + * *position* of a shared segment can be found on heavily decimated audio for a fraction of + * the cost, while whether it is really a match is still decided at full resolution. Searched + * flat at full resolution this took 199 seconds per end on a desktop -- measured, in + * [IntroOutroDetectorScaleTest] -- which is how fingerprinting came to burn a WorkManager + * worker's entire ten-minute allowance and persist nothing. * - * A step equal to the full window would almost always straddle a real embedded segment - * rather than land on it exactly (any embed position not a multiple of the window length - * gets split across two candidate windows, each only partially the real signal, which is - * often too weak to clear [MIN_SCORE]). A quarter-window step costs more candidate checks - * but actually lands on the real segment. + * The step within the search is a quarter-window, not a whole one. A step equal to the full + * window would almost always straddle a real embedded segment rather than land on it (any + * embed position not a multiple of the window length gets split across two candidate + * windows, each only partially the real signal, which is often too weak to clear + * [MIN_SCORE]). */ - private fun findCandidate(a: DoubleArray, b: DoubleArray, referenceLengthSamples: Int): DoubleArray? { + private fun findCandidate( + a: DoubleArray, + b: DoubleArray, + referenceLengthSamples: Int, + tuning: Tuning, + ): DoubleArray? { val win = referenceLengthSamples.coerceAtMost(a.size).coerceAtMost(b.size) if (win <= 0) return null + + val factor = coarseFactorFor(win, tuning.coarseFactor) + val positions = if (factor == 1) { + searchPositions(a, b, win, tuning.fineCandidates) + } else { + searchPositions(Decimate.byAveraging(a, factor), Decimate.byAveraging(b, factor), win / factor, tuning.fineCandidates) + .map { it * factor } + } + + return bestVerifiedWindow(a, b, win, positions) + } + + /** + * Decimation is skipped rather than allowed to shrink the search window into noise: a + * window of a handful of samples correlates with anything, and the clips the unit tests use + * are three hundred samples long in total. + */ + private fun coarseFactorFor(win: Int, requested: Int): Int { + val factor = requested.coerceAtLeast(1) + if (factor == 1) return 1 + return if (win / factor >= MIN_COARSE_WINDOW_SAMPLES) factor else 1 + } + + /** + * The best-scoring window start positions in [a], highest first -- in whatever resolution + * [a] and [b] are handed in at. + * + * No score threshold here on purpose. This pass only ranks positions; at a coarse resolution + * its absolute scores are not the ones detection is allowed to decide on, so rejecting here + * would be rejecting on a number the fine pass is about to compute properly. + */ + private fun searchPositions(a: DoubleArray, b: DoubleArray, win: Int, keep: Int): List<Int> { + if (win <= 0 || win > a.size || win > b.size) return emptyList() val step = (win / CANDIDATE_STEP_DIVISOR).coerceAtLeast(1) + val target = AudioCorrelation.prepareTarget(b, win) - var bestScore = MIN_SCORE - var bestWindow: DoubleArray? = null + val scored = mutableListOf<Pair<Int, Double>>() var start = 0 while (start + win <= a.size) { + AudioCorrelation.bestMatch(a.copyOfRange(start, start + win), target) + ?.let { scored += start to it.score } + start += step + } + return scored.sortedByDescending { it.second }.take(keep.coerceAtLeast(1)).map { it.first } + } + + /** + * The first candidate position whose full-resolution score clears [MIN_SCORE], as a window of + * [a] -- which is the clip the reference is kept from, exactly as before. + * + * This is where detection actually happens: every number the coarse pass produced is + * discarded and the decision is made on native-resolution audio. + */ + private fun bestVerifiedWindow(a: DoubleArray, b: DoubleArray, win: Int, positions: List<Int>): DoubleArray? { + val target = AudioCorrelation.prepareTarget(b, win) + var bestScore = MIN_SCORE + var bestWindow: DoubleArray? = null + + for (position in positions) { + val start = position.coerceIn(0, a.size - win) val window = a.copyOfRange(start, start + win) - val match = AudioCorrelation.bestMatch(window, b) + val match = AudioCorrelation.bestMatch(window, target) if (match != null && match.score > bestScore) { bestScore = match.score bestWindow = window } - start += step } return bestWindow } diff --git a/app/src/main/java/org/terst/listen/fingerprint/MediaCodecClipSource.kt b/app/src/main/java/org/terst/listen/fingerprint/MediaCodecClipSource.kt index 4daed7d..7294ba9 100644 --- a/app/src/main/java/org/terst/listen/fingerprint/MediaCodecClipSource.kt +++ b/app/src/main/java/org/terst/listen/fingerprint/MediaCodecClipSource.kt @@ -38,9 +38,12 @@ class MediaCodecClipSource : AudioClipSource { /** * Unlike [decodeHead]/[decodeTail], there is no fixed window: a midroll ad can be anywhere, - * so the whole episode is decoded. [EpisodeEntity.durationMs] sizes the buffer up front for - * the same reason [decodeToPcm] does for a short clip -- a boxed, growable collection for - * what can be hours of samples would be real, avoidable garbage on a phone. + * so the whole episode is decoded. [EpisodeEntity.durationMs] bounds how far to read rather + * than sizing an allocation -- see [MonoResampler] for why that distinction is the whole + * reason this path used to fail. + * + * How long an episode this is *worth* attempting is [MidrollUpdater]'s call, not this + * class's: the result is still held whole in memory by whoever asked for it. */ override suspend fun decodeFull(episode: EpisodeEntity): DecodedClip? { val path = episode.localPath ?: return null @@ -65,10 +68,9 @@ class MediaCodecClipSource : AudioClipSource { val channels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) extractor.seekTo(startMs * MICROS_PER_MS, MediaExtractor.SEEK_TO_CLOSEST_SYNC) - val raw = decodeToPcm(extractor, format, inputRate, channels, clipSeconds) - val mono = downmix(raw, channels) - val resampled = resample(mono, inputRate, TARGET_SAMPLE_RATE) - return DecodedClip(resampled, TARGET_SAMPLE_RATE, startMs) + val resampler = MonoResampler(inputRate, channels, TARGET_SAMPLE_RATE) + decodeInto(extractor, format, resampler, channels, clipSeconds.toLong() * inputRate * channels) + return DecodedClip(resampler.finish(), TARGET_SAMPLE_RATE, startMs) } finally { extractor.release() } @@ -82,35 +84,36 @@ class MediaCodecClipSource : AudioClipSource { error("No audio track in this file") } - /** Interleaved PCM16 samples for up to [clipSeconds] of decoded audio. */ - private fun decodeToPcm( + /** + * Pumps the decoder into [resampler] until [maxSamples] interleaved samples have been read + * or the track ends. + * + * Nothing here holds the decoded audio: each output buffer is copied into a small reused + * scratch array and handed straight on, so the bound is a stop condition rather than an + * allocation. That is the whole difference from the version this replaces, which asked for + * 145MB of native-rate PCM to produce 6MB of 8kHz mono and died doing it. + */ + private fun decodeInto( extractor: MediaExtractor, format: MediaFormat, - inputRate: Int, + resampler: MonoResampler, channels: Int, - clipSeconds: Int, - ): ShortArray { + maxSamples: Long, + ) { val mime = format.getString(MediaFormat.KEY_MIME) ?: error("No MIME on the audio track") val codec = MediaCodec.createDecoderByType(mime) try { codec.configure(format, null, null, 0) codec.start() - // Sized to the clip length up front rather than grown: a boxed collection for - // several million samples would be real, avoidable garbage on a phone. - val target = ShortArray((clipSeconds.toLong() * inputRate * channels).toInt()) - var written = 0 + val pump = OutputPump(codec, resampler, channels, maxSamples) var inputDone = false - var outputDone = false - val info = MediaCodec.BufferInfo() - - while (!outputDone && written < target.size) { + while (!pump.finished) { if (!inputDone) { inputDone = feedInput(extractor, codec) } - written = drainOutput(codec, info, target, written) { outputDone = true } + pump.drain() } - return if (written == target.size) target else target.copyOf(written) } finally { codec.stop() codec.release() @@ -133,59 +136,54 @@ class MediaCodecClipSource : AudioClipSource { } } - private fun drainOutput( - codec: MediaCodec, - info: MediaCodec.BufferInfo, - target: ShortArray, - writtenSoFar: Int, - onEndOfStream: () -> Unit, - ): Int { - val outputIndex = codec.dequeueOutputBuffer(info, TIMEOUT_US) - if (outputIndex < 0) return writtenSoFar - - var written = writtenSoFar - if (info.size > 0) { - val output = codec.getOutputBuffer(outputIndex) - if (output != null) { - output.position(info.offset) - output.limit(info.offset + info.size) - val shorts = output.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer() - while (shorts.hasRemaining() && written < target.size) { - target[written] = shorts.get() - written++ - } - } - } - codec.releaseOutputBuffer(outputIndex, false) - if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) onEndOfStream() - return written - } + /** + * One decoder's output side, with the running count it needs. + * + * A class rather than a function taking all of it: the loop needs a sample count, an + * end-of-stream flag and a reusable buffer carried between calls, and threading those + * through arguments produced a seven-parameter function that said less than this does. + */ + private class OutputPump( + private val codec: MediaCodec, + private val resampler: MonoResampler, + channels: Int, + private val maxSamples: Long, + ) { + private val info = MediaCodec.BufferInfo() - private fun downmix(interleaved: ShortArray, channels: Int): DoubleArray { - if (channels <= 1) return DoubleArray(interleaved.size) { interleaved[it].toDouble() } - val frames = interleaved.size / channels - return DoubleArray(frames) { frame -> - var sum = 0.0 - for (c in 0 until channels) sum += interleaved[frame * channels + c] - sum / channels + // A multiple of the channel count, so copying never splits a frame unless the decoder's + // own buffer already did -- which MonoResampler carries across anyway. + private val scratch = ShortArray(SCRATCH_FRAMES * channels) + + private var samplesRead = 0L + private var endOfStream = false + + /** Whether there is no more to read: the track ended, or enough has been taken. */ + val finished: Boolean get() = endOfStream || samplesRead >= maxSamples + + /** Hands one output buffer to the resampler, if one is ready. */ + fun drain() { + val outputIndex = codec.dequeueOutputBuffer(info, TIMEOUT_US) + if (outputIndex < 0) return + + if (info.size > 0) { + codec.getOutputBuffer(outputIndex)?.let { copyOut(it) } + } + codec.releaseOutputBuffer(outputIndex, false) + if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) endOfStream = true } - } - /** - * Linear-interpolation resample, not a proper anti-aliasing filter: matching only needs - * coarse spectral shape, not audiophile fidelity, and every source feed converges on the - * same [TARGET_SAMPLE_RATE] regardless of what it was encoded at. - */ - private fun resample(mono: DoubleArray, inputRate: Int, targetRate: Int): DoubleArray { - if (inputRate == targetRate || mono.isEmpty()) return mono - val ratio = targetRate.toDouble() / inputRate - val outLength = (mono.size * ratio).toInt() - return DoubleArray(outLength) { i -> - val sourcePosition = i / ratio - val index = sourcePosition.toInt().coerceIn(0, mono.size - 1) - val next = (index + 1).coerceAtMost(mono.size - 1) - val fraction = sourcePosition - index - mono[index] * (1 - fraction) + mono[next] * fraction + private fun copyOut(output: java.nio.ByteBuffer) { + output.position(info.offset) + output.limit(info.offset + info.size) + val shorts = output.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer() + while (shorts.hasRemaining() && samplesRead < maxSamples) { + val room = minOf(maxSamples - samplesRead, scratch.size.toLong()).toInt() + val count = minOf(shorts.remaining(), room) + shorts.get(scratch, 0, count) + resampler.accept(scratch, 0, count) + samplesRead += count + } } } @@ -200,6 +198,9 @@ class MediaCodecClipSource : AudioClipSource { const val FULL_DECODE_MARGIN_SECONDS = 30 const val TARGET_SAMPLE_RATE = 8_000 const val TIMEOUT_US = 10_000L + + /** Frames copied out of one decoder buffer at a time; see [decodeInto]. */ + const val SCRATCH_FRAMES = 4_096 const val MICROS_PER_MS = 1_000L const val MS_PER_SECOND = 1_000L } diff --git a/app/src/main/java/org/terst/listen/fingerprint/MidrollUpdater.kt b/app/src/main/java/org/terst/listen/fingerprint/MidrollUpdater.kt index e5e757c..95dcd3e 100644 --- a/app/src/main/java/org/terst/listen/fingerprint/MidrollUpdater.kt +++ b/app/src/main/java/org/terst/listen/fingerprint/MidrollUpdater.kt @@ -7,6 +7,7 @@ import org.terst.listen.data.EpisodeMidrollEntity import org.terst.listen.data.MidrollDao import org.terst.listen.data.ShowMidrollEntity import org.terst.listen.data.SubscriptionDao +import org.terst.listen.data.playbackDurationMs /** * Runs [MidrollDetector] for every downloaded episode a show has not yet been checked against, @@ -37,6 +38,22 @@ class MidrollUpdater( val discoverySamples: Int = DEFAULT_DISCOVERY_SAMPLES, val maxCandidates: Int = DEFAULT_MAX_CANDIDATES, val confirmThreshold: Int = DEFAULT_CONFIRM_THRESHOLD, + /** + * The longest episode a full-body scan will attempt. + * + * A scan holds the whole episode decoded in memory, so an episode's length is also an + * allocation: an hour of 8kHz mono is 28.8 million doubles, 230MB, on a phone with no + * largeHeap. Asking for that threw OutOfMemoryError inside [AudioClipSource.decodeFull], + * where `runCatching` swallowed it -- and the episode was then marked checked forever, so + * midroll detection could never find anything on any episode of normal length and said + * nothing about why. + * + * An over-budget episode is skipped *before* anything is spent, and deliberately left + * unchecked, so that lifting this limit (by decoding candidate ranges on demand instead + * of the whole body at once) picks those episodes up rather than finding them written + * off. + */ + val maxDecodableMs: Long = DEFAULT_MAX_DECODABLE_MS, val newId: () -> String = { UUID.randomUUID().toString() }, val now: () -> Long = System::currentTimeMillis, ) @@ -52,7 +69,12 @@ class MidrollUpdater( val show = subscriptions.getById(subscriptionId) ?: return var pendingEpisodeId = show.midrollPendingEpisodeId - for (episode in midrolls.getUncheckedDownloaded(subscriptionId)) { + // Over-budget episodes are dropped here rather than skipped inside the loop: nothing is + // spent on them and they stay unchecked, so lifting the limit picks them up again -- + // see Tuning.maxDecodableMs. + val scannable = midrolls.getUncheckedDownloaded(subscriptionId).filterNot(::exceedsDecodeBudget) + + for (episode in scannable) { val decoded = clips.decodeFull(episode) if (decoded == null) { midrolls.setMidrollCheckedAtMs(episode.id, now()) @@ -84,6 +106,18 @@ class MidrollUpdater( } } + /** + * Whether this episode is longer than a whole-body decode can hold. + * + * An unknown length is not over budget: it is the case [AudioClipSource.decodeFull] already + * declines for want of a size to allocate, and routing it through here would change what + * happens to it. + */ + private fun exceedsDecodeBudget(episode: EpisodeEntity): Boolean { + val durationMs = episode.playbackDurationMs ?: return false + return durationMs > tuning.maxDecodableMs + } + private suspend fun discoverAgainstPending( subscriptionId: String, pendingEpisodeId: String, @@ -249,6 +283,14 @@ class MidrollUpdater( const val DEFAULT_DISCOVERY_SAMPLES = 20_000 // 20s at the coarse (post-decimation) rate const val DEFAULT_MAX_CANDIDATES = 4 const val DEFAULT_CONFIRM_THRESHOLD = 3 + + /** + * Twenty minutes, which at 8kHz mono is 9.6 million doubles -- about 77MB, the most a + * background worker can reasonably hold on a phone with no largeHeap. Not a detection + * threshold: it is the ceiling of the current in-memory decode, and it rises when that + * decode learns to fetch candidate ranges on demand. + */ + const val DEFAULT_MAX_DECODABLE_MS = 20L * 60 * 1000 const val SEEDED_CONFIRM_COUNT = 2 const val COARSE_MIN_SCORE = 0.7 const val FINE_MIN_SCORE = 0.6 diff --git a/app/src/main/java/org/terst/listen/fingerprint/MonoResampler.kt b/app/src/main/java/org/terst/listen/fingerprint/MonoResampler.kt new file mode 100644 index 0000000..a98d2ab --- /dev/null +++ b/app/src/main/java/org/terst/listen/fingerprint/MonoResampler.kt @@ -0,0 +1,134 @@ +package org.terst.listen.fingerprint + +/** + * Turns a decoder's interleaved PCM16 output into mono at a fixed rate, a buffer at a time. + * + * The reason this is streaming rather than three passes over three arrays: a full-episode scan's + * decode used to buffer every interleaved sample at the source's native rate, then every mono + * frame, and only then resample. For a 13-minute stereo 44.1kHz episode that is 145MB and 289MB + * live at once to produce 6MB of output -- on a phone with no largeHeap. It threw + * OutOfMemoryError inside a `runCatching` that swallowed it, which is why midroll detection + * never ran on anything and never said why. Here the peak allocation is the output. + * + * Pure, so it is the part of [MediaCodecClipSource] that can actually be tested; what is left + * there is MediaCodec plumbing and nothing else. + * + * The interpolation is deliberately unchanged -- naive linear, no anti-aliasing filter. It was + * measured at 0.9991 correlation against a proper ffmpeg/swresample decode of the same audio + * (`docs/detection-findings.md`), so it was never the weak link, and a rewrite that both moved + * the allocation and changed the arithmetic would be untraceable if detection got worse. + */ +class MonoResampler( + private val inputRate: Int, + private val channels: Int, + private val targetRate: Int, +) { + + private val ratio = targetRate.toDouble() / inputRate + + private var out = DoubleArray(INITIAL_CAPACITY) + private var outCount = 0 + + /** Mono frames of the buffer being consumed, reused across calls to keep churn down. */ + private var frames = DoubleArray(0) + + private var framesAccepted = 0 + + /** + * The last mono frame accepted, kept because an output sample interpolates between two + * source frames and those two can straddle a decoder buffer boundary. One frame is all that + * is ever needed: output positions only move forward. + */ + private var previousFrame = 0.0 + + /** A frame split across two decoder buffers. Nothing guarantees they arrive frame-aligned. */ + private val splitFrame = ShortArray(channels.coerceAtLeast(1)) + private var splitCount = 0 + + /** Consumes [count] interleaved samples from [interleaved] starting at [offset]. */ + fun accept(interleaved: ShortArray, offset: Int, count: Int) { + if (count <= 0 || channels <= 0) return + + val accepted = toMonoFrames(interleaved, offset, count) + if (accepted == 0) return + + emitWhatIsReadable(start = framesAccepted, available = accepted) + framesAccepted += accepted + previousFrame = frames[accepted - 1] + } + + /** + * The resampled mono audio. + * + * Length is `(frames * ratio)` truncated, exactly as the batch implementation computed it, so + * that the two cannot disagree about where the audio ends. + */ + fun finish(): DoubleArray { + if (framesAccepted == 0) return DoubleArray(0) + + // Whatever is left reads past the final frame, which clamps to it -- so every remaining + // output is that frame, whatever its fractional position would have been. + val total = (framesAccepted * ratio).toInt() + while (outCount < total) append(previousFrame) + + return out.copyOf(outCount) + } + + /** + * Averages [count] interleaved samples down into [frames], completing any frame the previous + * buffer left split and keeping any this one does. Returns how many whole frames resulted. + */ + private fun toMonoFrames(source: ShortArray, offset: Int, count: Int): Int { + val total = splitCount + count + val whole = total / channels + if (frames.size < whole) frames = DoubleArray(whole) + + // The split frame's samples come first: together with this buffer they are one stream. + fun sampleAt(index: Int): Short = + if (index < splitCount) splitFrame[index] else source[offset + index - splitCount] + + var read = 0 + for (frame in 0 until whole) { + var sum = 0.0 + repeat(channels) { sum += sampleAt(read++) } + frames[frame] = sum / channels + } + + // Safe to overwrite while reading: with a whole frame produced, `read` is already past + // everything held in splitFrame; with none, this copies those samples onto themselves. + val leftover = total - whole * channels + for (i in 0 until leftover) splitFrame[i] = sampleAt(read + i) + splitCount = leftover + + return whole + } + + /** + * Emits every output sample whose two source frames are both readable now -- the newest + * buffer's frames, plus [previousFrame] sitting immediately before them. + */ + private fun emitWhatIsReadable(start: Int, available: Int) { + val lastReadable = start + available - 1 + while (true) { + val sourcePosition = outCount / ratio + val index = sourcePosition.toInt() + if (index + 1 > lastReadable) return + + // Output positions advance monotonically and everything readable from earlier + // buffers was already emitted, so `index` is never further back than previousFrame. + val before = if (index < start) previousFrame else frames[index - start] + val after = if (index + 1 < start) previousFrame else frames[index + 1 - start] + val fraction = sourcePosition - index + append(before * (1 - fraction) + after * fraction) + } + } + + private fun append(value: Double) { + if (outCount == out.size) out = out.copyOf(out.size * 2) + out[outCount++] = value + } + + private companion object { + const val INITIAL_CAPACITY = 8_192 + } +} diff --git a/app/src/main/java/org/terst/listen/fingerprint/SkipSummary.kt b/app/src/main/java/org/terst/listen/fingerprint/SkipSummary.kt index fedd3e4..63b3ec5 100644 --- a/app/src/main/java/org/terst/listen/fingerprint/SkipSummary.kt +++ b/app/src/main/java/org/terst/listen/fingerprint/SkipSummary.kt @@ -16,6 +16,18 @@ data class EpisodeSkips( val outro: String, val breaks: List<String>, val insertedAds: String?, + /** What the feed says the episode is, null if it does not say. */ + val declaredLength: String? = null, + /** What the player measured the file to be, null until it has opened it. */ + val measuredLength: String? = null, + /** + * How much of the episode the detected breaks add up to, null when none are detected. + * + * The point of showing it next to [insertedAds]: the length gap says how much advertising is + * in the file without any audio analysis, so these two should roughly agree. Far short means + * the scan is missing breaks; far over means it is matching programme. + */ + val breaksTotal: String? = null, ) /** @@ -41,6 +53,28 @@ object SkipSummary { private const val SECONDS_PER_MINUTE = 60 private const val MINUTES_PER_HOUR = 60 + /** Where one detected break sits in an episode, as fractions of its length. */ + data class BreakMarker(val start: Float, val end: Float) + + /** + * The detected breaks as positions on a 0..1 timeline, for the episode screen's scrubber. + * + * Here rather than in the composable because a marker drawn in the wrong place looks + * plausible in a screenshot and is obvious in a test -- and because a break can run past a + * duration that was measured on a different fetch of the same episode, dynamic ad insertion + * being per request. Clamped rather than dropped: the break is real, the length is the + * doubtful number. + */ + fun breakMarkers(ranges: List<EpisodeMidrollEntity>, durationMs: Long?): List<BreakMarker> { + if (durationMs == null || durationMs <= 0) return emptyList() + return ranges.sortedBy { it.startMs }.map { + BreakMarker( + start = (it.startMs.toFloat() / durationMs).coerceIn(0f, 1f), + end = (it.endMs.toFloat() / durationMs).coerceIn(0f, 1f), + ) + } + } + /** One end of a show's episodes -- its intro or its outro. */ fun candidateState(state: ShowFingerprintEntity?, threshold: Int): String { // No reference clip means no candidate: either nothing has been processed, or a single @@ -98,6 +132,11 @@ object SkipSummary { val breaks = ranges(ranges) return EpisodeSkips( insertedAds = episode.insertedAdMs?.let { clock(it) }, + declaredLength = episode.durationMs?.let { clock(it) }, + measuredLength = episode.actualDurationMs?.let { clock(it) }, + breaksTotal = ranges.sumOf { (it.endMs - it.startMs).coerceAtLeast(0) } + .takeIf { ranges.isNotEmpty() } + ?.let { clock(it) }, intro = offset(episode.detectedIntroMs, show?.skipIntroMs ?: 0), outro = offset(episode.detectedOutroMs, show?.skipOutroMs ?: 0), breaks = when { diff --git a/app/src/main/java/org/terst/listen/playback/PlaybackService.kt b/app/src/main/java/org/terst/listen/playback/PlaybackService.kt index 39af65d..3a10e4e 100644 --- a/app/src/main/java/org/terst/listen/playback/PlaybackService.kt +++ b/app/src/main/java/org/terst/listen/playback/PlaybackService.kt @@ -38,6 +38,12 @@ class PlaybackService : MediaLibraryService() { private var advancer: PlaybackAdvancer? = null private val progress = PlaybackProgress() + /** + * Times each start against the episode's length, for the open question about the pause + * before playback -- see [PrepareTrace]. Cheap enough to leave in: one line per episode. + */ + private val prepareTrace = PrepareTrace() + /** What the ticker runs against, refreshed whenever the playing item changes. */ private var context = PlayingContext() @@ -110,7 +116,18 @@ class PlaybackService : MediaLibraryService() { * component that survives backgrounding is the one that has to own this. */ private val completionListener = object : Player.Listener { + override fun onIsPlayingChanged(isPlaying: Boolean) { + if (!isPlaying) return + prepareTrace.mark(PrepareTrace.Stage.PLAYING) + prepareTrace.summary()?.let { android.util.Log.i(TAG, it) } + } + override fun onPlaybackStateChanged(playbackState: Int) { + if (playbackState == Player.STATE_READY) { + prepareTrace.length(player?.duration?.takeIf { it > 0 }) + prepareTrace.mark(PrepareTrace.Stage.READY) + prepareTrace.summary()?.let { android.util.Log.i(TAG, it) } + } if (playbackState != Player.STATE_ENDED) return // Read before anything else runs: starting the next episode replaces the media item, // so the id is gone by the time this returns. @@ -119,6 +136,13 @@ class PlaybackService : MediaLibraryService() { } override fun onMediaItemTransition(mediaItem: MediaItem?, reason: Int) { + // Where the prepare clock starts. This is the one hook both play paths reach: the + // service's own queue advance, and a tap in the UI, which sets the item through a + // MediaController and never calls this service's play(). + mediaItem?.mediaId?.let { id -> + val uri = mediaItem.localConfiguration?.uri?.scheme + prepareTrace.start(id, durationMs = null, local = uri == null || uri == "file") + } // The cached show and chapters belong to whatever is playing; refresh them whenever // that changes, including when something else in the app starts an episode. loadContextFor(mediaItem?.mediaId) @@ -241,6 +265,7 @@ class PlaybackService : MediaLibraryService() { } private companion object { + const val TAG = "ListenPlayback" const val POSITION_TICK_MS = 2_000L const val SEEK_BACK_MS = 15_000L const val SEEK_FORWARD_MS = 25_000L diff --git a/app/src/main/java/org/terst/listen/playback/PrepareTrace.kt b/app/src/main/java/org/terst/listen/playback/PrepareTrace.kt new file mode 100644 index 0000000..f565754 --- /dev/null +++ b/app/src/main/java/org/terst/listen/playback/PrepareTrace.kt @@ -0,0 +1,89 @@ +package org.terst.listen.playback + +/** + * Times how long starting an episode takes, against how long the episode is. + * + * Exists for a specific open question: playback takes a noticeable moment to start, and the + * delay looks proportional to the episode's length. Nothing in this app's play path scales with + * length -- it reads a few rows and calls `setMediaItem`/`prepare` -- so the delay is either + * inside Media3's preparation or it is the CPU being busy with something else. From outside + * those look identical, and picking between them by argument is how the wrong thing gets fixed. + * + * One line per start, carrying both numbers, settles it from a log: + * + * ``` + * prepare ep=abc123 len=1774s src=local ready=812ms playing=903ms + * ``` + * + * Deliberately a value rather than logging directly, so the format is pinned by + * [PrepareTraceTest] -- a line that quietly stopped reporting the length would make the whole + * measurement worthless while still looking like it worked. + */ +class PrepareTrace(private val now: () -> Long = System::currentTimeMillis) { + + /** The points worth timing between asking for an episode and hearing it. */ + enum class Stage(internal val label: String) { + /** Media3 has the media open and buffered enough to play. */ + READY("ready"), + + /** Audio is actually coming out. */ + PLAYING("playing"), + } + + private var episodeId: String? = null + private var durationMs: Long? = null + private var local = true + private var startedAt = 0L + private val reached = linkedMapOf<Stage, Long>() + + /** Called as the media item is handed to the player. */ + fun start(episodeId: String, durationMs: Long?, local: Boolean) { + this.episodeId = episodeId + this.durationMs = durationMs + this.local = local + startedAt = now() + reached.clear() + } + + /** + * Supplies the episode's length once the player knows it. + * + * The clock has to start when the media item is set -- the only point both play paths pass + * through, since a tap in the UI reaches the player through a MediaController and never + * calls the service's own `play` -- and a MediaItem carries no duration. What the player + * reports once it has opened the file is the better number anyway: the real length of the + * file rather than the feed's claim, which on a show using dynamic ad insertion is minutes + * short. + * + * A null is ignored rather than stored: Media3 reports no duration until it has one, and + * letting that clear a known length would lose it. + */ + fun length(durationMs: Long?) { + if (episodeId == null || durationMs == null) return + this.durationMs = durationMs + } + + /** + * Records when [stage] was first reached. + * + * First, not last: Media3 reports READY again after a rebuffer, and letting that overwrite + * the original would replace the delay being measured with a small number. + */ + fun mark(stage: Stage) { + if (episodeId == null || stage in reached) return + reached[stage] = now() - startedAt + } + + /** The line to log, or null if no start has been recorded yet. */ + fun summary(): String? { + val id = episodeId ?: return null + val length = durationMs?.let { "${it / MS_PER_SECOND}s" } ?: "?" + val source = if (local) "local" else "stream" + val stages = reached.entries.joinToString("") { " ${it.key.label}=${it.value}ms" } + return "prepare ep=$id len=$length src=$source$stages" + } + + private companion object { + const val MS_PER_SECOND = 1_000L + } +} diff --git a/app/src/main/java/org/terst/listen/ui/EpisodeDetailActions.kt b/app/src/main/java/org/terst/listen/ui/EpisodeDetailActions.kt index 5ef0ff1..009c8c0 100644 --- a/app/src/main/java/org/terst/listen/ui/EpisodeDetailActions.kt +++ b/app/src/main/java/org/terst/listen/ui/EpisodeDetailActions.kt @@ -43,4 +43,12 @@ data class EpisodeDetailActions( val download: () -> Unit, val togglePlayed: () -> Unit, val markOlderPlayed: () -> Unit, + /** Play/pause when this episode is the one playing; starts it when it is not. */ + val togglePlayPause: () -> Unit = {}, + /** + * Where the scrubber was dragged to. Seeks the player when this episode is the one playing, + * and otherwise moves where Play will start from -- the screen decides which, because only + * it knows whether the drag is fighting a live position tick. + */ + val seekTo: (Long) -> Unit = {}, ) diff --git a/app/src/main/java/org/terst/listen/ui/EpisodeScreen.kt b/app/src/main/java/org/terst/listen/ui/EpisodeScreen.kt index de828e2..649d24f 100644 --- a/app/src/main/java/org/terst/listen/ui/EpisodeScreen.kt +++ b/app/src/main/java/org/terst/listen/ui/EpisodeScreen.kt @@ -1,12 +1,15 @@ package org.terst.listen.ui +import androidx.compose.foundation.Canvas import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.FlowRow import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.rememberScrollState @@ -14,6 +17,8 @@ import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.filled.ArrowBack +import androidx.compose.material.icons.filled.Pause +import androidx.compose.material.icons.filled.PlayArrow import androidx.compose.material3.Button import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.Icon @@ -21,26 +26,33 @@ import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedButton import androidx.compose.material3.Scaffold +import androidx.compose.material3.Slider import androidx.compose.material3.Text import androidx.compose.material3.TopAppBar import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableFloatStateOf import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import coil3.compose.AsyncImage import org.terst.listen.data.EpisodeEntity import org.terst.listen.data.EpisodeMidrollEntity import org.terst.listen.data.SubscriptionEntity +import org.terst.listen.data.playbackDurationMs import org.terst.listen.feed.WatchLink import org.terst.listen.fingerprint.EpisodeSkips import org.terst.listen.fingerprint.SkipSummary +import org.terst.listen.playback.NowPlaying import org.terst.listen.playback.effectiveIntroMs import org.terst.listen.playback.effectiveOutroMs @@ -67,32 +79,29 @@ fun EpisodeScreen( } val rules by vm.rules.collectAsState() val shows by vm.subscriptions.collectAsState() + val nowPlaying by player.nowPlaying.collectAsState() val current = episode Scaffold( - topBar = { - TopAppBar( - title = { Text(current?.title ?: "Episode", maxLines = 1, overflow = TextOverflow.Ellipsis) }, - navigationIcon = { - IconButton(onClick = nav.back) { - Icon(Icons.AutoMirrored.Filled.ArrowBack, contentDescription = "Back") - } - }, - ) - }, + topBar = { EpisodeTopBar(current?.title, nav.back) }, bottomBar = { PlayerBar(player, nav.openPlayer, nav.openQueue) }, ) { padding -> if (current == null) { Text("Episode not found", Modifier.padding(padding).padding(24.dp)) return@Scaffold } + // Whether the transport controls drive the player or just move a stored position. An + // episode can be open on this screen while something else is playing, and a scrubber + // that silently drove the wrong episode would be worse than one that did nothing. + val isPlayingThis = nowPlaying.episodeId == current.id EpisodeDetail( episode = current, - ruleName = rules.firstOrNull { rule -> rule.id == current.appliedRuleId }?.name, - skips = SkipSummary.episode( - current, - shows.firstOrNull { it.id == current.subscriptionId }, - ranges, + detail = episodeDetailState( + episode = current, + show = shows.firstOrNull { it.id == current.subscriptionId }, + ruleName = rules.firstOrNull { rule -> rule.id == current.appliedRuleId }?.name, + ranges = ranges, + nowPlaying = nowPlaying, ), modifier = Modifier.padding(padding), actions = EpisodeDetailActions( @@ -104,6 +113,19 @@ fun EpisodeScreen( episode = current.copy(playedAtMs = if (current.playedAtMs == null) 1L else null) }, markOlderPlayed = { vm.markPlayedUpTo(current) }, + togglePlayPause = { + if (isPlayingThis) player.togglePlayPause() else player.play(current) + }, + seekTo = { positionMs -> + // Seeking the player only works for the item it has loaded; for any other + // episode the drag is a statement about where Play should begin. + if (isPlayingThis) { + player.seekTo(positionMs) + } else { + vm.setPosition(current.id, positionMs) + episode = current.copy(positionMs = positionMs) + } + }, ), ) } @@ -112,8 +134,7 @@ fun EpisodeScreen( @Composable private fun EpisodeDetail( episode: EpisodeEntity, - ruleName: String?, - skips: EpisodeSkips, + detail: EpisodeDetailState, modifier: Modifier, actions: EpisodeDetailActions, ) { @@ -136,6 +157,8 @@ private fun EpisodeDetail( } } + EpisodeScrubber(detail.playback, actions) + FlowRow( horizontalArrangement = Arrangement.spacedBy(8.dp), verticalArrangement = Arrangement.spacedBy(8.dp), @@ -155,9 +178,9 @@ private fun EpisodeDetail( Text("Mark this and older as played") } - RuleAttribution(episode, ruleName) + RuleAttribution(episode, detail.ruleName) - SkipReadout(skips) + SkipReadout(detail.skips) episode.description?.let { Text(stripHtml(it), style = MaterialTheme.typography.bodyMedium) @@ -165,6 +188,133 @@ private fun EpisodeDetail( } } +@OptIn(ExperimentalMaterial3Api::class) +@Composable +private fun EpisodeTopBar(title: String?, onBack: () -> Unit) { + TopAppBar( + title = { Text(title ?: "Episode", maxLines = 1, overflow = TextOverflow.Ellipsis) }, + navigationIcon = { + IconButton(onClick = onBack) { + Icon(Icons.AutoMirrored.Filled.ArrowBack, contentDescription = "Back") + } + }, + ) +} + +/** Everything the detail body renders but does not change. */ +data class EpisodeDetailState( + val ruleName: String?, + val skips: EpisodeSkips, + val playback: EpisodePlayback, +) + +/** + * Gathers what the detail body shows. + * + * Out of the composable because assembling it inline pushed `EpisodeScreen` past the length + * limit, and because the one decision in here is worth seeing plainly: a position and duration + * come from the player when this is the episode playing, and from the stored row when it is not. + */ +private fun episodeDetailState( + episode: EpisodeEntity, + show: SubscriptionEntity?, + ruleName: String?, + ranges: List<EpisodeMidrollEntity>, + nowPlaying: NowPlaying, +): EpisodeDetailState { + // Whether the transport controls drive the player or just describe a stored position. + val isPlayingThis = nowPlaying.episodeId == episode.id + val durationMs = if (isPlayingThis) nowPlaying.durationMs else episode.playbackDurationMs + return EpisodeDetailState( + ruleName = ruleName, + skips = SkipSummary.episode(episode, show, ranges), + playback = EpisodePlayback( + positionMs = if (isPlayingThis) nowPlaying.positionMs else episode.positionMs, + durationMs = durationMs, + isPlaying = isPlayingThis && nowPlaying.isPlaying, + breaks = SkipSummary.breakMarkers(ranges, durationMs), + ), + ) +} + +/** What the episode screen's transport controls act on. */ +data class EpisodePlayback( + val positionMs: Long, + val durationMs: Long?, + val isPlaying: Boolean, + /** Detected ad breaks as fractions of the episode -- see [SkipSummary.breakMarkers]. */ + val breaks: List<SkipSummary.BreakMarker>, +) + +/** + * This episode's position, with the detected ad breaks marked on it. + * + * Separate from the player screen's scrubber on purpose: that one always drives whatever is + * playing, and this one has to work for an episode that is merely open -- where a drag sets + * where Play will begin rather than seeking anything. The drag rule is the same, though: a + * finger should not fight the live position tick, so the local value only tracks the drag and + * the seek fires once on release. + */ +@Composable +private fun EpisodeScrubber(playback: EpisodePlayback, actions: EpisodeDetailActions) { + val duration = playback.durationMs?.takeIf { it > 0 } ?: 0L + var dragValue by remember { mutableFloatStateOf(-1f) } + val shown = if (dragValue >= 0f) dragValue else playback.positionMs.toFloat() + + Row(verticalAlignment = Alignment.CenterVertically) { + IconButton(onClick = actions.togglePlayPause) { + Icon( + if (playback.isPlaying) Icons.Default.Pause else Icons.Default.PlayArrow, + contentDescription = if (playback.isPlaying) "Pause" else "Play", + ) + } + Column(Modifier.weight(1f)) { + Box { + // Behind the slider, so the track reads as the episode and the marks read as + // parts of it. Drawn at the track's own inset, or they would not line up with + // the thumb positions they are describing. + BreakMarks(playback.breaks, Modifier.align(Alignment.Center)) + Slider( + value = shown.coerceIn(0f, duration.toFloat().coerceAtLeast(0f)), + onValueChange = { dragValue = it }, + onValueChangeFinished = { + actions.seekTo(dragValue.toLong()) + dragValue = -1f + }, + valueRange = 0f..duration.toFloat().coerceAtLeast(1f), + enabled = duration > 0, + ) + } + Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween) { + Text(formatClock(shown.toLong()), style = MaterialTheme.typography.bodySmall) + Text(formatClock(duration), style = MaterialTheme.typography.bodySmall) + } + } + } +} + +/** The detected ad breaks as bands across the track. */ +@Composable +private fun BreakMarks(breaks: List<SkipSummary.BreakMarker>, modifier: Modifier) { + if (breaks.isEmpty()) return + val colour = MaterialTheme.colorScheme.tertiary + Canvas(modifier.fillMaxWidth().height(BREAK_MARK_HEIGHT)) { + // Inset by the thumb's radius at each end: the slider's track is laid out inside it, so + // marks drawn edge to edge would sit a thumb-width off the positions they describe. + val inset = THUMB_RADIUS_PX + val usable = (size.width - inset * 2).coerceAtLeast(0f) + breaks.forEach { mark -> + val left = inset + usable * mark.start + val right = inset + usable * mark.end + drawRect( + color = colour, + topLeft = Offset(left, 0f), + size = Size((right - left).coerceAtLeast(MIN_BREAK_MARK_PX), size.height), + ) + } + } +} + /** * What is actually being skipped in this episode, and where the numbers came from. * @@ -182,12 +332,20 @@ private fun SkipReadout(skips: EpisodeSkips) { Text("Skips", style = MaterialTheme.typography.titleSmall) // Free, and a check on the detector: whatever it finds should eventually add up to // about this much. + // The three numbers only mean anything together: the feed's length against the file's + // says how much advertising is in it, and what the detector found says how much of that + // it has actually accounted for. Short means breaks are being missed; over means + // programme is being matched. + skips.declaredLength?.let { declared -> + ReadoutRow("Length", skips.measuredLength?.let { "$declared declared, $it measured" } ?: declared) + } skips.insertedAds?.let { ReadoutRow("Ads inserted", "~$it") } ReadoutRow("Intro", skips.intro) ReadoutRow("Outro", skips.outro) skips.breaks.forEachIndexed { index, line -> ReadoutRow(if (index == 0) "Ad breaks" else "", line) } + skips.breaksTotal?.let { ReadoutRow("Breaks found", it) } } } @@ -223,3 +381,12 @@ private fun RuleAttribution(episode: EpisodeEntity, ruleName: String?) { val label = if (ruleName != null) "Handled by \"$ruleName\" — $reason" else "Handled by a rule — $reason" Text(label, style = MaterialTheme.typography.bodySmall) } + +/** Thin enough to read as an annotation on the track rather than a second control. */ +private val BREAK_MARK_HEIGHT = 4.dp + +/** Material's slider thumb radius, which the track is inset by at both ends. */ +private const val THUMB_RADIUS_PX = 20f + +/** A break shorter than this would otherwise draw as nothing at all. */ +private const val MIN_BREAK_MARK_PX = 2f diff --git a/app/src/main/java/org/terst/listen/ui/ListenViewModel.kt b/app/src/main/java/org/terst/listen/ui/ListenViewModel.kt index e278f78..d03c7a5 100644 --- a/app/src/main/java/org/terst/listen/ui/ListenViewModel.kt +++ b/app/src/main/java/org/terst/listen/ui/ListenViewModel.kt @@ -85,6 +85,18 @@ class ListenViewModel(app: Application) : AndroidViewModel(app) { suspend fun midrollRanges(episodeId: String) = graph.midrolls.getRanges(episodeId) /** + * Moves where an episode will start from, for the episode screen's scrubber. + * + * Only for an episode that is *not* the one playing -- that one is dragged through the + * player, so the position comes back from Media3 rather than being written underneath it. + * Through [org.terst.listen.data.PlaybackStateDao] like every other position write, because + * this is the one piece of data in the app that cannot be reconstructed from anywhere. + */ + fun setPosition(episodeId: String, positionMs: Long) { + viewModelScope.launch { graph.playbackState.updatePosition(episodeId, positionMs.coerceAtLeast(0)) } + } + + /** * A show's episodes as it displays them. * * [dripEnabled] comes from the subscription rather than being looked up here, so the list diff --git a/app/src/main/java/org/terst/listen/ui/MainActivity.kt b/app/src/main/java/org/terst/listen/ui/MainActivity.kt index fedc34d..a4cc048 100644 --- a/app/src/main/java/org/terst/listen/ui/MainActivity.kt +++ b/app/src/main/java/org/terst/listen/ui/MainActivity.kt @@ -117,9 +117,12 @@ private fun ListenApp( QueueScreen( vm = viewModel, player = player, - onOpenPlayer = { nav.navigate("player") }, - onOpenQueue = {}, - onBack = { nav.popBackStack() }, + nav = QueueNav( + openEpisode = { nav.navigate("episode/$it") }, + openPlayer = { nav.navigate("player") }, + openQueue = {}, + back = { nav.popBackStack() }, + ), ) } composable("subscription/{id}") { entry -> diff --git a/app/src/main/java/org/terst/listen/ui/QueueScreen.kt b/app/src/main/java/org/terst/listen/ui/QueueScreen.kt index 75fbaca..c53fd5b 100644 --- a/app/src/main/java/org/terst/listen/ui/QueueScreen.kt +++ b/app/src/main/java/org/terst/listen/ui/QueueScreen.kt @@ -1,6 +1,7 @@ package org.terst.listen.ui import androidx.compose.foundation.background +import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxSize @@ -45,9 +46,7 @@ import sh.calvin.reorderable.rememberReorderableLazyListState @OptIn(ExperimentalMaterial3Api::class) @Composable -fun QueueScreen(vm: ListenViewModel, player: PlayerViewModel, - onOpenPlayer: () -> Unit, - onOpenQueue: () -> Unit, onBack: () -> Unit) { +fun QueueScreen(vm: ListenViewModel, player: PlayerViewModel, nav: QueueNav) { val queue by vm.queue.collectAsState() val nowPlaying by player.nowPlaying.collectAsState() val subscriptions by vm.subscriptions.collectAsState() @@ -70,13 +69,13 @@ fun QueueScreen(vm: ListenViewModel, player: PlayerViewModel, TopAppBar( title = { Text("Up Next") }, navigationIcon = { - IconButton(onClick = onBack) { + IconButton(onClick = nav.back) { Icon(Icons.AutoMirrored.Filled.ArrowBack, contentDescription = "Back") } }, ) }, - bottomBar = { PlayerBar(player, onOpenPlayer, onOpenQueue) }, + bottomBar = { PlayerBar(player, nav.openPlayer, nav.openQueue) }, ) { padding -> LazyColumn(state = lazyListState, modifier = Modifier.padding(padding).fillMaxSize()) { items(localOrder, key = { it.id }) { episode -> @@ -84,9 +83,12 @@ fun QueueScreen(vm: ListenViewModel, player: PlayerViewModel, QueueRow( episode = episode, artworkUrl = Artwork.preferredUrl(episode.imageUrl, subscriptionImages[episode.subscriptionId]), - onPlay = { player.play(episode) }, - onRemove = { vm.removeFromQueue(episode.id) }, - onDragStopped = { vm.reorderQueue(localOrder.map { it.id }) }, + actions = QueueRowActions( + play = { player.play(episode) }, + remove = { vm.removeFromQueue(episode.id) }, + dragStopped = { vm.reorderQueue(localOrder.map { it.id }) }, + openDetails = { nav.openEpisode(episode.id) }, + ), ) } HorizontalDivider() @@ -99,16 +101,17 @@ fun QueueScreen(vm: ListenViewModel, player: PlayerViewModel, private fun ReorderableCollectionItemScope.QueueRow( episode: EpisodeEntity, artworkUrl: String?, - onPlay: () -> Unit, - onRemove: () -> Unit, - onDragStopped: () -> Unit, + actions: QueueRowActions, ) { + // The row opens the episode, the same as the show screen's rows do. It used to do nothing, + // so where a tap landed decided whether anything happened -- the drag handle, play and + // remove keep their own handlers and sit above this. Row( - Modifier.fillMaxWidth().padding(horizontal = 16.dp, vertical = 8.dp), + Modifier.fillMaxWidth().clickable(onClick = actions.openDetails).padding(horizontal = 16.dp, vertical = 8.dp), verticalAlignment = Alignment.CenterVertically, ) { IconButton( - modifier = Modifier.draggableHandle(onDragStopped = onDragStopped), + modifier = Modifier.draggableHandle(onDragStopped = actions.dragStopped), onClick = {}, ) { Icon(Icons.Default.DragHandle, contentDescription = "Reorder") @@ -125,11 +128,27 @@ private fun ReorderableCollectionItemScope.QueueRow( Text(episode.title, maxLines = 2, overflow = TextOverflow.Ellipsis) Text(episode.progressLabel(), style = MaterialTheme.typography.bodySmall) } - IconButton(onClick = onPlay) { + IconButton(onClick = actions.play) { Icon(Icons.Default.PlayArrow, contentDescription = "Play") } - IconButton(onClick = onRemove) { + IconButton(onClick = actions.remove) { Icon(Icons.Default.RemoveCircleOutline, contentDescription = "Remove from queue") } } } + +/** Where the queue screen navigates to, bundled like [LibraryNav] and [SubscriptionNav]. */ +data class QueueNav( + val openEpisode: (String) -> Unit, + val openPlayer: () -> Unit, + val openQueue: () -> Unit, + val back: () -> Unit, +) + +/** One queue row's actions, bundled for the same reason [EpisodeActions] is. */ +data class QueueRowActions( + val play: () -> Unit, + val remove: () -> Unit, + val dragStopped: () -> Unit, + val openDetails: () -> Unit, +) diff --git a/app/src/test/java/org/terst/listen/fingerprint/AudioCorrelationTest.kt b/app/src/test/java/org/terst/listen/fingerprint/AudioCorrelationTest.kt index 6274e99..0d078cd 100644 --- a/app/src/test/java/org/terst/listen/fingerprint/AudioCorrelationTest.kt +++ b/app/src/test/java/org/terst/listen/fingerprint/AudioCorrelationTest.kt @@ -157,4 +157,41 @@ class AudioCorrelationTest { fun `silence in the query has no meaningful correlation with anything`() { assertNull(AudioCorrelation.bestMatch(DoubleArray(40), targetWithMatch)) } + + /** + * The target's transform is the expensive half of a correlation, and inside a window search + * the target never changes -- see [IntroOutroDetector]'s candidate search, which correlates + * every window of one clip against the *same* other clip. Preparing it once has to give + * bit-for-bit what the one-shot call gives, or the search would quietly change its answers + * as a side effect of going faster. + */ + @Test + fun `a prepared target gives the same match as a one-shot call`() { + val oneShot = AudioCorrelation.bestMatch(sting, targetWithMatch)!! + val reused = AudioCorrelation.bestMatch(sting, AudioCorrelation.prepareTarget(targetWithMatch, sting.size))!! + + assertEquals(oneShot.position, reused.position) + assertEquals(oneShot.score, reused.score, 1e-12) + } + + /** The whole point: one preparation, many queries, each scoring as it would alone. */ + @Test + fun `a prepared target serves several queries, each scoring as it would alone`() { + val prepared = AudioCorrelation.prepareTarget(targetWithMatch, sting.size) + + for (start in listOf(0, 40, 120, 200)) { + val query = targetWithMatch.copyOfRange(start, start + sting.size) + val alone = AudioCorrelation.bestMatch(query, targetWithMatch)!! + val viaPrepared = AudioCorrelation.bestMatch(query, prepared)!! + assertEquals("position at $start", alone.position, viaPrepared.position) + assertEquals("score at $start", alone.score, viaPrepared.score, 1e-12) + } + } + + /** A prepared target is sized for one query length; anything else is a programming error. */ + @Test + fun `a prepared target rejects a query of the wrong length`() { + val prepared = AudioCorrelation.prepareTarget(targetWithMatch, sting.size) + assertNull(AudioCorrelation.bestMatch(DoubleArray(sting.size + 1), prepared)) + } } diff --git a/app/src/test/java/org/terst/listen/fingerprint/IntroOutroDetectorScaleTest.kt b/app/src/test/java/org/terst/listen/fingerprint/IntroOutroDetectorScaleTest.kt new file mode 100644 index 0000000..a8fcaa5 --- /dev/null +++ b/app/src/test/java/org/terst/listen/fingerprint/IntroOutroDetectorScaleTest.kt @@ -0,0 +1,171 @@ +package org.terst.listen.fingerprint + +import kotlin.math.sin +import kotlin.random.Random +import kotlin.system.measureTimeMillis +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The guard whose absence let the detector ship unusable. + * + * [IntroOutroDetectorTest] proves the state machine on 300-sample clips. Production hands it + * 1,920,000 -- a 240-second window at 8kHz, 6,400 times larger -- and nothing anywhere measured + * what that costs. It cost minutes per episode: the candidate search correlated every one of 117 + * window positions against the whole of the other clip, re-transforming that unchanged clip + * every time. `FingerprintWorker` is a plain WorkManager worker, so Android stopped it at ten + * minutes, before it had persisted anything, and WorkManager re-ran it -- which is why Peter's + * phone ran hot and why no show ever got past its first processed episode. + * + * So these tests assert cost, at the real sizes, with real constants. The budget is deliberately + * generous: this runs on whatever machine happens to build the app, and a phone is slower again. + * It is not a benchmark, it is a tripwire -- the failure it exists to catch is minutes, not + * milliseconds. + */ +class IntroOutroDetectorScaleTest { + + private companion object { + const val SAMPLE_RATE = 8_000 + /** [MediaCodecClipSource.CLIP_SECONDS], the window production actually decodes. */ + const val CLIP_SECONDS = 240 + const val CLIP_SAMPLES = SAMPLE_RATE * CLIP_SECONDS + + /** [FingerprintUpdater.DEFAULT_REFERENCE_SECONDS] at [FingerprintUpdater.DEFAULT_SAMPLE_RATE]. */ + const val REFERENCE_SAMPLES = 8 * SAMPLE_RATE + + /** + * One episode's worth of candidate search. Both ends of an episode run inside the same + * ten-minute worker, so a single end has to come in far enough under that to leave room + * for the other end, two or four MediaCodec decodes, and a slower CPU than this one. + */ + const val BUDGET_MS = 60_000L + } + + private fun noise(seed: Int, n: Int) = Random(seed).let { r -> DoubleArray(n) { r.nextDouble(-50.0, 50.0) } } + + /** A tonal sting, the shape a real show's intro music has at 8kHz. */ + private fun sting(seed: Int, n: Int): DoubleArray { + val r = Random(seed) + return DoubleArray(n) { sin(it * 0.03) * 400 + r.nextDouble(-20.0, 20.0) } + } + + private fun embed(base: DoubleArray, segment: DoubleArray, at: Int): DoubleArray { + val out = base.copyOf() + segment.forEachIndexed { i, v -> out[at + i] = v } + return out + } + + /** The show's recurring intro, long enough that a reference window lands wholly inside it. */ + private val theShowsSting = sting(seed = 100, n = REFERENCE_SAMPLES * 2) + + /** + * Two episodes, the sting at a different offset in each -- the dynamic-pre-roll case + * `docs/detection-findings.md` measured on real Up First audio, where the intro landed + * anywhere between 44 and 94 seconds in. + */ + private fun episodeClip(seed: Int, stingAtSeconds: Int, clipSeconds: Int = CLIP_SECONDS) = + embed(noise(seed, clipSeconds * SAMPLE_RATE), theShowsSting, stingAtSeconds * SAMPLE_RATE) + + @Test + fun `a production-sized candidate search finds the sting inside the worker's budget`() { + val first = episodeClip(seed = 1, stingAtSeconds = 44) + val second = episodeClip(seed = 2, stingAtSeconds = 94) + + val waiting = IntroOutroDetector.check(FingerprintState(), "ep1", first, REFERENCE_SAMPLES) + assertTrue("first episode has nothing to compare against", waiting is FingerprintUpdate.Waiting) + + lateinit var update: FingerprintUpdate + val elapsed = measureTimeMillis { + update = IntroOutroDetector.check(waiting.state, "ep2", second, REFERENCE_SAMPLES) + } + + assertTrue( + "expected a candidate from two episodes sharing an intro, got $update", + update is FingerprintUpdate.Tentative, + ) + assertTrue( + "candidate search took ${elapsed}ms against a ${BUDGET_MS}ms budget -- a worker " + + "that cannot finish inside WorkManager's ten minutes never persists anything", + elapsed < BUDGET_MS, + ) + } + + /** + * The confirming episode's path, which runs once a reference exists and is the one that + * actually writes a skip offset. Cheap by construction -- a single correlation -- and pinned + * here so it stays that way. + */ + @Test + fun `confirming against an existing reference stays cheap at production size`() { + val reference = theShowsSting.copyOf(REFERENCE_SAMPLES) + val state = FingerprintState( + referenceClip = reference, + confirmedCount = 2, + lastConfirmedEpisodeId = "ep2", + ) + + lateinit var update: FingerprintUpdate + val elapsed = measureTimeMillis { + update = IntroOutroDetector.check(state, "ep3", episodeClip(seed = 3, stingAtSeconds = 61), REFERENCE_SAMPLES) + } + + assertTrue("expected the third episode to confirm, got $update", update is FingerprintUpdate.Confirmed) + assertEquals(61 * SAMPLE_RATE, (update as FingerprintUpdate.Confirmed).positionInClip) + assertTrue("single confirmation took ${elapsed}ms", elapsed < BUDGET_MS) + } + + /** + * Full resolution has to stay reachable, because that is what the current behaviour is + * defined by: a coarse factor of 1 means "search exactly as before". Having the knob proves + * the speed-up is a resolution choice and not a different algorithm wearing its name. + * + * Deliberately on a shorter clip than the budget tests. At the full 240 seconds this takes + * 137 seconds -- the cost the coarse pass exists to avoid -- and paying it on every run of + * the suite to re-prove something a 40-second clip proves just as well is the kind of thing + * that gets a test deleted rather than kept. + */ + @Test + fun `a coarse factor of one still finds the sting`() { + val first = episodeClip(seed = 4, stingAtSeconds = 8, clipSeconds = 40) + val second = episodeClip(seed = 5, stingAtSeconds = 24, clipSeconds = 40) + val tuning = IntroOutroDetector.Tuning(coarseFactor = 1) + + val waiting = IntroOutroDetector.check(FingerprintState(), "ep1", first, REFERENCE_SAMPLES, tuning = tuning) + val update = IntroOutroDetector.check(waiting.state, "ep2", second, REFERENCE_SAMPLES, tuning = tuning) + + assertTrue("expected a candidate at full resolution, got $update", update is FingerprintUpdate.Tentative) + } + + /** + * The coarse pass has to agree with full resolution about *where* the sting is, not just + * that there is one. A faster search that found a different segment would still report + * Tentative and would still be wrong -- and nothing downstream could tell. + */ + @Test + fun `the coarse search picks the same segment full resolution does`() { + val first = episodeClip(seed = 6, stingAtSeconds = 8, clipSeconds = 40) + val second = episodeClip(seed = 7, stingAtSeconds = 24, clipSeconds = 40) + + fun referenceWith(tuning: IntroOutroDetector.Tuning): DoubleArray { + val waiting = IntroOutroDetector.check(FingerprintState(), "ep1", first, REFERENCE_SAMPLES, tuning = tuning) + return IntroOutroDetector.check(waiting.state, "ep2", second, REFERENCE_SAMPLES, tuning = tuning) + .state.referenceClip!! + } + + val atFullResolution = referenceWith(IntroOutroDetector.Tuning(coarseFactor = 1)) + val viaCoarsePass = referenceWith(IntroOutroDetector.Tuning()) + + // Both searches step across the clip on the same native grid -- full resolution by + // win/4, the coarse pass by (win/factor)/4 scaled back up by factor, which is the same + // 16,000 samples -- so the window they pick is comparable sample for sample. Identical + // contents means the same position, and the position is what a skip offset comes from. + assertArrayEquals( + "coarse and full-resolution searches disagree about which segment is the sting", + atFullResolution, + viaCoarsePass, + 0.0, + ) + } +} diff --git a/app/src/test/java/org/terst/listen/fingerprint/IntroOutroDetectorTest.kt b/app/src/test/java/org/terst/listen/fingerprint/IntroOutroDetectorTest.kt index 91f2800..a9dfdbc 100644 --- a/app/src/test/java/org/terst/listen/fingerprint/IntroOutroDetectorTest.kt +++ b/app/src/test/java/org/terst/listen/fingerprint/IntroOutroDetectorTest.kt @@ -166,7 +166,7 @@ class IntroOutroDetectorTest { * confirmation genuinely depends on the threshold rather than being hardcoded to 3. */ @Test fun `a raised threshold keeps a third agreeing episode merely tentative`() { - val threshold = 4 + val threshold = IntroOutroDetector.Tuning(confirmThreshold = 4) var state = FingerprintState() state = IntroOutroDetector.check( state, "ep1", embed(noise(1, 300), theShowsSting, 50), REFERENCE_LENGTH, threshold, diff --git a/app/src/test/java/org/terst/listen/fingerprint/MidrollUpdaterTest.kt b/app/src/test/java/org/terst/listen/fingerprint/MidrollUpdaterTest.kt index e8e4474..ae9478e 100644 --- a/app/src/test/java/org/terst/listen/fingerprint/MidrollUpdaterTest.kt +++ b/app/src/test/java/org/terst/listen/fingerprint/MidrollUpdaterTest.kt @@ -3,6 +3,7 @@ package org.terst.listen.fingerprint import kotlin.random.Random import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test @@ -173,14 +174,76 @@ class MidrollUpdaterTest { assertEquals(fixedNow, episodes.rows.single().midrollCheckedAtMs) } + /** + * The decode a full-body scan needs is held entirely in memory, so an episode's length is + * also an allocation. An hour of 8kHz mono is 28.8 million doubles -- 230MB, on a phone with + * no largeHeap -- and before this the updater asked for it anyway: `decodeFull` threw + * OutOfMemoryError, `runCatching` swallowed it, and the episode was then marked checked + * forever, so no episode was ever scanned and nothing said why. + * + * Over budget now costs nothing and is left unchecked, so the slice-on-demand decode that + * lifts the limit picks these episodes up rather than finding them all written off. + */ + @Test + fun `an episode too long to hold in memory is never decoded`() = runTest { + show() + downloadedEpisode("marathon", downloadedAtMs = 1, durationMs = 3 * 60 * 60 * 1000L) + + updater.processShow("show1") + + assertEquals(emptyList<String>(), clips.decodeRequests) + } + + @Test + fun `an episode too long to hold in memory is left for later, not marked checked`() = runTest { + show() + downloadedEpisode("marathon", downloadedAtMs = 1, durationMs = 3 * 60 * 60 * 1000L) + + updater.processShow("show1") + + assertNull( + "an over-budget episode marked checked can never be scanned again", + episodes.rows.single { it.id == "marathon" }.midrollCheckedAtMs, + ) + } + + /** An over-budget episode must not become the pending basis either, or it blocks the show. */ + @Test + fun `an episode too long to hold in memory does not become the pending basis`() = runTest { + show() + downloadedEpisode("marathon", downloadedAtMs = 1, durationMs = 3 * 60 * 60 * 1000L) + + updater.processShow("show1") + + assertNull(subscriptions.rows.single().midrollPendingEpisodeId) + } + + /** A normal-length episode is unaffected -- the budget is a ceiling, not a new gate. */ + @Test + fun `an episode inside the budget is decoded as before`() = runTest { + show() + clips.full["e1"] = embed(noise(1, 3000), ad, at = 500) + downloadedEpisode("e1", downloadedAtMs = 1, durationMs = 600_000) + + updater.processShow("show1") + + assertEquals(listOf("e1"), clips.decodeRequests) + assertEquals(fixedNow, episodes.rows.single { it.id == "e1" }.midrollCheckedAtMs) + } + private class FakeFullClipSource : AudioClipSource { val full = mutableMapOf<String, DoubleArray>() val sampleRate = 8_000 + /** Which episodes were actually asked for, so "never attempted" can be asserted. */ + val decodeRequests = mutableListOf<String>() + override suspend fun decodeHead(episode: EpisodeEntity): DecodedClip? = null override suspend fun decodeTail(episode: EpisodeEntity): DecodedClip? = null - override suspend fun decodeFull(episode: EpisodeEntity): DecodedClip? = - full[episode.id]?.let { DecodedClip(it, sampleRate, startMs = 0) } + override suspend fun decodeFull(episode: EpisodeEntity): DecodedClip? { + decodeRequests += episode.id + return full[episode.id]?.let { DecodedClip(it, sampleRate, startMs = 0) } + } } } diff --git a/app/src/test/java/org/terst/listen/fingerprint/MonoResamplerTest.kt b/app/src/test/java/org/terst/listen/fingerprint/MonoResamplerTest.kt new file mode 100644 index 0000000..068585b --- /dev/null +++ b/app/src/test/java/org/terst/listen/fingerprint/MonoResamplerTest.kt @@ -0,0 +1,161 @@ +package org.terst.listen.fingerprint + +import kotlin.random.Random +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * The decode a full-episode scan needs, made to cost its output rather than its input. + * + * [MediaCodecClipSource] used to buffer the whole episode at its native rate -- a `ShortArray` + * of every interleaved sample, then a `DoubleArray` of every mono frame -- and only then + * resample to 8kHz. For a 13-minute stereo 44.1kHz episode that is 145MB plus 289MB live at + * once, on a phone with no largeHeap, to produce 6MB of output. It threw OutOfMemoryError, + * `runCatching` swallowed it, and midroll detection silently never ran on anything. + * + * This consumes the decoder's output chunk by chunk and keeps only the 8kHz mono result, so the + * peak allocation is the result. Being pure, it is also the half of that adapter that can + * actually be tested -- the rest is MediaCodec plumbing. + * + * The resampling is deliberately the same naive linear interpolation as before: measured at + * 0.9991 correlation against a proper ffmpeg/swresample decode of the same audio + * (`docs/detection-findings.md`), so it was never the weak link and this is not the change to + * start improving it in. + */ +class MonoResamplerTest { + + /** + * The batch formula the streaming one has to agree with, lifted from the implementation it + * replaces. Present as an oracle so the chunking is what gets tested -- the formula itself + * is pinned separately, by hand, in [matches hand-computed linear interpolation]. + */ + private fun batch(interleaved: ShortArray, channels: Int, inputRate: Int, targetRate: Int): DoubleArray { + val frames = interleaved.size / channels + val mono = DoubleArray(frames) { frame -> + var sum = 0.0 + for (c in 0 until channels) sum += interleaved[frame * channels + c] + sum / channels + } + if (inputRate == targetRate || mono.isEmpty()) return mono + + val ratio = targetRate.toDouble() / inputRate + val outLength = (mono.size * ratio).toInt() + return DoubleArray(outLength) { i -> + val sourcePosition = i / ratio + val index = sourcePosition.toInt().coerceIn(0, mono.size - 1) + val next = (index + 1).coerceAtMost(mono.size - 1) + val fraction = sourcePosition - index + mono[index] * (1 - fraction) + mono[next] * fraction + } + } + + private fun streamed( + interleaved: ShortArray, + channels: Int, + inputRate: Int, + targetRate: Int, + chunkFrames: Int, + ): DoubleArray { + val resampler = MonoResampler(inputRate = inputRate, channels = channels, targetRate = targetRate) + var offset = 0 + while (offset < interleaved.size) { + val count = minOf(chunkFrames * channels, interleaved.size - offset) + resampler.accept(interleaved, offset, count) + offset += count + } + return resampler.finish() + } + + @Test + fun `matches hand-computed linear interpolation`() { + // Mono, 4Hz in, 2Hz out: ratio 0.5, so output i reads source position 2i exactly and + // every fraction is zero -- every other sample, with no interpolation to obscure it. + val resampler = MonoResampler(inputRate = 4, channels = 1, targetRate = 2) + resampler.accept(shortArrayOf(10, 20, 30, 40), 0, 4) + + assertArrayEquals(doubleArrayOf(10.0, 30.0), resampler.finish(), 1e-9) + } + + @Test + fun `interpolates between neighbouring samples on a fractional position`() { + // 4Hz in, 3Hz out: ratio 0.75, so output 1 sits at source position 1.333... + val resampler = MonoResampler(inputRate = 4, channels = 1, targetRate = 3) + resampler.accept(shortArrayOf(0, 300, 600, 900), 0, 4) + + val out = resampler.finish() + assertEquals(3, out.size) + assertEquals(0.0, out[0], 1e-9) + assertEquals(300 * (1 - 1.0 / 3) + 600 * (1.0 / 3), out[1], 1e-9) + assertEquals(600 * (1 - 2.0 / 3) + 900 * (2.0 / 3), out[2], 1e-9) + } + + @Test + fun `averages channels down to mono`() { + val resampler = MonoResampler(inputRate = 8, channels = 2, targetRate = 8) + resampler.accept(shortArrayOf(10, 20, 100, 200), 0, 4) + + assertArrayEquals(doubleArrayOf(15.0, 150.0), resampler.finish(), 1e-9) + } + + /** + * The actual risk in streaming this: an output sample interpolates between two source + * frames, and those two can straddle a chunk boundary. A decoder hands over whatever buffer + * sizes it likes, so every split has to give the same answer as no split at all. + */ + @Test + fun `chunking the input does not change the output`() { + val random = Random(42) + val interleaved = ShortArray(20_000) { random.nextInt(-3000, 3000).toShort() } + val expected = batch(interleaved, channels = 2, inputRate = 44_100, targetRate = 8_000) + + for (chunkFrames in listOf(1, 2, 3, 7, 64, 999, 10_000)) { + assertArrayEquals( + "a $chunkFrames-frame chunking disagreed with the unchunked result", + expected, + streamed(interleaved, 2, 44_100, 8_000, chunkFrames), + 1e-9, + ) + } + } + + @Test + fun `chunking does not change the output when no resampling is needed`() { + val random = Random(7) + val interleaved = ShortArray(5_000) { random.nextInt(-3000, 3000).toShort() } + val expected = batch(interleaved, channels = 1, inputRate = 8_000, targetRate = 8_000) + + for (chunkFrames in listOf(1, 5, 333, 5_000)) { + assertArrayEquals(expected, streamed(interleaved, 1, 8_000, 8_000, chunkFrames), 1e-9) + } + } + + @Test + fun `upsampling agrees with the batch formula too`() { + val random = Random(9) + val interleaved = ShortArray(3_000) { random.nextInt(-3000, 3000).toShort() } + val expected = batch(interleaved, channels = 2, inputRate = 8_000, targetRate = 16_000) + + for (chunkFrames in listOf(1, 11, 1_500)) { + assertArrayEquals(expected, streamed(interleaved, 2, 8_000, 16_000, chunkFrames), 1e-9) + } + } + + @Test + fun `nothing accepted produces nothing`() { + assertEquals(0, MonoResampler(inputRate = 44_100, channels = 2, targetRate = 8_000).finish().size) + } + + /** + * The whole point, stated as a number: the result is sized by the output rate, so what the + * decoder was handed at its native rate is never all live at once. + */ + @Test + fun `the result is sized by the target rate, not the input rate`() { + val resampler = MonoResampler(inputRate = 44_100, channels = 2, targetRate = 8_000) + val oneSecondStereo = ShortArray(44_100 * 2) + resampler.accept(oneSecondStereo, 0, oneSecondStereo.size) + + assertEquals(8_000, resampler.finish().size) + } +} diff --git a/app/src/test/java/org/terst/listen/fingerprint/SkipSummaryTest.kt b/app/src/test/java/org/terst/listen/fingerprint/SkipSummaryTest.kt index bd10ece..efb01ae 100644 --- a/app/src/test/java/org/terst/listen/fingerprint/SkipSummaryTest.kt +++ b/app/src/test/java/org/terst/listen/fingerprint/SkipSummaryTest.kt @@ -247,4 +247,98 @@ class SkipSummaryTest { assertEquals(null, skips.insertedAds) } + + /** + * The sanity check Peter asked for, in one place: what the feed claims, what the file turned + * out to be, and how much of the gap the detector has actually accounted for. + * + * The three numbers only mean anything together. `insertedAds` says there are seven and a + * half minutes of advertising in Grass Apes; `breaksTotal` saying thirty seconds means the + * midroll scan is missing breaks, and saying nine minutes means it is matching programme. + * Either way the readout has to show both or it cannot be checked at all. + */ + @Test + fun `reports the declared and measured lengths side by side`() { + val skips = SkipSummary.episode( + episodeRow(durationMs = 1_325_000, actualDurationMs = 1_774_000), + showRow(), + emptyList(), + ) + + assertEquals("22:05", skips.declaredLength) + assertEquals("29:34", skips.measuredLength) + } + + /** An episode the player has never opened has no measured length to report. */ + @Test + fun `an unplayed episode reports only the declared length`() { + val skips = SkipSummary.episode(episodeRow(durationMs = 1_325_000), showRow(), emptyList()) + + assertEquals("22:05", skips.declaredLength) + assertEquals(null, skips.measuredLength) + } + + @Test + fun `totals up the detected ad breaks so they can be checked against the inserted ads`() { + val skips = SkipSummary.episode( + episodeRow(durationMs = 1_325_000, actualDurationMs = 1_774_000, midrollCheckedAtMs = 1), + showRow(), + listOf( + EpisodeMidrollEntity("e1", "c1", startMs = 120_000, endMs = 180_000), + EpisodeMidrollEntity("e1", "c2", startMs = 600_000, endMs = 690_000), + ), + ) + + // A minute plus ninety seconds, against the 7:29 the length gap implies. + assertEquals("2:30", skips.breaksTotal) + } + + /** Nothing detected is not a total of zero -- there is no figure to compare yet. */ + @Test + fun `no detected breaks means no total`() { + val skips = SkipSummary.episode( + episodeRow(durationMs = 1_325_000, actualDurationMs = 1_774_000, midrollCheckedAtMs = 1), + showRow(), + emptyList(), + ) + + assertEquals(null, skips.breaksTotal) + } + + /** + * Where the detected breaks sit, as fractions of the episode, so the scrubber can mark them. + * Computed here rather than in the composable for the usual reason: an off-by-a-factor that + * draws a marker in the wrong place is invisible in a screenshot and obvious in a test. + */ + @Test + fun `places detected breaks on the timeline as fractions of the measured length`() { + val markers = SkipSummary.breakMarkers( + listOf(EpisodeMidrollEntity("e1", "c1", startMs = 300_000, endMs = 600_000)), + durationMs = 1_200_000, + ) + + assertEquals(1, markers.size) + assertEquals(0.25f, markers.single().start, 1e-6f) + assertEquals(0.5f, markers.single().end, 1e-6f) + } + + @Test + fun `a break running past the end of the episode is clamped to it`() { + val markers = SkipSummary.breakMarkers( + listOf(EpisodeMidrollEntity("e1", "c1", startMs = 1_100_000, endMs = 9_000_000)), + durationMs = 1_200_000, + ) + + assertEquals(1.0f, markers.single().end, 1e-6f) + } + + @Test + fun `an unknown length places no markers`() { + val markers = SkipSummary.breakMarkers( + listOf(EpisodeMidrollEntity("e1", "c1", startMs = 1, endMs = 2)), + durationMs = null, + ) + + assertEquals(emptyList<SkipSummary.BreakMarker>(), markers) + } } diff --git a/app/src/test/java/org/terst/listen/playback/PrepareTraceTest.kt b/app/src/test/java/org/terst/listen/playback/PrepareTraceTest.kt new file mode 100644 index 0000000..b5b1d75 --- /dev/null +++ b/app/src/test/java/org/terst/listen/playback/PrepareTraceTest.kt @@ -0,0 +1,142 @@ +package org.terst.listen.playback + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +/** + * The measurement Peter's third report needs: "a longish pause when starting playback, + * proportional to episode length". + * + * Nothing in the app's own play path scales with an episode's length -- it is database reads and + * then `setMediaItem`/`prepare` -- so the candidates are Media3's own preparation and the CPU + * being saturated by something else. Those are indistinguishable from the outside, and guessing + * between them is how a fix gets shipped for the wrong one. + * + * So this emits one line per start carrying the length *and* the elapsed stages, which answers + * "is it proportional?" directly from a log rather than from an argument. The line is built here + * rather than in the service because a format that silently dropped the length would make the + * whole exercise pointless, and that is worth a test. + */ +class PrepareTraceTest { + + private var clock = 0L + private fun trace() = PrepareTrace { clock } + + @Test + fun `reports each stage as elapsed milliseconds since the start`() { + val trace = trace() + clock = 1_000 + trace.start(episodeId = "ep1", durationMs = 1_774_000, local = true) + clock = 1_812 + trace.mark(PrepareTrace.Stage.READY) + clock = 1_903 + trace.mark(PrepareTrace.Stage.PLAYING) + + assertEquals( + "prepare ep=ep1 len=1774s src=local ready=812ms playing=903ms", + trace.summary(), + ) + } + + /** A streamed episode is a different question from a downloaded one, so the line says which. */ + @Test + fun `distinguishes a streamed episode from a downloaded one`() { + val trace = trace() + clock = 500 + trace.start(episodeId = "ep2", durationMs = 60_000, local = false) + clock = 2_500 + trace.mark(PrepareTrace.Stage.READY) + + assertEquals("prepare ep=ep2 len=60s src=stream ready=2000ms", trace.summary()) + } + + /** An unknown length still traces: that it was unknown is itself worth seeing. */ + @Test + fun `reports an unknown length as unknown`() { + val trace = trace() + trace.start(episodeId = "ep3", durationMs = null, local = true) + clock = 40 + trace.mark(PrepareTrace.Stage.READY) + + assertEquals("prepare ep=ep3 len=? src=local ready=40ms", trace.summary()) + } + + @Test + fun `nothing to report before a start`() { + assertNull(trace().summary()) + } + + /** + * Media3 reports READY more than once for one item -- a rebuffer is another READY. Only the + * first is the preparation being measured; later ones would overwrite it with a small + * number and hide exactly the delay being chased. + */ + @Test + fun `keeps the first time a stage was reached, not the last`() { + val trace = trace() + trace.start(episodeId = "ep4", durationMs = 1_000, local = true) + clock = 900 + trace.mark(PrepareTrace.Stage.READY) + clock = 5_000 + trace.mark(PrepareTrace.Stage.READY) + + assertEquals("prepare ep=ep4 len=1s src=local ready=900ms", trace.summary()) + } + + /** A new start is a new measurement; the previous episode's stages must not bleed into it. */ + @Test + fun `starting again clears the previous episode's stages`() { + val trace = trace() + trace.start(episodeId = "ep5", durationMs = 1_000, local = true) + clock = 700 + trace.mark(PrepareTrace.Stage.READY) + + clock = 1_000 + trace.start(episodeId = "ep6", durationMs = 2_000, local = true) + + assertEquals("prepare ep=ep6 len=2s src=local", trace.summary()) + } + + /** + * The clock has to start when the media item is set, which is the only hook both play paths + * go through -- the service's own queue advance and a tap in the UI, which reaches the + * player through a MediaController and never calls the service's `play`. A MediaItem carries + * no duration, so the length arrives later, from the player, once it has opened the file. + * + * That length is also the better of the two available: it is what the file really is, not + * what the feed claimed, and on a show using dynamic ad insertion those differ by minutes. + */ + @Test + fun `a length supplied after the start still appears in the line`() { + val trace = trace() + clock = 1_000 + trace.start(episodeId = "ep7", durationMs = null, local = true) + clock = 1_500 + trace.length(1_774_000) + trace.mark(PrepareTrace.Stage.READY) + + assertEquals("prepare ep=ep7 len=1774s src=local ready=500ms", trace.summary()) + } + + /** A length learned for a previous episode must not be reported for the next one. */ + @Test + fun `a length does not survive into the next episode`() { + val trace = trace() + trace.start(episodeId = "ep8", durationMs = null, local = true) + trace.length(600_000) + trace.start(episodeId = "ep9", durationMs = null, local = true) + + assertEquals("prepare ep=ep9 len=? src=local", trace.summary()) + } + + /** Media3 reports no duration until it has one; that must not overwrite a real length. */ + @Test + fun `an unset length is ignored rather than clearing a known one`() { + val trace = trace() + trace.start(episodeId = "ep10", durationMs = 60_000, local = true) + trace.length(null) + + assertEquals("prepare ep=ep10 len=60s src=local", trace.summary()) + } +} |