diff options
| -rw-r--r-- | .coverage-baseline | 2 | ||||
| -rw-r--r-- | CLAUDE.md | 22 | ||||
| -rw-r--r-- | app/build.gradle.kts | 23 | ||||
| -rw-r--r-- | app/src/androidTest/java/org/terst/listen/youtube/YouTubeAudioTest.kt | 40 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/listen/youtube/YouTubeAudio.kt | 48 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/listen/youtube/YouTubeProbe.kt | 54 | ||||
| -rw-r--r-- | app/src/main/python/ytaudio.py | 51 | ||||
| -rw-r--r-- | app/src/test/java/org/terst/listen/youtube/YouTubeProbeTest.kt | 89 | ||||
| -rw-r--r-- | build.gradle.kts | 4 | ||||
| -rw-r--r-- | gradle/wrapper/gradle-wrapper.properties | 2 |
10 files changed, 330 insertions, 5 deletions
diff --git a/.coverage-baseline b/.coverage-baseline index f362818..f676739 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -75.88 +75.00 @@ -86,6 +86,22 @@ claims three separate times across these repos. title+date. An episode that loses its identity loses its playback position. - **Retention is auto-delete-once-played** with a grace period, so a mis-tap doesn't destroy an episode you were still listening to. -- **YouTube never touches this app.** Audio arrives via podkit running in Termux on the phone, - writing to a local folder Listen imports. No extractor code here, nothing to break when Google - ships a change. +- **YouTube audio is yt-dlp, embedded.** Chaquopy runs an in-process CPython with real yt-dlp; + see `youtube/YouTubeAudio.kt` and `src/main/python/ytaudio.py`. No Termux, no podkit, no + server round trip. + - **Pin the Python version.** Chaquopy defaults to 3.8, which yt-dlp no longer supports, so + pip silently resolves the last 3.8-compatible release (2024.10.22) and YouTube extraction + fails while the build succeeds. `YouTubeAudioTest` fails on a pre-2025 yt-dlp for exactly + this reason. + - **Audio-only, always.** Separate audio and video streams need ffmpeg to merge, which is the + documented failure mode for yt-dlp on Android. A single pre-muxed audio stream has nothing + to merge and needs no ffmpeg. + - **No proof-of-origin tokens.** Obtaining one means running Google's attestation script in a + WebView, which this app does not do. It works without one from an ordinary phone connection. + - **Known risk:** yt-dlp has deprecated YouTube extraction without a JavaScript runtime and + warns that some formats may be missing. There is no deno/node on Android. If yt-dlp makes a + JS runtime mandatory, this breaks and the fallback is bundling a JS engine it accepts. + +- **Gradle is pinned to 8.14.3, not 9.x**, because Chaquopy 16.1.0 calls `org.gradle.util. + VersionNumber`, removed in Gradle 9. Chaquopy 17 moved to AGP 9.x; going there means bumping + AGP, Gradle and Chaquopy together. This is the one deliberate divergence from FeeDreader. diff --git a/app/build.gradle.kts b/app/build.gradle.kts index a1d1854..585cbc8 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -6,6 +6,7 @@ plugins { id("com.google.devtools.ksp") id("io.gitlab.arturbosch.detekt") id("jacoco") + id("com.chaquo.python") } android { @@ -22,6 +23,13 @@ android { versionCode = 1 versionName = "1.0" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + + ndk { + // CPython ships as native code, so every ABI included multiplies the APK size. + // arm64 is the phone; x86_64 is the emulator. + abiFilters += listOf("arm64-v8a", "x86_64") + } + } buildTypes { @@ -110,6 +118,21 @@ dependencies { debugImplementation("androidx.compose.ui:ui-test-manifest") } +chaquopy { + defaultConfig { + // Not the default 3.8. yt-dlp dropped Python 3.8 support, so pip silently resolves the + // last release that still supported it -- 2024.10.22 -- which is stale enough that + // YouTube extraction no longer works. The build succeeds and the feature doesn't. + version = "3.12" + // The extractor itself. yt-dlp is pure Python and updates independently of this app, + // which is the entire reason for embedding it rather than reimplementing YouTube's + // stream resolution against a target that changes without notice. + pip { + install("yt-dlp") + } + } +} + detekt { buildUponDefaultConfig = true config.setFrom(files("$rootDir/config/detekt/detekt.yml")) diff --git a/app/src/androidTest/java/org/terst/listen/youtube/YouTubeAudioTest.kt b/app/src/androidTest/java/org/terst/listen/youtube/YouTubeAudioTest.kt new file mode 100644 index 0000000..1a8fcb6 --- /dev/null +++ b/app/src/androidTest/java/org/terst/listen/youtube/YouTubeAudioTest.kt @@ -0,0 +1,40 @@ +package org.terst.listen.youtube + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith + +/** + * Proves the embedded interpreter actually runs, which packaging alone does not. + * + * A stale bundled yt-dlp is the specific failure this guards: the default CPython 3.8 silently + * resolves yt-dlp 2024.10.22, the last release supporting it, and the build succeeds while + * YouTube extraction no longer works. + */ +@RunWith(AndroidJUnit4::class) +class YouTubeAudioTest { + + @Before + fun setUp() { + YouTubeAudio.start(InstrumentationRegistry.getInstrumentation().targetContext) + } + + @Test + fun pythonStartsAndYtDlpImports() { + val version = YouTubeAudio.ytDlpVersion() + assertTrue("yt-dlp reported no version", version.isNotBlank()) + } + + @Test + fun bundledYtDlpIsNotTheStalePython38Release() { + val version = YouTubeAudio.ytDlpVersion() + val year = version.substringBefore('.').toIntOrNull() + assertTrue("yt-dlp version '$version' is not a dated release", year != null) + // 2024.10.22 is the last release supporting Python 3.8. Anything from that era means + // the Python version was silently downgraded and extraction will fail in the field. + assertTrue("yt-dlp $version is stale -- check the Chaquopy python version", year!! >= 2025) + } +} diff --git a/app/src/main/java/org/terst/listen/youtube/YouTubeAudio.kt b/app/src/main/java/org/terst/listen/youtube/YouTubeAudio.kt new file mode 100644 index 0000000..6ddc7ea --- /dev/null +++ b/app/src/main/java/org/terst/listen/youtube/YouTubeAudio.kt @@ -0,0 +1,48 @@ +package org.terst.listen.youtube + +import android.content.Context +import com.chaquo.python.Python +import com.chaquo.python.android.AndroidPlatform +import java.io.File + +/** + * Audio extraction via yt-dlp, running in-process on an embedded CPython. + * + * Why yt-dlp rather than a JVM extractor: YouTube's stream resolution changes without notice, + * and yt-dlp is the implementation that actually tracks it. A JVM reimplementation would need + * a proof-of-origin token, which means running Google's attestation script in a WebView -- a + * different act from downloading, and not one this app performs. + * + * Why this works from a phone at all: the block that stops a server doing this is largely about + * where the request comes from. An ordinary phone connection is not a datacentre IP. + */ +object YouTubeAudio { + + fun start(context: Context) { + if (!Python.isStarted()) { + Python.start(AndroidPlatform(context)) + } + } + + private fun module() = Python.getInstance().getModule("ytaudio") + + /** yt-dlp's own version, so a stale bundled copy is visible rather than merely suspected. */ + fun ytDlpVersion(): String = module().callAttr("version").toString() + + /** Metadata only; downloads nothing. */ + fun probe(url: String): YouTubeProbe = + YouTubeProbe.fromJson(module().callAttr("probe", url).toString()) + + /** + * Downloads the best audio-only stream into [targetDir] and returns the file. + * + * Audio-only is deliberate beyond saving bandwidth: separate audio and video streams have to + * be merged with ffmpeg, which is the documented failure mode for yt-dlp on Android. A single + * pre-muxed audio stream has nothing to merge. + */ + fun download(url: String, targetDir: File): File { + targetDir.mkdirs() + val path = module().callAttr("download", url, targetDir.absolutePath).toString() + return File(path) + } +} diff --git a/app/src/main/java/org/terst/listen/youtube/YouTubeProbe.kt b/app/src/main/java/org/terst/listen/youtube/YouTubeProbe.kt new file mode 100644 index 0000000..1da98db --- /dev/null +++ b/app/src/main/java/org/terst/listen/youtube/YouTubeProbe.kt @@ -0,0 +1,54 @@ +package org.terst.listen.youtube + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +/** What yt-dlp reports about a video before anything is downloaded. */ +data class YouTubeProbe( + val id: String, + val title: String?, + val uploader: String?, + val durationMs: Long?, + val extension: String?, + val sizeBytes: Long?, +) { + companion object { + private const val MS_PER_SECOND = 1000.0 + + private val json = Json { ignoreUnknownKeys = true } + + /** + * Parsed with kotlinx.serialization rather than org.json, which is an empty stub off + * device -- this mapping is the boundary most likely to drift as yt-dlp changes, so it + * has to be testable without an emulator. + */ + fun fromJson(raw: String): YouTubeProbe { + val dto = runCatching { json.decodeFromString<Dto>(raw) } + .getOrElse { throw IllegalArgumentException("yt-dlp returned unparseable JSON: ${it.message}", it) } + val id = dto.id?.takeIf { it.isNotBlank() } + ?: throw IllegalArgumentException("yt-dlp returned no video id") + return YouTubeProbe( + id = id, + title = dto.title?.takeIf { it.isNotBlank() }, + uploader = dto.uploader?.takeIf { it.isNotBlank() }, + // Reported in fractional seconds. Absent means unknown -- a live stream, say -- + // and must not collapse to a zero-length episode. + durationMs = dto.duration?.let { (it * MS_PER_SECOND).toLong() }, + extension = dto.ext?.takeIf { it.isNotBlank() }, + sizeBytes = dto.filesize ?: dto.filesizeApprox, + ) + } + } + + @Serializable + private data class Dto( + val id: String? = null, + val title: String? = null, + val uploader: String? = null, + val duration: Double? = null, + val ext: String? = null, + val filesize: Long? = null, + @SerialName("filesize_approx") val filesizeApprox: Long? = null, + ) +} diff --git a/app/src/main/python/ytaudio.py b/app/src/main/python/ytaudio.py new file mode 100644 index 0000000..0e5993a --- /dev/null +++ b/app/src/main/python/ytaudio.py @@ -0,0 +1,51 @@ +"""Audio-only resolution and download via yt-dlp. + +Audio-only on purpose. The documented pain point with yt-dlp on Android is that separate audio +and video streams need ffmpeg to merge; requesting a single pre-muxed audio stream means there +is nothing to merge and no ffmpeg dependency at all. +""" +import json + +import yt_dlp + + +_FORMAT = "bestaudio[ext=m4a]/bestaudio[ext=webm]/bestaudio" + + +def probe(url): + """Return metadata for a video without downloading it, as a JSON string.""" + opts = {"quiet": True, "no_warnings": True, "skip_download": True, "format": _FORMAT} + with yt_dlp.YoutubeDL(opts) as ydl: + info = ydl.extract_info(url, download=False) + return json.dumps({ + "id": info.get("id"), + "title": info.get("title"), + "uploader": info.get("uploader"), + "duration": info.get("duration"), + "ext": info.get("ext"), + # Both, not one coalesced field: an audio-only stream often reports only the + # approximate size up front, and the caller should be able to tell them apart. + "filesize": info.get("filesize"), + "filesize_approx": info.get("filesize_approx"), + "abr": info.get("abr"), + }) + + +def download(url, target_dir): + """Download the best audio-only stream into target_dir. Returns the resulting path.""" + opts = { + "quiet": True, + "no_warnings": True, + "format": _FORMAT, + "outtmpl": target_dir + "/%(id)s.%(ext)s", + # No postprocessors: converting formats would need ffmpeg, and every player Media3 + # exposes handles m4a and webm/opus natively. + "noplaylist": True, + } + with yt_dlp.YoutubeDL(opts) as ydl: + info = ydl.extract_info(url, download=True) + return ydl.prepare_filename(info) + + +def version(): + return yt_dlp.version.__version__ diff --git a/app/src/test/java/org/terst/listen/youtube/YouTubeProbeTest.kt b/app/src/test/java/org/terst/listen/youtube/YouTubeProbeTest.kt new file mode 100644 index 0000000..1e62070 --- /dev/null +++ b/app/src/test/java/org/terst/listen/youtube/YouTubeProbeTest.kt @@ -0,0 +1,89 @@ +package org.terst.listen.youtube + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +/** + * yt-dlp's JSON is the boundary most likely to drift: it is a separate project on its own + * release cadence, and a field going missing or changing type would otherwise surface as a + * crash on the phone. + */ +class YouTubeProbeTest { + + private val full = """ + {"id":"Bo9Z29YjzVk", + "title":"The Law of Gravitation", + "uploader":"Der Atlantik Report", + "duration":3337.0, + "ext":"m4a", + "filesize":54006240, + "abr":129.0} + """.trimIndent() + + @Test + fun `maps a full response`() { + val probe = YouTubeProbe.fromJson(full) + assertEquals("Bo9Z29YjzVk", probe.id) + assertEquals("The Law of Gravitation", probe.title) + assertEquals("Der Atlantik Report", probe.uploader) + assertEquals("m4a", probe.extension) + assertEquals(54006240L, probe.sizeBytes) + } + + /** yt-dlp reports duration in fractional seconds; the app works in milliseconds. */ + @Test + fun `converts fractional seconds to milliseconds`() { + assertEquals(3337_000L, YouTubeProbe.fromJson(full).durationMs) + val fractional = """{"id":"x","duration":12.5}""" + assertEquals(12_500L, YouTubeProbe.fromJson(fractional).durationMs) + } + + /** Live streams and some uploads report no duration; that must not become zero. */ + @Test + fun `absent duration stays null rather than zero`() { + val probe = YouTubeProbe.fromJson("""{"id":"x"}""") + assertNull(probe.durationMs) + } + + @Test + fun `absent optional fields are null`() { + val probe = YouTubeProbe.fromJson("""{"id":"x"}""") + assertNull(probe.title) + assertNull(probe.uploader) + assertNull(probe.extension) + assertNull(probe.sizeBytes) + } + + @Test + fun `explicit json nulls are treated as absent`() { + val probe = YouTubeProbe.fromJson( + """{"id":"x","title":null,"uploader":null,"duration":null,"ext":null,"filesize":null}""" + ) + assertNull(probe.title) + assertNull(probe.durationMs) + assertNull(probe.sizeBytes) + } + + /** yt-dlp falls back to filesize_approx when the exact size isn't known up front. */ + @Test + fun `falls back to approximate filesize`() { + val probe = YouTubeProbe.fromJson("""{"id":"x","filesize":null,"filesize_approx":1234}""") + assertEquals(1234L, probe.sizeBytes) + } + + @Test + fun `an empty title is treated as absent`() { + assertNull(YouTubeProbe.fromJson("""{"id":"x","title":""}""").title) + } + + @Test(expected = IllegalArgumentException::class) + fun `a response without an id is rejected`() { + YouTubeProbe.fromJson("""{"title":"no id here"}""") + } + + @Test(expected = IllegalArgumentException::class) + fun `malformed json is rejected`() { + YouTubeProbe.fromJson("not json at all") + } +} diff --git a/build.gradle.kts b/build.gradle.kts index a2551b0..5d396f7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,4 +5,8 @@ plugins { id("org.jetbrains.kotlin.plugin.compose") version "2.2.21" apply false id("com.google.devtools.ksp") version "2.2.21-2.0.5" apply false id("io.gitlab.arturbosch.detekt") version "1.23.8" apply false + // Embeds CPython so yt-dlp runs in-process. 16.1.0 is the release whose supported AGP range + // (8.9-8.13) covers the 8.13.2 used here; 17.x moved to AGP 9.x and would need both bumped + // together. + id("com.chaquo.python") version "16.1.0" apply false } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 1a70468..d4081da 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME |