From 9210bf463a8818595f7ca42486a787d4f1941d55 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Wed, 21 Oct 2026 19:07:17 +0000 Subject: Fix stray promo headers, site-URL subscribing, and YouTube playback Three reported bugs. **"West Hawaii today articles get a bunch of stray Trending headers."** The publisher's widget platform injects promo blocks between the real paragraphs, inside the article's own container, so every structural signal tells Readability they belong. What survives is a heading labelling nothing, because the links under it are filled in by a script that never runs in the reader. Confirmed on 9 of the 12 articles in that feed, two each. Blocks marked `data-mrf-recirculation` are now removed -- the attribute says what the block is for in the widget platform's own words, which is a better signal than the word "Trending", since an article may legitimately discuss what is trending. Nothing longer than a label and a few headlines is ever removed. Also applied to already-cached articles, in place rather than by discarding them: the article was fine, only the litter has to go. **"Can't subscribe to heathercoxrichardson.substack.com."** Adding a feed handed whatever URL it was given straight to the XML parser, so a site's home page -- the only URL a reader can actually see and copy -- failed on the root element. That page advertises its feed the way sites have since 2002, in a in the head. It is now read, resolved against the page, and fetched. Only what the page declares: guessing /feed and /rss would add a burst of 404s to every failed add and eventually subscribe somebody to the wrong thing. **"YouTube play is misconfigured."** It was, and YouTube said so: the player dialog loaded the embed as its own top-level document, which answers with "Video player configuration error -- Error 153". An embed has to be embedded. It now sits in an iframe on a minimal host page. Measured in the app's own WebView: top-level, no requests for the video stream and the error on screen; framed, the stream loads and there is no error. **"Long click on YouTube placeholder should open the URL directly."** It does now. Worth recording why this is not simply the link's href: for an image inside a link, WebView's hit test reports the image's URL, so a long-press on the thumbnail arrives as the i.ytimg.com JPEG. The video id is resolved out of that. Verified on a device for all four: the Substack site URL adds "Letters from an American" and fetches 20 articles; a West Hawaii Today article extracts with no Trending headers and its body intact; the video plays; and a long-press reaches the YouTube app. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- .../java/org/terst/feedreader/ui/AddFeedScreen.kt | 65 +++++++++++++++++----- 1 file changed, 51 insertions(+), 14 deletions(-) (limited to 'app/src/main/java/org/terst/feedreader/ui/AddFeedScreen.kt') diff --git a/app/src/main/java/org/terst/feedreader/ui/AddFeedScreen.kt b/app/src/main/java/org/terst/feedreader/ui/AddFeedScreen.kt index 1e8814c..9f817ab 100644 --- a/app/src/main/java/org/terst/feedreader/ui/AddFeedScreen.kt +++ b/app/src/main/java/org/terst/feedreader/ui/AddFeedScreen.kt @@ -33,11 +33,13 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import okhttp3.Request import org.terst.feedreader.data.AppDatabase +import org.terst.feedreader.data.FeedDiscovery import org.terst.feedreader.data.FeedEntity import org.terst.feedreader.data.FeedParser import org.terst.feedreader.data.Net import org.terst.feedreader.opml.Opml import org.terst.feedreader.work.FeedFetchWorker +import java.io.ByteArrayInputStream @OptIn(ExperimentalMaterial3Api::class) @Composable @@ -168,25 +170,60 @@ fun AddFeedScreen(onDone: () -> Unit) { private fun preferHttps(url: String): String = if (url.startsWith("http://", ignoreCase = true)) "https://" + url.removePrefix("http://") else url +/** + * Fetches [url] and returns its bytes, or null when the server refused. + * + * Read into memory rather than parsed from the live stream because the bytes may need looking + * at twice: once as a feed, and -- when that is not what the URL turned out to be -- again as a + * web page advertising where its feed actually lives. A response body is a one-shot stream, so + * the second look is only possible if the first did not consume it. + */ +private class Fetched(val bytes: ByteArray?, val error: String?) + +private fun fetchBytes(url: String): Fetched = + Net.client.newCall(Request.Builder().url(url).build()).execute().use { response -> + when { + !response.isSuccessful -> Fetched(null, "Couldn't fetch feed: HTTP ${response.code}") + else -> response.body?.let { Fetched(it.bytes(), null) } ?: Fetched(null, "Empty response") + } + } + private suspend fun addFeedByUrl(context: android.content.Context, db: AppDatabase, feedUrl: String): String = withContext(Dispatchers.IO) { - val url = preferHttps(feedUrl) + val typedUrl = preferHttps(feedUrl) try { - val response = Net.client.newCall(Request.Builder().url(url).build()).execute() - response.use { - if (!it.isSuccessful) return@withContext "Couldn't fetch feed: HTTP ${it.code}" - val body = it.body ?: return@withContext "Empty response" - val parsed = FeedParser.parse(body.byteStream()) - // Same REPLACE-wipes-other-fields concern as the OPML import path. - val existing = db.feedDao().getById(url) - val entity = existing?.copy( - title = parsed.title.ifBlank { url }, - siteUrl = parsed.siteUrl ?: existing.siteUrl - ) ?: FeedEntity(id = url, title = parsed.title.ifBlank { url }, siteUrl = parsed.siteUrl) - db.feedDao().upsert(entity) + val fetched = fetchBytes(typedUrl) + val bytes = fetched.bytes ?: return@withContext fetched.error ?: "Empty response" + + // What the user had to hand is usually the site, not the feed -- a site advertises + // its feed in its head, it does not print the URL anywhere a reader can copy. So a + // failed parse is not the answer yet: look for what the page says its feed is and + // try that, once. See FeedDiscovery. + var url = typedUrl + var parsed = runCatching { FeedParser.parse(ByteArrayInputStream(bytes)) }.getOrNull() + if (parsed == null) { + val discovered = FeedDiscovery.findFeedUrl(String(bytes, Charsets.UTF_8), typedUrl) + ?: return@withContext "No feed found at that address" + val feed = fetchBytes(discovered) + val feedBytes = feed.bytes ?: return@withContext feed.error ?: "Empty response" + // One hop only: whatever the page pointed at has to be a feed. Following a + // second page would be a redirect chain nobody asked for. + parsed = FeedParser.parse(ByteArrayInputStream(feedBytes)) + url = discovered } + + // Same REPLACE-wipes-other-fields concern as the OPML import path. + val existing = db.feedDao().getById(url) + val entity = existing?.copy( + title = parsed.title.ifBlank { url }, + siteUrl = parsed.siteUrl ?: existing.siteUrl + ) ?: FeedEntity(id = url, title = parsed.title.ifBlank { url }, siteUrl = parsed.siteUrl) + db.feedDao().upsert(entity) + FeedFetchWorker.fetchNow(context) - "Added" + // Says which feed was added when it is not the URL that was typed, so a site URL + // quietly becoming a different feed URL is visible rather than surprising later. + if (url == typedUrl) "Added" else "Added ${parsed.title.ifBlank { url }}" } catch (e: Exception) { "Couldn't add feed: ${e.message}" } -- cgit v1.2.3