From 9210bf463a8818595f7ca42486a787d4f1941d55 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Wed, 21 Oct 2026 19:07:17 +0000 Subject: Fix stray promo headers, site-URL subscribing, and YouTube playback Three reported bugs. **"West Hawaii today articles get a bunch of stray Trending headers."** The publisher's widget platform injects promo blocks between the real paragraphs, inside the article's own container, so every structural signal tells Readability they belong. What survives is a heading labelling nothing, because the links under it are filled in by a script that never runs in the reader. Confirmed on 9 of the 12 articles in that feed, two each. Blocks marked `data-mrf-recirculation` are now removed -- the attribute says what the block is for in the widget platform's own words, which is a better signal than the word "Trending", since an article may legitimately discuss what is trending. Nothing longer than a label and a few headlines is ever removed. Also applied to already-cached articles, in place rather than by discarding them: the article was fine, only the litter has to go. **"Can't subscribe to heathercoxrichardson.substack.com."** Adding a feed handed whatever URL it was given straight to the XML parser, so a site's home page -- the only URL a reader can actually see and copy -- failed on the root element. That page advertises its feed the way sites have since 2002, in a in the head. It is now read, resolved against the page, and fetched. Only what the page declares: guessing /feed and /rss would add a burst of 404s to every failed add and eventually subscribe somebody to the wrong thing. **"YouTube play is misconfigured."** It was, and YouTube said so: the player dialog loaded the embed as its own top-level document, which answers with "Video player configuration error -- Error 153". An embed has to be embedded. It now sits in an iframe on a minimal host page. Measured in the app's own WebView: top-level, no requests for the video stream and the error on screen; framed, the stream loads and there is no error. **"Long click on YouTube placeholder should open the URL directly."** It does now. Worth recording why this is not simply the link's href: for an image inside a link, WebView's hit test reports the image's URL, so a long-press on the thumbnail arrives as the i.ytimg.com JPEG. The video id is resolved out of that. Verified on a device for all four: the Substack site URL adds "Letters from an American" and fetches 20 articles; a West Hawaii Today article extracts with no Trending headers and its body intact; the video plays; and a long-press reaches the YouTube app. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- .../org/terst/feedreader/data/CachedExtraction.kt | 10 ++- .../org/terst/feedreader/data/FeedDiscovery.kt | 88 ++++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 app/src/main/java/org/terst/feedreader/data/FeedDiscovery.kt (limited to 'app/src/main/java/org/terst/feedreader/data') diff --git a/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt b/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt index 3a791cc..63614d1 100644 --- a/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt +++ b/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt @@ -1,6 +1,7 @@ package org.terst.feedreader.data import org.terst.feedreader.readability.ConsentNotice +import org.terst.feedreader.readability.RecirculationWidgets import org.terst.feedreader.readability.UnrenderedTex /** What a cached `articles.extractedContent` value turned out to be worth. */ @@ -58,7 +59,14 @@ suspend fun loadCachedExtraction( discardedUnrenderedTex = true ) else -> null - } ?: return CachedExtraction(content = cached, discardedConsentNotice = false) + } ?: return CachedExtraction( + // Recirculation widgets are deliberately not in the `discard` list above. A cached + // article carrying the publisher's promo blocks is a good article with litter in it, + // not a poisoned row: cleaning it in place shows the right thing immediately, where + // discarding it would throw away a usable body and fetch the identical one back. + content = RecirculationWidgets.strip(cached), + discardedConsentNotice = false + ) clearCache() return discard diff --git a/app/src/main/java/org/terst/feedreader/data/FeedDiscovery.kt b/app/src/main/java/org/terst/feedreader/data/FeedDiscovery.kt new file mode 100644 index 0000000..98ce5b3 --- /dev/null +++ b/app/src/main/java/org/terst/feedreader/data/FeedDiscovery.kt @@ -0,0 +1,88 @@ +package org.terst.feedreader.data + +import java.net.URI + +/** + * Finds the feed a web page advertises. + * + * Reported live: "Can't subscribe to https://heathercoxrichardson.substack.com/". Adding a feed + * fetched whatever URL it was given and handed it straight to [FeedParser], so a site's home + * page -- the only URL a reader can actually see and copy -- failed on the XML parse with a + * message about an unrecognised root element. + * + * Nothing exotic is needed to fix it. Sites have advertised their feeds in the document head + * since RSS autodiscovery was agreed in 2002, and that Substack page does: + * + * ``` + * + * ``` + * + * Only what the page itself declares is used. Guessing paths (`/feed`, `/rss`, `/index.xml`) + * would add a burst of 404s on every failed add and would eventually subscribe somebody to the + * wrong thing; a site that declares nothing is reported as having no feed, which is the truth. + */ +object FeedDiscovery { + + /** Feed content types, as `type=` gives them. */ + private val FEED_TYPES = listOf("application/rss+xml", "application/atom+xml", "application/rdf+xml", "text/xml") + + private val LINK_TAG = Regex("]*>", RegexOption.IGNORE_CASE) + + /** `name="value"`, `name='value'` or `name=value`, in any order within the tag. */ + private val ATTRIBUTE = + Regex("""(?[a-zA-Z-]+)\s*=\s*("(?[^"]*)"|'(?[^']*)'|(?[^\s">]+))""") + + /** + * The feed URL [html] advertises, absolute, or null when it advertises none. + * + * [pageUrl] is the address the HTML came from, which a relative href resolves against -- + * against the page, not the site root, since a feed link on `/blog/` routinely says + * `feed.xml` and means `/blog/feed.xml`. + */ + fun findFeedUrl(html: String, pageUrl: String): String? { + // Already a feed: there is nothing to discover, and scanning XML for tags would + // only find Atom's own self-links and send the caller round in a circle. + if (html.trimStart().startsWith(" + attrs["rel"]?.contains("alternate", ignoreCase = true) == true && + attrs["type"]?.lowercase()?.trim() in FEED_TYPES + } + ?.get("href") + ?.takeIf { it.isNotBlank() } + ?.let { resolve(it, pageUrl) } + } + + private fun attributesOf(tag: String): Map = + ATTRIBUTE.findAll(tag).associate { match -> + // Named rather than numbered: which of the three quoting alternatives matched is + // the whole question here, and "groupValues[4]" does not say that to a reader. + val value = match.groups["dq"]?.value + ?: match.groups["sq"]?.value + ?: match.groups["bare"]?.value.orEmpty() + match.groups["name"]!!.value.lowercase() to value + } + + /** + * [href] as an absolute https URL. + * + * The https upgrade is the same rule the add-feed screen already applies to what the user + * types, for the same reason: Android blocks cleartext, so an http feed would fail every + * fetch forever. A site that advertises an http feed almost always serves it over https too. + */ + private fun resolve(href: String, pageUrl: String): String? = try { + val absolute = URI(pageUrl).resolve(href.trim()).toString() + if (absolute.startsWith("http://", ignoreCase = true)) { + "https://" + absolute.removePrefix("http://").removePrefix("HTTP://") + } else { + absolute + } + } catch (ignored: IllegalArgumentException) { + // A malformed href is a page problem, not a crash: report no feed and let the caller + // say so, the same as a page that advertises none. Nothing is lost by not reporting + // it -- "no feed found at that address" is exactly as true and as useful either way. + null + } +} -- cgit v1.2.3