diff options
Diffstat (limited to 'app/src/main/java/org/terst/feedreader/data')
| -rw-r--r-- | app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt | 10 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/feedreader/data/FeedDiscovery.kt | 88 |
2 files changed, 97 insertions, 1 deletions
diff --git a/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt b/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt index 3a791cc..63614d1 100644 --- a/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt +++ b/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt @@ -1,6 +1,7 @@ package org.terst.feedreader.data import org.terst.feedreader.readability.ConsentNotice +import org.terst.feedreader.readability.RecirculationWidgets import org.terst.feedreader.readability.UnrenderedTex /** What a cached `articles.extractedContent` value turned out to be worth. */ @@ -58,7 +59,14 @@ suspend fun loadCachedExtraction( discardedUnrenderedTex = true ) else -> null - } ?: return CachedExtraction(content = cached, discardedConsentNotice = false) + } ?: return CachedExtraction( + // Recirculation widgets are deliberately not in the `discard` list above. A cached + // article carrying the publisher's promo blocks is a good article with litter in it, + // not a poisoned row: cleaning it in place shows the right thing immediately, where + // discarding it would throw away a usable body and fetch the identical one back. + content = RecirculationWidgets.strip(cached), + discardedConsentNotice = false + ) clearCache() return discard diff --git a/app/src/main/java/org/terst/feedreader/data/FeedDiscovery.kt b/app/src/main/java/org/terst/feedreader/data/FeedDiscovery.kt new file mode 100644 index 0000000..98ce5b3 --- /dev/null +++ b/app/src/main/java/org/terst/feedreader/data/FeedDiscovery.kt @@ -0,0 +1,88 @@ +package org.terst.feedreader.data + +import java.net.URI + +/** + * Finds the feed a web page advertises. + * + * Reported live: "Can't subscribe to https://heathercoxrichardson.substack.com/". Adding a feed + * fetched whatever URL it was given and handed it straight to [FeedParser], so a site's home + * page -- the only URL a reader can actually see and copy -- failed on the XML parse with a + * message about an unrecognised root element. + * + * Nothing exotic is needed to fix it. Sites have advertised their feeds in the document head + * since RSS autodiscovery was agreed in 2002, and that Substack page does: + * + * ``` + * <link rel="alternate" type="application/rss+xml" href="/feed" title="Letters from an American"/> + * ``` + * + * Only what the page itself declares is used. Guessing paths (`/feed`, `/rss`, `/index.xml`) + * would add a burst of 404s on every failed add and would eventually subscribe somebody to the + * wrong thing; a site that declares nothing is reported as having no feed, which is the truth. + */ +object FeedDiscovery { + + /** Feed content types, as `type=` gives them. */ + private val FEED_TYPES = listOf("application/rss+xml", "application/atom+xml", "application/rdf+xml", "text/xml") + + private val LINK_TAG = Regex("<link\\b[^>]*>", RegexOption.IGNORE_CASE) + + /** `name="value"`, `name='value'` or `name=value`, in any order within the tag. */ + private val ATTRIBUTE = + Regex("""(?<name>[a-zA-Z-]+)\s*=\s*("(?<dq>[^"]*)"|'(?<sq>[^']*)'|(?<bare>[^\s">]+))""") + + /** + * The feed URL [html] advertises, absolute, or null when it advertises none. + * + * [pageUrl] is the address the HTML came from, which a relative href resolves against -- + * against the page, not the site root, since a feed link on `/blog/` routinely says + * `feed.xml` and means `/blog/feed.xml`. + */ + fun findFeedUrl(html: String, pageUrl: String): String? { + // Already a feed: there is nothing to discover, and scanning XML for <link> tags would + // only find Atom's own self-links and send the caller round in a circle. + if (html.trimStart().startsWith("<?xml", ignoreCase = true)) return null + + return LINK_TAG.findAll(html) + .map { attributesOf(it.value) } + .firstOrNull { attrs -> + attrs["rel"]?.contains("alternate", ignoreCase = true) == true && + attrs["type"]?.lowercase()?.trim() in FEED_TYPES + } + ?.get("href") + ?.takeIf { it.isNotBlank() } + ?.let { resolve(it, pageUrl) } + } + + private fun attributesOf(tag: String): Map<String, String> = + ATTRIBUTE.findAll(tag).associate { match -> + // Named rather than numbered: which of the three quoting alternatives matched is + // the whole question here, and "groupValues[4]" does not say that to a reader. + val value = match.groups["dq"]?.value + ?: match.groups["sq"]?.value + ?: match.groups["bare"]?.value.orEmpty() + match.groups["name"]!!.value.lowercase() to value + } + + /** + * [href] as an absolute https URL. + * + * The https upgrade is the same rule the add-feed screen already applies to what the user + * types, for the same reason: Android blocks cleartext, so an http feed would fail every + * fetch forever. A site that advertises an http feed almost always serves it over https too. + */ + private fun resolve(href: String, pageUrl: String): String? = try { + val absolute = URI(pageUrl).resolve(href.trim()).toString() + if (absolute.startsWith("http://", ignoreCase = true)) { + "https://" + absolute.removePrefix("http://").removePrefix("HTTP://") + } else { + absolute + } + } catch (ignored: IllegalArgumentException) { + // A malformed href is a page problem, not a crash: report no feed and let the caller + // say so, the same as a page that advertises none. Nothing is lost by not reporting + // it -- "no feed found at that address" is exactly as true and as useful either way. + null + } +} |