1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
|
package org.terst.feedreader.data
import java.net.URI
/**
* Finds the feed a web page advertises.
*
* Reported live: "Can't subscribe to https://heathercoxrichardson.substack.com/". Adding a feed
* fetched whatever URL it was given and handed it straight to [FeedParser], so a site's home
* page -- the only URL a reader can actually see and copy -- failed on the XML parse with a
* message about an unrecognised root element.
*
* Nothing exotic is needed to fix it. Sites have advertised their feeds in the document head
* since RSS autodiscovery was agreed in 2002, and that Substack page does:
*
* ```
* <link rel="alternate" type="application/rss+xml" href="/feed" title="Letters from an American"/>
* ```
*
* Only what the page itself declares is used. Guessing paths (`/feed`, `/rss`, `/index.xml`)
* would add a burst of 404s on every failed add and would eventually subscribe somebody to the
* wrong thing; a site that declares nothing is reported as having no feed, which is the truth.
*/
object FeedDiscovery {
/** Feed content types, as `type=` gives them. */
private val FEED_TYPES = listOf("application/rss+xml", "application/atom+xml", "application/rdf+xml", "text/xml")
private val LINK_TAG = Regex("<link\\b[^>]*>", RegexOption.IGNORE_CASE)
/** `name="value"`, `name='value'` or `name=value`, in any order within the tag. */
private val ATTRIBUTE =
Regex("""(?<name>[a-zA-Z-]+)\s*=\s*("(?<dq>[^"]*)"|'(?<sq>[^']*)'|(?<bare>[^\s">]+))""")
/**
* The feed URL [html] advertises, absolute, or null when it advertises none.
*
* [pageUrl] is the address the HTML came from, which a relative href resolves against --
* against the page, not the site root, since a feed link on `/blog/` routinely says
* `feed.xml` and means `/blog/feed.xml`.
*/
fun findFeedUrl(html: String, pageUrl: String): String? {
// Already a feed: there is nothing to discover, and scanning XML for <link> tags would
// only find Atom's own self-links and send the caller round in a circle.
if (html.trimStart().startsWith("<?xml", ignoreCase = true)) return null
return LINK_TAG.findAll(html)
.map { attributesOf(it.value) }
.firstOrNull { attrs ->
attrs["rel"]?.contains("alternate", ignoreCase = true) == true &&
attrs["type"]?.lowercase()?.trim() in FEED_TYPES
}
?.get("href")
?.takeIf { it.isNotBlank() }
?.let { resolve(it, pageUrl) }
}
private fun attributesOf(tag: String): Map<String, String> =
ATTRIBUTE.findAll(tag).associate { match ->
// Named rather than numbered: which of the three quoting alternatives matched is
// the whole question here, and "groupValues[4]" does not say that to a reader.
val value = match.groups["dq"]?.value
?: match.groups["sq"]?.value
?: match.groups["bare"]?.value.orEmpty()
match.groups["name"]!!.value.lowercase() to value
}
/**
* [href] as an absolute https URL.
*
* The https upgrade is the same rule the add-feed screen already applies to what the user
* types, for the same reason: Android blocks cleartext, so an http feed would fail every
* fetch forever. A site that advertises an http feed almost always serves it over https too.
*/
private fun resolve(href: String, pageUrl: String): String? = try {
val absolute = URI(pageUrl).resolve(href.trim()).toString()
if (absolute.startsWith("http://", ignoreCase = true)) {
"https://" + absolute.removePrefix("http://").removePrefix("HTTP://")
} else {
absolute
}
} catch (ignored: IllegalArgumentException) {
// A malformed href is a page problem, not a crash: report no feed and let the caller
// say so, the same as a page that advertises none. Nothing is lost by not reporting
// it -- "no feed found at that address" is exactly as true and as useful either way.
null
}
}
|
