1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
package org.terst.feedreader.data
/**
* A title for a feed item that doesn't have one.
*
* RSS requires an item to carry a title *or* a description, not both, and plenty of feeds take
* the second option. Mastodon's tag feeds are the reported case: every one of the 20 items in
* `mastodon.social/tags/giftlink.rss` has a `<description>` and no `<title>` at all, so
* subscribing to one produced a list of blank rows with nothing to read or tap.
*
* What a reader wants in that situation is what every other client does: name the item by its
* own opening words. The post says what it is; it just says it in the body.
*
* Deliberately free of `android.text.Html`, unlike [FeedParser.decodeTitleEntities] -- that
* drags in the Android framework, which is why the parser around this has no unit tests at all.
* This is the part worth testing, so it does its own stripping and decoding and can be exercised
* on the JVM.
*/
object ItemTitles {
/** Longest derived title. Two lines in the article list at the default text size. */
const val MAX_LENGTH = 100
/** Below this, a sentence is too short to be a title on its own; keep reading. */
private const val MIN_SENTENCE_LENGTH = 20
/** Unicode's highest code point, the ceiling on a numeric character reference. */
private const val MAX_CODE_POINT = 0x10FFFF
private const val DECIMAL = 10
private const val HEXADECIMAL = 16
private val SCRIPT_OR_STYLE = Regex("<(script|style)\\b[^>]*>.*?</\\1\\s*>", setOf(
RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL
))
/**
* Tags that mean a break between words. Everything else is inline and is removed without
* leaving a gap: Mastodon splits a hashtag as `#<span>giftlink</span>` and a URL as
* `<span class="invisible">https://</span><span>wapo.st/x</span>`, so turning every tag into
* a space pulled both apart -- seen live as "# HumaneRights" and "https:// wapo.".
*/
private val BLOCK_TAG = Regex(
"</?(p|div|br|li|ul|ol|tr|td|h[1-6]|blockquote|section|article|figure|figcaption|pre)\\b[^>]*>",
RegexOption.IGNORE_CASE
)
private val TAG = Regex("<[^>]*>")
private val WHITESPACE = Regex("\\s+")
private val NUMERIC_ENTITY = Regex("&#(x?)([0-9a-fA-F]+);")
/** The few named entities that actually turn up in feed prose. */
private val NAMED_ENTITIES = mapOf(
"amp" to "&", "lt" to "<", "gt" to ">", "quot" to "\"", "apos" to "'",
"nbsp" to " ", "hellip" to "…", "mdash" to "—", "ndash" to "–",
"lsquo" to "‘", "rsquo" to "’", "ldquo" to "“", "rdquo" to "”",
)
/**
* [rawTitle] when the item has one, else a title taken from its own words, else its link.
*
* The link is a poor title but a real one: it says where the item goes, which beats a blank
* row. "Untitled" is the last resort and should be unreachable in practice -- an item with
* no title, no words and no link has nothing in it at all.
*/
fun titleFor(rawTitle: String, contentHtml: String?, link: String): String =
rawTitle.trim().ifEmpty {
plainText(contentHtml).takeIf { it.isNotEmpty() }?.let(::shorten)
?: link.trim().takeIf { it.isNotEmpty() }?.let(::tidyLink)
?: "Untitled"
}
/** The item's text, with markup, entities and line breaks resolved away. */
private fun plainText(html: String?): String {
if (html.isNullOrBlank()) return ""
val withBreaks = BLOCK_TAG.replace(SCRIPT_OR_STYLE.replace(html, " "), " ")
val stripped = TAG.replace(withBreaks, "")
return WHITESPACE.replace(decodeEntities(stripped), " ").trim()
}
/**
* The opening sentence, or the opening words when the sentence is too long or never ends.
*
* Sentence-first because a post's first sentence is usually exactly the headline it would
* have written. No attempt is made to be clever about abbreviations -- "Mr. Smith" would cut
* early, which produces a slightly short title and nothing worse.
*/
private fun shorten(text: String): String =
firstSentence(text)?.takeIf { it.length <= MAX_LENGTH }
?: text.takeIf { it.length <= MAX_LENGTH }
?: truncateAtWord(text)
/** The opening words, cut on a space so no word is left half-written. */
private fun truncateAtWord(text: String): String {
val cut = text.take(MAX_LENGTH)
val lastSpace = cut.lastIndexOf(' ')
val kept = if (lastSpace >= MIN_SENTENCE_LENGTH) cut.take(lastSpace) else cut
return kept.trimEnd().trimEnd(',', ';', ':', '-') + "…"
}
/**
* The first sentence, or null when there isn't a usable one.
*
* A terminator only ends a sentence if whitespace or the end of the text follows it.
* Without that rule the dot in "wapo.st/abc" ended the sentence, and a post whose first
* words were a link was titled "The loophole https://wapo." -- seen live.
*/
private fun firstSentence(text: String): String? {
for (i in text.indices) {
if (text[i] != '.' && text[i] != '!' && text[i] != '?') continue
val endsHere = i == text.lastIndex || text[i + 1].isWhitespace()
if (endsHere && i >= MIN_SENTENCE_LENGTH) return text.take(i + 1)
}
return null
}
/** The link without the parts of it nobody reads. */
private fun tidyLink(link: String): String =
link.substringAfter("://", missingDelimiterValue = link).removeSuffix("/")
private fun decodeEntities(text: String): String {
if (!text.contains('&')) return text
var out = NUMERIC_ENTITY.replace(text) { match ->
val radix = if (match.groupValues[1].isEmpty()) DECIMAL else HEXADECIMAL
match.groupValues[2].toIntOrNull(radix)
?.takeIf { it in 1..MAX_CODE_POINT }
?.let { String(Character.toChars(it)) }
?: match.value
}
for ((name, value) in NAMED_ENTITIES) out = out.replace("&$name;", value)
return out
}
}
|
