diff options
Diffstat (limited to 'app/src/main/java/org/terst/feedreader')
| -rw-r--r-- | app/src/main/java/org/terst/feedreader/data/FeedParser.kt | 10 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/feedreader/data/ItemTitles.kt | 131 |
2 files changed, 139 insertions, 2 deletions
diff --git a/app/src/main/java/org/terst/feedreader/data/FeedParser.kt b/app/src/main/java/org/terst/feedreader/data/FeedParser.kt index 7a3ce93..ecb59dd 100644 --- a/app/src/main/java/org/terst/feedreader/data/FeedParser.kt +++ b/app/src/main/java/org/terst/feedreader/data/FeedParser.kt @@ -157,7 +157,11 @@ object FeedParser { // landing as the article's primary key. An empty-string id broke // Navigation Compose's route matching and crashed on tap. guid = itemGuid?.ifBlank { null } ?: itemLink.ifBlank { null }, - title = itemTitle, + // RSS requires a title OR a description, not both, and feeds that + // take the second option are common enough to matter: every item in + // a Mastodon tag feed is title-less, which showed as a list of blank + // rows. See ItemTitles. + title = ItemTitles.titleFor(itemTitle, content, itemLink), link = itemLink, author = itemAuthor, publishedAt = itemDate, @@ -245,7 +249,9 @@ object FeedParser { items += ParsedItem( // Same blank-vs-null fix as the RSS path above. guid = entryId?.ifBlank { null } ?: entryLink.ifBlank { null }, - title = entryTitle, + // Atom does require a title, but a feed that omits one should not + // render as a blank row either -- same treatment as RSS above. + title = ItemTitles.titleFor(entryTitle, content, entryLink), link = entryLink, author = entryAuthor, publishedAt = entryDate, diff --git a/app/src/main/java/org/terst/feedreader/data/ItemTitles.kt b/app/src/main/java/org/terst/feedreader/data/ItemTitles.kt new file mode 100644 index 0000000..880f39b --- /dev/null +++ b/app/src/main/java/org/terst/feedreader/data/ItemTitles.kt @@ -0,0 +1,131 @@ +package org.terst.feedreader.data + +/** + * A title for a feed item that doesn't have one. + * + * RSS requires an item to carry a title *or* a description, not both, and plenty of feeds take + * the second option. Mastodon's tag feeds are the reported case: every one of the 20 items in + * `mastodon.social/tags/giftlink.rss` has a `<description>` and no `<title>` at all, so + * subscribing to one produced a list of blank rows with nothing to read or tap. + * + * What a reader wants in that situation is what every other client does: name the item by its + * own opening words. The post says what it is; it just says it in the body. + * + * Deliberately free of `android.text.Html`, unlike [FeedParser.decodeTitleEntities] -- that + * drags in the Android framework, which is why the parser around this has no unit tests at all. + * This is the part worth testing, so it does its own stripping and decoding and can be exercised + * on the JVM. + */ +object ItemTitles { + + /** Longest derived title. Two lines in the article list at the default text size. */ + const val MAX_LENGTH = 100 + + /** Below this, a sentence is too short to be a title on its own; keep reading. */ + private const val MIN_SENTENCE_LENGTH = 20 + + /** Unicode's highest code point, the ceiling on a numeric character reference. */ + private const val MAX_CODE_POINT = 0x10FFFF + + private const val DECIMAL = 10 + private const val HEXADECIMAL = 16 + + private val SCRIPT_OR_STYLE = Regex("<(script|style)\\b[^>]*>.*?</\\1\\s*>", setOf( + RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL + )) + /** + * Tags that mean a break between words. Everything else is inline and is removed without + * leaving a gap: Mastodon splits a hashtag as `#<span>giftlink</span>` and a URL as + * `<span class="invisible">https://</span><span>wapo.st/x</span>`, so turning every tag into + * a space pulled both apart -- seen live as "# HumaneRights" and "https:// wapo.". + */ + private val BLOCK_TAG = Regex( + "</?(p|div|br|li|ul|ol|tr|td|h[1-6]|blockquote|section|article|figure|figcaption|pre)\\b[^>]*>", + RegexOption.IGNORE_CASE + ) + private val TAG = Regex("<[^>]*>") + private val WHITESPACE = Regex("\\s+") + private val NUMERIC_ENTITY = Regex("&#(x?)([0-9a-fA-F]+);") + + /** The few named entities that actually turn up in feed prose. */ + private val NAMED_ENTITIES = mapOf( + "amp" to "&", "lt" to "<", "gt" to ">", "quot" to "\"", "apos" to "'", + "nbsp" to " ", "hellip" to "…", "mdash" to "—", "ndash" to "–", + "lsquo" to "‘", "rsquo" to "’", "ldquo" to "“", "rdquo" to "”", + ) + + /** + * [rawTitle] when the item has one, else a title taken from its own words, else its link. + * + * The link is a poor title but a real one: it says where the item goes, which beats a blank + * row. "Untitled" is the last resort and should be unreachable in practice -- an item with + * no title, no words and no link has nothing in it at all. + */ + fun titleFor(rawTitle: String, contentHtml: String?, link: String): String = + rawTitle.trim().ifEmpty { + plainText(contentHtml).takeIf { it.isNotEmpty() }?.let(::shorten) + ?: link.trim().takeIf { it.isNotEmpty() }?.let(::tidyLink) + ?: "Untitled" + } + + /** The item's text, with markup, entities and line breaks resolved away. */ + private fun plainText(html: String?): String { + if (html.isNullOrBlank()) return "" + val withBreaks = BLOCK_TAG.replace(SCRIPT_OR_STYLE.replace(html, " "), " ") + val stripped = TAG.replace(withBreaks, "") + return WHITESPACE.replace(decodeEntities(stripped), " ").trim() + } + + /** + * The opening sentence, or the opening words when the sentence is too long or never ends. + * + * Sentence-first because a post's first sentence is usually exactly the headline it would + * have written. No attempt is made to be clever about abbreviations -- "Mr. Smith" would cut + * early, which produces a slightly short title and nothing worse. + */ + private fun shorten(text: String): String = + firstSentence(text)?.takeIf { it.length <= MAX_LENGTH } + ?: text.takeIf { it.length <= MAX_LENGTH } + ?: truncateAtWord(text) + + /** The opening words, cut on a space so no word is left half-written. */ + private fun truncateAtWord(text: String): String { + val cut = text.take(MAX_LENGTH) + val lastSpace = cut.lastIndexOf(' ') + val kept = if (lastSpace >= MIN_SENTENCE_LENGTH) cut.take(lastSpace) else cut + return kept.trimEnd().trimEnd(',', ';', ':', '-') + "…" + } + + /** + * The first sentence, or null when there isn't a usable one. + * + * A terminator only ends a sentence if whitespace or the end of the text follows it. + * Without that rule the dot in "wapo.st/abc" ended the sentence, and a post whose first + * words were a link was titled "The loophole https://wapo." -- seen live. + */ + private fun firstSentence(text: String): String? { + for (i in text.indices) { + if (text[i] != '.' && text[i] != '!' && text[i] != '?') continue + val endsHere = i == text.lastIndex || text[i + 1].isWhitespace() + if (endsHere && i >= MIN_SENTENCE_LENGTH) return text.take(i + 1) + } + return null + } + + /** The link without the parts of it nobody reads. */ + private fun tidyLink(link: String): String = + link.substringAfter("://", missingDelimiterValue = link).removeSuffix("/") + + private fun decodeEntities(text: String): String { + if (!text.contains('&')) return text + var out = NUMERIC_ENTITY.replace(text) { match -> + val radix = if (match.groupValues[1].isEmpty()) DECIMAL else HEXADECIMAL + match.groupValues[2].toIntOrNull(radix) + ?.takeIf { it in 1..MAX_CODE_POINT } + ?.let { String(Character.toChars(it)) } + ?: match.value + } + for ((name, value) in NAMED_ENTITIES) out = out.replace("&$name;", value) + return out + } +} |