package org.terst.feedreader.data /** * A title for a feed item that doesn't have one. * * RSS requires an item to carry a title *or* a description, not both, and plenty of feeds take * the second option. Mastodon's tag feeds are the reported case: every one of the 20 items in * `mastodon.social/tags/giftlink.rss` has a `` and no `` at all, so * subscribing to one produced a list of blank rows with nothing to read or tap. * * What a reader wants in that situation is what every other client does: name the item by its * own opening words. The post says what it is; it just says it in the body. * * Deliberately free of `android.text.Html`, unlike [FeedParser.decodeTitleEntities] -- that * drags in the Android framework, which is why the parser around this has no unit tests at all. * This is the part worth testing, so it does its own stripping and decoding and can be exercised * on the JVM. */ object ItemTitles { /** Longest derived title. Two lines in the article list at the default text size. */ const val MAX_LENGTH = 100 /** Below this, a sentence is too short to be a title on its own; keep reading. */ private const val MIN_SENTENCE_LENGTH = 20 /** Unicode's highest code point, the ceiling on a numeric character reference. */ private const val MAX_CODE_POINT = 0x10FFFF private const val DECIMAL = 10 private const val HEXADECIMAL = 16 private val SCRIPT_OR_STYLE = Regex("<(script|style)\\b[^>]*>.*?</\\1\\s*>", setOf( RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL )) /** * Tags that mean a break between words. Everything else is inline and is removed without * leaving a gap: Mastodon splits a hashtag as `#<span>giftlink</span>` and a URL as * `<span class="invisible">https://</span><span>wapo.st/x</span>`, so turning every tag into * a space pulled both apart -- seen live as "# HumaneRights" and "https:// wapo.". */ private val BLOCK_TAG = Regex( "</?(p|div|br|li|ul|ol|tr|td|h[1-6]|blockquote|section|article|figure|figcaption|pre)\\b[^>]*>", RegexOption.IGNORE_CASE ) private val TAG = Regex("<[^>]*>") private val WHITESPACE = Regex("\\s+") private val NUMERIC_ENTITY = Regex("&#(x?)([0-9a-fA-F]+);") /** The few named entities that actually turn up in feed prose. */ private val NAMED_ENTITIES = mapOf( "amp" to "&", "lt" to "<", "gt" to ">", "quot" to "\"", "apos" to "'", "nbsp" to " ", "hellip" to "…", "mdash" to "—", "ndash" to "–", "lsquo" to "‘", "rsquo" to "’", "ldquo" to "“", "rdquo" to "”", ) /** * [rawTitle] when the item has one, else a title taken from its own words, else its link. * * The link is a poor title but a real one: it says where the item goes, which beats a blank * row. "Untitled" is the last resort and should be unreachable in practice -- an item with * no title, no words and no link has nothing in it at all. */ fun titleFor(rawTitle: String, contentHtml: String?, link: String): String = rawTitle.trim().ifEmpty { plainText(contentHtml).takeIf { it.isNotEmpty() }?.let(::shorten) ?: link.trim().takeIf { it.isNotEmpty() }?.let(::tidyLink) ?: "Untitled" } /** The item's text, with markup, entities and line breaks resolved away. */ private fun plainText(html: String?): String { if (html.isNullOrBlank()) return "" val withBreaks = BLOCK_TAG.replace(SCRIPT_OR_STYLE.replace(html, " "), " ") val stripped = TAG.replace(withBreaks, "") return WHITESPACE.replace(decodeEntities(stripped), " ").trim() } /** * The opening sentence, or the opening words when the sentence is too long or never ends. * * Sentence-first because a post's first sentence is usually exactly the headline it would * have written. No attempt is made to be clever about abbreviations -- "Mr. Smith" would cut * early, which produces a slightly short title and nothing worse. */ private fun shorten(text: String): String = firstSentence(text)?.takeIf { it.length <= MAX_LENGTH } ?: text.takeIf { it.length <= MAX_LENGTH } ?: truncateAtWord(text) /** The opening words, cut on a space so no word is left half-written. */ private fun truncateAtWord(text: String): String { val cut = text.take(MAX_LENGTH) val lastSpace = cut.lastIndexOf(' ') val kept = if (lastSpace >= MIN_SENTENCE_LENGTH) cut.take(lastSpace) else cut return kept.trimEnd().trimEnd(',', ';', ':', '-') + "…" } /** * The first sentence, or null when there isn't a usable one. * * A terminator only ends a sentence if whitespace or the end of the text follows it. * Without that rule the dot in "wapo.st/abc" ended the sentence, and a post whose first * words were a link was titled "The loophole https://wapo." -- seen live. */ private fun firstSentence(text: String): String? { for (i in text.indices) { if (text[i] != '.' && text[i] != '!' && text[i] != '?') continue val endsHere = i == text.lastIndex || text[i + 1].isWhitespace() if (endsHere && i >= MIN_SENTENCE_LENGTH) return text.take(i + 1) } return null } /** The link without the parts of it nobody reads. */ private fun tidyLink(link: String): String = link.substringAfter("://", missingDelimiterValue = link).removeSuffix("/") private fun decodeEntities(text: String): String { if (!text.contains('&')) return text var out = NUMERIC_ENTITY.replace(text) { match -> val radix = if (match.groupValues[1].isEmpty()) DECIMAL else HEXADECIMAL match.groupValues[2].toIntOrNull(radix) ?.takeIf { it in 1..MAX_CODE_POINT } ?.let { String(Character.toChars(it)) } ?: match.value } for ((name, value) in NAMED_ENTITIES) out = out.replace("&$name;", value) return out } }