diff options
Diffstat (limited to 'app')
3 files changed, 318 insertions, 2 deletions
diff --git a/app/src/main/java/org/terst/feedreader/data/FeedParser.kt b/app/src/main/java/org/terst/feedreader/data/FeedParser.kt index 7a3ce93..ecb59dd 100644 --- a/app/src/main/java/org/terst/feedreader/data/FeedParser.kt +++ b/app/src/main/java/org/terst/feedreader/data/FeedParser.kt @@ -157,7 +157,11 @@ object FeedParser { // landing as the article's primary key. An empty-string id broke // Navigation Compose's route matching and crashed on tap. guid = itemGuid?.ifBlank { null } ?: itemLink.ifBlank { null }, - title = itemTitle, + // RSS requires a title OR a description, not both, and feeds that + // take the second option are common enough to matter: every item in + // a Mastodon tag feed is title-less, which showed as a list of blank + // rows. See ItemTitles. + title = ItemTitles.titleFor(itemTitle, content, itemLink), link = itemLink, author = itemAuthor, publishedAt = itemDate, @@ -245,7 +249,9 @@ object FeedParser { items += ParsedItem( // Same blank-vs-null fix as the RSS path above. guid = entryId?.ifBlank { null } ?: entryLink.ifBlank { null }, - title = entryTitle, + // Atom does require a title, but a feed that omits one should not + // render as a blank row either -- same treatment as RSS above. + title = ItemTitles.titleFor(entryTitle, content, entryLink), link = entryLink, author = entryAuthor, publishedAt = entryDate, diff --git a/app/src/main/java/org/terst/feedreader/data/ItemTitles.kt b/app/src/main/java/org/terst/feedreader/data/ItemTitles.kt new file mode 100644 index 0000000..880f39b --- /dev/null +++ b/app/src/main/java/org/terst/feedreader/data/ItemTitles.kt @@ -0,0 +1,131 @@ +package org.terst.feedreader.data + +/** + * A title for a feed item that doesn't have one. + * + * RSS requires an item to carry a title *or* a description, not both, and plenty of feeds take + * the second option. Mastodon's tag feeds are the reported case: every one of the 20 items in + * `mastodon.social/tags/giftlink.rss` has a `<description>` and no `<title>` at all, so + * subscribing to one produced a list of blank rows with nothing to read or tap. + * + * What a reader wants in that situation is what every other client does: name the item by its + * own opening words. The post says what it is; it just says it in the body. + * + * Deliberately free of `android.text.Html`, unlike [FeedParser.decodeTitleEntities] -- that + * drags in the Android framework, which is why the parser around this has no unit tests at all. + * This is the part worth testing, so it does its own stripping and decoding and can be exercised + * on the JVM. + */ +object ItemTitles { + + /** Longest derived title. Two lines in the article list at the default text size. */ + const val MAX_LENGTH = 100 + + /** Below this, a sentence is too short to be a title on its own; keep reading. */ + private const val MIN_SENTENCE_LENGTH = 20 + + /** Unicode's highest code point, the ceiling on a numeric character reference. */ + private const val MAX_CODE_POINT = 0x10FFFF + + private const val DECIMAL = 10 + private const val HEXADECIMAL = 16 + + private val SCRIPT_OR_STYLE = Regex("<(script|style)\\b[^>]*>.*?</\\1\\s*>", setOf( + RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL + )) + /** + * Tags that mean a break between words. Everything else is inline and is removed without + * leaving a gap: Mastodon splits a hashtag as `#<span>giftlink</span>` and a URL as + * `<span class="invisible">https://</span><span>wapo.st/x</span>`, so turning every tag into + * a space pulled both apart -- seen live as "# HumaneRights" and "https:// wapo.". + */ + private val BLOCK_TAG = Regex( + "</?(p|div|br|li|ul|ol|tr|td|h[1-6]|blockquote|section|article|figure|figcaption|pre)\\b[^>]*>", + RegexOption.IGNORE_CASE + ) + private val TAG = Regex("<[^>]*>") + private val WHITESPACE = Regex("\\s+") + private val NUMERIC_ENTITY = Regex("&#(x?)([0-9a-fA-F]+);") + + /** The few named entities that actually turn up in feed prose. */ + private val NAMED_ENTITIES = mapOf( + "amp" to "&", "lt" to "<", "gt" to ">", "quot" to "\"", "apos" to "'", + "nbsp" to " ", "hellip" to "…", "mdash" to "—", "ndash" to "–", + "lsquo" to "‘", "rsquo" to "’", "ldquo" to "“", "rdquo" to "”", + ) + + /** + * [rawTitle] when the item has one, else a title taken from its own words, else its link. + * + * The link is a poor title but a real one: it says where the item goes, which beats a blank + * row. "Untitled" is the last resort and should be unreachable in practice -- an item with + * no title, no words and no link has nothing in it at all. + */ + fun titleFor(rawTitle: String, contentHtml: String?, link: String): String = + rawTitle.trim().ifEmpty { + plainText(contentHtml).takeIf { it.isNotEmpty() }?.let(::shorten) + ?: link.trim().takeIf { it.isNotEmpty() }?.let(::tidyLink) + ?: "Untitled" + } + + /** The item's text, with markup, entities and line breaks resolved away. */ + private fun plainText(html: String?): String { + if (html.isNullOrBlank()) return "" + val withBreaks = BLOCK_TAG.replace(SCRIPT_OR_STYLE.replace(html, " "), " ") + val stripped = TAG.replace(withBreaks, "") + return WHITESPACE.replace(decodeEntities(stripped), " ").trim() + } + + /** + * The opening sentence, or the opening words when the sentence is too long or never ends. + * + * Sentence-first because a post's first sentence is usually exactly the headline it would + * have written. No attempt is made to be clever about abbreviations -- "Mr. Smith" would cut + * early, which produces a slightly short title and nothing worse. + */ + private fun shorten(text: String): String = + firstSentence(text)?.takeIf { it.length <= MAX_LENGTH } + ?: text.takeIf { it.length <= MAX_LENGTH } + ?: truncateAtWord(text) + + /** The opening words, cut on a space so no word is left half-written. */ + private fun truncateAtWord(text: String): String { + val cut = text.take(MAX_LENGTH) + val lastSpace = cut.lastIndexOf(' ') + val kept = if (lastSpace >= MIN_SENTENCE_LENGTH) cut.take(lastSpace) else cut + return kept.trimEnd().trimEnd(',', ';', ':', '-') + "…" + } + + /** + * The first sentence, or null when there isn't a usable one. + * + * A terminator only ends a sentence if whitespace or the end of the text follows it. + * Without that rule the dot in "wapo.st/abc" ended the sentence, and a post whose first + * words were a link was titled "The loophole https://wapo." -- seen live. + */ + private fun firstSentence(text: String): String? { + for (i in text.indices) { + if (text[i] != '.' && text[i] != '!' && text[i] != '?') continue + val endsHere = i == text.lastIndex || text[i + 1].isWhitespace() + if (endsHere && i >= MIN_SENTENCE_LENGTH) return text.take(i + 1) + } + return null + } + + /** The link without the parts of it nobody reads. */ + private fun tidyLink(link: String): String = + link.substringAfter("://", missingDelimiterValue = link).removeSuffix("/") + + private fun decodeEntities(text: String): String { + if (!text.contains('&')) return text + var out = NUMERIC_ENTITY.replace(text) { match -> + val radix = if (match.groupValues[1].isEmpty()) DECIMAL else HEXADECIMAL + match.groupValues[2].toIntOrNull(radix) + ?.takeIf { it in 1..MAX_CODE_POINT } + ?.let { String(Character.toChars(it)) } + ?: match.value + } + for ((name, value) in NAMED_ENTITIES) out = out.replace("&$name;", value) + return out + } +} diff --git a/app/src/test/java/org/terst/feedreader/data/ItemTitlesTest.kt b/app/src/test/java/org/terst/feedreader/data/ItemTitlesTest.kt new file mode 100644 index 0000000..63606ff --- /dev/null +++ b/app/src/test/java/org/terst/feedreader/data/ItemTitlesTest.kt @@ -0,0 +1,179 @@ +package org.terst.feedreader.data + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * A title for an item that doesn't have one. + * + * RSS only requires an item to carry a title *or* a description, and Mastodon's tag feeds + * exercise that: every one of the 20 items in `mastodon.social/tags/giftlink.rss` has a + * `<description>` and no `<title>` at all. Subscribing to one produced a list of blank rows. + */ +class ItemTitlesTest { + + /** Verbatim from the #giftlink feed, 2026-10-22, as the XML parser hands it over. */ + private val mastodonPost = + """<p>This is a nice story, with lovely photos. <a href="https://infosec.exchange/tags/giftlink" """ + + """class="mention hashtag" rel="nofollow noopener" target="_blank">#<span>giftlink</span></a> </p>""" + + """<p><a href="https://www.nytimes.com/2026/10/20/us/catherine-breed.html?unlocked_article_code=1.KlE" """ + + """rel="nofollow noopener" target="_blank"><span class="invisible">https://www.</span>""" + + """<span class="ellipsis">nytimes.com/2026/10/20/us/cath</span></a></p>""" + + @Test + fun `a real title is left exactly as it is`() { + assertEquals( + "The Flipbook of the Rings", + ItemTitles.titleFor("The Flipbook of the Rings", "<p>anything</p>", "https://kottke.org/x") + ) + } + + @Test + fun `a title of nothing but whitespace counts as absent`() { + assertEquals("Hello there.", ItemTitles.titleFor(" \n ", "<p>Hello there.</p>", "https://x/y")) + } + + @Test + fun `a titleless post is named by its own first sentence`() { + assertEquals( + "This is a nice story, with lovely photos.", + ItemTitles.titleFor("", mastodonPost, "https://infosec.exchange/@cremevax/117483063928862987") + ) + } + + @Test + fun `markup never reaches the title`() { + val title = ItemTitles.titleFor("", mastodonPost, "https://x/y") + + assertTrue("got: $title", !title.contains("<") && !title.contains(">")) + } + + @Test + fun `html entities are resolved, since a title is shown as plain text`() { + assertEquals( + "Smith & Co's \"deal\" — done", + ItemTitles.titleFor("", "<p>Smith & Co's "deal" — done</p>", "https://x/y") + ) + } + + @Test + fun `a non-breaking space is a space, not a stray glyph`() { + assertEquals("One two three", ItemTitles.titleFor("", "<p>One two three</p>", "https://x/y")) + } + + @Test + fun `runs of whitespace collapse, so a title stays one line`() { + assertEquals("One two", ItemTitles.titleFor("", "<p>One</p>\n\n <p>two</p>", "https://x/y")) + } + + @Test + fun `a long post is cut at a word boundary and marked as cut`() { + val long = "<p>" + "banana ".repeat(60) + "</p>" + + val title = ItemTitles.titleFor("", long, "https://x/y") + + assertTrue("should be cut: ${title.length}", title.length <= ItemTitles.MAX_LENGTH + 1) + assertTrue("should say it was cut: $title", title.endsWith("…")) + assertTrue("should not cut mid-word: $title", title.startsWith("banana banana")) + assertTrue("should not cut mid-word: $title", !title.contains("banan…")) + } + + @Test + fun `a very long first sentence is cut rather than used whole`() { + val long = "<p>" + "word ".repeat(80) + ". Second sentence.</p>" + + assertTrue(ItemTitles.titleFor("", long, "https://x/y").length <= ItemTitles.MAX_LENGTH + 1) + } + + @Test + fun `a post with no words falls back to its link`() { + val title = ItemTitles.titleFor("", """<p><img src="https://x/a.jpg"></p>""", "https://infosec.exchange/@a/1") + + assertEquals("infosec.exchange/@a/1", title) + } + + @Test + fun `no title, no content and no link still gives something to tap`() { + assertEquals("Untitled", ItemTitles.titleFor("", null, "")) + } + + @Test + fun `null content falls back to the link`() { + assertEquals("example.com/a", ItemTitles.titleFor("", null, "https://example.com/a")) + } + + @Test + fun `a trailing slash is not worth showing in a title`() { + assertEquals("example.com/a", ItemTitles.titleFor("", null, "https://example.com/a/")) + } + + @Test + fun `a script or style block contributes no words`() { + val html = "<style>p{color:red}</style><script>var x=1;</script><p>Actual words.</p>" + + assertEquals("Actual words.", ItemTitles.titleFor("", html, "https://x/y")) + } + + /** + * Mastodon splits a hashtag and a URL across inline spans -- `#<span>giftlink</span>` and + * `<span class="invisible">https://</span><span class="ellipsis">wapo.st/x</span>`. Turning + * every tag into a space broke both apart, which showed live as titles reading + * "# HumaneRights # USPolitics" and "the loophole https:// wapo.". + */ + @Test + fun `an inline tag does not insert a space into a word`() { + assertEquals( + "Tagged #giftlink today.", + ItemTitles.titleFor("", "<p>Tagged #<span>giftlink</span> today.</p>", "https://x/y") + ) + } + + @Test + fun `a url split across spans stays one url`() { + val html = """<p>Read <a href="https://wapo.st/x"><span class="invisible">https://</span>""" + + """<span class="ellipsis">wapo.st/x</span></a> now</p>""" + + assertEquals("Read https://wapo.st/x now", ItemTitles.titleFor("", html, "https://x/y")) + } + + @Test + fun `a block boundary still separates words`() { + assertEquals("One two", ItemTitles.titleFor("", "<p>One</p><p>two</p>", "https://x/y")) + } + + @Test + fun `a line break separates words`() { + assertEquals("One two", ItemTitles.titleFor("", "One<br>two", "https://x/y")) + } + + @Test + fun `a dot inside a domain does not end the sentence`() { + assertEquals( + "The $580 loophole https://wapo.st/abc is open.", + ItemTitles.titleFor("", "<p>The $580 loophole https://wapo.st/abc is open. And more.</p>", "https://x/y") + ) + } + + @Test + fun `a sentence still ends at a full stop followed by a space`() { + assertEquals( + "The first sentence here.", + ItemTitles.titleFor("", "<p>The first sentence here. Second one.</p>", "https://x/y") + ) + } + + @Test + fun `a two-word opening fragment is not a title on its own`() { + // "Wow." says nothing. Keep reading rather than name the article after an interjection. + assertEquals( + "Wow. Here is what the article actually says.", + ItemTitles.titleFor("", "<p>Wow. Here is what the article actually says.</p>", "https://x/y") + ) + } + + @Test + fun `a sentence ending the text needs no trailing space`() { + assertEquals("Only sentence here.", ItemTitles.titleFor("", "<p>Only sentence here.</p>", "https://x/y")) + } +} |