diff options
Diffstat (limited to 'app/src/test')
6 files changed, 410 insertions, 0 deletions
diff --git a/app/src/test/java/org/terst/feedreader/data/CachedExtractionTest.kt b/app/src/test/java/org/terst/feedreader/data/CachedExtractionTest.kt index fc9c32e..442663f 100644 --- a/app/src/test/java/org/terst/feedreader/data/CachedExtractionTest.kt +++ b/app/src/test/java/org/terst/feedreader/data/CachedExtractionTest.kt @@ -96,4 +96,23 @@ class CachedExtractionTest { assertFalse(result.discardedConsentNotice) assertFalse(cleared) } + + /** + * A cached article carrying the publisher's "Trending" promo widgets. Unlike a cookie + * notice, the article itself is fine -- only the widgets have to go, and re-extracting + * would be a pointless round trip that produced the same body again. + */ + @Test + fun `a cached article has its recirculation widgets stripped without being re-extracted`() = runTest { + var cleared = false + val body = "<p>" + "The actual article body. ".repeat(60) + "</p>" + val widget = "<div data-mrf-recirculation=\"IL_x\"><p><h3><span>Trending</span></h3></p></div>" + + val result = loadCachedExtraction(body + widget + body) { cleared = true } + + assertFalse("the stray header survived: ${result.content}", result.content!!.contains("Trending")) + assertTrue(result.content!!.contains("The actual article body.")) + assertFalse("the article was fine; nothing should be thrown away", result.discarded) + assertFalse("re-extracting would fetch the same body again", cleared) + } } diff --git a/app/src/test/java/org/terst/feedreader/data/FeedDiscoveryTest.kt b/app/src/test/java/org/terst/feedreader/data/FeedDiscoveryTest.kt new file mode 100644 index 0000000..eee238a --- /dev/null +++ b/app/src/test/java/org/terst/feedreader/data/FeedDiscoveryTest.kt @@ -0,0 +1,93 @@ +package org.terst.feedreader.data + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +/** + * Finding the feed when what you were given is a site. + * + * Reported live: "Can't subscribe to https://heathercoxrichardson.substack.com/". Pasting a + * site's home page was the only thing to hand -- a site does not advertise its feed URL + * anywhere a reader can see, it advertises it in a <link rel="alternate"> that every other + * reader looks for and this one did not. + */ +class FeedDiscoveryTest { + + /** Verbatim from the reported page, 2026-10-21. */ + private val substackHead = + """<!doctype html><html><head><title>Letters from an American</title>""" + + """<link rel="alternate" type="application/rss+xml" href="/feed" """ + + """title="Letters from an American"/></head><body>...</body></html>""" + + @Test + fun `finds the feed a Substack home page advertises, resolved against the site`() { + assertEquals( + "https://heathercoxrichardson.substack.com/feed", + FeedDiscovery.findFeedUrl(substackHead, "https://heathercoxrichardson.substack.com/") + ) + } + + @Test + fun `finds an Atom feed too`() { + val html = """<link rel="alternate" type="application/atom+xml" href="/atom.xml">""" + + assertEquals("https://example.com/atom.xml", FeedDiscovery.findFeedUrl(html, "https://example.com/")) + } + + @Test + fun `keeps an absolute href as it stands`() { + val html = """<link rel="alternate" type="application/rss+xml" href="https://feeds.example.net/x.xml">""" + + assertEquals("https://feeds.example.net/x.xml", FeedDiscovery.findFeedUrl(html, "https://example.com/blog/")) + } + + @Test + fun `resolves a relative href against the page, not the site root`() { + val html = """<link rel="alternate" type="application/rss+xml" href="feed.xml">""" + + assertEquals("https://example.com/blog/feed.xml", FeedDiscovery.findFeedUrl(html, "https://example.com/blog/")) + } + + @Test + fun `copes with the attributes in any order and single quotes`() { + val html = """<link type='application/rss+xml' href='/rss' rel='alternate'>""" + + assertEquals("https://example.com/rss", FeedDiscovery.findFeedUrl(html, "https://example.com")) + } + + @Test + fun `prefers the first feed when a page advertises several`() { + val html = + """<link rel="alternate" type="application/rss+xml" href="/feed" title="Posts">""" + + """<link rel="alternate" type="application/rss+xml" href="/comments/feed" title="Comments">""" + + assertEquals("https://example.com/feed", FeedDiscovery.findFeedUrl(html, "https://example.com/")) + } + + @Test + fun `ignores an alternate link that is not a feed`() { + val html = + """<link rel="alternate" hreflang="fr" href="/fr/">""" + + """<link rel="alternate" type="application/json" href="/feed.json">""" + + assertNull(FeedDiscovery.findFeedUrl(html, "https://example.com/")) + } + + @Test + fun `a page with no feed at all reports nothing rather than guessing a path`() { + assertNull(FeedDiscovery.findFeedUrl("<html><head></head><body>hi</body></html>", "https://example.com/")) + } + + @Test + fun `an http feed link is upgraded, since the app cannot fetch cleartext`() { + val html = """<link rel="alternate" type="application/rss+xml" href="http://example.com/feed">""" + + assertEquals("https://example.com/feed", FeedDiscovery.findFeedUrl(html, "https://example.com/")) + } + + @Test + fun `xml declaration means this was already a feed, so there is nothing to discover`() { + assertNull(FeedDiscovery.findFeedUrl("<?xml version=\"1.0\"?><rss><channel></channel></rss>", "https://x/f")) + } +} diff --git a/app/src/test/java/org/terst/feedreader/readability/EvalOutcomeTest.kt b/app/src/test/java/org/terst/feedreader/readability/EvalOutcomeTest.kt index 8995e22..3e2b28f 100644 --- a/app/src/test/java/org/terst/feedreader/readability/EvalOutcomeTest.kt +++ b/app/src/test/java/org/terst/feedreader/readability/EvalOutcomeTest.kt @@ -1,6 +1,7 @@ package org.terst.feedreader.readability import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test @@ -86,4 +87,21 @@ class EvalOutcomeTest { assertTrue(f.userMessage().lowercase().contains("cookie") || f.userMessage().lowercase().contains("consent")) assertTrue(f.userMessage() != ExtractionFailure.NoContent.userMessage()) } + + /** + * A fresh extraction is cleaned before it is accepted, so the widgets never reach the + * screen and never reach `articles.extractedContent` either. + */ + @Test + fun `a successful extraction has its recirculation widgets stripped`() { + val body = "<p>" + "Real article sentence. ".repeat(40) + "</p>" + val widget = "<div data-mrf-recirculation=\"IL_x\"><p><h3><span>Trending</span></h3></p></div>" + val article = ExtractedArticle(title = "T", content = body + widget + body) + + val decision = decideEvalOutcome(article, rawResult = "{}", attempt = 1, maxAttempts = 6) + + val content = ((decision as EvalDecision.Done).result as ExtractionResult.Success).article.content!! + assertFalse("the stray header reached the caller: $content", content.contains("Trending")) + assertTrue(content.contains("Real article sentence.")) + } } diff --git a/app/src/test/java/org/terst/feedreader/readability/RecirculationWidgetsTest.kt b/app/src/test/java/org/terst/feedreader/readability/RecirculationWidgetsTest.kt new file mode 100644 index 0000000..0926b80 --- /dev/null +++ b/app/src/test/java/org/terst/feedreader/readability/RecirculationWidgetsTest.kt @@ -0,0 +1,117 @@ +package org.terst.feedreader.readability + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * Promo blocks a publisher's widget platform injects into the middle of the article body. + * + * Reported live on West Hawaii Today: "articles get a bunch of stray 'Trending' headers". + * Confirmed on 9 of the 12 articles in its feed, two per article. + */ +class RecirculationWidgetsTest { + + /** The exact markup that survives extraction, from a West Hawaii Today article. */ + private val realWidget = + """<div data-mrf-recirculation="IL_gKyXYBmnRnamdDxChJw6lw" data-testid="text-html" """ + + """data-mrf-experience="IL_gKyXYBmnRnamdDxChJw6lw"><p><h3><span>Trending</span></h3></p></div>""" + + @Test + fun `strips a recirculation widget from the middle of an article`() { + val html = "<p>Real sentence one.</p>$realWidget<p>Real sentence two.</p>" + + val cleaned = RecirculationWidgets.strip(html)!! + + assertFalse("the stray header is still there: $cleaned", cleaned.contains("Trending")) + assertTrue(cleaned.contains("Real sentence one.")) + assertTrue(cleaned.contains("Real sentence two.")) + } + + @Test + fun `strips every widget, not just the first`() { + val html = "<p>One.</p>$realWidget<p>Two.</p>$realWidget<p>Three.</p>" + + val cleaned = RecirculationWidgets.strip(html)!! + + assertEquals(0, Regex("Trending").findAll(cleaned).count()) + assertTrue(cleaned.contains("Three.")) + } + + @Test + fun `leaves an article with no widgets exactly as it was`() { + val html = "<div><p>Nothing to do here.</p><h2>A real heading</h2><p>More.</p></div>" + + assertEquals(html, RecirculationWidgets.strip(html)) + } + + @Test + fun `never removes a block long enough to be real writing`() { + val paragraph = "Beaches across the island lost coral cover. ".repeat(20) + val html = """<div data-mrf-recirculation="X"><p>$paragraph</p></div>""" + + assertTrue( + "a block this long is article text whatever attribute it carries", + RecirculationWidgets.strip(html)!!.contains("Beaches across the island") + ) + } + + @Test + fun `handles an unquoted or single-quoted attribute`() { + val html = "<p>a</p><div data-mrf-recirculation='Y'><h3>Trending</h3></div><p>b</p>" + + assertFalse(RecirculationWidgets.strip(html)!!.contains("Trending")) + } + + @Test + fun `survives a nested div inside the widget`() { + val html = "<p>a</p><div data-mrf-recirculation=\"Z\"><div><h3>Trending</h3></div></div><p>b</p>" + + val cleaned = RecirculationWidgets.strip(html)!! + assertFalse(cleaned.contains("Trending")) + assertTrue(cleaned.contains("<p>a</p>")) + assertTrue(cleaned.contains("<p>b</p>")) + } + + @Test + fun `an unclosed widget div does not eat the rest of the article`() { + val html = "<p>Before.</p><div data-mrf-recirculation=\"W\"><h3>Trending</h3><p>After.</p>" + + assertTrue( + "a malformed widget must not take the article with it", + RecirculationWidgets.strip(html)!!.contains("Before.") + ) + } + + @Test + fun `null and blank are left alone`() { + assertEquals(null, RecirculationWidgets.strip(null)) + assertEquals("", RecirculationWidgets.strip("")) + } + + /** + * The whole extracted article, captured from the live page on 2026-10-21 by running the + * vendored Readability.js over it exactly as the app does. + * + * The unit cases above use the widget markup on its own, which proves the matcher but not + * that it meets reality in one piece. This is the article Peter would have been reading. + */ + @Test + fun `the real extracted article loses both widgets and keeps its text`() { + val extracted = checkNotNull( + javaClass.getResourceAsStream("/west-hawaii-today-extracted.html") + ).bufferedReader().use { it.readText() } + assertEquals("fixture should carry the reported bug", 2, Regex("Trending").findAll(extracted).count()) + + val cleaned = RecirculationWidgets.strip(extracted)!! + + assertEquals(0, Regex("Trending").findAll(cleaned).count()) + assertTrue(cleaned.contains("Leavitt officially stepped down as White House press secretary")) + assertTrue(cleaned.contains("The New York Times")) + assertTrue( + "barely any of the article should have gone with the widgets", + cleaned.length > extracted.length - 600 + ) + } +} diff --git a/app/src/test/java/org/terst/feedreader/ui/YouTubeLinksTest.kt b/app/src/test/java/org/terst/feedreader/ui/YouTubeLinksTest.kt new file mode 100644 index 0000000..3afba16 --- /dev/null +++ b/app/src/test/java/org/terst/feedreader/ui/YouTubeLinksTest.kt @@ -0,0 +1,139 @@ +package org.terst.feedreader.ui + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The YouTube placeholder: recognising one, and building a page its embed will actually play in. + * + * Both halves come from the same report: "YouTube play is misconfigured, long click on YouTube + * placeholder should open the URL directly." + */ +class YouTubeLinksTest { + + /** + * What a long-press on the reader's video placeholder actually reports. + * + * Not the link's href, which is what the first version of this assumed and got wrong on the + * device: the placeholder is `<a href="...watch?v=ID"><img src="...i.ytimg.com/vi/ID/..."></a>`, + * and for an image inside a link WebView's hit test reports SRC_IMAGE_ANCHOR_TYPE with the + * *image's* URL in `extra`, not the anchor's. Long-pressing the thumbnail produced the + * ordinary link menu showing `https://i.ytimg.com/vi/dQw4w9WgXcQ/hq720.jpg`. + * + * The thumbnail URL carries the video id, so it is resolved back to the video. + */ + @Test + fun `a long-press on the thumbnail resolves to the video, which is what WebView reports`() { + assertEquals( + "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + youTubeTargetForLongPress("https://i.ytimg.com/vi/dQw4w9WgXcQ/hq720.jpg") + ) + } + + @Test + fun `any thumbnail size resolves, since the reader has used more than one`() { + assertEquals( + "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + youTubeTargetForLongPress("https://i.ytimg.com/vi/dQw4w9WgXcQ/maxresdefault.jpg") + ) + assertEquals( + "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + youTubeTargetForLongPress("https://i9.ytimg.com/vi_webp/dQw4w9WgXcQ/hq720.webp") + ) + } + + @Test + fun `a watch link resolves to itself, for wherever one does come through`() { + assertEquals( + "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + youTubeTargetForLongPress("https://www.youtube.com/watch?v=dQw4w9WgXcQ") + ) + } + + @Test + fun `a short youtu-be link resolves to the video`() { + assertEquals( + "https://www.youtube.com/watch?v=dQw4w9WgXcQ", + youTubeTargetForLongPress("https://youtu.be/dQw4w9WgXcQ") + ) + } + + @Test + fun `an ordinary article link is not a video`() { + assertNull(youTubeTargetForLongPress("https://www.nytimes.com/2026/10/20/us/politics/x.html")) + } + + @Test + fun `an ordinary article image is not a video`() { + assertNull(youTubeTargetForLongPress("https://www.westhawaiitoday.com/wp-content/uploads/reef.jpg")) + } + + @Test + fun `a lookalike host is not YouTube`() { + assertNull(youTubeTargetForLongPress("https://notyoutube.com/watch?v=dQw4w9WgXcQ")) + assertNull(youTubeTargetForLongPress("https://youtube.com.evil.example/watch?v=dQw4w9WgXcQ")) + assertNull(youTubeTargetForLongPress("https://i.ytimg.com.evil.example/vi/dQw4w9WgXcQ/hq720.jpg")) + } + + @Test + fun `a YouTube link that is not one video stays on the ordinary menu`() { + assertNull(youTubeTargetForLongPress("https://www.youtube.com/@someone")) + assertNull(youTubeTargetForLongPress("https://www.youtube.com/playlist?list=PL123")) + } + + @Test + fun `rubbish is not a video`() { + assertNull(youTubeTargetForLongPress("")) + assertNull(youTubeTargetForLongPress("not a url at all")) + } + + // --- the embed page ------------------------------------------------------------------ + + @Test + fun `the embed page puts the player in an iframe, which is the whole point`() { + val html = youTubeEmbedPageHtml("dQw4w9WgXcQ") + + assertTrue("loading the embed top-level is what YouTube rejects: $html", html.contains("<iframe")) + assertTrue(html.contains("https://www.youtube-nocookie.com/embed/dQw4w9WgXcQ")) + } + + @Test + fun `the embed page keeps autoplay and inline playback`() { + val html = youTubeEmbedPageHtml("dQw4w9WgXcQ") + + assertTrue(html.contains("autoplay=1")) + assertTrue(html.contains("playsinline=1")) + assertTrue("without this the iframe is not allowed to autoplay", html.contains("allow=\"autoplay")) + } + + @Test + fun `the base URL is a real https origin, since that is what the embed requires`() { + assertTrue(YOUTUBE_EMBED_BASE_URL.startsWith("https://")) + } + + @Test + fun `the base URL is not YouTube's own origin`() { + assertFalse( + "the embed has to be a guest on somebody else's page, not its own", + YOUTUBE_EMBED_BASE_URL.contains("youtube") + ) + } + + @Test + fun `a video id that could break out of the attribute is scrubbed`() { + val html = youTubeEmbedPageHtml("""a"><script>alert(1)</script>""") + + assertFalse("markup from the id reached the page: $html", html.contains("<script")) + assertFalse(html.contains("alert(1)")) + // What is left is only ever id characters, so the src attribute still closes where it should. + assertTrue(html.contains("embed/ascriptalert1script?")) + } + + @Test + fun `the watch link for a video is the one a phone can hand to the YouTube app`() { + assertEquals("https://www.youtube.com/watch?v=dQw4w9WgXcQ", youTubeWatchUrl("dQw4w9WgXcQ")) + } +} diff --git a/app/src/test/resources/west-hawaii-today-extracted.html b/app/src/test/resources/west-hawaii-today-extracted.html new file mode 100644 index 0000000..5c52608 --- /dev/null +++ b/app/src/test/resources/west-hawaii-today-extracted.html @@ -0,0 +1,24 @@ +<div id="readability-page-1" class="page"><div id="article_info"> + + + + + + <p>Published 12:05 am Wednesday, October 21, 2026</p> + + <div> + <p> + <strong>By Maggie Haberman and Jonathan Swan NYTimes News Service</strong> + </p> + + </div><!-- END BYLINE --> + </div><div id="article_content"><p>WASHINGTON — In late August, during one of Karoline Leavitt’s final meetings with President Donald Trump in the Oval Office, he pitched her on an idea in front of several senior aides. He told her to move reporters out of the James S. Brady Press Briefing Room, which would effectively remove news organizations from the White House. It would be, he told her, her “last legacy.”</p> +<p>But the proposal never moved forward while she was there. The president had put the request to her several times over months, according to three people with knowledge of the discussions who were not authorized to speak publicly. Leavitt officially stepped down as White House press secretary on Aug. 27.</p> +<p>Trump has told aides he would prefer to bring back the indoor pool President Franklin D. Roosevelt added in 1933 as therapy for his polio-afflicted legs; President Richard Nixon later covered it over to make room for the news media.</p> +<div data-mrf-recirculation="IL_gKyXYBmnRnamdDxChJw6lw" data-testid="text-html" data-mrf-experience="IL_gKyXYBmnRnamdDxChJw6lw"><p><h3><span>Trending</span></h3></p></div><p>While moving the media out of the White House may be a relatively small debate considering Trump’s attacks on journalists more broadly, it would be highly symbolic. The proximity of the media to the seat of U.S. executive power has long been an acknowledgment of the importance of a free press in a constitutional democracy.</p> +<p>Trump has brought up the press-eviction idea before, dating to his first term, and he has never quite dropped it.</p> +<p>As recently as Sept. 27, Trump shared a pair of images on Truth Social showing the briefing room as it is now, captioned “Fake News,” alongside another that shows the space reimagined as a gilded swimming pool, captioned “FDR Pool.”</p> +<p>The Wall Street Journal reported last month that Trump had repeatedly discussed moving the press corps out of the White House, and that his advisers had so far either offered pushback or waited for him to move on.</p> +<p>What has changed since Trump’s first term or even earlier this term is the staff around him. The president’s core team has shrunk from the already-tight unit that he brought into the White House last year. During earlier discussions, most of Trump’s senior aides agreed it served him better to keep the media nearby. But Trump has grown more enraged about his press coverage, and other advisers of growing influence have encouraged him to take drastic measures to punish journalists.</p> +<p><ins>This article originally appeared in <a href="https://www.nytimes.com/2026/10/20/us/politics/trump-white-house-media.html" data-mrf-link="https://www.nytimes.com/2026/10/20/us/politics/trump-white-house-media.html" cmp-ltrk="Article Links" cmp-ltrk-idx="0" mrfobservableid="0c7048ed-f27d-4067-a945-5b92fb782ba2">The New York Times</a>. </ins></p> +<div data-mrf-recirculation="IL_p3LFgBtiQmCCCy-bXT17QA" data-testid="text-html" data-mrf-experience="IL_p3LFgBtiQmCCCy-bXT17QA"><p><h3><span>Trending</span></h3></p></div><p><em>© 2026 The New York Times Company</em></p></div></div>
\ No newline at end of file |