diff options
Diffstat (limited to 'app')
| -rw-r--r-- | app/build.gradle.kts | 10 | ||||
| -rw-r--r-- | app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt | 218 | ||||
| -rw-r--r-- | app/src/debug/res/xml/network_security_config.xml | 17 |
3 files changed, 245 insertions, 0 deletions
diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 03b240e..42bdbed 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -18,6 +18,7 @@ android { targetSdk = 34 versionCode = 1 versionName = "1.0" + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" } buildTypes { @@ -74,6 +75,15 @@ dependencies { testImplementation("junit:junit:4.13.2") testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.0") testImplementation("androidx.room:room-testing:2.6.1") + + // Instrumented tests: ReaderExtractor needs a real WebView to run Readability.js, so + // extraction-pipeline regression tests can't be plain JVM unit tests. MockWebServer serves + // fixture HTML over real http:// -- ReaderExtractor's WebView has allowFileAccess = false + // (deliberately, see its own comments), so file:// fixtures wouldn't load. + androidTestImplementation("androidx.test.ext:junit:1.2.1") + androidTestImplementation("androidx.test:runner:1.6.2") + androidTestImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.0") + androidTestImplementation("com.squareup.okhttp3:mockwebserver:4.12.0") } detekt { diff --git a/app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt b/app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt new file mode 100644 index 0000000..db5ed3a --- /dev/null +++ b/app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt @@ -0,0 +1,218 @@ +package org.terst.feedreader.readability + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import kotlinx.coroutines.runBlocking +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.After +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith + +/** + * Regression coverage for ReaderExtractor's WHT/Ars Technica-specific preprocessing (see + * BACKLOG.md "Extraction test coverage for WHT / Ars Technica"). Each fix so far had been + * diagnosed by hand against a single live article; these fixtures pin the specific markup + * shapes down so a future change to the extraction pipeline can't silently regress them. + * + * Needs a real WebView to run Readability.js (see ReaderExtractor's own class doc), so this is + * an instrumented test, not a plain JVM unit test. Fixtures are served over real http:// via + * MockWebServer -- ReaderExtractor's WebView has allowFileAccess = false (deliberately), so a + * file:// fixture URL wouldn't load; a debug-only network security config override permits + * cleartext to 127.0.0.1 for exactly this (see app/src/debug/res/xml/network_security_config.xml). + * + * Fixtures are minimal, hand-authored markup reproducing the exact selectors/attributes + * ReaderExtractor's preprocessing targets (grep the class for "paywall", "pswp-caption-content", + * "data-src", "post-content-double") -- not scraped live pages, which rot and would make these + * tests fail for reasons unrelated to the extraction logic. Real site markup drifts; these + * selectors are what the code actually keys off, confirmed by reading ReaderExtractor.kt itself. + */ +@RunWith(AndroidJUnit4::class) +class ReaderExtractorFixtureTest { + + private lateinit var server: MockWebServer + + @Before + fun startServer() { + server = MockWebServer() + server.start() + } + + @After + fun stopServer() { + server.shutdown() + } + + private fun extract(html: String): ExtractedArticle? { + server.enqueue(MockResponse().setBody(html).setResponseCode(200)) + val url = server.url("/article").toString() + val context = InstrumentationRegistry.getInstrumentation().targetContext + return runBlocking { ReaderExtractor.extract(context, url) } + } + + // West Hawaii Today's hidden paywall div is literally class="paywall paywall_<id>" -- + // ReaderExtractor only un-hides elements matching that (or premium-content/subscriber- + // content/metered-content) class/id pattern, and only keeps the un-hidden result if it's + // longer than the normal pass. This fixture's paywalled paragraph is the only place the + // marker sentence appears, so its presence in the output proves the un-hide path ran and + // actually won (not just that Readability found the visible teaser). + private val paywallFixture = """ + <html><head><title>Storm knocks out power across Kona</title></head> + <body> + <article> + <h1>Storm knocks out power across Kona</h1> + <p>A powerful overnight storm swept through West Hawaii, leaving thousands of + households without electricity as high winds toppled trees onto power lines across + the district. Hawaii Electric Light crews were dispatched before dawn to begin + assessing the damage.</p> + <p>County officials opened two emergency shelters at community centers in Kailua-Kona + and Captain Cook for residents without power, and urged drivers to treat every + intersection with a dark traffic signal as a four-way stop.</p> + <div class="paywall paywall_a91f2c" style="display:none"> + <p>PAYWALLED_MARKER_SENTENCE crews from Hawaii Electric worked through the night + restoring the substation feeding the hardest-hit neighborhoods near Kailua-Kona, with + utility officials now projecting full restoration by Thursday evening, a day earlier + than the initial estimate given at the county's morning briefing.</p> + </div> + </article> + </body></html> + """.trimIndent() + + // West Hawaii Today (and many sites) lazy-load images: the real photo sits in data-src and + // only gets copied into src by the site's own script, which never runs here (JS-off by + // design elsewhere in the app; this WebView runs JS only to drive Readability itself). + // ReaderExtractor resolves data-src into src before Readability ever sees the page. + private val lazyImageFixture = """ + <html><head><title>New pier opens at Kailua Bay</title></head> + <body> + <article> + <h1>New pier opens at Kailua Bay</h1> + <p>The county unveiled a rebuilt public pier at Kailua Bay on Saturday, replacing a + structure damaged in last year's storm season with a wider, ADA-accessible design + intended to serve both fishing charters and the town's canoe clubs.</p> + <img data-src="https://example.com/real-pier-photo.jpg" + src="https://example.com/placeholder-1x1.gif" alt="The new Kailua Bay pier"> + <p>Canoe club members said the wider dock apron would make it far easier to launch + during the busy early-morning practice hours, when as many as six crews share the + same stretch of water before work.</p> + </article> + </body></html> + """.trimIndent() + + // Ars Technica's lightbox (PhotoSwipe) keeps its own duplicate copy of a caption in a + // ".pswp-caption-content" div sitting beside the real, normally-visible <figcaption> -- + // Readability's hidden-element skip doesn't catch it (it's not display:none), so without + // ReaderExtractor's targeted removal it comes through as a genuine second copy of the text. + private val duplicateCaptionFixture = """ + <html><head><title>Reaction wheel failures leave Swift mission spinning in orbit</title></head> + <body> + <article> + <h1>Reaction wheel failures leave Swift mission spinning in orbit</h1> + <p>Engineers are racing to diagnose a second reaction wheel failure aboard NASA's + Swift observatory, a fault that has left the spacecraft unable to maintain the + precise pointing its gamma-ray burst instruments depend on.</p> + <figure> + <img src="https://example.com/swift-observatory.jpg" alt="The Swift observatory"> + <figcaption>UNIQUE_CAPTION_TEXT The Swift observatory, imaged before launch in 2004.</figcaption> + <div class="pswp-caption-content">UNIQUE_CAPTION_TEXT The Swift observatory, imaged before launch in 2004.</div> + </figure> + <p>A second, independent wheel dropping out of alignment within the same year points + engineers toward a shared root cause -- most likely accumulated wear in the same + bearing assembly design used across all six of the spacecraft's wheels.</p> + </article> + </body></html> + """.trimIndent() + + // Ars splits a longer article's body across multiple sibling + // ".post-content.post-content-double" divs (a full ad rail sits between them in the real + // page) instead of one continuous container. Readability scores each independently and + // keeps only the highest-scoring one, silently dropping the rest -- ReaderExtractor merges + // every such div's children into the first before Readability ever runs. + private val splitContentFixture = """ + <html><head><title>A deep dive into the new orbital refueling demo</title></head> + <body> + <article> + <h1>A deep dive into the new orbital refueling demo</h1> + <div class="post-content post-content-double"> + <p>FIRST_BLOCK_MARKER the mission's opening phase focused on a slow, careful approach + between the two spacecraft, with controllers on the ground validating relative + navigation sensors before allowing the vehicles to close to within a few meters of + each other for the first time.</p> + <p>That rendezvous alone took nearly six hours, deliberately paced far more slowly + than a typical crewed docking to leave wide margins for any sensor anomaly.</p> + </div> + <div class="post-content post-content-double"> + <p>SECOND_BLOCK_MARKER once docked, the actual propellant transfer ran for just under + two hours, moving a test quantity of cryogenic fluid across the connector while + instruments on both craft monitored boil-off rates and line pressure for any sign of + a leak at the coupling.</p> + <p>Engineers called the transfer itself the least eventful part of the whole + demonstration, with the docking and undocking sequences drawing far more scrutiny in + the post-mission review.</p> + </div> + </article> + </body></html> + """.trimIndent() + + @Test + fun paywalledContentIsUnhiddenAndExtracted() { + val result = extract(paywallFixture) + + assertNotNull("extraction should succeed", result) + assertNotNull("should have extracted content", result?.textContent) + assertTrue( + "paywalled paragraph should be present in extracted text", + result?.textContent?.contains("PAYWALLED_MARKER_SENTENCE") == true + ) + } + + @Test + fun lazyLoadedImageSrcIsResolved() { + val result = extract(lazyImageFixture) + + assertNotNull("extraction should succeed", result) + val content = result?.content + assertNotNull("should have extracted HTML content", content) + assertTrue( + "extracted content should reference the real photo URL, not the placeholder", + content?.contains("real-pier-photo.jpg") == true + ) + assertFalse( + "extracted content should not still reference the lazy placeholder", + content?.contains("placeholder-1x1.gif") == true + ) + } + + @Test + fun duplicatePswpCaptionIsRemovedWhenRealFigcaptionExists() { + val result = extract(duplicateCaptionFixture) + + assertNotNull("extraction should succeed", result) + val text = result?.textContent.orEmpty() + val occurrences = Regex("UNIQUE_CAPTION_TEXT").findAll(text).count() + assertTrue("caption marker should appear at least once (the real figcaption)", occurrences >= 1) + assertTrue( + "caption marker should appear exactly once, not duplicated by the pswp-caption-content div " + + "(found $occurrences occurrences)", + occurrences == 1 + ) + } + + @Test + fun splitPostContentDoubleBlocksAreMerged() { + val result = extract(splitContentFixture) + + assertNotNull("extraction should succeed", result) + val text = result?.textContent.orEmpty() + assertTrue("first .post-content-double block should be present", text.contains("FIRST_BLOCK_MARKER")) + assertTrue( + "second .post-content-double block should also be present (not dropped by Readability " + + "picking only the highest-scoring candidate)", + text.contains("SECOND_BLOCK_MARKER") + ) + } +} diff --git a/app/src/debug/res/xml/network_security_config.xml b/app/src/debug/res/xml/network_security_config.xml new file mode 100644 index 0000000..f953272 --- /dev/null +++ b/app/src/debug/res/xml/network_security_config.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + Debug-build-only override of app/src/main/res/xml/network_security_config.xml (AGP resource + merging picks the most specific source set for a same-named resource, so this fully replaces + the main one in debug builds; release is untouched). Adds cleartext-permitted 127.0.0.1 so + ReaderExtractor's WebView can load fixture HTML from MockWebServer in instrumented tests + (app/src/androidTest/.../ReaderExtractorFixtureTest.kt); the instrumented test and the WebView + it drives both run on-device, so the server is reached via loopback, not 10.0.2.2. +--> +<network-security-config> + <domain-config cleartextTrafficPermitted="true"> + <domain includeSubdomains="false">webpolicy.org</domain> + <domain includeSubdomains="false">bit-player.org</domain> + <domain includeSubdomains="false">127.0.0.1</domain> + <domain includeSubdomains="false">localhost</domain> + </domain-config> +</network-security-config> |