1 files changed, 218 insertions, 0 deletions
diff --git a/app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt b/app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt
new file mode 100644
index 0000000..db5ed3a
--- /dev/null
+++ b/app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt
@@ -0,0 +1,218 @@
+package org.terst.feedreader.readability
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import kotlinx.coroutines.runBlocking
+import okhttp3.mockwebserver.MockResponse
+import okhttp3.mockwebserver.MockWebServer
+import org.junit.After
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+
+/**
+ * Regression coverage for ReaderExtractor's WHT/Ars Technica-specific preprocessing (see
+ * BACKLOG.md "Extraction test coverage for WHT / Ars Technica"). Each fix so far had been
+ * diagnosed by hand against a single live article; these fixtures pin the specific markup
+ * shapes down so a future change to the extraction pipeline can't silently regress them.
+ *
+ * Needs a real WebView to run Readability.js (see ReaderExtractor's own class doc), so this is
+ * an instrumented test, not a plain JVM unit test. Fixtures are served over real http:// via
+ * MockWebServer -- ReaderExtractor's WebView has allowFileAccess = false (deliberately), so a
+ * file:// fixture URL wouldn't load; a debug-only network security config override permits
+ * cleartext to 127.0.0.1 for exactly this (see app/src/debug/res/xml/network_security_config.xml).
+ *
+ * Fixtures are minimal, hand-authored markup reproducing the exact selectors/attributes
+ * ReaderExtractor's preprocessing targets (grep the class for "paywall", "pswp-caption-content",
+ * "data-src", "post-content-double") -- not scraped live pages, which rot and would make these
+ * tests fail for reasons unrelated to the extraction logic. Real site markup drifts; these
+ * selectors are what the code actually keys off, confirmed by reading ReaderExtractor.kt itself.
+ */
+@RunWith(AndroidJUnit4::class)
+class ReaderExtractorFixtureTest {
+
+    private lateinit var server: MockWebServer
+
+    @Before
+    fun startServer() {
+        server = MockWebServer()
+        server.start()
+    }
+
+    @After
+    fun stopServer() {
+        server.shutdown()
+    }
+
+    private fun extract(html: String): ExtractedArticle? {
+        server.enqueue(MockResponse().setBody(html).setResponseCode(200))
+        val url = server.url("/article").toString()
+        val context = InstrumentationRegistry.getInstrumentation().targetContext
+        return runBlocking { ReaderExtractor.extract(context, url) }
+    }
+
+    // West Hawaii Today's hidden paywall div is literally class="paywall paywall_<id>" --
+    // ReaderExtractor only un-hides elements matching that (or premium-content/subscriber-
+    // content/metered-content) class/id pattern, and only keeps the un-hidden result if it's
+    // longer than the normal pass. This fixture's paywalled paragraph is the only place the
+    // marker sentence appears, so its presence in the output proves the un-hide path ran and
+    // actually won (not just that Readability found the visible teaser).
+    private val paywallFixture = """
+        <html><head><title>Storm knocks out power across Kona</title></head>
+        <body>
+        <article>
+        <h1>Storm knocks out power across Kona</h1>
+        <p>A powerful overnight storm swept through West Hawaii, leaving thousands of
+        households without electricity as high winds toppled trees onto power lines across
+        the district. Hawaii Electric Light crews were dispatched before dawn to begin
+        assessing the damage.</p>
+        <p>County officials opened two emergency shelters at community centers in Kailua-Kona
+        and Captain Cook for residents without power, and urged drivers to treat every
+        intersection with a dark traffic signal as a four-way stop.</p>
+        <div class="paywall paywall_a91f2c" style="display:none">
+        <p>PAYWALLED_MARKER_SENTENCE crews from Hawaii Electric worked through the night
+        restoring the substation feeding the hardest-hit neighborhoods near Kailua-Kona, with
+        utility officials now projecting full restoration by Thursday evening, a day earlier
+        than the initial estimate given at the county's morning briefing.</p>
+        </div>
+        </article>
+        </body></html>
+    """.trimIndent()
+
+    // West Hawaii Today (and many sites) lazy-load images: the real photo sits in data-src and
+    // only gets copied into src by the site's own script, which never runs here (JS-off by
+    // design elsewhere in the app; this WebView runs JS only to drive Readability itself).
+    // ReaderExtractor resolves data-src into src before Readability ever sees the page.
+    private val lazyImageFixture = """
+        <html><head><title>New pier opens at Kailua Bay</title></head>
+        <body>
+        <article>
+        <h1>New pier opens at Kailua Bay</h1>
+        <p>The county unveiled a rebuilt public pier at Kailua Bay on Saturday, replacing a
+        structure damaged in last year's storm season with a wider, ADA-accessible design
+        intended to serve both fishing charters and the town's canoe clubs.</p>
+        <img data-src="https://example.com/real-pier-photo.jpg"
+             src="https://example.com/placeholder-1x1.gif" alt="The new Kailua Bay pier">
+        <p>Canoe club members said the wider dock apron would make it far easier to launch
+        during the busy early-morning practice hours, when as many as six crews share the
+        same stretch of water before work.</p>
+        </article>
+        </body></html>
+    """.trimIndent()
+
+    // Ars Technica's lightbox (PhotoSwipe) keeps its own duplicate copy of a caption in a
+    // ".pswp-caption-content" div sitting beside the real, normally-visible <figcaption> --
+    // Readability's hidden-element skip doesn't catch it (it's not display:none), so without
+    // ReaderExtractor's targeted removal it comes through as a genuine second copy of the text.
+    private val duplicateCaptionFixture = """
+        <html><head><title>Reaction wheel failures leave Swift mission spinning in orbit</title></head>
+        <body>
+        <article>
+        <h1>Reaction wheel failures leave Swift mission spinning in orbit</h1>
+        <p>Engineers are racing to diagnose a second reaction wheel failure aboard NASA's
+        Swift observatory, a fault that has left the spacecraft unable to maintain the
+        precise pointing its gamma-ray burst instruments depend on.</p>
+        <figure>
+          <img src="https://example.com/swift-observatory.jpg" alt="The Swift observatory">
+          <figcaption>UNIQUE_CAPTION_TEXT The Swift observatory, imaged before launch in 2004.</figcaption>
+          <div class="pswp-caption-content">UNIQUE_CAPTION_TEXT The Swift observatory, imaged before launch in 2004.</div>
+        </figure>
+        <p>A second, independent wheel dropping out of alignment within the same year points
+        engineers toward a shared root cause -- most likely accumulated wear in the same
+        bearing assembly design used across all six of the spacecraft's wheels.</p>
+        </article>
+        </body></html>
+    """.trimIndent()
+
+    // Ars splits a longer article's body across multiple sibling
+    // ".post-content.post-content-double" divs (a full ad rail sits between them in the real
+    // page) instead of one continuous container. Readability scores each independently and
+    // keeps only the highest-scoring one, silently dropping the rest -- ReaderExtractor merges
+    // every such div's children into the first before Readability ever runs.
+    private val splitContentFixture = """
+        <html><head><title>A deep dive into the new orbital refueling demo</title></head>
+        <body>
+        <article>
+        <h1>A deep dive into the new orbital refueling demo</h1>
+        <div class="post-content post-content-double">
+        <p>FIRST_BLOCK_MARKER the mission's opening phase focused on a slow, careful approach
+        between the two spacecraft, with controllers on the ground validating relative
+        navigation sensors before allowing the vehicles to close to within a few meters of
+        each other for the first time.</p>
+        <p>That rendezvous alone took nearly six hours, deliberately paced far more slowly
+        than a typical crewed docking to leave wide margins for any sensor anomaly.</p>
+        </div>
+        <div class="post-content post-content-double">
+        <p>SECOND_BLOCK_MARKER once docked, the actual propellant transfer ran for just under
+        two hours, moving a test quantity of cryogenic fluid across the connector while
+        instruments on both craft monitored boil-off rates and line pressure for any sign of
+        a leak at the coupling.</p>
+        <p>Engineers called the transfer itself the least eventful part of the whole
+        demonstration, with the docking and undocking sequences drawing far more scrutiny in
+        the post-mission review.</p>
+        </div>
+        </article>
+        </body></html>
+    """.trimIndent()
+
+    @Test
+    fun paywalledContentIsUnhiddenAndExtracted() {
+        val result = extract(paywallFixture)
+
+        assertNotNull("extraction should succeed", result)
+        assertNotNull("should have extracted content", result?.textContent)
+        assertTrue(
+            "paywalled paragraph should be present in extracted text",
+            result?.textContent?.contains("PAYWALLED_MARKER_SENTENCE") == true
+        )
+    }
+
+    @Test
+    fun lazyLoadedImageSrcIsResolved() {
+        val result = extract(lazyImageFixture)
+
+        assertNotNull("extraction should succeed", result)
+        val content = result?.content
+        assertNotNull("should have extracted HTML content", content)
+        assertTrue(
+            "extracted content should reference the real photo URL, not the placeholder",
+            content?.contains("real-pier-photo.jpg") == true
+        )
+        assertFalse(
+            "extracted content should not still reference the lazy placeholder",
+            content?.contains("placeholder-1x1.gif") == true
+        )
+    }
+
+    @Test
+    fun duplicatePswpCaptionIsRemovedWhenRealFigcaptionExists() {
+        val result = extract(duplicateCaptionFixture)
+
+        assertNotNull("extraction should succeed", result)
+        val text = result?.textContent.orEmpty()
+        val occurrences = Regex("UNIQUE_CAPTION_TEXT").findAll(text).count()
+        assertTrue("caption marker should appear at least once (the real figcaption)", occurrences >= 1)
+        assertTrue(
+            "caption marker should appear exactly once, not duplicated by the pswp-caption-content div " +
+                "(found $occurrences occurrences)",
+            occurrences == 1
+        )
+    }
+
+    @Test
+    fun splitPostContentDoubleBlocksAreMerged() {
+        val result = extract(splitContentFixture)
+
+        assertNotNull("extraction should succeed", result)
+        val text = result?.textContent.orEmpty()
+        assertTrue("first .post-content-double block should be present", text.contains("FIRST_BLOCK_MARKER"))
+        assertTrue(
+            "second .post-content-double block should also be present (not dropped by Readability " +
+                "picking only the highest-scoring candidate)",
+            text.contains("SECOND_BLOCK_MARKER")
+        )
+    }
+}