Add instrumented extraction fixture tests for WHT/Ars Technica

BACKLOG's "Extraction test coverage for WHT / Ars Technica" item:
ReaderExtractor's site-specific preprocessing had only ever been
diagnosed by hand against single live articles across several rounds
of fixes. Four hand-authored fixtures now pin down the exact behavior:

- West Hawaii Today's paywall-div unhide (class="paywall paywall_*")
- Lazy-loaded image src resolution (data-src -> src)
- Ars Technica's duplicate PhotoSwipe caption removal
  (.pswp-caption-content vs. the real <figcaption>)
- Ars's split .post-content.post-content-double block merge

Needs a real WebView to run Readability.js, so these are instrumented
tests (app/src/androidTest/), not plain JVM unit tests -- that source
set didn't exist yet (testInstrumentationRunner + androidx.test deps
added to app/build.gradle.kts). Fixtures are minimal hand-authored
markup targeting the exact selectors ReaderExtractor's own code uses,
not scraped live pages, which rot and would fail for reasons unrelated
to the extraction logic.

Fixtures are served over real http:// via MockWebServer, since
ReaderExtractor's WebView has allowFileAccess = false and won't load
file:// URLs. Added a debug-only network security config override
(app/src/debug/res/xml/network_security_config.xml) permitting
cleartext to 127.0.0.1/localhost for this -- release is untouched.

Verified the tests actually catch regressions, not just pass
trivially: temporarily broke the paywall-unhide selector, confirmed
the corresponding test failed on both connected emulators, then
reverted and confirmed green again.

Ran on 2 connected devices (feedreader_test, doot_test_api36): 4/4
passing, 0 failures. Instrumented tests aren't counted by the JVM-only
jacoco coverage metric (expected, not a gate miss -- see CLAUDE.md).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017GMEkHeqKz6FLkmizowBTK

3 files changed, 245 insertions, 0 deletions

diff --git a/app/build.gradle.kts b/app/build.gradle.kts
index 03b240e..42bdbed 100644
--- a/app/build.gradle.kts
+++ b/app/build.gradle.kts
@@ -18,6 +18,7 @@ android {
         targetSdk = 34
         versionCode = 1
         versionName = "1.0"
+        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
     }
 
     buildTypes {
@@ -74,6 +75,15 @@ dependencies {
     testImplementation("junit:junit:4.13.2")
     testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.0")
     testImplementation("androidx.room:room-testing:2.6.1")
+
+    // Instrumented tests: ReaderExtractor needs a real WebView to run Readability.js, so
+    // extraction-pipeline regression tests can't be plain JVM unit tests. MockWebServer serves
+    // fixture HTML over real http:// -- ReaderExtractor's WebView has allowFileAccess = false
+    // (deliberately, see its own comments), so file:// fixtures wouldn't load.
+    androidTestImplementation("androidx.test.ext:junit:1.2.1")
+    androidTestImplementation("androidx.test:runner:1.6.2")
+    androidTestImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.0")
+    androidTestImplementation("com.squareup.okhttp3:mockwebserver:4.12.0")
 }
 
 detekt {
diff --git a/app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt b/app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt
new file mode 100644
index 0000000..db5ed3a
--- /dev/null
+++ b/app/src/androidTest/java/org/terst/feedreader/readability/ReaderExtractorFixtureTest.kt
@@ -0,0 +1,218 @@
+package org.terst.feedreader.readability
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import kotlinx.coroutines.runBlocking
+import okhttp3.mockwebserver.MockResponse
+import okhttp3.mockwebserver.MockWebServer
+import org.junit.After
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+
+/**
+ * Regression coverage for ReaderExtractor's WHT/Ars Technica-specific preprocessing (see
+ * BACKLOG.md "Extraction test coverage for WHT / Ars Technica"). Each fix so far had been
+ * diagnosed by hand against a single live article; these fixtures pin the specific markup
+ * shapes down so a future change to the extraction pipeline can't silently regress them.
+ *
+ * Needs a real WebView to run Readability.js (see ReaderExtractor's own class doc), so this is
+ * an instrumented test, not a plain JVM unit test. Fixtures are served over real http:// via
+ * MockWebServer -- ReaderExtractor's WebView has allowFileAccess = false (deliberately), so a
+ * file:// fixture URL wouldn't load; a debug-only network security config override permits
+ * cleartext to 127.0.0.1 for exactly this (see app/src/debug/res/xml/network_security_config.xml).
+ *
+ * Fixtures are minimal, hand-authored markup reproducing the exact selectors/attributes
+ * ReaderExtractor's preprocessing targets (grep the class for "paywall", "pswp-caption-content",
+ * "data-src", "post-content-double") -- not scraped live pages, which rot and would make these
+ * tests fail for reasons unrelated to the extraction logic. Real site markup drifts; these
+ * selectors are what the code actually keys off, confirmed by reading ReaderExtractor.kt itself.
+ */
+@RunWith(AndroidJUnit4::class)
+class ReaderExtractorFixtureTest {
+
+    private lateinit var server: MockWebServer
+
+    @Before
+    fun startServer() {
+        server = MockWebServer()
+        server.start()
+    }
+
+    @After
+    fun stopServer() {
+        server.shutdown()
+    }
+
+    private fun extract(html: String): ExtractedArticle? {
+        server.enqueue(MockResponse().setBody(html).setResponseCode(200))
+        val url = server.url("/article").toString()
+        val context = InstrumentationRegistry.getInstrumentation().targetContext
+        return runBlocking { ReaderExtractor.extract(context, url) }
+    }
+
+    // West Hawaii Today's hidden paywall div is literally class="paywall paywall_<id>" --
+    // ReaderExtractor only un-hides elements matching that (or premium-content/subscriber-
+    // content/metered-content) class/id pattern, and only keeps the un-hidden result if it's
+    // longer than the normal pass. This fixture's paywalled paragraph is the only place the
+    // marker sentence appears, so its presence in the output proves the un-hide path ran and
+    // actually won (not just that Readability found the visible teaser).
+    private val paywallFixture = """
+        <html><head><title>Storm knocks out power across Kona</title></head>
+        <body>
+        <article>
+        <h1>Storm knocks out power across Kona</h1>
+        <p>A powerful overnight storm swept through West Hawaii, leaving thousands of
+        households without electricity as high winds toppled trees onto power lines across
+        the district. Hawaii Electric Light crews were dispatched before dawn to begin
+        assessing the damage.</p>
+        <p>County officials opened two emergency shelters at community centers in Kailua-Kona
+        and Captain Cook for residents without power, and urged drivers to treat every
+        intersection with a dark traffic signal as a four-way stop.</p>
+        <div class="paywall paywall_a91f2c" style="display:none">
+        <p>PAYWALLED_MARKER_SENTENCE crews from Hawaii Electric worked through the night
+        restoring the substation feeding the hardest-hit neighborhoods near Kailua-Kona, with
+        utility officials now projecting full restoration by Thursday evening, a day earlier
+        than the initial estimate given at the county's morning briefing.</p>
+        </div>
+        </article>
+        </body></html>
+    """.trimIndent()
+
+    // West Hawaii Today (and many sites) lazy-load images: the real photo sits in data-src and
+    // only gets copied into src by the site's own script, which never runs here (JS-off by
+    // design elsewhere in the app; this WebView runs JS only to drive Readability itself).
+    // ReaderExtractor resolves data-src into src before Readability ever sees the page.
+    private val lazyImageFixture = """
+        <html><head><title>New pier opens at Kailua Bay</title></head>
+        <body>
+        <article>
+        <h1>New pier opens at Kailua Bay</h1>
+        <p>The county unveiled a rebuilt public pier at Kailua Bay on Saturday, replacing a
+        structure damaged in last year's storm season with a wider, ADA-accessible design
+        intended to serve both fishing charters and the town's canoe clubs.</p>
+        <img data-src="https://example.com/real-pier-photo.jpg"
+             src="https://example.com/placeholder-1x1.gif" alt="The new Kailua Bay pier">
+        <p>Canoe club members said the wider dock apron would make it far easier to launch
+        during the busy early-morning practice hours, when as many as six crews share the
+        same stretch of water before work.</p>
+        </article>
+        </body></html>
+    """.trimIndent()
+
+    // Ars Technica's lightbox (PhotoSwipe) keeps its own duplicate copy of a caption in a
+    // ".pswp-caption-content" div sitting beside the real, normally-visible <figcaption> --
+    // Readability's hidden-element skip doesn't catch it (it's not display:none), so without
+    // ReaderExtractor's targeted removal it comes through as a genuine second copy of the text.
+    private val duplicateCaptionFixture = """
+        <html><head><title>Reaction wheel failures leave Swift mission spinning in orbit</title></head>
+        <body>
+        <article>
+        <h1>Reaction wheel failures leave Swift mission spinning in orbit</h1>
+        <p>Engineers are racing to diagnose a second reaction wheel failure aboard NASA's
+        Swift observatory, a fault that has left the spacecraft unable to maintain the
+        precise pointing its gamma-ray burst instruments depend on.</p>
+        <figure>
+          <img src="https://example.com/swift-observatory.jpg" alt="The Swift observatory">
+          <figcaption>UNIQUE_CAPTION_TEXT The Swift observatory, imaged before launch in 2004.</figcaption>
+          <div class="pswp-caption-content">UNIQUE_CAPTION_TEXT The Swift observatory, imaged before launch in 2004.</div>
+        </figure>
+        <p>A second, independent wheel dropping out of alignment within the same year points
+        engineers toward a shared root cause -- most likely accumulated wear in the same
+        bearing assembly design used across all six of the spacecraft's wheels.</p>
+        </article>
+        </body></html>
+    """.trimIndent()
+
+    // Ars splits a longer article's body across multiple sibling
+    // ".post-content.post-content-double" divs (a full ad rail sits between them in the real
+    // page) instead of one continuous container. Readability scores each independently and
+    // keeps only the highest-scoring one, silently dropping the rest -- ReaderExtractor merges
+    // every such div's children into the first before Readability ever runs.
+    private val splitContentFixture = """
+        <html><head><title>A deep dive into the new orbital refueling demo</title></head>
+        <body>
+        <article>
+        <h1>A deep dive into the new orbital refueling demo</h1>
+        <div class="post-content post-content-double">
+        <p>FIRST_BLOCK_MARKER the mission's opening phase focused on a slow, careful approach
+        between the two spacecraft, with controllers on the ground validating relative
+        navigation sensors before allowing the vehicles to close to within a few meters of
+        each other for the first time.</p>
+        <p>That rendezvous alone took nearly six hours, deliberately paced far more slowly
+        than a typical crewed docking to leave wide margins for any sensor anomaly.</p>
+        </div>
+        <div class="post-content post-content-double">
+        <p>SECOND_BLOCK_MARKER once docked, the actual propellant transfer ran for just under
+        two hours, moving a test quantity of cryogenic fluid across the connector while
+        instruments on both craft monitored boil-off rates and line pressure for any sign of
+        a leak at the coupling.</p>
+        <p>Engineers called the transfer itself the least eventful part of the whole
+        demonstration, with the docking and undocking sequences drawing far more scrutiny in
+        the post-mission review.</p>
+        </div>
+        </article>
+        </body></html>
+    """.trimIndent()
+
+    @Test
+    fun paywalledContentIsUnhiddenAndExtracted() {
+        val result = extract(paywallFixture)
+
+        assertNotNull("extraction should succeed", result)
+        assertNotNull("should have extracted content", result?.textContent)
+        assertTrue(
+            "paywalled paragraph should be present in extracted text",
+            result?.textContent?.contains("PAYWALLED_MARKER_SENTENCE") == true
+        )
+    }
+
+    @Test
+    fun lazyLoadedImageSrcIsResolved() {
+        val result = extract(lazyImageFixture)
+
+        assertNotNull("extraction should succeed", result)
+        val content = result?.content
+        assertNotNull("should have extracted HTML content", content)
+        assertTrue(
+            "extracted content should reference the real photo URL, not the placeholder",
+            content?.contains("real-pier-photo.jpg") == true
+        )
+        assertFalse(
+            "extracted content should not still reference the lazy placeholder",
+            content?.contains("placeholder-1x1.gif") == true
+        )
+    }
+
+    @Test
+    fun duplicatePswpCaptionIsRemovedWhenRealFigcaptionExists() {
+        val result = extract(duplicateCaptionFixture)
+
+        assertNotNull("extraction should succeed", result)
+        val text = result?.textContent.orEmpty()
+        val occurrences = Regex("UNIQUE_CAPTION_TEXT").findAll(text).count()
+        assertTrue("caption marker should appear at least once (the real figcaption)", occurrences >= 1)
+        assertTrue(
+            "caption marker should appear exactly once, not duplicated by the pswp-caption-content div " +
+                "(found $occurrences occurrences)",
+            occurrences == 1
+        )
+    }
+
+    @Test
+    fun splitPostContentDoubleBlocksAreMerged() {
+        val result = extract(splitContentFixture)
+
+        assertNotNull("extraction should succeed", result)
+        val text = result?.textContent.orEmpty()
+        assertTrue("first .post-content-double block should be present", text.contains("FIRST_BLOCK_MARKER"))
+        assertTrue(
+            "second .post-content-double block should also be present (not dropped by Readability " +
+                "picking only the highest-scoring candidate)",
+            text.contains("SECOND_BLOCK_MARKER")
+        )
+    }
+}
diff --git a/app/src/debug/res/xml/network_security_config.xml b/app/src/debug/res/xml/network_security_config.xml
new file mode 100644
index 0000000..f953272
--- /dev/null
+++ b/app/src/debug/res/xml/network_security_config.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+  Debug-build-only override of app/src/main/res/xml/network_security_config.xml (AGP resource
+  merging picks the most specific source set for a same-named resource, so this fully replaces
+  the main one in debug builds; release is untouched). Adds cleartext-permitted 127.0.0.1 so
+  ReaderExtractor's WebView can load fixture HTML from MockWebServer in instrumented tests
+  (app/src/androidTest/.../ReaderExtractorFixtureTest.kt); the instrumented test and the WebView
+  it drives both run on-device, so the server is reached via loopback, not 10.0.2.2.
+-->
+<network-security-config>
+    <domain-config cleartextTrafficPermitted="true">
+        <domain includeSubdomains="false">webpolicy.org</domain>
+        <domain includeSubdomains="false">bit-player.org</domain>
+        <domain includeSubdomains="false">127.0.0.1</domain>
+        <domain includeSubdomains="false">localhost</domain>
+    </domain-config>
+</network-security-config>