1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
|
package org.terst.feedreader.readability
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import kotlinx.coroutines.runBlocking
import okhttp3.mockwebserver.MockResponse
import okhttp3.mockwebserver.MockWebServer
import org.junit.After
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
/**
* Regression coverage for ReaderExtractor's WHT/Ars Technica-specific preprocessing (see
* BACKLOG.md "Extraction test coverage for WHT / Ars Technica"). Each fix so far had been
* diagnosed by hand against a single live article; these fixtures pin the specific markup
* shapes down so a future change to the extraction pipeline can't silently regress them.
*
* Needs a real WebView to run Readability.js (see ReaderExtractor's own class doc), so this is
* an instrumented test, not a plain JVM unit test. Fixtures are served over real http:// via
* MockWebServer -- ReaderExtractor's WebView has allowFileAccess = false (deliberately), so a
* file:// fixture URL wouldn't load; a debug-only network security config override permits
* cleartext to 127.0.0.1 for exactly this (see app/src/debug/res/xml/network_security_config.xml).
*
* Fixtures are minimal, hand-authored markup reproducing the exact selectors/attributes
* ReaderExtractor's preprocessing targets (grep the class for "paywall", "pswp-caption-content",
* "data-src", "post-content-double") -- not scraped live pages, which rot and would make these
* tests fail for reasons unrelated to the extraction logic. Real site markup drifts; these
* selectors are what the code actually keys off, confirmed by reading ReaderExtractor.kt itself.
*/
@RunWith(AndroidJUnit4::class)
class ReaderExtractorFixtureTest {
private lateinit var server: MockWebServer
@Before
fun startServer() {
server = MockWebServer()
server.start()
}
@After
fun stopServer() {
server.shutdown()
}
private fun extract(html: String): ExtractedArticle? {
server.enqueue(MockResponse().setBody(html).setResponseCode(200))
val url = server.url("/article").toString()
val context = InstrumentationRegistry.getInstrumentation().targetContext
return runBlocking { ReaderExtractor.extract(context, url) }
}
// West Hawaii Today's hidden paywall div is literally class="paywall paywall_<id>" --
// ReaderExtractor only un-hides elements matching that (or premium-content/subscriber-
// content/metered-content) class/id pattern, and only keeps the un-hidden result if it's
// longer than the normal pass. This fixture's paywalled paragraph is the only place the
// marker sentence appears, so its presence in the output proves the un-hide path ran and
// actually won (not just that Readability found the visible teaser).
private val paywallFixture = """
<html><head><title>Storm knocks out power across Kona</title></head>
<body>
<article>
<h1>Storm knocks out power across Kona</h1>
<p>A powerful overnight storm swept through West Hawaii, leaving thousands of
households without electricity as high winds toppled trees onto power lines across
the district. Hawaii Electric Light crews were dispatched before dawn to begin
assessing the damage.</p>
<p>County officials opened two emergency shelters at community centers in Kailua-Kona
and Captain Cook for residents without power, and urged drivers to treat every
intersection with a dark traffic signal as a four-way stop.</p>
<div class="paywall paywall_a91f2c" style="display:none">
<p>PAYWALLED_MARKER_SENTENCE crews from Hawaii Electric worked through the night
restoring the substation feeding the hardest-hit neighborhoods near Kailua-Kona, with
utility officials now projecting full restoration by Thursday evening, a day earlier
than the initial estimate given at the county's morning briefing.</p>
</div>
</article>
</body></html>
""".trimIndent()
// West Hawaii Today (and many sites) lazy-load images: the real photo sits in data-src and
// only gets copied into src by the site's own script, which never runs here (JS-off by
// design elsewhere in the app; this WebView runs JS only to drive Readability itself).
// ReaderExtractor resolves data-src into src before Readability ever sees the page.
private val lazyImageFixture = """
<html><head><title>New pier opens at Kailua Bay</title></head>
<body>
<article>
<h1>New pier opens at Kailua Bay</h1>
<p>The county unveiled a rebuilt public pier at Kailua Bay on Saturday, replacing a
structure damaged in last year's storm season with a wider, ADA-accessible design
intended to serve both fishing charters and the town's canoe clubs.</p>
<img data-src="https://example.com/real-pier-photo.jpg"
src="https://example.com/placeholder-1x1.gif" alt="The new Kailua Bay pier">
<p>Canoe club members said the wider dock apron would make it far easier to launch
during the busy early-morning practice hours, when as many as six crews share the
same stretch of water before work.</p>
</article>
</body></html>
""".trimIndent()
// Ars Technica's lightbox (PhotoSwipe) keeps its own duplicate copy of a caption in a
// ".pswp-caption-content" div sitting beside the real, normally-visible <figcaption> --
// Readability's hidden-element skip doesn't catch it (it's not display:none), so without
// ReaderExtractor's targeted removal it comes through as a genuine second copy of the text.
private val duplicateCaptionFixture = """
<html><head><title>Reaction wheel failures leave Swift mission spinning in orbit</title></head>
<body>
<article>
<h1>Reaction wheel failures leave Swift mission spinning in orbit</h1>
<p>Engineers are racing to diagnose a second reaction wheel failure aboard NASA's
Swift observatory, a fault that has left the spacecraft unable to maintain the
precise pointing its gamma-ray burst instruments depend on.</p>
<figure>
<img src="https://example.com/swift-observatory.jpg" alt="The Swift observatory">
<figcaption>UNIQUE_CAPTION_TEXT The Swift observatory, imaged before launch in 2004.</figcaption>
<div class="pswp-caption-content">UNIQUE_CAPTION_TEXT The Swift observatory, imaged before launch in 2004.</div>
</figure>
<p>A second, independent wheel dropping out of alignment within the same year points
engineers toward a shared root cause -- most likely accumulated wear in the same
bearing assembly design used across all six of the spacecraft's wheels.</p>
</article>
</body></html>
""".trimIndent()
// Ars splits a longer article's body across multiple sibling
// ".post-content.post-content-double" divs (a full ad rail sits between them in the real
// page) instead of one continuous container. Readability scores each independently and
// keeps only the highest-scoring one, silently dropping the rest -- ReaderExtractor merges
// every such div's children into the first before Readability ever runs.
private val splitContentFixture = """
<html><head><title>A deep dive into the new orbital refueling demo</title></head>
<body>
<article>
<h1>A deep dive into the new orbital refueling demo</h1>
<div class="post-content post-content-double">
<p>FIRST_BLOCK_MARKER the mission's opening phase focused on a slow, careful approach
between the two spacecraft, with controllers on the ground validating relative
navigation sensors before allowing the vehicles to close to within a few meters of
each other for the first time.</p>
<p>That rendezvous alone took nearly six hours, deliberately paced far more slowly
than a typical crewed docking to leave wide margins for any sensor anomaly.</p>
</div>
<div class="post-content post-content-double">
<p>SECOND_BLOCK_MARKER once docked, the actual propellant transfer ran for just under
two hours, moving a test quantity of cryogenic fluid across the connector while
instruments on both craft monitored boil-off rates and line pressure for any sign of
a leak at the coupling.</p>
<p>Engineers called the transfer itself the least eventful part of the whole
demonstration, with the docking and undocking sequences drawing far more scrutiny in
the post-mission review.</p>
</div>
</article>
</body></html>
""".trimIndent()
@Test
fun paywalledContentIsUnhiddenAndExtracted() {
val result = extract(paywallFixture)
assertNotNull("extraction should succeed", result)
assertNotNull("should have extracted content", result?.textContent)
assertTrue(
"paywalled paragraph should be present in extracted text",
result?.textContent?.contains("PAYWALLED_MARKER_SENTENCE") == true
)
}
@Test
fun lazyLoadedImageSrcIsResolved() {
val result = extract(lazyImageFixture)
assertNotNull("extraction should succeed", result)
val content = result?.content
assertNotNull("should have extracted HTML content", content)
assertTrue(
"extracted content should reference the real photo URL, not the placeholder",
content?.contains("real-pier-photo.jpg") == true
)
assertFalse(
"extracted content should not still reference the lazy placeholder",
content?.contains("placeholder-1x1.gif") == true
)
}
@Test
fun duplicatePswpCaptionIsRemovedWhenRealFigcaptionExists() {
val result = extract(duplicateCaptionFixture)
assertNotNull("extraction should succeed", result)
val text = result?.textContent.orEmpty()
val occurrences = Regex("UNIQUE_CAPTION_TEXT").findAll(text).count()
assertTrue("caption marker should appear at least once (the real figcaption)", occurrences >= 1)
assertTrue(
"caption marker should appear exactly once, not duplicated by the pswp-caption-content div " +
"(found $occurrences occurrences)",
occurrences == 1
)
}
@Test
fun splitPostContentDoubleBlocksAreMerged() {
val result = extract(splitContentFixture)
assertNotNull("extraction should succeed", result)
val text = result?.textContent.orEmpty()
assertTrue("first .post-content-double block should be present", text.contains("FIRST_BLOCK_MARKER"))
assertTrue(
"second .post-content-double block should also be present (not dropped by Readability " +
"picking only the highest-scoring candidate)",
text.contains("SECOND_BLOCK_MARKER")
)
}
}
|
