1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
package org.terst.feedreader.readability
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
/**
* Promo blocks a publisher's widget platform injects into the middle of the article body.
*
* Reported live on West Hawaii Today: "articles get a bunch of stray 'Trending' headers".
* Confirmed on 9 of the 12 articles in its feed, two per article.
*/
class RecirculationWidgetsTest {
/** The exact markup that survives extraction, from a West Hawaii Today article. */
private val realWidget =
"""<div data-mrf-recirculation="IL_gKyXYBmnRnamdDxChJw6lw" data-testid="text-html" """ +
"""data-mrf-experience="IL_gKyXYBmnRnamdDxChJw6lw"><p><h3><span>Trending</span></h3></p></div>"""
@Test
fun `strips a recirculation widget from the middle of an article`() {
val html = "<p>Real sentence one.</p>$realWidget<p>Real sentence two.</p>"
val cleaned = RecirculationWidgets.strip(html)!!
assertFalse("the stray header is still there: $cleaned", cleaned.contains("Trending"))
assertTrue(cleaned.contains("Real sentence one."))
assertTrue(cleaned.contains("Real sentence two."))
}
@Test
fun `strips every widget, not just the first`() {
val html = "<p>One.</p>$realWidget<p>Two.</p>$realWidget<p>Three.</p>"
val cleaned = RecirculationWidgets.strip(html)!!
assertEquals(0, Regex("Trending").findAll(cleaned).count())
assertTrue(cleaned.contains("Three."))
}
@Test
fun `leaves an article with no widgets exactly as it was`() {
val html = "<div><p>Nothing to do here.</p><h2>A real heading</h2><p>More.</p></div>"
assertEquals(html, RecirculationWidgets.strip(html))
}
@Test
fun `never removes a block long enough to be real writing`() {
val paragraph = "Beaches across the island lost coral cover. ".repeat(20)
val html = """<div data-mrf-recirculation="X"><p>$paragraph</p></div>"""
assertTrue(
"a block this long is article text whatever attribute it carries",
RecirculationWidgets.strip(html)!!.contains("Beaches across the island")
)
}
@Test
fun `handles an unquoted or single-quoted attribute`() {
val html = "<p>a</p><div data-mrf-recirculation='Y'><h3>Trending</h3></div><p>b</p>"
assertFalse(RecirculationWidgets.strip(html)!!.contains("Trending"))
}
@Test
fun `survives a nested div inside the widget`() {
val html = "<p>a</p><div data-mrf-recirculation=\"Z\"><div><h3>Trending</h3></div></div><p>b</p>"
val cleaned = RecirculationWidgets.strip(html)!!
assertFalse(cleaned.contains("Trending"))
assertTrue(cleaned.contains("<p>a</p>"))
assertTrue(cleaned.contains("<p>b</p>"))
}
@Test
fun `an unclosed widget div does not eat the rest of the article`() {
val html = "<p>Before.</p><div data-mrf-recirculation=\"W\"><h3>Trending</h3><p>After.</p>"
assertTrue(
"a malformed widget must not take the article with it",
RecirculationWidgets.strip(html)!!.contains("Before.")
)
}
@Test
fun `null and blank are left alone`() {
assertEquals(null, RecirculationWidgets.strip(null))
assertEquals("", RecirculationWidgets.strip(""))
}
/**
* The whole extracted article, captured from the live page on 2026-10-21 by running the
* vendored Readability.js over it exactly as the app does.
*
* The unit cases above use the widget markup on its own, which proves the matcher but not
* that it meets reality in one piece. This is the article Peter would have been reading.
*/
@Test
fun `the real extracted article loses both widgets and keeps its text`() {
val extracted = checkNotNull(
javaClass.getResourceAsStream("/west-hawaii-today-extracted.html")
).bufferedReader().use { it.readText() }
assertEquals("fixture should carry the reported bug", 2, Regex("Trending").findAll(extracted).count())
val cleaned = RecirculationWidgets.strip(extracted)!!
assertEquals(0, Regex("Trending").findAll(cleaned).count())
assertTrue(cleaned.contains("Leavitt officially stepped down as White House press secretary"))
assertTrue(cleaned.contains("The New York Times"))
assertTrue(
"barely any of the article should have gone with the widgets",
cleaned.length > extracted.length - 600
)
}
}
|
