1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
package org.terst.feedreader.data
import kotlinx.coroutines.test.runTest
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
/**
* The reported bug came with "extractedContent cached: true", which is the half
* that a fixed extractor cannot reach: the cookie disclaimer was already written
* to `articles.extractedContent`, and every open after that read it straight back
* out without ever looking at it again.
*
* So a poisoned cache has to be recognised and dropped on read, not just kept out
* on write.
*/
class CachedExtractionTest {
private val cookieNotice =
"<div id=\"readability-page-1\" class=\"page\"><div id=\"didomi-host\"><p>This website uses " +
"cookies and other tracking technologies (“Cookies”) to enhance user experience, to " +
"display targeted ads, and to analyze performance and user activity on our website. For " +
"more information about our use of Cookies and our privacy practices, see our Privacy " +
"Policy. By continuing to use this website, you agree to our Terms and Conditions.</p>" +
"</div></div>"
@Test
fun `real cached content is used as-is and the cache is left alone`() = runTest {
var cleared = false
val content = "<p>" + "The actual article body. ".repeat(60) + "</p>"
val result = loadCachedExtraction(content) { cleared = true }
assertEquals(content, result.content)
assertFalse(result.discardedConsentNotice)
assertFalse(result.discarded)
assertFalse("a good cache must not be cleared", cleared)
}
@Test
fun `a cached consent notice is discarded and deleted`() = runTest {
var cleared = false
val result = loadCachedExtraction(cookieNotice) { cleared = true }
assertNull("the notice must not be shown as the article", result.content)
assertTrue(result.discardedConsentNotice)
assertTrue("the poisoned row must be cleared so a re-extract can replace it", cleared)
}
@Test
fun `nothing cached touches nothing`() = runTest {
var cleared = false
val result = loadCachedExtraction(null) { cleared = true }
assertNull(result.content)
assertFalse(result.discardedConsentNotice)
assertFalse(cleared)
}
// Same shape of problem as the consent notice, different cause: this content was
// extracted before extraction knew how to render maths, so it holds the raw TeX source
// Substack ships. A fixed extractor never gets a look at it -- the cached copy is
// returned first and the article is never re-extracted.
@Test
fun `a cached extraction holding raw tex source is discarded and deleted`() = runTest {
var cleared = false
val rawTex = "<div id=\"readability-page-1\"><p>Imagine the following sentence:</p>" +
"<div data-component-name=\"Latex\"><p><span>" +
"\\(p = \\text{\u201Cif }p\\text{ is true, then pigs can fly\u201D}\\)</span></p></div>" +
"<p>" + "Considered as a whole, the statement cannot be true. ".repeat(20) + "</p></div>"
val result = loadCachedExtraction(rawTex) { cleared = true }
assertNull("raw TeX source must not be shown as the article", result.content)
assertTrue(result.discardedUnrenderedTex)
assertTrue("the stale row must be cleared so a re-extract can replace it", cleared)
assertTrue("the caller must be told to extract again", result.discarded)
}
@Test
fun `an extraction with rendered maths is kept`() = runTest {
var cleared = false
val withMathMl = "<p>A slice of the set looks like this:</p>" +
"<math xmlns=\"http://www.w3.org/1998/Math/MathML\" display=\"block\"><semantics>" +
"<mrow><mi>p</mi><mo>=</mo></mrow></semantics></math>" +
"<p>" + "The rest of the article continues from there. ".repeat(20) + "</p>"
val result = loadCachedExtraction(withMathMl) { cleared = true }
assertEquals(withMathMl, result.content)
assertFalse(result.discardedUnrenderedTex)
assertFalse("a good cache must not be cleared", cleared)
}
// A blank value is as good as nothing cached -- it is not a consent notice, so
// reporting it as a discarded one would make the caller re-extract on a false
// premise.
@Test
fun `a blank cached value is not reported as a discarded notice`() = runTest {
var cleared = false
val result = loadCachedExtraction(" ") { cleared = true }
assertFalse(result.discardedConsentNotice)
assertFalse(cleared)
}
/**
* A cached article carrying the publisher's "Trending" promo widgets. Unlike a cookie
* notice, the article itself is fine -- only the widgets have to go, and re-extracting
* would be a pointless round trip that produced the same body again.
*/
@Test
fun `a cached article has its recirculation widgets stripped without being re-extracted`() = runTest {
var cleared = false
val body = "<p>" + "The actual article body. ".repeat(60) + "</p>"
val widget = "<div data-mrf-recirculation=\"IL_x\"><p><h3><span>Trending</span></h3></p></div>"
val result = loadCachedExtraction(body + widget + body) { cleared = true }
assertFalse("the stray header survived: ${result.content}", result.content!!.contains("Trending"))
assertTrue(result.content!!.contains("The actual article body."))
assertFalse("the article was fine; nothing should be thrown away", result.discarded)
assertFalse("re-extracting would fetch the same body again", cleared)
}
}
|
