1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
package org.terst.feedreader.readability
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
/**
* Reported live: reader mode on a New Republic feature article showed nothing but
* the site's cookie disclaimer, and cached it.
*
* Reproduced outside the app by running the extractor's own injected script
* (ReaderExtractor's `script`, Readability.js and all) against the real page: when
* the article body is present, extraction is perfect. When the body is NOT in the
* DOM at the moment Readability runs, Readability's own fallback pass kicks in --
* its first pass strips "unlikely candidates" (which the consent banner is, its
* class says `didomi-notice-banner`), comes back under its 500-character
* threshold, and so it retries with that stripping disabled. The Didomi notice's
* single paragraph is 510 characters, i.e. just over that threshold, so it is
* accepted as the article.
*
* [EXTRACTED_COOKIE_NOTICE] below is not a hand-written approximation of that --
* it is the verbatim `content` string that reproduction produced.
*
* Nothing on the Kotlin side looked at that: any non-blank `content` counted as a
* successful extraction, got written to `articles.extractedContent`, and was never
* re-checked. Hence a permanently cached cookie disclaimer.
*/
class ConsentNoticeTest {
@Test
fun `the reported New Republic extraction is recognised as a consent notice`() {
assertTrue(ConsentNotice.looksLikeConsentNotice(EXTRACTED_COOKIE_NOTICE))
}
// Readability strips class attributes but keeps ids, so the CMP container's id
// survives into the extracted HTML and is the most precise signal available.
@Test
fun `other consent-management platforms are recognised by their container id`() {
val cmpIds = listOf(
"didomi-host",
"onetrust-banner-sdk",
"onetrust-consent-sdk",
"CybotCookiebotDialog",
"qc-cmp2-container",
"sp_message_container_1234",
"truste-consent-track",
"usercentrics-root",
"osano-cm-dialog",
"iubenda-cs-banner",
"optanon-alert-box-wrapper",
"cookie-law-info-bar",
)
for (id in cmpIds) {
val html = """<div id="readability-page-1"><div id="$id"><p>Please review your choices.</p></div></div>"""
assertTrue("$id should be recognised as a consent container", ConsentNotice.looksLikeConsentNotice(html))
}
}
// Not every cookie banner is a known CMP. A short result whose text is
// unmistakably consent boilerplate counts too, with no id to go on.
@Test
fun `a short unbranded cookie banner is recognised from its wording`() {
val html = """
<div id="readability-page-1"><div><p>We use cookies and similar tracking technologies
to personalise content and ads. By continuing to use this site you consent to our use
of cookies. You can manage your preferences at any time.</p></div></div>
""".trimIndent()
assertTrue(ConsentNotice.looksLikeConsentNotice(html))
}
// One stray phrase is not enough -- plenty of ordinary writing mentions cookies.
@Test
fun `a single incidental phrase is not enough`() {
val html = """<div id="readability-page-1"><div><p>The recipe calls for cookies.
We use cookies in the frosting too.</p></div></div>"""
assertFalse(ConsentNotice.looksLikeConsentNotice(html))
}
// The whole point is to not throw away real articles. A genuine article ABOUT
// cookie-consent law would hit every phrase there is; length is what separates
// it from a banner. Nothing long is ever rejected.
@Test
fun `a long article about cookie consent is never rejected`() {
val paragraph = "<p>We use cookies and other tracking technologies, and by continuing " +
"to use this website you consent to our use of cookies -- that is the boilerplate " +
"the GDPR produced, and this article is about where it came from. You can manage " +
"your preferences, in theory.</p>"
val html = "<div id=\"readability-page-1\"><div>${paragraph.repeat(12)}</div></div>"
assertTrue("fixture must be past the length cap", html.length > 2_000)
assertFalse(ConsentNotice.looksLikeConsentNotice(html))
}
// Short posts are normal (kottke-style linkblogging is called out in
// ReaderExtractor's own comments) and must survive.
@Test
fun `an ordinary short post is not a consent notice`() {
val html = """
<div id="readability-page-1"><div><p>A lovely short note about a bridge in Lisbon,
with one link and nothing else. This is the entire post.</p></div></div>
""".trimIndent()
assertFalse(ConsentNotice.looksLikeConsentNotice(html))
}
@Test
fun `null and blank content are not consent notices`() {
assertFalse(ConsentNotice.looksLikeConsentNotice(null))
assertFalse(ConsentNotice.looksLikeConsentNotice(""))
assertFalse(ConsentNotice.looksLikeConsentNotice(" \n "))
}
// Markup must not count towards the length budget: a banner wrapped in enough
// nested divs and inline styles to exceed the cap in raw characters is still a
// banner.
@Test
fun `length is measured on text not markup`() {
val padding = "<div style=\"${"border-color: rgb(0, 0, 0); ".repeat(80)}\">"
val html = padding + EXTRACTED_COOKIE_NOTICE + "</div>"
assertTrue("fixture must be past the length cap in raw characters", html.length > 2_000)
assertTrue(ConsentNotice.looksLikeConsentNotice(html))
}
companion object {
/**
* Verbatim output of running ReaderExtractor's injected script against the
* reported page in a state where the article body was absent. Do not tidy
* this up -- its value is that it is exactly what the app produced.
*/
const val EXTRACTED_COOKIE_NOTICE: String =
"<div id=\"readability-page-1\" class=\"page\"><div id=\"didomi-host\" data-testid=\"notice\" " +
"data-nosnippet=\"true\" lang=\"en\" data-lang-dir=\"ltr\"><p>This website uses cookies and " +
"other tracking technologies (“Cookies”) to enhance user experience, to display " +
"targeted ads, and to analyze performance and user activity on our website. We also may " +
"share information about your activity on our website with our advertising and analytics " +
"providers. For more information about our use of Cookies and our privacy practices, see " +
"our <a href=\"https://newrepublic.com/pages/privacy\" target=\"_blank\">Privacy Policy</a>. " +
"By continuing to use this website, you agree to our " +
"<a href=\"https://newrepublic.com/pages/terms-and-conditions\" target=\"_blank\">Terms and " +
"Conditions</a>, including our dispute resolution provisions.</p></div></div>"
}
}
|
