1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
package org.terst.feedreader.readability
/**
* Recognises an extraction result that is the page's cookie/consent notice rather
* than its article.
*
* Why this is needed at all: Readability's first pass discards "unlikely
* candidates" by class/id, which a consent banner normally is. But when that pass
* comes back under Readability's own 500-character threshold -- which is what
* happens whenever the real article body is not in the DOM at the moment the
* script runs -- it retries with that stripping switched OFF. A consent banner is
* then a perfectly ordinary candidate, and the ones sites actually ship are long
* enough to clear the threshold (the reported New Republic/Didomi notice is 510
* characters). Readability hands it back as the article, with the right title
* attached, and nothing downstream could tell the difference.
*
* The bar for rejecting something has to be high: throwing away a real article is
* worse than showing a cookie notice. So two independent, deliberately narrow
* signals, and a hard length cap on both -- a real article body is far longer than
* any banner, so nothing long is ever rejected no matter what it says.
*/
object ConsentNotice {
/**
* Plain-text length past which a result is never treated as a banner, however
* much consent boilerplate it contains. Readability's own `charThreshold` is
* 500; this leaves generous room above it for the longest real banners
* (multi-paragraph "we and our 847 partners" notices) while staying far below
* any actual article.
*/
private const val MAX_NOTICE_TEXT_LENGTH = 2_000
/** How many distinct consent phrases a result needs before wording alone convicts it. */
private const val MIN_PHRASE_MATCHES = 2
/**
* Container ids belonging to consent-management platforms. Readability strips
* class attributes but preserves ids, so the CMP's own container id survives
* into the extracted HTML -- the most precise signal available, since no real
* article body is ever inside one of these.
*/
private val CMP_CONTAINER_IDS = listOf(
"didomi-host",
"didomi-notice",
"didomi-popup",
"onetrust-banner-sdk",
"onetrust-consent-sdk",
"onetrust-policy-text",
"optanon-alert-box-wrapper",
"cybotcookiebotdialog",
"qc-cmp2-container",
"sp_message_container",
"truste-consent-track",
"usercentrics-root",
"usercentrics-cmp-ui",
"osano-cm-dialog",
"iubenda-cs-banner",
"cookie-law-info-bar",
"cookie-notice",
"cookie-banner",
"cookieconsent",
"cmpbox",
)
/**
* Phrases that only really occur in consent boilerplate. Individually weak --
* an article can mention any one of them -- so [MIN_PHRASE_MATCHES] are
* required, and only within the length cap.
*/
private val CONSENT_PHRASES = listOf(
"uses cookies",
"use cookies",
"using cookies",
"tracking technologies",
"by continuing to use this",
"by continuing to browse",
"accept all cookies",
"reject all",
"manage your preferences",
"manage preferences",
"cookie preferences",
"cookie policy",
"cookie settings",
"we and our partners",
"our advertising and analytics",
"your privacy choices",
"consent to our use of cookies",
"personalise content and ads",
"personalize content and ads",
)
/**
* True when [html] is a consent/cookie notice rather than article content.
*
* Takes the stored HTML rather than Readability's `textContent` so the same
* check works on a value already cached in `articles.extractedContent`, where
* the text form was never kept.
*/
fun looksLikeConsentNotice(html: String?): Boolean {
if (html.isNullOrBlank()) return false
// Length is judged on the text, not the markup: a banner wrapped in enough
// nested divs and inline styles clears 2KB of characters while still being
// three sentences of text. Checked first, so nothing long is ever rejected
// however much consent boilerplate it happens to quote.
val text = plainText(html)
val withinNoticeLength = text.isNotBlank() && text.length <= MAX_NOTICE_TEXT_LENGTH
return withinNoticeLength && (hasCmpContainer(html) || readsAsConsentBoilerplate(text))
}
private fun hasCmpContainer(html: String): Boolean {
val markup = html.lowercase()
return CMP_CONTAINER_IDS.any { markup.contains("id=\"$it") || markup.contains("id='$it") }
}
private fun readsAsConsentBoilerplate(text: String): Boolean {
val lowerText = text.lowercase()
return CONSENT_PHRASES.count { lowerText.contains(it) } >= MIN_PHRASE_MATCHES
}
/**
* Crude tag strip -- enough to measure length and match phrases. Deliberately
* not a parser: this runs on every article open, and getting the character
* count roughly right is all the decision needs.
*/
private fun plainText(html: String): String =
html.replace(TAG_REGEX, " ").replace(WHITESPACE_REGEX, " ").trim()
private val TAG_REGEX = Regex("<[^>]*>")
private val WHITESPACE_REGEX = Regex("\\s+")
}
|
