1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
package org.terst.feedreader.readability
/**
* Removes "recirculation" widgets -- the promo blocks a publisher's widget platform injects
* into the middle of the article body to send the reader off to other stories.
*
* Reported live: West Hawaii Today articles "get a bunch of stray 'Trending' headers".
* Confirmed by running the vendored Readability.js over the rendered page for the 12 articles
* in that feed: 9 carried two surviving widgets each. The three that didn't were paywalled
* stubs too short for the widgets to be reached.
*
* Readability keeps them because they are not in a sidebar -- they are inside the article's own
* container, between real paragraphs, so every structural signal says they belong. What arrives
* is a heading labelling nothing, because the list of links under it is filled in later by a
* script that never runs in the reader:
*
* ```
* <div data-mrf-recirculation="IL_..." data-testid="text-html"><p><h3><span>Trending</span></h3></p></div>
* ```
*
* The signal is the attribute, not the word "Trending": `data-mrf-recirculation` says what the
* block is for, in the widget platform's own vocabulary, and no article body is ever inside one.
* Matching on the heading text instead would be both narrower (the label varies by site and
* section) and more dangerous (an article may legitimately discuss what is trending).
*
* [MAX_WIDGET_TEXT_LENGTH] is the same safety valve [ConsentNotice] uses: however the attribute
* is applied, nothing with real writing in it is ever thrown away.
*/
object RecirculationWidgets {
/**
* Text length past which a block is kept whatever attribute it carries. A widget is a label
* and a few headlines; anything longer is somebody's writing and is not ours to delete.
*/
private const val MAX_WIDGET_TEXT_LENGTH = 600
/**
* Attributes marking a block as recirculation. Only ones whose own name says so -- this list
* grows on evidence from a real article, never on a guess about what a vendor prefix means.
*/
private val WIDGET_ATTRIBUTES = listOf("data-mrf-recirculation")
private val OPENING_DIV = Regex("<div\\b[^>]*>", RegexOption.IGNORE_CASE)
private val TAG_REGEX = Regex("<[^>]*>")
private val WHITESPACE_REGEX = Regex("\\s+")
private const val OPEN_DIV = "<div"
private const val CLOSE_DIV = "</div"
/** [html] with any recirculation widget removed. */
fun strip(html: String?): String? =
if (html.isNullOrBlank() || WIDGET_ATTRIBUTES.none { html.contains(it, ignoreCase = true) }) {
html
} else {
removeWidgets(html)
}
private fun removeWidgets(html: String): String {
val out = StringBuilder(html.length)
var cursor = 0
var open = OPENING_DIV.find(html, cursor)
while (open != null) {
val end = widgetEnd(html, open)
if (end == null) {
// Not a widget, or not safe to remove. Copy up to and including this opening
// tag: an ordinary div that merely sits before a widget must not go with it.
out.append(html, cursor, open.range.last + 1)
cursor = open.range.last + 1
} else {
out.append(html, cursor, open.range.first)
cursor = end
}
open = OPENING_DIV.find(html, cursor)
}
out.append(html, cursor, html.length)
return out.toString()
}
/**
* Where the block opened at [open] ends, or null when it should be left exactly as it is --
* because it is not a widget, because the markup never closes it, or because it holds more
* text than a widget ever does and is therefore somebody's writing.
*/
private fun widgetEnd(html: String, open: MatchResult): Int? =
if (!marksRecirculation(open.value)) {
null
} else {
matchingClose(html, open.range.last + 1)
?.takeIf { !tooLongToBeAWidget(html.substring(open.range.first, it)) }
}
private fun marksRecirculation(openingTag: String) =
WIDGET_ATTRIBUTES.any { openingTag.contains(it, ignoreCase = true) }
/**
* The index just past the `</div>` closing the div that opened before [from], or null when
* the markup never closes it.
*/
private fun matchingClose(html: String, from: Int): Int? {
var depth = 1
var i = from
var end: Int? = null
while (end == null && i < html.length) {
val next = html.indexOf('<', i)
when {
next < 0 -> i = html.length
html.startsWith(CLOSE_DIV, next, ignoreCase = true) -> {
val close = html.indexOf('>', next)
depth--
when {
close < 0 -> i = html.length
depth == 0 -> end = close + 1
else -> i = close + 1
}
}
html.startsWith(OPEN_DIV, next, ignoreCase = true) -> {
depth++
i = next + OPEN_DIV.length
}
else -> i = next + 1
}
}
return end
}
private fun tooLongToBeAWidget(block: String): Boolean =
block.replace(TAG_REGEX, " ").replace(WHITESPACE_REGEX, " ").trim().length > MAX_WIDGET_TEXT_LENGTH
}
|
