1 files changed, 128 insertions, 0 deletions
diff --git a/app/src/main/java/org/terst/feedreader/readability/RecirculationWidgets.kt b/app/src/main/java/org/terst/feedreader/readability/RecirculationWidgets.kt
new file mode 100644
index 0000000..a6515c1
--- /dev/null
+++ b/app/src/main/java/org/terst/feedreader/readability/RecirculationWidgets.kt
@@ -0,0 +1,128 @@
+package org.terst.feedreader.readability
+
+/**
+ * Removes "recirculation" widgets -- the promo blocks a publisher's widget platform injects
+ * into the middle of the article body to send the reader off to other stories.
+ *
+ * Reported live: West Hawaii Today articles "get a bunch of stray 'Trending' headers".
+ * Confirmed by running the vendored Readability.js over the rendered page for the 12 articles
+ * in that feed: 9 carried two surviving widgets each. The three that didn't were paywalled
+ * stubs too short for the widgets to be reached.
+ *
+ * Readability keeps them because they are not in a sidebar -- they are inside the article's own
+ * container, between real paragraphs, so every structural signal says they belong. What arrives
+ * is a heading labelling nothing, because the list of links under it is filled in later by a
+ * script that never runs in the reader:
+ *
+ * ```
+ * <div data-mrf-recirculation="IL_..." data-testid="text-html"><p><h3><span>Trending</span></h3></p></div>
+ * ```
+ *
+ * The signal is the attribute, not the word "Trending": `data-mrf-recirculation` says what the
+ * block is for, in the widget platform's own vocabulary, and no article body is ever inside one.
+ * Matching on the heading text instead would be both narrower (the label varies by site and
+ * section) and more dangerous (an article may legitimately discuss what is trending).
+ *
+ * [MAX_WIDGET_TEXT_LENGTH] is the same safety valve [ConsentNotice] uses: however the attribute
+ * is applied, nothing with real writing in it is ever thrown away.
+ */
+object RecirculationWidgets {
+
+    /**
+     * Text length past which a block is kept whatever attribute it carries. A widget is a label
+     * and a few headlines; anything longer is somebody's writing and is not ours to delete.
+     */
+    private const val MAX_WIDGET_TEXT_LENGTH = 600
+
+    /**
+     * Attributes marking a block as recirculation. Only ones whose own name says so -- this list
+     * grows on evidence from a real article, never on a guess about what a vendor prefix means.
+     */
+    private val WIDGET_ATTRIBUTES = listOf("data-mrf-recirculation")
+
+    private val OPENING_DIV = Regex("<div\\b[^>]*>", RegexOption.IGNORE_CASE)
+    private val TAG_REGEX = Regex("<[^>]*>")
+    private val WHITESPACE_REGEX = Regex("\\s+")
+
+    private const val OPEN_DIV = "<div"
+    private const val CLOSE_DIV = "</div"
+
+    /** [html] with any recirculation widget removed. */
+    fun strip(html: String?): String? =
+        if (html.isNullOrBlank() || WIDGET_ATTRIBUTES.none { html.contains(it, ignoreCase = true) }) {
+            html
+        } else {
+            removeWidgets(html)
+        }
+
+    private fun removeWidgets(html: String): String {
+        val out = StringBuilder(html.length)
+        var cursor = 0
+        var open = OPENING_DIV.find(html, cursor)
+        while (open != null) {
+            val end = widgetEnd(html, open)
+            if (end == null) {
+                // Not a widget, or not safe to remove. Copy up to and including this opening
+                // tag: an ordinary div that merely sits before a widget must not go with it.
+                out.append(html, cursor, open.range.last + 1)
+                cursor = open.range.last + 1
+            } else {
+                out.append(html, cursor, open.range.first)
+                cursor = end
+            }
+            open = OPENING_DIV.find(html, cursor)
+        }
+        out.append(html, cursor, html.length)
+        return out.toString()
+    }
+
+    /**
+     * Where the block opened at [open] ends, or null when it should be left exactly as it is --
+     * because it is not a widget, because the markup never closes it, or because it holds more
+     * text than a widget ever does and is therefore somebody's writing.
+     */
+    private fun widgetEnd(html: String, open: MatchResult): Int? =
+        if (!marksRecirculation(open.value)) {
+            null
+        } else {
+            matchingClose(html, open.range.last + 1)
+                ?.takeIf { !tooLongToBeAWidget(html.substring(open.range.first, it)) }
+        }
+
+    private fun marksRecirculation(openingTag: String) =
+        WIDGET_ATTRIBUTES.any { openingTag.contains(it, ignoreCase = true) }
+
+    /**
+     * The index just past the `</div>` closing the div that opened before [from], or null when
+     * the markup never closes it.
+     */
+    private fun matchingClose(html: String, from: Int): Int? {
+        var depth = 1
+        var i = from
+        var end: Int? = null
+        while (end == null && i < html.length) {
+            val next = html.indexOf('<', i)
+            when {
+                next < 0 -> i = html.length
+                html.startsWith(CLOSE_DIV, next, ignoreCase = true) -> {
+                    val close = html.indexOf('>', next)
+                    depth--
+                    when {
+                        close < 0 -> i = html.length
+                        depth == 0 -> end = close + 1
+                        else -> i = close + 1
+                    }
+                }
+                html.startsWith(OPEN_DIV, next, ignoreCase = true) -> {
+                    depth++
+                    i = next + OPEN_DIV.length
+                }
+                else -> i = next + 1
+            }
+        }
+        return end
+    }
+
+    private fun tooLongToBeAWidget(block: String): Boolean =
+        block.replace(TAG_REGEX, " ").replace(WHITESPACE_REGEX, " ").trim().length > MAX_WIDGET_TEXT_LENGTH
+}