1 files changed, 132 insertions, 0 deletions
diff --git a/app/src/main/java/org/terst/feedreader/readability/ConsentNotice.kt b/app/src/main/java/org/terst/feedreader/readability/ConsentNotice.kt
new file mode 100644
index 0000000..5699595
--- /dev/null
+++ b/app/src/main/java/org/terst/feedreader/readability/ConsentNotice.kt
@@ -0,0 +1,132 @@
+package org.terst.feedreader.readability
+
+/**
+ * Recognises an extraction result that is the page's cookie/consent notice rather
+ * than its article.
+ *
+ * Why this is needed at all: Readability's first pass discards "unlikely
+ * candidates" by class/id, which a consent banner normally is. But when that pass
+ * comes back under Readability's own 500-character threshold -- which is what
+ * happens whenever the real article body is not in the DOM at the moment the
+ * script runs -- it retries with that stripping switched OFF. A consent banner is
+ * then a perfectly ordinary candidate, and the ones sites actually ship are long
+ * enough to clear the threshold (the reported New Republic/Didomi notice is 510
+ * characters). Readability hands it back as the article, with the right title
+ * attached, and nothing downstream could tell the difference.
+ *
+ * The bar for rejecting something has to be high: throwing away a real article is
+ * worse than showing a cookie notice. So two independent, deliberately narrow
+ * signals, and a hard length cap on both -- a real article body is far longer than
+ * any banner, so nothing long is ever rejected no matter what it says.
+ */
+object ConsentNotice {
+
+    /**
+     * Plain-text length past which a result is never treated as a banner, however
+     * much consent boilerplate it contains. Readability's own `charThreshold` is
+     * 500; this leaves generous room above it for the longest real banners
+     * (multi-paragraph "we and our 847 partners" notices) while staying far below
+     * any actual article.
+     */
+    private const val MAX_NOTICE_TEXT_LENGTH = 2_000
+
+    /** How many distinct consent phrases a result needs before wording alone convicts it. */
+    private const val MIN_PHRASE_MATCHES = 2
+
+    /**
+     * Container ids belonging to consent-management platforms. Readability strips
+     * class attributes but preserves ids, so the CMP's own container id survives
+     * into the extracted HTML -- the most precise signal available, since no real
+     * article body is ever inside one of these.
+     */
+    private val CMP_CONTAINER_IDS = listOf(
+        "didomi-host",
+        "didomi-notice",
+        "didomi-popup",
+        "onetrust-banner-sdk",
+        "onetrust-consent-sdk",
+        "onetrust-policy-text",
+        "optanon-alert-box-wrapper",
+        "cybotcookiebotdialog",
+        "qc-cmp2-container",
+        "sp_message_container",
+        "truste-consent-track",
+        "usercentrics-root",
+        "usercentrics-cmp-ui",
+        "osano-cm-dialog",
+        "iubenda-cs-banner",
+        "cookie-law-info-bar",
+        "cookie-notice",
+        "cookie-banner",
+        "cookieconsent",
+        "cmpbox",
+    )
+
+    /**
+     * Phrases that only really occur in consent boilerplate. Individually weak --
+     * an article can mention any one of them -- so [MIN_PHRASE_MATCHES] are
+     * required, and only within the length cap.
+     */
+    private val CONSENT_PHRASES = listOf(
+        "uses cookies",
+        "use cookies",
+        "using cookies",
+        "tracking technologies",
+        "by continuing to use this",
+        "by continuing to browse",
+        "accept all cookies",
+        "reject all",
+        "manage your preferences",
+        "manage preferences",
+        "cookie preferences",
+        "cookie policy",
+        "cookie settings",
+        "we and our partners",
+        "our advertising and analytics",
+        "your privacy choices",
+        "consent to our use of cookies",
+        "personalise content and ads",
+        "personalize content and ads",
+    )
+
+    /**
+     * True when [html] is a consent/cookie notice rather than article content.
+     *
+     * Takes the stored HTML rather than Readability's `textContent` so the same
+     * check works on a value already cached in `articles.extractedContent`, where
+     * the text form was never kept.
+     */
+    fun looksLikeConsentNotice(html: String?): Boolean {
+        if (html.isNullOrBlank()) return false
+
+        // Length is judged on the text, not the markup: a banner wrapped in enough
+        // nested divs and inline styles clears 2KB of characters while still being
+        // three sentences of text. Checked first, so nothing long is ever rejected
+        // however much consent boilerplate it happens to quote.
+        val text = plainText(html)
+        val withinNoticeLength = text.isNotBlank() && text.length <= MAX_NOTICE_TEXT_LENGTH
+
+        return withinNoticeLength && (hasCmpContainer(html) || readsAsConsentBoilerplate(text))
+    }
+
+    private fun hasCmpContainer(html: String): Boolean {
+        val markup = html.lowercase()
+        return CMP_CONTAINER_IDS.any { markup.contains("id=\"$it") || markup.contains("id='$it") }
+    }
+
+    private fun readsAsConsentBoilerplate(text: String): Boolean {
+        val lowerText = text.lowercase()
+        return CONSENT_PHRASES.count { lowerText.contains(it) } >= MIN_PHRASE_MATCHES
+    }
+
+    /**
+     * Crude tag strip -- enough to measure length and match phrases. Deliberately
+     * not a parser: this runs on every article open, and getting the character
+     * count roughly right is all the decision needs.
+     */
+    private fun plainText(html: String): String =
+        html.replace(TAG_REGEX, " ").replace(WHITESPACE_REGEX, " ").trim()
+
+    private val TAG_REGEX = Regex("<[^>]*>")
+    private val WHITESPACE_REGEX = Regex("\\s+")
+}