diff options
Diffstat (limited to 'app/src/main/java/org/terst/feedreader/readability')
4 files changed, 215 insertions, 20 deletions
diff --git a/app/src/main/java/org/terst/feedreader/readability/ConsentNotice.kt b/app/src/main/java/org/terst/feedreader/readability/ConsentNotice.kt new file mode 100644 index 0000000..5699595 --- /dev/null +++ b/app/src/main/java/org/terst/feedreader/readability/ConsentNotice.kt @@ -0,0 +1,132 @@ +package org.terst.feedreader.readability + +/** + * Recognises an extraction result that is the page's cookie/consent notice rather + * than its article. + * + * Why this is needed at all: Readability's first pass discards "unlikely + * candidates" by class/id, which a consent banner normally is. But when that pass + * comes back under Readability's own 500-character threshold -- which is what + * happens whenever the real article body is not in the DOM at the moment the + * script runs -- it retries with that stripping switched OFF. A consent banner is + * then a perfectly ordinary candidate, and the ones sites actually ship are long + * enough to clear the threshold (the reported New Republic/Didomi notice is 510 + * characters). Readability hands it back as the article, with the right title + * attached, and nothing downstream could tell the difference. + * + * The bar for rejecting something has to be high: throwing away a real article is + * worse than showing a cookie notice. So two independent, deliberately narrow + * signals, and a hard length cap on both -- a real article body is far longer than + * any banner, so nothing long is ever rejected no matter what it says. + */ +object ConsentNotice { + + /** + * Plain-text length past which a result is never treated as a banner, however + * much consent boilerplate it contains. Readability's own `charThreshold` is + * 500; this leaves generous room above it for the longest real banners + * (multi-paragraph "we and our 847 partners" notices) while staying far below + * any actual article. + */ + private const val MAX_NOTICE_TEXT_LENGTH = 2_000 + + /** How many distinct consent phrases a result needs before wording alone convicts it. */ + private const val MIN_PHRASE_MATCHES = 2 + + /** + * Container ids belonging to consent-management platforms. Readability strips + * class attributes but preserves ids, so the CMP's own container id survives + * into the extracted HTML -- the most precise signal available, since no real + * article body is ever inside one of these. + */ + private val CMP_CONTAINER_IDS = listOf( + "didomi-host", + "didomi-notice", + "didomi-popup", + "onetrust-banner-sdk", + "onetrust-consent-sdk", + "onetrust-policy-text", + "optanon-alert-box-wrapper", + "cybotcookiebotdialog", + "qc-cmp2-container", + "sp_message_container", + "truste-consent-track", + "usercentrics-root", + "usercentrics-cmp-ui", + "osano-cm-dialog", + "iubenda-cs-banner", + "cookie-law-info-bar", + "cookie-notice", + "cookie-banner", + "cookieconsent", + "cmpbox", + ) + + /** + * Phrases that only really occur in consent boilerplate. Individually weak -- + * an article can mention any one of them -- so [MIN_PHRASE_MATCHES] are + * required, and only within the length cap. + */ + private val CONSENT_PHRASES = listOf( + "uses cookies", + "use cookies", + "using cookies", + "tracking technologies", + "by continuing to use this", + "by continuing to browse", + "accept all cookies", + "reject all", + "manage your preferences", + "manage preferences", + "cookie preferences", + "cookie policy", + "cookie settings", + "we and our partners", + "our advertising and analytics", + "your privacy choices", + "consent to our use of cookies", + "personalise content and ads", + "personalize content and ads", + ) + + /** + * True when [html] is a consent/cookie notice rather than article content. + * + * Takes the stored HTML rather than Readability's `textContent` so the same + * check works on a value already cached in `articles.extractedContent`, where + * the text form was never kept. + */ + fun looksLikeConsentNotice(html: String?): Boolean { + if (html.isNullOrBlank()) return false + + // Length is judged on the text, not the markup: a banner wrapped in enough + // nested divs and inline styles clears 2KB of characters while still being + // three sentences of text. Checked first, so nothing long is ever rejected + // however much consent boilerplate it happens to quote. + val text = plainText(html) + val withinNoticeLength = text.isNotBlank() && text.length <= MAX_NOTICE_TEXT_LENGTH + + return withinNoticeLength && (hasCmpContainer(html) || readsAsConsentBoilerplate(text)) + } + + private fun hasCmpContainer(html: String): Boolean { + val markup = html.lowercase() + return CMP_CONTAINER_IDS.any { markup.contains("id=\"$it") || markup.contains("id='$it") } + } + + private fun readsAsConsentBoilerplate(text: String): Boolean { + val lowerText = text.lowercase() + return CONSENT_PHRASES.count { lowerText.contains(it) } >= MIN_PHRASE_MATCHES + } + + /** + * Crude tag strip -- enough to measure length and match phrases. Deliberately + * not a parser: this runs on every article open, and getting the character + * count roughly right is all the decision needs. + */ + private fun plainText(html: String): String = + html.replace(TAG_REGEX, " ").replace(WHITESPACE_REGEX, " ").trim() + + private val TAG_REGEX = Regex("<[^>]*>") + private val WHITESPACE_REGEX = Regex("\\s+") +} diff --git a/app/src/main/java/org/terst/feedreader/readability/EvalDecision.kt b/app/src/main/java/org/terst/feedreader/readability/EvalDecision.kt new file mode 100644 index 0000000..ed0033d --- /dev/null +++ b/app/src/main/java/org/terst/feedreader/readability/EvalDecision.kt @@ -0,0 +1,61 @@ +package org.terst.feedreader.readability + +/** What to do with one `evaluateJavascript` result. */ +sealed interface EvalDecision { + /** Nothing more to try: this is the extraction's outcome. */ + data class Done(val result: ExtractionResult) : EvalDecision + + /** The page probably was not ready; inject the script again. */ + data object Retry : EvalDecision +} + +/** + * Decides, for one script result, whether extraction succeeded, should be + * retried, or has failed and why. + * + * Lifted out of the WebView callback it used to live inside so it can be + * exercised without a device. The bug that prompted that: a result consisting + * only of the page's cookie-consent banner was non-blank and error-free, so it + * counted as a success, was cached to `articles.extractedContent`, and was never + * reconsidered. Reported live as "reader mode just extracts cookie disclaimer". + */ +internal fun decideEvalOutcome( + article: ExtractedArticle?, + rawResult: String, + attempt: Int, + maxAttempts: Int +): EvalDecision { + val readabilityError = article?.error?.takeIf { it.isNotBlank() } + // A banner is exactly what a half-loaded page produces -- the consent script + // runs early, the article body may not be in the DOM yet -- so this is not + // usable, which routes it into the same retry the extractor already does for + // an empty result rather than being accepted. + val isConsentNotice = article != null && + readabilityError == null && + ConsentNotice.looksLikeConsentNotice(article.content) + val usable = article != null && + readabilityError == null && + !article.content.isNullOrBlank() && + !isConsentNotice + + return when { + usable -> EvalDecision.Done(ExtractionResult.Success(article!!)) + + // Readability ran and objected: that verdict will not change. + readabilityError != null -> + EvalDecision.Done(ExtractionResult.Failure(ExtractionFailure.ReadabilityError(readabilityError))) + + attempt < maxAttempts -> EvalDecision.Retry + + // Every attempt came back with nothing but the consent notice, so this is + // what the page actually serves -- report it as its own failure rather + // than passing a cookie disclaimer off as the article. + isConsentNotice -> + EvalDecision.Done(ExtractionResult.Failure(ExtractionFailure.ConsentInterstitial)) + + article == null -> + EvalDecision.Done(ExtractionResult.Failure(ExtractionFailure.BadResult(rawResult))) + + else -> EvalDecision.Done(ExtractionResult.Failure(ExtractionFailure.NoContent)) + } +} diff --git a/app/src/main/java/org/terst/feedreader/readability/ExtractionFailure.kt b/app/src/main/java/org/terst/feedreader/readability/ExtractionFailure.kt index 532b109..63fbdb2 100644 --- a/app/src/main/java/org/terst/feedreader/readability/ExtractionFailure.kt +++ b/app/src/main/java/org/terst/feedreader/readability/ExtractionFailure.kt @@ -28,6 +28,12 @@ sealed interface ExtractionFailure { /** The JS result could not be decoded. */ data class BadResult(val raw: String) : ExtractionFailure + + /** + * Readability returned content, but it was the page's cookie/consent notice, + * not the article -- see [ConsentNotice]. + */ + data object ConsentInterstitial : ExtractionFailure } private const val MILLIS_PER_SECOND = 1_000 @@ -47,6 +53,7 @@ fun ExtractionFailure.userMessage(): String = when (this) { is ExtractionFailure.ReadabilityError -> "Couldn't parse the article ($message)" ExtractionFailure.NoContent -> "No article text found on the page" is ExtractionFailure.BadResult -> "Unexpected result from the page" + ExtractionFailure.ConsentInterstitial -> "Only the site's cookie notice came through, not the article" } /** @@ -61,4 +68,6 @@ fun ExtractionFailure.diagnostic(): String = when (this) { is ExtractionFailure.ReadabilityError -> "ReadabilityError $message" ExtractionFailure.NoContent -> "NoContent (Readability returned no usable body)" is ExtractionFailure.BadResult -> "BadResult raw=${raw.take(RAW_DIAGNOSTIC_CHARS)}" + ExtractionFailure.ConsentInterstitial -> + "ConsentInterstitial (extracted content was the page's cookie/consent notice)" } diff --git a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt index 5061444..21633b4 100644 --- a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt +++ b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt @@ -436,10 +436,9 @@ object ReaderExtractor { * Runs the extraction script, retrying while the result looks like * the page simply was not ready yet. * - * A Readability error is final -- retrying will not change what the - * parser thinks of the markup. A null result or empty content is - * not: both are what an injection against a half-loaded document - * produces, and the next attempt usually succeeds. + * The accept/retry/give-up decision itself lives in + * [decideEvalOutcome], which is testable without a WebView; this + * only decodes the payload and carries the decision out. */ private fun evaluate(view: WebView, script: String, attempt: Int) { if (finished) return @@ -451,24 +450,18 @@ object ReaderExtractor { val article = unwrapped?.let { runCatching { json.decodeFromString<ExtractedArticle>(it) }.getOrNull() } - val usable = article != null && - article.error.isNullOrBlank() && - !article.content.isNullOrBlank() - when { - usable -> finish(ExtractionResult.Success(article!!)) - - // Readability ran and objected: that verdict will not change. - article != null && !article.error.isNullOrBlank() -> - fail(ExtractionFailure.ReadabilityError(article.error)) - - attempt < MAX_EVAL_ATTEMPTS -> + when ( + val decision = decideEvalOutcome( + article = article, + rawResult = rawResult ?: "null", + attempt = attempt, + maxAttempts = MAX_EVAL_ATTEMPTS + ) + ) { + is EvalDecision.Done -> finish(decision.result) + EvalDecision.Retry -> view.postDelayed({ evaluate(view, script, attempt + 1) }, EVAL_RETRY_DELAY_MS) - - article == null -> - fail(ExtractionFailure.BadResult(rawResult ?: "null")) - - else -> fail(ExtractionFailure.NoContent) } } } |