path: root/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt

package org.terst.feedreader.readability

import android.annotation.SuppressLint
import android.content.Context
import android.os.Handler
import android.os.Looper
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebView
import android.webkit.WebViewClient
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import kotlin.coroutines.resume

@Serializable
data class ExtractedArticle(
    val title: String? = null,
    val content: String? = null,       // HTML
    val textContent: String? = null,
    val excerpt: String? = null,
    val error: String? = null
)

/**
 * The outcome of an extraction: either usable content, or a reason it failed.
 *
 * extract() used to return a bare ExtractedArticle?, so every distinct failure
 * — a dead host, an HTTP 403, a timeout, a Readability throw — arrived at the
 * caller as the same null, and the caller dropped it.
 */
sealed interface ExtractionResult {
    data class Success(val article: ExtractedArticle) : ExtractionResult
    data class Failure(val reason: ExtractionFailure) : ExtractionResult
}

/**
 * Runs Mozilla's Readability.js (vendored in assets/Readability.js, pinned
 * to release 0.6.0) against a page loaded in a hidden WebView, for feed
 * articles whose feedContentHtml is missing or too short to be the real
 * article body. WebView APIs are main-thread-only, so this hops to
 * Dispatchers.Main for the load+eval and suspends the calling coroutine
 * until the JS callback fires (or the timeout elapses).
 */
object ReaderExtractor {
    // Confirmed live: a heavy, ad/tracker-laden real page (several Ars Technica articles) can
    // legitimately take longer than 15s to finish loading everything before onPageFinished
    // fires, causing extraction to time out and silently fall back to no content -- not a
    // parsing failure, just not enough runway. 25s costs little (a spinner stays up slightly
    // longer on a truly dead page) against a real reduction in spurious extraction failures.
    private const val TIMEOUT_MS = 25_000L

    // onPageFinished fires while the page is still loading -- observed firing at
    // progress=25 on a perfectly ordinary article. Injecting the script then is a
    // race: sometimes the DOM is far enough along for Readability to work, and
    // sometimes evaluateJavascript returns a bare "null" because the document is
    // still being replaced underneath it. Measured at roughly 2 failures in 6 on
    // the same URL, which is exactly what a user sees as "extraction randomly
    // fails on some articles".
    //
    // So an unusable result is not final: retry a few times before giving up.
    // The outer TIMEOUT_MS remains the real backstop.
    private const val MAX_EVAL_ATTEMPTS = 6
    private const val EVAL_RETRY_DELAY_MS = 500L
    private val json = Json { ignoreUnknownKeys = true }

    // The vendored asset (~90KB) is identical every time regardless of which Context reads
    // it -- caching it once avoids a disk read + UTF-8 decode on the MAIN thread (this
    // whole class runs its WebView work on Dispatchers.Main) on every single extraction,
    // which adds up over a reading session that opens/re-extracts many articles. No
    // synchronization needed: extractOnMainThread only ever runs on Dispatchers.Main, so
    // this is never accessed from more than one thread.
    private var cachedReadabilityJs: String? = null

    private fun loadReadabilityJs(context: Context): String =
        cachedReadabilityJs ?: context.assets.open("Readability.js").bufferedReader().use { it.readText() }
            .also { cachedReadabilityJs = it }

    // KaTeX (~267KB vendored, see assets/katex.min.js), cached for the same reason and
    // under the same single-threaded guarantee as Readability.js above.
    private var cachedKatexJs: String? = null

    private fun loadKatexJs(context: Context): String =
        cachedKatexJs ?: context.assets.open("katex.min.js").bufferedReader().use { it.readText() }
            .also { cachedKatexJs = it }

    /**
     * Extracts article content, or explains why it could not.
     *
     * A null from the inner call means the timeout fired, which is a distinct
     * and common failure (a page whose load never completes) and deserves to
     * say so rather than look like every other failure.
     */
    suspend fun extractResult(context: Context, url: String): ExtractionResult {
        val outcome = withTimeoutOrNull(TIMEOUT_MS) {
            withContext(Dispatchers.Main) {
                extractOnMainThread(context, url)
            }
        } ?: return ExtractionResult.Failure(ExtractionFailure.TimedOut(TIMEOUT_MS))
        return outcome
    }

    /** Back-compatible shim: content when it worked, null when it did not. */
    suspend fun extract(context: Context, url: String): ExtractedArticle? =
        when (val r = extractResult(context, url)) {
            is ExtractionResult.Success -> r.article
            is ExtractionResult.Failure -> null
        }

    @SuppressLint("SetJavaScriptEnabled")
    private suspend fun extractOnMainThread(context: Context, url: String): ExtractionResult =
        suspendCancellableCoroutine { continuation ->
            val webView = WebView(context)
            webView.settings.javaScriptEnabled = true
            // This WebView runs full JS against whatever URL it's given -- every article
            // link, every feed URL, and (via ShareReceiverActivity) any URL another
            // installed app hands this one. It has no legitimate reason to ever load a
            // file:// URL, so don't leave that door open for a malicious page to redirect
            // through (WebView defaults allowFileAccess to true pre-API 30).
            webView.settings.allowFileAccess = false
            webView.settings.allowContentAccess = false
            webView.settings.allowFileAccessFromFileURLs = false
            webView.settings.allowUniversalAccessFromFileURLs = false

            // A slow-loading page's evaluateJavascript callback can arrive right as the
            // outer withTimeoutOrNull cancels this coroutine, racing finish() (JS-callback
            // path) against invokeOnCancellation (timeout path). Both used to call
            // webView.destroy() unconditionally -- a double destroy() crashes natively.
            // This flag makes teardown happen exactly once no matter which path wins.
            var finished = false

            fun finish(result: ExtractionResult) {
                if (finished) return
                finished = true
                if (continuation.isActive) continuation.resume(result)
                webView.stopLoading()
                webView.destroy()
            }

            fun fail(reason: ExtractionFailure) = finish(ExtractionResult.Failure(reason))

            webView.webViewClient = object : WebViewClient() {
                override fun onPageFinished(view: WebView, finishedUrl: String) {
                    val readabilityJs = loadReadabilityJs(context)
                    val katexJs = loadKatexJs(context)
                    val script = """
                        (function() {
                            try {
                                $readabilityJs
                                function plainLength(article) {
                                    return (article && article.textContent) ? article.textContent.trim().length : 0;
                                }

                                // A common lazy-load pattern pairs a placeholder <img> with a
                                // sibling <noscript> holding the REAL <img> tag, meant for
                                // browsers with JS off. But our WebView runs WITH JS enabled
                                // (Readability.js itself needs it) -- and per the HTML spec,
                                // when scripting IS enabled, a <noscript> element's contents are
                                // parsed as opaque text, not as DOM, so document.querySelectorAll
                                // never even sees that real <img>. Pull it out manually: parse
                                // each <noscript>'s raw text as markup and promote any <img>
                                // inside to a real, visible sibling (replacing the placeholder if
                                // one immediately precedes it).
                                var noscripts = document.querySelectorAll('noscript');
                                for (var i = 0; i < noscripts.length; i++) {
                                    var ns = noscripts[i];
                                    var raw = ns.textContent || '';
                                    if (raw.indexOf('<img') === -1) continue;
                                    var temp = document.createElement('div');
                                    temp.innerHTML = raw;
                                    var realImg = temp.querySelector('img');
                                    if (!realImg) continue;
                                    var prev = ns.previousElementSibling;
                                    if (prev && prev.tagName === 'IMG') {
                                        prev.setAttribute('src', realImg.getAttribute('src') || '');
                                        if (realImg.getAttribute('srcset')) prev.setAttribute('srcset', realImg.getAttribute('srcset'));
                                        prev.removeAttribute('data-src');
                                        prev.removeAttribute('data-original');
                                        prev.removeAttribute('data-lazy-src');
                                    } else {
                                        ns.parentNode.insertBefore(realImg, ns);
                                    }
                                }

                                // Many sites lazy-load media via JS: the real URL sits in a
                                // data-src/data-srcset attribute and only gets copied into
                                // src/srcset once the site's own script runs -- which never
                                // happens here (JS is off in the reader WebView by design, so
                                // articles can't run arbitrary third-party scripts). Left alone
                                // this shows either a tiny placeholder/category icon stretched to
                                // fill the article width (the site's width/height attributes,
                                // sized for the real photo, still apply to whatever's in src), or
                                // nothing but the img's alt text if src is empty. Resolve these
                                // before Readability runs so it captures real media, not the
                                // unresolved lazy attribute.
                                var lazyAttrs = ['data-src', 'data-original', 'data-lazy-src'];
                                var imgs = document.querySelectorAll('img');
                                for (var i = 0; i < imgs.length; i++) {
                                    var img = imgs[i];
                                    for (var j = 0; j < lazyAttrs.length; j++) {
                                        var lazySrc = img.getAttribute(lazyAttrs[j]);
                                        if (lazySrc) { img.setAttribute('src', lazySrc); break; }
                                    }
                                    var lazySrcset = img.getAttribute('data-srcset');
                                    if (lazySrcset) img.setAttribute('srcset', lazySrcset);
                                    if (!img.getAttribute('src') && img.getAttribute('srcset')) {
                                        var first = img.getAttribute('srcset').split(',')[0].trim().split(/\s+/)[0];
                                        if (first) img.setAttribute('src', first);
                                    }
                                    // Drop width/height attributes now that src may point at a
                                    // differently-sized image than whatever they were set for --
                                    // our own stylesheet's max-width:100%/height:auto sizes
                                    // correctly from the (now-resolved) image's real dimensions.
                                    img.removeAttribute('width');
                                    img.removeAttribute('height');
                                    // The browser's native loading="lazy" only fetches an image
                                    // once it's near the (real, on-screen) viewport -- meaningless
                                    // in this hidden, off-screen extraction WebView, where whether
                                    // that ever counts as "near enough" is inconsistent run to run.
                                    // Confirmed live: the same Ars Technica article's gallery photos
                                    // came through in one extraction pass and were silently missing
                                    // in another. Force every image to load immediately instead of
                                    // leaving it to viewport-intersection timing that doesn't apply.
                                    img.removeAttribute('loading');
                                }
                                var sources = document.querySelectorAll('source');
                                for (var i = 0; i < sources.length; i++) {
                                    var lazySrcset = sources[i].getAttribute('data-srcset') || sources[i].getAttribute('data-src');
                                    if (lazySrcset) sources[i].setAttribute('srcset', lazySrcset);
                                }

                                // A lightbox (PhotoSwipe, and others like it) commonly keeps its own
                                // copy of an image's caption text in a separate div meant to only
                                // ever be shown inside the lightbox overlay -- confirmed live on Ars
                                // Technica: a ".pswp-caption-content" div sits right next to a real,
                                // normally-visible <figcaption> with the exact same text, inside the
                                // same <figure>. Readability's hidden-element skip only catches
                                // display:none/visibility:hidden, not however that div is actually
                                // kept offscreen until the lightbox opens, so it was coming through
                                // as a genuine second copy. Only remove it when that real figcaption
                                // sibling actually exists -- some of Ars's own gallery images use
                                // this same div as their ONLY caption (no figcaption at all), and
                                // removing those unconditionally just deleted their caption outright
                                // (confirmed live: two gallery photos lost their captions entirely
                                // rather than losing a duplicate).
                                var pswpCaptions = document.querySelectorAll('.pswp-caption-content');
                                for (var i = 0; i < pswpCaptions.length; i++) {
                                    var pc = pswpCaptions[i];
                                    var figure = pc.closest('figure');
                                    if (figure && figure.querySelector('figcaption')) {
                                        pc.parentNode.removeChild(pc);
                                    }
                                }

                                // Some sites lazy-load the embed iframe itself, not just images --
                                // leaving `src` blank or a generic loader URL until their own script
                                // runs, which never happens here. Resolve iframe data-src first (same
                                // idea as the image lazy-load fix above) so the YouTube detection below
                                // has a real URL to work with instead of nothing.
                                var lazyFrames = document.querySelectorAll('iframe[data-src], iframe[data-lazy-src]');
                                for (var i = 0; i < lazyFrames.length; i++) {
                                    var lazyFrameSrc = lazyFrames[i].getAttribute('data-src') || lazyFrames[i].getAttribute('data-lazy-src');
                                    if (lazyFrameSrc) lazyFrames[i].setAttribute('src', lazyFrameSrc);
                                }

                                // A YouTube iframe embed is dead weight here -- JS is off in the WebView
                                // that actually displays this content, so the real embedded player could
                                // never initialize anyway (just a blank/broken box). Swap it for a
                                // tap-to-open thumbnail (a real <img>, which Readability treats like any
                                // other content image) linking to the video -- newReaderWebView sends a
                                // YouTube watch link straight to the YouTube app/browser instead of our
                                // own article-extraction pipeline, since there's no article to extract
                                // from a video page.
                                var iframes = document.querySelectorAll(
                                    'iframe[src*="youtube.com/embed/"], iframe[src*="youtube-nocookie.com/embed/"]'
                                );
                                for (var i = 0; i < iframes.length; i++) {
                                    var frame = iframes[i];
                                    var frameSrc = frame.getAttribute('src') || '';
                                    // A real video ID is always exactly 11 chars -- a playlist/live
                                    // embed's path is literally ".../embed/videoseries?list=...", which
                                    // (coincidentally exactly 11 characters itself) previously slipped
                                    // past a length-only check and built a thumbnail URL for a video
                                    // that doesn't exist (confirmed live: rendered as a broken-image
                                    // glyph, not a still frame -- exactly what was reported).
                                    var match = frameSrc.match(/embed\/([a-zA-Z0-9_-]{11})(?:[/?]|$)/);
                                    if (match && match[1] === 'videoseries') match = null;
                                    // Neither a class NOR an inline style survives here -- Readability
                                    // strips both as part of its own cleanup. A bare <a><img></a> with
                                    // no surrounding text didn't survive either, for a different reason:
                                    // Readability's link-density cleanup treats a node that's almost
                                    // entirely inside an <a> as a probable ad/share-button unit and
                                    // discards it (confirmed live: the iframe was there, the transform
                                    // ran, and the result still vanished from the final extracted
                                    // content). A <figure>/<figcaption> pair -- the exact shape a normal
                                    // captioned photo takes, which reliably survives elsewhere in this
                                    // pipeline -- with the caption text OUTSIDE the link keeps the
                                    // node's link-density low enough to read as real content.
                                    var figure = document.createElement('figure');
                                    var link = document.createElement('a');
                                    figure.appendChild(link);
                                    if (match) {
                                        var videoId = match[1];
                                        link.href = 'https://www.youtube.com/watch?v=' + videoId;
                                        var thumb = document.createElement('img');
                                        // hq720 (true 16:9 crop) beats hqdefault (a 480x360 4:3
                                        // crop that letterboxes any widescreen video with black
                                        // bars) and, unlike maxresdefault, is reliably present for
                                        // essentially every video -- no existence check needed.
                                        thumb.src = 'https://i.ytimg.com/vi/' + videoId + '/hq720.jpg';
                                        thumb.alt = 'YouTube video';
                                        link.appendChild(thumb);
                                        var caption = document.createElement('figcaption');
                                        caption.textContent = '▶ Watch on YouTube';
                                        figure.appendChild(caption);
                                    } else {
                                        // No confident single video ID (a playlist, a live stream, or
                                        // lazy-load resolution above still came up empty) -- a plain
                                        // link beats a broken thumbnail or a dead iframe. frameSrc is
                                        // attacker-controlled article content though (it only needed to
                                        // match the querySelector's substring src filter, not the strict
                                        // video-ID regex) -- only use it verbatim if it's actually an
                                        // http(s) URL, so a crafted file://, content://, or intent://
                                        // iframe src can't ride along into this href and downstream into
                                        // an ACTION_VIEW intent or a JS-enabled WebView.loadUrl.
                                        link.href = /^https?:\/\//i.test(frameSrc) ? frameSrc : 'https://www.youtube.com/';
                                        link.textContent = 'Watch on YouTube';
                                    }
                                    frame.parentNode.replaceChild(figure, frame);
                                }

                                // Ars Technica splits a longer article's body across MULTIPLE sibling
                                // ".post-content.post-content-double" divs -- each its own separate grid
                                // row, with a full mid-content ad rail sitting between them -- rather than
                                // keeping one continuous content container. Readability scores each as an
                                // independent candidate and keeps only the highest-scoring one, silently
                                // dropping the rest (confirmed live: "Reaction wheel failures leave Swift
                                // rescue mission spinning in orbit" lost its first .post-content entirely --
                                // exactly its lede through the reaction-wheel explainer, 7 paragraphs --
                                // while the second block, continuing the story, came through fine). Merge
                                // every such div's children into the first one before Readability ever sees
                                // the page, so it scores one complete candidate instead of splitting its
                                // vote across several partial ones.
                                var splitBlocks = document.querySelectorAll('.post-content.post-content-double');
                                if (splitBlocks.length > 1) {
                                    var mergeTarget = splitBlocks[0];
                                    for (var i = 1; i < splitBlocks.length; i++) {
                                        while (splitBlocks[i].firstChild) {
                                            mergeTarget.appendChild(splitBlocks[i].firstChild);
                                        }
                                        splitBlocks[i].parentNode.removeChild(splitBlocks[i]);
                                    }
                                }

                                // A news/blog article page's <header> conventionally holds everything
                                // BEFORE the real body: a section/category kicker, the dek/subhead, the
                                // byline, and the hero image + its caption. <footer> (site nav/link
                                // menus) is the same story at the other end. Readability doesn't exclude
                                // either by tag outright (only deprioritizes them in scoring), so on
                                // some articles enough of one scores highly enough to get pulled in
                                // alongside the real body -- confirmed live: a kicker icon, the dek
                                // paragraph, AND an orphaned duplicate hero-image caption (a mobile-only
                                // copy with no adjacent <img> at all) all leaking in above the real lede
                                // on an Ars Technica article. This is a pre-processing decision on our
                                // side, not a change to Readability's own algorithm: treat <header>/
                                // <footer> as page chrome, never article content, and remove them before
                                // Readability ever sees the page. (The article's actual title is shown
                                // separately -- see ReaderWebView.wrapReaderHtml -- so losing the in-body
                                // <h1> here is fine, it was never meant to double as the body text.)
                                var chrome = document.querySelectorAll('header, footer');
                                for (var i = 0; i < chrome.length; i++) {
                                    chrome[i].parentNode.removeChild(chrome[i]);
                                }

                                // Two more sibling widgets confirmed live, same "not article content"
                                // story: an author bio card (photo + name + one-paragraph bio, after the
                                // real body -- its photo has the same Tailwind-dependent absolute-fill
                                // styling as the section-badge icon and hero image, so it'd render
                                // oversized for the same underlying reason even if kept) and a "Most
                                // Read"/trending-stories widget (a list of OTHER articles' thumbnails --
                                // confirmed live: one showed up as a giant unrelated photo followed by an
                                // empty numbered list, since the widget's later items only populate their
                                // thumbnail on scroll-into-view, which never happens in this hidden,
                                // off-screen extraction WebView).
                                var sequels = document.querySelectorAll(
                                    '[class*="author-mini-bio" i], [class*="author-bio" i], [class*="most-read" i]'
                                );
                                for (var i = 0; i < sequels.length; i++) {
                                    if (sequels[i].parentNode) sequels[i].parentNode.removeChild(sequels[i]);
                                }

                                // A page that uses TeX ships the SOURCE of its maths and
                                // renders it in the browser with its own script -- which
                                // never runs here, so the source is all there is. Confirmed
                                // live on lcamtuf's Substack (blog.coredump.cx/p/yours-truly):
                                // all four of that article's equations arrive as literal
                                // "\(p = \text{...}\)" text in a span inside a
                                // "latex-rendered" div, and Readability keeps that text
                                // verbatim (it strips the class, not the text) -- which is
                                // exactly what the reader was showing. So render it HERE,
                                // where JS is already running for Readability's sake, and
                                // hand the reader finished markup.
                                //
                                // Output is MathML, not KaTeX's usual HTML: the WebView
                                // renders MathML natively, so nothing downstream needs
                                // KaTeX's stylesheet, its webfonts, or any script -- the
                                // reader WebView keeps JavaScript OFF, as it must for
                                // untrusted article content. KaTeX's HTML output would not
                                // have survived anyway: it is entirely class-driven, and
                                // Readability strips class attributes.
                                var katex = (function() {
                                    // Hand KaTeX's UMD wrapper a CommonJS target of our own.
                                    // Left to itself it would take the AMD branch on any page
                                    // that happens to ship an AMD loader (leaving `katex`
                                    // undefined), or publish itself on the page's window --
                                    // and this is a tool we are using on the document, not
                                    // something the document should see or be able to redefine.
                                    var module = { exports: {} };
                                    var exports = module.exports;
                                    var define = undefined;
                                    $katexJs
                                    return module.exports;
                                })();

                                (function renderTexSource() {
                                    if (!katex || typeof katex.renderToString !== 'function') return;
                                    // Delimiter pairs, scanned left to right. Single-$ inline
                                    // maths is deliberately NOT supported: prices and currency
                                    // in ordinary prose would be mangled far more often than
                                    // real maths would be found.
                                    var DELIMITERS = [
                                        { left: '$$', right: '$$', display: true },
                                        { left: '\\[', right: '\\]', display: true },
                                        { left: '\\(', right: '\\)', display: false }
                                    ];
                                    // Where TeX-looking text is quoted on purpose rather than
                                    // meant as maths -- an article ABOUT TeX shows its source.
                                    var SKIP_TAGS = {
                                        SCRIPT: 1, STYLE: 1, PRE: 1, CODE: 1, KBD: 1,
                                        SAMP: 1, TEXTAREA: 1, NOSCRIPT: 1, MATH: 1
                                    };
                                    // A page is untrusted content: cap the work it can ask for.
                                    // Far above any real article (the reported one has four).
                                    var MAX_EXPRESSIONS = 300;
                                    var rendered = 0;

                                    function renderOnce(tex, display) {
                                        var holder = document.createElement('div');
                                        try {
                                            holder.innerHTML = katex.renderToString(tex, {
                                                output: 'mathml',
                                                displayMode: display,
                                                throwOnError: false,
                                                strict: false
                                            });
                                        } catch (e) {
                                            return null;
                                        }
                                        var math = holder.querySelector('math');
                                        if (!math) return null;
                                        // KaTeX attaches an <annotation> holding the TeX source
                                        // again, for copy/paste tools. Nothing renders it, but it
                                        // does count toward Readability's text-length scoring --
                                        // and it is the source we just replaced, so drop it.
                                        var annotations = math.querySelectorAll('annotation');
                                        for (var a = 0; a < annotations.length; a++) {
                                            annotations[a].parentNode.removeChild(annotations[a]);
                                        }
                                        return math;
                                    }

                                    function renderTex(tex, display) {
                                        // KaTeX refuses {align} and friends outside display mode
                                        // (with throwOnError off it returns an error span and no
                                        // <math> at all) -- and Substack wraps DISPLAY maths in
                                        // INLINE delimiters, leaving the centring to a
                                        // block-styled span. Confirmed live: both of the reported
                                        // article's multi-line align blocks are \( ... \). So a
                                        // mode that produces nothing is retried in the other one,
                                        // and only if both come up empty is the source left
                                        // exactly as it was -- no worse than before.
                                        return renderOnce(tex, display) || renderOnce(tex, !display);
                                    }

                                    function nextDelimiter(text, from) {
                                        var best = null;
                                        for (var d = 0; d < DELIMITERS.length; d++) {
                                            var at = text.indexOf(DELIMITERS[d].left, from);
                                            if (at === -1) continue;
                                            if (!best || at < best.at) best = { at: at, delimiter: DELIMITERS[d] };
                                        }
                                        return best;
                                    }

                                    function isSkipped(node) {
                                        for (var p = node.parentNode; p && p.nodeType === 1; p = p.parentNode) {
                                            if (SKIP_TAGS[p.tagName]) return true;
                                        }
                                        return false;
                                    }

                                    // Collect first, rewrite after: replacing a text node while
                                    // the walker is still pointing into it invalidates the walk.
                                    var walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, null, false);
                                    var candidates = [];
                                    var node;
                                    while ((node = walker.nextNode())) {
                                        var data = node.nodeValue || '';
                                        if (data.indexOf('\\(') === -1 && data.indexOf('\\[') === -1 &&
                                            data.indexOf('$$') === -1) continue;
                                        if (isSkipped(node)) continue;
                                        candidates.push(node);
                                    }

                                    for (var c = 0; c < candidates.length; c++) {
                                        var target = candidates[c];
                                        var text = target.nodeValue || '';
                                        var fragment = document.createDocumentFragment();
                                        var cursor = 0;
                                        var replacedAny = false;
                                        while (cursor < text.length && rendered < MAX_EXPRESSIONS) {
                                            var found = nextDelimiter(text, cursor);
                                            if (!found) break;
                                            var innerStart = found.at + found.delimiter.left.length;
                                            var closeAt = text.indexOf(found.delimiter.right, innerStart);
                                            // An unpaired delimiter is just prose that happens to
                                            // contain one. Leave the rest of the node alone.
                                            if (closeAt === -1) break;
                                            var tex = text.slice(innerStart, closeAt);
                                            var math = tex.trim() ? renderTex(tex, found.delimiter.display) : null;
                                            if (!math) {
                                                // Not renderable: step past this opening delimiter
                                                // and keep looking, rather than dropping the text.
                                                cursor = innerStart;
                                                continue;
                                            }
                                            if (found.at > cursor) {
                                                fragment.appendChild(document.createTextNode(text.slice(cursor, found.at)));
                                            }
                                            fragment.appendChild(math);
                                            cursor = closeAt + found.delimiter.right.length;
                                            replacedAny = true;
                                            rendered++;
                                        }
                                        if (!replacedAny) continue;
                                        if (cursor < text.length) {
                                            fragment.appendChild(document.createTextNode(text.slice(cursor)));
                                        }
                                        target.parentNode.replaceChild(fragment, target);
                                    }
                                })();

                                // Readability's default hidden-element skip (treating display:none /
                                // visibility:hidden nodes as boilerplate) is correct for the vast
                                // majority of sites -- it's what keeps collapsed nav/footer/sidebar
                                // chrome out of the result. Try it first, unmodified.
                                var normalArticle = new Readability(document.cloneNode(true)).parse();

                                // "Thin normal-pass result" is NOT a reliable paywall signal on its
                                // own -- plenty of ordinary, non-paywalled posts (kottke's linkblog
                                // style, for one) are also genuinely just short, and un-hiding
                                // EVERYTHING for those pulled in an unrelated hidden footer/social-
                                // links block instead of any real article content (reproduced live:
                                // the same kottke article kept coming back wrapped in site-footer
                                // boilerplate even with reader mode explicitly enabled for the feed).
                                // Target the actual, specific signal instead: elements whose class/id
                                // names the page itself labels as paywall/premium/subscriber content.
                                // West Hawaii Today's hidden div is literally class="paywall
                                // paywall_<id>" -- this only ever fires for genuine paywall markup,
                                // never for a plain short post.
                                var paywallEls = document.querySelectorAll(
                                    '[class*="paywall" i], [id*="paywall" i], [class*="premium-content" i], ' +
                                    '[class*="subscriber-content" i], [class*="metered-content" i]'
                                );
                                var unhidAny = false;
                                for (var i = 0; i < paywallEls.length; i++) {
                                    var el = paywallEls[i];
                                    var cs = window.getComputedStyle(el);
                                    if (cs.display === 'none' || cs.visibility === 'hidden') {
                                        el.style.setProperty('display', 'block', 'important');
                                        el.style.setProperty('visibility', 'visible', 'important');
                                        unhidAny = true;
                                    }
                                }
                                if (unhidAny) {
                                    var unhiddenArticle = new Readability(document.cloneNode(true)).parse();
                                    if (plainLength(unhiddenArticle) > plainLength(normalArticle)) {
                                        return JSON.stringify(unhiddenArticle);
                                    }
                                }
                                return JSON.stringify(normalArticle);
                            } catch (e) {
                                return JSON.stringify({error: e.toString()});
                            }
                        })();
                    """.trimIndent()

                    evaluate(view, script, attempt = 1)
                }

                /**
                 * Runs the extraction script, retrying while the result looks like
                 * the page simply was not ready yet.
                 *
                 * The accept/retry/give-up decision itself lives in
                 * [decideEvalOutcome], which is testable without a WebView; this
                 * only decodes the payload and carries the decision out.
                 */
                private fun evaluate(view: WebView, script: String, attempt: Int) {
                    if (finished) return
                    view.evaluateJavascript(script) { rawResult ->
                        // evaluateJavascript's callback value is itself JSON-encoded (our
                        // JSON.stringify(...) string comes back wrapped in an extra layer
                        // of JSON string-quoting), so this needs decoding twice.
                        val unwrapped = runCatching { json.decodeFromString<String>(rawResult) }.getOrNull()
                        val article = unwrapped?.let {
                            runCatching { json.decodeFromString<ExtractedArticle>(it) }.getOrNull()
                        }

                        when (
                            val decision = decideEvalOutcome(
                                article = article,
                                rawResult = rawResult ?: "null",
                                attempt = attempt,
                                maxAttempts = MAX_EVAL_ATTEMPTS
                            )
                        ) {
                            is EvalDecision.Done -> finish(decision.result)
                            EvalDecision.Retry ->
                                view.postDelayed({ evaluate(view, script, attempt + 1) }, EVAL_RETRY_DELAY_MS)
                        }
                    }
                }

                // The four-argument onReceivedError this used to override is only
                // called below API 23. minSdk is 26, so it never fired on any
                // supported device: a failed load invoked nothing, hung for the
                // full TIMEOUT_MS, and then surfaced as an indistinguishable
                // null. These are the callbacks Android actually uses.
                //
                // isForMainFrame matters: a page routinely fails to load some
                // tracker or font, and aborting extraction because an analytics
                // script 404'd would break articles that render perfectly well.
                override fun onReceivedError(
                    view: WebView,
                    request: WebResourceRequest,
                    error: WebResourceError
                ) {
                    if (!request.isForMainFrame) return
                    fail(ExtractionFailure.LoadError(error.errorCode, error.description?.toString()))
                }

                override fun onReceivedHttpError(
                    view: WebView,
                    request: WebResourceRequest,
                    errorResponse: WebResourceResponse
                ) {
                    if (!request.isForMainFrame) return
                    fail(
                        ExtractionFailure.HttpError(
                            errorResponse.statusCode,
                            errorResponse.reasonPhrase
                        )
                    )
                }
            }

            // withTimeoutOrNull's cancellation is driven by a delayed task on
            // kotlinx.coroutines.DefaultExecutor, NOT the calling coroutine's dispatcher --
            // so this handler can run on a background thread despite the whole extraction
            // being wrapped in withContext(Dispatchers.Main). Touching the WebView there
            // crashes ("A WebView method was called on thread ..."), which is exactly what
            // a slow/never-finishing article load (timing out after TIMEOUT_MS) hit every
            // time -- reproduced here as the real "crash loop" root cause. Hop back to the
            // main looper before tearing down.
            continuation.invokeOnCancellation {
                Handler(Looper.getMainLooper()).post {
                    if (!finished) {
                        finished = true
                        webView.stopLoading()
                        webView.destroy()
                    }
                }
            }
            webView.loadUrl(url)
        }
}