Render an article's maths instead of showing its TeX source

Reported against lcamtuf's Substack ("we should render tex",
blog.coredump.cx/p/yours-truly): reader mode showed raw
"\(p = \text{...}\)" where equations should be.

What the page actually ships is the TeX SOURCE, as text, inside a
"latex-rendered" div -- Substack renders it in the browser with its own
script, which never runs here. Readability keeps that text verbatim (it
strips the class, not the text), so nothing was being discarded during
extraction and no amount of extraction-side repair would have helped:
there was simply never any rendered markup to keep.

So render it during extraction, where JavaScript is already running for
Readability's sake. KaTeX (vendored in assets, never fetched -- this
runs against untrusted third-party pages) converts each expression to
MathML before Readability sees the document, and the reader WebView
renders that natively. The reader keeps JavaScript OFF, as it must, and
needs no stylesheet, webfont or script of KaTeX's; the only reader-side
change is containing a wide equation so it scrolls within its own box
instead of dragging the whole article sideways.

Two details worth knowing:

- KaTeX refuses {align} outside display mode, and Substack wraps display
  maths in INLINE delimiters (a block-styled span does the centring), so
  a mode that renders nothing is retried in the other one. An expression
  neither mode can render is left as source -- no worse than before.
- The reported article came with "extractedContent cached: true", so the
  raw source was already on disk and every open read it back. Recognised
  and dropped on read (same mechanism as the cached consent notice),
  which re-extracts the article the next time it is opened.

APK grows by ~90KB: the vendored katex.min.js (267KB) compresses to 91KB
in the APK. No new permissions, no schema change, no new JS-enabled
surface.

Verified on the emulator against the live article: stale raw-TeX cache
discarded, article re-extracted, all four equations rendered, and a wide
align block scrolls inside itself.

5 files changed, 295 insertions, 9 deletions

diff --git a/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt b/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt
index 313d467..3a791cc 100644
--- a/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt
+++ b/app/src/main/java/org/terst/feedreader/data/CachedExtraction.kt
@@ -1,6 +1,7 @@
 package org.terst.feedreader.data
 
 import org.terst.feedreader.readability.ConsentNotice
+import org.terst.feedreader.readability.UnrenderedTex
 
 /** What a cached `articles.extractedContent` value turned out to be worth. */
 data class CachedExtraction(
@@ -11,12 +12,24 @@ data class CachedExtraction(
      * show and should extract again -- the cache is the reason the article looked
      * broken, so dropping it silently would just leave a blank reader.
      */
-    val discardedConsentNotice: Boolean
-)
+    val discardedConsentNotice: Boolean,
+    /**
+     * True when a cached extraction holding raw TeX source was thrown away -- it was
+     * extracted before extraction knew how to render maths (see [UnrenderedTex]).
+     */
+    val discardedUnrenderedTex: Boolean = false
+) {
+    /**
+     * True when something was thrown away, whatever it was: there is nothing to show and
+     * the caller has to extract again.
+     */
+    val discarded: Boolean get() = discardedConsentNotice || discardedUnrenderedTex
+}
 
 /**
  * Reads a cached extraction, dropping it if it is the site's cookie/consent notice
- * rather than the article.
+ * rather than the article, or if it is an older extraction whose maths never got
+ * rendered.
  *
  * Needed because the extractor's own guard can only keep a bad extraction from
  * being written in future. The reported bug ("reader mode just extracts cookie
@@ -33,9 +46,20 @@ suspend fun loadCachedExtraction(
     cached: String?,
     clearCache: suspend () -> Unit
 ): CachedExtraction {
-    if (!ConsentNotice.looksLikeConsentNotice(cached)) {
-        return CachedExtraction(content = cached, discardedConsentNotice = false)
-    }
+    val discard = when {
+        ConsentNotice.looksLikeConsentNotice(cached) ->
+            CachedExtraction(content = null, discardedConsentNotice = true)
+        // Same shape of problem, different cause: this one was extracted before extraction
+        // rendered maths, so it holds the TeX source the page shipped.
+        UnrenderedTex.looksLikeUnrenderedTex(cached) ->
+            CachedExtraction(
+                content = null,
+                discardedConsentNotice = false,
+                discardedUnrenderedTex = true
+            )
+        else -> null
+    } ?: return CachedExtraction(content = cached, discardedConsentNotice = false)
+
     clearCache()
-    return CachedExtraction(content = null, discardedConsentNotice = true)
+    return discard
 }
diff --git a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt
index 21633b4..4858007 100644
--- a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt
+++ b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt
@@ -80,6 +80,14 @@ object ReaderExtractor {
         cachedReadabilityJs ?: context.assets.open("Readability.js").bufferedReader().use { it.readText() }
             .also { cachedReadabilityJs = it }
 
+    // KaTeX (~267KB vendored, see assets/katex.min.js), cached for the same reason and
+    // under the same single-threaded guarantee as Readability.js above.
+    private var cachedKatexJs: String? = null
+
+    private fun loadKatexJs(context: Context): String =
+        cachedKatexJs ?: context.assets.open("katex.min.js").bufferedReader().use { it.readText() }
+            .also { cachedKatexJs = it }
+
     /**
      * Extracts article content, or explains why it could not.
      *
@@ -138,6 +146,7 @@ object ReaderExtractor {
             webView.webViewClient = object : WebViewClient() {
                 override fun onPageFinished(view: WebView, finishedUrl: String) {
                     val readabilityJs = loadReadabilityJs(context)
+                    val katexJs = loadKatexJs(context)
                     val script = """
                         (function() {
                             try {
@@ -384,6 +393,167 @@ object ReaderExtractor {
                                     if (sequels[i].parentNode) sequels[i].parentNode.removeChild(sequels[i]);
                                 }
 
+                                // A page that uses TeX ships the SOURCE of its maths and
+                                // renders it in the browser with its own script -- which
+                                // never runs here, so the source is all there is. Confirmed
+                                // live on lcamtuf's Substack (blog.coredump.cx/p/yours-truly):
+                                // all four of that article's equations arrive as literal
+                                // "\(p = \text{...}\)" text in a span inside a
+                                // "latex-rendered" div, and Readability keeps that text
+                                // verbatim (it strips the class, not the text) -- which is
+                                // exactly what the reader was showing. So render it HERE,
+                                // where JS is already running for Readability's sake, and
+                                // hand the reader finished markup.
+                                //
+                                // Output is MathML, not KaTeX's usual HTML: the WebView
+                                // renders MathML natively, so nothing downstream needs
+                                // KaTeX's stylesheet, its webfonts, or any script -- the
+                                // reader WebView keeps JavaScript OFF, as it must for
+                                // untrusted article content. KaTeX's HTML output would not
+                                // have survived anyway: it is entirely class-driven, and
+                                // Readability strips class attributes.
+                                var katex = (function() {
+                                    // Hand KaTeX's UMD wrapper a CommonJS target of our own.
+                                    // Left to itself it would take the AMD branch on any page
+                                    // that happens to ship an AMD loader (leaving `katex`
+                                    // undefined), or publish itself on the page's window --
+                                    // and this is a tool we are using on the document, not
+                                    // something the document should see or be able to redefine.
+                                    var module = { exports: {} };
+                                    var exports = module.exports;
+                                    var define = undefined;
+                                    $katexJs
+                                    return module.exports;
+                                })();
+
+                                (function renderTexSource() {
+                                    if (!katex || typeof katex.renderToString !== 'function') return;
+                                    // Delimiter pairs, scanned left to right. Single-$ inline
+                                    // maths is deliberately NOT supported: prices and currency
+                                    // in ordinary prose would be mangled far more often than
+                                    // real maths would be found.
+                                    var DELIMITERS = [
+                                        { left: '$$', right: '$$', display: true },
+                                        { left: '\\[', right: '\\]', display: true },
+                                        { left: '\\(', right: '\\)', display: false }
+                                    ];
+                                    // Where TeX-looking text is quoted on purpose rather than
+                                    // meant as maths -- an article ABOUT TeX shows its source.
+                                    var SKIP_TAGS = {
+                                        SCRIPT: 1, STYLE: 1, PRE: 1, CODE: 1, KBD: 1,
+                                        SAMP: 1, TEXTAREA: 1, NOSCRIPT: 1, MATH: 1
+                                    };
+                                    // A page is untrusted content: cap the work it can ask for.
+                                    // Far above any real article (the reported one has four).
+                                    var MAX_EXPRESSIONS = 300;
+                                    var rendered = 0;
+
+                                    function renderOnce(tex, display) {
+                                        var holder = document.createElement('div');
+                                        try {
+                                            holder.innerHTML = katex.renderToString(tex, {
+                                                output: 'mathml',
+                                                displayMode: display,
+                                                throwOnError: false,
+                                                strict: false
+                                            });
+                                        } catch (e) {
+                                            return null;
+                                        }
+                                        var math = holder.querySelector('math');
+                                        if (!math) return null;
+                                        // KaTeX attaches an <annotation> holding the TeX source
+                                        // again, for copy/paste tools. Nothing renders it, but it
+                                        // does count toward Readability's text-length scoring --
+                                        // and it is the source we just replaced, so drop it.
+                                        var annotations = math.querySelectorAll('annotation');
+                                        for (var a = 0; a < annotations.length; a++) {
+                                            annotations[a].parentNode.removeChild(annotations[a]);
+                                        }
+                                        return math;
+                                    }
+
+                                    function renderTex(tex, display) {
+                                        // KaTeX refuses {align} and friends outside display mode
+                                        // (with throwOnError off it returns an error span and no
+                                        // <math> at all) -- and Substack wraps DISPLAY maths in
+                                        // INLINE delimiters, leaving the centring to a
+                                        // block-styled span. Confirmed live: both of the reported
+                                        // article's multi-line align blocks are \( ... \). So a
+                                        // mode that produces nothing is retried in the other one,
+                                        // and only if both come up empty is the source left
+                                        // exactly as it was -- no worse than before.
+                                        return renderOnce(tex, display) || renderOnce(tex, !display);
+                                    }
+
+                                    function nextDelimiter(text, from) {
+                                        var best = null;
+                                        for (var d = 0; d < DELIMITERS.length; d++) {
+                                            var at = text.indexOf(DELIMITERS[d].left, from);
+                                            if (at === -1) continue;
+                                            if (!best || at < best.at) best = { at: at, delimiter: DELIMITERS[d] };
+                                        }
+                                        return best;
+                                    }
+
+                                    function isSkipped(node) {
+                                        for (var p = node.parentNode; p && p.nodeType === 1; p = p.parentNode) {
+                                            if (SKIP_TAGS[p.tagName]) return true;
+                                        }
+                                        return false;
+                                    }
+
+                                    // Collect first, rewrite after: replacing a text node while
+                                    // the walker is still pointing into it invalidates the walk.
+                                    var walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, null, false);
+                                    var candidates = [];
+                                    var node;
+                                    while ((node = walker.nextNode())) {
+                                        var data = node.nodeValue || '';
+                                        if (data.indexOf('\\(') === -1 && data.indexOf('\\[') === -1 &&
+                                            data.indexOf('$$') === -1) continue;
+                                        if (isSkipped(node)) continue;
+                                        candidates.push(node);
+                                    }
+
+                                    for (var c = 0; c < candidates.length; c++) {
+                                        var target = candidates[c];
+                                        var text = target.nodeValue || '';
+                                        var fragment = document.createDocumentFragment();
+                                        var cursor = 0;
+                                        var replacedAny = false;
+                                        while (cursor < text.length && rendered < MAX_EXPRESSIONS) {
+                                            var found = nextDelimiter(text, cursor);
+                                            if (!found) break;
+                                            var innerStart = found.at + found.delimiter.left.length;
+                                            var closeAt = text.indexOf(found.delimiter.right, innerStart);
+                                            // An unpaired delimiter is just prose that happens to
+                                            // contain one. Leave the rest of the node alone.
+                                            if (closeAt === -1) break;
+                                            var tex = text.slice(innerStart, closeAt);
+                                            var math = tex.trim() ? renderTex(tex, found.delimiter.display) : null;
+                                            if (!math) {
+                                                // Not renderable: step past this opening delimiter
+                                                // and keep looking, rather than dropping the text.
+                                                cursor = innerStart;
+                                                continue;
+                                            }
+                                            if (found.at > cursor) {
+                                                fragment.appendChild(document.createTextNode(text.slice(cursor, found.at)));
+                                            }
+                                            fragment.appendChild(math);
+                                            cursor = closeAt + found.delimiter.right.length;
+                                            replacedAny = true;
+                                            rendered++;
+                                        }
+                                        if (!replacedAny) continue;
+                                        if (cursor < text.length) {
+                                            fragment.appendChild(document.createTextNode(text.slice(cursor)));
+                                        }
+                                        target.parentNode.replaceChild(fragment, target);
+                                    }
+                                })();
+
                                 // Readability's default hidden-element skip (treating display:none /
                                 // visibility:hidden nodes as boilerplate) is correct for the vast
                                 // majority of sites -- it's what keeps collapsed nav/footer/sidebar
diff --git a/app/src/main/java/org/terst/feedreader/readability/UnrenderedTex.kt b/app/src/main/java/org/terst/feedreader/readability/UnrenderedTex.kt
new file mode 100644
index 0000000..aa46f7e
--- /dev/null
+++ b/app/src/main/java/org/terst/feedreader/readability/UnrenderedTex.kt
@@ -0,0 +1,84 @@
+package org.terst.feedreader.readability
+
+/**
+ * Recognises a cached extraction that still holds raw TeX source instead of rendered maths.
+ *
+ * Needed for the same reason [ConsentNotice] is: fixing the extractor only changes what
+ * FUTURE extractions produce. The reported article ("we should render tex", lcamtuf's
+ * Substack) came with `extractedContent cached: true` -- its raw `\(p = \text{...}\)`
+ * source was already on disk, and every open read it straight back out, so the reader
+ * would have gone on showing TeX source no matter what the extractor learned to do.
+ * Recognising it on read re-extracts the article the next time it is opened, including
+ * everything extracted before maths rendering existed.
+ *
+ * Over-detecting is the real risk here, and it is a specific one: a false positive makes
+ * an article re-extract on EVERY open, for ever, because re-extracting cannot change a
+ * verdict the content itself causes. So the bar is a complete delimiter pair around
+ * something that actually looks like an expression, and never anything that already
+ * carries rendered MathML.
+ */
+object UnrenderedTex {
+
+    /**
+     * Length past which a delimiter pair is read as two unrelated delimiters rather than
+     * one expression. Generous: the reported article's longest block (a ten-line `align`
+     * environment) is well under half this.
+     */
+    private const val MAX_EXPRESSION_LENGTH = 2_000
+
+    /** Delimiter pairs, matching the ones extraction renders (see ReaderExtractor). */
+    private val DELIMITERS = listOf("\\(" to "\\)", "\\[" to "\\]", "$$" to "$$")
+
+    /**
+     * Characters no plain sentence between two stray delimiters is likely to contain but
+     * essentially every real expression does -- a command, a group, a sub- or superscript.
+     */
+    private const val TEX_MARKERS = "\\{_^"
+
+    /**
+     * Quoted source, not maths: extraction deliberately leaves `<pre>`/`<code>` alone, so
+     * an article demonstrating TeX would otherwise be convicted on every open and still,
+     * correctly, come back unchanged.
+     */
+    private val CODE_BLOCK_REGEX = Regex(
+        "<(pre|code)\\b[^>]*>.*?</\\1\\s*>",
+        setOf(RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL)
+    )
+
+    /**
+     * True when [html] is an extraction whose maths never got rendered.
+     *
+     * Rendered maths anywhere in it means extraction already had its go at this page, and
+     * nothing is to be gained by sending it round again: an expression KaTeX could not
+     * render is still in there as source and always will be.
+     */
+    fun looksLikeUnrenderedTex(html: String?): Boolean {
+        if (html.isNullOrBlank() || html.contains("<math", ignoreCase = true)) return false
+
+        val prose = html.replace(CODE_BLOCK_REGEX, " ")
+        return DELIMITERS.any { (open, close) -> containsExpression(prose, open, close) }
+    }
+
+    private fun containsExpression(text: String, open: String, close: String): Boolean {
+        var opensAt = text.indexOf(open)
+        while (opensAt >= 0) {
+            val innerStart = opensAt + open.length
+            val closesAt = text.indexOf(close, innerStart)
+            // No closing delimiter left in the content: whatever follows is prose that
+            // happens to contain an opening one.
+            if (closesAt < 0) break
+            if (isExpression(text.substring(innerStart, closesAt))) return true
+            opensAt = text.indexOf(open, innerStart)
+        }
+        return false
+    }
+
+    private fun isExpression(inner: String): Boolean =
+        inner.isNotBlank() &&
+            inner.length <= MAX_EXPRESSION_LENGTH &&
+            // The span between two stray delimiters is markup, and markup brings its own
+            // braces and underscores along (ids, filenames, inline styles). A tag anywhere
+            // inside is on its own enough to say the two delimiters are unrelated.
+            !inner.contains('<') &&
+            inner.any { it in TEX_MARKERS }
+}
diff --git a/app/src/main/java/org/terst/feedreader/ui/ReaderScreen.kt b/app/src/main/java/org/terst/feedreader/ui/ReaderScreen.kt
index f08088b..9952dc5 100644
--- a/app/src/main/java/org/terst/feedreader/ui/ReaderScreen.kt
+++ b/app/src/main/java/org/terst/feedreader/ui/ReaderScreen.kt
@@ -221,7 +221,8 @@ private fun ReaderContent(
 
         feedContent = loaded.feedContentHtml
         // A previously-cached extraction can be the site's cookie-consent notice rather
-        // than the article (reported live on a New Republic feature). That is dropped
+        // than the article (reported live on a New Republic feature), or an older
+        // extraction holding raw TeX source where its maths should be. Either is dropped
         // from the cache on read, which leaves nothing to show -- so re-extract, or the
         // repair would just turn a wrong article into a blank one.
         val cachedExtraction = loadCachedExtraction(loaded.extractedContent) {
@@ -239,7 +240,7 @@ private fun ReaderContent(
         if (loadedFeed?.readerModeEnabled == true) {
             isReaderMode = true
         }
-        if (loadedFeed?.readerModeEnabled == true || cachedExtraction.discardedConsentNotice) {
+        if (loadedFeed?.readerModeEnabled == true || cachedExtraction.discarded) {
             if (readerContent == null) {
                 isExtracting = true
                 val result = ReaderExtractor.extractResult(context, loaded.link)
diff --git a/app/src/main/java/org/terst/feedreader/ui/ReaderWebView.kt b/app/src/main/java/org/terst/feedreader/ui/ReaderWebView.kt
index 6d96e79..f1400d3 100644
--- a/app/src/main/java/org/terst/feedreader/ui/ReaderWebView.kt
+++ b/app/src/main/java/org/terst/feedreader/ui/ReaderWebView.kt
@@ -514,6 +514,13 @@ fun wrapReaderHtml(
        explicit width/height was deliberately sized by the page itself, so leave
        those alone -- just cap the ones that would otherwise size themselves. */
     svg:not([width]):not([height]) { max-width: 32px; max-height: 32px; }
+    /* Maths arrives as MathML, which the WebView renders itself -- no script, no webfont,
+       no stylesheet of KaTeX's (see ReaderExtractor). It only needs containing: an aligned
+       multi-line derivation is routinely wider than a phone screen, and <math> obeys
+       neither the img nor the svg clamp above, so without this one wide equation drags the
+       whole article into horizontal scrolling. */
+    math { max-width: 100%; }
+    math[display="block"] { display: block; overflow-x: auto; overflow-y: hidden; margin: 1em 0; }
     </style></head>
     <body>${if (!title.isNullOrBlank()) "<h1 class=\"article-title\">${escapeHtml(title)}</h1>" else ""}${
         if (publishedAt != null) "<p class=\"article-date\">${escapeHtml(absoluteDateText(publishedAt))}</p>" else ""