diff options
Diffstat (limited to 'app/src/main/java/org/terst/feedreader/readability')
| -rw-r--r-- | app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt | 170 | ||||
| -rw-r--r-- | app/src/main/java/org/terst/feedreader/readability/UnrenderedTex.kt | 84 |
2 files changed, 254 insertions, 0 deletions
diff --git a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt index 21633b4..4858007 100644 --- a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt +++ b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt @@ -80,6 +80,14 @@ object ReaderExtractor { cachedReadabilityJs ?: context.assets.open("Readability.js").bufferedReader().use { it.readText() } .also { cachedReadabilityJs = it } + // KaTeX (~267KB vendored, see assets/katex.min.js), cached for the same reason and + // under the same single-threaded guarantee as Readability.js above. + private var cachedKatexJs: String? = null + + private fun loadKatexJs(context: Context): String = + cachedKatexJs ?: context.assets.open("katex.min.js").bufferedReader().use { it.readText() } + .also { cachedKatexJs = it } + /** * Extracts article content, or explains why it could not. * @@ -138,6 +146,7 @@ object ReaderExtractor { webView.webViewClient = object : WebViewClient() { override fun onPageFinished(view: WebView, finishedUrl: String) { val readabilityJs = loadReadabilityJs(context) + val katexJs = loadKatexJs(context) val script = """ (function() { try { @@ -384,6 +393,167 @@ object ReaderExtractor { if (sequels[i].parentNode) sequels[i].parentNode.removeChild(sequels[i]); } + // A page that uses TeX ships the SOURCE of its maths and + // renders it in the browser with its own script -- which + // never runs here, so the source is all there is. Confirmed + // live on lcamtuf's Substack (blog.coredump.cx/p/yours-truly): + // all four of that article's equations arrive as literal + // "\(p = \text{...}\)" text in a span inside a + // "latex-rendered" div, and Readability keeps that text + // verbatim (it strips the class, not the text) -- which is + // exactly what the reader was showing. So render it HERE, + // where JS is already running for Readability's sake, and + // hand the reader finished markup. + // + // Output is MathML, not KaTeX's usual HTML: the WebView + // renders MathML natively, so nothing downstream needs + // KaTeX's stylesheet, its webfonts, or any script -- the + // reader WebView keeps JavaScript OFF, as it must for + // untrusted article content. KaTeX's HTML output would not + // have survived anyway: it is entirely class-driven, and + // Readability strips class attributes. + var katex = (function() { + // Hand KaTeX's UMD wrapper a CommonJS target of our own. + // Left to itself it would take the AMD branch on any page + // that happens to ship an AMD loader (leaving `katex` + // undefined), or publish itself on the page's window -- + // and this is a tool we are using on the document, not + // something the document should see or be able to redefine. + var module = { exports: {} }; + var exports = module.exports; + var define = undefined; + $katexJs + return module.exports; + })(); + + (function renderTexSource() { + if (!katex || typeof katex.renderToString !== 'function') return; + // Delimiter pairs, scanned left to right. Single-$ inline + // maths is deliberately NOT supported: prices and currency + // in ordinary prose would be mangled far more often than + // real maths would be found. + var DELIMITERS = [ + { left: '$$', right: '$$', display: true }, + { left: '\\[', right: '\\]', display: true }, + { left: '\\(', right: '\\)', display: false } + ]; + // Where TeX-looking text is quoted on purpose rather than + // meant as maths -- an article ABOUT TeX shows its source. + var SKIP_TAGS = { + SCRIPT: 1, STYLE: 1, PRE: 1, CODE: 1, KBD: 1, + SAMP: 1, TEXTAREA: 1, NOSCRIPT: 1, MATH: 1 + }; + // A page is untrusted content: cap the work it can ask for. + // Far above any real article (the reported one has four). + var MAX_EXPRESSIONS = 300; + var rendered = 0; + + function renderOnce(tex, display) { + var holder = document.createElement('div'); + try { + holder.innerHTML = katex.renderToString(tex, { + output: 'mathml', + displayMode: display, + throwOnError: false, + strict: false + }); + } catch (e) { + return null; + } + var math = holder.querySelector('math'); + if (!math) return null; + // KaTeX attaches an <annotation> holding the TeX source + // again, for copy/paste tools. Nothing renders it, but it + // does count toward Readability's text-length scoring -- + // and it is the source we just replaced, so drop it. + var annotations = math.querySelectorAll('annotation'); + for (var a = 0; a < annotations.length; a++) { + annotations[a].parentNode.removeChild(annotations[a]); + } + return math; + } + + function renderTex(tex, display) { + // KaTeX refuses {align} and friends outside display mode + // (with throwOnError off it returns an error span and no + // <math> at all) -- and Substack wraps DISPLAY maths in + // INLINE delimiters, leaving the centring to a + // block-styled span. Confirmed live: both of the reported + // article's multi-line align blocks are \( ... \). So a + // mode that produces nothing is retried in the other one, + // and only if both come up empty is the source left + // exactly as it was -- no worse than before. + return renderOnce(tex, display) || renderOnce(tex, !display); + } + + function nextDelimiter(text, from) { + var best = null; + for (var d = 0; d < DELIMITERS.length; d++) { + var at = text.indexOf(DELIMITERS[d].left, from); + if (at === -1) continue; + if (!best || at < best.at) best = { at: at, delimiter: DELIMITERS[d] }; + } + return best; + } + + function isSkipped(node) { + for (var p = node.parentNode; p && p.nodeType === 1; p = p.parentNode) { + if (SKIP_TAGS[p.tagName]) return true; + } + return false; + } + + // Collect first, rewrite after: replacing a text node while + // the walker is still pointing into it invalidates the walk. + var walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, null, false); + var candidates = []; + var node; + while ((node = walker.nextNode())) { + var data = node.nodeValue || ''; + if (data.indexOf('\\(') === -1 && data.indexOf('\\[') === -1 && + data.indexOf('$$') === -1) continue; + if (isSkipped(node)) continue; + candidates.push(node); + } + + for (var c = 0; c < candidates.length; c++) { + var target = candidates[c]; + var text = target.nodeValue || ''; + var fragment = document.createDocumentFragment(); + var cursor = 0; + var replacedAny = false; + while (cursor < text.length && rendered < MAX_EXPRESSIONS) { + var found = nextDelimiter(text, cursor); + if (!found) break; + var innerStart = found.at + found.delimiter.left.length; + var closeAt = text.indexOf(found.delimiter.right, innerStart); + // An unpaired delimiter is just prose that happens to + // contain one. Leave the rest of the node alone. + if (closeAt === -1) break; + var tex = text.slice(innerStart, closeAt); + var math = tex.trim() ? renderTex(tex, found.delimiter.display) : null; + if (!math) { + // Not renderable: step past this opening delimiter + // and keep looking, rather than dropping the text. + cursor = innerStart; + continue; + } + if (found.at > cursor) { + fragment.appendChild(document.createTextNode(text.slice(cursor, found.at))); + } + fragment.appendChild(math); + cursor = closeAt + found.delimiter.right.length; + replacedAny = true; + rendered++; + } + if (!replacedAny) continue; + if (cursor < text.length) { + fragment.appendChild(document.createTextNode(text.slice(cursor))); + } + target.parentNode.replaceChild(fragment, target); + } + })(); + // Readability's default hidden-element skip (treating display:none / // visibility:hidden nodes as boilerplate) is correct for the vast // majority of sites -- it's what keeps collapsed nav/footer/sidebar diff --git a/app/src/main/java/org/terst/feedreader/readability/UnrenderedTex.kt b/app/src/main/java/org/terst/feedreader/readability/UnrenderedTex.kt new file mode 100644 index 0000000..aa46f7e --- /dev/null +++ b/app/src/main/java/org/terst/feedreader/readability/UnrenderedTex.kt @@ -0,0 +1,84 @@ +package org.terst.feedreader.readability + +/** + * Recognises a cached extraction that still holds raw TeX source instead of rendered maths. + * + * Needed for the same reason [ConsentNotice] is: fixing the extractor only changes what + * FUTURE extractions produce. The reported article ("we should render tex", lcamtuf's + * Substack) came with `extractedContent cached: true` -- its raw `\(p = \text{...}\)` + * source was already on disk, and every open read it straight back out, so the reader + * would have gone on showing TeX source no matter what the extractor learned to do. + * Recognising it on read re-extracts the article the next time it is opened, including + * everything extracted before maths rendering existed. + * + * Over-detecting is the real risk here, and it is a specific one: a false positive makes + * an article re-extract on EVERY open, for ever, because re-extracting cannot change a + * verdict the content itself causes. So the bar is a complete delimiter pair around + * something that actually looks like an expression, and never anything that already + * carries rendered MathML. + */ +object UnrenderedTex { + + /** + * Length past which a delimiter pair is read as two unrelated delimiters rather than + * one expression. Generous: the reported article's longest block (a ten-line `align` + * environment) is well under half this. + */ + private const val MAX_EXPRESSION_LENGTH = 2_000 + + /** Delimiter pairs, matching the ones extraction renders (see ReaderExtractor). */ + private val DELIMITERS = listOf("\\(" to "\\)", "\\[" to "\\]", "$$" to "$$") + + /** + * Characters no plain sentence between two stray delimiters is likely to contain but + * essentially every real expression does -- a command, a group, a sub- or superscript. + */ + private const val TEX_MARKERS = "\\{_^" + + /** + * Quoted source, not maths: extraction deliberately leaves `<pre>`/`<code>` alone, so + * an article demonstrating TeX would otherwise be convicted on every open and still, + * correctly, come back unchanged. + */ + private val CODE_BLOCK_REGEX = Regex( + "<(pre|code)\\b[^>]*>.*?</\\1\\s*>", + setOf(RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL) + ) + + /** + * True when [html] is an extraction whose maths never got rendered. + * + * Rendered maths anywhere in it means extraction already had its go at this page, and + * nothing is to be gained by sending it round again: an expression KaTeX could not + * render is still in there as source and always will be. + */ + fun looksLikeUnrenderedTex(html: String?): Boolean { + if (html.isNullOrBlank() || html.contains("<math", ignoreCase = true)) return false + + val prose = html.replace(CODE_BLOCK_REGEX, " ") + return DELIMITERS.any { (open, close) -> containsExpression(prose, open, close) } + } + + private fun containsExpression(text: String, open: String, close: String): Boolean { + var opensAt = text.indexOf(open) + while (opensAt >= 0) { + val innerStart = opensAt + open.length + val closesAt = text.indexOf(close, innerStart) + // No closing delimiter left in the content: whatever follows is prose that + // happens to contain an opening one. + if (closesAt < 0) break + if (isExpression(text.substring(innerStart, closesAt))) return true + opensAt = text.indexOf(open, innerStart) + } + return false + } + + private fun isExpression(inner: String): Boolean = + inner.isNotBlank() && + inner.length <= MAX_EXPRESSION_LENGTH && + // The span between two stray delimiters is markup, and markup brings its own + // braces and underscores along (ids, filenames, inline styles). A tag anywhere + // inside is on its own enough to say the two delimiters are unrelated. + !inner.contains('<') && + inner.any { it in TEX_MARKERS } +} |