1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
|
package org.terst.feedreader.readability
/**
* Recognises a cached extraction that still holds raw TeX source instead of rendered maths.
*
* Needed for the same reason [ConsentNotice] is: fixing the extractor only changes what
* FUTURE extractions produce. The reported article ("we should render tex", lcamtuf's
* Substack) came with `extractedContent cached: true` -- its raw `\(p = \text{...}\)`
* source was already on disk, and every open read it straight back out, so the reader
* would have gone on showing TeX source no matter what the extractor learned to do.
* Recognising it on read re-extracts the article the next time it is opened, including
* everything extracted before maths rendering existed.
*
* Over-detecting is the real risk here, and it is a specific one: a false positive makes
* an article re-extract on EVERY open, for ever, because re-extracting cannot change a
* verdict the content itself causes. So the bar is a complete delimiter pair around
* something that actually looks like an expression, and never anything that already
* carries rendered MathML.
*/
object UnrenderedTex {
/**
* Length past which a delimiter pair is read as two unrelated delimiters rather than
* one expression. Generous: the reported article's longest block (a ten-line `align`
* environment) is well under half this.
*/
private const val MAX_EXPRESSION_LENGTH = 2_000
/** Delimiter pairs, matching the ones extraction renders (see ReaderExtractor). */
private val DELIMITERS = listOf("\\(" to "\\)", "\\[" to "\\]", "$$" to "$$")
/**
* Characters no plain sentence between two stray delimiters is likely to contain but
* essentially every real expression does -- a command, a group, a sub- or superscript.
*/
private const val TEX_MARKERS = "\\{_^"
/**
* Quoted source, not maths: extraction deliberately leaves `<pre>`/`<code>` alone, so
* an article demonstrating TeX would otherwise be convicted on every open and still,
* correctly, come back unchanged.
*/
private val CODE_BLOCK_REGEX = Regex(
"<(pre|code)\\b[^>]*>.*?</\\1\\s*>",
setOf(RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL)
)
/**
* True when [html] is an extraction whose maths never got rendered.
*
* Rendered maths anywhere in it means extraction already had its go at this page, and
* nothing is to be gained by sending it round again: an expression KaTeX could not
* render is still in there as source and always will be.
*/
fun looksLikeUnrenderedTex(html: String?): Boolean {
if (html.isNullOrBlank() || html.contains("<math", ignoreCase = true)) return false
val prose = html.replace(CODE_BLOCK_REGEX, " ")
return DELIMITERS.any { (open, close) -> containsExpression(prose, open, close) }
}
private fun containsExpression(text: String, open: String, close: String): Boolean {
var opensAt = text.indexOf(open)
while (opensAt >= 0) {
val innerStart = opensAt + open.length
val closesAt = text.indexOf(close, innerStart)
// No closing delimiter left in the content: whatever follows is prose that
// happens to contain an opening one.
if (closesAt < 0) break
if (isExpression(text.substring(innerStart, closesAt))) return true
opensAt = text.indexOf(open, innerStart)
}
return false
}
private fun isExpression(inner: String): Boolean =
inner.isNotBlank() &&
inner.length <= MAX_EXPRESSION_LENGTH &&
// The span between two stray delimiters is markup, and markup brings its own
// braces and underscores along (ids, filenames, inline styles). A tag anywhere
// inside is on its own enough to say the two delimiters are unrelated.
!inner.contains('<') &&
inner.any { it in TEX_MARKERS }
}
|
