diff options
Diffstat (limited to 'app/src/main/java/org/terst/feedreader/readability')
| -rw-r--r-- | app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt | 26 |
1 files changed, 25 insertions, 1 deletions
diff --git a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt index c41bbfb..f9adbb4 100644 --- a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt +++ b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt @@ -32,7 +32,12 @@ data class ExtractedArticle( * until the JS callback fires (or the timeout elapses). */ object ReaderExtractor { - private const val TIMEOUT_MS = 15_000L + // Confirmed live: a heavy, ad/tracker-laden real page (several Ars Technica articles) can + // legitimately take longer than 15s to finish loading everything before onPageFinished + // fires, causing extraction to time out and silently fall back to no content -- not a + // parsing failure, just not enough runway. 25s costs little (a spinner stays up slightly + // longer on a truly dead page) against a real reduction in spurious extraction failures. + private const val TIMEOUT_MS = 25_000L private val json = Json { ignoreUnknownKeys = true } suspend fun extract(context: Context, url: String): ExtractedArticle? = @@ -175,6 +180,25 @@ object ReaderExtractor { } } + // A news/blog article page's <header> conventionally holds everything + // BEFORE the real body: a section/category kicker, the dek/subhead, the + // byline, and the hero image + its caption. Readability doesn't exclude + // <header> by tag outright (only deprioritizes it in scoring), so on + // some articles enough of it scores highly enough to get pulled in + // alongside the real body -- confirmed live: a kicker icon, the dek + // paragraph, AND an orphaned duplicate hero-image caption (a mobile-only + // copy with no adjacent <img> at all) all leaking in above the real + // lede on an Ars Technica article. This is a pre-processing decision on + // our side, not a change to Readability's own algorithm: treat <header> + // as page chrome, never article content, and remove it before Readability + // ever sees the page. (The article's actual title is shown separately -- + // see ReaderWebView.wrapReaderHtml -- so losing the in-body <h1> here is + // fine, it was never meant to double as the body text anyway.) + var headers = document.querySelectorAll('header'); + for (var i = 0; i < headers.length; i++) { + headers[i].parentNode.removeChild(headers[i]); + } + // Readability's default hidden-element skip (treating display:none / // visibility:hidden nodes as boilerplate) is correct for the vast // majority of sites -- it's what keeps collapsed nav/footer/sidebar |