diff options
Diffstat (limited to 'app/src/main/java/org/terst/feedreader/readability')
| -rw-r--r-- | app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt | 76 |
1 files changed, 58 insertions, 18 deletions
diff --git a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt index 82300ac..3eaa158 100644 --- a/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt +++ b/app/src/main/java/org/terst/feedreader/readability/ReaderExtractor.kt @@ -74,6 +74,37 @@ object ReaderExtractor { return (article && article.textContent) ? article.textContent.trim().length : 0; } + // A common lazy-load pattern pairs a placeholder <img> with a + // sibling <noscript> holding the REAL <img> tag, meant for + // browsers with JS off. But our WebView runs WITH JS enabled + // (Readability.js itself needs it) -- and per the HTML spec, + // when scripting IS enabled, a <noscript> element's contents are + // parsed as opaque text, not as DOM, so document.querySelectorAll + // never even sees that real <img>. Pull it out manually: parse + // each <noscript>'s raw text as markup and promote any <img> + // inside to a real, visible sibling (replacing the placeholder if + // one immediately precedes it). + var noscripts = document.querySelectorAll('noscript'); + for (var i = 0; i < noscripts.length; i++) { + var ns = noscripts[i]; + var raw = ns.textContent || ''; + if (raw.indexOf('<img') === -1) continue; + var temp = document.createElement('div'); + temp.innerHTML = raw; + var realImg = temp.querySelector('img'); + if (!realImg) continue; + var prev = ns.previousElementSibling; + if (prev && prev.tagName === 'IMG') { + prev.setAttribute('src', realImg.getAttribute('src') || ''); + if (realImg.getAttribute('srcset')) prev.setAttribute('srcset', realImg.getAttribute('srcset')); + prev.removeAttribute('data-src'); + prev.removeAttribute('data-original'); + prev.removeAttribute('data-lazy-src'); + } else { + ns.parentNode.insertBefore(realImg, ns); + } + } + // Many sites lazy-load media via JS: the real URL sits in a // data-src/data-srcset attribute and only gets copied into // src/srcset once the site's own script runs -- which never @@ -117,31 +148,40 @@ object ReaderExtractor { // majority of sites -- it's what keeps collapsed nav/footer/sidebar // chrome out of the result. Try it first, unmodified. var normalArticle = new Readability(document.cloneNode(true)).parse(); - if (plainLength(normalArticle) >= 500) { - return JSON.stringify(normalArticle); - } - // Only reached when the normal pass came back thin. Some "soft" - // paywalls deliver the full article in the initial HTML response to - // every visitor, then hide it with CSS (inline or via a stylesheet - // rule) pending a client-side subscription check -- the content isn't - // withheld by the server, only its visibility is, and Readability's - // hidden-element skip discards it the same as any other hidden chrome. - // Un-hide and retry; keep whichever pass actually found more text, - // since un-hiding everything indiscriminately can just as easily pull - // in an unrelated hidden footer/notice on a normal (non-paywalled) page. - var all = document.querySelectorAll('*'); - for (var i = 0; i < all.length; i++) { - var el = all[i]; + // "Thin normal-pass result" is NOT a reliable paywall signal on its + // own -- plenty of ordinary, non-paywalled posts (kottke's linkblog + // style, for one) are also genuinely just short, and un-hiding + // EVERYTHING for those pulled in an unrelated hidden footer/social- + // links block instead of any real article content (reproduced live: + // the same kottke article kept coming back wrapped in site-footer + // boilerplate even with reader mode explicitly enabled for the feed). + // Target the actual, specific signal instead: elements whose class/id + // names the page itself labels as paywall/premium/subscriber content. + // West Hawaii Today's hidden div is literally class="paywall + // paywall_<id>" -- this only ever fires for genuine paywall markup, + // never for a plain short post. + var paywallEls = document.querySelectorAll( + '[class*="paywall" i], [id*="paywall" i], [class*="premium-content" i], ' + + '[class*="subscriber-content" i], [class*="metered-content" i]' + ); + var unhidAny = false; + for (var i = 0; i < paywallEls.length; i++) { + var el = paywallEls[i]; var cs = window.getComputedStyle(el); if (cs.display === 'none' || cs.visibility === 'hidden') { el.style.setProperty('display', 'block', 'important'); el.style.setProperty('visibility', 'visible', 'important'); + unhidAny = true; + } + } + if (unhidAny) { + var unhiddenArticle = new Readability(document.cloneNode(true)).parse(); + if (plainLength(unhiddenArticle) > plainLength(normalArticle)) { + return JSON.stringify(unhiddenArticle); } } - var unhiddenArticle = new Readability(document.cloneNode(true)).parse(); - var best = plainLength(unhiddenArticle) > plainLength(normalArticle) ? unhiddenArticle : normalArticle; - return JSON.stringify(best); + return JSON.stringify(normalArticle); } catch (e) { return JSON.stringify({error: e.toString()}); } |