From 6a1847b391b87969a0fc4682a8fe33642d98105d Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Sat, 26 Sep 2026 19:13:09 +0000 Subject: Drop ID3 CHAP frames, having measured the real library Peter asked how expensive it would be to check before dropping them. Checking turned out to be cheaper than arguing about it, so here is the answer from his own 61 subscriptions rather than from general knowledge. Every feed's newest episode was fetched far enough to read its entire ID3 tag -- the declared size from the header, not a guessed window, which matters because several are multi-megabyte (Cruising the Waves carries 10MB of cover art) and a first pass with a 400KB window left thirteen of them inconclusive. ID3v2.2's 3-byte frame headers needed handling separately from v2.3/2.4's 4-byte ones, which is what made Dan Carlin's feeds look truncated at first. Not one of the 61 carries a CHAP frame. The frames present are all artwork and metadata: APIC, TALB, TIT2 and friends. Exactly one show publishes chapters at all -- Wards Way Podcast -- and it does so as Podcasting 2.0 JSON, which has been supported since the skip work landed. So CHAP is dropped rather than deferred. The two survey scripts are kept under tools/ so this can be rerun in a minute if a new subscription ever suggests otherwise, rather than re-derived. Two things fell out of the survey worth writing down. A feed can carry two chapters elements per item -- the podcastindex one with a url and a podlove one without; Buzzsprout emits both. The parser takes the first non-null url so it is unaffected, but a last-wins read would silently lose chapters. And the one show with chapters marks none of them toc:false, so ad-chapter skipping -- on by default as of this morning -- currently has nothing to act on anywhere in the library. It is correct and idle, waiting for a publisher who marks breaks. Documentation and tooling only; no behaviour change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- tools/chaprecheck.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tools/chaprecheck.py (limited to 'tools/chaprecheck.py') diff --git a/tools/chaprecheck.py b/tools/chaprecheck.py new file mode 100644 index 0000000..c130acf --- /dev/null +++ b/tools/chaprecheck.py @@ -0,0 +1,106 @@ +"""Second pass over the feeds whose ID3 tag did not fit the first window. + +Reads the declared tag size from the 10-byte header, then fetches exactly that much, so the +answer does not depend on a guessed window. Also handles ID3v2.2, whose frame headers are +6 bytes with 3-byte ids rather than 10 with 4 -- misparsing those is what made some of them +look truncated in the first pass. +""" +import sys +import urllib.request +import xml.etree.ElementTree as ET + +UA = ("Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Mobile Safari/537.36 Listen/1.0") + +WANTED = { + "Cruising the Waves Podcast", "Conversations with Tyler", "Darknet Diaries", + "Dan Carlin's Hardcore History", "1619", "Common Sense with Dan Carlin", "Reply All", + "Dan Carlin's Hardcore History: Addendum", "Flesh and Code", + "Talk Easy with Sam Fragoso", "This American Life", "Hooked On Wooden Boats", "Humans", + "Dream Chasers and Eccentrics", "Paranormal Investigations Hawaii", +} + + +def get(url, rangebytes=None, timeout=40): + req = urllib.request.Request(url, headers={"User-Agent": UA}) + if rangebytes: + req.add_header("Range", f"bytes={rangebytes[0]}-{rangebytes[1]}") + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.read() + + +def enclosure_of(feed_xml): + root = ET.fromstring(feed_xml) + for item in root.iter(): + if item.tag.split('}')[-1] not in ("item", "entry"): + continue + for child in item: + if child.tag.split('}')[-1] == "enclosure": + return child.get("url") + return None + + +def synchsafe(b): + return (b[0] << 21) | (b[1] << 14) | (b[2] << 7) | b[3] + + +def scan(blob, major): + """Walk the frame list. v2.2 uses 3-byte ids and 3-byte sizes; v2.3/2.4 use 4 and 4.""" + pos, ids = 10, [] + while True: + if major == 2: + if pos + 6 > len(blob): + break + fid = blob[pos:pos + 3] + if not fid.strip(b"\x00"): + break + size = int.from_bytes(blob[pos + 3:pos + 6], "big") + step = 6 + size + else: + if pos + 10 > len(blob): + break + fid = blob[pos:pos + 4] + if not fid.strip(b"\x00"): + break + raw = blob[pos + 4:pos + 8] + size = synchsafe(raw) if major == 4 else int.from_bytes(raw, "big") + step = 10 + size + if size <= 0: + break + ids.append(fid.decode("latin-1")) + pos += step + return ids + + +def check(name, url): + feed = get(url) + enclosure = enclosure_of(feed) + if not enclosure: + return name, "no enclosure", [] + head = get(enclosure, rangebytes=(0, 9)) + if not head.startswith(b"ID3"): + return name, "no ID3 tag at all", [] + major = head[3] + size = synchsafe(head[6:10]) + blob = get(enclosure, rangebytes=(0, 10 + size - 1)) + got = len(blob) >= 10 + size + ids = scan(blob, major) + verdict = "CHAP PRESENT" if "CHAP" in ids else "no CHAP" + detail = f"v2.{major}, tag {size / 1024:.0f}KB, {'complete' if got else 'SHORT'}, frames: {sorted(set(ids))}" + return name, verdict, detail + + +def main(path): + tree = ET.parse(path) + feeds = [(o.get("text"), o.get("xmlUrl")) + for o in tree.iter("outline") if o.get("xmlUrl") and o.get("text") in WANTED] + for name, url in feeds: + try: + n, verdict, detail = check(name, url) + print(f"{verdict:<14} {n[:42]:<42} {detail}", flush=True) + except Exception as e: + print(f"{'error':<14} {name[:42]:<42} {type(e).__name__}", flush=True) + + +if __name__ == "__main__": + main(sys.argv[1]) -- cgit v1.2.3