From 6a1847b391b87969a0fc4682a8fe33642d98105d Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Sat, 26 Sep 2026 19:13:09 +0000 Subject: Drop ID3 CHAP frames, having measured the real library Peter asked how expensive it would be to check before dropping them. Checking turned out to be cheaper than arguing about it, so here is the answer from his own 61 subscriptions rather than from general knowledge. Every feed's newest episode was fetched far enough to read its entire ID3 tag -- the declared size from the header, not a guessed window, which matters because several are multi-megabyte (Cruising the Waves carries 10MB of cover art) and a first pass with a 400KB window left thirteen of them inconclusive. ID3v2.2's 3-byte frame headers needed handling separately from v2.3/2.4's 4-byte ones, which is what made Dan Carlin's feeds look truncated at first. Not one of the 61 carries a CHAP frame. The frames present are all artwork and metadata: APIC, TALB, TIT2 and friends. Exactly one show publishes chapters at all -- Wards Way Podcast -- and it does so as Podcasting 2.0 JSON, which has been supported since the skip work landed. So CHAP is dropped rather than deferred. The two survey scripts are kept under tools/ so this can be rerun in a minute if a new subscription ever suggests otherwise, rather than re-derived. Two things fell out of the survey worth writing down. A feed can carry two chapters elements per item -- the podcastindex one with a url and a podlove one without; Buzzsprout emits both. The parser takes the first non-null url so it is unaffected, but a last-wins read would silently lose chapters. And the one show with chapters marks none of them toc:false, so ad-chapter skipping -- on by default as of this morning -- currently has nothing to act on anywhere in the library. It is correct and idle, waiting for a publisher who marks breaks. Documentation and tooling only; no behaviour change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- CLAUDE.md | 19 +++++++- tools/chaprecheck.py | 106 ++++++++++++++++++++++++++++++++++++++++++++ tools/chapsurvey.py | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 245 insertions(+), 2 deletions(-) create mode 100644 tools/chaprecheck.py create mode 100644 tools/chapsurvey.py diff --git a/CLAUDE.md b/CLAUDE.md index 0316ec3..ed4373d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -129,11 +129,26 @@ downloaded at once, Facebook videos are supported, the library ranks shows by th **Tier 3 skip detection — audio fingerprinting of theme music — is still not built.** Peter deferred it; the design plan puts it in Phase 3 and wants a desktop prototype against real -episodes before any Android code. ID3 `CHAP` frames are also not read: that means parsing ID3 -out of a possibly-partial MP3, and the JSON path covers the shows that mark chapters at all. +episodes before any Android code. + +**ID3 `CHAP` frames are dropped, not deferred** — decided 2026-09-26 by measuring Peter's actual +61 subscriptions rather than guessing. Every feed's newest episode was fetched far enough to read +its whole ID3 tag (declared size, not a guessed window — several are multi-megabyte, Cruising the +Waves is 10MB of cover art). **Not one carries a CHAP frame.** The frames present are all artwork +and metadata: APIC, TALB, TIT2 and friends. Exactly one show, Wards Way Podcast, publishes +chapters at all, and it does so as Podcasting 2.0 JSON, which is already supported. The survey +scripts are worth rerunning rather than re-deriving if this ever comes back up. + +**Ad-chapter skipping has nothing to act on in Peter's library yet.** That same survey: the one +show with chapters has six navigable chapters and *zero* marked `toc: false`. The feature is on +by default and correct; it is simply waiting for a publisher who marks breaks. **Known gaps in what is built** (deliberate, not forgotten): +- **A feed can carry two `<...:chapters>` elements per item** — the podcastindex one with a + `url`, and a podlove one with none. Buzzsprout emits both. The parser takes the first non-null + url, so order does not matter; a naive last-wins read gets null and silently loses chapters. + - **Ad-chapter skipping trusts `toc: false` and nothing else.** No title matching: a show discussing its own sponsorship deal has a chapter called "Sponsors" that is the actual programme. It shipped off by default out of caution and Peter asked for it on after living diff --git a/tools/chaprecheck.py b/tools/chaprecheck.py new file mode 100644 index 0000000..c130acf --- /dev/null +++ b/tools/chaprecheck.py @@ -0,0 +1,106 @@ +"""Second pass over the feeds whose ID3 tag did not fit the first window. + +Reads the declared tag size from the 10-byte header, then fetches exactly that much, so the +answer does not depend on a guessed window. Also handles ID3v2.2, whose frame headers are +6 bytes with 3-byte ids rather than 10 with 4 -- misparsing those is what made some of them +look truncated in the first pass. +""" +import sys +import urllib.request +import xml.etree.ElementTree as ET + +UA = ("Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Mobile Safari/537.36 Listen/1.0") + +WANTED = { + "Cruising the Waves Podcast", "Conversations with Tyler", "Darknet Diaries", + "Dan Carlin's Hardcore History", "1619", "Common Sense with Dan Carlin", "Reply All", + "Dan Carlin's Hardcore History: Addendum", "Flesh and Code", + "Talk Easy with Sam Fragoso", "This American Life", "Hooked On Wooden Boats", "Humans", + "Dream Chasers and Eccentrics", "Paranormal Investigations Hawaii", +} + + +def get(url, rangebytes=None, timeout=40): + req = urllib.request.Request(url, headers={"User-Agent": UA}) + if rangebytes: + req.add_header("Range", f"bytes={rangebytes[0]}-{rangebytes[1]}") + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.read() + + +def enclosure_of(feed_xml): + root = ET.fromstring(feed_xml) + for item in root.iter(): + if item.tag.split('}')[-1] not in ("item", "entry"): + continue + for child in item: + if child.tag.split('}')[-1] == "enclosure": + return child.get("url") + return None + + +def synchsafe(b): + return (b[0] << 21) | (b[1] << 14) | (b[2] << 7) | b[3] + + +def scan(blob, major): + """Walk the frame list. v2.2 uses 3-byte ids and 3-byte sizes; v2.3/2.4 use 4 and 4.""" + pos, ids = 10, [] + while True: + if major == 2: + if pos + 6 > len(blob): + break + fid = blob[pos:pos + 3] + if not fid.strip(b"\x00"): + break + size = int.from_bytes(blob[pos + 3:pos + 6], "big") + step = 6 + size + else: + if pos + 10 > len(blob): + break + fid = blob[pos:pos + 4] + if not fid.strip(b"\x00"): + break + raw = blob[pos + 4:pos + 8] + size = synchsafe(raw) if major == 4 else int.from_bytes(raw, "big") + step = 10 + size + if size <= 0: + break + ids.append(fid.decode("latin-1")) + pos += step + return ids + + +def check(name, url): + feed = get(url) + enclosure = enclosure_of(feed) + if not enclosure: + return name, "no enclosure", [] + head = get(enclosure, rangebytes=(0, 9)) + if not head.startswith(b"ID3"): + return name, "no ID3 tag at all", [] + major = head[3] + size = synchsafe(head[6:10]) + blob = get(enclosure, rangebytes=(0, 10 + size - 1)) + got = len(blob) >= 10 + size + ids = scan(blob, major) + verdict = "CHAP PRESENT" if "CHAP" in ids else "no CHAP" + detail = f"v2.{major}, tag {size / 1024:.0f}KB, {'complete' if got else 'SHORT'}, frames: {sorted(set(ids))}" + return name, verdict, detail + + +def main(path): + tree = ET.parse(path) + feeds = [(o.get("text"), o.get("xmlUrl")) + for o in tree.iter("outline") if o.get("xmlUrl") and o.get("text") in WANTED] + for name, url in feeds: + try: + n, verdict, detail = check(name, url) + print(f"{verdict:<14} {n[:42]:<42} {detail}", flush=True) + except Exception as e: + print(f"{'error':<14} {name[:42]:<42} {type(e).__name__}", flush=True) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/tools/chapsurvey.py b/tools/chapsurvey.py new file mode 100644 index 0000000..da4aa84 --- /dev/null +++ b/tools/chapsurvey.py @@ -0,0 +1,122 @@ +"""Survey Peter's real subscriptions for chapter markers. + +Answers one question: of the shows he actually listens to, how many publish Podcasting 2.0 +chapter JSON (already supported), how many only have ID3 CHAP frames (not supported), and how +many have nothing at all. + +ID3v2 tags sit at the very start of an MP3, so a Range request for the first chunk is enough -- +no need to pull a whole episode. +""" +import concurrent.futures as cf +import re +import sys +import urllib.request +import xml.etree.ElementTree as ET + +UA = ("Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Mobile Safari/537.36 Listen/1.0") +HEAD_BYTES = 400_000 + + +def get(url, timeout=25, rangebytes=None): + req = urllib.request.Request(url, headers={"User-Agent": UA}) + if rangebytes: + req.add_header("Range", f"bytes=0-{rangebytes - 1}") + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.read() + + +def newest_item(feed_xml): + """The newest item's enclosure URL and whether the feed declares chapter JSON.""" + root = ET.fromstring(feed_xml) + for item in root.iter(): + if item.tag.split('}')[-1] not in ("item", "entry"): + continue + enclosure, chapters, mime = None, False, None + for child in item: + tag = child.tag.split('}')[-1] + if tag == "enclosure": + enclosure = child.get("url") + mime = child.get("type") + elif tag == "chapters": + chapters = True + return enclosure, chapters, mime + return None, False, None + + +def synchsafe(b): + return (b[0] << 21) | (b[1] << 14) | (b[2] << 7) | b[3] + + +def has_chap(blob): + """Parse the ID3v2 frame list looking for a CHAP frame. Returns (found, note).""" + if not blob.startswith(b"ID3"): + return False, "no ID3 tag" + major = blob[3] + tag_size = synchsafe(blob[6:10]) + end = min(10 + tag_size, len(blob)) + pos = 10 + saw_any = False + while pos + 10 <= end: + fid = blob[pos:pos + 4] + if not re.fullmatch(rb"[A-Z0-9]{4}", fid): + break # padding, or the end of the frame list + raw = blob[pos + 4:pos + 8] + size = synchsafe(raw) if major == 4 else int.from_bytes(raw, "big") + if size <= 0: + break + saw_any = True + if fid == b"CHAP": + return True, f"ID3v2.{major}" + pos += 10 + size + truncated = 10 + tag_size > len(blob) + if truncated: + return False, f"ID3v2.{major}, tag larger than fetched window" + return False, f"ID3v2.{major}" if saw_any else "ID3 tag unreadable" + + +def survey(name, url): + try: + feed = get(url) + except Exception as e: + return name, "feed-error", type(e).__name__, "" + try: + enclosure, json_chapters, mime = newest_item(feed) + except Exception as e: + return name, "parse-error", type(e).__name__, "" + if not enclosure: + return name, "no-enclosure", "", "" + if json_chapters: + return name, "json-chapters", "", mime or "" + if mime and "mpeg" not in mime and "mp3" not in mime: + return name, "not-mp3", mime, mime or "" + try: + blob = get(enclosure, rangebytes=HEAD_BYTES) + except Exception as e: + return name, "media-error", type(e).__name__, mime or "" + found, note = has_chap(blob) + return name, ("id3-chap" if found else "none"), note, mime or "" + + +def main(path): + tree = ET.parse(path) + feeds = [(o.get("text"), o.get("xmlUrl")) + for o in tree.iter("outline") if o.get("xmlUrl")] + print(f"{len(feeds)} feeds\n", flush=True) + + results = [] + with cf.ThreadPoolExecutor(max_workers=8) as pool: + for r in pool.map(lambda f: survey(*f), feeds): + results.append(r) + print(f" {r[1]:<14} {r[0][:52]:<52} {r[2]}", flush=True) + + print("\n=== summary ===") + counts = {} + for _, verdict, _, _ in results: + counts[verdict] = counts.get(verdict, 0) + 1 + for k in sorted(counts, key=lambda k: -counts[k]): + print(f" {counts[k]:>3} {k}") + + +if __name__ == "__main__": + main(sys.argv[1]) -- cgit v1.2.3