// Package transcribe turns an episode's audio into its words, using whisper.cpp. // // The other half of the transcript story. A publisher's own transcript is exact and free and // describes the publisher's ad-free master, so the server can only use one where no advertising // was inserted -- 38 of 342 mirrored episodes, measured 2026-10-28. Recognition is approximate // and costs CPU, and describes exactly the file that was mirrored, which is the other 304. Those // are also the episodes where a transcript is worth most: they are the ones carrying inserted // advertising, and the ones the ad scan has to work on. // // Shelling out rather than binding the library. The same call ffmpeg gets in this project, for // the same reasons: one process per episode is killable, its memory is its own, and a crash in // a C++ decoder cannot take the server down with it. package transcribe import ( "context" "encoding/json" "fmt" "os" "os/exec" "path/filepath" "strings" "time" "github.com/terst/listen/server/internal/transcript" ) // Result is one episode's recognised words. type Result struct { LanguageCode string Cues []transcript.Cue } // Whisper runs whisper.cpp over an episode's audio. type Whisper struct { // Binary is the whisper-cli executable; Model the ggml model file. Both empty means // recognition is not configured, which is a working state -- the server still takes // publishers' transcripts. Binary string Model string // Threads is how many cores one run may use. Fewer than the machine has, deliberately: // detection is running on the same box and this must never be the thing that makes it slow. Threads int // WorkDir is where the decoded audio and the output land. Defaults to the system temp dir. WorkDir string // Timeout bounds one episode. A run that hangs must not stop every later episode from // being done. Timeout time.Duration } const ( defaultThreads = 6 // defaultTimeout is generous against the measured rate. whisper.cpp does about 11x // realtime on this machine with base.en on six threads, so an hour-long episode takes // about five minutes; an hour means something has gone wrong, not that the episode is long. defaultTimeout = time.Hour // sampleRate and channels are what whisper.cpp requires of its input, which is why the // audio is decoded first rather than handed over as an mp3. sampleRate = "16000" channels = "1" ) // Available reports whether recognition is configured and its files are present. // // Checked once rather than discovered per episode: without this the first attempt reports // whatever the shell says about a missing binary, once for every episode in the library. func (w Whisper) Available() bool { if w.Binary == "" || w.Model == "" { return false } for _, path := range []string{w.Binary, w.Model} { if _, err := os.Stat(path); err != nil { return false } } return true } // Transcribe recognises the words in one audio file. func (w Whisper) Transcribe(ctx context.Context, audioPath string) (Result, error) { if !w.Available() { return Result{}, fmt.Errorf("transcribe: recognition is not configured") } ctx, cancel := context.WithTimeout(ctx, w.timeout()) defer cancel() dir := w.WorkDir if dir == "" { dir = os.TempDir() } // Created here rather than assumed. The first real run failed on every episode because this // directory did not exist, and each failure was recorded as "do not try this one again". if err := os.MkdirAll(dir, 0o755); err != nil { return Result{}, fmt.Errorf("transcribe: %w", err) } base := filepath.Join(dir, workingName(audioPath)) wav := base + ".wav" // Named for the episode so two runs can never collide, and removed on the way out so a // leftover from a killed run is never read as this episode's words. defer os.Remove(wav) defer os.Remove(base + ".json") if err := w.decode(ctx, audioPath, wav); err != nil { return Result{}, err } if err := w.run(ctx, wav, base); err != nil { return Result{}, err } raw, err := os.ReadFile(base + ".json") if err != nil { return Result{}, fmt.Errorf("transcribe: %w", err) } return parseWhisperJSON(raw) } // decode turns the mirrored audio into the 16kHz mono PCM whisper.cpp requires. func (w Whisper) decode(ctx context.Context, audioPath, wav string) error { cmd := exec.CommandContext(ctx, "ffmpeg", "-nostdin", "-loglevel", "error", "-i", audioPath, "-ar", sampleRate, "-ac", channels, "-c:a", "pcm_s16le", "-y", wav) if out, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("transcribe: decode %s: %w: %s", filepath.Base(audioPath), err, trim(out)) } return nil } func (w Whisper) run(ctx context.Context, wav, base string) error { cmd := exec.CommandContext(ctx, w.Binary, "-m", w.Model, "-f", wav, "-t", fmt.Sprint(w.threads()), "-oj", "-of", base, "--no-prints") if out, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("transcribe: %s: %w: %s", filepath.Base(base), err, trim(out)) } return nil } func (w Whisper) threads() int { if w.Threads > 0 { return w.Threads } return defaultThreads } func (w Whisper) timeout() time.Duration { if w.Timeout > 0 { return w.Timeout } return defaultTimeout } // whisperOutput is the shape whisper-cli writes with -oj. type whisperOutput struct { Result struct { Language string `json:"language"` } `json:"result"` Transcription []struct { Offsets struct { From int64 `json:"from"` To int64 `json:"to"` } `json:"offsets"` Text string `json:"text"` } `json:"transcription"` } // ParseOutput reads whisper-cli's output into cues. Exported for tools that work from a file // whisper wrote earlier rather than running it. func ParseOutput(raw []byte) (Result, error) { return parseWhisperJSON(raw) } // parseWhisperJSON reads whisper-cli's output into cues. // // The offsets are milliseconds, which is what both the app and the ad scan want; the timestamps // field beside them is the same numbers written as text. func parseWhisperJSON(raw []byte) (Result, error) { var out whisperOutput if err := json.Unmarshal(raw, &out); err != nil { return Result{}, fmt.Errorf("transcribe: reading output: %w", err) } cues := make([]transcript.Cue, 0, len(out.Transcription)) for _, segment := range out.Transcription { // whisper pads every segment with a leading space; a transcript full of them reads // badly and matches ad phrases differently. text := strings.TrimSpace(segment.Text) if text == "" || segment.Offsets.To <= segment.Offsets.From { continue } cues = append(cues, transcript.Cue{ StartMs: segment.Offsets.From, EndMs: segment.Offsets.To, Text: text, }) } if len(cues) == 0 { // A run that produced nothing is a failure, not an empty transcript. Stored as one the // episode would be marked done for ever -- exactly how the midroll scanner came to // report "none found" for episodes it had never managed to decode. return Result{}, fmt.Errorf("transcribe: the run produced no words") } return Result{LanguageCode: out.Result.Language, Cues: cues}, nil } // workingName is the episode's id, which is what its audio file is named after. Used for the // decoded audio and the output so two runs can never collide. func workingName(audioPath string) string { base := filepath.Base(audioPath) return strings.TrimSuffix(base, filepath.Ext(base)) } func trim(out []byte) string { s := strings.TrimSpace(string(out)) if len(s) > 300 { return s[:300] + "..." } return s }