path: root/server/internal/transcribe/whisper.go

// Package transcribe turns an episode's audio into its words, using whisper.cpp.
//
// The other half of the transcript story. A publisher's own transcript is exact and free and
// describes the publisher's ad-free master, so the server can only use one where no advertising
// was inserted -- 38 of 342 mirrored episodes, measured 2026-10-28. Recognition is approximate
// and costs CPU, and describes exactly the file that was mirrored, which is the other 304. Those
// are also the episodes where a transcript is worth most: they are the ones carrying inserted
// advertising, and the ones the ad scan has to work on.
//
// Shelling out rather than binding the library. The same call ffmpeg gets in this project, for
// the same reasons: one process per episode is killable, its memory is its own, and a crash in
// a C++ decoder cannot take the server down with it.
package transcribe

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/terst/listen/server/internal/transcript"
)

// Result is one episode's recognised words.
type Result struct {
	LanguageCode string
	Cues         []transcript.Cue
}

// Whisper runs whisper.cpp over an episode's audio.
type Whisper struct {
	// Binary is the whisper-cli executable; Model the ggml model file. Both empty means
	// recognition is not configured, which is a working state -- the server still takes
	// publishers' transcripts.
	Binary string
	Model  string
	// Threads is how many cores one run may use. Fewer than the machine has, deliberately:
	// detection is running on the same box and this must never be the thing that makes it slow.
	Threads int
	// WorkDir is where the decoded audio and the output land. Defaults to the system temp dir.
	WorkDir string
	// Timeout bounds one episode. A run that hangs must not stop every later episode from
	// being done.
	Timeout time.Duration
}

const (
	defaultThreads = 6
	// defaultTimeout is generous against the measured rate. whisper.cpp does about 11x
	// realtime on this machine with base.en on six threads, so an hour-long episode takes
	// about five minutes; an hour means something has gone wrong, not that the episode is long.
	defaultTimeout = time.Hour
	// sampleRate and channels are what whisper.cpp requires of its input, which is why the
	// audio is decoded first rather than handed over as an mp3.
	sampleRate = "16000"
	channels   = "1"
)

// Available reports whether recognition is configured and its files are present.
//
// Checked once rather than discovered per episode: without this the first attempt reports
// whatever the shell says about a missing binary, once for every episode in the library.
func (w Whisper) Available() bool {
	if w.Binary == "" || w.Model == "" {
		return false
	}
	for _, path := range []string{w.Binary, w.Model} {
		if _, err := os.Stat(path); err != nil {
			return false
		}
	}
	return true
}

// Transcribe recognises the words in one audio file.
func (w Whisper) Transcribe(ctx context.Context, audioPath string) (Result, error) {
	if !w.Available() {
		return Result{}, fmt.Errorf("transcribe: recognition is not configured")
	}

	ctx, cancel := context.WithTimeout(ctx, w.timeout())
	defer cancel()

	dir := w.WorkDir
	if dir == "" {
		dir = os.TempDir()
	}
	// Created here rather than assumed. The first real run failed on every episode because this
	// directory did not exist, and each failure was recorded as "do not try this one again".
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return Result{}, fmt.Errorf("transcribe: %w", err)
	}
	base := filepath.Join(dir, workingName(audioPath))
	wav := base + ".wav"
	// Named for the episode so two runs can never collide, and removed on the way out so a
	// leftover from a killed run is never read as this episode's words.
	defer os.Remove(wav)
	defer os.Remove(base + ".json")

	if err := w.decode(ctx, audioPath, wav); err != nil {
		return Result{}, err
	}
	if err := w.run(ctx, wav, base); err != nil {
		return Result{}, err
	}

	raw, err := os.ReadFile(base + ".json")
	if err != nil {
		return Result{}, fmt.Errorf("transcribe: %w", err)
	}
	return parseWhisperJSON(raw)
}

// decode turns the mirrored audio into the 16kHz mono PCM whisper.cpp requires.
func (w Whisper) decode(ctx context.Context, audioPath, wav string) error {
	cmd := exec.CommandContext(ctx, "ffmpeg",
		"-nostdin", "-loglevel", "error",
		"-i", audioPath,
		"-ar", sampleRate, "-ac", channels, "-c:a", "pcm_s16le",
		"-y", wav)
	if out, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("transcribe: decode %s: %w: %s", filepath.Base(audioPath), err, trim(out))
	}
	return nil
}

func (w Whisper) run(ctx context.Context, wav, base string) error {
	cmd := exec.CommandContext(ctx, w.Binary,
		"-m", w.Model,
		"-f", wav,
		"-t", fmt.Sprint(w.threads()),
		"-oj", "-of", base,
		"--no-prints")
	if out, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("transcribe: %s: %w: %s", filepath.Base(base), err, trim(out))
	}
	return nil
}

func (w Whisper) threads() int {
	if w.Threads > 0 {
		return w.Threads
	}
	return defaultThreads
}

func (w Whisper) timeout() time.Duration {
	if w.Timeout > 0 {
		return w.Timeout
	}
	return defaultTimeout
}

// whisperOutput is the shape whisper-cli writes with -oj.
type whisperOutput struct {
	Result struct {
		Language string `json:"language"`
	} `json:"result"`
	Transcription []struct {
		Offsets struct {
			From int64 `json:"from"`
			To   int64 `json:"to"`
		} `json:"offsets"`
		Text string `json:"text"`
	} `json:"transcription"`
}

// ParseOutput reads whisper-cli's output into cues. Exported for tools that work from a file
// whisper wrote earlier rather than running it.
func ParseOutput(raw []byte) (Result, error) { return parseWhisperJSON(raw) }

// parseWhisperJSON reads whisper-cli's output into cues.
//
// The offsets are milliseconds, which is what both the app and the ad scan want; the timestamps
// field beside them is the same numbers written as text.
func parseWhisperJSON(raw []byte) (Result, error) {
	var out whisperOutput
	if err := json.Unmarshal(raw, &out); err != nil {
		return Result{}, fmt.Errorf("transcribe: reading output: %w", err)
	}

	cues := make([]transcript.Cue, 0, len(out.Transcription))
	for _, segment := range out.Transcription {
		// whisper pads every segment with a leading space; a transcript full of them reads
		// badly and matches ad phrases differently.
		text := strings.TrimSpace(segment.Text)
		if text == "" || segment.Offsets.To <= segment.Offsets.From {
			continue
		}
		cues = append(cues, transcript.Cue{
			StartMs: segment.Offsets.From,
			EndMs:   segment.Offsets.To,
			Text:    text,
		})
	}
	if len(cues) == 0 {
		// A run that produced nothing is a failure, not an empty transcript. Stored as one the
		// episode would be marked done for ever -- exactly how the midroll scanner came to
		// report "none found" for episodes it had never managed to decode.
		return Result{}, fmt.Errorf("transcribe: the run produced no words")
	}
	return Result{LanguageCode: out.Result.Language, Cues: cues}, nil
}

// workingName is the episode's id, which is what its audio file is named after. Used for the
// decoded audio and the output so two runs can never collide.
func workingName(audioPath string) string {
	base := filepath.Base(audioPath)
	return strings.TrimSuffix(base, filepath.Ext(base))
}

func trim(out []byte) string {
	s := strings.TrimSpace(string(out))
	if len(s) > 300 {
		return s[:300] + "..."
	}
	return s
}