Skip to content

Commit 1b6697f

Browse files
committed
feat: configurable words per caption segment (#3275)
Adds a segmentWords caption setting (1-8, default 6) that controls how many words a generated caption segment holds, threaded from the captions panels through both transcription pipelines into the shared chunker. At 1 the chunker switches to the one word on screen at a time style: every word becomes its own segment and the multi-word rules (weak boundary extension, short tail merge) are bypassed. Above 1 the existing rules apply relative to the chosen size, so the default of 6 reproduces current behavior exactly. Settings surface: cap_project::CaptionSettings.segmentWords (JSON segmentWords), a Words Per Caption slider in the Tauri editor's captions tab and in the gpui captions sidebar, and the tauri transcribe_audio command gains an optional segmentWords argument.
1 parent 7f84158 commit 1b6697f

12 files changed

Lines changed: 147 additions & 21 deletions

File tree

‎apps/desktop-gpui/src/editor_tabs.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -454,6 +454,7 @@ pub enum CaptionSlider {
454454
Size,
455455
BackgroundOpacity,
456456
FadeDuration,
457+
SegmentWords,
457458
ManualX,
458459
ManualY,
459460
}
@@ -519,6 +520,7 @@ impl CaptionSlider {
519520
Self::BackgroundOpacity => (0., 100., 1.),
520521
// `value={[getSetting("fadeDuration") * 100]}` over 0..50.
521522
Self::FadeDuration => (0., 50., 1.),
523+
Self::SegmentWords => (1., 8., 1.),
522524
Self::ManualX | Self::ManualY => (0., 100., 1.),
523525
}
524526
}
@@ -529,6 +531,7 @@ impl CaptionSlider {
529531
Self::Size => settings.size as f32,
530532
Self::BackgroundOpacity => settings.background_opacity as f32,
531533
Self::FadeDuration => settings.fade_duration * 100.,
534+
Self::SegmentWords => settings.segment_words as f32,
532535
Self::ManualX => caption_manual_center(&settings).0 * 100.,
533536
Self::ManualY => caption_manual_center(&settings).1 * 100.,
534537
}
@@ -1145,6 +1148,7 @@ impl EditorWindow {
11451148
CaptionSlider::Size => settings.size = value.round() as u32,
11461149
CaptionSlider::BackgroundOpacity => settings.background_opacity = value.round() as u32,
11471150
CaptionSlider::FadeDuration => settings.fade_duration = value / 100.,
1151+
CaptionSlider::SegmentWords => settings.segment_words = (value.round() as u32).max(1),
11481152
CaptionSlider::ManualX => set_caption_manual_center(settings, Some(value), None),
11491153
CaptionSlider::ManualY => set_caption_manual_center(settings, None, Some(value)),
11501154
});
@@ -2959,6 +2963,12 @@ impl EditorWindow {
29592963
SliderKey::Caption(CaptionSlider::FadeDuration),
29602964
format!("{:.0}ms", settings.fade_duration * 1000.),
29612965
cx,
2966+
))
2967+
.child(self.labelled_readout(
2968+
"Words Per Caption",
2969+
SliderKey::Caption(CaptionSlider::SegmentWords),
2970+
format!("{:.0}", settings.segment_words),
2971+
cx,
29622972
)),
29632973
),
29642974
)
@@ -3270,6 +3280,7 @@ impl EditorWindow {
32703280
let project_path = self.project_path.clone();
32713281
let model = self.selected_caption_model().name;
32723282
let language = self.sidebar.caption_language;
3283+
let segment_words = caption_settings(&self.project).segment_words.max(1) as usize;
32733284

32743285
if !transcription::begin_generation(&project_path) {
32753286
return;
@@ -3280,7 +3291,7 @@ impl EditorWindow {
32803291
let task_path = project_path.clone();
32813292
let task = gpui_tokio::Tokio::spawn(cx, async move {
32823293
tokio::task::spawn_blocking(move || {
3283-
transcription::transcribe_blocking(&task_path, model, language)
3294+
transcription::transcribe_blocking(&task_path, model, language, segment_words)
32843295
})
32853296
.await
32863297
.map_err(|error| format!("Transcription task panicked: {error}"))?

‎apps/desktop-gpui/src/transcription.rs‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1430,6 +1430,7 @@ fn process_with_whisper(
14301430
context: Arc<WhisperContext>,
14311431
language: &str,
14321432
transcription_hints: &[String],
1433+
segment_words: usize,
14331434
) -> Result<Vec<CaptionSegment>, String> {
14341435
let mut params = FullParams::new(SamplingStrategy::BeamSearch {
14351436
beam_size: 5,
@@ -1555,7 +1556,7 @@ fn process_with_whisper(
15551556
continue;
15561557
}
15571558

1558-
for (chunk_idx, chunk_words) in caption_word_chunks(&words).into_iter().enumerate() {
1559+
for (chunk_idx, chunk_words) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() {
15591560
let segment_text = caption_text_from_words(chunk_words);
15601561
let segment_start = chunk_words
15611562
.first()
@@ -1623,6 +1624,7 @@ fn parakeet_chunk_ranges(samples: &[f32], sample_rate: usize) -> Vec<std::ops::R
16231624
fn process_with_parakeet(
16241625
audio_path: &Path,
16251626
model_dir: &str,
1627+
segment_words: usize,
16261628
) -> Result<Vec<CaptionSegment>, String> {
16271629
let cached_model = {
16281630
let guard = PARAKEET_CONTEXT
@@ -1707,14 +1709,15 @@ fn process_with_parakeet(
17071709

17081710
tracing::info!("Parakeet produced {} timed tokens", tokens.len());
17091711

1710-
caption_segments_from_tokens(tokens)
1712+
caption_segments_from_tokens(tokens, segment_words)
17111713
}
17121714

17131715
/// Parakeet's timed word tokens -> caption segments: drop empty tokens,
17141716
/// normalize the words and chunk them.
17151717
#[cfg(any(test, not(all(target_os = "macos", target_arch = "x86_64"))))]
17161718
fn caption_segments_from_tokens(
17171719
tokens: impl IntoIterator<Item = CaptionWord>,
1720+
segment_words: usize,
17181721
) -> Result<Vec<CaptionSegment>, String> {
17191722
let words = normalize_caption_words(
17201723
tokens
@@ -1733,7 +1736,7 @@ fn caption_segments_from_tokens(
17331736
}
17341737

17351738
let mut segments = Vec::new();
1736-
for (chunk_idx, chunk) in caption_word_chunks(&words).into_iter().enumerate() {
1739+
for (chunk_idx, chunk) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() {
17371740
segments.push(CaptionSegment {
17381741
id: format!("segment-{chunk_idx}"),
17391742
start: chunk.first().map_or(0.0, |word| word.start),
@@ -1750,6 +1753,7 @@ fn caption_segments_from_tokens(
17501753
fn process_with_parakeet(
17511754
_audio_path: &Path,
17521755
_model_dir: &str,
1756+
_segment_words: usize,
17531757
) -> Result<Vec<CaptionSegment>, String> {
17541758
Err(PARAKEET_UNSUPPORTED_MESSAGE.to_string())
17551759
}
@@ -1766,6 +1770,7 @@ pub fn transcribe_blocking(
17661770
project_path: &Path,
17671771
model: &str,
17681772
language: &str,
1773+
segment_words: usize,
17691774
) -> Result<Vec<CaptionSegment>, String> {
17701775
let requested = model;
17711776
// A download writes straight to the model's final path, so a model still
@@ -1794,14 +1799,15 @@ pub fn transcribe_blocking(
17941799
let _guard = lock_transcription_worker_slot();
17951800
if is_parakeet_model(model) {
17961801
tracing::info!("Using Parakeet TDT engine");
1797-
let result = process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref());
1802+
let result =
1803+
process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref(), segment_words);
17981804
schedule_parakeet_release();
17991805
result
18001806
} else {
18011807
tracing::info!("Using Whisper engine");
18021808
let hints = transcription_hints();
18031809
let context = get_whisper_context(model_file.to_string_lossy().as_ref())?;
1804-
let result = process_with_whisper(audio_path, context, language, &hints);
1810+
let result = process_with_whisper(audio_path, context, language, &hints, segment_words);
18051811
release_whisper_context_after_transcription();
18061812
result
18071813
}
@@ -1857,7 +1863,7 @@ fn extract_audio_from_video(video_path: &Path, output_path: &Path) -> Result<(),
18571863
// ---------------------------------------------------------------------------
18581864

18591865
use cap_project::caption_words::{
1860-
caption_text_from_words, caption_word_chunks, normalize_caption_words,
1866+
caption_text_from_words, caption_word_chunks, caption_word_chunks_with, normalize_caption_words,
18611867
MAX_CAPTION_WORD_DURATION,
18621868
};
18631869

‎apps/desktop/src-tauri/src/captions.rs‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ lazy_static::lazy_static! {
7676
const WHISPER_SAMPLE_RATE: u32 = 16000;
7777

7878
use cap_project::caption_words::{
79-
caption_text_from_words, caption_word_chunks, normalize_caption_words,
79+
caption_text_from_words, caption_word_chunks, caption_word_chunks_with, normalize_caption_words,
8080
MAX_CAPTION_WORD_DURATION,
8181
};
8282

@@ -802,6 +802,7 @@ fn process_with_whisper(
802802
context: Arc<WhisperContext>,
803803
language: &str,
804804
transcription_hints: &[String],
805+
segment_words: usize,
805806
) -> Result<CaptionData, String> {
806807
log::info!("=== WHISPER TRANSCRIPTION START ===");
807808
log::info!("Processing audio file: {audio_path:?}");
@@ -1023,7 +1024,7 @@ fn process_with_whisper(
10231024
continue;
10241025
}
10251026

1026-
for (chunk_idx, chunk_words) in caption_word_chunks(&words).into_iter().enumerate() {
1027+
for (chunk_idx, chunk_words) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() {
10271028
let segment_text = caption_text_from_words(chunk_words);
10281029

10291030
let segment_start = chunk_words
@@ -1091,6 +1092,7 @@ fn build_initial_prompt(transcription_hints: &[String]) -> Option<String> {
10911092
fn process_with_parakeet(
10921093
audio_path: &std::path::Path,
10931094
model_dir: &str,
1095+
segment_words: usize,
10941096
) -> Result<CaptionData, String> {
10951097
tracing::info!("Processing audio file: {audio_path:?}");
10961098
tracing::info!("Model directory: {model_dir}");
@@ -1164,7 +1166,7 @@ fn process_with_parakeet(
11641166

11651167
let mut segments = Vec::new();
11661168

1167-
for (chunk_idx, chunk) in caption_word_chunks(&words).into_iter().enumerate() {
1169+
for (chunk_idx, chunk) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() {
11681170
let segment_text = caption_text_from_words(chunk);
11691171
let segment_start = chunk.first().map(|w| w.start).unwrap_or(0.0);
11701172
let segment_end = chunk.last().map(|w| w.end).unwrap_or(0.0);
@@ -1194,6 +1196,7 @@ fn process_with_parakeet(
11941196
fn process_with_parakeet(
11951197
_audio_path: &std::path::Path,
11961198
_model_dir: &str,
1199+
_segment_words: usize,
11971200
) -> Result<CaptionData, String> {
11981201
Err(PARAKEET_UNSUPPORTED_MESSAGE.to_string())
11991202
}
@@ -1207,11 +1210,13 @@ pub async fn transcribe_audio(
12071210
model_path: String,
12081211
language: String,
12091212
engine: TranscriptionEngine,
1213+
segment_words: Option<u32>,
12101214
) -> Result<CaptionData, String> {
12111215
log::info!("=== TRANSCRIBE AUDIO COMMAND START ===");
12121216
log::info!("Video path: {}", video_path);
12131217
log::info!("Model path: {}", model_path);
12141218
log::info!("Language: {}", language);
1219+
let segment_words = segment_words.unwrap_or(6).max(1) as usize;
12151220

12161221
let validated_model_path = validate_model_path(&app, &model_path)?;
12171222

@@ -1259,7 +1264,7 @@ pub async fn transcribe_audio(
12591264
let model_dir = model_path.clone();
12601265
tokio::task::spawn_blocking(move || {
12611266
let _guard = lock_transcription_worker_slot();
1262-
process_with_parakeet(&audio_path, &model_dir)
1267+
process_with_parakeet(&audio_path, &model_dir, segment_words)
12631268
})
12641269
.await
12651270
.map_err(|e| format!("Parakeet task panicked: {e}"))?
@@ -1284,8 +1289,13 @@ pub async fn transcribe_audio(
12841289
return Err(format!("Failed to initialize transcription model: {e}"));
12851290
}
12861291
};
1287-
let result =
1288-
process_with_whisper(&audio_path, context, &language, &transcription_hints);
1292+
let result = process_with_whisper(
1293+
&audio_path,
1294+
context,
1295+
&language,
1296+
&transcription_hints,
1297+
segment_words,
1298+
);
12891299
release_whisper_context_after_transcription();
12901300
result
12911301
})
@@ -1499,6 +1509,10 @@ pub async fn save_captions(
14991509
"activeWordHighlight".to_string(),
15001510
serde_json::Value::Bool(settings.active_word_highlight),
15011511
);
1512+
settings_obj.insert(
1513+
"segmentWords".to_string(),
1514+
serde_json::Value::Number(settings.segment_words.into()),
1515+
);
15021516

15031517
json_obj.insert(
15041518
"settings".to_string(),
@@ -1655,6 +1669,12 @@ pub fn parse_captions_json(json: &str) -> Result<cap_project::CaptionsData, Stri
16551669
.and_then(|v| v.as_bool())
16561670
.unwrap_or(false);
16571671

1672+
let segment_words = settings_obj
1673+
.get("segmentWords")
1674+
.or_else(|| settings_obj.get("segment_words"))
1675+
.and_then(|v| v.as_u64())
1676+
.unwrap_or(6) as u32;
1677+
16581678
let preset = settings_obj
16591679
.get("preset")
16601680
.and_then(|v| v.as_str())
@@ -1707,6 +1727,7 @@ pub fn parse_captions_json(json: &str) -> Result<cap_project::CaptionsData, Stri
17071727
linger_duration,
17081728
word_transition_duration,
17091729
active_word_highlight,
1730+
segment_words,
17101731
manual_position,
17111732
preset,
17121733
animation,

‎apps/desktop/src/routes/editor/CaptionsRegenerateBadge.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ export function CaptionsRegenerateBadge(props: { class?: string }) {
2929
editorInstance.path,
3030
model,
3131
language,
32+
project?.captions?.settings?.segmentWords ?? 6,
3233
);
3334
if (result.segments.length < 1) {
3435
toast.error(

‎apps/desktop/src/routes/editor/CaptionsTab.tsx‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -731,6 +731,7 @@ export function CaptionsTab(props: {
731731
editorInstance.path,
732732
selectedModel(),
733733
selectedLanguage(),
734+
getSetting("segmentWords"),
734735
);
735736

736737
if (result && result.segments.length > 0) {
@@ -1138,6 +1139,18 @@ export function CaptionsTab(props: {
11381139
/>
11391140
</Field>
11401141

1142+
<Field name="Words Per Caption" inline>
1143+
<Slider
1144+
value={[getSetting("segmentWords")]}
1145+
onChange={(v: number[]) =>
1146+
updateCaptionSetting("segmentWords", Math.max(1, v[0]))
1147+
}
1148+
minValue={1}
1149+
maxValue={8}
1150+
step={1}
1151+
/>
1152+
</Field>
1153+
11411154
<Field name="Uppercase" inline>
11421155
<Toggle
11431156
checked={getSetting("uppercase")}

‎apps/desktop/src/routes/editor/captions.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -540,13 +540,20 @@ export async function transcribeEditorCaptions(
540540
videoPath: string,
541541
modelName = DEFAULT_CAPTION_MODEL,
542542
language = DEFAULT_CAPTION_LANGUAGE,
543+
segmentWords = 6,
543544
): Promise<CaptionData> {
544545
const resolvedModelName = resolveCaptionModel(modelName);
545546
const modelPath = await getModelPath(resolvedModelName);
546547
const engine = PARAKEET_DIR_MODELS.has(resolvedModelName)
547548
? "Parakeet"
548549
: "Whisper";
549-
return await commands.transcribeAudio(videoPath, modelPath, language, engine);
550+
return await commands.transcribeAudio(
551+
videoPath,
552+
modelPath,
553+
language,
554+
engine,
555+
segmentWords,
556+
);
550557
}
551558

552559
const CAPTION_ATTACHING_PUNCTUATION = new Set([

‎apps/desktop/src/store/captions.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ export const defaultCaptionSettings: EditorCaptionSettings = {
180180
exportWithSubtitles: false,
181181
lingerDuration: 0.4,
182182
wordTransitionDuration: 0.25,
183+
segmentWords: 6,
183184
manualPosition: null,
184185
preset: classicPreset.id,
185186
...classicPreset.style,

‎apps/desktop/src/utils/tauri.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -489,8 +489,8 @@ async createDir(path: string, recursive: boolean) : Promise<null> {
489489
async saveModelFile(path: string, data: number[]) : Promise<null> {
490490
return await TAURI_INVOKE("save_model_file", { path, data });
491491
},
492-
async transcribeAudio(videoPath: string, modelPath: string, language: string, engine: TranscriptionEngine) : Promise<CaptionData> {
493-
return await TAURI_INVOKE("transcribe_audio", { videoPath, modelPath, language, engine });
492+
async transcribeAudio(videoPath: string, modelPath: string, language: string, engine: TranscriptionEngine, segmentWords: number | null) : Promise<CaptionData> {
493+
return await TAURI_INVOKE("transcribe_audio", { videoPath, modelPath, language, engine, segmentWords });
494494
},
495495
async saveCaptions(videoId: string, captions: CaptionData) : Promise<null> {
496496
return await TAURI_INVOKE("save_captions", { videoId, captions });
@@ -892,7 +892,7 @@ export type CameraXPosition = "left" | "center" | "right"
892892
export type CameraYPosition = "top" | "bottom"
893893
export type CaptionData = { segments: CaptionSegment[]; settings: CaptionSettings | null }
894894
export type CaptionSegment = { id: string; start: number; end: number; text: string; words?: CaptionWord[] }
895-
export type CaptionSettings = { enabled: boolean; font: string; size: number; color: string; backgroundColor: string; backgroundOpacity: number; position: string; italic: boolean; fontWeight: number; outline: boolean; outlineColor: string; exportWithSubtitles: boolean; highlightColor: string; fadeDuration: number; lingerDuration: number; wordTransitionDuration: number; activeWordHighlight: boolean; manualPosition: XY<number> | null; preset: string; animation: string; highlightStyle: string; uppercase: boolean }
895+
export type CaptionSettings = { enabled: boolean; font: string; size: number; color: string; backgroundColor: string; backgroundOpacity: number; position: string; italic: boolean; fontWeight: number; outline: boolean; outlineColor: string; exportWithSubtitles: boolean; highlightColor: string; fadeDuration: number; lingerDuration: number; wordTransitionDuration: number; activeWordHighlight: boolean; segmentWords: number; manualPosition: XY<number> | null; preset: string; animation: string; highlightStyle: string; uppercase: boolean }
896896
export type CaptionTrackSegment = { id: string; start: number; end: number; text: string; words?: CaptionWord[]; fadeDurationOverride?: number | null; lingerDurationOverride?: number | null; positionOverride?: string | null; colorOverride?: string | null; backgroundColorOverride?: string | null; fontSizeOverride?: number | null }
897897
export type CaptionWord = { text: string; start: number; end: number }
898898
export type CaptionsData = { segments: CaptionSegment[]; settings: CaptionSettings;

0 commit comments

Comments
 (0)