super-stt-daemon/src/output/typer.rs
Line | Count | Source |
1 | | // SPDX-License-Identifier: GPL-3.0-only |
2 | | |
3 | | //! The preview typer state machine: session/stabilization state plus the |
4 | | //! keyboard-driving update logic. The pure text-diff algorithms it builds on |
5 | | //! live in [`crate::output::preview`]. |
6 | | |
7 | | use crate::output::keyboard::Simulator; |
8 | | use crate::output::preview::{ |
9 | | find_common_prefix, find_tail_match_in_text, preprocess_text, sanitize_for_typing, |
10 | | }; |
11 | | use log::{debug, info, warn}; |
12 | | |
13 | | /// State for tracking preview updates |
14 | | pub struct State { |
15 | | pub last_transcription: String, |
16 | | pub prev_text: String, |
17 | | /// Complete transcription built from all audio (for final output) |
18 | | pub full_session_text: String, |
19 | | /// When we last saw substantial text growth (to commit to full session) |
20 | | pub last_growth_time: std::time::Instant, |
21 | | /// History of transcriptions for stabilization |
22 | | pub text_storage: Vec<String>, |
23 | | /// Text confirmed by appearing in multiple transcriptions |
24 | | pub stabilized_text: String, |
25 | | } |
26 | | |
27 | | impl Default for State { |
28 | 58 | fn default() -> Self { |
29 | 58 | Self { |
30 | 58 | last_transcription: String::new(), |
31 | 58 | prev_text: String::new(), |
32 | 58 | full_session_text: String::new(), |
33 | 58 | last_growth_time: std::time::Instant::now(), |
34 | 58 | text_storage: Vec::new(), |
35 | 58 | stabilized_text: String::new(), |
36 | 58 | } |
37 | 58 | } |
38 | | } |
39 | | |
40 | | impl State { |
41 | | /// Stabilization and session text update (Phase 1). |
42 | | /// |
43 | | /// Keyboard-free: mutates only session/stabilization state, so it is |
44 | | /// independently testable. |
45 | 4 | fn update_with_stabilization(&mut self, new_preview_text: &str) { |
46 | | // Add current text to storage |
47 | 4 | self.text_storage.push(new_preview_text.to_string()); |
48 | | |
49 | | // Keep only recent texts for stabilization (prevent unbounded growth) |
50 | 4 | if self.text_storage.len() > 10 { |
51 | 0 | self.text_storage.remove(0); |
52 | 4 | } |
53 | | |
54 | | // Find common prefix between last two texts |
55 | 4 | if self.text_storage.len() >= 2 { |
56 | 2 | let last_two = &self.text_storage[self.text_storage.len() - 2..]; |
57 | 2 | let common_prefix = find_common_prefix(&last_two[0], &last_two[1]); |
58 | 2 | let prefix_text = last_two[0].chars().take(common_prefix).collect::<String>(); |
59 | | |
60 | | // Only update stabilized text if we found a longer stable prefix |
61 | 2 | if prefix_text.len() > self.stabilized_text.len() { |
62 | 2 | self.stabilized_text = prefix_text; |
63 | 2 | debug!( |
64 | | "Updated stabilized text: '{}'", |
65 | 0 | self.stabilized_text.chars().take(30).collect::<String>() |
66 | | ); |
67 | 0 | } |
68 | 2 | } |
69 | | |
70 | | // Update full session text using stabilized text + tail matching |
71 | 4 | self.update_full_session_text(new_preview_text); |
72 | 4 | } |
73 | | |
74 | | /// Update the full session text using stabilized text as base. |
75 | 10 | fn update_full_session_text(&mut self, new_preview_text: &str) { |
76 | | // If we have stabilized text, use it as our base |
77 | 10 | if !self.stabilized_text.is_empty() |
78 | 2 | && self.stabilized_text.len() > self.full_session_text.len() |
79 | | { |
80 | 0 | self.full_session_text = self.stabilized_text.clone(); |
81 | 0 | self.last_growth_time = std::time::Instant::now(); |
82 | 0 | debug!( |
83 | | "Updated session from stabilized: '{}'", |
84 | 0 | self.full_session_text.chars().take(30).collect::<String>() |
85 | | ); |
86 | 10 | } |
87 | | |
88 | | // Only grow the session text, never shrink it |
89 | 10 | if self.full_session_text.is_empty() { |
90 | 4 | self.full_session_text = new_preview_text.to_string(); |
91 | 4 | self.last_growth_time = std::time::Instant::now(); |
92 | 4 | debug!( |
93 | | "Started session text: '{}'", |
94 | 0 | self.full_session_text.chars().take(30).collect::<String>() |
95 | | ); |
96 | 4 | return; |
97 | 6 | } |
98 | | |
99 | | // Check if preview text extends our session text |
100 | 6 | if new_preview_text.len() > self.full_session_text.len() |
101 | 4 | && new_preview_text.starts_with(&self.full_session_text) |
102 | | { |
103 | | // Perfect extension - just grow |
104 | 2 | self.full_session_text = new_preview_text.to_string(); |
105 | 2 | self.last_growth_time = std::time::Instant::now(); |
106 | 2 | debug!( |
107 | | "Extended session text to: '{}'", |
108 | 0 | self.full_session_text.chars().take(40).collect::<String>() |
109 | | ); |
110 | 2 | return; |
111 | 4 | } |
112 | | |
113 | | // Use tail matching to extend session with new content |
114 | 4 | if let Some(pos2 ) = find_tail_match_in_text(&self.full_session_text, new_preview_text, 3) { |
115 | 2 | let extended = format!("{}{}", self.full_session_text, &new_preview_text[pos..]); |
116 | 2 | if extended.len() > self.full_session_text.len() { |
117 | 2 | self.full_session_text = extended; |
118 | 2 | self.last_growth_time = std::time::Instant::now(); |
119 | 2 | debug!( |
120 | | "Extended session via tail match: '{}'", |
121 | 0 | self.full_session_text.chars().take(40).collect::<String>() |
122 | | ); |
123 | 0 | } |
124 | 2 | } |
125 | 10 | } |
126 | | |
127 | | /// Build the display text (Phase 2) - what actually shows on screen. |
128 | 8 | fn build_display_text(&self, preview_text: &str) -> String { |
129 | | // Use stabilized text as base, but be smart about it |
130 | | |
131 | | // If no stabilized text yet, show the preview |
132 | 8 | if self.stabilized_text.is_empty() { |
133 | 2 | return preview_text.to_string(); |
134 | 6 | } |
135 | | |
136 | | // Try tail matching first |
137 | 6 | if let Some(pos2 ) = find_tail_match_in_text(&self.stabilized_text, preview_text, 3) { |
138 | | // Found overlap - combine stabilized text with new part from preview |
139 | 2 | return format!("{}{}", self.stabilized_text, &preview_text[pos..]); |
140 | 4 | } |
141 | | |
142 | | // No tail match found - be conservative to avoid text loss |
143 | | // Prefer the longer text (session text or preview) to avoid disappearing words |
144 | 4 | let best_text = if self.full_session_text.len() >= preview_text.len() { |
145 | 2 | &self.full_session_text |
146 | | } else { |
147 | 2 | preview_text |
148 | | }; |
149 | | |
150 | 4 | best_text.to_string() |
151 | 8 | } |
152 | | } |
153 | | |
154 | | /// How long [`Typer::type_notice`] waits before typing, so the user's shortcut |
155 | | /// keys can be fully released first. |
156 | | /// |
157 | | /// A failure notice can be emitted within milliseconds of a keypress, at which |
158 | | /// point the shortcut's modifier keys (Ctrl/Alt/Super/Shift) are usually still |
159 | | /// physically held down. Simulated keystrokes sent during that window arrive |
160 | | /// *modified*, so the focused application interprets the notice as shortcuts |
161 | | /// rather than text. |
162 | | /// |
163 | | /// Two distinct cases hit this window, which is why the wait applies to every |
164 | | /// notice rather than just the first: |
165 | | /// |
166 | | /// - The no-model preflight rejects the request before capture even starts, so |
167 | | /// the notice follows the *start* keypress almost immediately. |
168 | | /// - In manual stop mode the user presses the hotkey to end the recording, and |
169 | | /// a failure that surfaces quickly after that — a model unloaded mid-cycle, |
170 | | /// say — puts the notice right behind the *stop* keypress. |
171 | | /// |
172 | | /// This does not apply to transcription output: that is typed after speech has |
173 | | /// been captured and transcribed, long past any plausible key-release window. |
174 | | const NOTICE_KEY_RELEASE_DELAY: std::time::Duration = std::time::Duration::from_secs(1); |
175 | | |
176 | | /// Unified, simplified preview typer that combines the best of both approaches |
177 | | pub struct Typer { |
178 | | keyboard_simulator: Simulator, |
179 | | state: State, |
180 | | } |
181 | | |
182 | | impl Typer { |
183 | | #[must_use] |
184 | 42 | pub fn new(keyboard_simulator: Simulator) -> Self { |
185 | 42 | Self { |
186 | 42 | keyboard_simulator, |
187 | 42 | state: State::default(), |
188 | 42 | } |
189 | 42 | } |
190 | | |
191 | | #[must_use] |
192 | 0 | pub fn write_method_name(&self) -> &'static str { |
193 | 0 | self.keyboard_simulator.name() |
194 | 0 | } |
195 | | |
196 | | /// Extract the simulator so it can be cached for reuse. |
197 | | #[must_use] |
198 | 0 | pub fn take_simulator(self) -> Simulator { |
199 | 0 | self.keyboard_simulator |
200 | 0 | } |
201 | | |
202 | | /// Apply a simple differential update by backspacing to the first differing |
203 | | /// character and retyping the rest. Returns the **net change in screen |
204 | | /// characters** (chars typed minus chars deleted) so callers accounting in |
205 | | /// chars stay consistent — mixing this with a byte length would drift on any |
206 | | /// multibyte text. |
207 | 0 | pub async fn apply_simple_diff(&mut self, old_text: &str, new_text: &str) -> isize { |
208 | | // Safety checks |
209 | 0 | if old_text == new_text { |
210 | 0 | return 0; |
211 | 0 | } |
212 | | |
213 | 0 | if old_text.is_empty() && !new_text.is_empty() { |
214 | 0 | if let Err(e) = self.keyboard_simulator.type_text(new_text).await { |
215 | 0 | debug!("Failed to type new text: {e}"); |
216 | 0 | } |
217 | 0 | return isize::try_from(new_text.chars().count()).unwrap_or(isize::MAX); |
218 | 0 | } |
219 | | |
220 | 0 | if new_text.is_empty() { |
221 | | // Skip |
222 | 0 | return 0; |
223 | 0 | } |
224 | | |
225 | 0 | let old_chars: Vec<char> = old_text.chars().collect(); |
226 | 0 | let new_chars: Vec<char> = new_text.chars().collect(); |
227 | | |
228 | | // Find first different character position |
229 | 0 | let common_prefix = find_common_prefix(old_text, new_text); |
230 | | |
231 | | // Calculate what to delete and what to type |
232 | 0 | let chars_to_delete = old_chars.len() - common_prefix; |
233 | 0 | let text_to_type: String = new_chars[common_prefix..].iter().collect(); |
234 | 0 | let chars_to_type = new_chars.len() - common_prefix; |
235 | | |
236 | 0 | debug!( |
237 | | "Simple diff: prefix={}, delete={}, type='{}'", |
238 | | common_prefix, |
239 | | chars_to_delete, |
240 | 0 | text_to_type.chars().take(20).collect::<String>() |
241 | | ); |
242 | | |
243 | | // Backspace to the first different position |
244 | 0 | let _ = self.keyboard_simulator.backspace_n(chars_to_delete).await; |
245 | | |
246 | | // Type the new part |
247 | 0 | let _ = self.keyboard_simulator.type_text(&text_to_type).await; |
248 | | |
249 | | // Net screen delta in chars: what we added minus what we removed. |
250 | 0 | isize::try_from(chars_to_type).unwrap_or(isize::MAX) |
251 | 0 | - isize::try_from(chars_to_delete).unwrap_or(isize::MAX) |
252 | 0 | } |
253 | | |
254 | | /// Update preview text using two-phase approach |
255 | 0 | pub async fn update_preview(&mut self, new_text: &str, actually_typed: &mut String) { |
256 | 0 | let processed_text = preprocess_text(new_text, true); |
257 | | |
258 | 0 | info!( |
259 | | "Preview update: new='{}', prev='{}', typed='{}'", |
260 | 0 | processed_text.chars().take(30).collect::<String>(), |
261 | 0 | self.state.prev_text.chars().take(30).collect::<String>(), |
262 | 0 | actually_typed.chars().take(30).collect::<String>() |
263 | | ); |
264 | | |
265 | | // Skip if text hasn't changed |
266 | 0 | if processed_text == self.state.prev_text { |
267 | 0 | debug!("Text unchanged, skipping"); |
268 | 0 | return; |
269 | 0 | } |
270 | | |
271 | | // Skip empty text |
272 | 0 | if processed_text.is_empty() { |
273 | 0 | debug!("Empty text, skipping"); |
274 | 0 | return; |
275 | 0 | } |
276 | | |
277 | | // PHASE 1: Stabilization and session text update |
278 | 0 | self.state.update_with_stabilization(&processed_text); |
279 | | |
280 | | // PHASE 2: Decide what to show on screen |
281 | 0 | let display_text = self.state.build_display_text(&processed_text); |
282 | | |
283 | 0 | info!( |
284 | | "Display logic: display='{}', session='{}', stabilized='{}'", |
285 | 0 | display_text.chars().take(30).collect::<String>(), |
286 | 0 | self.state |
287 | 0 | .full_session_text |
288 | 0 | .chars() |
289 | 0 | .take(30) |
290 | 0 | .collect::<String>(), |
291 | 0 | self.state |
292 | 0 | .stabilized_text |
293 | 0 | .chars() |
294 | 0 | .take(30) |
295 | 0 | .collect::<String>() |
296 | | ); |
297 | | |
298 | | // Apply the update to screen |
299 | 0 | self.apply_text_update(&display_text, actually_typed).await; |
300 | 0 | self.state.prev_text = processed_text; |
301 | 0 | } |
302 | | |
303 | | /// Process final text (completed sentence) - Uses full session audio |
304 | 8 | pub async fn process_final_text(&mut self, transcription_result: &str) { |
305 | | // No preview typing, type directly |
306 | 8 | let processed_text = preprocess_text(transcription_result, false); |
307 | | |
308 | | // An empty transcript has nothing to type. Without this guard the |
309 | | // `format!("{processed_text} ")` below deposits a bare space into the |
310 | | // user's focused window every time a recording produces no text. |
311 | 8 | if processed_text.trim().is_empty() { |
312 | 6 | info!("Final transcription is empty; typing nothing"); |
313 | 6 | self.reset_after_recording(processed_text); |
314 | 6 | return; |
315 | 2 | } |
316 | | |
317 | 2 | let final_text = format!("{processed_text} "); |
318 | 2 | if let Err(e0 ) = self.keyboard_simulator.type_text(&final_text).await { |
319 | 0 | warn!("Failed to type final transcription: {e}"); |
320 | | } else { |
321 | 2 | info!("Step 6 complete: Final transcription typed directly"); |
322 | | } |
323 | | |
324 | 2 | self.reset_after_recording(processed_text); |
325 | 8 | } |
326 | | |
327 | | /// Clear the per-recording transcript state so the next recording starts |
328 | | /// clean. Preview tail-matching reads `full_session_text` and `prev_text`, |
329 | | /// so anything left here would be treated as a prefix to extend. |
330 | | /// |
331 | | /// Split out of [`Self::process_final_text`] because the no-speech path |
332 | | /// finishes a recording without typing anything and still has to reset. |
333 | 10 | pub fn reset_after_recording(&mut self, last_transcription: String) { |
334 | 10 | self.state.prev_text.clear(); |
335 | 10 | self.state.last_transcription = last_transcription; |
336 | 10 | self.state.last_growth_time = std::time::Instant::now(); |
337 | | |
338 | 10 | info!( |
339 | | "Completed sentence. Session text: '{}'", |
340 | 0 | self.state |
341 | 0 | .full_session_text |
342 | 0 | .chars() |
343 | 0 | .take(50) |
344 | 0 | .collect::<String>() |
345 | | ); |
346 | | |
347 | | // Clear session for next recording |
348 | 10 | self.state.full_session_text.clear(); |
349 | 10 | } |
350 | | |
351 | | /// Type a fixed daemon-authored notice into the focused window. |
352 | | /// |
353 | | /// Deliberately **not** `process_final_text`. That method mutates |
354 | | /// transcript state (`last_transcription`, `prev_text`, `full_session_text`) |
355 | | /// which feeds preview tail-matching on the next recording, and applies |
356 | | /// transcript semantics — capitalization, a trailing period, a trailing |
357 | | /// space — that a fixed marker must not inherit. A notice is typed verbatim |
358 | | /// and leaves session state alone. |
359 | | /// |
360 | | /// Routed through the same [`sanitize_for_typing`] choke point as every |
361 | | /// other write path (audit 2 Tier 3 #8). The callers pass constants, so |
362 | | /// this is a no-op today; it is here so the property holds by |
363 | | /// construction. |
364 | | /// |
365 | | /// Waits [`NOTICE_KEY_RELEASE_DELAY`] before typing — see that constant for |
366 | | /// why. |
367 | 14 | pub async fn type_notice(&mut self, notice: &str) { |
368 | | // Let the user's hotkey come back up first. The no-model preflight |
369 | | // rejects before capture starts, so this can run within milliseconds of |
370 | | // the press, while the shortcut's modifiers are still physically held. |
371 | | // Typing then delivers *modified* keystrokes to the focused window — |
372 | | // the notice would fire shortcuts in the user's application instead of |
373 | | // inserting text. |
374 | 14 | tokio::time::sleep(NOTICE_KEY_RELEASE_DELAY).await; |
375 | | |
376 | 14 | let sanitized = sanitize_for_typing(notice); |
377 | 14 | if let Err(e0 ) = self.keyboard_simulator.type_text(&sanitized).await { |
378 | 0 | warn!("Failed to type notice: {e}"); |
379 | | } else { |
380 | 14 | info!("Typed failure notice: {sanitized}"); |
381 | | } |
382 | 14 | } |
383 | | |
384 | | /// Apply text update to screen (common logic) |
385 | 0 | async fn apply_text_update(&mut self, new_text: &str, actually_typed: &mut String) { |
386 | 0 | info!( |
387 | | "Typing logic: old_typed='{}', new_display='{}'", |
388 | 0 | actually_typed.chars().take(30).collect::<String>(), |
389 | 0 | new_text.chars().take(30).collect::<String>(), |
390 | | ); |
391 | | |
392 | 0 | if actually_typed.is_empty() { |
393 | | // Screen is empty — type the whole thing. |
394 | 0 | info!( |
395 | | "Screen empty, typing new text: '{}'", |
396 | 0 | new_text.chars().take(30).collect::<String>() |
397 | | ); |
398 | 0 | let _ = self |
399 | 0 | .keyboard_simulator |
400 | 0 | .type_text(&format!("{new_text} ")) |
401 | 0 | .await; |
402 | 0 | } else if new_text.starts_with(actually_typed.as_str()) |
403 | 0 | && new_text.len() > actually_typed.len() |
404 | | { |
405 | | // Perfect extension — append only the new suffix. |
406 | 0 | let suffix = &new_text[actually_typed.len()..]; |
407 | 0 | info!("Perfect extension, adding suffix: '{suffix}'"); |
408 | 0 | let _ = self |
409 | 0 | .keyboard_simulator |
410 | 0 | .type_text(&format!("{suffix} ")) |
411 | 0 | .await; |
412 | | } else { |
413 | | // Replacement — backspace to the first difference and retype. |
414 | 0 | let net_change = self.apply_simple_diff(actually_typed, new_text).await; |
415 | 0 | info!("Diff replacement: net {net_change} char(s)"); |
416 | | } |
417 | | |
418 | | // `actually_typed` mirrors what we drove onto the screen so |
419 | | // `clear_preview` backspaces the right count next time. Every branch |
420 | | // above leaves the screen showing `new_text`. The keyboard results are |
421 | | // best-effort and unchecked, so there is no measured count to reconcile |
422 | | // against — the old byte-vs-char reconciliation was both wrong (it added |
423 | | // `apply_simple_diff`'s byte length to a char count) and dead (both of |
424 | | // its branches did exactly this assignment). |
425 | 0 | actually_typed.clear(); |
426 | 0 | actually_typed.push_str(new_text); |
427 | 0 | } |
428 | | |
429 | | /// Clear all typed text and reset state |
430 | 0 | pub async fn clear_preview(&mut self, actually_typed: &mut String) { |
431 | 0 | info!("clear_preview called with actually_typed: '{actually_typed}'"); |
432 | | |
433 | 0 | if actually_typed.is_empty() { |
434 | 0 | info!("actually_typed is empty, nothing to clear"); |
435 | 0 | return; |
436 | 0 | } |
437 | | |
438 | 0 | let chars_to_delete = actually_typed.chars().count(); |
439 | 0 | info!("Backspacing {chars_to_delete} characters"); |
440 | | |
441 | 0 | if let Err(e) = self.keyboard_simulator.backspace_n(chars_to_delete).await { |
442 | 0 | warn!("Failed to backspace preview text: {e}"); |
443 | | } else { |
444 | 0 | info!("Successfully backspaced {chars_to_delete} characters"); |
445 | | } |
446 | | |
447 | 0 | actually_typed.clear(); |
448 | | |
449 | | // Also clear state when explicitly clearing preview |
450 | 0 | self.state.prev_text.clear(); |
451 | 0 | self.state.last_transcription.clear(); |
452 | 0 | self.state.full_session_text.clear(); |
453 | 0 | self.state.last_growth_time = std::time::Instant::now(); |
454 | | |
455 | 0 | info!("Cleared all {chars_to_delete} characters and reset state"); |
456 | 0 | } |
457 | | } |
458 | | |
459 | | #[cfg(test)] |
460 | | #[path = "typer_tests.rs"] |
461 | | mod tests; |