super-stt-daemon/src/output/keyboard/mod.rs
Line | Count | Source |
1 | | // SPDX-License-Identifier: GPL-3.0-only |
2 | | |
3 | | mod enigo_backend; |
4 | | #[cfg(target_os = "linux")] |
5 | | mod xdg_portal_backend; |
6 | | #[cfg(target_os = "linux")] |
7 | | mod ydotool_backend; |
8 | | |
9 | | use anyhow::Result; |
10 | | use log::debug; |
11 | | #[cfg(target_os = "linux")] |
12 | | use log::warn; |
13 | | use super_stt_shared::models::write_method::WriteMethod; |
14 | | |
15 | | use enigo_backend::EnigoBackend; |
16 | | #[cfg(target_os = "linux")] |
17 | | use xdg_portal_backend::XdgPortalBackend; |
18 | | #[cfg(target_os = "linux")] |
19 | | use ydotool_backend::YdotoolBackend; |
20 | | |
21 | | /// Keyboard simulation backend. |
22 | | /// |
23 | | /// # Safety |
24 | | /// |
25 | | /// `Simulator` is `Send + Sync` because it is only ever accessed by one |
26 | | /// recording session at a time (guarded by `busy`). The `!Send` |
27 | | /// inner type (`Enigo` with raw xkbcommon pointers) is never used |
28 | | /// concurrently. |
29 | | pub enum Simulator { |
30 | | /// [`WriteMethod::BuiltIn`] — enigo, which drives |
31 | | /// `zwp_virtual_keyboard_manager_v1` on Linux and CoreGraphics on macOS. |
32 | | /// The only backend that exists on both. |
33 | | BuiltIn(Box<EnigoBackend>), |
34 | | #[cfg(target_os = "linux")] |
35 | | Ydotool(YdotoolBackend), |
36 | | #[cfg(target_os = "linux")] |
37 | | XdgPortal(XdgPortalBackend), |
38 | | /// Test-only backend that records what *would* have been typed. Every |
39 | | /// real backend needs something a test host does not have — a live |
40 | | /// compositor, a portal, or the macOS Accessibility grant — so without |
41 | | /// this the typing path cannot be asserted on at all. |
42 | | #[cfg(test)] |
43 | | Capture(std::sync::Arc<std::sync::Mutex<String>>), |
44 | | } |
45 | | |
46 | | // SAFETY: see Simulator doc comment — single-writer access enforced by daemon. |
47 | | unsafe impl Send for Simulator {} |
48 | | unsafe impl Sync for Simulator {} |
49 | | |
50 | | impl Simulator { |
51 | | /// Create a simulator for the requested write method. |
52 | | /// |
53 | | /// # Errors |
54 | | /// Returns an error when a *specific* method is requested and fails, or — |
55 | | /// for `Auto` — when every backend in the chain is unavailable. |
56 | 1 | pub async fn new(method: WriteMethod) -> Result<Self> { |
57 | 1 | let sim0 = match method { |
58 | 1 | WriteMethod::Auto => Self::auto().await?, |
59 | | #[cfg(target_os = "linux")] |
60 | | WriteMethod::XdgDesktopPortal => { |
61 | 0 | let backend = XdgPortalBackend::new().await?; |
62 | 0 | Self::XdgPortal(backend) |
63 | | } |
64 | | #[cfg(target_os = "linux")] |
65 | | WriteMethod::Ydotool => { |
66 | 0 | anyhow::ensure!(YdotoolBackend::is_available(), "ydotool is not available"); |
67 | 0 | Self::Ydotool(YdotoolBackend::new()) |
68 | | } |
69 | 0 | WriteMethod::BuiltIn => Self::BuiltIn(Box::new(EnigoBackend::new()?)), |
70 | | // Asking for a Linux session technology on macOS is not a runtime |
71 | | // failure to retry but a request that can never be satisfied, so |
72 | | // it is named as such rather than reported as "unavailable". |
73 | | #[cfg(not(target_os = "linux"))] |
74 | | other => anyhow::bail!( |
75 | | "write method `{other}` is Linux-only and cannot be used on this platform" |
76 | | ), |
77 | | }; |
78 | 0 | Ok(sim) |
79 | 1 | } |
80 | | |
81 | | /// Auto-detect: built-in → XDG Portal → ydotool. |
82 | | /// |
83 | | /// The built-in backend leads because it needs nothing installed beyond a |
84 | | /// compositor exposing `zwp_virtual_keyboard_manager_v1`, types the user's |
85 | | /// actual layout, and costs no D-Bus round-trips or authorization prompt. |
86 | | /// The portal follows for sessions that withhold the virtual-keyboard |
87 | | /// global, and ydotool last since it needs a running `ydotoold` and types a |
88 | | /// hardcoded US-QWERTY map. |
89 | | /// |
90 | | /// # Errors |
91 | | /// Only when every backend is unavailable. The message carries each rung's |
92 | | /// reason: a failed recording is the daemon's sole chance to explain why |
93 | | /// nothing can type. |
94 | | #[cfg(target_os = "linux")] |
95 | 1 | async fn auto() -> Result<Self> { |
96 | 1 | debug!("Auto-detecting write method..."); |
97 | 1 | let mut unavailable = Vec::new(); |
98 | | |
99 | 1 | match EnigoBackend::new() { |
100 | 0 | Ok(backend) => return Ok(Self::BuiltIn(Box::new(backend))), |
101 | 1 | Err(e) => { |
102 | 1 | debug!("Built-in write method unavailable: {e}"); |
103 | 1 | unavailable.push(format!("built-in ({e})")); |
104 | | } |
105 | | } |
106 | | |
107 | 1 | let portal_available = XdgPortalBackend::is_available().await; |
108 | 1 | debug!("XDG Desktop Portal available: {portal_available}"); |
109 | 1 | if portal_available { |
110 | 0 | match XdgPortalBackend::new().await { |
111 | 0 | Ok(backend) => return Ok(Self::XdgPortal(backend)), |
112 | 0 | Err(e) => { |
113 | 0 | warn!("XDG Portal available but session failed: {e}"); |
114 | 0 | unavailable.push(format!("XDG Desktop Portal ({e})")); |
115 | | } |
116 | | } |
117 | 1 | } else { |
118 | 1 | unavailable |
119 | 1 | .push("XDG Desktop Portal (RemoteDesktop interface not on the session bus)".into()); |
120 | 1 | } |
121 | | |
122 | 1 | let ydotool_available = YdotoolBackend::is_available(); |
123 | 1 | debug!("ydotool available: {ydotool_available}"); |
124 | 1 | if ydotool_available { |
125 | 0 | return Ok(Self::Ydotool(YdotoolBackend::new())); |
126 | 1 | } |
127 | 1 | unavailable.push("ydotool (not installed or ydotoold not running)".into()); |
128 | | |
129 | 1 | anyhow::bail!( |
130 | | "no write method available — tried {}", |
131 | 1 | unavailable.join(", ") |
132 | | ) |
133 | 1 | } |
134 | | |
135 | | /// Auto-detect on macOS, where the chain has one rung. |
136 | | /// |
137 | | /// Not a degenerate case of the Linux chain. The two fallbacks there |
138 | | /// exist because a Wayland compositor may withhold the virtual-keyboard |
139 | | /// global, leaving a working session with no way to type; CoreGraphics |
140 | | /// event posting has no such variation between machines. What gates it |
141 | | /// instead is a permission — the daemon must hold Accessibility — and a |
142 | | /// second backend would not help, because every way of synthesizing a |
143 | | /// key event on macOS is behind the same grant. |
144 | | /// |
145 | | /// So there is nothing to fall back *to*, and the failure is reported |
146 | | /// rather than walked around. enigo checks the grant itself |
147 | | /// (`AXIsProcessTrusted`) and reports it distinctly, so the message a |
148 | | /// user sees already names the checkbox — see |
149 | | /// `enigo_backend::describe_connection_failure`. |
150 | | /// |
151 | | /// # Errors |
152 | | /// When enigo cannot open a keyboard connection: no Accessibility grant, |
153 | | /// or no window server to talk to at all (an ssh session, or a |
154 | | /// `LaunchDaemon` in the system context rather than a `LaunchAgent` in the |
155 | | /// user's). |
156 | | // `allow`, not `expect`: clippy 1.98 files these under `unused_async_trait_impl` |
157 | | // and later releases under `unused_async`, so either `expect` goes unfulfilled |
158 | | // on some toolchain. |
159 | | #[cfg(target_os = "macos")] |
160 | | #[allow( |
161 | | clippy::unused_async, |
162 | | clippy::unused_async_trait_impl, |
163 | | reason = "shares a signature with the Linux arm, which awaits the portal probe" |
164 | | )] |
165 | | async fn auto() -> Result<Self> { |
166 | | debug!("Auto-detecting write method..."); |
167 | | match EnigoBackend::new() { |
168 | | Ok(backend) => Ok(Self::BuiltIn(Box::new(backend))), |
169 | | // Not prefixed with "no write method available — tried …" the way |
170 | | // the Linux arm is: there is one method here, and its own error |
171 | | // already says what to do about it. |
172 | | Err(e) => Err(e), |
173 | | } |
174 | | } |
175 | | |
176 | | /// Whether this backend may be held across recordings. |
177 | | /// |
178 | | /// Everything except the built-in backend is cached. Rebuilding the |
179 | | /// portal session costs |
180 | | /// three D-Bus round-trips before capture can start and may prompt the |
181 | | /// user for authorization each time, so paying it per recording is not an |
182 | | /// option. enigo is the exception: Wayland compositors recycle idle |
183 | | /// connections, leaving a stale `Con` that fails silently on the next |
184 | | /// recording, and recreating it is cheap. |
185 | | #[must_use] |
186 | 2 | pub fn is_cacheable(&self) -> bool { |
187 | 2 | !matches!(self, Self::BuiltIn(_)) |
188 | 2 | } |
189 | | |
190 | | /// The concrete method this simulator drives. |
191 | | /// |
192 | | /// Never `Auto` in a shipped build: `auto()` resolves the chain at |
193 | | /// construction, and this is the only way a client can learn which rung it |
194 | | /// landed on (`POST /write_method/test`). The test-only capture backend |
195 | | /// types through no real method and so reports the unresolved `Auto`. |
196 | | #[must_use] |
197 | 1 | pub fn resolved_method(&self) -> WriteMethod { |
198 | 1 | match self { |
199 | | #[cfg(target_os = "linux")] |
200 | 0 | Self::XdgPortal(_) => WriteMethod::XdgDesktopPortal, |
201 | | #[cfg(target_os = "linux")] |
202 | 0 | Self::Ydotool(_) => WriteMethod::Ydotool, |
203 | 0 | Self::BuiltIn(_) => WriteMethod::BuiltIn, |
204 | | #[cfg(test)] |
205 | 1 | Self::Capture(_) => WriteMethod::Auto, |
206 | | } |
207 | 1 | } |
208 | | |
209 | | /// Human-readable name for logging. |
210 | | #[must_use] |
211 | 1 | pub fn name(&self) -> &'static str { |
212 | 1 | match self { |
213 | | #[cfg(target_os = "linux")] |
214 | 0 | Self::XdgPortal(_) => "XDG Desktop Portal", |
215 | | #[cfg(target_os = "linux")] |
216 | 0 | Self::Ydotool(_) => "ydotool", |
217 | 0 | Self::BuiltIn(_) => "built-in", |
218 | | #[cfg(test)] |
219 | 1 | Self::Capture(_) => "capture (test)", |
220 | | } |
221 | 1 | } |
222 | | |
223 | | /// Type text using the active backend. Async so the portal backend awaits |
224 | | /// its D-Bus calls directly and the blocking backends yield the worker |
225 | | /// (audit Tier 3 #35). |
226 | | /// |
227 | | /// # Errors |
228 | | /// Returns an error if the backend fails to simulate key input. |
229 | | /// |
230 | | /// # Panics |
231 | | /// The test-only capture backend panics if its buffer mutex is poisoned. |
232 | | // `allow`, not `expect`: clippy 1.98 files these under `unused_async_trait_impl` |
233 | | // and later releases under `unused_async`, so either `expect` goes unfulfilled |
234 | | // on some toolchain. |
235 | | #[cfg_attr( |
236 | | target_os = "macos", |
237 | | allow( |
238 | | clippy::unused_async, |
239 | | clippy::unused_async_trait_impl, |
240 | | reason = "the awaiting arm is the XDG portal backend, which is Linux-only" |
241 | | ) |
242 | | )] |
243 | 37 | pub async fn type_text(&mut self, text: &str) -> Result<()> { |
244 | 37 | match self { |
245 | | // The enigo/ydotool backends are synchronous and `!Send`; run them |
246 | | // under `block_in_place` so their handle never crosses an await and |
247 | | // the runtime spins up a replacement worker rather than stalling. The |
248 | | // portal backend is genuinely async — await it directly. |
249 | 0 | Self::BuiltIn(b) => tokio::task::block_in_place(|| b.type_text(text)), |
250 | | #[cfg(target_os = "linux")] |
251 | 0 | Self::Ydotool(_) => tokio::task::block_in_place(|| YdotoolBackend::type_text(text)), |
252 | | #[cfg(target_os = "linux")] |
253 | 0 | Self::XdgPortal(b) => b.type_text(text).await, |
254 | | #[cfg(test)] |
255 | 37 | Self::Capture(buf) => { |
256 | 37 | buf.lock().expect("capture buffer poisoned").push_str(text); |
257 | 37 | Ok(()) |
258 | | } |
259 | | } |
260 | 37 | } |
261 | | |
262 | | /// Backspace N characters using the active backend. |
263 | | /// |
264 | | /// # Errors |
265 | | /// Returns an error if the backend fails to simulate key input. |
266 | | /// |
267 | | /// # Panics |
268 | | /// The test-only capture backend panics if its buffer mutex is poisoned. |
269 | | // `allow`, not `expect`: clippy 1.98 files these under `unused_async_trait_impl` |
270 | | // and later releases under `unused_async`, so either `expect` goes unfulfilled |
271 | | // on some toolchain. |
272 | | #[cfg_attr( |
273 | | target_os = "macos", |
274 | | allow( |
275 | | clippy::unused_async, |
276 | | clippy::unused_async_trait_impl, |
277 | | reason = "the awaiting arm is the XDG portal backend, which is Linux-only" |
278 | | ) |
279 | | )] |
280 | 8 | pub async fn backspace_n(&mut self, n: usize) -> Result<()> { |
281 | 8 | match self { |
282 | 0 | Self::BuiltIn(b) => tokio::task::block_in_place(|| b.backspace_n(n)), |
283 | | #[cfg(target_os = "linux")] |
284 | 0 | Self::Ydotool(_) => tokio::task::block_in_place(|| YdotoolBackend::backspace_n(n)), |
285 | | #[cfg(target_os = "linux")] |
286 | 0 | Self::XdgPortal(b) => b.backspace_n(n).await, |
287 | | #[cfg(test)] |
288 | 8 | Self::Capture(buf) => { |
289 | 8 | let mut guard = buf.lock().expect("capture buffer poisoned"); |
290 | | // Truncate by chars, not bytes — a real backspace removes one |
291 | | // grapheme, and truncating mid-UTF-8 would panic. |
292 | 8 | let keep = guard.chars().count().saturating_sub(n); |
293 | 8 | *guard = guard.chars().take(keep).collect(); |
294 | 8 | Ok(()) |
295 | | } |
296 | | } |
297 | 8 | } |
298 | | } |
299 | | |
300 | | #[cfg(test)] |
301 | | impl Simulator { |
302 | | /// A simulator that accumulates typed text instead of driving a keyboard. |
303 | | /// Returns the simulator and a handle to the accumulated text. |
304 | 37 | pub(crate) fn capture() -> (Self, std::sync::Arc<std::sync::Mutex<String>>) { |
305 | 37 | let buf = std::sync::Arc::new(std::sync::Mutex::new(String::new())); |
306 | 37 | (Self::Capture(std::sync::Arc::clone(&buf)), buf) |
307 | 37 | } |
308 | | } |
309 | | |
310 | | #[cfg(test)] |
311 | | mod tests { |
312 | | use super::Simulator; |
313 | | |
314 | | /// The capture backend has to behave like a real one — text accumulates and |
315 | | /// backspace removes trailing *characters* (not bytes) — or tests written |
316 | | /// against it will not reflect what lands in a user's window. |
317 | | #[tokio::test] |
318 | 1 | async fn capture_backend_accumulates_text_and_honors_backspace() { |
319 | 1 | let (mut sim, buf) = Simulator::capture(); |
320 | | |
321 | 1 | sim.type_text("hello").await.expect("type"); |
322 | 1 | sim.type_text(" wörld").await.expect("type"); |
323 | 1 | assert_eq!(*buf.lock().unwrap(), "hello wörld"); |
324 | | |
325 | | // Multi-byte char must be removed whole. |
326 | 1 | sim.backspace_n(4).await.expect("backspace"); |
327 | 1 | assert_eq!(*buf.lock().unwrap(), "hello w"); |
328 | | |
329 | 1 | assert_eq!(sim.name(), "capture (test)"); |
330 | 1 | } |
331 | | |
332 | | /// Caching is the default; only enigo opts out. A regression that inverts |
333 | | /// this rebuilds the portal session before every recording, costing three |
334 | | /// D-Bus round-trips and possibly an authorization prompt. enigo itself |
335 | | /// needs a live compositor to construct, so this pins the side of the rule |
336 | | /// that is reachable in a test. |
337 | | #[test] |
338 | 1 | fn backends_are_cached_by_default() { |
339 | 1 | let (sim, _buf) = Simulator::capture(); |
340 | 1 | assert!(sim.is_cacheable()); |
341 | 1 | } |
342 | | } |
343 | | |
344 | | /// Ask macOS, once at daemon startup, whether the daemon may type — and let |
345 | | /// the system's Accessibility prompt appear if it may not. |
346 | | /// |
347 | | /// Without Accessibility the daemon transcribes perfectly and types nothing, |
348 | | /// which is the worst shape a failure can take: everything looks like it |
349 | | /// worked. The grant cannot be requested programmatically — only the user can |
350 | | /// give it, in System Settings — so the most the daemon can do is ask at a |
351 | | /// moment when asking is not disruptive, and say clearly what is wrong when |
352 | | /// the answer is no. |
353 | | /// |
354 | | /// Startup is that moment. The alternative, prompting from the recording |
355 | | /// path, puts a modal dialog on screen in the instant between the user |
356 | | /// finishing a sentence and expecting it to appear, stealing focus from the |
357 | | /// window they were dictating into. So this is the one place that prompts, |
358 | | /// and [`EnigoBackend::new`] never does. |
359 | | /// |
360 | | /// Advisory only: a `false` here does not stop the daemon. Transcription over |
361 | | /// the HTTP protocol works without ever typing anything, and a client that |
362 | | /// only reads transcripts has no use for the grant. |
363 | | #[cfg(target_os = "macos")] |
364 | | pub fn probe_accessibility_permission() { |
365 | | match EnigoBackend::probe_accessibility_permission() { |
366 | | Ok(()) => debug!("Accessibility permission granted; write mode can type"), |
367 | | Err(e) => log::warn!( |
368 | | "Write mode will not be able to type: {e} \ |
369 | | (transcription itself is unaffected)" |
370 | | ), |
371 | | } |
372 | | } |