Repository navigation
Expand file tree
/
Copy pathuia.rs
More file actions
3408 lines (3175 loc) · 136 KB
/
Copy pathuia.rs
File metadata and controls
3408 lines (3175 loc) · 136 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//! Windows UI Automation accessibility provider.
use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use windows::core::{implement, BOOL};
use windows::Win32::Foundation::*;
use windows::Win32::System::Com::{CoInitializeEx, COINIT};
use windows::Win32::System::Variant::VARIANT;
use windows::Win32::UI::Accessibility::*;
use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, STATE_SYSTEM_SELECTED};
use xa11y_core::{
selector::{matches_simple, Combinator, Selector, SelectorSegment},
CancelHandle, ElementData, Error, Event, EventKind, EventReceiver, Provider, Rect, Result,
Role, StateFlag, StateSet, Subscription, Toggled,
};
static NEXT_HANDLE: AtomicU64 = AtomicU64::new(1);
/// `EVENT_E_ALL_SUBSCRIBERS_FAILED` (0x80040201) — returned by UIA when an
/// action fires a notification and all registered event subscribers fail to
/// handle it. This means the UI action itself completed; only the notification
/// layer had a transient failure. Certain providers (notably Qt's UIA backend
/// for QTabBar items) propagate this error back through action methods like
/// `Invoke()` and `Select()`, which is incorrect — callers should treat it as
/// success. See: https://github.com/xa11y/xa11y/issues/169
const EVENT_E_ALL_SUBSCRIBERS_FAILED: windows::core::HRESULT =
windows::core::HRESULT(0x80040201u32 as i32);
fn is_event_subscriber_failure(e: &windows::core::Error) -> bool {
e.code() == EVENT_E_ALL_SUBSCRIBERS_FAILED
}
/// Initialize COM for UIA. Called once per WindowsProvider creation.
/// Does not uninitialize on drop — COM lifetime is managed by the process.
fn ensure_com_initialized() -> windows::core::Result<()> {
// Use MTA (0x0) — same mode as the Rust runtime default.
// STA (0x2) would conflict with Rust's thread pool.
let hr = unsafe { CoInitializeEx(None, COINIT(0x0)) };
// S_OK, S_FALSE (already initialized), or RPC_E_CHANGED_MODE are all fine
if hr.is_err() && hr.0 as u32 != 0x80010106 {
hr.ok()?;
}
Ok(())
}
/// Windows accessibility provider using UI Automation.
pub struct WindowsProvider {
automation: IUIAutomation,
/// Describes which properties and patterns to pre-fetch in bulk queries.
/// Not a cache — each FindAllBuildCache call takes a fresh snapshot.
batch_request: IUIAutomationCacheRequest,
/// Raw-view tree walker, created once. Snapshot builds use it to probe a
/// pattern-less `DataItem`'s parent (the cell-vs-row disambiguation in
/// `map_uia_role`); raw view matches `batch_request`'s TrueCondition so
/// the probe sees the same tree the traversal does.
raw_walker: IUIAutomationTreeWalker,
/// UIA elements retained for action dispatch (keyed by handle ID).
handle_cache: Mutex<HashMap<u64, IUIAutomationElement>>,
}
// IUIAutomation is COM and thread-safe via proxy
unsafe impl Send for WindowsProvider {}
unsafe impl Sync for WindowsProvider {}
impl WindowsProvider {
pub fn new() -> Result<Self> {
// Establish Per-Monitor-V2 DPI awareness before the first bounds read
// so UIA reports coordinates in a stable (physical) space that we can
// convert to logical. Shared once-only init with the screenshot
// backend — see `crate::dpi` and issue #300.
crate::dpi::ensure_process_dpi_aware();
ensure_com_initialized().map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("COM initialization failed: {}", e),
})?;
let automation: IUIAutomation = unsafe {
windows::Win32::System::Com::CoCreateInstance(
&CUIAutomation8,
None,
windows::Win32::System::Com::CLSCTX_ALL,
)
}
.map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("Failed to create IUIAutomation: {}", e),
})?;
let batch_request = create_batch_request(&automation)?;
let raw_walker = unsafe { automation.RawViewWalker() }.map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("Failed to get RawViewWalker: {}", e),
})?;
Ok(Self {
automation,
batch_request,
raw_walker,
handle_cache: Mutex::new(HashMap::new()),
})
}
/// Re-acquire a UIA element via its native window handle.
/// This triggers WM_GETOBJECT which activates AccessKit's UIA provider,
/// ensuring the element's children include virtual accessibility elements.
fn reacquire_via_hwnd(
&self,
element: &IUIAutomationElement,
) -> std::result::Result<IUIAutomationElement, ()> {
let hwnd = unsafe { element.CurrentNativeWindowHandle() }.map_err(|_| ())?;
if hwnd.0.is_null() {
return Err(());
}
// Callers fall back to the un-reacquired element on `Err`, which for a
// transiently-busy COM server would silently hand back an element whose
// AccessKit provider was never activated. Retry first so a foreign
// app's momentary busy-ness doesn't quietly degrade the result.
retry_transient(|| unsafe { self.automation.ElementFromHandle(hwnd) }).map_err(|_| ())
}
/// Find an application's root UIA element + window name by PID.
///
/// Used by `subscribe_impl` to scope native UIA event handlers to a
/// single application's subtree.
fn find_app_by_pid(&self, pid: u32) -> Result<(IUIAutomationElement, String)> {
let root = uia_call(|| unsafe { self.automation.GetRootElement() })?;
let condition = uia_call(|| unsafe {
self.automation
.CreatePropertyCondition(UIA_ProcessIdPropertyId, &VARIANT::from(pid as i32))
})?;
let el = unsafe { root.FindFirst(TreeScope_Children, &condition) }.map_err(|_| {
Error::Platform {
code: -1,
message: format!("No window found for PID {}", pid),
}
})?;
// Re-acquire via HWND to activate AccessKit provider
let el = self.reacquire_via_hwnd(&el).unwrap_or(el);
let name = unsafe { el.CurrentName() }
.map(|s| s.to_string())
.unwrap_or_default();
Ok((el, name))
}
/// Cache a UIA element and return its handle ID.
fn cache_element(&self, uia: IUIAutomationElement) -> u64 {
let handle = NEXT_HANDLE.fetch_add(1, Ordering::Relaxed);
self.handle_cache
.lock()
.unwrap_or_else(|e| e.into_inner())
.insert(handle, uia);
handle
}
/// Look up a cached UIA element by handle.
fn get_cached(&self, handle: u64) -> Result<IUIAutomationElement> {
self.handle_cache
.lock()
.unwrap_or_else(|e| e.into_inner())
.get(&handle)
.cloned()
.ok_or(Error::ElementStale {
selector: format!("handle:{}", handle),
})
}
/// Query UIA patterns from the element once, sharing across
/// `get_value`, `get_actions`, and `parse_states` to avoid duplicate COM calls.
fn query_patterns(element: &IUIAutomationElement) -> ElementPatterns {
ElementPatterns {
invoke: unsafe {
element.GetCurrentPatternAs::<IUIAutomationInvokePattern>(UIA_InvokePatternId)
}
.ok(),
toggle: unsafe {
element.GetCurrentPatternAs::<IUIAutomationTogglePattern>(UIA_TogglePatternId)
}
.ok(),
expand_collapse: unsafe {
element.GetCurrentPatternAs::<IUIAutomationExpandCollapsePattern>(
UIA_ExpandCollapsePatternId,
)
}
.ok(),
value: unsafe {
element.GetCurrentPatternAs::<IUIAutomationValuePattern>(UIA_ValuePatternId)
}
.ok(),
range_value: unsafe {
element
.GetCurrentPatternAs::<IUIAutomationRangeValuePattern>(UIA_RangeValuePatternId)
}
.ok(),
selection_item: unsafe {
element.GetCurrentPatternAs::<IUIAutomationSelectionItemPattern>(
UIA_SelectionItemPatternId,
)
}
.ok(),
}
}
/// Build an ElementData from a pre-fetched UIA element snapshot.
///
/// The element MUST have been obtained via `FindAllBuildCache` or
/// `BuildUpdatedCache` so that Cached* accessors are populated.
/// Every query takes a fresh snapshot — callers never see stale data.
fn build_element_data(&self, element: &IUIAutomationElement, pid: Option<u32>) -> ElementData {
let handle = self.cache_element(element.clone());
build_snapshot_data(element, pid, handle, Some(&self.raw_walker))
}
/// Populate a UIA element's snapshot so Cached* accessors work.
/// Used for single-element reads (e.g., get_parent) that don't go
/// through FindAllBuildCache.
fn populate_cache(
&self,
element: &IUIAutomationElement,
) -> windows::core::Result<IUIAutomationElement> {
retry_transient(|| unsafe { element.BuildUpdatedCache(&self.batch_request) })
}
/// Get direct UIA children of an element with properties pre-fetched.
/// batch_request uses raw view (TrueCondition), so FindAllBuildCache sees
/// all elements including virtual/fragment elements.
fn uia_children(&self, element: &IUIAutomationElement) -> Vec<IUIAutomationElement> {
let true_cond = match unsafe { self.automation.CreateTrueCondition() } {
Ok(c) => c,
Err(_) => return Vec::new(),
};
// An empty Vec here reads downstream as "this element has no children",
// so a transiently-busy provider would make a populated subtree look
// empty. Retry the classified-transient HRESULTs before degrading.
match retry_transient(|| unsafe {
element.FindAllBuildCache(TreeScope_Children, &true_cond, &self.batch_request)
}) {
Ok(arr) => (0..uia_len(&arr))
.filter_map(|i| uia_get(&arr, i))
.collect(),
Err(_) => Vec::new(),
}
}
/// Fetch the entire subtree with all properties pre-fetched in one COM call.
fn find_all_subtree(&self, root: &IUIAutomationElement) -> Result<IUIAutomationElementArray> {
let true_cond = uia_call(|| unsafe { self.automation.CreateTrueCondition() })?;
uia_call(|| unsafe {
root.FindAllBuildCache(TreeScope_Subtree, &true_cond, &self.batch_request)
})
}
/// Extract a UIA element's RuntimeId as a `Vec<i32>` for use as a stable
/// cross-call identity key. `GetRuntimeId` returns a SAFEARRAY of i32 that
/// uniquely identifies an element within the UIA tree session — the only
/// identifier safe to use for dedup across `narrow_multi_segment` walks
/// within a single `find_elements_group` call.
///
/// Returns `None` if the COM call fails or the SAFEARRAY shape isn't what
/// UIA documents (1D, VT_I4). Callers treat `None` as "untracked" — the
/// element falls through dedup, which is harmless because untracked
/// duplicates would only over-report, never under-report.
fn runtime_id_key(element: &IUIAutomationElement) -> Option<Vec<i32>> {
use windows::Win32::System::Com::SAFEARRAY;
use windows::Win32::System::Ole::{SafeArrayAccessData, SafeArrayUnaccessData};
let sa: *mut SAFEARRAY = match unsafe { element.GetRuntimeId() } {
Ok(p) if !p.is_null() => p,
_ => return None,
};
// Safety: GetRuntimeId hands us ownership of a SAFEARRAY*. We must
// free it via SafeArrayDestroy when done, otherwise we leak. Lock
// the data via SafeArrayAccessData, copy out the i32s, unlock,
// destroy.
let result = unsafe {
let mut data_ptr: *mut core::ffi::c_void = core::ptr::null_mut();
if SafeArrayAccessData(sa, &mut data_ptr as *mut _).is_err() {
let _ = windows::Win32::System::Ole::SafeArrayDestroy(sa);
return None;
}
// SAFEARRAY of i32 — rgsabound[0].cElements is the count.
let bounds = (*sa).rgsabound.as_ptr();
let count = (*bounds).cElements as usize;
let slice = std::slice::from_raw_parts(data_ptr as *const i32, count);
let v = slice.to_vec();
let _ = SafeArrayUnaccessData(sa);
let _ = windows::Win32::System::Ole::SafeArrayDestroy(sa);
v
};
Some(result)
}
}
// ── Safe UIA helpers ────────────────────────────────────────────────────────
/// Maximum number of attempts for a UIA call that keeps failing with a
/// classified-transient HRESULT, and the delay between attempts.
const TRANSIENT_RETRY_ATTEMPTS: u32 = 3;
const TRANSIENT_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
/// True for COM "the server can't take this call right now" HRESULTs.
///
/// App discovery reaches well beyond xa11y's own target: `get_children(None)`
/// enumerates *every* top-level window on the desktop and makes a cross-process
/// call into each one. Those foreign processes are ordinary applications with
/// their own threading models, and COM refuses an incoming cross-apartment call
/// while the target is busy:
///
/// - `RPC_E_CALL_REJECTED` (0x80010001) — the callee rejected the call.
/// - `RPC_E_SERVERCALL_RETRYLATER` (0x8001010A) — the server is in a state
/// that cannot process it, and says so explicitly.
/// - `RPC_E_CANTCALLOUT_ININPUTSYNCCALL` (0x8001010D) — the target STA thread
/// is dispatching an input-synchronous call (it is inside a cross-thread
/// `SendMessage` handler), so COM will not let it call out.
///
/// None of these say anything about the app the caller actually asked for —
/// an unrelated busy process on the machine must not fail their query. Windows
/// raises them as first-chance SEH exceptions in the *calling* process before
/// COM converts them to a failed HRESULT, which is why they surface in CI logs
/// as `Windows fatal exception: code 0x8001010d` under pytest's faulthandler.
/// Those lines are handled exceptions, not crashes.
///
/// Observed on the `winforms` Windows integ cell in #328's CI run.
fn is_com_server_busy(e: &windows::core::Error) -> bool {
let code = e.code();
code == RPC_E_CALL_REJECTED
|| code == RPC_E_SERVERCALL_RETRYLATER
|| code == RPC_E_CANTCALLOUT_ININPUTSYNCCALL
}
/// The complete set of HRESULTs worth another attempt.
///
/// Deliberately a closed list. Every other error propagates on the first
/// attempt, so this stays a retry of specifically-classified transient
/// failures rather than a fallback chain (tenet 1).
fn is_transient(e: &windows::core::Error) -> bool {
is_event_subscriber_failure(e) || is_com_server_busy(e)
}
/// Retry a COM call while it fails transiently, preserving the raw HRESULT.
///
/// Used by the call sites that need the original `windows::core::Error` (or
/// that degrade rather than propagate). [`uia_call`] wraps this for the common
/// case of mapping into [`Error::Platform`].
fn retry_transient<T>(f: impl Fn() -> windows::core::Result<T>) -> windows::core::Result<T> {
let mut attempts_left = TRANSIENT_RETRY_ATTEMPTS;
loop {
attempts_left -= 1;
match f() {
Ok(v) => return Ok(v),
Err(e) if is_transient(&e) && attempts_left > 0 => {
std::thread::sleep(TRANSIENT_RETRY_DELAY);
}
Err(e) => return Err(e),
}
}
}
/// Wrap a UIA COM call, mapping the error to xa11y Error::Platform.
///
/// Two families of HRESULT are retried before the error is propagated:
///
/// `EVENT_E_ALL_SUBSCRIBERS_FAILED` (0x80040201) — some providers (notably
/// Qt's UIA backend) surface it from query calls like `FindAllBuildCache` even
/// though only the notification layer hiccupped. The action paths
/// (`press`/`toggle`/`select`) can swallow it outright because the action
/// already completed (#169); a query needs a value, so it is retried.
/// See: https://github.com/xa11y/xa11y/issues/257
///
/// The COM server-busy family — see [`is_com_server_busy`].
///
/// Any other error is returned immediately.
fn uia_call<T>(f: impl Fn() -> windows::core::Result<T>) -> Result<T> {
retry_transient(f).map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: e.to_string(),
})
}
/// Read a BSTR VARIANT property from the element's pre-fetched snapshot.
fn uia_cached_bstr(element: &IUIAutomationElement, prop: UIA_PROPERTY_ID) -> Option<String> {
unsafe { element.GetCachedPropertyValue(prop) }
.ok()
.and_then(|v| windows::core::BSTR::try_from(&v).ok())
.map(|b| b.to_string())
.filter(|s| !s.is_empty())
}
/// Read a VT_BOOL VARIANT property from the element's pre-fetched snapshot.
fn uia_cached_bool(element: &IUIAutomationElement, prop: UIA_PROPERTY_ID) -> Option<bool> {
unsafe { element.GetCachedPropertyValue(prop) }
.ok()
.and_then(|v| variant_bool(&v))
}
/// Read a VT_I4 VARIANT property from the element's pre-fetched snapshot.
fn uia_cached_i32(element: &IUIAutomationElement, prop: UIA_PROPERTY_ID) -> Option<i32> {
unsafe { element.GetCachedPropertyValue(prop) }
.ok()
.and_then(|v| variant_i32(&v))
}
/// Build an ElementData snapshot from a pre-fetched UIA element without
/// retaining the live reference in the provider's handle cache.
///
/// Used both by [`WindowsProvider::build_element_data`] (which allocates a
/// handle and wraps this call) and by event handlers (which pass `handle=0`
/// because event targets are snapshots — callers don't act on them directly).
fn build_snapshot_data(
element: &IUIAutomationElement,
pid: Option<u32>,
handle: u64,
walker: Option<&IUIAutomationTreeWalker>,
) -> ElementData {
let control_type = unsafe { element.CachedControlType() }.unwrap_or(UIA_CONTROLTYPE_ID(0));
let is_table_item = (control_type == UIA_DataItemControlTypeId
|| control_type == UIA_CustomControlTypeId)
&& uia_cached_bool(element, UIA_IsTableItemPatternAvailablePropertyId).unwrap_or(false);
// The parent probe costs two live COM calls, so it only runs for the one
// ambiguous case: a DataItem that doesn't implement TableItem. All other
// control types resolve from the cached snapshot alone.
let parent_is_data_item = control_type == UIA_DataItemControlTypeId
&& !is_table_item
&& walker.and_then(|w| parent_control_type(w, element)) == Some(UIA_DataItemControlTypeId);
// Only read for `Custom`, where it is the provider's sole role signal
// (see `map_msaa_role`); every other control type answers from the UIA
// control type alone.
let legacy_role = if control_type == UIA_CustomControlTypeId {
uia_cached_i32(element, UIA_LegacyIAccessibleRolePropertyId)
} else {
None
};
let mut role = map_uia_role(
control_type,
is_table_item,
parent_is_data_item,
legacy_role,
);
// Refine role using AriaRole property for elements that UIA maps ambiguously
// (e.g., Alert/Heading both become ControlType.Text, Dialog becomes Window)
if matches!(
role,
Role::StaticText | Role::Window | Role::Group | Role::Unknown
) {
if let Some(aria_str) = uia_cached_bstr(element, UIA_AriaRolePropertyId) {
match aria_str.as_str() {
"alert" => role = Role::Alert,
"dialog" | "alertdialog" => role = Role::Dialog,
"heading" => role = Role::Heading,
"separator" => role = Role::Separator,
"progressbar" => role = Role::ProgressBar,
"link" => role = Role::Link,
_ => {}
}
}
// Native (non-ARIA) dialogs: UIA_IsDialogPropertyId is a first-class
// UIA property (Windows 10 1703+) that native frameworks such as Qt
// set without populating AriaRole. Only apply when AriaRole hasn't
// already resolved the role to Dialog.
if role == Role::Window && uia_cached_bool(element, UIA_IsDialogPropertyId).unwrap_or(false)
{
role = Role::Dialog;
}
}
let name = unsafe { element.CachedName() }
.ok()
.map(|s| s.to_string())
.filter(|s| !s.is_empty());
let patterns = WindowsProvider::query_patterns(element);
let value = get_value(role, &patterns);
// Try FullDescription first (AccessKit's description), then HelpText
let description = uia_cached_bstr(element, UIA_FullDescriptionPropertyId)
.or_else(|| uia_cached_bstr(element, UIA_HelpTextPropertyId));
let states = parse_states(element, role, &patterns);
let bounds = unsafe { element.CachedBoundingRectangle() }
.ok()
.and_then(|r| {
let width = (r.right - r.left).max(0) as u32;
let height = (r.bottom - r.top).max(0) as u32;
if width == 0 && height == 0 {
None
} else {
// Under Per-Monitor-V2 awareness UIA reports physical pixels.
// Convert to logical coordinates so `Element::bounds` matches
// the cross-platform contract (logical points, same space as
// the screenshot/input layers). Scale is the DPI of the
// monitor the element sits on.
let scale = crate::dpi::scale_for_physical_point(r.left, r.top);
Some(
Rect {
x: r.left,
y: r.top,
width,
height,
}
.to_logical(scale),
)
}
});
let actions = get_actions(element, role, &patterns);
let automation_id = unsafe { element.CachedAutomationId() }
.ok()
.map(|s| s.to_string())
.filter(|s| !s.is_empty());
let class_name = unsafe { element.CachedClassName() }
.ok()
.map(|s| s.to_string())
.filter(|s| !s.is_empty());
let raw = {
let mut raw = std::collections::HashMap::new();
raw.insert(
"control_type_id".into(),
serde_json::Value::Number(serde_json::Number::from(control_type.0)),
);
if let Some(ref aid) = automation_id {
raw.insert(
"automation_id".into(),
serde_json::Value::String(aid.clone()),
);
}
if let Some(ref cn) = class_name {
raw.insert("class_name".into(), serde_json::Value::String(cn.clone()));
}
// Preserve unstripped originals so callers who need bidi marks can
// recover them after the strip below.
if let Some(ref n) = name {
raw.insert("uia_name".into(), serde_json::Value::String(n.clone()));
}
if let Some(ref v) = value {
raw.insert("uia_value".into(), serde_json::Value::String(v.clone()));
}
if let Some(ref d) = description {
raw.insert("uia_help_text".into(), serde_json::Value::String(d.clone()));
}
raw
};
// Strip Unicode bidi format controls. RTL apps on Windows embed LRM/RLM
// marks into reported strings; the originals are preserved in `raw`.
let name = xa11y_core::text::strip_bidi_opt(name);
let value = xa11y_core::text::strip_bidi_opt(value);
let description = xa11y_core::text::strip_bidi_opt(description);
let (numeric_value, min_value, max_value) = if matches!(
role,
Role::Slider | Role::ProgressBar | Role::ScrollBar | Role::SpinButton
) {
if let Some(ref pattern) = patterns.range_value {
(
unsafe { pattern.CurrentValue() }.ok(),
unsafe { pattern.CurrentMinimum() }.ok(),
unsafe { pattern.CurrentMaximum() }.ok(),
)
} else {
(None, None, None)
}
} else {
(None, None, None)
};
ElementData {
role,
name,
value,
description,
bounds,
actions,
states,
stable_id: automation_id,
numeric_value,
min_value,
max_value,
pid,
raw,
handle,
}
}
/// Build the batch request that describes which properties and patterns
/// to pre-fetch. Created once per provider, used on every query.
fn create_batch_request(automation: &IUIAutomation) -> Result<IUIAutomationCacheRequest> {
let request = uia_call(|| unsafe { automation.CreateCacheRequest() })?;
// The property list is a fixed constant array of valid UIA property IDs.
// If AddProperty ever fails here, something is structurally wrong with the
// UIA environment (e.g. COM state corrupted). Propagate rather than
// silently producing a half-configured cache request (tenet 1).
for prop in BATCH_PROPERTIES {
unsafe { request.AddProperty(*prop) }.map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("AddProperty({:?}) failed: {e}", prop),
})?;
}
// Use raw view (TrueCondition) so FindAllBuildCache sees all UIA elements,
// including virtual/fragment elements from Qt, AccessKit, etc. that don't
// set IsControlElement=true and are silently excluded by the default
// Control View tree filter.
let raw_view = uia_call(|| unsafe { automation.CreateTrueCondition() })?;
uia_call(|| unsafe { request.SetTreeFilter(&raw_view) })?;
Ok(request)
}
/// Properties pre-fetched in every bulk query.
const BATCH_PROPERTIES: &[UIA_PROPERTY_ID] = &[
UIA_ControlTypePropertyId,
UIA_AriaRolePropertyId,
UIA_IsDialogPropertyId,
UIA_IsTableItemPatternAvailablePropertyId,
UIA_NamePropertyId,
UIA_FullDescriptionPropertyId,
UIA_HelpTextPropertyId,
UIA_BoundingRectanglePropertyId,
UIA_AutomationIdPropertyId,
UIA_ClassNamePropertyId,
UIA_ProcessIdPropertyId,
UIA_IsEnabledPropertyId,
UIA_IsOffscreenPropertyId,
UIA_HasKeyboardFocusPropertyId,
UIA_IsKeyboardFocusablePropertyId,
UIA_NativeWindowHandlePropertyId,
// MSAA state bitmask — the only place pre-SelectionItem frameworks report
// selection (see the `selected` derivation in `parse_states`).
UIA_LegacyIAccessibleStatePropertyId,
// MSAA role — the only role signal for providers that publish no UIA
// control type (see `map_msaa_role`).
UIA_LegacyIAccessibleRolePropertyId,
];
/// Safe wrapper for IUIAutomationElementArray::Length.
fn uia_len(arr: &IUIAutomationElementArray) -> i32 {
unsafe { arr.Length() }.unwrap_or(0)
}
/// Safe wrapper for IUIAutomationElementArray::GetElement.
fn uia_get(arr: &IUIAutomationElementArray, index: i32) -> Option<IUIAutomationElement> {
unsafe { arr.GetElement(index) }.ok()
}
/// Locate the caret within a control's TextPattern, as a character offset
/// from the start of the document.
///
/// Returns:
/// - `Ok(Some(n))` if TextPattern reports a selection whose start lies `n`
/// characters into the document. For a collapsed caret, `n` is the caret
/// position. For a non-empty selection, `n` is the selection's start —
/// mirroring macOS/AT-SPI semantics of "insert at selection start".
/// - `Ok(None)` if the control has no TextPattern, or its selection array is
/// empty (no caret available). The caller should fall back to "append at
/// end" — the behaviour documented in design/README.md.
/// - `Err(..)` if TextPattern is present but a COM call to walk the range
/// fails. These are propagated rather than silently falling back, so
/// genuine platform errors surface (tenet 1).
fn caret_char_offset(uia_element: &IUIAutomationElement) -> Result<Option<usize>> {
let text_pattern = match unsafe {
uia_element.GetCurrentPatternAs::<IUIAutomationTextPattern>(UIA_TextPatternId)
} {
Ok(p) => p,
Err(_) => return Ok(None), // TextPattern not supported on this control.
};
let selection = match unsafe { text_pattern.GetSelection() } {
Ok(s) => s,
// No active selection (e.g. control never focused) — treat as "no caret".
Err(_) => return Ok(None),
};
if unsafe { selection.Length() }.unwrap_or(0) == 0 {
return Ok(None);
}
let selection_range = unsafe { selection.GetElement(0) }.map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("TextRangeArray::GetElement(0) failed: {}", e),
})?;
let doc_range = unsafe { text_pattern.DocumentRange() }.map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("TextPattern::DocumentRange failed: {}", e),
})?;
// Clone the document range and clip it so it spans [doc start .. selection start].
// The length of its text (in Unicode characters) is the caret offset.
let prefix = unsafe { doc_range.Clone() }.map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("TextRange::Clone failed: {}", e),
})?;
unsafe {
prefix.MoveEndpointByRange(
TextPatternRangeEndpoint_End,
&selection_range,
TextPatternRangeEndpoint_Start,
)
}
.map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("TextRange::MoveEndpointByRange failed: {}", e),
})?;
let prefix_text = unsafe { prefix.GetText(-1) }
.map(|s| s.to_string())
.map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("TextRange::GetText failed: {}", e),
})?;
Ok(Some(prefix_text.chars().count()))
}
/// Pre-queried UIA patterns for an element, avoiding redundant COM calls
/// across `get_value`, `get_actions`, and `parse_states`.
struct ElementPatterns {
invoke: Option<IUIAutomationInvokePattern>,
toggle: Option<IUIAutomationTogglePattern>,
expand_collapse: Option<IUIAutomationExpandCollapsePattern>,
value: Option<IUIAutomationValuePattern>,
range_value: Option<IUIAutomationRangeValuePattern>,
selection_item: Option<IUIAutomationSelectionItemPattern>,
}
impl Provider for WindowsProvider {
fn get_children(&self, element: Option<&ElementData>) -> Result<Vec<ElementData>> {
match element {
None => {
// Top-level: list all GUI application windows
let root = uia_call(|| unsafe { self.automation.GetRootElement() })?;
let condition = uia_call(|| unsafe {
self.automation.CreatePropertyCondition(
UIA_ControlTypePropertyId,
&VARIANT::from(UIA_WindowControlTypeId.0),
)
})?;
let found = uia_call(|| unsafe {
root.FindAllBuildCache(TreeScope_Children, &condition, &self.batch_request)
})?;
let mut results = Vec::new();
for i in 0..uia_len(&found) {
let Some(el) = uia_get(&found, i) else {
continue;
};
let pid = unsafe { el.CachedProcessId() }.unwrap_or(0) as u32;
// A process may own several top-level windows (e.g. a main
// window plus a modal dialog) and each is returned as its
// own entry. Deduping by pid silently dropped every window
// after the first, hiding modals from `App::list`/`find`
// (issue #304). The `pid == 0` skip still drops windows with
// no resolvable owning process; the empty-name skip below
// drops windows that are still unnamed mid-startup.
// Each entry's `states.active` marks the actual foreground
// window (HWND == GetForegroundWindow); that is what lets
// the core's foreground tagging pick the right window when a
// single process owns several top-level entries.
if pid == 0 {
continue;
}
let name = unsafe { el.CachedName() }
.map(|s| s.to_string())
.unwrap_or_default();
if name.is_empty() {
continue;
}
// Re-acquire via HWND to activate AccessKit provider,
// then populate snapshot for build_element_data.
let el = self
.reacquire_via_hwnd(&el)
.and_then(|e| self.populate_cache(&e).map_err(|_| ()))
.unwrap_or(el);
let mut data = self.build_element_data(&el, Some(pid));
if data.name.is_none() {
data.name = Some(name);
}
results.push(data);
}
Ok(results)
}
Some(element_data) => {
let uia = self.get_cached(element_data.handle)?;
let children = self.uia_children(&uia);
let pid = element_data.pid;
Ok(children
.iter()
.map(|child| self.build_element_data(child, pid))
.collect())
}
}
}
fn get_parent(&self, element: &ElementData) -> Result<Option<ElementData>> {
let uia = self.get_cached(element.handle)?;
if let Ok(walker) = unsafe { self.automation.RawViewWalker() } {
if let Ok(parent) = unsafe { walker.GetParentElement(&uia) } {
// Check if the parent is the desktop root (no further parent)
let parent_parent = unsafe { walker.GetParentElement(&parent) };
if parent_parent.is_err() {
return Ok(None);
}
// Populate snapshot so build_element_data can read Cached* props
let parent = self.populate_cache(&parent).map_err(|e| Error::Platform {
code: e.code().0 as i64,
message: format!("BuildUpdatedCache failed: {}", e),
})?;
let data = self.build_element_data(&parent, element.pid);
return Ok(Some(data));
}
}
Ok(None)
}
/// Enumerate top-level applications. UIA exposes apps as top-level
/// `Window` control-type elements under the desktop root — there's no
/// dedicated `Application` accessible — so we list the desktop's direct
/// named window children, one entry per top-level window. A process that
/// owns several top-level windows (e.g. an app showing a modal dialog)
/// therefore yields several entries, not one per PID (issue #304). This
/// is the canonical app discovery primitive (replaces the old
/// `find_elements(None, "application"/"window", …, depth=0)` idiom).
fn list_apps(&self) -> Result<Vec<ElementData>> {
self.get_children(None)
}
/// Attach to an application directly by pid via a UIA `ProcessId`
/// property search over the desktop root's children.
///
/// `list_apps()` enumerates desktop-root children of control type
/// `Window` and skips windows whose name is still empty — which is
/// exactly the state a freshly launched app's top-level window is in
/// while the process boots. Matching on the pid property alone closes
/// that blind spot: any top-level element owned by the process counts,
/// named or not.
fn app_by_pid(&self, pid: u32) -> Result<ElementData> {
let root = uia_call(|| unsafe { self.automation.GetRootElement() })?;
let condition = uia_call(|| unsafe {
self.automation
.CreatePropertyCondition(UIA_ProcessIdPropertyId, &VARIANT::from(pid as i32))
})?;
// FindFirstBuildCache returns S_OK with a null element when nothing
// matches; windows-rs surfaces that null as an `Err` carrying the
// S_OK HRESULT. That case is "process not in the UIA tree yet" —
// SelectorNotMatched, so the core poll loop retries — while a
// failing HRESULT is a genuine UIA error and short-circuits.
let el = match unsafe {
root.FindFirstBuildCache(TreeScope_Children, &condition, &self.batch_request)
} {
Ok(el) => el,
Err(e) if e.code().is_ok() => {
return Err(
Error::selector_not_matched(format!("application[pid={pid}]")).diagnose(
xa11y_core::Diagnosis {
last_observed: Some(
"no top-level UIA element owned by the process yet".to_string(),
),
..Default::default()
},
),
);
}
Err(e) => {
return Err(Error::Platform {
code: e.code().0 as i64,
message: format!("FindFirstBuildCache(ProcessId={pid}) failed: {e}"),
});
}
};
// Mirror get_children(None): re-acquire via HWND to activate
// AccessKit's UIA provider, then repopulate the property snapshot.
let el = self
.reacquire_via_hwnd(&el)
.and_then(|e| self.populate_cache(&e).map_err(|_| ()))
.unwrap_or(el);
Ok(self.build_element_data(&el, Some(pid)))
}
/// Identify the foreground application via `GetForegroundWindow` +
/// `ElementFromHandle` — the canonical Win32 foreground query mapped into
/// the UIA tree. UIA exposes apps as top-level `Window` elements (see
/// [`list_apps`](Self::list_apps)), and the foreground HWND is exactly such
/// a top-level window, so the resolved element's pid lines up with a
/// `list_apps` entry for the core to tag.
///
/// A NULL foreground window (nothing active — e.g. the desktop has focus,
/// or during a fast app switch) maps to [`Error::SelectorNotMatched`]
/// ("nothing focused"); a failing `ElementFromHandle` is a genuine UIA
/// error and propagates.
fn focused_app(&self) -> Result<ElementData> {
let hwnd = unsafe { GetForegroundWindow() };
if hwnd.0.is_null() {
return Err(Error::selector_not_matched("focused application"));
}
let el = uia_call(|| unsafe { self.automation.ElementFromHandle(hwnd) })?;
let pid = unsafe { el.CurrentProcessId() }.unwrap_or(0) as u32;
let pid_opt = (pid != 0).then_some(pid);
// Populate the snapshot so build_element_data's Cached* reads work,
// falling back to the live element if caching fails.
let el = self.populate_cache(&el).unwrap_or(el);
Ok(self.build_element_data(&el, pid_opt))
}
/// Override the default `narrow_multi_segment` so that the Descendant
/// combinator uses `find_elements_in_tree` (tree-walking via `get_children`)
/// rather than `self.find_elements` (which would invoke `find_all_subtree`).
///
/// `find_all_subtree` calls `FindAllBuildCache(TreeScope_Subtree)`. When the
/// candidate element is a UIA *fragment element* (not the HWND fragment root
/// — e.g. a Qt QFormLayout virtual group), that call can return an incomplete
/// array due to provider-activation boundaries, regardless of the tree view
/// filter. Walking level-by-level via `get_children` avoids that problem.
fn narrow_multi_segment(
&self,
mut candidates: Vec<ElementData>,
segments: &[SelectorSegment],
max_depth: u32,
limit: Option<usize>,
) -> Result<Vec<ElementData>> {
for segment in segments {
let mut next_candidates = Vec::new();
for candidate in &candidates {
match segment.combinator {
Combinator::Child => {
let children = self.get_children(Some(candidate))?;
for child in children {
if matches_simple(&child, &segment.simple) {
next_candidates.push(child);
}
}
}
Combinator::Descendant => {
// Walk level-by-level to avoid provider-activation boundary
// issues with FindAllBuildCache(Subtree) on fragment elements.
let sub_selector = Selector {
segments: vec![SelectorSegment {
combinator: Combinator::Root,
simple: segment.simple.clone(),
}],
};
let mut sub_results = xa11y_core::selector::find_elements_in_tree(
|el| self.get_children(el),
Some(candidate),
&sub_selector,
None,
Some(max_depth),
)?;
next_candidates.append(&mut sub_results);
}
Combinator::Root => unreachable!(),
}
}
let mut seen = std::collections::HashSet::new();
next_candidates.retain(|e| seen.insert(e.handle));
candidates = next_candidates;
}
if let Some(nth) = segments.last().and_then(|s| s.simple.nth) {
if nth <= candidates.len() {
candidates = vec![candidates.remove(nth - 1)];
} else {
candidates.clear();
}
}
if let Some(limit) = limit {
candidates.truncate(limit);
}
Ok(candidates)
}
fn find_elements_group(
&self,
root: &ElementData,
group: &xa11y_core::selector::SelectorGroup,
limit: Option<usize>,
max_depth: Option<u32>,
) -> Result<Vec<ElementData>> {
if group.clauses.is_empty() {
return Ok(vec![]);
}
// Reject any clause with zero segments early — `clause.segments[0]`
// below would otherwise panic. The parser doesn't produce empty
// clauses, but be defensive against direct `SelectorGroup` builders.
if group.clauses.iter().any(|c| c.segments.is_empty()) {
return Ok(vec![]);
}