Skip to content

Commit 67bb2a5

Browse files
committed
[iOS] Announce scroll position to VoiceOver
Fixes flutter#189285
1 parent ccf262e commit 67bb2a5

3 files changed

Lines changed: 612 additions & 32 deletions

File tree

‎engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,31 @@
2626
namespace flutter {
2727
class PlatformViewIOS;
2828

29+
/**
30+
* The numbers behind a spoken scroll status.
31+
*
32+
* Retained instead of the rendered string so that consecutive animation frames
33+
* can be compared with integer equality. A scroll animates over many frames and
34+
* only a few of them change what would be said, so building and localizing a
35+
* string every frame just to discover that would be wasted work on the platform
36+
* thread.
37+
*/
38+
struct AccessibilityScrollStatus {
39+
enum class Form { kNone, kRows, kPage };
40+
41+
Form form = Form::kNone;
42+
int32_t uid = -1;
43+
int64_t first = 0;
44+
int64_t last = 0;
45+
int64_t total = 0;
46+
47+
bool operator==(const AccessibilityScrollStatus& other) const {
48+
return form == other.form && uid == other.uid && first == other.first && last == other.last &&
49+
total == other.total;
50+
}
51+
bool operator!=(const AccessibilityScrollStatus& other) const { return !(*this == other); }
52+
};
53+
2954
/**
3055
* An accessibility instance is bound to one `FlutterViewController` and
3156
* `FlutterView` instance.
@@ -99,6 +124,10 @@ class AccessibilityBridge final : public AccessibilityBridgeIos {
99124
static constexpr int32_t kSemanticObjectIdInvalid = -1;
100125
int32_t last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
101126

127+
// The most recently announced scroll position, used to suppress consecutive
128+
// announcements that would say exactly the same thing. See UpdateSemantics.
129+
AccessibilityScrollStatus last_scroll_status_;
130+
102131
NSMutableDictionary<NSNumber*, SemanticsObject*>* objects_;
103132
FlutterBasicMessageChannel* accessibility_channel_;
104133
int32_t previous_route_id_ = 0;

‎engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm‎

Lines changed: 259 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
#import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h"
66

7+
#include <algorithm>
8+
#include <cmath>
79
#include <utility>
810

911
#include "flutter/fml/logging.h"
@@ -37,6 +39,212 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
3739
UIAccessibilityPostNotification(notification, argument);
3840
}
3941
};
42+
43+
// Keys into UIKit's own accessibility string table. These are the strings the
44+
// system itself speaks when VoiceOver scrolls a UIScrollView or UITableView, so
45+
// reusing them makes Flutter announce exactly what a native app announces, in
46+
// every language iOS ships.
47+
//
48+
// They are undocumented, so every lookup carries an English fallback and the
49+
// substitution below is written so that an unexpected format string degrades
50+
// rather than misbehaves. Both keys have been stable and byte-identical across
51+
// iOS 18.0 through 27.0, which added keys to this table but removed none.
52+
//
53+
// TODO(LouiseHsu): Consider having the framework supply these strings through
54+
// the semantics tree, localized like the rest of the app, which would remove
55+
// the dependency on undocumented keys and on the app declaring its
56+
// localizations to iOS. https://github.com/flutter/flutter/issues/189285
57+
constexpr char kUIKitAccessibilityBundleId[] = "com.apple.UIKit.axbundle";
58+
constexpr char kUIKitAccessibilityTable[] = "Accessibility";
59+
constexpr char kScrollPageStatusKey[] = "scroll.page.summary";
60+
constexpr char kScrollRowStatusKey[] = "table.scrollbypage.status";
61+
constexpr char kScrollPageStatusFallback[] = "page %1$@ of %2$@";
62+
constexpr char kScrollRowStatusFallback[] = "rows %1$@ to %2$@ of %3$@";
63+
64+
// UIKit's accessibility strings bundle, or nil if it has not been loaded.
65+
//
66+
// The system loads this bundle into the process when an assistive technology
67+
// such as VoiceOver starts, which is the only time a scroll status is spoken.
68+
// Looking it up by identifier therefore needs no knowledge of where it lives.
69+
//
70+
// Because loading is lazy, the lookup can legitimately fail early in the
71+
// process lifetime. A failed lookup is deliberately not cached, since doing so
72+
// would strand the app in the English fallback for as long as it runs; only a
73+
// successful lookup is memoized.
74+
//
75+
// Called only from the platform thread, so the unsynchronized cache is safe.
76+
NSBundle* UIKitAccessibilityBundle() {
77+
static NSBundle* bundle = nil;
78+
if (!bundle) {
79+
bundle = [NSBundle bundleWithIdentifier:@(kUIKitAccessibilityBundleId)];
80+
}
81+
return bundle;
82+
}
83+
84+
// Renders `value` using the current locale's digits and grouping, so that e.g.
85+
// Arabic and Devanagari locales do not get Western digits spliced into an
86+
// otherwise localized sentence.
87+
NSString* LocalizedCount(int64_t value) {
88+
return [NSNumberFormatter localizedStringFromNumber:@(value)
89+
numberStyle:NSNumberFormatterDecimalStyle];
90+
}
91+
92+
// Substitutes `arguments` into the positional placeholders (%1$@, %2$@, ...) of
93+
// `format`, returning nil if any placeholder is left over.
94+
//
95+
// Deliberately avoids +stringWithFormat:. `format` originates from a system
96+
// resource file rather than from source, so a format string with more
97+
// specifiers than we supply arguments for would read past the argument list.
98+
// Explicit token replacement cannot do that whatever the input.
99+
NSString* SubstitutePositionalArguments(NSString* format, NSArray<NSString*>* arguments) {
100+
NSMutableString* result = [format mutableCopy];
101+
for (NSUInteger i = 0; i < arguments.count; ++i) {
102+
NSString* token = [NSString stringWithFormat:@"%%%lu$@", static_cast<unsigned long>(i + 1)];
103+
[result replaceOccurrencesOfString:token
104+
withString:arguments[i]
105+
options:0
106+
range:NSMakeRange(0, result.length)];
107+
}
108+
// A leftover specifier means the format did not have the shape expected,
109+
// so the caller should use its own.
110+
if ([result rangeOfString:@"$@"].location != NSNotFound ||
111+
[result rangeOfString:@"%@"].location != NSNotFound) {
112+
return nil;
113+
}
114+
return result;
115+
}
116+
117+
// Looks `key` up in UIKit's table and fills in `arguments`, falling back to
118+
// `fallback_format` if the bundle is unavailable, the key is absent, or the
119+
// system format does not have the expected placeholders.
120+
NSString* LocalizedScrollStatus(const char* key,
121+
const char* fallback_format,
122+
NSArray<NSString*>* arguments) {
123+
NSString* fallback = @(fallback_format);
124+
NSString* format = fallback;
125+
if (NSBundle* bundle = UIKitAccessibilityBundle()) {
126+
NSString* localized = [bundle localizedStringForKey:@(key)
127+
value:fallback
128+
table:@(kUIKitAccessibilityTable)];
129+
if (localized.length > 0) {
130+
format = localized;
131+
}
132+
}
133+
NSString* result = SubstitutePositionalArguments(format, arguments);
134+
if (result) {
135+
return result;
136+
}
137+
return SubstitutePositionalArguments(fallback, arguments);
138+
}
139+
140+
// Derives the numbers that describe `object`'s current scroll position.
141+
//
142+
// Pure arithmetic: no allocation, no bundle access, no number formatting, so it
143+
// is cheap enough to run on every semantics update during a scroll animation.
144+
// The result compares equal across frames that would say the same thing, which
145+
// is what suppresses repeat announcements.
146+
AccessibilityScrollStatus ComputeScrollStatus(SemanticsObject* object) {
147+
const flutter::SemanticsNode& node = object.node;
148+
149+
AccessibilityScrollStatus status;
150+
status.uid = object.uid;
151+
152+
// Preferred form: "rows x to y of z", matching what UIKit says for table
153+
// views. Only possible when the scrollable reports child index information.
154+
//
155+
// UIKit has no singular variant, so a single visible row yields "rows 6 to 6
156+
// of 50". That is what a native table view announces, and matching it is
157+
// preferred over inventing a better-sounding string that no other iOS app
158+
// produces.
159+
if (node.scrollChildren > 0) {
160+
// The framework keeps rows inside the cache extent in the tree, flagged
161+
// hidden, on both sides of the viewport. Counting them would overstate the
162+
// last visible row. Android's bridge excludes them the same way.
163+
int64_t visible = 0;
164+
for (SemanticsObject* child in object.children) {
165+
if (!child.node.flags.isHidden) {
166+
++visible;
167+
}
168+
}
169+
int64_t total = node.scrollChildren;
170+
int64_t first = std::clamp<int64_t>(node.scrollIndex + 1, 1, total);
171+
status.form = AccessibilityScrollStatus::Form::kRows;
172+
status.first = first;
173+
status.last = std::min(total, first + std::max<int64_t>(visible, 1) - 1);
174+
status.total = total;
175+
return status;
176+
}
177+
178+
// Fallback: "page x of y" derived from the pixel extents.
179+
double position = node.scrollPosition;
180+
double extent_min = node.scrollExtentMin;
181+
double extent_max = node.scrollExtentMax;
182+
if (std::isnan(position) || std::isnan(extent_max) || !std::isfinite(extent_max)) {
183+
// Infinite or unknown extent: any total page count would be a fabrication.
184+
return status;
185+
}
186+
if (std::isnan(extent_min)) {
187+
extent_min = 0.0;
188+
}
189+
190+
const bool horizontal = node.HasAction(flutter::SemanticsAction::kScrollLeft) ||
191+
node.HasAction(flutter::SemanticsAction::kScrollRight);
192+
const double viewport = horizontal ? node.rect.width() : node.rect.height();
193+
if (viewport <= 0.0) {
194+
return status;
195+
}
196+
197+
// The content is `range + viewport` long, so a partial trailing screen counts
198+
// as a page of its own. The tolerance stops floating point error in the
199+
// extents, e.g. a range of 200.0000001 for exactly two further screens, from
200+
// producing a phantom extra page.
201+
constexpr double kPageTolerance = 1e-6;
202+
const double range = std::max(extent_max - extent_min, 0.0);
203+
int64_t total_pages = std::max<int64_t>(
204+
static_cast<int64_t>(std::ceil((range + viewport) / viewport - kPageTolerance)), 1);
205+
206+
// Maps how far through the scroll range the viewport is onto the pages, so
207+
// the start is always the first page and the end always the last.
208+
//
209+
// Deriving the page from `position / viewport` instead cannot reach the last
210+
// page unless the content is an exact multiple of the viewport: 2.5 screens
211+
// of content is 3 pages, but scrolled fully to the end it would say page 2.
212+
// For content that is an exact multiple, such as a PageView, this gives the
213+
// same answer, and rounding absorbs floating point error at page boundaries.
214+
int64_t current_page = 1;
215+
if (range > 0.0 && total_pages > 1) {
216+
const double progress = std::clamp((position - extent_min) / range, 0.0, 1.0);
217+
current_page = 1 + static_cast<int64_t>(std::round(progress * (total_pages - 1)));
218+
}
219+
status.form = AccessibilityScrollStatus::Form::kPage;
220+
status.first = std::clamp<int64_t>(current_page, 1, total_pages);
221+
status.total = total_pages;
222+
return status;
223+
}
224+
225+
// Renders `status` as the description carried by
226+
// UIAccessibilityPageScrolledNotification.
227+
//
228+
// Apple's contract for that notification is that the argument is an NSString
229+
// describing the new scroll position; posting an element instead produces no
230+
// speech at all.
231+
//
232+
// Returns nil for a status with no meaningful description, in which case the
233+
// caller should post without an argument rather than invent a misleading
234+
// number.
235+
NSString* FormatScrollStatus(const AccessibilityScrollStatus& status) {
236+
switch (status.form) {
237+
case AccessibilityScrollStatus::Form::kNone:
238+
return nil;
239+
case AccessibilityScrollStatus::Form::kRows:
240+
return LocalizedScrollStatus(kScrollRowStatusKey, kScrollRowStatusFallback, @[
241+
LocalizedCount(status.first), LocalizedCount(status.last), LocalizedCount(status.total)
242+
]);
243+
case AccessibilityScrollStatus::Form::kPage:
244+
return LocalizedScrollStatus(kScrollPageStatusKey, kScrollPageStatusFallback,
245+
@[ LocalizedCount(status.first), LocalizedCount(status.total) ]);
246+
}
247+
}
40248
} // namespace
41249

42250
AccessibilityBridge::AccessibilityBridge(
@@ -88,7 +296,10 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
88296
flutter::SemanticsNodeUpdates nodes,
89297
const flutter::CustomAccessibilityActionUpdates& actions) {
90298
BOOL layoutChanged = NO;
91-
BOOL scrollOccured = NO;
299+
// The object whose scroll position changed in this update, if any. Retained
300+
// (rather than a bare BOOL) so its new scroll position can be described in
301+
// the UIAccessibilityPageScrolledNotification argument below.
302+
SemanticsObject* scrolledObject = nil;
92303
BOOL needsAnnouncement = NO;
93304
for (const auto& entry : actions) {
94305
const flutter::CustomAccessibilityAction& action = entry.second;
@@ -98,7 +309,9 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
98309
const flutter::SemanticsNode& node = entry.second;
99310
SemanticsObject* object = GetOrCreateObject(node.id, nodes);
100311
layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
101-
scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
312+
if ([object nodeWillCauseScroll:&node]) {
313+
scrolledObject = object;
314+
}
102315
needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
103316
[object setSemanticsNode:&node];
104317
NSUInteger newChildCountInTraversalOrder = node.childrenInTraversalOrder.size();
@@ -218,7 +431,49 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
218431
routeName);
219432
}
220433

221-
if (layoutChanged) {
434+
if (scrolledObject) {
435+
// A scroll necessarily moves child nodes and recycles others, so
436+
// `layoutChanged` is effectively always true when a scroll happens.
437+
// Testing `scrolledObject` first is therefore required: while this was
438+
// written as `else if (scrollOccured)` the scroll notification was
439+
// unreachable in any real app.
440+
//
441+
// The argument must be an NSString describing the new scroll position.
442+
// A nil argument still moves VoiceOver's scroll cursor, it just does not
443+
// speak, which is the correct degradation when the position is unknown.
444+
//
445+
// A scroll is animated, so it arrives as a run of semantics updates with
446+
// slightly different scroll positions - an overscroll bounce alone can
447+
// produce dozens. Most of those describe the same visible rows, and an
448+
// announcement is only useful when what it says actually changes, so
449+
// repeats of the previous status for the same scrollable are dropped.
450+
// Comparing the computed numbers rather than the rendered string keeps
451+
// the discarded frames off the localization path entirely.
452+
//
453+
// This also suppresses the duplicate "page x of y" that the framework's
454+
// 0.8 viewport scroll factor would otherwise produce on consecutive
455+
// scrolls.
456+
AccessibilityScrollStatus status = ComputeScrollStatus(scrolledObject);
457+
if (status != last_scroll_status_) {
458+
last_scroll_status_ = status;
459+
ios_delegate_->PostAccessibilityNotification(UIAccessibilityPageScrolledNotification,
460+
FormatScrollStatus(status));
461+
}
462+
463+
// Scrolling can destroy the focused element, because rows leaving the
464+
// viewport are removed from the tree. Only then does focus need
465+
// re-anchoring; otherwise posting a layout change here would drag
466+
// VoiceOver away from the element the user was reading.
467+
//
468+
// This is deliberately outside the announcement check above: whether
469+
// focus was destroyed is unrelated to whether the status string changed.
470+
if (last_focused_semantics_object_id_ != kSemanticObjectIdInvalid &&
471+
![objects_ objectForKey:@(last_focused_semantics_object_id_)]) {
472+
ios_delegate_->PostAccessibilityNotification(
473+
UIAccessibilityLayoutChangedNotification,
474+
FindNextFocusableIfNecessary().nativeAccessibility);
475+
}
476+
} else if (layoutChanged) {
222477
SemanticsObject* next = FindNextFocusableIfNecessary();
223478
SemanticsObject* lastFocused = [objects_ objectForKey:@(last_focused_semantics_object_id_)];
224479
// Only specify the focus item if the new focus is different, avoiding double focuses on the
@@ -227,13 +482,6 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
227482
ios_delegate_->PostAccessibilityNotification(
228483
UIAccessibilityLayoutChangedNotification,
229484
(routeChanged || next != lastFocused) ? next.nativeAccessibility : NULL);
230-
} else if (scrollOccured) {
231-
// TODO(chunhtai): figure out what string to use for notification. At this
232-
// point, it is guarantee the previous focused object is still in the tree
233-
// so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
234-
ios_delegate_->PostAccessibilityNotification(
235-
UIAccessibilityPageScrolledNotification,
236-
FindNextFocusableIfNecessary().nativeAccessibility);
237485
}
238486
}
239487
}
@@ -380,6 +628,7 @@ static void ReplaceSemanticsObject(SemanticsObject* oldObject,
380628
[objects_ removeAllObjects];
381629
previous_route_id_ = 0;
382630
previous_routes_.clear();
631+
last_scroll_status_ = {};
383632
view_controller_.viewIfLoaded.accessibilityElements = nil;
384633
}
385634

0 commit comments

Comments
 (0)