Skip to content

Commit 2f856f6

Browse files
committed
[iOS] Announce scroll position to VoiceOver
Fixes flutter#189285
1 parent ccf262e commit 2f856f6

3 files changed

Lines changed: 616 additions & 32 deletions

File tree

‎engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,31 @@
2626
namespace flutter {
2727
class PlatformViewIOS;
2828

29+
/**
30+
* The numbers behind a spoken scroll status.
31+
*
32+
* Retained instead of the rendered string so that consecutive animation frames
33+
* can be compared with integer equality. A scroll animates over many frames and
34+
* only a few of them change what would be said, so building and localizing a
35+
* string every frame just to discover that would be wasted work on the platform
36+
* thread.
37+
*/
38+
struct AccessibilityScrollStatus {
39+
enum class Form { kNone, kRows, kPage };
40+
41+
Form form = Form::kNone;
42+
int32_t uid = -1;
43+
int64_t first = 0;
44+
int64_t last = 0;
45+
int64_t total = 0;
46+
47+
bool operator==(const AccessibilityScrollStatus& other) const {
48+
return form == other.form && uid == other.uid && first == other.first && last == other.last &&
49+
total == other.total;
50+
}
51+
bool operator!=(const AccessibilityScrollStatus& other) const { return !(*this == other); }
52+
};
53+
2954
/**
3055
* An accessibility instance is bound to one `FlutterViewController` and
3156
* `FlutterView` instance.
@@ -99,6 +124,10 @@ class AccessibilityBridge final : public AccessibilityBridgeIos {
99124
static constexpr int32_t kSemanticObjectIdInvalid = -1;
100125
int32_t last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
101126

127+
// The most recently announced scroll position, used to suppress consecutive
128+
// announcements that would say exactly the same thing. See UpdateSemantics.
129+
AccessibilityScrollStatus last_scroll_status_;
130+
102131
NSMutableDictionary<NSNumber*, SemanticsObject*>* objects_;
103132
FlutterBasicMessageChannel* accessibility_channel_;
104133
int32_t previous_route_id_ = 0;

‎engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm‎

Lines changed: 263 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
#import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h"
66

7+
#include <algorithm>
8+
#include <cmath>
79
#include <utility>
810

911
#include "flutter/fml/logging.h"
@@ -84,11 +86,224 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
8486
}
8587
}
8688

89+
namespace {
90+
91+
// Keys into UIKit's own accessibility string table. These are the strings the
92+
// system itself speaks when VoiceOver scrolls a UIScrollView or UITableView, so
93+
// reusing them makes Flutter announce exactly what a native app announces, in
94+
// every language iOS ships.
95+
//
96+
// They are undocumented, so every lookup carries an English fallback and the
97+
// substitution below is written so that an unexpected format string degrades
98+
// rather than misbehaves. Both keys have been stable and byte-identical across
99+
// iOS 18.0 through 27.0, which added keys to this table but removed none.
100+
//
101+
// TODO(LouiseHsu): Consider having the framework supply these strings through
102+
// the semantics tree, localized like the rest of the app, which would remove
103+
// the dependency on undocumented keys and on the app declaring its
104+
// localizations to iOS. https://github.com/flutter/flutter/issues/189285
105+
constexpr char kUIKitAccessibilityBundleId[] = "com.apple.UIKit.axbundle";
106+
constexpr char kUIKitAccessibilityTable[] = "Accessibility";
107+
constexpr char kScrollPageStatusKey[] = "scroll.page.summary";
108+
constexpr char kScrollRowStatusKey[] = "table.scrollbypage.status";
109+
constexpr char kScrollPageStatusFallback[] = "page %1$@ of %2$@";
110+
constexpr char kScrollRowStatusFallback[] = "rows %1$@ to %2$@ of %3$@";
111+
112+
// UIKit's accessibility strings bundle, or nil if it has not been loaded.
113+
//
114+
// The system loads this bundle into the process when an assistive technology
115+
// such as VoiceOver starts, which is the only time a scroll status is spoken.
116+
// Looking it up by identifier therefore needs no knowledge of where it lives.
117+
//
118+
// Because loading is lazy, the lookup can legitimately fail early in the
119+
// process lifetime. A failed lookup is deliberately not cached, since doing so
120+
// would strand the app in the English fallback for as long as it runs; only a
121+
// successful lookup is memoized.
122+
//
123+
// Called only from the platform thread, so the unsynchronized cache is safe.
124+
NSBundle* UIKitAccessibilityBundle() {
125+
static NSBundle* bundle = nil;
126+
if (!bundle) {
127+
bundle = [NSBundle bundleWithIdentifier:@(kUIKitAccessibilityBundleId)];
128+
}
129+
return bundle;
130+
}
131+
132+
// Renders `value` using the current locale's digits and grouping, so that e.g.
133+
// Arabic and Devanagari locales do not get Western digits spliced into an
134+
// otherwise localized sentence.
135+
NSString* LocalizedCount(int64_t value) {
136+
return [NSNumberFormatter localizedStringFromNumber:@(value)
137+
numberStyle:NSNumberFormatterDecimalStyle];
138+
}
139+
140+
// Substitutes `arguments` into the positional placeholders (%1$@, %2$@, ...) of
141+
// `format`, returning nil if any placeholder is left over.
142+
//
143+
// Deliberately avoids +stringWithFormat:. `format` originates from a system
144+
// resource file rather than from source, so a format string with more
145+
// specifiers than we supply arguments for would read past the argument list.
146+
// Explicit token replacement cannot do that whatever the input.
147+
NSString* SubstitutePositionalArguments(NSString* format, NSArray<NSString*>* arguments) {
148+
NSMutableString* result = [format mutableCopy];
149+
for (NSUInteger i = 0; i < arguments.count; ++i) {
150+
NSString* token = [NSString stringWithFormat:@"%%%lu$@", static_cast<unsigned long>(i + 1)];
151+
[result replaceOccurrencesOfString:token
152+
withString:arguments[i]
153+
options:0
154+
range:NSMakeRange(0, result.length)];
155+
}
156+
// A leftover specifier means the format did not have the shape expected,
157+
// so the caller should use its own.
158+
if ([result rangeOfString:@"$@"].location != NSNotFound ||
159+
[result rangeOfString:@"%@"].location != NSNotFound) {
160+
return nil;
161+
}
162+
return result;
163+
}
164+
165+
// Looks `key` up in UIKit's table and fills in `arguments`, falling back to
166+
// `fallback_format` if the bundle is unavailable, the key is absent, or the
167+
// system format does not have the expected placeholders.
168+
NSString* LocalizedScrollStatus(const char* key,
169+
const char* fallback_format,
170+
NSArray<NSString*>* arguments) {
171+
NSString* fallback = @(fallback_format);
172+
NSString* format = fallback;
173+
if (NSBundle* bundle = UIKitAccessibilityBundle()) {
174+
NSString* localized = [bundle localizedStringForKey:@(key)
175+
value:fallback
176+
table:@(kUIKitAccessibilityTable)];
177+
if (localized.length > 0) {
178+
format = localized;
179+
}
180+
}
181+
NSString* result = SubstitutePositionalArguments(format, arguments);
182+
if (result) {
183+
return result;
184+
}
185+
return SubstitutePositionalArguments(fallback, arguments);
186+
}
187+
188+
// Derives the numbers that describe `object`'s current scroll position.
189+
//
190+
// Pure arithmetic: no allocation, no bundle access, no number formatting, so it
191+
// is cheap enough to run on every semantics update during a scroll animation.
192+
// The result compares equal across frames that would say the same thing, which
193+
// is what suppresses repeat announcements.
194+
AccessibilityScrollStatus ComputeScrollStatus(SemanticsObject* object) {
195+
const flutter::SemanticsNode& node = object.node;
196+
197+
AccessibilityScrollStatus status;
198+
status.uid = object.uid;
199+
200+
// Preferred form: "rows x to y of z", matching what UIKit says for table
201+
// views. Only possible when the scrollable reports child index information.
202+
//
203+
// UIKit has no singular variant, so a single visible row yields "rows 6 to 6
204+
// of 50". That is what a native table view announces, and matching it is
205+
// preferred over inventing a better-sounding string that no other iOS app
206+
// produces.
207+
if (node.scrollChildren > 0) {
208+
// The framework keeps rows inside the cache extent in the tree, flagged
209+
// hidden, on both sides of the viewport. Counting them would overstate the
210+
// last visible row. Android's bridge excludes them the same way.
211+
int64_t visible = 0;
212+
for (SemanticsObject* child in object.children) {
213+
if (!child.node.flags.isHidden) {
214+
++visible;
215+
}
216+
}
217+
int64_t total = node.scrollChildren;
218+
int64_t first = std::clamp<int64_t>(node.scrollIndex + 1, 1, total);
219+
status.form = AccessibilityScrollStatus::Form::kRows;
220+
status.first = first;
221+
status.last = std::min(total, first + std::max<int64_t>(visible, 1) - 1);
222+
status.total = total;
223+
return status;
224+
}
225+
226+
// Fallback: "page x of y" derived from the pixel extents.
227+
double position = node.scrollPosition;
228+
double extent_min = node.scrollExtentMin;
229+
double extent_max = node.scrollExtentMax;
230+
if (std::isnan(position) || std::isnan(extent_max) || !std::isfinite(extent_max)) {
231+
// Infinite or unknown extent: any total page count would be a fabrication.
232+
return status;
233+
}
234+
if (std::isnan(extent_min)) {
235+
extent_min = 0.0;
236+
}
237+
238+
const bool horizontal = node.HasAction(flutter::SemanticsAction::kScrollLeft) ||
239+
node.HasAction(flutter::SemanticsAction::kScrollRight);
240+
const double viewport = horizontal ? node.rect.width() : node.rect.height();
241+
if (viewport <= 0.0) {
242+
return status;
243+
}
244+
245+
// The content is `range + viewport` long, so a partial trailing screen counts
246+
// as a page of its own. The tolerance stops floating point error in the
247+
// extents, e.g. a range of 200.0000001 for exactly two further screens, from
248+
// producing a phantom extra page.
249+
constexpr double kPageTolerance = 1e-6;
250+
const double range = std::max(extent_max - extent_min, 0.0);
251+
int64_t total_pages = std::max<int64_t>(
252+
static_cast<int64_t>(std::ceil((range + viewport) / viewport - kPageTolerance)), 1);
253+
254+
// Maps how far through the scroll range the viewport is onto the pages, so
255+
// the start is always the first page and the end always the last.
256+
//
257+
// Deriving the page from `position / viewport` instead cannot reach the last
258+
// page unless the content is an exact multiple of the viewport: 2.5 screens
259+
// of content is 3 pages, but scrolled fully to the end it would say page 2.
260+
// For content that is an exact multiple, such as a PageView, this gives the
261+
// same answer, and rounding absorbs floating point error at page boundaries.
262+
int64_t current_page = 1;
263+
if (range > 0.0 && total_pages > 1) {
264+
const double progress = std::clamp((position - extent_min) / range, 0.0, 1.0);
265+
current_page = 1 + static_cast<int64_t>(std::round(progress * (total_pages - 1)));
266+
}
267+
status.form = AccessibilityScrollStatus::Form::kPage;
268+
status.first = std::clamp<int64_t>(current_page, 1, total_pages);
269+
status.total = total_pages;
270+
return status;
271+
}
272+
273+
// Renders `status` as the description carried by
274+
// UIAccessibilityPageScrolledNotification.
275+
//
276+
// Apple's contract for that notification is that the argument is an NSString
277+
// describing the new scroll position; posting an element instead produces no
278+
// speech at all.
279+
//
280+
// Returns nil for a status with no meaningful description, in which case the
281+
// caller should post without an argument rather than invent a misleading
282+
// number.
283+
NSString* FormatScrollStatus(const AccessibilityScrollStatus& status) {
284+
switch (status.form) {
285+
case AccessibilityScrollStatus::Form::kNone:
286+
return nil;
287+
case AccessibilityScrollStatus::Form::kRows:
288+
return LocalizedScrollStatus(kScrollRowStatusKey, kScrollRowStatusFallback, @[
289+
LocalizedCount(status.first), LocalizedCount(status.last), LocalizedCount(status.total)
290+
]);
291+
case AccessibilityScrollStatus::Form::kPage:
292+
return LocalizedScrollStatus(kScrollPageStatusKey, kScrollPageStatusFallback,
293+
@[ LocalizedCount(status.first), LocalizedCount(status.total) ]);
294+
}
295+
}
296+
297+
} // namespace
298+
87299
void AccessibilityBridge::UpdateSemantics(
88300
flutter::SemanticsNodeUpdates nodes,
89301
const flutter::CustomAccessibilityActionUpdates& actions) {
90302
BOOL layoutChanged = NO;
91-
BOOL scrollOccured = NO;
303+
// The object whose scroll position changed in this update, if any. Retained
304+
// (rather than a bare BOOL) so its new scroll position can be described in
305+
// the UIAccessibilityPageScrolledNotification argument below.
306+
SemanticsObject* scrolledObject = nil;
92307
BOOL needsAnnouncement = NO;
93308
for (const auto& entry : actions) {
94309
const flutter::CustomAccessibilityAction& action = entry.second;
@@ -98,7 +313,9 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
98313
const flutter::SemanticsNode& node = entry.second;
99314
SemanticsObject* object = GetOrCreateObject(node.id, nodes);
100315
layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
101-
scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
316+
if ([object nodeWillCauseScroll:&node]) {
317+
scrolledObject = object;
318+
}
102319
needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
103320
[object setSemanticsNode:&node];
104321
NSUInteger newChildCountInTraversalOrder = node.childrenInTraversalOrder.size();
@@ -218,7 +435,49 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
218435
routeName);
219436
}
220437

221-
if (layoutChanged) {
438+
if (scrolledObject) {
439+
// A scroll necessarily moves child nodes and recycles others, so
440+
// `layoutChanged` is effectively always true when a scroll happens.
441+
// Testing `scrolledObject` first is therefore required: while this was
442+
// written as `else if (scrollOccured)` the scroll notification was
443+
// unreachable in any real app.
444+
//
445+
// The argument must be an NSString describing the new scroll position.
446+
// A nil argument still moves VoiceOver's scroll cursor, it just does not
447+
// speak, which is the correct degradation when the position is unknown.
448+
//
449+
// A scroll is animated, so it arrives as a run of semantics updates with
450+
// slightly different scroll positions - an overscroll bounce alone can
451+
// produce dozens. Most of those describe the same visible rows, and an
452+
// announcement is only useful when what it says actually changes, so
453+
// repeats of the previous status for the same scrollable are dropped.
454+
// Comparing the computed numbers rather than the rendered string keeps
455+
// the discarded frames off the localization path entirely.
456+
//
457+
// This also suppresses the duplicate "page x of y" that the framework's
458+
// 0.8 viewport scroll factor would otherwise produce on consecutive
459+
// scrolls.
460+
AccessibilityScrollStatus status = ComputeScrollStatus(scrolledObject);
461+
if (status != last_scroll_status_) {
462+
last_scroll_status_ = status;
463+
ios_delegate_->PostAccessibilityNotification(UIAccessibilityPageScrolledNotification,
464+
FormatScrollStatus(status));
465+
}
466+
467+
// Scrolling can destroy the focused element, because rows leaving the
468+
// viewport are removed from the tree. Only then does focus need
469+
// re-anchoring; otherwise posting a layout change here would drag
470+
// VoiceOver away from the element the user was reading.
471+
//
472+
// This is deliberately outside the announcement check above: whether
473+
// focus was destroyed is unrelated to whether the status string changed.
474+
if (last_focused_semantics_object_id_ != kSemanticObjectIdInvalid &&
475+
![objects_ objectForKey:@(last_focused_semantics_object_id_)]) {
476+
ios_delegate_->PostAccessibilityNotification(
477+
UIAccessibilityLayoutChangedNotification,
478+
FindNextFocusableIfNecessary().nativeAccessibility);
479+
}
480+
} else if (layoutChanged) {
222481
SemanticsObject* next = FindNextFocusableIfNecessary();
223482
SemanticsObject* lastFocused = [objects_ objectForKey:@(last_focused_semantics_object_id_)];
224483
// Only specify the focus item if the new focus is different, avoiding double focuses on the
@@ -227,13 +486,6 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
227486
ios_delegate_->PostAccessibilityNotification(
228487
UIAccessibilityLayoutChangedNotification,
229488
(routeChanged || next != lastFocused) ? next.nativeAccessibility : NULL);
230-
} else if (scrollOccured) {
231-
// TODO(chunhtai): figure out what string to use for notification. At this
232-
// point, it is guarantee the previous focused object is still in the tree
233-
// so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
234-
ios_delegate_->PostAccessibilityNotification(
235-
UIAccessibilityPageScrolledNotification,
236-
FindNextFocusableIfNecessary().nativeAccessibility);
237489
}
238490
}
239491
}
@@ -380,6 +632,7 @@ static void ReplaceSemanticsObject(SemanticsObject* oldObject,
380632
[objects_ removeAllObjects];
381633
previous_route_id_ = 0;
382634
previous_routes_.clear();
635+
last_scroll_status_ = {};
383636
view_controller_.viewIfLoaded.accessibilityElements = nil;
384637
}
385638

0 commit comments

Comments
 (0)