Skip to content

Commit e876ccf

Browse files
committed
[iOS] Announce scroll position to VoiceOver
Fixes flutter#189285
1 parent ccf262e commit e876ccf

3 files changed

Lines changed: 523 additions & 32 deletions

File tree

‎engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,26 @@
2626
namespace flutter {
2727
class PlatformViewIOS;
2828

29+
/**
30+
* The numbers behind a spoken scroll status. Compared across frames to skip
31+
* repeated announcements without building strings.
32+
*/
33+
struct AccessibilityScrollStatus {
34+
enum class Form { kNone, kRows, kPage };
35+
36+
Form form = Form::kNone;
37+
int32_t uid = -1;
38+
int64_t first = 0;
39+
int64_t last = 0;
40+
int64_t total = 0;
41+
42+
bool operator==(const AccessibilityScrollStatus& other) const {
43+
return form == other.form && uid == other.uid && first == other.first && last == other.last &&
44+
total == other.total;
45+
}
46+
bool operator!=(const AccessibilityScrollStatus& other) const { return !(*this == other); }
47+
};
48+
2949
/**
3050
* An accessibility instance is bound to one `FlutterViewController` and
3151
* `FlutterView` instance.
@@ -99,6 +119,9 @@ class AccessibilityBridge final : public AccessibilityBridgeIos {
99119
static constexpr int32_t kSemanticObjectIdInvalid = -1;
100120
int32_t last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
101121

122+
// The last announced scroll status, used to skip repeats.
123+
AccessibilityScrollStatus last_scroll_status_;
124+
102125
NSMutableDictionary<NSNumber*, SemanticsObject*>* objects_;
103126
FlutterBasicMessageChannel* accessibility_channel_;
104127
int32_t previous_route_id_ = 0;

‎engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm‎

Lines changed: 186 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
#import "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h"
66

7+
#include <algorithm>
8+
#include <cmath>
79
#include <utility>
810

911
#include "flutter/fml/logging.h"
@@ -84,11 +86,169 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
8486
}
8587
}
8688

89+
namespace {
90+
91+
// Keys into UIKit's accessibility strings, so scroll announcements match native
92+
// scroll views in every language iOS supports. The keys are undocumented, so
93+
// each has an English fallback.
94+
//
95+
// TODO(LouiseHsu): Have the framework supply these strings instead, removing
96+
// the dependency on undocumented keys.
97+
// https://github.com/flutter/flutter/issues/189285
98+
constexpr char kUIKitAccessibilityBundleId[] = "com.apple.UIKit.axbundle";
99+
constexpr char kUIKitAccessibilityTable[] = "Accessibility";
100+
constexpr char kScrollPageStatusKey[] = "scroll.page.summary";
101+
constexpr char kScrollRowStatusKey[] = "table.scrollbypage.status";
102+
constexpr char kScrollPageStatusFallback[] = "page %1$@ of %2$@";
103+
constexpr char kScrollRowStatusFallback[] = "rows %1$@ to %2$@ of %3$@";
104+
105+
// UIKit's accessibility strings bundle, or nil if not loaded yet. iOS loads it
106+
// when VoiceOver starts, so only a successful lookup is cached.
107+
NSBundle* UIKitAccessibilityBundle() {
108+
static NSBundle* bundle = nil;
109+
if (!bundle) {
110+
bundle = [NSBundle bundleWithIdentifier:@(kUIKitAccessibilityBundleId)];
111+
}
112+
return bundle;
113+
}
114+
115+
// Formats `value` with the current locale's digits.
116+
NSString* LocalizedCount(int64_t value) {
117+
return [NSNumberFormatter localizedStringFromNumber:@(value)
118+
numberStyle:NSNumberFormatterDecimalStyle];
119+
}
120+
121+
// Replaces %1$@, %2$@, ... in `format` with `arguments`, or returns nil if a
122+
// placeholder is left over. Avoids +stringWithFormat: because `format` comes
123+
// from a system file and could have more specifiers than arguments.
124+
NSString* SubstitutePositionalArguments(NSString* format, NSArray<NSString*>* arguments) {
125+
NSMutableString* result = [format mutableCopy];
126+
for (NSUInteger i = 0; i < arguments.count; ++i) {
127+
NSString* token = [NSString stringWithFormat:@"%%%lu$@", static_cast<unsigned long>(i + 1)];
128+
[result replaceOccurrencesOfString:token
129+
withString:arguments[i]
130+
options:0
131+
range:NSMakeRange(0, result.length)];
132+
}
133+
if ([result rangeOfString:@"$@"].location != NSNotFound ||
134+
[result rangeOfString:@"%@"].location != NSNotFound) {
135+
return nil;
136+
}
137+
return result;
138+
}
139+
140+
// Looks up `key` in UIKit's table and fills in `arguments`, falling back to
141+
// `fallback_format` if that fails.
142+
NSString* LocalizedScrollStatus(const char* key,
143+
const char* fallback_format,
144+
NSArray<NSString*>* arguments) {
145+
NSString* fallback = @(fallback_format);
146+
NSString* format = fallback;
147+
if (NSBundle* bundle = UIKitAccessibilityBundle()) {
148+
NSString* localized = [bundle localizedStringForKey:@(key)
149+
value:fallback
150+
table:@(kUIKitAccessibilityTable)];
151+
if (localized.length > 0) {
152+
format = localized;
153+
}
154+
}
155+
NSString* result = SubstitutePositionalArguments(format, arguments);
156+
if (result) {
157+
return result;
158+
}
159+
return SubstitutePositionalArguments(fallback, arguments);
160+
}
161+
162+
// Computes the numbers to announce for `object`'s scroll position. Builds no
163+
// strings, so it is cheap enough to run on every frame of a scroll.
164+
AccessibilityScrollStatus ComputeScrollStatus(SemanticsObject* object) {
165+
const flutter::SemanticsNode& node = object.node;
166+
167+
AccessibilityScrollStatus status;
168+
status.uid = object.uid;
169+
170+
// "rows x to y of z", when the scrollable reports child indexes.
171+
if (node.scrollChildren > 0) {
172+
// Skip off-screen cache extent rows, which the framework marks hidden.
173+
int64_t visible = 0;
174+
for (SemanticsObject* child in object.children) {
175+
if (!child.node.flags.isHidden) {
176+
++visible;
177+
}
178+
}
179+
int64_t total = node.scrollChildren;
180+
int64_t first = std::clamp<int64_t>(node.scrollIndex + 1, 1, total);
181+
status.form = AccessibilityScrollStatus::Form::kRows;
182+
status.first = first;
183+
status.last = std::min(total, first + std::max<int64_t>(visible, 1) - 1);
184+
status.total = total;
185+
return status;
186+
}
187+
188+
// Otherwise "page x of y", from the scroll extents.
189+
double position = node.scrollPosition;
190+
double extent_min = node.scrollExtentMin;
191+
double extent_max = node.scrollExtentMax;
192+
if (std::isnan(position) || std::isnan(extent_max) || !std::isfinite(extent_max)) {
193+
// No page count exists for an infinite or unknown extent.
194+
return status;
195+
}
196+
if (std::isnan(extent_min)) {
197+
extent_min = 0.0;
198+
}
199+
200+
const bool horizontal = node.HasAction(flutter::SemanticsAction::kScrollLeft) ||
201+
node.HasAction(flutter::SemanticsAction::kScrollRight);
202+
const double viewport = horizontal ? node.rect.width() : node.rect.height();
203+
if (viewport <= 0.0) {
204+
return status;
205+
}
206+
207+
// A partial last screen counts as a page. The tolerance stops floating point
208+
// error in the extents from adding a page.
209+
constexpr double kPageTolerance = 1e-6;
210+
const double range = std::max(extent_max - extent_min, 0.0);
211+
int64_t total_pages = std::max<int64_t>(
212+
static_cast<int64_t>(std::ceil((range + viewport) / viewport - kPageTolerance)), 1);
213+
214+
// Map scroll progress onto the pages so the end is always the last page;
215+
// `position / viewport` can't reach it unless the content is a whole number
216+
// of screens.
217+
int64_t current_page = 1;
218+
if (range > 0.0 && total_pages > 1) {
219+
const double progress = std::clamp((position - extent_min) / range, 0.0, 1.0);
220+
current_page = 1 + static_cast<int64_t>(std::round(progress * (total_pages - 1)));
221+
}
222+
status.form = AccessibilityScrollStatus::Form::kPage;
223+
status.first = std::clamp<int64_t>(current_page, 1, total_pages);
224+
status.total = total_pages;
225+
return status;
226+
}
227+
228+
// Renders `status` as the string UIAccessibilityPageScrolledNotification
229+
// expects, or nil if there is nothing accurate to say.
230+
NSString* FormatScrollStatus(const AccessibilityScrollStatus& status) {
231+
switch (status.form) {
232+
case AccessibilityScrollStatus::Form::kNone:
233+
return nil;
234+
case AccessibilityScrollStatus::Form::kRows:
235+
return LocalizedScrollStatus(kScrollRowStatusKey, kScrollRowStatusFallback, @[
236+
LocalizedCount(status.first), LocalizedCount(status.last), LocalizedCount(status.total)
237+
]);
238+
case AccessibilityScrollStatus::Form::kPage:
239+
return LocalizedScrollStatus(kScrollPageStatusKey, kScrollPageStatusFallback,
240+
@[ LocalizedCount(status.first), LocalizedCount(status.total) ]);
241+
}
242+
}
243+
244+
} // namespace
245+
87246
void AccessibilityBridge::UpdateSemantics(
88247
flutter::SemanticsNodeUpdates nodes,
89248
const flutter::CustomAccessibilityActionUpdates& actions) {
90249
BOOL layoutChanged = NO;
91-
BOOL scrollOccured = NO;
250+
// The object whose scroll position changed in this update, if any.
251+
SemanticsObject* scrolledObject = nil;
92252
BOOL needsAnnouncement = NO;
93253
for (const auto& entry : actions) {
94254
const flutter::CustomAccessibilityAction& action = entry.second;
@@ -98,7 +258,9 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
98258
const flutter::SemanticsNode& node = entry.second;
99259
SemanticsObject* object = GetOrCreateObject(node.id, nodes);
100260
layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
101-
scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
261+
if ([object nodeWillCauseScroll:&node]) {
262+
scrolledObject = object;
263+
}
102264
needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
103265
[object setSemanticsNode:&node];
104266
NSUInteger newChildCountInTraversalOrder = node.childrenInTraversalOrder.size();
@@ -218,7 +380,27 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
218380
routeName);
219381
}
220382

221-
if (layoutChanged) {
383+
if (scrolledObject) {
384+
// Checked before `layoutChanged` because a scroll moves its children,
385+
// which almost always sets `layoutChanged` too.
386+
//
387+
// A scroll spans many frames, so only announce when the status changes.
388+
AccessibilityScrollStatus status = ComputeScrollStatus(scrolledObject);
389+
if (status != last_scroll_status_) {
390+
last_scroll_status_ = status;
391+
ios_delegate_->PostAccessibilityNotification(UIAccessibilityPageScrolledNotification,
392+
FormatScrollStatus(status));
393+
}
394+
395+
// Move focus only if the scroll removed the focused element; otherwise
396+
// VoiceOver would be pulled away from what the user is reading.
397+
if (last_focused_semantics_object_id_ != kSemanticObjectIdInvalid &&
398+
![objects_ objectForKey:@(last_focused_semantics_object_id_)]) {
399+
ios_delegate_->PostAccessibilityNotification(
400+
UIAccessibilityLayoutChangedNotification,
401+
FindNextFocusableIfNecessary().nativeAccessibility);
402+
}
403+
} else if (layoutChanged) {
222404
SemanticsObject* next = FindNextFocusableIfNecessary();
223405
SemanticsObject* lastFocused = [objects_ objectForKey:@(last_focused_semantics_object_id_)];
224406
// Only specify the focus item if the new focus is different, avoiding double focuses on the
@@ -227,13 +409,6 @@ void PostAccessibilityNotification(UIAccessibilityNotifications notification,
227409
ios_delegate_->PostAccessibilityNotification(
228410
UIAccessibilityLayoutChangedNotification,
229411
(routeChanged || next != lastFocused) ? next.nativeAccessibility : NULL);
230-
} else if (scrollOccured) {
231-
// TODO(chunhtai): figure out what string to use for notification. At this
232-
// point, it is guarantee the previous focused object is still in the tree
233-
// so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
234-
ios_delegate_->PostAccessibilityNotification(
235-
UIAccessibilityPageScrolledNotification,
236-
FindNextFocusableIfNecessary().nativeAccessibility);
237412
}
238413
}
239414
}
@@ -380,6 +555,7 @@ static void ReplaceSemanticsObject(SemanticsObject* oldObject,
380555
[objects_ removeAllObjects];
381556
previous_route_id_ = 0;
382557
previous_routes_.clear();
558+
last_scroll_status_ = {};
383559
view_controller_.viewIfLoaded.accessibilityElements = nil;
384560
}
385561

0 commit comments

Comments
 (0)