Skip to content

Commit 198e1ce

Browse files
authored
Add service extension getSemanticsTree (flutter#189635)
tracking issue: flutter/devtools#9893 Add a service extension ,it will be used to visualize semantics tree in devtool. ## Pre-launch Checklist - [ ] I read the [Contributor Guide] and followed the process outlined there for submitting PRs. - [ ] I read the [AI contribution guidelines] and understand my responsibilities, or I am not using AI tools. - [ ] I read the [Tree Hygiene] wiki page, which explains my responsibilities. - [ ] I read and followed the [Flutter Style Guide], including [Features we expect every widget to implement]. - [ ] I signed the [CLA]. - [ ] I listed at least one issue that this PR fixes in the description above. tracking issue: flutter/devtools#9893 It will used to visualize semantics tree in devtool - [ ] I updated/added relevant documentation (doc comments with `///`). - [ ] I added new tests to check the change I am making, or this PR is [test-exempt]. - [ ] I followed the [breaking change policy] and added [Data Driven Fixes] where supported. - [ ] All existing and new tests are passing. If you need help, consider asking for advice on the #hackers-new channel on [Discord]. If this change needs to override an active code freeze, provide a comment explaining why. The code freeze workflow can be overridden by code reviewers. See pinned issues for any active code freezes with guidance. **Note**: The Flutter team is currently trialing the use of [Gemini Code Assist for GitHub](https://developers.google.com/gemini-code-assist/docs/review-github-code). Comments from the `gemini-code-assist` bot should not be taken as authoritative feedback from the Flutter team. If you find its comments useful you can update your code accordingly, but if you are unsure or disagree with the feedback, please feel free to wait for a Flutter team member's review for guidance on which automated comments should be addressed. <!-- Links --> [Contributor Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#overview [AI contribution guidelines]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#ai-contribution-guidelines [Tree Hygiene]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md [test-exempt]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#tests [Flutter Style Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md [Features we expect every widget to implement]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md#features-we-expect-every-widget-to-implement [CLA]: https://cla.developers.google.com/ [flutter/tests]: https://github.com/flutter/tests [breaking change policy]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#handling-breaking-changes [Discord]: https://github.com/flutter/flutter/blob/main/docs/contributing/Chat.md [Data Driven Fixes]: https://github.com/flutter/flutter/blob/main/docs/contributing/Data-driven-Fixes.md
1 parent 68bd116 commit 198e1ce

7 files changed

Lines changed: 421 additions & 3 deletions

File tree

‎packages/flutter/lib/src/semantics/semantics.dart‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1377,6 +1377,38 @@ class SemanticsData with Diagnosticable {
13771377
/// Whether [actions] contains the given action.
13781378
bool hasAction(SemanticsAction action) => (actions & action.index) != 0;
13791379

1380+
/// Returns a JSON-compatible map representation of this object.
1381+
///
1382+
/// Used by debugging tools and VM service extensions such as
1383+
/// `ext.flutter.accessibility.getSemanticsTree`.
1384+
Map<String, Object?> toJson() {
1385+
final flagsList = <String>[
1386+
for (final SemanticsFlag flag in SemanticsFlag.values)
1387+
if (hasFlag(flag)) flag.name,
1388+
];
1389+
final actionsList = <String>[
1390+
for (final SemanticsAction action in SemanticsAction.values)
1391+
if (hasAction(action)) action.name,
1392+
];
1393+
return <String, Object?>{
1394+
'label': label,
1395+
'value': value,
1396+
'hint': hint,
1397+
'tooltip': tooltip,
1398+
'increasedValue': increasedValue,
1399+
'decreasedValue': decreasedValue,
1400+
'flags': flagsList,
1401+
'actions': actionsList,
1402+
'rect': <String, double>{
1403+
'left': rect.left,
1404+
'top': rect.top,
1405+
'width': rect.width,
1406+
'height': rect.height,
1407+
},
1408+
'transform': ?transform?.storage.toList(),
1409+
};
1410+
}
1411+
13801412
@override
13811413
String toStringShort() => objectRuntimeType(this, 'SemanticsData');
13821414

@@ -4566,6 +4598,29 @@ class SemanticsNode with DiagnosticableTreeMixin {
45664598
DebugSemanticsDumpOrder.traversalOrder => _childrenInTraversalOrder(),
45674599
};
45684600
}
4601+
4602+
/// Returns a JSON-compatible map representation of this node and its children
4603+
/// identifiers.
4604+
///
4605+
/// Used by debugging tools and VM service extensions such as
4606+
/// `ext.flutter.accessibility.getSemanticsTree`.
4607+
Map<String, Object?> toJson() {
4608+
final SemanticsData data = getSemanticsData();
4609+
final List<SemanticsNode> traversalChildren = debugListChildrenInOrder(
4610+
DebugSemanticsDumpOrder.traversalOrder,
4611+
);
4612+
final List<SemanticsNode> hitTestChildren = debugListChildrenInOrder(
4613+
DebugSemanticsDumpOrder.inverseHitTest,
4614+
);
4615+
return <String, Object?>{
4616+
'id': id,
4617+
...data.toJson(),
4618+
'childrenInTraversalOrder': <int>[
4619+
for (final SemanticsNode child in traversalChildren) child.id,
4620+
],
4621+
'childrenInHitTestOrder': <int>[for (final SemanticsNode child in hitTestChildren) child.id],
4622+
};
4623+
}
45694624
}
45704625

45714626
/// An edge of a box, such as top, bottom, left or right, used to compute
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'package:flutter/foundation.dart';
6+
import 'package:flutter/rendering.dart';
7+
8+
import 'service_extensions.dart';
9+
10+
/// Service that handles accessibility and semantics inspection.
11+
class AccessibilityInspector {
12+
AccessibilityInspector._();
13+
14+
/// The active [AccessibilityInspector] instance.
15+
static final AccessibilityInspector instance = AccessibilityInspector._();
16+
17+
SemanticsHandle? _semanticsHandle;
18+
19+
/// Registers accessibility-related VM service extensions.
20+
void initServiceExtensions(
21+
void Function({required String name, required ServiceExtensionCallback callback})
22+
registerServiceExtension,
23+
) {
24+
registerServiceExtension(
25+
name: AccessibilityServiceExtensions.getSemanticsTree.extensionName,
26+
callback: _getSemanticsTree,
27+
);
28+
registerServiceExtension(
29+
name: AccessibilityServiceExtensions.enableSemantics.extensionName,
30+
callback: _enableSemantics,
31+
);
32+
registerServiceExtension(
33+
name: AccessibilityServiceExtensions.disposeSemantics.extensionName,
34+
callback: _disposeSemantics,
35+
);
36+
}
37+
38+
/// Reset the helper state (primarily used in tests).
39+
@visibleForTesting
40+
void resetAllState() {
41+
_semanticsHandle?.dispose();
42+
_semanticsHandle = null;
43+
}
44+
45+
Future<Map<String, Object?>> _enableSemantics(Map<String, String> parameters) async {
46+
_semanticsHandle ??= SemanticsBinding.instance.ensureSemantics();
47+
return const <String, Object?>{};
48+
}
49+
50+
Future<Map<String, Object?>> _disposeSemantics(Map<String, String> parameters) async {
51+
resetAllState();
52+
return const <String, Object?>{};
53+
}
54+
55+
Future<Map<String, Object?>> _getSemanticsTree(Map<String, String> parameters) async {
56+
if (!SemanticsBinding.instance.semanticsEnabled) {
57+
return <String, Object?>{'error': 'Semantics not enabled.'};
58+
}
59+
final PipelineOwner? pipelineOwner = _findPipelineOwner();
60+
final SemanticsOwner? semanticsOwner = pipelineOwner?.semanticsOwner;
61+
if (semanticsOwner == null) {
62+
return <String, Object?>{'error': 'No PipelineOwner with SemanticsOwner found'};
63+
}
64+
final SemanticsNode? root = semanticsOwner.rootSemanticsNode;
65+
if (root == null) {
66+
RendererBinding.instance.ensureVisualUpdate();
67+
return <String, Object?>{'error': 'rootSemanticsNode is null', 'needsFrame': true};
68+
}
69+
70+
final nodes = <String, Object?>{};
71+
final visited = <int>{};
72+
final queue = <SemanticsNode>[root];
73+
while (queue.isNotEmpty) {
74+
final SemanticsNode node = queue.removeLast();
75+
if (!visited.add(node.id)) {
76+
continue;
77+
}
78+
nodes[node.id.toString()] = node.toJson();
79+
for (final SemanticsNode child in node.debugListChildrenInOrder(
80+
DebugSemanticsDumpOrder.traversalOrder,
81+
)) {
82+
if (!visited.contains(child.id)) {
83+
queue.add(child);
84+
}
85+
}
86+
for (final SemanticsNode child in node.debugListChildrenInOrder(
87+
DebugSemanticsDumpOrder.inverseHitTest,
88+
)) {
89+
if (!visited.contains(child.id)) {
90+
queue.add(child);
91+
}
92+
}
93+
}
94+
95+
return <String, Object?>{'data': nodes};
96+
}
97+
98+
// TODO(hannahjin): This returns the first SemanticsOwner of any RenderView.
99+
// This getSemanticsTree feature is used in DevTools, which currently only supports
100+
// single-view inspection. Add multi-view support when DevTools needs it.
101+
PipelineOwner? _findPipelineOwner() {
102+
for (final RenderView renderView in RendererBinding.instance.renderViews) {
103+
if (renderView.owner?.semanticsOwner != null) {
104+
return renderView.owner;
105+
}
106+
}
107+
final PipelineOwner deprecatedOwner = RendererBinding.instance.pipelineOwner;
108+
if (deprecatedOwner.semanticsOwner != null) {
109+
return deprecatedOwner;
110+
}
111+
return null;
112+
}
113+
}

‎packages/flutter/lib/src/widgets/binding.dart‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import 'package:flutter/services.dart';
3838
import '../foundation/_features.dart';
3939
import '_accessibility_evaluations.dart';
4040
import '_window.dart';
41+
import 'accessibility_inspector.dart';
4142
import 'app.dart';
4243
import 'debug.dart';
4344
import 'focus_manager.dart';
@@ -807,7 +808,7 @@ mixin WidgetsBinding
807808
return _forceRebuild();
808809
},
809810
);
810-
811+
AccessibilityInspector.instance.initServiceExtensions(registerServiceExtension);
811812
WidgetInspectorService.instance.initServiceExtensions(registerServiceExtension);
812813

813814
return true;

‎packages/flutter/lib/src/widgets/service_extensions.dart‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
/// @docImport 'package:flutter/foundation.dart';
77
/// @docImport 'package:flutter/rendering.dart';
88
///
9+
/// @docImport 'accessibility_inspector.dart';
910
/// @docImport 'app.dart';
1011
/// @docImport 'binding.dart';
1112
/// @docImport 'debug.dart';
@@ -537,3 +538,46 @@ enum WidgetInspectorServiceExtensions {
537538
/// extension is registered.
538539
setFlexProperties,
539540
}
541+
542+
/// Service extension constants for accessibility and semantics.
543+
///
544+
/// These constants will be used when registering service extensions in the
545+
/// framework, and they will also be used by tools and services that call these
546+
/// service extensions.
547+
///
548+
/// The String value for each of these extension names should be accessed by
549+
/// calling the [extensionName] property on the enum value.
550+
enum AccessibilityServiceExtensions {
551+
/// Name of service extension that, when called, returns the JSON serialized
552+
/// semantics tree.
553+
///
554+
/// This extension should only be called after semantics has been enabled
555+
/// (for example, by calling [enableSemantics]).
556+
///
557+
/// See also:
558+
///
559+
/// * [AccessibilityInspector.initServiceExtensions], where the service
560+
/// extension is registered.
561+
getSemanticsTree,
562+
563+
/// Name of service extension that, when called, enables semantics in the app
564+
/// by creating a [SemanticsHandle].
565+
///
566+
/// See also:
567+
///
568+
/// * [AccessibilityInspector.initServiceExtensions], where the service
569+
/// extension is registered.
570+
enableSemantics,
571+
572+
/// Name of service extension that, when called, disposes the [SemanticsHandle]
573+
/// created by [enableSemantics].
574+
///
575+
/// See also:
576+
///
577+
/// * [AccessibilityInspector.initServiceExtensions], where the service
578+
/// extension is registered.
579+
disposeSemantics;
580+
581+
/// The full name of the service extension, including the `accessibility.` prefix.
582+
String get extensionName => 'accessibility.$name';
583+
}

‎packages/flutter/test/foundation/service_extensions_test.dart‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,14 @@ void main() {
194194
hasLength(widgetInspectorExtensionCount),
195195
);
196196

197+
// See accessibility_inspector_test.dart for tests of the ext.flutter.accessibility
198+
// service extensions included in this count.
199+
const accessibilityExtensionCount = 3;
200+
expect(
201+
binding.extensions.keys.where((String name) => name.startsWith('accessibility.')),
202+
hasLength(accessibilityExtensionCount),
203+
);
204+
197205
// The following service extensions are disabled in web:
198206
// 1. exit
199207
// 2. showPerformanceOverlay
@@ -202,7 +210,8 @@ void main() {
202210
// The expected number of registered service extensions in the Flutter
203211
// framework, excluding any that are for the widget inspector (see
204212
// widget_inspector_test.dart for tests of the ext.flutter.inspector service
205-
// extensions). Any test counted here must be tested in this file!
213+
// extensions) or accessibility inspector (see accessibility_inspector_test.dart).
214+
// Any test counted here must be tested in this file!
206215
const serviceExtensionCount = 31;
207216

208217
// The tests are in the widgets/accessibility_evaluations_service_extension_test.dart
@@ -211,7 +220,10 @@ void main() {
211220

212221
expect(
213222
binding.extensions.length,
214-
serviceExtensionCount + widgetInspectorExtensionCount - disabledExtensions,
223+
serviceExtensionCount +
224+
widgetInspectorExtensionCount +
225+
accessibilityExtensionCount -
226+
disabledExtensions,
215227
);
216228
expect(testedExtensions, hasLength(serviceExtensionCount));
217229

‎packages/flutter/test/semantics/semantics_test.dart‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1342,6 +1342,47 @@ void main() {
13421342
expect(label, 'Emoji: 😀🎉 Math: ∑∆π Currency: €£¥');
13431343
});
13441344
});
1345+
1346+
test('SemanticsData.toJson and SemanticsNode.toJson generate expected maps', () {
1347+
final node = SemanticsNode()
1348+
..rect = const Rect.fromLTRB(0.0, 0.0, 100.0, 50.0)
1349+
..updateWith(
1350+
config: SemanticsConfiguration()
1351+
..label = 'Test Label'
1352+
..textDirection = TextDirection.ltr
1353+
..value = 'Test Value'
1354+
..hint = 'Test Hint'
1355+
..isButton = true,
1356+
);
1357+
1358+
final SemanticsData data = node.getSemanticsData();
1359+
final Map<String, Object?> dataJsonMap = data.toJson();
1360+
expect(dataJsonMap['label'], 'Test Label');
1361+
expect(dataJsonMap['value'], 'Test Value');
1362+
expect(dataJsonMap['hint'], 'Test Hint');
1363+
expect(dataJsonMap['flags'], contains('isButton'));
1364+
expect(dataJsonMap['rect'], <String, double>{
1365+
'left': 0.0,
1366+
'top': 0.0,
1367+
'width': 100.0,
1368+
'height': 50.0,
1369+
});
1370+
1371+
final Map<String, Object?> nodeJsonMap = node.toJson();
1372+
expect(nodeJsonMap['id'], node.id);
1373+
expect(nodeJsonMap['label'], 'Test Label');
1374+
expect(nodeJsonMap['value'], 'Test Value');
1375+
expect(nodeJsonMap['hint'], 'Test Hint');
1376+
expect(nodeJsonMap['flags'], contains('isButton'));
1377+
expect(nodeJsonMap['rect'], <String, double>{
1378+
'left': 0.0,
1379+
'top': 0.0,
1380+
'width': 100.0,
1381+
'height': 50.0,
1382+
});
1383+
expect(nodeJsonMap['childrenInTraversalOrder'], isEmpty);
1384+
expect(nodeJsonMap['childrenInHitTestOrder'], isEmpty);
1385+
});
13451386
}
13461387

13471388
class TestRender extends RenderProxyBox {

0 commit comments

Comments
 (0)