Skip to content

webdriver: Add touch support for all platforms #41067

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yezhizhen merged 4 commits into from
Dec 5, 2025
Merged

webdriver: Add touch support for all platforms #41067

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
039f834
Android/OHOS touch webdriver support
yezhizhen Dec 5, 2025
aeb566a
Fix OHOS/Android build
yezhizhen Dec 5, 2025
f6f9671
Update WPT
yezhizhen Dec 5, 2025
e0d99b7
Update comment+ pointerid
yezhizhen Dec 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 72 additions & 30 deletions components/webdriver_server/actions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@ use std::time::{Duration, Instant};
use base::id::BrowsingContextId;
use crossbeam_channel::Select;
use embedder_traits::{
InputEvent, KeyboardEvent, MouseButtonAction, MouseButtonEvent, MouseMoveEvent,
WebDriverCommandMsg, WebDriverScriptCommand, WebViewPoint, WheelDelta, WheelEvent, WheelMode,
InputEvent, KeyboardEvent, MouseButtonAction, MouseButtonEvent, MouseMoveEvent, TouchEvent,
TouchEventType, TouchId, WebDriverCommandMsg, WebDriverScriptCommand, WebViewPoint, WheelDelta,
WheelEvent, WheelMode,
};
use euclid::Point2D;
use ipc_channel::ipc;
Expand Down Expand Up @@ -62,9 +63,6 @@ pub(crate) enum InputSourceState {
}

/// <https://w3c.github.io/webdriver/#dfn-pointer-input-source>
/// TODO: subtype is used for <https://w3c.github.io/webdriver/#dfn-get-a-pointer-id>
/// Need to add pointer-id to the following struct
#[expect(dead_code)]
pub(crate) struct PointerInputState {
subtype: PointerType,
pressed: FxHashSet<u64>,
Expand All @@ -75,13 +73,18 @@ pub(crate) struct PointerInputState {

impl PointerInputState {
/// <https://w3c.github.io/webdriver/#dfn-create-a-pointer-input-source>
pub(crate) fn new(subtype: PointerType, pointer_ids: FxHashSet<u32>) -> PointerInputState {
pub(crate) fn new(
subtype: PointerType,
pointer_ids: FxHashSet<u32>,
x: f64,
y: f64,
) -> PointerInputState {
PointerInputState {
subtype,
pressed: FxHashSet::default(),
pointer_id: Self::get_pointer_id(subtype, pointer_ids),
x: 0.0,
y: 0.0,
x,
y,
}
}

Expand Down Expand Up @@ -347,18 +350,27 @@ impl Handler {
return;
}

let PointerInputState { x, y, .. } = *pointer_input_state;
let PointerInputState { x, y, subtype, .. } = *pointer_input_state;
// Step 6. Add button to the set corresponding to source's pressed property
pointer_input_state.pressed.insert(action.button);
// Step 7 - 15: Variable namings already done.

// Step 16. Perform implementation-specific action dispatch steps
// TODO: We have not considered pen/touch pointer type
self.send_blocking_input_event_to_embedder(InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Down,
action.button.into(),
WebViewPoint::Page(Point2D::new(x as f32, y as f32)),
)));
// TODO: We have not considered pen pointer type
let point = WebViewPoint::Page(Point2D::new(x as f32, y as f32));
let input_event = match subtype {
PointerType::Mouse => InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Down,
action.button.into(),
point,
)),
PointerType::Pen | PointerType::Touch => InputEvent::Touch(TouchEvent::new(
TouchEventType::Down,
TouchId(pointer_input_state.pointer_id as i32),
point,
)),
};
self.send_blocking_input_event_to_embedder(input_event);

// Step 17. Return success with data null.
}
Expand All @@ -373,7 +385,13 @@ impl Handler {

// Step 6. Remove button from the set corresponding to source's pressed property,
pointer_input_state.pressed.remove(&action.button);
let PointerInputState { x, y, .. } = *pointer_input_state;
let PointerInputState {
x,
y,
subtype,
pointer_id,
..
} = *pointer_input_state;

// Remove matching pointerUp(must be unique) from `[input_cancel_list]` due to bugs in spec
// See https://github.com/w3c/webdriver/issues/1905 &&
Expand All @@ -390,11 +408,20 @@ impl Handler {
}

// Step 7. Perform implementation-specific action dispatch steps
self.send_blocking_input_event_to_embedder(InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Up,
action.button.into(),
WebViewPoint::Page(Point2D::new(x as f32, y as f32)),
)));
let point = WebViewPoint::Page(Point2D::new(x as f32, y as f32));
let input_event = match subtype {
PointerType::Mouse => InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Up,
action.button.into(),
point,
)),
PointerType::Pen | PointerType::Touch => InputEvent::Touch(TouchEvent::new(
TouchEventType::Up,
TouchId(pointer_id as i32),
point,
)),
};
self.send_blocking_input_event_to_embedder(input_event);

// Step 8. Return success with data null.
}
Expand Down Expand Up @@ -449,7 +476,7 @@ impl Handler {
// Step 9 - 18
// Perform a pointer move with arguments source, global key state, duration, start x, start y,
// x, y, width, height, pressure, tangentialPressure, tiltX, tiltY, twist, altitudeAngle, azimuthAngle.
// TODO: We have not considered pen/touch pointer type
// TODO: We have not considered pen pointer type
self.perform_pointer_move(source_id, duration, start_x, start_y, x, y, tick_start);

// Step 19. Return success with data null.
Expand Down Expand Up @@ -500,18 +527,29 @@ impl Handler {
};

// Step 5 - 6: Let current x/y equal the x/y property of input state.
let (current_x, current_y) = {
let pointer_input_state = self.get_pointer_input_state(source_id);
(pointer_input_state.x, pointer_input_state.y)
};
let PointerInputState {
x: current_x,
y: current_y,
subtype,
pointer_id,
..
} = *self.get_pointer_input_state(source_id);

// Step 7. If x != current x or y != current y, run the following steps:
// FIXME: Actually "last" should not be checked here based on spec.
if x != current_x || y != current_y || last {
// Step 7.1. Let buttons be equal to input state's buttons property.
// Step 7.2. Perform implementation-specific action dispatch steps
let point = WebViewPoint::Page(Point2D::new(x as f32, y as f32));
let input_event = InputEvent::MouseMove(MouseMoveEvent::new(point));

let input_event = match subtype {
PointerType::Mouse => InputEvent::MouseMove(MouseMoveEvent::new(point)),
PointerType::Pen | PointerType::Touch => InputEvent::Touch(TouchEvent::new(
TouchEventType::Move,
TouchId(pointer_id as i32),
point,
)),
};
if last {
self.send_blocking_input_event_to_embedder(input_event);
} else {
Expand Down Expand Up @@ -730,7 +768,7 @@ impl Handler {
}

/// <https://w3c.github.io/webdriver/#dfn-get-coordinates-relative-to-an-origin>
fn get_origin_relative_coordinates(
pub(crate) fn get_origin_relative_coordinates(
&self,
origin: &PointerOrigin,
x_offset: f64,
Expand Down Expand Up @@ -840,15 +878,19 @@ impl Handler {
key_actions.into_iter().map(ActionItem::Key).collect()
},
ActionsType::Pointer {
parameters: _,
parameters,
actions: pointer_actions,
} => {
let pointer_ids = self.session().unwrap().pointer_ids();
// Get or create a pointer input source with subtype, and other iterms
// set to default values.
self.input_state_table_mut()
.entry(id)
.or_insert(InputSourceState::Pointer(PointerInputState::new(
PointerType::Mouse,
parameters.pointer_type,
Comment on lines 889 to +890
Copy link
Copy Markdown
Member Author

@yezhizhen yezhizhen Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are no longer ignoring requested pointer type.
This allows us to dispatch touch actions correctly for Perform Actions.

pointer_ids,
0.0,
0.0,
)));
pointer_actions
.into_iter()
Expand Down
86 changes: 83 additions & 3 deletions components/webdriver_server/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,12 @@ use servo_url::ServoUrl;
use style_traits::CSSPixel;
use time::OffsetDateTime;
use uuid::Uuid;
#[cfg(not(any(target_env = "ohos", target_os = "android")))]
use webdriver::actions::PointerMoveAction;
use webdriver::actions::{
ActionSequence, ActionsType, KeyAction, KeyActionItem, KeyDownAction, KeyUpAction,
PointerAction, PointerActionItem, PointerActionParameters, PointerDownAction,
PointerMoveAction, PointerOrigin, PointerType, PointerUpAction,
PointerAction, PointerActionItem, PointerActionParameters, PointerDownAction, PointerOrigin,
PointerType, PointerUpAction,
};
use webdriver::capabilities::CapabilitiesMatching;
use webdriver::command::{
Expand Down Expand Up @@ -2281,14 +2283,92 @@ impl Handler {

/// <https://w3c.github.io/webdriver/#element-click>
/// Step 8 for elements other than <option>
/// There is currently no spec for touchscreen webdriver support.
/// There is an ongoing discussion in W3C:
/// <https://github.com/w3c/webdriver/issues/1925>
#[cfg(any(target_env = "ohos", target_os = "android"))]
Copy link
Copy Markdown
Member

@jschwe jschwe Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer if we did a runtime check for device type, since ohos / android devices can be run in desktop mode.
I don't think we need ot change this now, would just prefer if we didn't add to many cfgs like this when it shouldn't be a compiletime option.

Copy link
Copy Markdown
Member Author

@yezhizhen yezhizhen Dec 5, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks! Will do when I got time.

Another way is to process "user_agent" of webdriver session to determine at runtime.

Copy link
Copy Markdown
Member Author

@yezhizhen yezhizhen Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TODO later: When we have touch simulation for Desktop, "Element click" should also dispatch pointer touch.

We always have a plan to do this in runtime.

fn perform_element_click(&mut self, element: String) -> WebDriverResult<WebDriverResponse> {
// Step 8.1 - 8.4: Create UUID, create input source "pointer".
let id = Uuid::new_v4().to_string();

let pointer_ids = self.session()?.pointer_ids();
let (x, y) = self
.get_origin_relative_coordinates(
&PointerOrigin::Element(WebElement(element.clone())),
0.0,
0.0,
&id,
)
.map_err(|err| WebDriverError::new(err, ""))?;

// Difference with Desktop: Create an input source with coordinates directly at the
// element centre.
self.session_mut()?.input_state_table.insert(
id.clone(),
InputSourceState::Pointer(PointerInputState::new(PointerType::Mouse, pointer_ids)),
InputSourceState::Pointer(PointerInputState::new(
PointerType::Touch,
pointer_ids,
x,
y,
)),
);

// Step 8.11. Construct pointer down action.
// Step 8.12. Set a property button to 0 on pointer down action.
let pointer_down_action = PointerDownAction {
button: i16::from(MouseButton::Left) as u64,
..Default::default()
};

// Step 8.13. Construct pointer up action.
// Step 8.14. Set a property button to 0 on pointer up action.
let pointer_up_action = PointerUpAction {
button: i16::from(MouseButton::Left) as u64,
..Default::default()
};

// Difference with Desktop: We only need pointerdown and pointerup for touchscreen.
let action_sequence = ActionSequence {
id: id.clone(),
actions: ActionsType::Pointer {
parameters: PointerActionParameters {
pointer_type: PointerType::Touch,
},
actions: vec![
PointerActionItem::Pointer(PointerAction::Down(pointer_down_action)),
PointerActionItem::Pointer(PointerAction::Up(pointer_up_action)),
],
},
};

// Step 8.16. Dispatch a list of actions with session's current browsing context
let actions_by_tick = self.extract_an_action_sequence(vec![action_sequence]);
if let Err(e) = self.dispatch_actions(actions_by_tick, self.browsing_context_id()?) {
log::error!("handle_element_click: dispatch_actions failed: {:?}", e);
}

// Step 8.17 Remove an input source with input state and input id.
self.session_mut()?.input_state_table.remove(&id);

Ok(WebDriverResponse::Void)
}

/// <https://w3c.github.io/webdriver/#element-click>
/// Step 8 for elements other than <option>,
#[cfg(not(any(target_env = "ohos", target_os = "android")))]
fn perform_element_click(&mut self, element: String) -> WebDriverResult<WebDriverResponse> {
// Step 8.1 - 8.4: Create UUID, create input source "pointer".
let id = Uuid::new_v4().to_string();

let pointer_ids = self.session()?.pointer_ids();
self.session_mut()?.input_state_table.insert(
id.clone(),
InputSourceState::Pointer(PointerInputState::new(
PointerType::Mouse,
pointer_ids,
0.0,
0.0,
)),
);

// Step 8.7. Construct a pointer move action.
Expand Down
1 change: 1 addition & 0 deletions .../events/non-cancelable-when-passive/non-passive-touchmove-event-listener-on-body.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchmove-event-listener-on-body.html]
expected: CRASH
[non-passive touchmove event listener on body]
expected: FAIL
1 change: 1 addition & 0 deletions ...m/events/non-cancelable-when-passive/non-passive-touchmove-event-listener-on-div.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchmove-event-listener-on-div.html]
expected: CRASH
[non-passive touchmove event listener on div]
expected: FAIL
1 change: 1 addition & 0 deletions ...nts/non-cancelable-when-passive/non-passive-touchmove-event-listener-on-document.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchmove-event-listener-on-document.html]
expected: CRASH
[non-passive touchmove event listener on document]
expected: FAIL
1 change: 1 addition & 0 deletions .../events/non-cancelable-when-passive/non-passive-touchmove-event-listener-on-root.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchmove-event-listener-on-root.html]
expected: CRASH
[non-passive touchmove event listener on root]
expected: FAIL
1 change: 1 addition & 0 deletions ...vents/non-cancelable-when-passive/non-passive-touchmove-event-listener-on-window.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchmove-event-listener-on-window.html]
expected: CRASH
[non-passive-touchmove-event-listener-on-window]
expected: FAIL
1 change: 1 addition & 0 deletions ...events/non-cancelable-when-passive/non-passive-touchstart-event-listener-on-body.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchstart-event-listener-on-body.html]
expected: CRASH
[non-passive touchstart event listener on body]
expected: FAIL
1 change: 1 addition & 0 deletions .../events/non-cancelable-when-passive/non-passive-touchstart-event-listener-on-div.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchstart-event-listener-on-div.html]
expected: CRASH
[non-passive touchstart event listener on div]
expected: FAIL
1 change: 1 addition & 0 deletions ...ts/non-cancelable-when-passive/non-passive-touchstart-event-listener-on-document.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchstart-event-listener-on-document.html]
expected: CRASH
[non-passive touchstart event listener on document]
expected: FAIL
1 change: 1 addition & 0 deletions ...events/non-cancelable-when-passive/non-passive-touchstart-event-listener-on-root.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchstart-event-listener-on-root.html]
expected: CRASH
[non-passive touchstart event listener on root]
expected: FAIL
1 change: 1 addition & 0 deletions ...ents/non-cancelable-when-passive/non-passive-touchstart-event-listener-on-window.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[non-passive-touchstart-event-listener-on-window.html]
expected: CRASH
[non-passive touchstart event listener on window]
expected: FAIL
1 change: 1 addition & 0 deletions .../dom/events/non-cancelable-when-passive/passive-touchmove-event-listener-on-body.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[passive-touchmove-event-listener-on-body.html]
expected: CRASH
[passive touchmove event listener on body]
expected: FAIL
1 change: 1 addition & 0 deletions ...a/dom/events/non-cancelable-when-passive/passive-touchmove-event-listener-on-div.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[passive-touchmove-event-listener-on-div.html]
expected: CRASH
[passive touchmove event listener on div]
expected: FAIL
1 change: 1 addition & 0 deletions .../events/non-cancelable-when-passive/passive-touchmove-event-listener-on-document.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[passive-touchmove-event-listener-on-document.html]
expected: CRASH
[passive touchmove event listener on document]
expected: FAIL
1 change: 1 addition & 0 deletions .../dom/events/non-cancelable-when-passive/passive-touchmove-event-listener-on-root.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[passive-touchmove-event-listener-on-root.html]
expected: CRASH
[passive touchmove event listener on root]
expected: FAIL
1 change: 1 addition & 0 deletions ...om/events/non-cancelable-when-passive/passive-touchmove-event-listener-on-window.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[passive-touchmove-event-listener-on-window.html]
expected: CRASH
[passive touchmove event listener on window]
expected: FAIL
1 change: 1 addition & 0 deletions ...dom/events/non-cancelable-when-passive/passive-touchstart-event-listener-on-body.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[passive-touchstart-event-listener-on-body.html]
expected: CRASH
[passive touchstart event listener on body]
expected: FAIL
1 change: 1 addition & 0 deletions .../dom/events/non-cancelable-when-passive/passive-touchstart-event-listener-on-div.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[passive-touchstart-event-listener-on-div.html]
expected: CRASH
[passive touchstart event listener on div]
expected: FAIL
1 change: 1 addition & 0 deletions ...events/non-cancelable-when-passive/passive-touchstart-event-listener-on-document.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[passive-touchstart-event-listener-on-document.html]
expected: CRASH
[passive touchstart event listener on document]
expected: FAIL
1 change: 1 addition & 0 deletions ...dom/events/non-cancelable-when-passive/passive-touchstart-event-listener-on-root.html.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[passive-touchstart-event-listener-on-root.html]
expected: CRASH
[passive touchstart event listener on root]
expected: FAIL
Loading