vi-mode: vi-compliant word movements - #12269

diff --git a/src/reader/mod.rs b/src/reader/mod.rs
index afd1c30be3..551ba9a254 100644
--- a/src/reader/mod.rs
+++ b/src/reader/mod.rs
@@ -4,5 +4,6 @@
 pub mod iothreads;
 #[allow(clippy::module_inception)]
 pub mod reader;
+mod word_motion;
 
 pub use reader::*;
diff --git a/src/reader/reader.rs b/src/reader/reader.rs
index 23fcbe2328..498502728c 100644
--- a/src/reader/reader.rs
+++ b/src/reader/reader.rs
@@ -49,6 +49,7 @@
 
 use super::history_search::{ReaderHistorySearch, SearchMode, smartcase_flags};
 use super::iothreads::{self, Debouncers};
+use super::word_motion::{MoveWordDir, MoveWordStyle, move_word_state_machine};
 use crate::abbrs::abbrs_match;
 use crate::ast::{self, Kind, is_same_node};
 use crate::builtins::shared::ErrorCode;
@@ -143,8 +144,7 @@
 use crate::tokenizer::quote_end;
 use crate::tokenizer::variable_assignment_equals_pos;
 use crate::tokenizer::{
-    MoveWordDir, MoveWordStateMachine, MoveWordStyle, TOK_ACCEPT_UNFINISHED, TOK_SHOW_COMMENTS,
-    TokenType, Tokenizer, tok_command,
+    TOK_ACCEPT_UNFINISHED, TOK_SHOW_COMMENTS, TokenType, Tokenizer, tok_command,
 };
 use crate::tty_handoff::SCROLL_CONTENT_UP_TERMINFO_CODE;
 use crate::tty_handoff::XTGETTCAP_QUERY_OS_NAME;
@@ -2207,7 +2207,7 @@
         }
 
         // When moving left, a value of 1 means the character at index 0.
-        let mut state = MoveWordStateMachine::new(style, state_machine_dir);
+        let mut sm = move_word_state_machine(style, state_machine_dir);
         let start_buff_pos = el.position();
 
         let mut buff_pos = el.position();
@@ -2231,7 +2231,7 @@
             } else {
                 buff_pos.wrapping_sub(1)
             };
-            let consumed = state.consume_char(el.text(), char_pos);
+            let consumed = sm(el.text(), char_pos);
             if consumed {
                 buff_pos = if move_right {
                     buff_pos + 1
@@ -5607,11 +5607,11 @@
                 } else {
                     MoveWordDir::Right
                 };
-                let mut state = MoveWordStateMachine::new(style, state_machine_dir);
+                let mut sm = move_word_state_machine(style, state_machine_dir);
                 let have = search_string_range.len();
                 let mut want = have;
                 while want < autosuggestion_text.len() {
-                    let consumed = state.consume_char(autosuggestion_text, want);
+                    let consumed = sm(autosuggestion_text, want);
                     if consumed {
                         want += 1;
                     } else {
diff --git a/src/reader/word_motion.rs b/src/reader/word_motion.rs
new file mode 100644
index 0000000000..4aa91e2cee
--- /dev/null
+++ b/src/reader/word_motion.rs
@@ -0,0 +1,438 @@
+use std::ops::ControlFlow;
+
+use crate::prelude::*;
+use crate::tokenizer::tok_is_string_character;
+use fish_wchar::word_char::{WordCharClass, is_blank};
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum MoveWordStyle {
+    /// stop at punctuation
+    Punctuation,
+    /// stops at path components
+    PathComponents,
+    /// stops at whitespace
+    Whitespace,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum MoveWordDir {
+    Left,
+    Right,
+}
+
+type StateMachine = dyn FnMut(&wstr, usize) -> bool;
+
+pub fn move_word_state_machine(style: MoveWordStyle, direction: MoveWordDir) -> Box<StateMachine> {
+    use MoveWordStyle as Style;
+    match style {
+        Style::Punctuation => Box::new(state_machine(move |state, text, idx| {
+            let c = text.as_char_slice()[idx];
+            consume_char_word_movement(state, false, direction, c)
+        })),
+        Style::PathComponents => match direction {
+            MoveWordDir::Left => Box::new(state_machine(move |state, text, idx| {
+                let ControlFlow::Continue(transition) = path_component_state_transition(text, idx)
+                else {
+                    return true;
+                };
+                consume_char_path_component_backward(state, transition)
+            })),
+            MoveWordDir::Right => Box::new(state_machine(move |state, text, idx| {
+                let ControlFlow::Continue(transition) = path_component_state_transition(text, idx)
+                else {
+                    return true;
+                };
+                consume_char_path_component_forward(state, transition)
+            })),
+        },
+        Style::Whitespace => Box::new(state_machine(move |state, text, idx| {
+            let c = text.as_char_slice()[idx];
+            consume_char_word_movement(state, true, direction, c)
+        })),
+    }
+}
+
+enum State<T> {
+    Initial,
+    Live(T),
+    Final,
+}
+impl<T> State<T> {
+    fn is_final(&self) -> bool {
+        matches!(self, State::Final)
+    }
+}
+
+// TODO(MSRV=?) Define trait alias to extract return type.
+fn state_machine<T>(
+    mut consumer: impl FnMut(&mut State<T>, &wstr, usize) -> bool,
+) -> impl FnMut(&wstr, usize) -> bool {
+    let mut state = State::Initial;
+    move |text: &wstr, idx: usize| {
+        assert!(!state.is_final());
+        consumer(&mut state, text, idx)
+    }
+}
+
+struct WordMovementState {
+    last_char_class: WordCharClass,
+}
+fn consume_char_word_movement(
+    state: &mut State<WordMovementState>,
+    is_bigword: bool,
+    direction: MoveWordDir,
+    c: char,
+) -> bool {
+    let cur_class = if is_bigword {
+        if c == '\n' {
+            WordCharClass::Newline
+        } else if is_blank(c) {
+            WordCharClass::Blank
+        } else {
+            WordCharClass::Word
+        }
+    } else {
+        WordCharClass::from_char(c)
+    };
+    enum Transition {
+        Blank,
+        Newline,
+        SameClass,
+        DifferentClass,
+    }
+    use Transition as T;
+    let transition = if cur_class == WordCharClass::Blank {
+        T::Blank
+    } else if cur_class == WordCharClass::Newline {
+        T::Newline
+    } else if matches!(
+        *state, State::Live(WordMovementState { last_char_class }) if last_char_class == cur_class
+    ) {
+        T::SameClass
+    } else {
+        T::DifferentClass
+    };
+
+    use State as S;
+    let finalize = match *state {
+        S::Initial => false,
+        S::Live(WordMovementState { last_char_class }) => match direction {
+            MoveWordDir::Left => match last_char_class {
+                WordCharClass::Blank => false,
+                WordCharClass::Newline => matches!(transition, T::Newline),
+                _ => matches!(transition, T::Blank | T::Newline | T::DifferentClass),
+            },
+            MoveWordDir::Right => match last_char_class {
+                WordCharClass::Blank => {
+                    matches!(transition, T::SameClass | T::DifferentClass)
+                }
+                WordCharClass::Newline => {
+                    matches!(transition, T::Newline | T::SameClass | T::DifferentClass)
+                }
+                _ => matches!(transition, T::DifferentClass),
+            },
+        },
+        S::Final => unreachable!(),
+    };
+    if finalize {
+        return to_final(state);
+    }
+
+    *state = S::Live(WordMovementState {
+        last_char_class: cur_class,
+    });
+    true
+}
+
+enum PathComponentTransition {
+    Blank,
+    PathComponent,
+    Punctuation,
+}
+
+fn path_component_state_transition(
+    text: &wstr,
+    idx: usize,
+) -> ControlFlow<(), PathComponentTransition> {
+    let c = text.as_char_slice()[idx];
+    let mut escaped: bool = false;
+    for j in (0..idx).rev() {
+        if text.as_char_slice()[j] == '\\' {
+            escaped = !escaped;
+        } else {
+            break;
+        }
+    }
+
+    if c == '\\' && !escaped {
+        return ControlFlow::Break(());
+    };
+
+    use PathComponentTransition as T;
+    ControlFlow::Continue(if is_blank(c) && !escaped {
+        T::Blank
+    } else if is_path_component_character(c) || (is_blank(c) && escaped) {
+        T::PathComponent
+    } else {
+        T::Punctuation
+    })
+}
+
+fn is_path_component_character(c: char) -> bool {
+    tok_is_string_character(c, None) && !L!("/={,}'\":@#").as_char_slice().contains(&c)
+}
+
+fn to_final<T>(state: &mut State<T>) -> bool {
+    *state = State::Final;
+    false
+}
+
+enum PathComponentForwardState {
+    PathComponent,
+    Punctuation,
+    Blank,
+}
+fn consume_char_path_component_forward(
+    state: &mut State<PathComponentForwardState>,
+    transition: PathComponentTransition,
+) -> bool {
+    // Forward path component movement: skip current homogeneous component, stop at next.
+    // Each component type (word, slash, punctuation, whitespace) is a unit.
+
+    use {PathComponentForwardState as LS, PathComponentTransition as T, State as S};
+
+    *state = S::Live(match *state {
+        S::Initial | S::Live(LS::PathComponent) => match transition {
+            T::Blank => LS::Blank,
+            T::PathComponent => LS::PathComponent,
+            T::Punctuation => LS::Punctuation,
+        },
+        S::Live(LS::Punctuation) => match transition {
+            T::Blank => LS::Blank,
+            T::PathComponent => return to_final(state),
+            T::Punctuation => LS::Punctuation,
+        },
+        S::Live(LS::Blank) => match transition {
+            T::Blank => LS::Blank,
+            T::PathComponent => return to_final(state),
+            T::Punctuation => return to_final(state),
+        },
+        S::Final => unreachable!(),
+    });
+    true
+}
+
+enum PathComponentBackwardState {
+    Punctuation,
+    PathComponent,
+    Space,
+    EndPathComponent,
+    EndPunctuation,
+}
+fn consume_char_path_component_backward(
+    state: &mut State<PathComponentBackwardState>,
+    transition: PathComponentTransition,
+) -> bool {
+    // a path component contains either
+    // - [space +] end word
+    // - punc + end word
+    // - space + end punc
+
+    use {PathComponentBackwardState as LS, PathComponentTransition as T, State as S};
+
+    *state = S::Live(match state {
+        S::Initial => match transition {
+            T::Blank => LS::Space,
+            T::PathComponent => LS::PathComponent,
+            T::Punctuation => LS::Punctuation,
+        },
+        S::Live(ls) => match ls {
+            LS::Punctuation => match transition {
+                T::Blank => return to_final(state),
+                T::PathComponent => LS::EndPathComponent,
+                T::Punctuation => LS::Punctuation,
+            },
+            LS::PathComponent => match transition {
+                T::Blank => return to_final(state),
+                T::PathComponent => LS::EndPathComponent,
+                T::Punctuation => return to_final(state),
+            },
+            LS::Space => match transition {
+                T::Blank => LS::Space,
+                T::PathComponent => LS::PathComponent,
+                T::Punctuation => LS::EndPunctuation,
+            },
+            LS::EndPathComponent => match transition {
+                T::Blank => return to_final(state),
+                T::PathComponent => LS::EndPathComponent,
+                T::Punctuation => return to_final(state),
+            },
+            LS::EndPunctuation => match transition {
+                T::Blank => return to_final(state),
+                T::PathComponent => return to_final(state),
+                T::Punctuation => LS::EndPunctuation,
+            },
+        },
+        S::Final => unreachable!(),
+    });
+
+    true
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{MoveWordDir, MoveWordStyle, move_word_state_machine};
+    use crate::prelude::*;
+    use std::collections::VecDeque;
+
+    #[test]
+    fn test_word_motion() {
+        fn setup(direction: MoveWordDir, line: &str) -> (WString, VecDeque<usize>, usize, usize) {
+            let mut command = WString::new();
+            let mut stops = VecDeque::new();
+
+            // Carets represent stops and should be cut out of the command.
+            for c in line.chars() {
+                if c == '^' {
+                    if direction == MoveWordDir::Left {
+                        stops.push_front(command.len());
+                    } else {
+                        stops.push_back(command.len());
+                    }
+                } else {
+                    command.push(c);
+                }
+            }
+            stops.pop_back();
+
+            let idx = stops.pop_front().unwrap();
+            let end = if direction == MoveWordDir::Left {
+                0
+            } else {
+                command.len()
+            };
+
+            (command, stops, idx, end)
+        }
+
+        macro_rules! validate {
+             ($direction:expr, $style:expr, $line:expr) => {
+                let direction = $direction;
+                let (command, mut stops, mut idx, end) = setup(direction, $line);
+                assert!(!command.is_empty());
+                let mut sm = move_word_state_machine($style, direction);
+                while idx != end {
+                    let word_idx = if direction == MoveWordDir::Left {
+                        idx - 1
+                    } else {
+                        idx
+                    };
+                    let consumed = sm(&command, word_idx);
+                    if consumed {
+                        idx = if direction == MoveWordDir::Left {
+                            idx - 1
+                        } else {
+                            idx + 1
+                        };
+                    } else {
+                        assert!(
+                            !stops.is_empty(),
+                            "unexpected stop at {idx}. String: {command:?}"
+                        );
+                        let expected_idx = stops.front().unwrap();
+                        assert_eq!(
+                            idx, *expected_idx,
+                            "Expected to stop={expected_idx} but stopped at {idx}. String: {command:?}"
+                        );
+                        stops.pop_front();
+                        sm = move_word_state_machine($style, direction);
+                    }
+                }
+                assert!(
+                    stops.is_empty(),
+                    "expected to stop at {stops:?} but not. String: {command:?}"
+                );
+             }
+        }
+
+        use MoveWordDir::*;
+        use MoveWordStyle::*;
+
+        // PathComponents tests
+        validate!(
+            Left,
+            PathComponents,
+            "^echo ^/^foo/^bar{^aaa,^bbb,^ccc}^bak/^"
+        );
+        validate!(Left, PathComponents, "^echo ^bak ^///^");
+        validate!(Left, PathComponents, "^aaa ^@ ^@^aaa^");
+        validate!(Left, PathComponents, "^aaa ^a ^@^aaa^");
+        validate!(Left, PathComponents, "^aaa ^@@@ ^@@^aa^");
+        validate!(Left, PathComponents, "^aa^@@  ^aa@@^a^");
+        validate!(Left, PathComponents, r#"^a\  ^b\ c/^d"^e\ f"^g"#);
+        validate!(Left, PathComponents, r#"^a\  ^b\ c/^d"^e\\\ f"^g"#);
+        validate!(Left, PathComponents, r#"^a\"^bc^"#);
+
+        validate!(Right, PathComponents, "^/^foo/^bar/^baz/^");
+        validate!(Right, PathComponents, "^echo ^--foo ^--bar^");
+        validate!(Right, PathComponents, "^echo ^hi ^> ^/^dev/^null^");
+        validate!(Right, PathComponents, "^echo ^/^foo/^bar{^aaa,^ccc}^bak/^");
+        validate!(Right, PathComponents, "^echo ^bak ^///^");
+        validate!(Right, PathComponents, "^aaa ^@ ^@^aaa^");
+        validate!(Right, PathComponents, "^aa@@ ^aa@@^a^");
+
+        // General punctuation tests
+        validate!(Left, Punctuation, "^a ^bcd^");
+        validate!(Left, Punctuation, "^ab ^cd^e");
+        validate!(Left, Punctuation, "^ab ^c^de");
+        validate!(Left, Punctuation, "^ab ^cde^");
+        validate!(Right, Punctuation, "^a ^bcd^");
+        validate!(Right, Punctuation, "a^b ^cde^");
+        validate!(Right, Punctuation, "ab^ ^cde^");
+        validate!(Right, Punctuation, "^ab ^cde^");
+
+        validate!(Left, Punctuation, "^echo ^hello^_^world^.^txt^");
+        validate!(Right, Punctuation, "^echo ^hello^_^world^.^txt^");
+
+        validate!(Left, Punctuation, "^echo ^foo^__^foo^_^foo^//    ^");
+        validate!(Right, Punctuation, "^echo ^foo^__^foo^_^foo^//^");
+
+        validate!(Right, Punctuation, "^ab^&^cd ^& ^e ^f^&^");
+        validate!(Left, Punctuation, "^ab^&^cd ^& ^e ^f^&^");
+
+        // General Whiltespace tests
+        validate!(Left, Whitespace, "^a ^bcd^");
+        validate!(Left, Whitespace, "^ab ^cd^e");
+        validate!(Left, Whitespace, "^ab ^c^de");
+        validate!(Left, Whitespace, "^ab ^cde^");
+        validate!(Right, Whitespace, "^a ^bcd^");
+        validate!(Right, Whitespace, "a^b ^cde^");
+        validate!(Right, Whitespace, "ab^ ^cde^");
+        validate!(Right, Whitespace, "^ab ^cde^");
+
+        // Newline-related tests
+        validate!(Right, Punctuation, "^a \n ^bcd^");
+        validate!(Left, Punctuation, "^a \n ^bcd^");
+        validate!(Right, Whitespace, "^a \n ^bcd^");
+        validate!(Left, Whitespace, "^a \n ^bcd^");
+
+        validate!(Right, Punctuation, "^a\n^\n^b^-^cd^");
+        validate!(Left, Punctuation, "^a\n^\n^b^_^cd^");
+        validate!(Right, Whitespace, "^a\n^\n^b_cd^");
+        validate!(Left, Whitespace, "^a\n^\n^b_cd^");
+
+        validate!(Right, Punctuation, "^a \n  \n \n^\n ^bcd^");
+        validate!(Left, Punctuation, "^a \n  \n \n^\n ^bcd^");
+        validate!(Right, Whitespace, "^a \n  \n \n^\n ^bcd^");
+        validate!(Left, Whitespace, "^a \n  \n \n^\n ^bcd^");
+
+        // Unicode-related tests
+        validate!(Right, Punctuation, "^hello ^中^@^文^あいう^漢字^");
+        validate!(Left, Punctuation, "^hello ^中^@^文^あいう^漢字^");
+        validate!(Right, Whitespace, "^hello ^中文あいう漢字^");
+        validate!(Left, Whitespace, "^hello ^中文あいう漢字^");
+
+        validate!(Right, Punctuation, "^café ^naïve^");
+        validate!(Left, Punctuation, "^café ^naïve^");
+    }
+}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index d3f244b82e..b4990a5d8f 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -8,7 +8,6 @@
 use crate::parser_keywords::parser_keywords_is_subcommand;
 use crate::prelude::*;
 use crate::redirection::RedirectionMode;
-use fish_wchar::word_char::{WordCharClass, is_blank};
 use libc::{STDIN_FILENO, STDOUT_FILENO};
 use nix::fcntl::OFlag;
 use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, Not, Range};
@@ -103,30 +102,6 @@
     pub consumed: usize,
 }
 
-#[derive(Clone, Copy, PartialEq, Eq)]
-pub enum MoveWordStyle {
-    /// stop at punctuation
-    Punctuation,
-    /// stops at path components
-    PathComponents,
-    /// stops at whitespace
-    Whitespace,
-}
-
-#[derive(Clone, Copy, PartialEq, Eq)]
-pub enum MoveWordDir {
-    Left,
-    Right,
-}
-
-/// Our state machine that implements "one word" movement or erasure.
-pub struct MoveWordStateMachine {
-    state: u8,
-    style: MoveWordStyle,
-    direction: MoveWordDir,
-    last_char_cls: Option<WordCharClass>,
-}
-
 #[derive(Clone, Copy)]
 pub struct TokFlags(pub u8);
 
@@ -902,7 +877,7 @@
 
 /// Tests if this character can be a part of a string. Hash (#) starts a comment if it's the first
 /// character in a token; otherwise it is considered a string character. See issue #953.
-fn tok_is_string_character(c: char, next: Option<char>) -> bool {
+pub fn tok_is_string_character(c: char, next: Option<char>) -> bool {
     match c {
         // Unconditional separators.
         '\0' | ' ' | '\n' | '|' | '\t' | ';' | '\r' | '<' | '>' => false,
@@ -1183,181 +1158,6 @@
     s.parse().unwrap_or(-1)
 }
 
-const S_INIT: u8 = 0;
-
-impl MoveWordStateMachine {
-    pub fn new(style: MoveWordStyle, direction: MoveWordDir) -> Self {
-        MoveWordStateMachine {
-            state: S_INIT,
-            last_char_cls: None,
-            style,
-            direction,
-        }
-    }
-
-    pub fn consume_char(&mut self, text: &wstr, idx: usize) -> bool {
-        let c = text.as_char_slice()[idx];
-        match self.style {
-            MoveWordStyle::Punctuation => self.consume_char_word_movement(c, false),
-            MoveWordStyle::PathComponents => {
-                let mut escaped: bool = false;
-                for j in (0..idx).rev() {
-                    if text.as_char_slice()[j] == '\\' {
-                        escaped = !escaped;
-                    } else {
-                        break;
-                    }
-                }
-                match self.direction {
-                    MoveWordDir::Right => self.consume_char_path_component_forward(c, escaped),
-                    MoveWordDir::Left => self.consume_char_path_component_backward(c, escaped),
-                }
-            }
-            MoveWordStyle::Whitespace => self.consume_char_word_movement(c, true),
-        }
-    }
-
-    pub fn reset(&mut self) {
-        self.state = S_INIT;
-        self.last_char_cls = None;
-    }
-
-    fn consume_char_path_component_forward(&mut self, c: char, escaped: bool) -> bool {
-        // Forward path component movement: skip current homogeneous component, stop at next.
-        // Each component type (word, slash, punctuation, whitespace) is a unit.
-        const S_WORD: u8 = 1;
-        const S_SEP: u8 = 2;
-        const S_SPACE: u8 = 3;
-        const S_END: u8 = 4;
-
-        if c == '\\' && !escaped {
-            return true;
-        };
-
-        let transition_idx = if is_blank(c) && !escaped {
-            0
-        } else if is_path_component_character(c) || (is_blank(c) && escaped) {
-            1
-        } else {
-            2
-        };
-
-        #[rustfmt::skip]
-        const TRANSITION_TABLE: &[[(u8, bool); 3]] = &[
-            // space,         word,           sep,
-            [(S_SPACE, true), (S_WORD, true), (S_SEP, true)],  // S_INIT
-            [(S_SPACE, true), (S_WORD, true), (S_SEP, true)],  // S_WORD
-            [(S_SPACE, true), (S_END, false), (S_SEP, true)],  // S_SEP
-            [(S_SPACE, true), (S_END, false), (S_END, false)], // S_SPACE
-            [(S_END, false),  (S_END, false), (S_END, false)], // S_END
-        ];
-
-        let (new_state, result) = TRANSITION_TABLE[usize::from(self.state)][transition_idx];
-        self.state = new_state;
-        result
-    }
-
-    fn consume_char_path_component_backward(&mut self, c: char, escaped: bool) -> bool {
-        const S_PUNC: u8 = 1;
-        const S_WORD: u8 = 2;
-        const S_SPACE: u8 = 3;
-        const S_END_WORD: u8 = 4;
-        const S_END_PUNC: u8 = 5;
-        const S_END: u8 = 7;
-
-        // a path component contains either
-        // - [space +] end word
-        // - punc + end word
-        // - space + end punc
-
-        if c == '\\' && !escaped {
-            return true;
-        };
-
-        let transition_idx = if is_blank(c) && !escaped {
-            0
-        } else if is_path_component_character(c) || (is_blank(c) && escaped) {
-            1
-        } else {
-            2
-        };
-
-        #[rustfmt::skip]
-        const TRANSITION_TABLE: &[[(u8, bool); 3]] = &[
-            // space,         word,               punc,
-            [(S_SPACE, true), (S_WORD, true),     (S_PUNC, true)],     // S_INIT
-            [(S_END, false),  (S_END_WORD, true), (S_PUNC, true)],     // S_PUNC
-            [(S_END, false),  (S_WORD, true),     (S_SPACE, false)],   // S_WORD
-            [(S_SPACE, true), (S_WORD, true),     (S_END_PUNC, true)], // S_SPACE
-            [(S_END, false),  (S_END_WORD, true), (S_SPACE, false)],   // S_END_WORD
-            [(S_END, false),  (S_END, false),     (S_END_PUNC, true)], // S_END_PUNC
-            [(S_END, true),   (S_WORD, true),     (S_PUNC, true)],     // S_END
-        ];
-
-        let (new_state, result) = TRANSITION_TABLE[usize::from(self.state)][transition_idx];
-        self.state = new_state;
-        result
-    }
-
-    fn consume_char_word_movement(&mut self, c: char, is_bigword: bool) -> bool {
-        const S_SPACE: u8 = 1; // when seeing initial or midway space
-        const S_NEWLINE: u8 = 2; // when seeing a newline
-        const S_WORD: u8 = 3; // when seeing the word that will jump over
-        const S_END: u8 = 4;
-        let cur_cls = if is_bigword {
-            if c == '\n' {
-                WordCharClass::Newline
-            } else if is_blank(c) {
-                WordCharClass::Blank
-            } else {
-                WordCharClass::Word
-            }
-        } else {
-            WordCharClass::from_char(c)
-        };
-        let transition_idx = if cur_cls == WordCharClass::Blank {
-            0
-        } else if cur_cls == WordCharClass::Newline {
-            1
-        } else if self.last_char_cls == Some(cur_cls) {
-            2
-        } else {
-            3
-        };
-        // Transition tables for forward and backward word movement
-        #[rustfmt::skip]
-        const FORWARD_TABLE: &[[(u8, bool); 4]] = &[
-            // space,         newline,           same-class,     different-class
-            [(S_SPACE, true), (S_NEWLINE, true), (S_WORD, true), (S_WORD, true)], // S_INIT
-            [(S_SPACE, true), (S_NEWLINE, true), (S_END, false), (S_END, false)], // S_SPACE
-            [(S_SPACE, true), (S_END, false),    (S_END, false), (S_END, false)], // S_NEWLINE
-            [(S_SPACE, true), (S_NEWLINE, true), (S_WORD, true), (S_END, false)], // S_WORD
-            [(S_END, false),  (S_END, false),    (S_END, false), (S_END, false)], // S_END
-        ];
-        #[rustfmt::skip]
-        const BACKWARD_TABLE: &[[(u8, bool); 4]] = &[
-            // space,         newline,           same-class,     different-class
-            [(S_SPACE, true), (S_NEWLINE, true), (S_WORD, true), (S_WORD, true)], // S_INIT
-            [(S_SPACE, true), (S_NEWLINE, true), (S_WORD, true), (S_WORD, true)], // S_SPACE
-            [(S_SPACE, true), (S_END, false),    (S_WORD, true), (S_WORD, true)], // S_NEWLINE
-            [(S_END, false),  (S_END, false),    (S_WORD, true), (S_END, false)], // S_WORD
-            [(S_END, false),  (S_END, false),    (S_END, false), (S_END, false)], // S_END
-        ];
-        let table = match self.direction {
-            MoveWordDir::Right => FORWARD_TABLE,
-            MoveWordDir::Left => BACKWARD_TABLE,
-        };
-        let (new_state, result) = table[usize::from(self.state)][transition_idx];
-        self.state = new_state;
-        self.last_char_cls = Some(cur_cls);
-        result
-    }
-}
-
-fn is_path_component_character(c: char) -> bool {
-    tok_is_string_character(c, None) && !L!("/={,}'\":@#").as_char_slice().contains(&c)
-}
-
 /// The position of the equal sign in a variable assignment like foo=bar.
 ///
 /// Return the location of the equals sign, or none if the string does
@@ -1388,14 +1188,10 @@
 
 #[cfg(test)]
 mod tests {
-    use super::{
-        MoveWordDir, MoveWordStateMachine, MoveWordStyle, PipeOrRedir, TokFlags, TokenType,
-        Tokenizer, TokenizerError,
-    };
+    use super::{PipeOrRedir, TokFlags, TokenType, Tokenizer, TokenizerError};
     use crate::prelude::*;
     use crate::redirection::RedirectionMode;
     use libc::{STDERR_FILENO, STDOUT_FILENO};
-    use std::collections::VecDeque;
 
     #[test]
     fn test_tokenizer() {
@@ -1583,156 +1379,4 @@
         assert_eq!(get_redir_mode!("3<&0"), RedirectionMode::Fd);
         assert_eq!(get_redir_mode!("3</tmp/filetxt"), RedirectionMode::Input);
     }
-
-    /// Test word motion (forward-word, etc.). Carets represent cursor stops.
-    #[test]
-    fn test_word_motion() {
-        fn setup(direction: MoveWordDir, line: &str) -> (WString, VecDeque<usize>, usize, usize) {
-            let mut command = WString::new();
-            let mut stops = VecDeque::new();
-
-            // Carets represent stops and should be cut out of the command.
-            for c in line.chars() {
-                if c == '^' {
-                    if direction == MoveWordDir::Left {
-                        stops.push_front(command.len());
-                    } else {
-                        stops.push_back(command.len());
-                    }
-                } else {
-                    command.push(c);
-                }
-            }
-            stops.pop_back();
-
-            let idx = stops.pop_front().unwrap();
-            let end = if direction == MoveWordDir::Left {
-                0
-            } else {
-                command.len()
-            };
-
-            (command, stops, idx, end)
-        }
-
-        macro_rules! validate {
-             ($direction:expr, $style:expr, $line:expr) => {
-                let direction = $direction;
-                let (command, mut stops, mut idx, end) = setup(direction, $line);
-                assert!(!command.is_empty());
-                let mut sm = MoveWordStateMachine::new($style, direction);
-                while idx != end {
-                    let word_idx = if direction == MoveWordDir::Left {
-                        idx - 1
-                    } else {
-                        idx
-                    };
-                    let consumed = sm.consume_char(&command, word_idx);
-                    if consumed {
-                        idx = if direction == MoveWordDir::Left {
-                            idx - 1
-                        } else {
-                            idx + 1
-                        };
-                    } else {
-                        assert!(
-                            !stops.is_empty(),
-                            "unexpected stop at {idx}. String: {command:?}"
-                        );
-                        let expected_idx = stops.front().unwrap();
-                        assert_eq!(
-                            idx, *expected_idx,
-                            "Expected to stop={expected_idx} but stopped at {idx}. String: {command:?}"
-                        );
-                        stops.pop_front();
-                        sm.reset();
-                    }
-                }
-                assert!(
-                    stops.is_empty(),
-                    "expected to stop at {stops:?} but not. String: {command:?}"
-                );
-             }
-        }
-
-        use MoveWordDir::*;
-        use MoveWordStyle::*;
-
-        // PathComponents tests
-        validate!(
-            Left,
-            PathComponents,
-            "^echo ^/^foo/^bar{^aaa,^bbb,^ccc}^bak/^"
-        );
-        validate!(Left, PathComponents, "^echo ^bak ^///^");
-        validate!(Left, PathComponents, "^aaa ^@ ^@^aaa^");
-        validate!(Left, PathComponents, "^aaa ^a ^@^aaa^");
-        validate!(Left, PathComponents, "^aaa ^@@@ ^@@^aa^");
-        validate!(Left, PathComponents, "^aa^@@  ^aa@@^a^");
-        validate!(Left, PathComponents, r#"^a\  ^b\ c/^d"^e\ f"^g"#);
-        validate!(Left, PathComponents, r#"^a\  ^b\ c/^d"^e\\\ f"^g"#);
-        validate!(Left, PathComponents, r#"^a\"^bc^"#);
-
-        validate!(Right, PathComponents, "^/^foo/^bar/^baz/^");
-        validate!(Right, PathComponents, "^echo ^--foo ^--bar^");
-        validate!(Right, PathComponents, "^echo ^hi ^> ^/^dev/^null^");
-        validate!(Right, PathComponents, "^echo ^/^foo/^bar{^aaa,^ccc}^bak/^");
-        validate!(Right, PathComponents, "^echo ^bak ^///^");
-        validate!(Right, PathComponents, "^aaa ^@ ^@^aaa^");
-        validate!(Right, PathComponents, "^aa@@ ^aa@@^a^");
-
-        // General punctuation tests
-        validate!(Left, Punctuation, "^a ^bcd^");
-        validate!(Left, Punctuation, "^ab ^cd^e");
-        validate!(Left, Punctuation, "^ab ^c^de");
-        validate!(Left, Punctuation, "^ab ^cde^");
-        validate!(Right, Punctuation, "^a ^bcd^");
-        validate!(Right, Punctuation, "a^b ^cde^");
-        validate!(Right, Punctuation, "ab^ ^cde^");
-        validate!(Right, Punctuation, "^ab ^cde^");
-
-        validate!(Left, Punctuation, "^echo ^hello^_^world^.^txt^");
-        validate!(Right, Punctuation, "^echo ^hello^_^world^.^txt^");
-
-        validate!(Left, Punctuation, "^echo ^foo^__^foo^_^foo^//    ^");
-        validate!(Right, Punctuation, "^echo ^foo^__^foo^_^foo^//^");
-
-        validate!(Right, Punctuation, "^ab^&^cd ^& ^e ^f^&^");
-        validate!(Left, Punctuation, "^ab^&^cd ^& ^e ^f^&^");
-
-        // General Whiltespace tests
-        validate!(Left, Whitespace, "^a ^bcd^");
-        validate!(Left, Whitespace, "^ab ^cd^e");
-        validate!(Left, Whitespace, "^ab ^c^de");
-        validate!(Left, Whitespace, "^ab ^cde^");
-        validate!(Right, Whitespace, "^a ^bcd^");
-        validate!(Right, Whitespace, "a^b ^cde^");
-        validate!(Right, Whitespace, "ab^ ^cde^");
-        validate!(Right, Whitespace, "^ab ^cde^");
-
-        // Newline-related tests
-        validate!(Right, Punctuation, "^a \n ^bcd^");
-        validate!(Left, Punctuation, "^a \n ^bcd^");
-        validate!(Right, Whitespace, "^a \n ^bcd^");
-        validate!(Left, Whitespace, "^a \n ^bcd^");
-
-        validate!(Right, Punctuation, "^a\n^\n^b^-^cd^");
-        validate!(Left, Punctuation, "^a\n^\n^b^_^cd^");
-        validate!(Right, Whitespace, "^a\n^\n^b_cd^");
-        validate!(Left, Whitespace, "^a\n^\n^b_cd^");
-
-        validate!(Right, Punctuation, "^a \n  \n \n^\n ^bcd^");
-        validate!(Left, Punctuation, "^a \n  \n \n^\n ^bcd^");
-        validate!(Right, Whitespace, "^a \n  \n \n^\n ^bcd^");
-        validate!(Left, Whitespace, "^a \n  \n \n^\n ^bcd^");
-
-        // Unicode-related tests
-        validate!(Right, Punctuation, "^hello ^中^@^文^あいう^漢字^");
-        validate!(Left, Punctuation, "^hello ^中^@^文^あいう^漢字^");
-        validate!(Right, Whitespace, "^hello ^中文あいう漢字^");
-        validate!(Left, Whitespace, "^hello ^中文あいう漢字^");
-
-        validate!(Right, Punctuation, "^café ^naïve^");
-        validate!(Left, Punctuation, "^café ^naïve^");
-    }
 }