Sort the tokens if jdk version >= 21

butterunderflow · butterunderflow · commit 86390d0d569d · 2023-12-15T21:03:01.000+08:00
diff --git a/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java b/core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java
@@ -27,6 +27,7 @@
 import com.sun.tools.javac.parser.Tokens.TokenKind;
 import com.sun.tools.javac.parser.UnicodeReader;
 import com.sun.tools.javac.util.Context;
+import java.util.Comparator;
 import java.util.Set;
 
 /** A wrapper around javac's lexer. */
@@ -71,22 +72,40 @@ public String stringVal() {
     }
   }
 
-  /** Lex the input and return a list of {@link RawTok}s. */
-  public static ImmutableList<RawTok> getTokens(
-      String source, Context context, Set<TokenKind> stopTokens) {
+  private static ImmutableList<Token> readAllToken(String source, Context context) {
     if (source == null) {
       return ImmutableList.of();
     }
     ScannerFactory fac = ScannerFactory.instance(context);
     char[] buffer = (source + EOF_COMMENT).toCharArray();
     Scanner scanner =
         new AccessibleScanner(fac, new CommentSavingTokenizer(fac, buffer, buffer.length));
+    ImmutableList.Builder<Token> tokens = ImmutableList.builder();
+    do {
+      scanner.nextToken();
+      tokens.add(scanner.token());
+    } while (scanner.token().kind != TokenKind.EOF);
+    if (Runtime.version().feature() < 21) {
+      return tokens.build();
+    } else {
+      return ImmutableList.sortedCopyOf(Comparator.comparingInt(t -> t.pos), tokens.build());
+    }
+  }
+
+  /** Lex the input and return a list of {@link RawTok}s. */
+  public static ImmutableList<RawTok> getTokens(
+      String source, Context context, Set<TokenKind> stopTokens) {
+    if (source == null) {
+      return ImmutableList.of();
+    }
+    ImmutableList<Token> javacTokens = readAllToken(source, context);
+
+    var javacTokenIt = javacTokens.iterator();
     ImmutableList.Builder<RawTok> tokens = ImmutableList.builder();
     int end = source.length();
     int last = 0;
     do {
-      scanner.nextToken();
-      Token t = scanner.token();
+      Token t = javacTokenIt.next();
       if (t.comments != null) {
         for (Comment c : Lists.reverse(t.comments)) {
           if (last < c.getSourcePos(0)) {
@@ -104,6 +123,8 @@ public static ImmutableList<RawTok> getTokens(
         break;
       }
       if (last < t.pos) {
+        /* If the current token is not immediately following the previous one,
+        treat the gap as a token */
         tokens.add(new RawTok(null, null, last, t.pos));
       }
       tokens.add(
@@ -113,7 +134,7 @@ public static ImmutableList<RawTok> getTokens(
               t.pos,
               t.endPos));
       last = t.endPos;
-    } while (scanner.token().kind != TokenKind.EOF);
+    } while (javacTokenIt.hasNext());
     if (last < end) {
       tokens.add(new RawTok(null, null, last, end));
     }
