prepend UnmatchedToken error tokens

Dan Rubel · commit-bot@chromium.org · commit b41b8687978d · 2019-07-23T15:21:04.000Z
This change ensures that instances of UnmatchedToken are always prepended to the token stream. This should prevent the exception that occurs in #37491. This addresses only one type of error token and we may see a similar exception with other types of error tokens. Future CLs will prepend other and eventually all error tokens. Change-Id: I99cdc331c03f47b21c5a67d436a0e62ebf416e94 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/110100 Reviewed-by: Brian Wilkerson <brianwilkerson@google.com> Commit-Queue: Dan Rubel <danrubel@google.com>
diff --git a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
@@ -1090,8 +1090,8 @@ abstract class AbstractScanner implements Scanner {
    */
   void appendToken(Token token) {
     tail.next = token;
-    tail.next.previous = tail;
-    tail = tail.next;
+    token.previous = tail;
+    tail = token;
     if (comments != null && comments == token.precedingComments) {
       comments = null;
       commentsTail = null;
diff --git a/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
@@ -33,11 +33,14 @@ import '../util/link.dart' show Link;
 
 abstract class ArrayBasedScanner extends AbstractScanner {
   bool hasErrors = false;
+  Token _errorTail;
 
   ArrayBasedScanner(ScannerConfiguration config, bool includeComments,
       LanguageVersionChanged languageVersionChanged, {int numberOfBytesHint})
       : super(config, includeComments, languageVersionChanged,
-            numberOfBytesHint: numberOfBytesHint);
+            numberOfBytesHint: numberOfBytesHint) {
+    this._errorTail = this.tokens;
+  }
 
   /**
    * The stack of open groups, e.g [: { ... ( .. :]
@@ -257,6 +260,20 @@ abstract class ArrayBasedScanner extends AbstractScanner {
     appendToken(token);
   }
 
+  void prependErrorToken(ErrorToken token) {
+    hasErrors = true;
+    if (_errorTail == tail) {
+      appendToken(token);
+      _errorTail = tail;
+    } else {
+      token.next = _errorTail.next;
+      token.next.previous = token;
+      _errorTail.next = token;
+      token.previous = _errorTail;
+      _errorTail = _errorTail.next;
+    }
+  }
+
   @override
   void appendSubstringToken(TokenType type, int start, bool asciiOnly,
       [int extraOffset = 0]) {
@@ -374,6 +391,6 @@ abstract class ArrayBasedScanner extends AbstractScanner {
     TokenType type = closeBraceInfoFor(begin);
     appendToken(new SyntheticToken(type, tokenStart)..beforeSynthetic = tail);
     begin.endGroup = tail;
-    appendErrorToken(new UnmatchedToken(begin));
+    prependErrorToken(new UnmatchedToken(begin));
   }
 }
diff --git a/pkg/front_end/lib/src/fasta/scanner/recover.dart b/pkg/front_end/lib/src/fasta/scanner/recover.dart
@@ -151,10 +151,20 @@ Token scannerRecovery(List<int> bytes, Token tokens, List<int> lineStarts) {
   recoverUnmatched() {
     // TODO(ahe): Try to use top-level keywords (such as `class`, `typedef`,
     // and `enum`) and indentation to recover.
-    return errorTail.next;
+    throw "Internal error: Unmatched error token should have been prepended";
+  }
+
+  // All unmatched error tokens should have been prepended
+  Token current = tokens;
+  while (current is ErrorToken && current.errorCode == codeUnmatchedToken) {
+    if (errorTail == null) {
+      error = current;
+    }
+    errorTail = current;
+    current = current.next;
   }
 
-  for (Token current = tokens; !current.isEof; current = current.next) {
+  for (; !current.isEof; current = current.next) {
     while (current is ErrorToken) {
       ErrorToken first = current;
       Token next = current;
diff --git a/pkg/front_end/testcases/rasta/bad_interpolation.dart.legacy.expect b/pkg/front_end/testcases/rasta/bad_interpolation.dart.legacy.expect
@@ -2,14 +2,14 @@ library;
 //
 // Problems in library:
 //
-// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
-//   print(" $x.);
-//             ^^^
-//
 // pkg/front_end/testcases/rasta/bad_interpolation.dart:6:8: Error: Can't find ')' to match '('.
 //   print(" $x.);
 //        ^
 //
+// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
+//   print(" $x.);
+//             ^^^
+//
 // pkg/front_end/testcases/rasta/bad_interpolation.dart:6:12: Warning: Getter not found: 'x'.
 //   print(" $x.);
 //            ^
diff --git a/pkg/front_end/testcases/rasta/bad_interpolation.dart.legacy.transformed.expect b/pkg/front_end/testcases/rasta/bad_interpolation.dart.legacy.transformed.expect
@@ -2,14 +2,14 @@ library;
 //
 // Problems in library:
 //
-// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
-//   print(" $x.);
-//             ^^^
-//
 // pkg/front_end/testcases/rasta/bad_interpolation.dart:6:8: Error: Can't find ')' to match '('.
 //   print(" $x.);
 //        ^
 //
+// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
+//   print(" $x.);
+//             ^^^
+//
 // pkg/front_end/testcases/rasta/bad_interpolation.dart:6:12: Warning: Getter not found: 'x'.
 //   print(" $x.);
 //            ^
diff --git a/pkg/front_end/testcases/rasta/bad_interpolation.dart.outline.expect b/pkg/front_end/testcases/rasta/bad_interpolation.dart.outline.expect
@@ -2,14 +2,14 @@ library;
 //
 // Problems in library:
 //
-// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
-//   print(" $x.);
-//             ^^^
-//
 // pkg/front_end/testcases/rasta/bad_interpolation.dart:6:8: Error: Can't find ')' to match '('.
 //   print(" $x.);
 //        ^
 //
+// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
+//   print(" $x.);
+//             ^^^
+//
 import self as self;
 
 static method main() → dynamic
diff --git a/pkg/front_end/testcases/rasta/bad_interpolation.dart.strong.expect b/pkg/front_end/testcases/rasta/bad_interpolation.dart.strong.expect
@@ -2,14 +2,14 @@ library;
 //
 // Problems in library:
 //
-// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
-//   print(" $x.);
-//             ^^^
-//
 // pkg/front_end/testcases/rasta/bad_interpolation.dart:6:8: Error: Can't find ')' to match '('.
 //   print(" $x.);
 //        ^
 //
+// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
+//   print(" $x.);
+//             ^^^
+//
 // pkg/front_end/testcases/rasta/bad_interpolation.dart:6:12: Error: Getter not found: 'x'.
 //   print(" $x.);
 //            ^
diff --git a/pkg/front_end/testcases/rasta/bad_interpolation.dart.strong.transformed.expect b/pkg/front_end/testcases/rasta/bad_interpolation.dart.strong.transformed.expect
@@ -2,14 +2,14 @@ library;
 //
 // Problems in library:
 //
-// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
-//   print(" $x.);
-//             ^^^
-//
 // pkg/front_end/testcases/rasta/bad_interpolation.dart:6:8: Error: Can't find ')' to match '('.
 //   print(" $x.);
 //        ^
 //
+// pkg/front_end/testcases/rasta/bad_interpolation.dart:6:13: Error: String starting with " must end with ".
+//   print(" $x.);
+//             ^^^
+//
 // pkg/front_end/testcases/rasta/bad_interpolation.dart:6:12: Error: Getter not found: 'x'.
 //   print(" $x.);
 //            ^
diff --git a/pkg/front_end/testcases/regress/issue_29976.dart.legacy.expect b/pkg/front_end/testcases/regress/issue_29976.dart.legacy.expect
@@ -2,6 +2,10 @@ library;
 //
 // Problems in library:
 //
+// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
+//     "x${x*"'"é'}x
+//       ^
+//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:14: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
 // Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
 //     "x${x*"'"é'}x
@@ -11,10 +15,6 @@ library;
 //     "x${x*"'"é'}x
 //               ^^^
 //
-// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
-//     "x${x*"'"é'}x
-//       ^
-//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:5: Error: String starting with " must end with ".
 //     "x${x*"'"é'}x
 //     ^^^^^^^^^^^^^^...
diff --git a/pkg/front_end/testcases/regress/issue_29976.dart.legacy.transformed.expect b/pkg/front_end/testcases/regress/issue_29976.dart.legacy.transformed.expect
@@ -2,6 +2,10 @@ library;
 //
 // Problems in library:
 //
+// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
+//     "x${x*"'"é'}x
+//       ^
+//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:14: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
 // Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
 //     "x${x*"'"é'}x
@@ -11,10 +15,6 @@ library;
 //     "x${x*"'"é'}x
 //               ^^^
 //
-// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
-//     "x${x*"'"é'}x
-//       ^
-//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:5: Error: String starting with " must end with ".
 //     "x${x*"'"é'}x
 //     ^^^^^^^^^^^^^^...
diff --git a/pkg/front_end/testcases/regress/issue_29976.dart.outline.expect b/pkg/front_end/testcases/regress/issue_29976.dart.outline.expect
@@ -2,6 +2,10 @@ library;
 //
 // Problems in library:
 //
+// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
+//     "x${x*"'"é'}x
+//       ^
+//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:14: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
 // Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
 //     "x${x*"'"é'}x
@@ -11,10 +15,6 @@ library;
 //     "x${x*"'"é'}x
 //               ^^^
 //
-// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
-//     "x${x*"'"é'}x
-//       ^
-//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:5: Error: String starting with " must end with ".
 //     "x${x*"'"é'}x
 //     ^^^^^^^^^^^^^^...
diff --git a/pkg/front_end/testcases/regress/issue_29976.dart.strong.expect b/pkg/front_end/testcases/regress/issue_29976.dart.strong.expect
@@ -2,6 +2,10 @@ library;
 //
 // Problems in library:
 //
+// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
+//     "x${x*"'"é'}x
+//       ^
+//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:14: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
 // Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
 //     "x${x*"'"é'}x
@@ -11,10 +15,6 @@ library;
 //     "x${x*"'"é'}x
 //               ^^^
 //
-// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
-//     "x${x*"'"é'}x
-//       ^
-//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:5: Error: String starting with " must end with ".
 //     "x${x*"'"é'}x
 //     ^^^^^^^^^^^^^^...
diff --git a/pkg/front_end/testcases/regress/issue_29976.dart.strong.transformed.expect b/pkg/front_end/testcases/regress/issue_29976.dart.strong.transformed.expect
@@ -2,6 +2,10 @@ library;
 //
 // Problems in library:
 //
+// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
+//     "x${x*"'"é'}x
+//       ^
+//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:14: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
 // Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
 //     "x${x*"'"é'}x
@@ -11,10 +15,6 @@ library;
 //     "x${x*"'"é'}x
 //               ^^^
 //
-// pkg/front_end/testcases/regress/issue_29976.dart:9:7: Error: Can't find '}' to match '${'.
-//     "x${x*"'"é'}x
-//       ^
-//
 // pkg/front_end/testcases/regress/issue_29976.dart:9:5: Error: String starting with " must end with ".
 //     "x${x*"'"é'}x
 //     ^^^^^^^^^^^^^^...
diff --git a/pkg/front_end/testcases/regress/issue_29982.dart.legacy.expect b/pkg/front_end/testcases/regress/issue_29982.dart.legacy.expect
@@ -2,11 +2,6 @@ library;
 //
 // Problems in library:
 //
-// pkg/front_end/testcases/regress/issue_29982.dart:7:15: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
-// Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
-//   print('${eh[éh']}');
-//               ^
-//
 // pkg/front_end/testcases/regress/issue_29982.dart:7:14: Error: Can't find ']' to match '['.
 //   print('${eh[éh']}');
 //              ^
@@ -15,6 +10,11 @@ library;
 //   print('${eh[éh']}');
 //          ^
 //
+// pkg/front_end/testcases/regress/issue_29982.dart:7:15: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
+// Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
+//   print('${eh[éh']}');
+//               ^
+//
 // pkg/front_end/testcases/regress/issue_29982.dart:7:11: Error: String starting with ' must end with '.
 //   print('${eh[éh']}');
 //           ^^^^^^^^^^^^^...
diff --git a/pkg/front_end/testcases/regress/issue_29982.dart.legacy.transformed.expect b/pkg/front_end/testcases/regress/issue_29982.dart.legacy.transformed.expect
@@ -2,11 +2,6 @@ library;
 //
 // Problems in library:
 //
-// pkg/front_end/testcases/regress/issue_29982.dart:7:15: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
-// Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
-//   print('${eh[éh']}');
-//               ^
-//
 // pkg/front_end/testcases/regress/issue_29982.dart:7:14: Error: Can't find ']' to match '['.
 //   print('${eh[éh']}');
 //              ^
@@ -15,6 +10,11 @@ library;
 //   print('${eh[éh']}');
 //          ^
 //
+// pkg/front_end/testcases/regress/issue_29982.dart:7:15: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
+// Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
+//   print('${eh[éh']}');
+//               ^
+//
 // pkg/front_end/testcases/regress/issue_29982.dart:7:11: Error: String starting with ' must end with '.
 //   print('${eh[éh']}');
 //           ^^^^^^^^^^^^^...
diff --git a/pkg/front_end/testcases/regress/issue_29982.dart.outline.expect b/pkg/front_end/testcases/regress/issue_29982.dart.outline.expect
@@ -2,11 +2,6 @@ library;
 //
 // Problems in library:
 //
-// pkg/front_end/testcases/regress/issue_29982.dart:7:15: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
-// Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
-//   print('${eh[éh']}');
-//               ^
-//
 // pkg/front_end/testcases/regress/issue_29982.dart:7:14: Error: Can't find ']' to match '['.
 //   print('${eh[éh']}');
 //              ^
@@ -15,6 +10,11 @@ library;
 //   print('${eh[éh']}');
 //          ^
 //
+// pkg/front_end/testcases/regress/issue_29982.dart:7:15: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
+// Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
+//   print('${eh[éh']}');
+//               ^
+//
 // pkg/front_end/testcases/regress/issue_29982.dart:7:11: Error: String starting with ' must end with '.
 //   print('${eh[éh']}');
 //           ^^^^^^^^^^^^^...
diff --git a/pkg/front_end/testcases/regress/issue_29982.dart.strong.expect b/pkg/front_end/testcases/regress/issue_29982.dart.strong.expect
@@ -2,11 +2,6 @@ library;
 //
 // Problems in library:
 //
-// pkg/front_end/testcases/regress/issue_29982.dart:7:15: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
-// Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
-//   print('${eh[éh']}');
-//               ^
-//
 // pkg/front_end/testcases/regress/issue_29982.dart:7:14: Error: Can't find ']' to match '['.
 //   print('${eh[éh']}');
 //              ^
@@ -15,6 +10,11 @@ library;
 //   print('${eh[éh']}');
 //          ^
 //
+// pkg/front_end/testcases/regress/issue_29982.dart:7:15: Error: The non-ASCII character 'é' (U+00E9) can't be used in identifiers, only in strings and comments.
+// Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign).
+//   print('${eh[éh']}');
+//               ^
+//
 // pkg/front_end/testcases/regress/issue_29982.dart:7:11: Error: String starting with ' must end with '.
 //   print('${eh[éh']}');
 //           ^^^^^^^^^^^^^...
diff --git a/pkg/front_end/testcases/regress/issue_29982.dart.strong.transformed.expect b/pkg/front_end/testcases/regress/issue_29982.dart.strong.transformed.expect
