prepend all remaining scanner error tokens

danrubel · commit-bot@chromium.org · commit 5713caaf09db · 2019-07-25T01:51:43.000Z
Change-Id: I325f0681a26e336a41ade9250a2ee0e90a658492 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/110360 Reviewed-by: Brian Wilkerson <brianwilkerson@google.com> Commit-Queue: Dan Rubel <danrubel@google.com>
diff --git a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
@@ -421,15 +421,6 @@ abstract class AbstractScanner implements Scanner {
     }
   }
 
-  /// Append [token] to the token stream.
-  /// DEPRECATED: Use prependErrorToken instead.
-  void appendErrorToken(ErrorToken token) {
-    // TODO(danrubel): Update scanner to use prependErrorToken everywhere
-    // then remove this method
-    hasErrors = true;
-    appendToken(token);
-  }
-
   /// Prepend [token] to the token stream.
   void prependErrorToken(ErrorToken token) {
     hasErrors = true;
@@ -994,7 +985,7 @@ abstract class AbstractScanner implements Scanner {
       int next = advance();
       if (identical(next, $EQ)) {
         appendPrecedenceToken(TokenType.BANG_EQ_EQ);
-        appendErrorToken(new UnsupportedOperator(tail, tokenStart));
+        prependErrorToken(new UnsupportedOperator(tail, tokenStart));
         return advance();
       } else {
         appendPrecedenceToken(TokenType.BANG_EQ);
@@ -1019,7 +1010,7 @@ abstract class AbstractScanner implements Scanner {
       int next = advance();
       if (identical(next, $EQ)) {
         appendPrecedenceToken(TokenType.EQ_EQ_EQ);
-        appendErrorToken(new UnsupportedOperator(tail, tokenStart));
+        prependErrorToken(new UnsupportedOperator(tail, tokenStart));
         return advance();
       } else {
         appendPrecedenceToken(TokenType.EQ_EQ);
diff --git a/pkg/front_end/lib/src/fasta/scanner/recover.dart b/pkg/front_end/lib/src/fasta/scanner/recover.dart
@@ -6,162 +6,48 @@ library fasta.scanner.recover;
 
 import '../../scanner/token.dart' show TokenType;
 
-import '../fasta_codes.dart'
-    show
-        Code,
-        codeAsciiControlCharacter,
-        codeEncoding,
-        codeExpectedHexDigit,
-        codeMissingExponent,
-        codeNonAsciiIdentifier,
-        codeNonAsciiWhitespace,
-        codeUnexpectedDollarInString,
-        codeUnmatchedToken,
-        codeUnterminatedComment,
-        codeUnterminatedString;
-
 import '../../scanner/token.dart' show Token;
 
 import 'token.dart' show StringToken;
 
-import 'error_token.dart' show NonAsciiIdentifierToken, ErrorToken;
+import 'error_token.dart' show ErrorToken;
 
 /// Recover from errors in [tokens]. The original sources are provided as
 /// [bytes]. [lineStarts] are the beginning character offsets of lines, and
 /// must be updated if recovery is performed rewriting the original source
 /// code.
 Token scannerRecovery(List<int> bytes, Token tokens, List<int> lineStarts) {
-  // See [Parser.reportErrorToken](../parser/src/parser.dart) for how
-  // it currently handles lexical errors. In addition, notice how the parser
-  // calls [handleInvalidExpression], [handleInvalidFunctionBody], and
-  // [handleInvalidTypeReference] to allow the listener to recover its internal
-  // state. See [package:compiler/src/parser/element_listener.dart] for an
-  // example of how these events are used.
-  //
-  // In addition, the scanner will attempt a bit of recovery when braces don't
-  // match up during brace grouping. For more details on brace grouping see
-  // [AbstractScanner.discardBeginGroupUntil] and
-  // [AbstractScanner.unmatchedBeginGroup].
-
-  /// Tokens with errors.
-  ErrorToken error;
-
-  /// Used for appending to [error].
-  ErrorToken errorTail;
-
-  /// Tokens without errors.
-  Token good;
-
-  /// Used for appending to [good].
-  Token goodTail;
-
-  recoverIdentifier(NonAsciiIdentifierToken first) {
-    throw "Internal error: Identifier error token should have been prepended";
-  }
+  // Sanity check that all error tokens are prepended.
 
-  recoverExponent() {
-    throw "Internal error: Exponent error token should have been prepended";
-  }
-
-  recoverString() {
-    throw "Internal error: String error token should have been prepended";
-  }
-
-  recoverHexDigit() {
-    throw "Internal error: Hex digit error token should have been prepended";
-  }
-
-  recoverStringInterpolation() {
-    throw "Internal error: Interpolation error token should have been prepended";
-  }
+  // TODO(danrubel): Remove this in a while after the dust has settled.
 
-  recoverComment() {
-    throw "Internal error: Comment error token should have been prepended";
-  }
-
-  recoverUnmatched() {
-    // TODO(ahe): Try to use top-level keywords (such as `class`, `typedef`,
-    // and `enum`) and indentation to recover.
-    throw "Internal error: Unmatched error token should have been prepended";
-  }
-
-  // All unmatched error tokens should have been prepended
-  Token current = tokens;
-  while (current is ErrorToken) {
-    if (errorTail == null) {
-      error = current;
-    }
-    errorTail = current;
-    current = current.next;
+  // Skip over prepended error tokens
+  var token = tokens;
+  while (token is ErrorToken) {
+    token = token.next;
   }
 
-  for (; !current.isEof; current = current.next) {
-    while (current is ErrorToken) {
-      ErrorToken first = current;
-      Token next = current;
-      do {
-        current = next;
-        if (errorTail == null) {
-          error = next;
-        } else {
-          errorTail.setNext(next);
-        }
-        errorTail = next;
-        next = next.next;
-      } while (next is ErrorToken && first.errorCode == next.errorCode);
-
-      Code code = first.errorCode;
-      if (code == codeEncoding ||
-          code == codeNonAsciiWhitespace ||
-          code == codeAsciiControlCharacter) {
-        current = errorTail.next;
-      } else if (code == codeNonAsciiIdentifier) {
-        current = recoverIdentifier(first);
-        assert(current.next != null);
-      } else if (code == codeMissingExponent) {
-        current = recoverExponent();
-        assert(current.next != null);
-      } else if (code == codeUnterminatedString) {
-        current = recoverString();
-        assert(current.next != null);
-      } else if (code == codeExpectedHexDigit) {
-        current = recoverHexDigit();
-        assert(current.next != null);
-      } else if (code == codeUnexpectedDollarInString) {
-        current = recoverStringInterpolation();
-        assert(current.next != null);
-      } else if (code == codeUnterminatedComment) {
-        current = recoverComment();
-        assert(current.next != null);
-      } else if (code == codeUnmatchedToken) {
-        current = recoverUnmatched();
-        assert(current.next != null);
-      } else {
-        current = errorTail.next;
+  // Assert no error tokens in the remaining tokens
+  while (!token.isEof) {
+    if (token is ErrorToken) {
+      for (int count = 0; count < 3; ++count) {
+        var previous = token.previous;
+        if (previous.isEof) break;
+        token = previous;
       }
+      var msg = new StringBuffer(
+          "Internal error: All error tokens should have been prepended:");
+      for (int count = 0; count < 7; ++count) {
+        if (token.isEof) break;
+        msg.write(' ${token.runtimeType},');
+        token = token.next;
+      }
+      throw msg.toString();
     }
-    if (goodTail == null) {
-      good = current;
-    } else {
-      goodTail.setNext(current);
-    }
-    goodTail = current;
+    token = token.next;
   }
 
-  if (error == null) {
-    // All of the errors are in the scanner's error list.
-    return tokens;
-  }
-  new Token.eof(-1).setNext(error);
-  Token tail;
-  if (good != null) {
-    errorTail.setNext(good);
-    tail = goodTail;
-  } else {
-    tail = errorTail;
-  }
-  if (!tail.isEof) tail.setNext(new Token.eof(tail.end));
-  return error;
+  return tokens;
 }
 
 Token synthesizeToken(int charOffset, String value, TokenType type) {
