rustc_save_analysis: don't pollute the codemap with fake files.

eddyb · eddyb · commit f702b20dfd22 · 2017-02-28T08:30:07.000+02:00
diff --git a/src/librustc_save_analysis/span_utils.rs b/src/librustc_save_analysis/span_utils.rs
@@ -17,7 +17,6 @@ use std::env;
 use std::path::Path;
 
 use syntax::ast;
-use syntax::parse::filemap_to_tts;
 use syntax::parse::lexer::{self, StringReader};
 use syntax::parse::token::{self, Token};
 use syntax::symbol::keywords;
@@ -49,23 +48,6 @@ impl<'a> SpanUtils<'a> {
         }
     }
 
-    // sub_span starts at span.lo, so we need to adjust the positions etc.
-    // If sub_span is None, we don't need to adjust.
-    pub fn make_sub_span(&self, span: Span, sub_span: Option<Span>) -> Option<Span> {
-        match sub_span {
-            None => None,
-            Some(sub) => {
-                let FileMapAndBytePos {fm, pos} = self.sess.codemap().lookup_byte_offset(span.lo);
-                let base = pos + fm.start_pos;
-                Some(Span {
-                    lo: base + self.sess.codemap().lookup_byte_offset(sub.lo).pos,
-                    hi: base + self.sess.codemap().lookup_byte_offset(sub.hi).pos,
-                    expn_id: span.expn_id,
-                })
-            }
-        }
-    }
-
     pub fn snippet(&self, span: Span) -> String {
         match self.sess.codemap().span_to_snippet(span) {
             Ok(s) => s,
@@ -74,24 +56,7 @@ impl<'a> SpanUtils<'a> {
     }
 
     pub fn retokenise_span(&self, span: Span) -> StringReader<'a> {
-        // sadness - we don't have spans for sub-expressions nor access to the tokens
-        // so in order to get extents for the function name itself (which dxr expects)
-        // we need to re-tokenise the fn definition
-
-        // Note: this is a bit awful - it adds the contents of span to the end of
-        // the codemap as a new filemap. This is mostly OK, but means we should
-        // not iterate over the codemap. Also, any spans over the new filemap
-        // are incompatible with spans over other filemaps.
-        let filemap = self.sess
-                          .codemap()
-                          .new_filemap(String::from("<anon-dxr>"), None, self.snippet(span));
-        lexer::StringReader::new(&self.sess.parse_sess, filemap)
-    }
-
-    fn span_to_tts(&self, span: Span) -> Vec<TokenTree> {
-        let filename = String::from("<anon-dxr>");
-        let filemap = self.sess.codemap().new_filemap(filename, None, self.snippet(span));
-        filemap_to_tts(&self.sess.parse_sess, filemap)
+        lexer::StringReader::retokenize(&self.sess.parse_sess, span)
     }
 
     // Re-parses a path and returns the span for the last identifier in the path
@@ -103,7 +68,7 @@ impl<'a> SpanUtils<'a> {
         loop {
             let ts = toks.real_token();
             if ts.tok == token::Eof {
-                return self.make_sub_span(span, result)
+                return result
             }
             if bracket_count == 0 && (ts.tok.is_ident() || ts.tok.is_keyword(keywords::SelfValue)) {
                 result = Some(ts.sp);
@@ -128,7 +93,7 @@ impl<'a> SpanUtils<'a> {
                 return None;
             }
             if bracket_count == 0 && (ts.tok.is_ident() || ts.tok.is_keyword(keywords::SelfValue)) {
-                return self.make_sub_span(span, Some(ts.sp));
+                return Some(ts.sp);
             }
 
             bracket_count += match ts.tok {
@@ -178,10 +143,7 @@ impl<'a> SpanUtils<'a> {
             }
             prev = next;
         }
-        if result.is_none() && prev_span.is_some() {
-            return self.make_sub_span(span, prev_span);
-        }
-        return self.make_sub_span(span, result);
+        result.or(prev_span)
     }
 
     // Return the span for the last ident before a `<` and outside any
@@ -241,9 +203,9 @@ impl<'a> SpanUtils<'a> {
                       loc.line);
         }
         if result.is_none() && prev.tok.is_ident() && angle_count == 0 {
-            return self.make_sub_span(span, Some(prev.sp));
+            return Some(prev.sp);
         }
-        self.make_sub_span(span, result)
+        result
     }
 
     // Reparse span and return an owned vector of sub spans of the first limit
@@ -310,7 +272,7 @@ impl<'a> SpanUtils<'a> {
                 angle_count += 1;
             }
             if ts.tok.is_ident() && angle_count == nesting {
-                result.push(self.make_sub_span(span, Some(ts.sp)).unwrap());
+                result.push(ts.sp);
             }
         }
     }
@@ -320,8 +282,11 @@ impl<'a> SpanUtils<'a> {
     /// end of the 'signature' part, that is up to, but not including an opening
     /// brace or semicolon.
     pub fn signature_string_for_span(&self, span: Span) -> String {
-        let mut toks = self.span_to_tts(span).into_iter();
+        let mut toks = self.retokenise_span(span);
+        toks.real_token();
+        let mut toks = toks.parse_all_token_trees().unwrap().into_iter();
         let mut prev = toks.next().unwrap();
+
         let first_span = prev.get_span();
         let mut angle_count = 0;
         for tok in toks {
@@ -360,7 +325,7 @@ impl<'a> SpanUtils<'a> {
             }
             let next = toks.real_token();
             if next.tok == tok {
-                return self.make_sub_span(span, Some(prev.sp));
+                return Some(prev.sp);
             }
             prev = next;
         }
@@ -374,7 +339,7 @@ impl<'a> SpanUtils<'a> {
                 return None;
             }
             if next.tok == tok {
-                return self.make_sub_span(span, Some(next.sp));
+                return Some(next.sp);
             }
         }
     }
@@ -399,7 +364,7 @@ impl<'a> SpanUtils<'a> {
                 if ts.tok == token::Eof {
                     return None
                 } else {
-                    return self.make_sub_span(span, Some(ts.sp));
+                    return Some(ts.sp);
                 }
             }
         }
@@ -444,7 +409,7 @@ impl<'a> SpanUtils<'a> {
             if ts.tok == token::Not {
                 let ts = toks.real_token();
                 if ts.tok.is_ident() {
-                    return self.make_sub_span(span, Some(ts.sp));
+                    return Some(ts.sp);
                 } else {
                     return None;
                 }
@@ -463,7 +428,7 @@ impl<'a> SpanUtils<'a> {
             let ts = toks.real_token();
             if ts.tok == token::Not {
                 if prev.tok.is_ident() {
-                    return self.make_sub_span(span, Some(prev.sp));
+                    return Some(prev.sp);
                 } else {
                     return None;
                 }
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
@@ -51,10 +51,10 @@ pub struct StringReader<'a> {
     pub filemap: Rc<syntax_pos::FileMap>,
     /// If Some, stop reading the source at this position (inclusive).
     pub terminator: Option<BytePos>,
-    /// Whether to record new-lines in filemap. This is only necessary the first
-    /// time a filemap is lexed. If part of a filemap is being re-lexed, this
-    /// should be set to false.
-    pub save_new_lines: bool,
+    /// Whether to record new-lines and multibyte chars in filemap.
+    /// This is only necessary the first time a filemap is lexed.
+    /// If part of a filemap is being re-lexed, this should be set to false.
+    pub save_new_lines_and_multibyte: bool,
     // cached:
     pub peek_tok: token::Token,
     pub peek_span: Span,
@@ -162,7 +162,7 @@ impl<'a> StringReader<'a> {
             ch: Some('\n'),
             filemap: filemap,
             terminator: None,
-            save_new_lines: true,
+            save_new_lines_and_multibyte: true,
             // dummy values; not read
             peek_tok: token::Eof,
             peek_span: syntax_pos::DUMMY_SP,
@@ -183,6 +183,31 @@ impl<'a> StringReader<'a> {
         sr
     }
 
+    pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
+        let begin = sess.codemap().lookup_byte_offset(span.lo);
+        let end = sess.codemap().lookup_byte_offset(span.hi);
+
+        // Make the range zero-length if the span is invalid.
+        if span.lo > span.hi || begin.fm.start_pos != end.fm.start_pos {
+            span.hi = span.lo;
+        }
+
+        let mut sr = StringReader::new_raw_internal(sess, begin.fm);
+
+        // Seek the lexer to the right byte range.
+        sr.save_new_lines_and_multibyte = false;
+        sr.next_pos = span.lo;
+        sr.terminator = Some(span.hi);
+
+        sr.bump();
+
+        if let Err(_) = sr.advance_token() {
+            sr.emit_fatal_errors();
+            panic!(FatalError);
+        }
+        sr
+    }
+
     pub fn ch_is(&self, c: char) -> bool {
         self.ch == Some(c)
     }
@@ -378,7 +403,10 @@ impl<'a> StringReader<'a> {
     pub fn bump(&mut self) {
         let new_pos = self.next_pos;
         let new_byte_offset = self.byte_offset(new_pos).to_usize();
-        if new_byte_offset < self.source_text.len() {
+        let end = self.terminator.map_or(self.source_text.len(), |t| {
+            self.byte_offset(t).to_usize()
+        });
+        if new_byte_offset < end {
             let old_ch_is_newline = self.ch.unwrap() == '\n';
             let new_ch = char_at(&self.source_text, new_byte_offset);
             let new_ch_len = new_ch.len_utf8();
@@ -387,15 +415,17 @@ impl<'a> StringReader<'a> {
             self.pos = new_pos;
             self.next_pos = new_pos + Pos::from_usize(new_ch_len);
             if old_ch_is_newline {
-                if self.save_new_lines {
+                if self.save_new_lines_and_multibyte {
                     self.filemap.next_line(self.pos);
                 }
                 self.col = CharPos(0);
             } else {
                 self.col = self.col + CharPos(1);
             }
             if new_ch_len > 1 {
-                self.filemap.record_multibyte_char(self.pos, new_ch_len);
+                if self.save_new_lines_and_multibyte {
+                    self.filemap.record_multibyte_char(self.pos, new_ch_len);
+                }
             }
         } else {
             self.ch = None;

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,6 @@ use std::env;`
`17`	`17`	`use std::path::Path;`
`18`	`18`
`19`	`19`	`use syntax::ast;`
`20`		`-use syntax::parse::filemap_to_tts;`
`21`	`20`	`use syntax::parse::lexer::{self, StringReader};`
`22`	`21`	`use syntax::parse::token::{self, Token};`
`23`	`22`	`use syntax::symbol::keywords;`
`@@ -49,23 +48,6 @@ impl<'a> SpanUtils<'a> {`
`49`	`48`	`}`
`50`	`49`	`}`
`51`	`50`
`52`		`- // sub_span starts at span.lo, so we need to adjust the positions etc.`
`53`		`- // If sub_span is None, we don't need to adjust.`
`54`		`- pub fn make_sub_span(&self, span: Span, sub_span: Option<Span>) -> Option<Span> {`
`55`		`- match sub_span {`
`56`		`- None => None,`
`57`		`- Some(sub) => {`
`58`		`- let FileMapAndBytePos {fm, pos} = self.sess.codemap().lookup_byte_offset(span.lo);`
`59`		`- let base = pos + fm.start_pos;`
`60`		`- Some(Span {`
`61`		`- lo: base + self.sess.codemap().lookup_byte_offset(sub.lo).pos,`
`62`		`- hi: base + self.sess.codemap().lookup_byte_offset(sub.hi).pos,`
`63`		`- expn_id: span.expn_id,`
`64`		`- })`
`65`		`- }`
`66`		`- }`
`67`		`- }`
`68`		`-`
`69`	`51`	`pub fn snippet(&self, span: Span) -> String {`
`70`	`52`	`match self.sess.codemap().span_to_snippet(span) {`
`71`	`53`	`Ok(s) => s,`
`@@ -74,24 +56,7 @@ impl<'a> SpanUtils<'a> {`
`74`	`56`	`}`
`75`	`57`
`76`	`58`	`pub fn retokenise_span(&self, span: Span) -> StringReader<'a> {`
`77`		`- // sadness - we don't have spans for sub-expressions nor access to the tokens`
`78`		`- // so in order to get extents for the function name itself (which dxr expects)`
`79`		`- // we need to re-tokenise the fn definition`
`80`		`-`
`81`		`- // Note: this is a bit awful - it adds the contents of span to the end of`
`82`		`- // the codemap as a new filemap. This is mostly OK, but means we should`
`83`		`- // not iterate over the codemap. Also, any spans over the new filemap`
`84`		`- // are incompatible with spans over other filemaps.`
`85`		`- let filemap = self.sess`
`86`		`- .codemap()`
`87`		`- .new_filemap(String::from("<anon-dxr>"), None, self.snippet(span));`
`88`		`- lexer::StringReader::new(&self.sess.parse_sess, filemap)`
`89`		`- }`
`90`		`-`
`91`		`- fn span_to_tts(&self, span: Span) -> Vec<TokenTree> {`
`92`		`- let filename = String::from("<anon-dxr>");`
`93`		`- let filemap = self.sess.codemap().new_filemap(filename, None, self.snippet(span));`
`94`		`- filemap_to_tts(&self.sess.parse_sess, filemap)`
	`59`	`+ lexer::StringReader::retokenize(&self.sess.parse_sess, span)`
`95`	`60`	`}`
`96`	`61`
`97`	`62`	`// Re-parses a path and returns the span for the last identifier in the path`
`@@ -103,7 +68,7 @@ impl<'a> SpanUtils<'a> {`
`103`	`68`	`loop {`
`104`	`69`	`let ts = toks.real_token();`
`105`	`70`	`if ts.tok == token::Eof {`
`106`		`- return self.make_sub_span(span, result)`
	`71`	`+ return result`
`107`	`72`	`}`
`108`	`73`	`if bracket_count == 0 && (ts.tok.is_ident() \|\| ts.tok.is_keyword(keywords::SelfValue)) {`
`109`	`74`	`result = Some(ts.sp);`
`@@ -128,7 +93,7 @@ impl<'a> SpanUtils<'a> {`
`128`	`93`	`return None;`
`129`	`94`	`}`
`130`	`95`	`if bracket_count == 0 && (ts.tok.is_ident() \|\| ts.tok.is_keyword(keywords::SelfValue)) {`
`131`		`- return self.make_sub_span(span, Some(ts.sp));`
	`96`	`+ return Some(ts.sp);`
`132`	`97`	`}`
`133`	`98`
`134`	`99`	`bracket_count += match ts.tok {`
`@@ -178,10 +143,7 @@ impl<'a> SpanUtils<'a> {`
`178`	`143`	`}`
`179`	`144`	`prev = next;`
`180`	`145`	`}`
`181`		`- if result.is_none() && prev_span.is_some() {`
`182`		`- return self.make_sub_span(span, prev_span);`
`183`		`- }`
`184`		`- return self.make_sub_span(span, result);`
	`146`	`+ result.or(prev_span)`
`185`	`147`	`}`
`186`	`148`
`187`	`149`	// Return the span for the last ident before a `<` and outside any
`@@ -241,9 +203,9 @@ impl<'a> SpanUtils<'a> {`
`241`	`203`	`loc.line);`
`242`	`204`	`}`
`243`	`205`	`if result.is_none() && prev.tok.is_ident() && angle_count == 0 {`
`244`		`- return self.make_sub_span(span, Some(prev.sp));`
	`206`	`+ return Some(prev.sp);`
`245`	`207`	`}`
`246`		`- self.make_sub_span(span, result)`
	`208`	`+ result`
`247`	`209`	`}`
`248`	`210`
`249`	`211`	`// Reparse span and return an owned vector of sub spans of the first limit`
`@@ -310,7 +272,7 @@ impl<'a> SpanUtils<'a> {`
`310`	`272`	`angle_count += 1;`
`311`	`273`	`}`
`312`	`274`	`if ts.tok.is_ident() && angle_count == nesting {`
`313`		`- result.push(self.make_sub_span(span, Some(ts.sp)).unwrap());`
	`275`	`+ result.push(ts.sp);`
`314`	`276`	`}`
`315`	`277`	`}`
`316`	`278`	`}`
`@@ -320,8 +282,11 @@ impl<'a> SpanUtils<'a> {`
`320`	`282`	`/// end of the 'signature' part, that is up to, but not including an opening`
`321`	`283`	`/// brace or semicolon.`
`322`	`284`	`pub fn signature_string_for_span(&self, span: Span) -> String {`
`323`		`- let mut toks = self.span_to_tts(span).into_iter();`
	`285`	`+ let mut toks = self.retokenise_span(span);`
	`286`	`+ toks.real_token();`
	`287`	`+ let mut toks = toks.parse_all_token_trees().unwrap().into_iter();`
`324`	`288`	`let mut prev = toks.next().unwrap();`
	`289`	`+`
`325`	`290`	`let first_span = prev.get_span();`
`326`	`291`	`let mut angle_count = 0;`
`327`	`292`	`for tok in toks {`
`@@ -360,7 +325,7 @@ impl<'a> SpanUtils<'a> {`
`360`	`325`	`}`
`361`	`326`	`let next = toks.real_token();`
`362`	`327`	`if next.tok == tok {`
`363`		`- return self.make_sub_span(span, Some(prev.sp));`
	`328`	`+ return Some(prev.sp);`
`364`	`329`	`}`
`365`	`330`	`prev = next;`
`366`	`331`	`}`
`@@ -374,7 +339,7 @@ impl<'a> SpanUtils<'a> {`
`374`	`339`	`return None;`
`375`	`340`	`}`
`376`	`341`	`if next.tok == tok {`
`377`		`- return self.make_sub_span(span, Some(next.sp));`
	`342`	`+ return Some(next.sp);`
`378`	`343`	`}`
`379`	`344`	`}`
`380`	`345`	`}`
`@@ -399,7 +364,7 @@ impl<'a> SpanUtils<'a> {`
`399`	`364`	`if ts.tok == token::Eof {`
`400`	`365`	`return None`
`401`	`366`	`} else {`
`402`		`- return self.make_sub_span(span, Some(ts.sp));`
	`367`	`+ return Some(ts.sp);`
`403`	`368`	`}`
`404`	`369`	`}`
`405`	`370`	`}`
`@@ -444,7 +409,7 @@ impl<'a> SpanUtils<'a> {`
`444`	`409`	`if ts.tok == token::Not {`
`445`	`410`	`let ts = toks.real_token();`
`446`	`411`	`if ts.tok.is_ident() {`
`447`		`- return self.make_sub_span(span, Some(ts.sp));`
	`412`	`+ return Some(ts.sp);`
`448`	`413`	`} else {`
`449`	`414`	`return None;`
`450`	`415`	`}`
`@@ -463,7 +428,7 @@ impl<'a> SpanUtils<'a> {`
`463`	`428`	`let ts = toks.real_token();`
`464`	`429`	`if ts.tok == token::Not {`
`465`	`430`	`if prev.tok.is_ident() {`
`466`		`- return self.make_sub_span(span, Some(prev.sp));`
	`431`	`+ return Some(prev.sp);`
`467`	`432`	`} else {`
`468`	`433`	`return None;`
`469`	`434`	`}`