From 257d279fe47bbf3431c76f0942654c1bcf60d501 Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Wed, 23 May 2018 15:59:42 +0200 Subject: Make FileMap::{lines, multibyte_chars, non_narrow_chars} non-mutable. --- src/libsyntax/codemap.rs | 40 ++++++++++++++++------------------------ src/libsyntax/parse/lexer/mod.rs | 20 +------------------- 2 files changed, 17 insertions(+), 43 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/codemap.rs b/src/libsyntax/codemap.rs index 8e4b7660a1c..000f1607514 100644 --- a/src/libsyntax/codemap.rs +++ b/src/libsyntax/codemap.rs @@ -250,16 +250,7 @@ impl CodeMap { /// Creates a new filemap and sets its line information. /// This does not ensure that only one FileMap exists per file name. pub fn new_filemap_and_lines(&self, filename: &Path, src: &str) -> Lrc { - let fm = self.new_filemap(filename.to_owned().into(), src.to_owned()); - let mut byte_pos: u32 = fm.start_pos.0; - for line in src.lines() { - // register the start of this line - fm.next_line(BytePos(byte_pos)); - - // update byte_pos to include this line and the \n at the end - byte_pos += line.len() as u32 + 1; - } - fm + self.new_filemap(filename.to_owned().into(), src.to_owned()) } @@ -305,9 +296,9 @@ impl CodeMap { external_src: Lock::new(ExternalSource::AbsentOk), start_pos, end_pos, - lines: Lock::new(file_local_lines), - multibyte_chars: Lock::new(file_local_multibyte_chars), - non_narrow_chars: Lock::new(file_local_non_narrow_chars), + lines: file_local_lines, + multibyte_chars: file_local_multibyte_chars, + non_narrow_chars: file_local_non_narrow_chars, name_hash, }); @@ -345,21 +336,22 @@ impl CodeMap { match self.lookup_line(pos) { Ok(FileMapAndLine { fm: f, line: a }) => { let line = a + 1; // Line numbers start at 1 - let linebpos = (*f.lines.borrow())[a]; + let linebpos = f.lines[a]; let linechpos = self.bytepos_to_file_charpos(linebpos); let col = chpos - linechpos; let col_display = { - let non_narrow_chars = f.non_narrow_chars.borrow(); - let start_width_idx = non_narrow_chars + let start_width_idx = f + .non_narrow_chars .binary_search_by_key(&linebpos, |x| x.pos()) .unwrap_or_else(|x| x); - let end_width_idx = non_narrow_chars + let end_width_idx = f + .non_narrow_chars .binary_search_by_key(&pos, |x| x.pos()) .unwrap_or_else(|x| x); let special_chars = end_width_idx - start_width_idx; - let non_narrow: usize = - non_narrow_chars[start_width_idx..end_width_idx] + let non_narrow: usize = f + .non_narrow_chars[start_width_idx..end_width_idx] .into_iter() .map(|x| x.width()) .sum(); @@ -380,12 +372,12 @@ impl CodeMap { } Err(f) => { let col_display = { - let non_narrow_chars = f.non_narrow_chars.borrow(); - let end_width_idx = non_narrow_chars + let end_width_idx = f + .non_narrow_chars .binary_search_by_key(&pos, |x| x.pos()) .unwrap_or_else(|x| x); - let non_narrow: usize = - non_narrow_chars[0..end_width_idx] + let non_narrow: usize = f + .non_narrow_chars[0..end_width_idx] .into_iter() .map(|x| x.width()) .sum(); @@ -830,7 +822,7 @@ impl CodeMap { // The number of extra bytes due to multibyte chars in the FileMap let mut total_extra_bytes = 0; - for mbc in map.multibyte_chars.borrow().iter() { + for mbc in map.multibyte_chars.iter() { debug!("{}-byte char at {:?}", mbc.bytes, mbc.pos); if mbc.pos < bpos { // every character is at least one byte, so we only diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index c09cfd910d2..dcc71e78778 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -51,11 +51,7 @@ pub struct StringReader<'a> { pub ch: Option, pub filemap: Lrc, /// Stop reading src at this index. - end_src_index: usize, - /// Whether to record new-lines and multibyte chars in filemap. - /// This is only necessary the first time a filemap is lexed. - /// If part of a filemap is being re-lexed, this should be set to false. - save_new_lines_and_multibyte: bool, + pub end_src_index: usize, // cached: peek_tok: token::Token, peek_span: Span, @@ -188,7 +184,6 @@ impl<'a> StringReader<'a> { ch: Some('\n'), filemap, end_src_index: src.len(), - save_new_lines_and_multibyte: true, // dummy values; not read peek_tok: token::Eof, peek_span: syntax_pos::DUMMY_SP, @@ -225,7 +220,6 @@ impl<'a> StringReader<'a> { let mut sr = StringReader::new_raw_internal(sess, begin.fm, None); // Seek the lexer to the right byte range. - sr.save_new_lines_and_multibyte = false; sr.next_pos = span.lo(); sr.end_src_index = sr.src_index(span.hi()); @@ -458,18 +452,6 @@ impl<'a> StringReader<'a> { let next_ch = char_at(&self.src, next_src_index); let next_ch_len = next_ch.len_utf8(); - if self.ch.unwrap() == '\n' { - if self.save_new_lines_and_multibyte { - self.filemap.next_line(self.next_pos); - } - } - if next_ch_len > 1 { - if self.save_new_lines_and_multibyte { - self.filemap.record_multibyte_char(self.next_pos, next_ch_len); - } - } - self.filemap.record_width(self.next_pos, next_ch); - self.ch = Some(next_ch); self.pos = self.next_pos; self.next_pos = self.next_pos + Pos::from_usize(next_ch_len); -- cgit 1.4.1-3-g733a5 From 095a339bec62e81e38727d16cc8f275c6818061e Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Wed, 23 May 2018 16:19:20 +0200 Subject: Remove the now redundant CodeMap::new_filemap_with_lines() method. --- src/libsyntax/codemap.rs | 14 +++----------- src/libsyntax/ext/expand.rs | 6 ++++-- src/libsyntax/ext/source_util.rs | 8 +++++--- src/libsyntax/test_snippet.rs | 2 +- 4 files changed, 13 insertions(+), 17 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/codemap.rs b/src/libsyntax/codemap.rs index 000f1607514..5e23c1b0d62 100644 --- a/src/libsyntax/codemap.rs +++ b/src/libsyntax/codemap.rs @@ -211,8 +211,7 @@ impl CodeMap { } } - /// Creates a new filemap without setting its line information. If you don't - /// intend to set the line information yourself, you should use new_filemap_and_lines. + /// Creates a new filemap. /// This does not ensure that only one FileMap exists per file name. pub fn new_filemap(&self, filename: FileName, src: String) -> Lrc { let start_pos = self.next_start_pos(); @@ -247,13 +246,6 @@ impl CodeMap { filemap } - /// Creates a new filemap and sets its line information. - /// This does not ensure that only one FileMap exists per file name. - pub fn new_filemap_and_lines(&self, filename: &Path, src: &str) -> Lrc { - self.new_filemap(filename.to_owned().into(), src.to_owned()) - } - - /// Allocates a new FileMap representing a source file from an external /// crate. The source code of such an "imported filemap" is not available, /// but we still know enough to generate accurate debuginfo location @@ -1188,7 +1180,7 @@ mod tests { let cm = CodeMap::new(FilePathMapping::empty()); let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n"; let selection = " \n ~~\n~~~\n~~~~~ \n \n"; - cm.new_filemap_and_lines(Path::new("blork.rs"), inputtext); + cm.new_filemap(Path::new("blork.rs").to_owned().into(), inputtext.to_string()); let span = span_from_selection(inputtext, selection); // check that we are extracting the text we thought we were extracting @@ -1231,7 +1223,7 @@ mod tests { let inputtext = "bbbb BB\ncc CCC\n"; let selection1 = " ~~\n \n"; let selection2 = " \n ~~~\n"; - cm.new_filemap_and_lines(Path::new("blork.rs"), inputtext); + cm.new_filemap(Path::new("blork.rs").to_owned().into(), inputtext.to_owned()); let span1 = span_from_selection(inputtext, selection1); let span2 = span_from_selection(inputtext, selection2); diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 69c99c63aaf..094e572693c 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1487,9 +1487,11 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { match String::from_utf8(buf) { Ok(src) => { + let src_interned = Symbol::intern(&src); + // Add this input file to the code map to make it available as // dependency information - self.cx.codemap().new_filemap_and_lines(&filename, &src); + self.cx.codemap().new_filemap(filename.into(), src); let include_info = vec![ dummy_spanned(ast::NestedMetaItemKind::MetaItem( @@ -1497,7 +1499,7 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> { dummy_spanned(file)))), dummy_spanned(ast::NestedMetaItemKind::MetaItem( attr::mk_name_value_item_str(Ident::from_str("contents"), - dummy_spanned(Symbol::intern(&src))))), + dummy_spanned(src_interned)))), ]; let include_ident = Ident::from_str("include"); diff --git a/src/libsyntax/ext/source_util.rs b/src/libsyntax/ext/source_util.rs index d6dce63ea5e..669536f519c 100644 --- a/src/libsyntax/ext/source_util.rs +++ b/src/libsyntax/ext/source_util.rs @@ -150,11 +150,13 @@ pub fn expand_include_str(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenT }; match String::from_utf8(bytes) { Ok(src) => { + let interned_src = Symbol::intern(&src); + // Add this input file to the code map to make it available as // dependency information - cx.codemap().new_filemap_and_lines(&file, &src); + cx.codemap().new_filemap(file.into(), src); - base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&src))) + base::MacEager::expr(cx.expr_str(sp, interned_src)) } Err(_) => { cx.span_err(sp, @@ -182,7 +184,7 @@ pub fn expand_include_bytes(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::Toke Ok(..) => { // Add this input file to the code map to make it available as // dependency information, but don't enter it's contents - cx.codemap().new_filemap_and_lines(&file, ""); + cx.codemap().new_filemap(file.into(), "".to_string()); base::MacEager::expr(cx.expr_lit(sp, ast::LitKind::ByteStr(Lrc::new(bytes)))) } diff --git a/src/libsyntax/test_snippet.rs b/src/libsyntax/test_snippet.rs index 81dcc1998ed..c7e4fbd1073 100644 --- a/src/libsyntax/test_snippet.rs +++ b/src/libsyntax/test_snippet.rs @@ -51,7 +51,7 @@ fn test_harness(file_text: &str, span_labels: Vec, expected_output: & let output = Arc::new(Mutex::new(Vec::new())); let code_map = Lrc::new(CodeMap::new(FilePathMapping::empty())); - code_map.new_filemap_and_lines(Path::new("test.rs"), &file_text); + code_map.new_filemap(Path::new("test.rs").to_owned().into(), file_text.to_owned()); let primary_span = make_span(&file_text, &span_labels[0].start, &span_labels[0].end); let mut msp = MultiSpan::from_span(primary_span); -- cgit 1.4.1-3-g733a5 From 04d4da1bf9df23f1bc2ad22115642735bda8b39e Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Thu, 24 May 2018 11:30:30 +0200 Subject: Update CodeMap tests after changing FileMap construction. --- src/libsyntax/codemap.rs | 71 +++++++----------------------------------------- 1 file changed, 10 insertions(+), 61 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/codemap.rs b/src/libsyntax/codemap.rs index 5e23c1b0d62..2aa2564b097 100644 --- a/src/libsyntax/codemap.rs +++ b/src/libsyntax/codemap.rs @@ -1012,51 +1012,16 @@ impl FilePathMapping { #[cfg(test)] mod tests { use super::*; - use std::borrow::Cow; use rustc_data_structures::sync::Lrc; - #[test] - fn t1 () { - let cm = CodeMap::new(FilePathMapping::empty()); - let fm = cm.new_filemap(PathBuf::from("blork.rs").into(), - "first line.\nsecond line".to_string()); - fm.next_line(BytePos(0)); - // Test we can get lines with partial line info. - assert_eq!(fm.get_line(0), Some(Cow::from("first line."))); - // TESTING BROKEN BEHAVIOR: line break declared before actual line break. - fm.next_line(BytePos(10)); - assert_eq!(fm.get_line(1), Some(Cow::from("."))); - fm.next_line(BytePos(12)); - assert_eq!(fm.get_line(2), Some(Cow::from("second line"))); - } - - #[test] - #[should_panic] - fn t2 () { - let cm = CodeMap::new(FilePathMapping::empty()); - let fm = cm.new_filemap(PathBuf::from("blork.rs").into(), - "first line.\nsecond line".to_string()); - // TESTING *REALLY* BROKEN BEHAVIOR: - fm.next_line(BytePos(0)); - fm.next_line(BytePos(10)); - fm.next_line(BytePos(2)); - } - fn init_code_map() -> CodeMap { let cm = CodeMap::new(FilePathMapping::empty()); - let fm1 = cm.new_filemap(PathBuf::from("blork.rs").into(), - "first line.\nsecond line".to_string()); - let fm2 = cm.new_filemap(PathBuf::from("empty.rs").into(), - "".to_string()); - let fm3 = cm.new_filemap(PathBuf::from("blork2.rs").into(), - "first line.\nsecond line".to_string()); - - fm1.next_line(BytePos(0)); - fm1.next_line(BytePos(12)); - fm2.next_line(fm2.start_pos); - fm3.next_line(fm3.start_pos); - fm3.next_line(fm3.start_pos + BytePos(12)); - + cm.new_filemap(PathBuf::from("blork.rs").into(), + "first line.\nsecond line".to_string()); + cm.new_filemap(PathBuf::from("empty.rs").into(), + "".to_string()); + cm.new_filemap(PathBuf::from("blork2.rs").into(), + "first line.\nsecond line".to_string()); cm } @@ -1109,26 +1074,10 @@ mod tests { fn init_code_map_mbc() -> CodeMap { let cm = CodeMap::new(FilePathMapping::empty()); // € is a three byte utf8 char. - let fm1 = - cm.new_filemap(PathBuf::from("blork.rs").into(), - "fir€st €€€€ line.\nsecond line".to_string()); - let fm2 = cm.new_filemap(PathBuf::from("blork2.rs").into(), - "first line€€.\n€ second line".to_string()); - - fm1.next_line(BytePos(0)); - fm1.next_line(BytePos(28)); - fm2.next_line(fm2.start_pos); - fm2.next_line(fm2.start_pos + BytePos(20)); - - fm1.record_multibyte_char(BytePos(3), 3); - fm1.record_multibyte_char(BytePos(9), 3); - fm1.record_multibyte_char(BytePos(12), 3); - fm1.record_multibyte_char(BytePos(15), 3); - fm1.record_multibyte_char(BytePos(18), 3); - fm2.record_multibyte_char(fm2.start_pos + BytePos(10), 3); - fm2.record_multibyte_char(fm2.start_pos + BytePos(13), 3); - fm2.record_multibyte_char(fm2.start_pos + BytePos(18), 3); - + cm.new_filemap(PathBuf::from("blork.rs").into(), + "fir€st €€€€ line.\nsecond line".to_string()); + cm.new_filemap(PathBuf::from("blork2.rs").into(), + "first line€€.\n€ second line".to_string()); cm } -- cgit 1.4.1-3-g733a5 From 3497138634bf58a7c29ef35f1f677dbde0633af8 Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Tue, 29 May 2018 17:49:35 +0200 Subject: Use u32 instead of usize of encoding byte count of multi-byte chars. --- src/libsyntax/codemap.rs | 6 +++--- src/libsyntax_pos/lib.rs | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/codemap.rs b/src/libsyntax/codemap.rs index 2aa2564b097..68882c4f063 100644 --- a/src/libsyntax/codemap.rs +++ b/src/libsyntax/codemap.rs @@ -822,14 +822,14 @@ impl CodeMap { total_extra_bytes += mbc.bytes - 1; // We should never see a byte position in the middle of a // character - assert!(bpos.to_usize() >= mbc.pos.to_usize() + mbc.bytes); + assert!(bpos.to_u32() >= mbc.pos.to_u32() + mbc.bytes); } else { break; } } - assert!(map.start_pos.to_usize() + total_extra_bytes <= bpos.to_usize()); - CharPos(bpos.to_usize() - map.start_pos.to_usize() - total_extra_bytes) + assert!(map.start_pos.to_u32() + total_extra_bytes <= bpos.to_u32()); + CharPos(bpos.to_usize() - map.start_pos.to_usize() - total_extra_bytes as usize) } // Return the index of the filemap (in self.files) which contains pos. diff --git a/src/libsyntax_pos/lib.rs b/src/libsyntax_pos/lib.rs index 266737dd7b6..93b65dac288 100644 --- a/src/libsyntax_pos/lib.rs +++ b/src/libsyntax_pos/lib.rs @@ -657,7 +657,7 @@ pub struct MultiByteChar { /// The absolute offset of the character in the CodeMap pub pos: BytePos, /// The number of bytes, >=2 - pub bytes: usize, + pub bytes: u32, } /// Identifies an offset of a non-narrow character in a FileMap @@ -1174,6 +1174,8 @@ fn remove_bom(src: &mut String) { pub trait Pos { fn from_usize(n: usize) -> Self; fn to_usize(&self) -> usize; + fn from_u32(n: u32) -> Self; + fn to_u32(&self) -> u32; } /// A byte offset. Keep this small (currently 32-bits), as AST contains @@ -1195,7 +1197,13 @@ impl Pos for BytePos { fn from_usize(n: usize) -> BytePos { BytePos(n as u32) } #[inline(always)] - fn to_usize(&self) -> usize { let BytePos(n) = *self; n as usize } + fn to_usize(&self) -> usize { self.0 as usize } + + #[inline(always)] + fn from_u32(n: u32) -> BytePos { BytePos(n) } + + #[inline(always)] + fn to_u32(&self) -> u32 { self.0 } } impl Add for BytePos { @@ -1233,7 +1241,13 @@ impl Pos for CharPos { fn from_usize(n: usize) -> CharPos { CharPos(n) } #[inline(always)] - fn to_usize(&self) -> usize { let CharPos(n) = *self; n } + fn to_usize(&self) -> usize { self.0 } + + #[inline(always)] + fn from_u32(n: u32) -> CharPos { CharPos(n as usize) } + + #[inline(always)] + fn to_u32(&self) -> u32 { self.0 as u32} } impl Add for CharPos { -- cgit 1.4.1-3-g733a5 From ba30c1dac9d0af45836403b2da89561a627b9a6e Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Tue, 26 Jun 2018 15:37:09 +0200 Subject: syntax_pos: Store multibyte char size as u8 instead of u32. --- src/libsyntax/codemap.rs | 4 ++-- src/libsyntax_pos/analyze_filemap.rs | 2 +- src/libsyntax_pos/lib.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/codemap.rs b/src/libsyntax/codemap.rs index 68882c4f063..1d5429bdf8f 100644 --- a/src/libsyntax/codemap.rs +++ b/src/libsyntax/codemap.rs @@ -819,10 +819,10 @@ impl CodeMap { if mbc.pos < bpos { // every character is at least one byte, so we only // count the actual extra bytes. - total_extra_bytes += mbc.bytes - 1; + total_extra_bytes += mbc.bytes as u32 - 1; // We should never see a byte position in the middle of a // character - assert!(bpos.to_u32() >= mbc.pos.to_u32() + mbc.bytes); + assert!(bpos.to_u32() >= mbc.pos.to_u32() + mbc.bytes as u32); } else { break; } diff --git a/src/libsyntax_pos/analyze_filemap.rs b/src/libsyntax_pos/analyze_filemap.rs index 99fc3f19009..c7c0263e459 100644 --- a/src/libsyntax_pos/analyze_filemap.rs +++ b/src/libsyntax_pos/analyze_filemap.rs @@ -263,7 +263,7 @@ fn analyze_filemap_generic(src: &str, assert!(char_len >=2 && char_len <= 4); let mbc = MultiByteChar { pos, - bytes: char_len as u32, + bytes: char_len as u8, }; multi_byte_chars.push(mbc); } diff --git a/src/libsyntax_pos/lib.rs b/src/libsyntax_pos/lib.rs index 90f3ae90c2f..5502b30e488 100644 --- a/src/libsyntax_pos/lib.rs +++ b/src/libsyntax_pos/lib.rs @@ -663,7 +663,7 @@ pub struct MultiByteChar { /// The absolute offset of the character in the CodeMap pub pos: BytePos, /// The number of bytes, >=2 - pub bytes: u32, + pub bytes: u8, } /// Identifies an offset of a non-narrow character in a FileMap -- cgit 1.4.1-3-g733a5 From a1f8a6ce80a340d51074071c0d9e30eb14f65d25 Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Thu, 28 Jun 2018 10:45:57 +0200 Subject: Fix FileMap::line_begin_pos(). The method relied on the FileMap still being under construction in order for it to do what the name promises. It's now independent of the current state. --- src/libsyntax/parse/lexer/comments.rs | 6 ++++-- src/libsyntax_pos/lib.rs | 8 +++----- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index 7da0d816d0f..3995a9b8689 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -240,9 +240,11 @@ fn read_block_comment(rdr: &mut StringReader, let mut lines: Vec = Vec::new(); // Count the number of chars since the start of the line by rescanning. - let mut src_index = rdr.src_index(rdr.filemap.line_begin_pos()); + let mut src_index = rdr.src_index(rdr.filemap.line_begin_pos(rdr.pos)); let end_src_index = rdr.src_index(rdr.pos); - assert!(src_index <= end_src_index); + assert!(src_index <= end_src_index, + "src_index={}, end_src_index={}, line_begin_pos={}", + src_index, end_src_index, rdr.filemap.line_begin_pos(rdr.pos).to_u32()); let mut n = 0; while src_index < end_src_index { let c = char_at(&rdr.src, src_index); diff --git a/src/libsyntax_pos/lib.rs b/src/libsyntax_pos/lib.rs index 5502b30e488..55dec31511c 100644 --- a/src/libsyntax_pos/lib.rs +++ b/src/libsyntax_pos/lib.rs @@ -976,11 +976,9 @@ impl FileMap { } /// Return the BytePos of the beginning of the current line. - pub fn line_begin_pos(&self) -> BytePos { - match self.lines.last() { - Some(&line_pos) => line_pos, - None => self.start_pos, - } + pub fn line_begin_pos(&self, pos: BytePos) -> BytePos { + let line_index = self.lookup_line(pos).unwrap(); + self.lines[line_index] } /// Add externally loaded source. -- cgit 1.4.1-3-g733a5