Rollup merge of #65074 - Rantanen:json-byte-pos, r=matklad

Fix the start/end byte positions in the compiler JSON output

Track the changes made during normalization in the `SourceFile` and use this information to correct the `start_byte` and `end_byte` fields in the JSON output.

This should ensure the start/end byte fields can be used to index the original file, even if Rust normalized the source code for parsing purposes. Both CRLF to LF and BOM removal are handled with this one.

The rough plan was discussed with @matklad in rust-lang-nursery/rustfix#176 - although I ended up going with `u32` offset tracking so I wouldn't need to deal with `u32 + i32` arithmetics when applying the offset to the span byte positions.

Fixes #65029

14 files changed, 543 insertions, 22 deletions

diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs
index dc1f6fd3131..2510d7efb59 100644
--- a/src/librustc/ich/impls_syntax.rs
+++ b/src/librustc/ich/impls_syntax.rs
@@ -425,6 +425,7 @@ impl<'a> HashStable<StableHashingContext<'a>> for SourceFile {
             ref lines,
             ref multibyte_chars,
             ref non_narrow_chars,
+            ref normalized_pos,
         } = *self;
 
         (name_hash as u64).hash_stable(hcx, hasher);
@@ -453,6 +454,12 @@ impl<'a> HashStable<StableHashingContext<'a>> for SourceFile {
         for &char_pos in non_narrow_chars.iter() {
             stable_non_narrow_char(char_pos, start_pos).hash_stable(hcx, hasher);
         }
+
+        normalized_pos.len().hash_stable(hcx, hasher);
+        for &char_pos in normalized_pos.iter() {
+            stable_normalized_pos(char_pos, start_pos).hash_stable(hcx, hasher);
+        }
+
     }
 }
 
@@ -482,6 +489,18 @@ fn stable_non_narrow_char(swc: ::syntax_pos::NonNarrowChar,
     (pos.0 - source_file_start.0, width as u32)
 }
 
+fn stable_normalized_pos(np: ::syntax_pos::NormalizedPos,
+                         source_file_start: ::syntax_pos::BytePos)
+                         -> (u32, u32) {
+    let ::syntax_pos::NormalizedPos {
+        pos,
+        diff
+    } = np;
+
+    (pos.0 - source_file_start.0, diff)
+}
+
+
 impl<'tcx> HashStable<StableHashingContext<'tcx>> for feature_gate::Features {
     fn hash_stable(&self, hcx: &mut StableHashingContext<'tcx>, hasher: &mut StableHasher) {
         // Unfortunately we cannot exhaustively list fields here, since the
diff --git a/src/librustc_metadata/decoder.rs b/src/librustc_metadata/decoder.rs
index 8c52168b418..09ff7891a9d 100644
--- a/src/librustc_metadata/decoder.rs
+++ b/src/librustc_metadata/decoder.rs
@@ -1319,6 +1319,7 @@ impl<'a, 'tcx> CrateMetadata {
                                       mut lines,
                                       mut multibyte_chars,
                                       mut non_narrow_chars,
+                                      mut normalized_pos,
                                       name_hash,
                                       .. } = source_file_to_import;
 
@@ -1338,6 +1339,9 @@ impl<'a, 'tcx> CrateMetadata {
             for swc in &mut non_narrow_chars {
                 *swc = *swc - start_pos;
             }
+            for np in &mut normalized_pos {
+                np.pos = np.pos - start_pos;
+            }
 
             let local_version = local_source_map.new_imported_source_file(name,
                                                                    name_was_remapped,
@@ -1347,7 +1351,8 @@ impl<'a, 'tcx> CrateMetadata {
                                                                    source_length,
                                                                    lines,
                                                                    multibyte_chars,
-                                                                   non_narrow_chars);
+                                                                   non_narrow_chars,
+                                                                   normalized_pos);
             debug!("CrateMetaData::imported_source_files alloc \
                     source_file {:?} original (start_pos {:?} end_pos {:?}) \
                     translated (start_pos {:?} end_pos {:?})",
diff --git a/src/libsyntax/json.rs b/src/libsyntax/json.rs
index e3296788d9f..0b157938375 100644
--- a/src/libsyntax/json.rs
+++ b/src/libsyntax/json.rs
@@ -25,6 +25,9 @@ use std::sync::{Arc, Mutex};
 
 use rustc_serialize::json::{as_json, as_pretty_json};
 
+#[cfg(test)]
+mod tests;
+
 pub struct JsonEmitter {
     dst: Box<dyn Write + Send>,
     registry: Option<Registry>,
@@ -336,8 +339,8 @@ impl DiagnosticSpan {
 
         DiagnosticSpan {
             file_name: start.file.name.to_string(),
-            byte_start: span.lo().0 - start.file.start_pos.0,
-            byte_end: span.hi().0 - start.file.start_pos.0,
+            byte_start: start.file.original_relative_byte_pos(span.lo()).0,
+            byte_end: start.file.original_relative_byte_pos(span.hi()).0,
             line_start: start.line,
             line_end: end.line,
             column_start: start.col.0 + 1,
diff --git a/src/libsyntax/json/tests.rs b/src/libsyntax/json/tests.rs
new file mode 100644
index 00000000000..eb0d9ef3947
--- /dev/null
+++ b/src/libsyntax/json/tests.rs
@@ -0,0 +1,186 @@
+use super::*;
+
+use crate::json::JsonEmitter;
+use crate::source_map::{FilePathMapping, SourceMap};
+use crate::tests::Shared;
+use crate::with_default_globals;
+
+use errors::emitter::{ColorConfig, HumanReadableErrorType};
+use errors::Handler;
+use rustc_serialize::json::decode;
+use syntax_pos::{BytePos, Span};
+
+use std::str;
+
+#[derive(RustcDecodable, Debug, PartialEq, Eq)]
+struct TestData {
+    spans: Vec<SpanTestData>,
+}
+
+#[derive(RustcDecodable, Debug, PartialEq, Eq)]
+struct SpanTestData {
+    pub byte_start: u32,
+    pub byte_end: u32,
+    pub line_start: u32,
+    pub column_start: u32,
+    pub line_end: u32,
+    pub column_end: u32,
+}
+
+/// Test the span yields correct positions in JSON.
+fn test_positions(code: &str, span: (u32, u32), expected_output: SpanTestData) {
+    let expected_output = TestData { spans: vec![expected_output] };
+
+    with_default_globals(|| {
+        let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
+        sm.new_source_file(Path::new("test.rs").to_owned().into(), code.to_owned());
+
+        let output = Arc::new(Mutex::new(Vec::new()));
+        let je = JsonEmitter::new(
+            Box::new(Shared { data: output.clone() }),
+            None,
+            sm,
+            true,
+            HumanReadableErrorType::Short(ColorConfig::Never),
+            false,
+        );
+
+        let span = Span::with_root_ctxt(BytePos(span.0), BytePos(span.1));
+        let handler = Handler::with_emitter(true, None, Box::new(je));
+        handler.span_err(span, "foo");
+
+        let bytes = output.lock().unwrap();
+        let actual_output = str::from_utf8(&bytes).unwrap();
+        let actual_output: TestData = decode(actual_output).unwrap();
+
+        assert_eq!(expected_output, actual_output)
+    })
+}
+
+#[test]
+fn empty() {
+    test_positions(
+        " ",
+        (0, 1),
+        SpanTestData {
+            byte_start: 0,
+            byte_end: 1,
+            line_start: 1,
+            column_start: 1,
+            line_end: 1,
+            column_end: 2,
+        },
+    )
+}
+
+#[test]
+fn bom() {
+    test_positions(
+        "\u{feff} ",
+        (0, 1),
+        SpanTestData {
+            byte_start: 3,
+            byte_end: 4,
+            line_start: 1,
+            column_start: 1,
+            line_end: 1,
+            column_end: 2,
+        },
+    )
+}
+
+#[test]
+fn lf_newlines() {
+    test_positions(
+        "\nmod foo;\nmod bar;\n",
+        (5, 12),
+        SpanTestData {
+            byte_start: 5,
+            byte_end: 12,
+            line_start: 2,
+            column_start: 5,
+            line_end: 3,
+            column_end: 3,
+        },
+    )
+}
+
+#[test]
+fn crlf_newlines() {
+    test_positions(
+        "\r\nmod foo;\r\nmod bar;\r\n",
+        (5, 12),
+        SpanTestData {
+            byte_start: 6,
+            byte_end: 14,
+            line_start: 2,
+            column_start: 5,
+            line_end: 3,
+            column_end: 3,
+        },
+    )
+}
+
+#[test]
+fn crlf_newlines_with_bom() {
+    test_positions(
+        "\u{feff}\r\nmod foo;\r\nmod bar;\r\n",
+        (5, 12),
+        SpanTestData {
+            byte_start: 9,
+            byte_end: 17,
+            line_start: 2,
+            column_start: 5,
+            line_end: 3,
+            column_end: 3,
+        },
+    )
+}
+
+#[test]
+fn span_before_crlf() {
+    test_positions(
+        "foo\r\nbar",
+        (2, 3),
+        SpanTestData {
+            byte_start: 2,
+            byte_end: 3,
+            line_start: 1,
+            column_start: 3,
+            line_end: 1,
+            column_end: 4,
+        },
+    )
+}
+
+#[test]
+fn span_on_crlf() {
+    test_positions(
+        "foo\r\nbar",
+        (3, 4),
+        SpanTestData {
+            byte_start: 3,
+            byte_end: 5,
+            line_start: 1,
+            column_start: 4,
+            line_end: 2,
+            column_end: 1,
+        },
+    )
+}
+
+#[test]
+fn span_after_crlf() {
+    test_positions(
+        "foo\r\nbar",
+        (4, 5),
+        SpanTestData {
+            byte_start: 5,
+            byte_end: 6,
+            line_start: 2,
+            column_start: 1,
+            line_end: 2,
+            column_end: 2,
+        },
+    )
+}
diff --git a/src/libsyntax/source_map.rs b/src/libsyntax/source_map.rs
index a1d147637e2..d7760e0cf9e 100644
--- a/src/libsyntax/source_map.rs
+++ b/src/libsyntax/source_map.rs
@@ -283,6 +283,7 @@ impl SourceMap {
         mut file_local_lines: Vec<BytePos>,
         mut file_local_multibyte_chars: Vec<MultiByteChar>,
         mut file_local_non_narrow_chars: Vec<NonNarrowChar>,
+        mut file_local_normalized_pos: Vec<NormalizedPos>,
     ) -> Lrc<SourceFile> {
         let start_pos = self.next_start_pos();
 
@@ -301,6 +302,10 @@ impl SourceMap {
             *swc = *swc + start_pos;
         }
 
+        for nc in &mut file_local_normalized_pos {
+            nc.pos = nc.pos + start_pos;
+        }
+
         let source_file = Lrc::new(SourceFile {
             name: filename,
             name_was_remapped,
@@ -314,6 +319,7 @@ impl SourceMap {
             lines: file_local_lines,
             multibyte_chars: file_local_multibyte_chars,
             non_narrow_chars: file_local_non_narrow_chars,
+            normalized_pos: file_local_normalized_pos,
             name_hash,
         });
 
diff --git a/src/libsyntax/tests.rs b/src/libsyntax/tests.rs
index 881bdaa84d0..e73c8b43bcc 100644
--- a/src/libsyntax/tests.rs
+++ b/src/libsyntax/tests.rs
@@ -111,8 +111,8 @@ struct SpanLabel {
     label: &'static str,
 }
 
-struct Shared<T: Write> {
-    data: Arc<Mutex<T>>,
+crate struct Shared<T: Write> {
+    pub data: Arc<Mutex<T>>,
 }
 
 impl<T: Write> Write for Shared<T> {
diff --git a/src/libsyntax_pos/lib.rs b/src/libsyntax_pos/lib.rs
index a307a5c0b37..9034f8c1afd 100644
--- a/src/libsyntax_pos/lib.rs
+++ b/src/libsyntax_pos/lib.rs
@@ -855,6 +855,15 @@ impl Sub<BytePos> for NonNarrowChar {
     }
 }
 
+/// Identifies an offset of a character that was normalized away from `SourceFile`.
+#[derive(Copy, Clone, RustcEncodable, RustcDecodable, Eq, PartialEq, Debug)]
+pub struct NormalizedPos {
+    /// The absolute offset of the character in the `SourceMap`.
+    pub pos: BytePos,
+    /// The difference between original and normalized string at position.
+    pub diff: u32,
+}
+
 /// The state of the lazy external source loading mechanism of a `SourceFile`.
 #[derive(PartialEq, Eq, Clone)]
 pub enum ExternalSource {
@@ -918,6 +927,8 @@ pub struct SourceFile {
     pub multibyte_chars: Vec<MultiByteChar>,
     /// Width of characters that are not narrow in the source code.
     pub non_narrow_chars: Vec<NonNarrowChar>,
+    /// Locations of characters removed during normalization.
+    pub normalized_pos: Vec<NormalizedPos>,
     /// A hash of the filename, used for speeding up hashing in incremental compilation.
     pub name_hash: u128,
 }
@@ -984,6 +995,9 @@ impl Encodable for SourceFile {
             })?;
             s.emit_struct_field("name_hash", 8, |s| {
                 self.name_hash.encode(s)
+            })?;
+            s.emit_struct_field("normalized_pos", 9, |s| {
+                self.normalized_pos.encode(s)
             })
         })
     }
@@ -1034,6 +1048,8 @@ impl Decodable for SourceFile {
                 d.read_struct_field("non_narrow_chars", 7, |d| Decodable::decode(d))?;
             let name_hash: u128 =
                 d.read_struct_field("name_hash", 8, |d| Decodable::decode(d))?;
+            let normalized_pos: Vec<NormalizedPos> =
+                d.read_struct_field("normalized_pos", 9, |d| Decodable::decode(d))?;
             Ok(SourceFile {
                 name,
                 name_was_remapped,
@@ -1050,6 +1066,7 @@ impl Decodable for SourceFile {
                 lines,
                 multibyte_chars,
                 non_narrow_chars,
+                normalized_pos,
                 name_hash,
             })
         })
@@ -1068,8 +1085,7 @@ impl SourceFile {
                unmapped_path: FileName,
                mut src: String,
                start_pos: BytePos) -> Result<SourceFile, OffsetOverflowError> {
-        remove_bom(&mut src);
-        normalize_newlines(&mut src);
+        let normalized_pos = normalize_src(&mut src, start_pos);
 
         let src_hash = {
             let mut hasher: StableHasher = StableHasher::new();
@@ -1102,6 +1118,7 @@ impl SourceFile {
             lines,
             multibyte_chars,
             non_narrow_chars,
+            normalized_pos,
             name_hash,
         })
     }
@@ -1228,12 +1245,44 @@ impl SourceFile {
     pub fn contains(&self, byte_pos: BytePos) -> bool {
         byte_pos >= self.start_pos && byte_pos <= self.end_pos
     }
+
+    /// Calculates the original byte position relative to the start of the file
+    /// based on the given byte position.
+    pub fn original_relative_byte_pos(&self, pos: BytePos) -> BytePos {
+
+        // Diff before any records is 0. Otherwise use the previously recorded
+        // diff as that applies to the following characters until a new diff
+        // is recorded.
+        let diff = match self.normalized_pos.binary_search_by(
+                            |np| np.pos.cmp(&pos)) {
+            Ok(i) => self.normalized_pos[i].diff,
+            Err(i) if i == 0 => 0,
+            Err(i) => self.normalized_pos[i-1].diff,
+        };
+
+        BytePos::from_u32(pos.0 - self.start_pos.0 + diff)
+    }
+}
+
+/// Normalizes the source code and records the normalizations.
+fn normalize_src(src: &mut String, start_pos: BytePos) -> Vec<NormalizedPos> {
+    let mut normalized_pos = vec![];
+    remove_bom(src, &mut normalized_pos);
+    normalize_newlines(src, &mut normalized_pos);
+
+    // Offset all the positions by start_pos to match the final file positions.
+    for np in &mut normalized_pos {
+        np.pos.0 += start_pos.0;
+    }
+
+    normalized_pos
 }
 
 /// Removes UTF-8 BOM, if any.
-fn remove_bom(src: &mut String) {
+fn remove_bom(src: &mut String, normalized_pos: &mut Vec<NormalizedPos>) {
     if src.starts_with("\u{feff}") {
         src.drain(..3);
+        normalized_pos.push(NormalizedPos { pos: BytePos(0), diff: 3 });
     }
 }
 
@@ -1241,7 +1290,7 @@ fn remove_bom(src: &mut String) {
 /// Replaces `\r\n` with `\n` in-place in `src`.
 ///
 /// Returns error if there's a lone `\r` in the string
-fn normalize_newlines(src: &mut String) {
+fn normalize_newlines(src: &mut String, normalized_pos: &mut Vec<NormalizedPos>) {
     if !src.as_bytes().contains(&b'\r') {
         return;
     }
@@ -1254,6 +1303,8 @@ fn normalize_newlines(src: &mut String) {
     let mut buf = std::mem::replace(src, String::new()).into_bytes();
     let mut gap_len = 0;
     let mut tail = buf.as_mut_slice();
+    let mut cursor = 0;
+    let original_gap = normalized_pos.last().map_or(0, |l| l.diff);
     loop {
         let idx = match find_crlf(&tail[gap_len..]) {
             None => tail.len(),
@@ -1264,7 +1315,12 @@ fn normalize_newlines(src: &mut String) {
         if tail.len() == gap_len {
             break;
         }
+        cursor += idx - gap_len;
         gap_len += 1;
+        normalized_pos.push(NormalizedPos {
+            pos: BytePos::from_usize(cursor + 1),
+            diff: original_gap + gap_len as u32,
+        });
     }
 
     // Account for removed `\r`.
diff --git a/src/libsyntax_pos/tests.rs b/src/libsyntax_pos/tests.rs
index 6bd6016020a..87cc3505e38 100644
--- a/src/libsyntax_pos/tests.rs
+++ b/src/libsyntax_pos/tests.rs
@@ -19,20 +19,25 @@ fn test_lookup_line() {
 
 #[test]
 fn test_normalize_newlines() {
-    fn check(before: &str, after: &str) {
+    fn check(before: &str, after: &str, expected_positions: &[u32]) {
         let mut actual = before.to_string();
-        normalize_newlines(&mut actual);
+        let mut actual_positions = vec![];
+        normalize_newlines(&mut actual, &mut actual_positions);
+        let actual_positions : Vec<_> = actual_positions
+            .into_iter()
+            .map(|nc| nc.pos.0).collect();
         assert_eq!(actual.as_str(), after);
+        assert_eq!(actual_positions, expected_positions);
     }
-    check("", "");
-    check("\n", "\n");
-    check("\r", "\r");
-    check("\r\r", "\r\r");
-    check("\r\n", "\n");
-    check("hello world", "hello world");
-    check("hello\nworld", "hello\nworld");
-    check("hello\r\nworld", "hello\nworld");
-    check("\r\nhello\r\nworld\r\n", "\nhello\nworld\n");
-    check("\r\r\n", "\r\n");
-    check("hello\rworld", "hello\rworld");
+    check("", "", &[]);
+    check("\n", "\n", &[]);
+    check("\r", "\r", &[]);
+    check("\r\r", "\r\r", &[]);
+    check("\r\n", "\n", &[1]);
+    check("hello world", "hello world", &[]);
+    check("hello\nworld", "hello\nworld", &[]);
+    check("hello\r\nworld", "hello\nworld", &[6]);
+    check("\r\nhello\r\nworld\r\n", "\nhello\nworld\n", &[1, 7, 13]);
+    check("\r\r\n", "\r\n", &[2]);
+    check("hello\rworld", "hello\rworld", &[]);
 }
diff --git a/src/test/ui/.gitattributes b/src/test/ui/.gitattributes
index 489dc8ad111..9ea3d3fb0e1 100644
--- a/src/test/ui/.gitattributes
+++ b/src/test/ui/.gitattributes
@@ -1,3 +1,6 @@
 lexer-crlf-line-endings-string-literal-doc-comment.rs -text
+json-bom-plus-crlf.rs -text
+json-bom-plus-crlf-multifile.rs -text
+json-bom-plus-crlf-multifile-aux.rs -text
 trailing-carriage-return-in-string.rs -text
 *.bin -text
diff --git a/src/test/ui/json-bom-plus-crlf-multifile-aux.rs b/src/test/ui/json-bom-plus-crlf-multifile-aux.rs
new file mode 100644
index 00000000000..991ea1d85d2
--- /dev/null
+++ b/src/test/ui/json-bom-plus-crlf-multifile-aux.rs
@@ -0,0 +1,27 @@
+// (This line has BOM so it's ignored by compiletest for directives)

+//

+// ignore-test Not a test. Used by other tests

+// ignore-tidy-cr

+

+// For easier verifying, the byte offsets in this file should match those

+// in the json-bom-plus-crlf.rs - given the actual fn is identical (just with

+// a different, but equally sized name), the easiest way to do this is to

+// ensure the two files are of equal size on disk.

+// Padding............................

+

+// N.B., this file needs CRLF line endings. The .gitattributes file in

+// this directory should enforce it.

+

+pub fn test() {

+

+    let s : String = 1;  // Error in the middle of line.

+

+    let s : String = 1

+    ;  // Error before the newline.

+

+    let s : String =

+1;  // Error after the newline.

+

+    let s : String = (

+    );  // Error spanning the newline.

+}

diff --git a/src/test/ui/json-bom-plus-crlf-multifile.rs b/src/test/ui/json-bom-plus-crlf-multifile.rs
new file mode 100644
index 00000000000..c71dd325f44
--- /dev/null
+++ b/src/test/ui/json-bom-plus-crlf-multifile.rs
@@ -0,0 +1,12 @@
+// (This line has BOM so it's ignored by compiletest for directives)

+//

+// build-fail

+// compile-flags: --json=diagnostic-short --error-format=json

+// ignore-tidy-cr

+

+#[path = "json-bom-plus-crlf-multifile-aux.rs"]

+mod json_bom_plus_crlf_multifile_aux;

+

+fn main() {

+    json_bom_plus_crlf_multifile_aux::test();

+}

diff --git a/src/test/ui/json-bom-plus-crlf-multifile.stderr b/src/test/ui/json-bom-plus-crlf-multifile.stderr
new file mode 100644
index 00000000000..8472f161089
--- /dev/null
+++ b/src/test/ui/json-bom-plus-crlf-multifile.stderr
@@ -0,0 +1,86 @@
+{"message":"mismatched types","code":{"code":"E0308","explanation":"
+This error occurs when the compiler was unable to infer the concrete type of a
+variable. It can occur for several cases, the most common of which is a
+mismatch in the expected type that the compiler inferred for a variable's
+initializing expression, and the actual type explicitly assigned to the
+variable.
+
+For example:
+
+```compile_fail,E0308
+let x: i32 = \"I am not a number!\";
+//     ~~~   ~~~~~~~~~~~~~~~~~~~~
+//      |             |
+//      |    initializing expression;
+//      |    compiler infers type `&str`
+//      |
+//    type `i32` assigned to variable `x`
+```
+"},"level":"error","spans":[{"file_name":"$DIR/json-bom-plus-crlf-multifile-aux.rs","byte_start":621,"byte_end":622,"line_start":17,"line_end":17,"column_start":22,"column_end":23,"is_primary":true,"text":[{"text":"    let s : String = 1;  // Error in the middle of line.","highlight_start":22,"highlight_end":23}],"label":"expected struct `std::string::String`, found integer","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"expected type `std::string::String`
+   found type `{integer}`","code":null,"level":"note","spans":[],"children":[],"rendered":null},{"message":"try using a conversion method","code":null,"level":"help","spans":[{"file_name":"$DIR/json-bom-plus-crlf-multifile-aux.rs","byte_start":621,"byte_end":622,"line_start":17,"line_end":17,"column_start":22,"column_end":23,"is_primary":true,"text":[{"text":"    let s : String = 1;  // Error in the middle of line.","highlight_start":22,"highlight_end":23}],"label":null,"suggested_replacement":"1.to_string()","suggestion_applicability":"MaybeIncorrect","expansion":null}],"children":[],"rendered":null}],"rendered":"$DIR/json-bom-plus-crlf-multifile-aux.rs:17:22: error[E0308]: mismatched types
+"}
+{"message":"mismatched types","code":{"code":"E0308","explanation":"
+This error occurs when the compiler was unable to infer the concrete type of a
+variable. It can occur for several cases, the most common of which is a
+mismatch in the expected type that the compiler inferred for a variable's
+initializing expression, and the actual type explicitly assigned to the
+variable.
+
+For example:
+
+```compile_fail,E0308
+let x: i32 = \"I am not a number!\";
+//     ~~~   ~~~~~~~~~~~~~~~~~~~~
+//      |             |
+//      |    initializing expression;
+//      |    compiler infers type `&str`
+//      |
+//    type `i32` assigned to variable `x`
+```
+"},"level":"error","spans":[{"file_name":"$DIR/json-bom-plus-crlf-multifile-aux.rs","byte_start":681,"byte_end":682,"line_start":19,"line_end":19,"column_start":22,"column_end":23,"is_primary":true,"text":[{"text":"    let s : String = 1","highlight_start":22,"highlight_end":23}],"label":"expected struct `std::string::String`, found integer","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"expected type `std::string::String`
+   found type `{integer}`","code":null,"level":"note","spans":[],"children":[],"rendered":null},{"message":"try using a conversion method","code":null,"level":"help","spans":[{"file_name":"$DIR/json-bom-plus-crlf-multifile-aux.rs","byte_start":681,"byte_end":682,"line_start":19,"line_end":19,"column_start":22,"column_end":23,"is_primary":true,"text":[{"text":"    let s : String = 1","highlight_start":22,"highlight_end":23}],"label":null,"suggested_replacement":"1.to_string()","suggestion_applicability":"MaybeIncorrect","expansion":null}],"children":[],"rendered":null}],"rendered":"$DIR/json-bom-plus-crlf-multifile-aux.rs:19:22: error[E0308]: mismatched types
+"}
+{"message":"mismatched types","code":{"code":"E0308","explanation":"
+This error occurs when the compiler was unable to infer the concrete type of a
+variable. It can occur for several cases, the most common of which is a
+mismatch in the expected type that the compiler inferred for a variable's
+initializing expression, and the actual type explicitly assigned to the
+variable.
+
+For example:
+
+```compile_fail,E0308
+let x: i32 = \"I am not a number!\";
+//     ~~~   ~~~~~~~~~~~~~~~~~~~~
+//      |             |
+//      |    initializing expression;
+//      |    compiler infers type `&str`
+//      |
+//    type `i32` assigned to variable `x`
+```
+"},"level":"error","spans":[{"file_name":"$DIR/json-bom-plus-crlf-multifile-aux.rs","byte_start":745,"byte_end":746,"line_start":23,"line_end":23,"column_start":1,"column_end":2,"is_primary":true,"text":[{"text":"1;  // Error after the newline.","highlight_start":1,"highlight_end":2}],"label":"expected struct `std::string::String`, found integer","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"expected type `std::string::String`
+   found type `{integer}`","code":null,"level":"note","spans":[],"children":[],"rendered":null},{"message":"try using a conversion method","code":null,"level":"help","spans":[{"file_name":"$DIR/json-bom-plus-crlf-multifile-aux.rs","byte_start":745,"byte_end":746,"line_start":23,"line_end":23,"column_start":1,"column_end":2,"is_primary":true,"text":[{"text":"1;  // Error after the newline.","highlight_start":1,"highlight_end":2}],"label":null,"suggested_replacement":"1.to_string()","suggestion_applicability":"MaybeIncorrect","expansion":null}],"children":[],"rendered":null}],"rendered":"$DIR/json-bom-plus-crlf-multifile-aux.rs:23:1: error[E0308]: mismatched types
+"}
+{"message":"mismatched types","code":{"code":"E0308","explanation":"
+This error occurs when the compiler was unable to infer the concrete type of a
+variable. It can occur for several cases, the most common of which is a
+mismatch in the expected type that the compiler inferred for a variable's
+initializing expression, and the actual type explicitly assigned to the
+variable.
+
+For example:
+
+```compile_fail,E0308
+let x: i32 = \"I am not a number!\";
+//     ~~~   ~~~~~~~~~~~~~~~~~~~~
+//      |             |
+//      |    initializing expression;
+//      |    compiler infers type `&str`
+//      |
+//    type `i32` assigned to variable `x`
+```
+"},"level":"error","spans":[{"file_name":"$DIR/json-bom-plus-crlf-multifile-aux.rs","byte_start":801,"byte_end":809,"line_start":25,"line_end":26,"column_start":22,"column_end":6,"is_primary":true,"text":[{"text":"    let s : String = (","highlight_start":22,"highlight_end":23},{"text":"    );  // Error spanning the newline.","highlight_start":1,"highlight_end":6}],"label":"expected struct `std::string::String`, found ()","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"expected type `std::string::String`
+   found type `()`","code":null,"level":"note","spans":[],"children":[],"rendered":null}],"rendered":"$DIR/json-bom-plus-crlf-multifile-aux.rs:25:22: error[E0308]: mismatched types
+"}
+{"message":"aborting due to 4 previous errors","code":null,"level":"error","spans":[],"children":[],"rendered":"error: aborting due to 4 previous errors
+"}
diff --git a/src/test/ui/json-bom-plus-crlf.rs b/src/test/ui/json-bom-plus-crlf.rs
new file mode 100644
index 00000000000..ae54a35d480
--- /dev/null
+++ b/src/test/ui/json-bom-plus-crlf.rs
@@ -0,0 +1,27 @@
+// (This line has BOM so it's ignored by compiletest for directives)

+//

+// build-fail

+// compile-flags: --json=diagnostic-short --error-format=json

+// ignore-tidy-cr

+

+// For easier verifying, the byte offsets in this file should match those

+// in the json_bom_plus_crlf_multifile_aux.rs - given the actual fn is

+// identical (just with a different, but equally sized name), the easiest way

+// to do this is to ensure the two files are of equal size on disk.

+

+// N.B., this file needs CRLF line endings. The .gitattributes file in

+// this directory should enforce it.

+

+fn main() {

+

+    let s : String = 1;  // Error in the middle of line.

+

+    let s : String = 1

+    ;  // Error before the newline.

+

+    let s : String =

+1;  // Error after the newline.

+

+    let s : String = (

+    );  // Error spanning the newline.

+}

diff --git a/src/test/ui/json-bom-plus-crlf.stderr b/src/test/ui/json-bom-plus-crlf.stderr
new file mode 100644
index 00000000000..17775f59fd1
--- /dev/null
+++ b/src/test/ui/json-bom-plus-crlf.stderr
@@ -0,0 +1,86 @@
+{"message":"mismatched types","code":{"code":"E0308","explanation":"
+This error occurs when the compiler was unable to infer the concrete type of a
+variable. It can occur for several cases, the most common of which is a
+mismatch in the expected type that the compiler inferred for a variable's
+initializing expression, and the actual type explicitly assigned to the
+variable.
+
+For example:
+
+```compile_fail,E0308
+let x: i32 = \"I am not a number!\";
+//     ~~~   ~~~~~~~~~~~~~~~~~~~~
+//      |             |
+//      |    initializing expression;
+//      |    compiler infers type `&str`
+//      |
+//    type `i32` assigned to variable `x`
+```
+"},"level":"error","spans":[{"file_name":"$DIR/json-bom-plus-crlf.rs","byte_start":621,"byte_end":622,"line_start":17,"line_end":17,"column_start":22,"column_end":23,"is_primary":true,"text":[{"text":"    let s : String = 1;  // Error in the middle of line.","highlight_start":22,"highlight_end":23}],"label":"expected struct `std::string::String`, found integer","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"expected type `std::string::String`
+   found type `{integer}`","code":null,"level":"note","spans":[],"children":[],"rendered":null},{"message":"try using a conversion method","code":null,"level":"help","spans":[{"file_name":"$DIR/json-bom-plus-crlf.rs","byte_start":621,"byte_end":622,"line_start":17,"line_end":17,"column_start":22,"column_end":23,"is_primary":true,"text":[{"text":"    let s : String = 1;  // Error in the middle of line.","highlight_start":22,"highlight_end":23}],"label":null,"suggested_replacement":"1.to_string()","suggestion_applicability":"MaybeIncorrect","expansion":null}],"children":[],"rendered":null}],"rendered":"$DIR/json-bom-plus-crlf.rs:17:22: error[E0308]: mismatched types
+"}
+{"message":"mismatched types","code":{"code":"E0308","explanation":"
+This error occurs when the compiler was unable to infer the concrete type of a
+variable. It can occur for several cases, the most common of which is a
+mismatch in the expected type that the compiler inferred for a variable's
+initializing expression, and the actual type explicitly assigned to the
+variable.
+
+For example:
+
+```compile_fail,E0308
+let x: i32 = \"I am not a number!\";
+//     ~~~   ~~~~~~~~~~~~~~~~~~~~
+//      |             |
+//      |    initializing expression;
+//      |    compiler infers type `&str`
+//      |
+//    type `i32` assigned to variable `x`
+```
+"},"level":"error","spans":[{"file_name":"$DIR/json-bom-plus-crlf.rs","byte_start":681,"byte_end":682,"line_start":19,"line_end":19,"column_start":22,"column_end":23,"is_primary":true,"text":[{"text":"    let s : String = 1","highlight_start":22,"highlight_end":23}],"label":"expected struct `std::string::String`, found integer","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"expected type `std::string::String`
+   found type `{integer}`","code":null,"level":"note","spans":[],"children":[],"rendered":null},{"message":"try using a conversion method","code":null,"level":"help","spans":[{"file_name":"$DIR/json-bom-plus-crlf.rs","byte_start":681,"byte_end":682,"line_start":19,"line_end":19,"column_start":22,"column_end":23,"is_primary":true,"text":[{"text":"    let s : String = 1","highlight_start":22,"highlight_end":23}],"label":null,"suggested_replacement":"1.to_string()","suggestion_applicability":"MaybeIncorrect","expansion":null}],"children":[],"rendered":null}],"rendered":"$DIR/json-bom-plus-crlf.rs:19:22: error[E0308]: mismatched types
+"}
+{"message":"mismatched types","code":{"code":"E0308","explanation":"
+This error occurs when the compiler was unable to infer the concrete type of a
+variable. It can occur for several cases, the most common of which is a
+mismatch in the expected type that the compiler inferred for a variable's
+initializing expression, and the actual type explicitly assigned to the
+variable.
+
+For example:
+
+```compile_fail,E0308
+let x: i32 = \"I am not a number!\";
+//     ~~~   ~~~~~~~~~~~~~~~~~~~~
+//      |             |
+//      |    initializing expression;
+//      |    compiler infers type `&str`
+//      |
+//    type `i32` assigned to variable `x`
+```
+"},"level":"error","spans":[{"file_name":"$DIR/json-bom-plus-crlf.rs","byte_start":745,"byte_end":746,"line_start":23,"line_end":23,"column_start":1,"column_end":2,"is_primary":true,"text":[{"text":"1;  // Error after the newline.","highlight_start":1,"highlight_end":2}],"label":"expected struct `std::string::String`, found integer","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"expected type `std::string::String`
+   found type `{integer}`","code":null,"level":"note","spans":[],"children":[],"rendered":null},{"message":"try using a conversion method","code":null,"level":"help","spans":[{"file_name":"$DIR/json-bom-plus-crlf.rs","byte_start":745,"byte_end":746,"line_start":23,"line_end":23,"column_start":1,"column_end":2,"is_primary":true,"text":[{"text":"1;  // Error after the newline.","highlight_start":1,"highlight_end":2}],"label":null,"suggested_replacement":"1.to_string()","suggestion_applicability":"MaybeIncorrect","expansion":null}],"children":[],"rendered":null}],"rendered":"$DIR/json-bom-plus-crlf.rs:23:1: error[E0308]: mismatched types
+"}
+{"message":"mismatched types","code":{"code":"E0308","explanation":"
+This error occurs when the compiler was unable to infer the concrete type of a
+variable. It can occur for several cases, the most common of which is a
+mismatch in the expected type that the compiler inferred for a variable's
+initializing expression, and the actual type explicitly assigned to the
+variable.
+
+For example:
+
+```compile_fail,E0308
+let x: i32 = \"I am not a number!\";
+//     ~~~   ~~~~~~~~~~~~~~~~~~~~
+//      |             |
+//      |    initializing expression;
+//      |    compiler infers type `&str`
+//      |
+//    type `i32` assigned to variable `x`
+```
+"},"level":"error","spans":[{"file_name":"$DIR/json-bom-plus-crlf.rs","byte_start":801,"byte_end":809,"line_start":25,"line_end":26,"column_start":22,"column_end":6,"is_primary":true,"text":[{"text":"    let s : String = (","highlight_start":22,"highlight_end":23},{"text":"    );  // Error spanning the newline.","highlight_start":1,"highlight_end":6}],"label":"expected struct `std::string::String`, found ()","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"expected type `std::string::String`
+   found type `()`","code":null,"level":"note","spans":[],"children":[],"rendered":null}],"rendered":"$DIR/json-bom-plus-crlf.rs:25:22: error[E0308]: mismatched types
+"}
+{"message":"aborting due to 4 previous errors","code":null,"level":"error","spans":[],"children":[],"rendered":"error: aborting due to 4 previous errors
+"}