diff options
| -rw-r--r-- | .travis.yml | 10 | ||||
| -rw-r--r-- | src/Cargo.lock | 2 | ||||
| -rw-r--r-- | src/ci/docker/README.md | 50 | ||||
| -rwxr-xr-x | src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh | 6 | ||||
| -rwxr-xr-x | src/ci/docker/run.sh | 13 | ||||
| -rw-r--r-- | src/librustc_trans/back/link.rs | 11 | ||||
| -rw-r--r-- | src/libtest/lib.rs | 17 | ||||
| -rw-r--r-- | src/tools/compiletest/Cargo.toml | 4 | ||||
| -rw-r--r-- | src/tools/compiletest/src/main.rs | 4 | ||||
| -rw-r--r-- | src/tools/compiletest/src/read2.rs | 208 | ||||
| -rw-r--r-- | src/tools/compiletest/src/runtest.rs | 91 | ||||
| -rw-r--r-- | src/tools/tidy/src/bins.rs | 6 |
12 files changed, 394 insertions, 28 deletions
diff --git a/.travis.yml b/.travis.yml index db34f140448..33982838eae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -253,7 +253,14 @@ after_failure: # Random attempt at debugging currently. Just poking around in here to see if # anything shows up. - - ls $HOME/Library/Logs/DiagnosticReports/ + - ls -lat $HOME/Library/Logs/DiagnosticReports/ + - find $HOME/Library/Logs/DiagnosticReports/ ! \( + -name '*.stage2-*.crash' + -name 'com.apple.CoreSimulator.CoreSimulatorService-*.crash' + \) + -exec echo -e travis_fold":start:crashlog\n\033[31;1m" {} "\033[0m" \; + -exec head -750 {} \; + -exec echo travis_fold":"end:crashlog \; # attempt to debug anything killed by the oom killer on linux, just to see if # it happened @@ -286,6 +293,7 @@ before_deploy: rm -rf obj/build/dist/doc && cp -r obj/build/dist/* deploy/$TRAVIS_COMMIT; fi + - travis_retry gem update --system deploy: - provider: s3 diff --git a/src/Cargo.lock b/src/Cargo.lock index 79aea99a5a9..ff97de681a1 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -348,7 +348,9 @@ dependencies = [ "getopts 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/src/ci/docker/README.md b/src/ci/docker/README.md index adce6a00d46..922deba7367 100644 --- a/src/ci/docker/README.md +++ b/src/ci/docker/README.md @@ -22,6 +22,48 @@ Images will output artifacts in an `obj` dir at the root of a repository. - `scripts` contains files shared by docker images - `disabled` contains images that are not built on travis +## Docker Toolbox on Windows + +For Windows before Windows 10, the docker images can be run on Windows via +[Docker Toolbox]. There are several preparation needs to be made before running +a Docker image. + +1. Stop the virtual machine from the terminal with `docker-machine stop` + +2. If your Rust source is placed outside of `C:\Users\**`, e.g. if you place the + repository in the `E:\rust` folder, please add a shared folder from + VirtualBox by: + + 1. Select the "default" virtual machine inside VirtualBox, then click + "Settings" + 2. Go to "Shared Folders", click "Add shared foldrer" (the folder icon with + a plus sign), fill in the following information, then click "OK": + + * Folder path: `E:\rust` + * Folder name: `e/rust` + * Read-only: ☐ *unchecked* + * Auto-mount: ☑ *checked* + * Make Permanant: ☑ *checked* + +3. VirtualBox might not support creating symbolic links inside a shared folder + by default. You can enable it manually by running these from `cmd.exe`: + + ```bat + cd "C:\Program Files\Oracle\VirtualBox" + VBoxManage setextradata default VBoxInternal2/SharedFoldersEnableSymlinksCreate/e/rust 1 + :: ^~~~~~ + :: folder name + ``` + +4. Restart the virtual machine from terminal with `docker-machine start`. + +To run the image, + +1. Launch the "Docker Quickstart Terminal". +2. Execute `./src/ci/docker/run.sh $image_name` as explained at the beginning. + +[Docker Toolbox]: https://www.docker.com/products/docker-toolbox + ## Cross toolchains A number of these images take quite a long time to compile as they're building @@ -137,7 +179,7 @@ For targets: `armv7-unknown-linux-gnueabihf` libraries like jemalloc. See the mk/cfg/arm(v7)-uknown-linux-gnueabi{,hf}.mk file in Rust's source code. -## `aarch64-linux-gnu.config` +### `aarch64-linux-gnu.config` For targets: `aarch64-unknown-linux-gnu` @@ -150,7 +192,7 @@ For targets: `aarch64-unknown-linux-gnu` - C compiler > gcc version = 5.2.0 - C compiler > C++ = ENABLE -- to cross compile LLVM -## `powerpc-linux-gnu.config` +### `powerpc-linux-gnu.config` For targets: `powerpc-unknown-linux-gnu` @@ -165,7 +207,7 @@ For targets: `powerpc-unknown-linux-gnu` - C compiler > gcc version = 4.9.3 - C compiler > C++ = ENABLE -- to cross compile LLVM -## `powerpc64-linux-gnu.config` +### `powerpc64-linux-gnu.config` For targets: `powerpc64-unknown-linux-gnu` @@ -184,7 +226,7 @@ For targets: `powerpc64-unknown-linux-gnu` (+) These CPU options match the configuration of the toolchains in RHEL6. -## `s390x-linux-gnu.config` +### `s390x-linux-gnu.config` For targets: `s390x-unknown-linux-gnu` diff --git a/src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh b/src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh index f231d20b197..5f556b67081 100755 --- a/src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh +++ b/src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh @@ -23,9 +23,9 @@ SYSROOT=/usr/local/$TARGET/sysroot mkdir -p $SYSROOT pushd $SYSROOT -centos_base=http://mirror.centos.org/altarch/7/os/ppc64le/Packages -glibc_v=2.17-196.el7 -kernel_v=3.10.0-693.el7 +centos_base=http://vault.centos.org/altarch/7.3.1611/os/ppc64le/Packages/ +glibc_v=2.17-157.el7 +kernel_v=3.10.0-514.el7 for package in glibc{,-devel,-headers}-$glibc_v kernel-headers-$kernel_v; do curl $centos_base/$package.ppc64le.rpm | \ rpm2cpio - | cpio -idm diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index b2560c6b95b..dc02310b4f2 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -11,6 +11,8 @@ set -e +export MSYS_NO_PATHCONV=1 + script=`cd $(dirname $0) && pwd`/`basename $0` image=$1 @@ -25,12 +27,19 @@ travis_fold start build_docker travis_time_start if [ -f "$docker_dir/$image/Dockerfile" ]; then + dockerfile="$docker_dir/$image/Dockerfile" + if [ -x /usr/bin/cygpath ]; then + context="`cygpath -w $docker_dir`" + dockerfile="`cygpath -w $dockerfile`" + else + context="$docker_dir" + fi retry docker \ build \ --rm \ -t rust-ci \ - -f "$docker_dir/$image/Dockerfile" \ - "$docker_dir" + -f "$dockerfile" \ + "$context" elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then if [ -n "$TRAVIS_OS_NAME" ]; then echo Cannot run disabled images on travis! diff --git a/src/librustc_trans/back/link.rs b/src/librustc_trans/back/link.rs index 907693ea8a3..1961acf53a6 100644 --- a/src/librustc_trans/back/link.rs +++ b/src/librustc_trans/back/link.rs @@ -671,11 +671,12 @@ fn link_natively(sess: &Session, break } - sess.struct_warn("looks like the linker segfaulted when we tried to \ - call it, automatically retrying again") - .note(&format!("{:?}", cmd)) - .note(&out) - .emit(); + warn!( + "looks like the linker segfaulted when we tried to call it, \ + automatically retrying again. cmd = {:?}, out = {}.", + cmd, + out, + ); } match prog { diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index e8a1242c814..76abcb83edc 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -71,6 +71,7 @@ use std::thread; use std::time::{Instant, Duration}; const TEST_WARN_TIMEOUT_S: u64 = 60; +const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode // to be used by rustc to compile tests in libtest pub mod test { @@ -614,7 +615,14 @@ impl<T: Write> ConsoleTestState<T> { pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::color::Color) -> io::Result<()> { if self.quiet { - self.write_pretty(quiet, color) + self.write_pretty(quiet, color)?; + if self.current_test_count() % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { + // we insert a new line every 100 dots in order to flush the + // screen when dealing with line-buffered output (e.g. piping to + // `stamp` in the rust CI). + self.write_plain("\n")?; + } + Ok(()) } else { self.write_pretty(verbose, color)?; self.write_plain("\n") @@ -771,9 +779,12 @@ impl<T: Write> ConsoleTestState<T> { Ok(()) } + fn current_test_count(&self) -> usize { + self.passed + self.failed + self.ignored + self.measured + self.allowed_fail + } + pub fn write_run_finish(&mut self) -> io::Result<bool> { - assert!(self.passed + self.failed + self.ignored + self.measured + - self.allowed_fail == self.total); + assert!(self.current_test_count() == self.total); if self.options.display_output { self.write_outputs()?; diff --git a/src/tools/compiletest/Cargo.toml b/src/tools/compiletest/Cargo.toml index f8282cc5f8d..d4d567e63c0 100644 --- a/src/tools/compiletest/Cargo.toml +++ b/src/tools/compiletest/Cargo.toml @@ -11,3 +11,7 @@ getopts = "0.2" log = "0.3" rustc-serialize = "0.3" libc = "0.2" + +[target.'cfg(windows)'.dependencies] +miow = "0.2" +winapi = "0.2" diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 1701c8a3e43..9fb6a3f5e07 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -11,10 +11,11 @@ #![crate_name = "compiletest"] #![feature(test)] +#![feature(slice_rotate)] #![deny(warnings)] -#[cfg(any(target_os = "macos", target_os = "ios"))] +#[cfg(unix)] extern crate libc; extern crate test; extern crate getopts; @@ -47,6 +48,7 @@ pub mod runtest; pub mod common; pub mod errors; mod raise_fd_limit; +mod read2; fn main() { env_logger::init().unwrap(); diff --git a/src/tools/compiletest/src/read2.rs b/src/tools/compiletest/src/read2.rs new file mode 100644 index 00000000000..1d8816c7db1 --- /dev/null +++ b/src/tools/compiletest/src/read2.rs @@ -0,0 +1,208 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// FIXME: This is a complete copy of `cargo/src/cargo/util/read2.rs` +// Consider unify the read2() in libstd, cargo and this to prevent further code duplication. + +pub use self::imp::read2; + +#[cfg(not(any(unix, windows)))] +mod imp { + use std::io::{self, Read}; + use std::process::{ChildStdout, ChildStderr}; + + pub fn read2(out_pipe: ChildStdout, + err_pipe: ChildStderr, + data: &mut FnMut(bool, &mut Vec<u8>, bool)) -> io::Result<()> { + let mut buffer = Vec::new(); + out_pipe.read_to_end(&mut buffer)?; + data(true, &mut buffer, true); + buffer.clear(); + err_pipe.read_to_end(&mut buffer)?; + data(false, &mut buffer, true); + Ok(()) + } +} + +#[cfg(unix)] +mod imp { + use std::io::prelude::*; + use std::io; + use std::mem; + use std::os::unix::prelude::*; + use std::process::{ChildStdout, ChildStderr}; + use libc; + + pub fn read2(mut out_pipe: ChildStdout, + mut err_pipe: ChildStderr, + data: &mut FnMut(bool, &mut Vec<u8>, bool)) -> io::Result<()> { + unsafe { + libc::fcntl(out_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK); + libc::fcntl(err_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK); + } + + let mut out_done = false; + let mut err_done = false; + let mut out = Vec::new(); + let mut err = Vec::new(); + + let mut fds: [libc::pollfd; 2] = unsafe { mem::zeroed() }; + fds[0].fd = out_pipe.as_raw_fd(); + fds[0].events = libc::POLLIN; + fds[1].fd = err_pipe.as_raw_fd(); + fds[1].events = libc::POLLIN; + loop { + // wait for either pipe to become readable using `select` + let r = unsafe { libc::poll(fds.as_mut_ptr(), 2, -1) }; + if r == -1 { + let err = io::Error::last_os_error(); + if err.kind() == io::ErrorKind::Interrupted { + continue + } + return Err(err) + } + + // Read as much as we can from each pipe, ignoring EWOULDBLOCK or + // EAGAIN. If we hit EOF, then this will happen because the underlying + // reader will return Ok(0), in which case we'll see `Ok` ourselves. In + // this case we flip the other fd back into blocking mode and read + // whatever's leftover on that file descriptor. + let handle = |res: io::Result<_>| { + match res { + Ok(_) => Ok(true), + Err(e) => { + if e.kind() == io::ErrorKind::WouldBlock { + Ok(false) + } else { + Err(e) + } + } + } + }; + if !out_done && fds[0].revents != 0 && handle(out_pipe.read_to_end(&mut out))? { + out_done = true; + } + data(true, &mut out, out_done); + if !err_done && fds[1].revents != 0 && handle(err_pipe.read_to_end(&mut err))? { + err_done = true; + } + data(false, &mut err, err_done); + + if out_done && err_done { + return Ok(()) + } + } + } +} + +#[cfg(windows)] +mod imp { + extern crate miow; + extern crate winapi; + + use std::io; + use std::os::windows::prelude::*; + use std::process::{ChildStdout, ChildStderr}; + use std::slice; + + use self::miow::iocp::{CompletionPort, CompletionStatus}; + use self::miow::pipe::NamedPipe; + use self::miow::Overlapped; + use self::winapi::ERROR_BROKEN_PIPE; + + struct Pipe<'a> { + dst: &'a mut Vec<u8>, + overlapped: Overlapped, + pipe: NamedPipe, + done: bool, + } + + pub fn read2(out_pipe: ChildStdout, + err_pipe: ChildStderr, + data: &mut FnMut(bool, &mut Vec<u8>, bool)) -> io::Result<()> { + let mut out = Vec::new(); + let mut err = Vec::new(); + + let port = CompletionPort::new(1)?; + port.add_handle(0, &out_pipe)?; + port.add_handle(1, &err_pipe)?; + + unsafe { + let mut out_pipe = Pipe::new(out_pipe, &mut out); + let mut err_pipe = Pipe::new(err_pipe, &mut err); + + out_pipe.read()?; + err_pipe.read()?; + + let mut status = [CompletionStatus::zero(), CompletionStatus::zero()]; + + while !out_pipe.done || !err_pipe.done { + for status in port.get_many(&mut status, None)? { + if status.token() == 0 { + out_pipe.complete(status); + data(true, out_pipe.dst, out_pipe.done); + out_pipe.read()?; + } else { + err_pipe.complete(status); + data(false, err_pipe.dst, err_pipe.done); + err_pipe.read()?; + } + } + } + + Ok(()) + } + } + + impl<'a> Pipe<'a> { + unsafe fn new<P: IntoRawHandle>(p: P, dst: &'a mut Vec<u8>) -> Pipe<'a> { + Pipe { + dst: dst, + pipe: NamedPipe::from_raw_handle(p.into_raw_handle()), + overlapped: Overlapped::zero(), + done: false, + } + } + + unsafe fn read(&mut self) -> io::Result<()> { + let dst = slice_to_end(self.dst); + match self.pipe.read_overlapped(dst, self.overlapped.raw()) { + Ok(_) => Ok(()), + Err(e) => { + if e.raw_os_error() == Some(ERROR_BROKEN_PIPE as i32) { + self.done = true; + Ok(()) + } else { + Err(e) + } + } + } + } + + unsafe fn complete(&mut self, status: &CompletionStatus) { + let prev = self.dst.len(); + self.dst.set_len(prev + status.bytes_transferred() as usize); + if status.bytes_transferred() == 0 { + self.done = true; + } + } + } + + unsafe fn slice_to_end(v: &mut Vec<u8>) -> &mut [u8] { + if v.capacity() == 0 { + v.reserve(16); + } + if v.capacity() == v.len() { + v.reserve(1); + } + slice::from_raw_parts_mut(v.as_mut_ptr().offset(v.len() as isize), + v.capacity() - v.len()) + } +} diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index f02525c118a..80ca0afe72b 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -29,7 +29,7 @@ use std::fmt; use std::io::prelude::*; use std::io::{self, BufReader}; use std::path::{Path, PathBuf}; -use std::process::{Command, Output, ExitStatus, Stdio}; +use std::process::{Command, Output, ExitStatus, Stdio, Child}; use std::str; use extract_gdb_version; @@ -1344,12 +1344,14 @@ actual:\n\ if let Some(input) = input { child.stdin.as_mut().unwrap().write_all(input.as_bytes()).unwrap(); } - let Output { status, stdout, stderr } = child.wait_with_output().unwrap(); + + let Output { status, stdout, stderr } = read2_abbreviated(child) + .expect("failed to read output"); let result = ProcRes { status, - stdout: String::from_utf8(stdout).unwrap(), - stderr: String::from_utf8(stderr).unwrap(), + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), cmdline, }; @@ -1635,7 +1637,9 @@ actual:\n\ cmd.arg("-a").arg("-u"); cmd.arg(filename); cmd.arg("-nobanner"); - let output = match cmd.output() { + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + let output = match cmd.spawn().and_then(read2_abbreviated) { Ok(output) => output, Err(_) => return, }; @@ -2095,6 +2099,8 @@ actual:\n\ let mut cmd = Command::new(make); cmd.current_dir(&self.testpaths.file) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) .env("TARGET", &self.config.target) .env("PYTHON", &self.config.docck_python) .env("S", src_root) @@ -2143,7 +2149,7 @@ actual:\n\ } } - let output = cmd.output().expect("failed to spawn `make`"); + let output = cmd.spawn().and_then(read2_abbreviated).expect("failed to spawn `make`"); if !output.status.success() { let res = ProcRes { status: output.status, @@ -2536,3 +2542,76 @@ fn nocomment_mir_line(line: &str) -> &str { line } } + +fn read2_abbreviated(mut child: Child) -> io::Result<Output> { + use std::mem::replace; + use read2::read2; + + const HEAD_LEN: usize = 160 * 1024; + const TAIL_LEN: usize = 256 * 1024; + + enum ProcOutput { + Full(Vec<u8>), + Abbreviated { + head: Vec<u8>, + skipped: usize, + tail: Box<[u8]>, + } + } + + impl ProcOutput { + fn extend(&mut self, data: &[u8]) { + let new_self = match *self { + ProcOutput::Full(ref mut bytes) => { + bytes.extend_from_slice(data); + let new_len = bytes.len(); + if new_len <= HEAD_LEN + TAIL_LEN { + return; + } + let tail = bytes.split_off(new_len - TAIL_LEN).into_boxed_slice(); + let head = replace(bytes, Vec::new()); + let skipped = new_len - HEAD_LEN - TAIL_LEN; + ProcOutput::Abbreviated { head, skipped, tail } + } + ProcOutput::Abbreviated { ref mut skipped, ref mut tail, .. } => { + *skipped += data.len(); + if data.len() <= TAIL_LEN { + tail[..data.len()].copy_from_slice(data); + tail.rotate(data.len()); + } else { + tail.copy_from_slice(&data[(data.len() - TAIL_LEN)..]); + } + return; + } + }; + *self = new_self; + } + + fn into_bytes(self) -> Vec<u8> { + match self { + ProcOutput::Full(bytes) => bytes, + ProcOutput::Abbreviated { mut head, skipped, tail } => { + write!(&mut head, "\n\n<<<<<< SKIPPED {} BYTES >>>>>>\n\n", skipped).unwrap(); + head.extend_from_slice(&tail); + head + } + } + } + } + + let mut stdout = ProcOutput::Full(Vec::new()); + let mut stderr = ProcOutput::Full(Vec::new()); + + drop(child.stdin.take()); + read2(child.stdout.take().unwrap(), child.stderr.take().unwrap(), &mut |is_stdout, data, _| { + if is_stdout { &mut stdout } else { &mut stderr }.extend(data); + data.clear(); + })?; + let status = child.wait()?; + + Ok(Output { + status, + stdout: stdout.into_bytes(), + stderr: stderr.into_bytes(), + }) +} \ No newline at end of file diff --git a/src/tools/tidy/src/bins.rs b/src/tools/tidy/src/bins.rs index 11d5dbe736e..f6e42c8dc17 100644 --- a/src/tools/tidy/src/bins.rs +++ b/src/tools/tidy/src/bins.rs @@ -31,9 +31,9 @@ pub fn check(path: &Path, bad: &mut bool) { if let Ok(mut file) = fs::File::open("/proc/version") { let mut contents = String::new(); file.read_to_string(&mut contents).unwrap(); - // Probably on Windows Linux Subsystem, all files will be marked as - // executable, so skip checking. - if contents.contains("Microsoft") { + // Probably on Windows Linux Subsystem or Docker via VirtualBox, + // all files will be marked as executable, so skip checking. + if contents.contains("Microsoft") || contents.contains("boot2docker") { return; } } |