Auto merge of #34366 - Diggsey:rust-src-pkg, r=brson

Produce source package in rust-installer format

See rust-lang/rust-buildbot#102

There may be a better way to do this, wasn't sure how to clean-up the `rust-src-image` directory when it's used by multiple make rules.

7 files changed, 213 insertions, 6 deletions

diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock
index 02698d6f7a1..d52577eb228 100644
--- a/src/bootstrap/Cargo.lock
+++ b/src/bootstrap/Cargo.lock
@@ -11,12 +11,21 @@ dependencies = [
  "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 0.1.73 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "toml 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)",
  "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
+name = "aho-corasick"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
 name = "build_helper"
 version = "0.1.0"
 
@@ -71,6 +80,14 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
+name = "memchr"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
 name = "num_cpus"
 version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -79,11 +96,45 @@ dependencies = [
 ]
 
 [[package]]
+name = "regex"
+version = "0.1.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "aho-corasick 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thread_local 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
 name = "rustc-serialize"
 version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
+name = "thread-id"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "thread_local"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
 name = "toml"
 version = "0.1.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -92,6 +143,11 @@ dependencies = [
 ]
 
 [[package]]
+name = "utf8-ranges"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
 name = "winapi"
 version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml
index 02746034cca..b19545590b9 100644
--- a/src/bootstrap/Cargo.toml
+++ b/src/bootstrap/Cargo.toml
@@ -32,3 +32,4 @@ kernel32-sys = "0.2"
 gcc = { git = "https://github.com/alexcrichton/gcc-rs" }
 libc = "0.2"
 md5 = "0.1"
+regex = "0.1.73"
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 1cf71c3aaec..9d18901eb00 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -24,7 +24,8 @@ use std::path::{PathBuf, Path};
 use std::process::Command;
 
 use {Build, Compiler};
-use util::{cp_r, libdir, is_dylib};
+use util::{cp_r, libdir, is_dylib, cp_filtered, copy};
+use regex::{RegexSet, quote};
 
 fn package_vers(build: &Build) -> &str {
     match &build.config.channel[..] {
@@ -284,6 +285,119 @@ pub fn std(build: &Build, compiler: &Compiler, target: &str) {
     t!(fs::remove_dir_all(&image));
 }
 
+/// Creates the `rust-src` installer component and the plain source tarball
+pub fn rust_src(build: &Build) {
+    println!("Dist src");
+    let plain_name = format!("rustc-{}-src", package_vers(build));
+    let name = format!("rust-src-{}", package_vers(build));
+    let image = tmpdir(build).join(format!("{}-image", name));
+    let _ = fs::remove_dir_all(&image);
+
+    let dst = image.join("lib/rustlib/src");
+    let dst_src = dst.join("rust");
+    let plain_dst_src = dst.join(&plain_name);
+    t!(fs::create_dir_all(&dst_src));
+
+    // This is the set of root paths which will become part of the source package
+    let src_files = [
+        "COPYRIGHT",
+        "LICENSE-APACHE",
+        "LICENSE-MIT",
+        "CONTRIBUTING.md",
+        "README.md",
+        "RELEASES.md",
+        "configure",
+        "Makefile.in"
+    ];
+    let src_dirs = [
+        "man",
+        "src",
+        "mk"
+    ];
+
+    // Exclude paths matching these wildcard expressions
+    let excludes = [
+        // exclude-vcs
+        "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules", ".gitattributes", ".cvsignore",
+        ".svn", ".arch-ids", "{arch}", "=RELEASE-ID", "=meta-update", "=update", ".bzr",
+        ".bzrignore", ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
+        // extensions
+        "*~", "*.pyc",
+        // misc
+        "llvm/test/*/*.ll",
+        "llvm/test/*/*.td",
+        "llvm/test/*/*.s",
+        "llvm/test/*/*/*.ll",
+        "llvm/test/*/*/*.td",
+        "llvm/test/*/*/*.s"
+    ];
+
+    // Construct a set of regexes for efficiently testing whether paths match one of the above
+    // expressions.
+    let regex_set = t!(RegexSet::new(
+        // This converts a wildcard expression to a regex
+        excludes.iter().map(|&s| {
+            // Prefix ensures that matching starts on a path separator boundary
+            r"^(.*[\\/])?".to_owned() + (
+                // Escape the expression to produce a regex matching exactly that string
+                &quote(s)
+                // Replace slashes with a pattern matching either forward or backslash
+                .replace(r"/", r"[\\/]")
+                // Replace wildcards with a pattern matching a single path segment, ie. containing
+                // no slashes.
+                .replace(r"\*", r"[^\\/]*")
+            // Suffix anchors to the end of the path
+            ) + "$"
+        })
+    ));
+
+    // Create a filter which skips files which match the regex set or contain invalid unicode
+    let filter_fn = move |path: &Path| {
+        if let Some(path) = path.to_str() {
+            !regex_set.is_match(path)
+        } else {
+            false
+        }
+    };
+
+    // Copy the directories using our filter
+    for item in &src_dirs {
+        let dst = &dst_src.join(item);
+        t!(fs::create_dir(dst));
+        cp_filtered(&build.src.join(item), dst, &filter_fn);
+    }
+    // Copy the files normally
+    for item in &src_files {
+        copy(&build.src.join(item), &dst_src.join(item));
+    }
+
+    // Create source tarball in rust-installer format
+    let mut cmd = Command::new("sh");
+    cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
+       .arg("--product-name=Rust")
+       .arg("--rel-manifest-dir=rustlib")
+       .arg("--success-message=Awesome-Source.")
+       .arg(format!("--image-dir={}", sanitize_sh(&image)))
+       .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build))))
+       .arg(format!("--output-dir={}", sanitize_sh(&distdir(build))))
+       .arg(format!("--package-name={}", name))
+       .arg("--component-name=rust-src")
+       .arg("--legacy-manifest-dirs=rustlib,cargo");
+    build.run(&mut cmd);
+
+    // Rename directory, so that root folder of tarball has the correct name
+    t!(fs::rename(&dst_src, &plain_dst_src));
+
+    // Create plain source tarball
+    let mut cmd = Command::new("tar");
+    cmd.arg("-czf").arg(sanitize_sh(&distdir(build).join(&format!("{}.tar.gz", plain_name))))
+       .arg(&plain_name)
+       .current_dir(&dst);
+    build.run(&mut cmd);
+
+    t!(fs::remove_dir_all(&image));
+}
+
 fn install(src: &Path, dstdir: &Path, perms: u32) {
     let dst = dstdir.join(src.file_name().unwrap());
     t!(fs::create_dir_all(dstdir));
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index 5d61abe5e08..3ef8cb74dfa 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -26,6 +26,7 @@ extern crate md5;
 extern crate num_cpus;
 extern crate rustc_serialize;
 extern crate toml;
+extern crate regex;
 
 use std::cell::RefCell;
 use std::collections::HashMap;
@@ -451,6 +452,7 @@ impl Build {
                 DistMingw { _dummy } => dist::mingw(self, target.target),
                 DistRustc { stage } => dist::rustc(self, stage, target.target),
                 DistStd { compiler } => dist::std(self, &compiler, target.target),
+                DistSrc { _dummy } => dist::rust_src(self),
 
                 DebuggerScripts { stage } => {
                     let compiler = Compiler::new(stage, target.target);
diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs
index f715ceb16d7..8d3cb36166b 100644
--- a/src/bootstrap/step.rs
+++ b/src/bootstrap/step.rs
@@ -140,6 +140,7 @@ macro_rules! targets {
             (dist_mingw, DistMingw { _dummy: () }),
             (dist_rustc, DistRustc { stage: u32 }),
             (dist_std, DistStd { compiler: Compiler<'a> }),
+            (dist_src, DistSrc { _dummy: () }),
 
             // Misc targets
             (android_copy_libs, AndroidCopyLibs { compiler: Compiler<'a> }),
@@ -568,12 +569,14 @@ impl<'a> Step<'a> {
                     vec![self.libtest(compiler)]
                 }
             }
+            Source::DistSrc { _dummy: _ } => Vec::new(),
 
             Source::Dist { stage } => {
                 let mut base = Vec::new();
 
                 for host in build.config.host.iter() {
                     let host = self.target(host);
+                    base.push(host.dist_src(()));
                     base.push(host.dist_rustc(stage));
                     if host.target.contains("windows-gnu") {
                         base.push(host.dist_mingw(()));
diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs
index b5230132bcb..dfc1c7a243b 100644
--- a/src/bootstrap/util.rs
+++ b/src/bootstrap/util.rs
@@ -67,6 +67,35 @@ pub fn cp_r(src: &Path, dst: &Path) {
     }
 }
 
+/// Copies the `src` directory recursively to `dst`. Both are assumed to exist
+/// when this function is called. Unwanted files or directories can be skipped
+/// by returning `false` from the filter function.
+pub fn cp_filtered<F: Fn(&Path) -> bool>(src: &Path, dst: &Path, filter: &F) {
+    // Inner function does the actual work
+    fn recurse<F: Fn(&Path) -> bool>(src: &Path, dst: &Path, relative: &Path, filter: &F) {
+        for f in t!(fs::read_dir(src)) {
+            let f = t!(f);
+            let path = f.path();
+            let name = path.file_name().unwrap();
+            let dst = dst.join(name);
+            let relative = relative.join(name);
+            // Only copy file or directory if the filter function returns true
+            if filter(&relative) {
+                if t!(f.file_type()).is_dir() {
+                    let _ = fs::remove_dir_all(&dst);
+                    t!(fs::create_dir(&dst));
+                    recurse(&path, &dst, &relative, filter);
+                } else {
+                    let _ = fs::remove_file(&dst);
+                    copy(&path, &dst);
+                }
+            }
+        }
+    }
+    // Immediately recurse with an empty relative path
+    recurse(src, dst, Path::new(""), filter)
+}
+
 /// Given an executable called `name`, return the filename for the
 /// executable for a particular target.
 pub fn exe(name: &str, target: &str) -> String {
diff --git a/src/tools/tidy/src/cargo_lock.rs b/src/tools/tidy/src/cargo_lock.rs
index 4324db489b7..165dd52758e 100644
--- a/src/tools/tidy/src/cargo_lock.rs
+++ b/src/tools/tidy/src/cargo_lock.rs
@@ -9,6 +9,7 @@
 // except according to those terms.
 
 use std::path::Path;
+use std::ffi::OsStr;
 
 const CARGO_LOCK: &'static str = "Cargo.lock";
 
@@ -18,14 +19,15 @@ pub fn check(path: &Path, bad: &mut bool) {
     super::walk(path,
                 &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
                 &mut |file| {
-        let name = file.file_name().unwrap().to_string_lossy();
-        if name == CARGO_LOCK {
+        if let Some(CARGO_LOCK) = file.file_name().and_then(OsStr::to_str) {
             let rel_path = file.strip_prefix(path).unwrap();
+            let git_friendly_path = rel_path.to_str().unwrap().replace("\\", "/");
             let ret_code = Command::new("git")
-                                        .arg("diff-index")
-                                        .arg("--quiet")
+                                        .arg("diff")
+                                        .arg("--exit-code")
+                                        .arg("--patch")
                                         .arg("HEAD")
-                                        .arg(rel_path)
+                                        .arg(&git_friendly_path)
                                         .current_dir(path)
                                         .status()
                                         .unwrap_or_else(|e| {