From 44a01b8a54b078d15620d1133b94ee21ee7a6915 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 15 Feb 2017 15:57:06 -0800 Subject: [PATCH] rustbuild: Add support for compiling Cargo This commit adds support to rustbuild for compiling Cargo as part of the release process. Previously rustbuild would simply download a Cargo snapshot and repackage it. With this change we should be able to turn off artifacts from the rust-lang/cargo repository and purely rely on the artifacts Cargo produces here. The infrastructure added here is intended to be extensible to other components, such as the RLS. It won't exactly be a one-line addition, but the addition of Cargo didn't require too much hooplah anyway. The process for release Cargo will now look like: * The rust-lang/rust repository has a Cargo submodule which is used to build a Cargo to pair with the rust-lang/rust release * Periodically we'll update the cargo submodule as necessary on rust-lang/rust's master branch * When branching beta we'll create a new branch of Cargo (as we do today), and the first commit to the beta branch will be to update the Cargo submodule to this exact revision. * When branching stable, we'll ensure that the Cargo submodule is updated and then make a stable release. Backports to Cargo will look like: * Send a PR to cargo's master branch * Send a PR to cargo's release branch (e.g. rust-1.16.0) * Send a PR to rust-lang/rust's beta branch updating the submodule * Eventually send a PR to rust-lang/rust's master branch updating the submodule For reference, the process to add a new component to the rust-lang/rust release would look like: * Add `$foo` as a submodule in `src/tools` * Add a `tool-$foo` step which compiles `$foo` with the specified compiler, likely mirroring what Cargo does. * Add a `dist-$foo` step which uses `src/tools/$foo` and the `tool-$foo` output to create a rust-installer package for `$foo` likely mirroring what Cargo does. * Update the `dist-extended` step with a new dependency on `dist-$foo` * Update `src/tools/build-manifest` for the new component. --- configure | 1 + src/Cargo.lock | 84 +++++++++++++--- src/bootstrap/channel.rs | 95 ++++++++++--------- src/bootstrap/compile.rs | 34 +++++-- src/bootstrap/config.rs | 4 + src/bootstrap/config.toml.example | 5 + src/bootstrap/dist.rs | 134 ++++++++++++++------------ src/bootstrap/doc.rs | 9 +- src/bootstrap/install.rs | 4 +- src/bootstrap/lib.rs | 137 ++++++++++++++++++++++----- src/bootstrap/metadata.rs | 2 + src/bootstrap/native.rs | 93 ++++++++++++++++++ src/bootstrap/step.rs | 30 +++++- src/build_helper/lib.rs | 18 ++++ src/ci/run.sh | 1 + src/tools/build-manifest/src/main.rs | 19 ++-- src/tools/cargo | 2 +- src/tools/tidy/src/cargo.rs | 2 +- src/tools/tidy/src/deps.rs | 2 + src/tools/tidy/src/main.rs | 1 + 20 files changed, 511 insertions(+), 166 deletions(-) diff --git a/configure b/configure index 70952438a3559..be8628de62832 100755 --- a/configure +++ b/configure @@ -651,6 +651,7 @@ opt locked-deps 0 "force Cargo.lock to be up to date" opt vendor 0 "enable usage of vendored Rust crates" opt sanitizers 0 "build the sanitizer runtimes (asan, lsan, msan, tsan)" opt dist-src 1 "when building tarballs enables building a source tarball" +opt cargo-openssl-static 0 "static openssl in cargo" # Optimization and debugging options. These may be overridden by the release channel, etc. opt_nosave optimize 1 "build optimized rust code" diff --git a/src/Cargo.lock b/src/Cargo.lock index 9a4fd3a1d5594..c463f2bb747cc 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -140,11 +140,15 @@ dependencies = [ "psapi-sys 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_ignored 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)", "shell-escape 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "term 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "toml 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -163,6 +167,8 @@ dependencies = [ "libc 0.2.20 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "term 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -238,7 +244,9 @@ name = "crates-io" version = "0.7.0" dependencies = [ "curl 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -784,6 +792,11 @@ name = "quick-error" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "quote" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "rand" version = "0.0.0" @@ -1222,6 +1235,32 @@ name = "serde" version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "serde_codegen_internals" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "syn 0.11.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_codegen_internals 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_ignored" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde_json" version = "0.9.7" @@ -1278,6 +1317,24 @@ name = "strsim" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "syn" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synom" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "syntax" version = "0.0.0" @@ -1404,14 +1461,6 @@ dependencies = [ "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "toml" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "toml" version = "0.3.0" @@ -1443,6 +1492,11 @@ name = "unicode-width" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unreachable" version = "0.1.1" @@ -1547,8 +1601,8 @@ dependencies = [ "checksum libssh2-sys 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "91e135645c2e198a39552c8c7686bb5b83b1b99f64831c040a6c2798a1195934" "checksum libz-sys 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e5ee912a45d686d393d5ac87fac15ba0ba18daae14e8e7543c63ebf7fb7e970c" "checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054" -"checksum mdbook 0.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "dbba458ca886cb082d026afd704eeeeb0531f7e4ffd6c619f72dc309c1c18fe4" "checksum matches 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efd7622e3022e1a6eaa602c4cea8912254e5582c9c692e9167714182244801b1" +"checksum mdbook 0.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "dbba458ca886cb082d026afd704eeeeb0531f7e4ffd6c619f72dc309c1c18fe4" "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" "checksum memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1dbccc0e46f1ea47b9f17e6d67c5a96bd27030519c519c9c91327e31275a47b4" "checksum miniz-sys 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "28eaee17666671fa872e567547e8428e83308ebe5808cdf6a0e28397dbe2c726" @@ -1572,6 +1626,7 @@ dependencies = [ "checksum psapi-sys 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "abcd5d1a07d360e29727f757a9decb3ce8bc6e0efa8969cfaad669a8317a2478" "checksum pulldown-cmark 0.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1058d7bb927ca067656537eec4e02c2b4b70eaaa129664c5b90c111e20326f41" "checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c" +"checksum quote 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7375cf7ad34a92e8fd18dd9c42f58b9a11def59ab48bec955bf359a788335592" "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" "checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f" "checksum regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4278c17d0f6d62dfef0ab00028feb45bd7d2102843f80763474eeb1be8a10c01" @@ -1581,9 +1636,14 @@ dependencies = [ "checksum semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a3186ec9e65071a2095434b1f5bb24838d4e8e130f584c790f6033c79943537" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1e0ed773960f90a78567fcfbe935284adf50c5d7cf119aa2cf43bb0b4afa69bb" +"checksum serde_codegen_internals 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4d52006899f910528a10631e5b727973fe668f3228109d1707ccf5bad5490b6e" +"checksum serde_derive 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)" = "789ee9f3cd78c850948b94121020147f5220b47dafbf230d7098a93a58f726cf" +"checksum serde_ignored 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4b3f5576874721d14690657e9f0ed286e72a52be2f6fdc0cf2f024182bd8f64" "checksum serde_json 0.9.7 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb96d30e4e6f9fc52e08f51176d078b6f79b981dc3ed4134f7b850be9f446a8" "checksum shell-escape 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "dd5cc96481d54583947bfe88bf30c23d53f883c6cd0145368b69989d97b84ef8" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" +"checksum syn 0.11.8 (registry+https://github.com/rust-lang/crates.io-index)" = "37c279fb816210c9bb28b2c292664581e7b87b4561e86b94df462664d8620bb8" +"checksum synom 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "27e31aa4b09b9f4cb12dff3c30ba503e17b1a624413d764d32dab76e3920e5bc" "checksum tar 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "1eb3bf6ec92843ca93f4fcfb5fc6dfe30534815b147885db4b5759b8e2ff7d52" "checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" "checksum term 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d168af3930b369cfe245132550579d47dfd873d69470755a19c2c6568dbbd989" @@ -1593,12 +1653,12 @@ dependencies = [ "checksum thread_local 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8576dbbfcaef9641452d5cf0df9b0e7eeab7694956dd33bb61515fb8f18cfdd5" "checksum thread_local 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c85048c6260d17cf486ceae3282d9fb6b90be220bf5b28c400f5485ffc29f0c7" "checksum toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "0590d72182e50e879c4da3b11c6488dae18fccb1ae0c7a3eda18e16795844796" -"checksum toml 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "736b60249cb25337bc196faa43ee12c705e426f3d55c214d73a4e7be06f92cb4" "checksum toml 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08272367dd2e766db3fa38f068067d17aa6a9dfd7259af24b3927db92f1e0c2f" "checksum unicode-bidi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d3a078ebdd62c0e71a709c3d53d2af693fe09fe93fbff8344aebe289b78f9032" "checksum unicode-normalization 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e28fa37426fceeb5cf8f41ee273faa7c82c47dc8fba5853402841e665fcd86ff" "checksum unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18127285758f0e2c6cf325bb3f3d138a12fee27de4f23e146cd6a179f26c2cf3" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" +"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" "checksum url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f5ba8a749fb4479b043733416c244fa9d1d3af3d7c23804944651c8a448cb87e" "checksum user32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ef4711d107b21b410a3a974b1204d9accc8b10dad75d8324b5d755de1617d47" diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs index 81e745bc76c9e..c126c076a3d4e 100644 --- a/src/bootstrap/channel.rs +++ b/src/bootstrap/channel.rs @@ -15,6 +15,7 @@ //! `package_vers`, and otherwise indicating to the compiler what it should //! print out as part of its version information. +use std::path::Path; use std::process::Command; use build_helper::output; @@ -22,65 +23,69 @@ use build_helper::output; use Build; // The version number -const CFG_RELEASE_NUM: &'static str = "1.17.0"; +pub const CFG_RELEASE_NUM: &'static str = "1.17.0"; // An optional number to put after the label, e.g. '.2' -> '-beta.2' // Be sure to make this starts with a dot to conform to semver pre-release // versions (section 9) -const CFG_PRERELEASE_VERSION: &'static str = ".1"; +pub const CFG_PRERELEASE_VERSION: &'static str = ".1"; -pub fn collect(build: &mut Build) { - build.release_num = CFG_RELEASE_NUM.to_string(); - build.prerelease_version = CFG_RELEASE_NUM.to_string(); +pub struct GitInfo { + inner: Option, +} - // Depending on the channel, passed in `./configure --release-channel`, - // determine various properties of the build. - match &build.config.channel[..] { - "stable" => { - build.release = CFG_RELEASE_NUM.to_string(); - build.package_vers = build.release.clone(); - build.unstable_features = false; - } - "beta" => { - build.release = format!("{}-beta{}", CFG_RELEASE_NUM, - CFG_PRERELEASE_VERSION); - build.package_vers = "beta".to_string(); - build.unstable_features = false; - } - "nightly" => { - build.release = format!("{}-nightly", CFG_RELEASE_NUM); - build.package_vers = "nightly".to_string(); - build.unstable_features = true; - } - _ => { - build.release = format!("{}-dev", CFG_RELEASE_NUM); - build.package_vers = build.release.clone(); - build.unstable_features = true; - } - } - build.version = build.release.clone(); +struct Info { + commit_date: String, + sha: String, + short_sha: String, +} - // If we have a git directory, add in some various SHA information of what - // commit this compiler was compiled from. - if build.src.join(".git").is_dir() { - let ver_date = output(Command::new("git").current_dir(&build.src) +impl GitInfo { + pub fn new(dir: &Path) -> GitInfo { + if !dir.join(".git").is_dir() { + return GitInfo { inner: None } + } + let ver_date = output(Command::new("git").current_dir(dir) .arg("log").arg("-1") .arg("--date=short") .arg("--pretty=format:%cd")); - let ver_hash = output(Command::new("git").current_dir(&build.src) + let ver_hash = output(Command::new("git").current_dir(dir) .arg("rev-parse").arg("HEAD")); let short_ver_hash = output(Command::new("git") - .current_dir(&build.src) + .current_dir(dir) .arg("rev-parse") .arg("--short=9") .arg("HEAD")); - let ver_date = ver_date.trim().to_string(); - let ver_hash = ver_hash.trim().to_string(); - let short_ver_hash = short_ver_hash.trim().to_string(); - build.version.push_str(&format!(" ({} {})", short_ver_hash, - ver_date)); - build.ver_date = Some(ver_date.to_string()); - build.ver_hash = Some(ver_hash); - build.short_ver_hash = Some(short_ver_hash); + GitInfo { + inner: Some(Info { + commit_date: ver_date.trim().to_string(), + sha: ver_hash.trim().to_string(), + short_sha: short_ver_hash.trim().to_string(), + }), + } + } + + pub fn sha(&self) -> Option<&str> { + self.inner.as_ref().map(|s| &s.sha[..]) + } + + pub fn sha_short(&self) -> Option<&str> { + self.inner.as_ref().map(|s| &s.short_sha[..]) + } + + pub fn commit_date(&self) -> Option<&str> { + self.inner.as_ref().map(|s| &s.commit_date[..]) + } + + pub fn version(&self, build: &Build, num: &str) -> String { + let mut version = build.release(num); + if let Some(ref inner) = self.inner { + version.push_str(" ("); + version.push_str(&inner.short_sha); + version.push_str(" "); + version.push_str(&inner.commit_date); + version.push_str(")"); + } + return version } } diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index cea8b13366657..46d8d4b4aab2d 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -24,6 +24,7 @@ use std::process::Command; use build_helper::{output, mtime, up_to_date}; use filetime::FileTime; +use channel::GitInfo; use util::{exe, libdir, is_dylib, copy}; use {Build, Compiler, Mode}; @@ -210,9 +211,9 @@ pub fn rustc(build: &Build, target: &str, compiler: &Compiler) { // Set some configuration variables picked up by build scripts and // the compiler alike - cargo.env("CFG_RELEASE", &build.release) + cargo.env("CFG_RELEASE", build.rust_release()) .env("CFG_RELEASE_CHANNEL", &build.config.channel) - .env("CFG_VERSION", &build.version) + .env("CFG_VERSION", build.rust_version()) .env("CFG_PREFIX", build.config.prefix.clone().unwrap_or(PathBuf::new())); if compiler.stage == 0 { @@ -229,13 +230,13 @@ pub fn rustc(build: &Build, target: &str, compiler: &Compiler) { cargo.env_remove("RUSTC_DEBUGINFO_LINES"); } - if let Some(ref ver_date) = build.ver_date { + if let Some(ref ver_date) = build.rust_info.commit_date() { cargo.env("CFG_VER_DATE", ver_date); } - if let Some(ref ver_hash) = build.ver_hash { + if let Some(ref ver_hash) = build.rust_info.sha() { cargo.env("CFG_VER_HASH", ver_hash); } - if !build.unstable_features { + if !build.unstable_features() { cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); } // Flag that rust llvm is in use @@ -416,13 +417,32 @@ pub fn tool(build: &Build, stage: u32, target: &str, tool: &str) { // build.clear_if_dirty(&out_dir, &libstd_stamp(build, stage, &host, target)); let mut cargo = build.cargo(&compiler, Mode::Tool, target, "build"); - cargo.arg("--manifest-path") - .arg(build.src.join(format!("src/tools/{}/Cargo.toml", tool))); + let dir = build.src.join("src/tools").join(tool); + cargo.arg("--manifest-path").arg(dir.join("Cargo.toml")); // We don't want to build tools dynamically as they'll be running across // stages and such and it's just easier if they're not dynamically linked. cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1"); + if let Some(dir) = build.openssl_install_dir(target) { + cargo.env("OPENSSL_STATIC", "1"); + cargo.env("OPENSSL_DIR", dir); + cargo.env("LIBZ_SYS_STATIC", "1"); + } + + cargo.env("CFG_RELEASE_CHANNEL", &build.config.channel); + + let info = GitInfo::new(&dir); + if let Some(sha) = info.sha() { + cargo.env("CFG_COMMIT_HASH", sha); + } + if let Some(sha_short) = info.sha_short() { + cargo.env("CFG_SHORT_COMMIT_HASH", sha_short); + } + if let Some(date) = info.commit_date() { + cargo.env("CFG_COMMIT_DATE", date); + } + build.run(&mut cargo); } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 8308dc3202fff..438ce6103d624 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -106,6 +106,7 @@ pub struct Config { pub gdb: Option, pub python: Option, pub configure_args: Vec, + pub openssl_static: bool, } /// Per-target configuration stored in the global configuration structure. @@ -155,6 +156,7 @@ struct Build { extended: Option, verbose: Option, sanitizers: Option, + openssl_static: Option, } /// TOML representation of various global install decisions. @@ -305,6 +307,7 @@ impl Config { set(&mut config.extended, build.extended); set(&mut config.verbose, build.verbose); set(&mut config.sanitizers, build.sanitizers); + set(&mut config.openssl_static, build.openssl_static); if let Some(ref install) = toml.install { config.prefix = install.prefix.clone().map(PathBuf::from); @@ -453,6 +456,7 @@ impl Config { ("EXTENDED", self.extended), ("SANITIZERS", self.sanitizers), ("DIST_SRC", self.rust_dist_src), + ("CARGO_OPENSSL_STATIC", self.openssl_static), } match key { diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index f95e890f346ee..30763e38a336f 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -134,6 +134,11 @@ # Build the sanitizer runtimes #sanitizers = false +# Indicates whether the OpenSSL linked into Cargo will be statically linked or +# not. If static linkage is specified then the build system will download a +# known-good version of OpenSSL, compile it, and link it to Cargo. +#openssl-static = false + # ============================================================================= # General install configuration options # ============================================================================= diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index c468d4896a614..67e4dad83ce88 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -33,19 +33,12 @@ const SH_CMD: &'static str = "sh"; const SH_CMD: &'static str = "bash"; use {Build, Compiler, Mode}; -use util::{cp_r, libdir, is_dylib, cp_filtered, copy}; - -pub fn package_vers(build: &Build) -> &str { - match &build.config.channel[..] { - "stable" => &build.release, - "beta" => "beta", - "nightly" => "nightly", - _ => &build.release, - } -} +use channel; +use util::{cp_r, libdir, is_dylib, cp_filtered, copy, exe}; fn pkgname(build: &Build, component: &str) -> String { - format!("{}-{}", component, package_vers(build)) + assert!(component.starts_with("rust")); // does not work with cargo + format!("{}-{}", component, build.rust_package_vers()) } fn distdir(build: &Build) -> PathBuf { @@ -93,7 +86,7 @@ pub fn docs(build: &Build, stage: u32, host: &str) { // As part of this step, *also* copy the docs directory to a directory which // buildbot typically uploads. if host == build.config.build { - let dst = distdir(build).join("doc").join(&build.package_vers); + let dst = distdir(build).join("doc").join(build.rust_package_vers()); t!(fs::create_dir_all(&dst)); cp_r(&src, &dst); } @@ -162,7 +155,7 @@ pub fn rustc(build: &Build, stage: u32, host: &str) { cp("LICENSE-MIT"); cp("README.md"); // tiny morsel of metadata is used by rust-packaging - let version = &build.version; + let version = build.rust_version(); t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); // On MinGW we've got a few runtime DLL dependencies that we need to @@ -312,7 +305,7 @@ pub fn std(build: &Build, compiler: &Compiler, target: &str) { } pub fn rust_src_location(build: &Build) -> PathBuf { - let plain_name = format!("rustc-{}-src", package_vers(build)); + let plain_name = format!("rustc-{}-src", build.rust_package_vers()); distdir(build).join(&format!("{}.tar.gz", plain_name)) } @@ -477,14 +470,14 @@ pub fn rust_src(build: &Build) { build.run(&mut cmd); // Rename directory, so that root folder of tarball has the correct name - let plain_name = format!("rustc-{}-src", package_vers(build)); + let plain_name = format!("rustc-{}-src", build.rust_package_vers()); let plain_dst_src = tmpdir(build).join(&plain_name); let _ = fs::remove_dir_all(&plain_dst_src); t!(fs::create_dir_all(&plain_dst_src)); cp_r(&dst_src, &plain_dst_src); // Create the version file - write_file(&plain_dst_src.join("version"), build.version.as_bytes()); + write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes()); // Create plain source tarball let mut cmd = Command::new("tar"); @@ -536,43 +529,64 @@ fn write_file(path: &Path, data: &[u8]) { t!(vf.write_all(data)); } -// FIXME(#38531) eventually this should package up a Cargo that we just compiled -// and tested locally, but for now we're downloading cargo -// artifacts from their compiled location. pub fn cargo(build: &Build, stage: u32, target: &str) { println!("Dist cargo stage{} ({})", stage, target); + let compiler = Compiler::new(stage, &build.config.build); - let branch = match &build.config.channel[..] { - "stable" | - "beta" => format!("rust-{}", build.release_num), - _ => "master".to_string(), - }; + let src = build.src.join("src/tools/cargo"); + let etc = src.join("src/etc"); + let release_num = &build.crates["cargo"].version; + let name = format!("cargo-{}", build.package_vers(release_num)); + let version = build.cargo_info.version(build, release_num); + + let tmp = tmpdir(build); + let image = tmp.join("cargo-image"); + drop(fs::remove_dir_all(&image)); + t!(fs::create_dir_all(&image)); - let dst = tmpdir(build).join("cargo"); - let _ = fs::remove_dir_all(&dst); - build.run(Command::new("git") - .arg("clone") - .arg("--depth").arg("1") - .arg("--branch").arg(&branch) - .arg("https://github.com/rust-lang/cargo") - .current_dir(dst.parent().unwrap())); - let sha = output(Command::new("git") - .arg("rev-parse") - .arg("HEAD") - .current_dir(&dst)); - let sha = sha.trim(); - println!("\tgot cargo sha: {}", sha); - - let input = format!("https://s3.amazonaws.com/rust-lang-ci/cargo-builds\ - /{}/cargo-nightly-{}.tar.gz", sha, target); - let output = distdir(build).join(format!("cargo-nightly-{}.tar.gz", target)); - println!("\tdownloading {}", input); - let mut curl = Command::new("curl"); - curl.arg("-f") - .arg("-o").arg(&output) - .arg(&input) - .arg("--retry").arg("3"); - build.run(&mut curl); + // Prepare the image directory + t!(fs::create_dir_all(image.join("share/zsh/site-functions"))); + t!(fs::create_dir_all(image.join("etc/bash_completions.d"))); + let cargo = build.cargo_out(&compiler, Mode::Tool, target) + .join(exe("cargo", target)); + install(&cargo, &image.join("bin"), 0o755); + for man in t!(etc.join("man").read_dir()) { + let man = t!(man); + install(&man.path(), &image.join("share/man/man1"), 0o644); + } + install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644); + copy(&etc.join("cargo.bashcomp.sh"), + &image.join("etc/bash_completions.d/cargo")); + let doc = image.join("share/doc/cargo"); + install(&src.join("README.md"), &doc, 0o644); + install(&src.join("LICENSE-MIT"), &doc, 0o644); + install(&src.join("LICENSE-APACHE"), &doc, 0o644); + install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644); + + // Prepare the overlay + let overlay = tmp.join("cargo-overlay"); + drop(fs::remove_dir_all(&overlay)); + t!(fs::create_dir_all(&overlay)); + install(&src.join("README.md"), &overlay, 0o644); + install(&src.join("LICENSE-MIT"), &overlay, 0o644); + install(&src.join("LICENSE-APACHE"), &overlay, 0o644); + install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644); + t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); + + // Generate the installer tarball + let mut cmd = Command::new("sh"); + cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh"))) + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=Rust-is-ready-to-roll.") + .arg(format!("--image-dir={}", sanitize_sh(&image))) + .arg(format!("--work-dir={}", sanitize_sh(&tmpdir(build)))) + .arg(format!("--output-dir={}", sanitize_sh(&distdir(build)))) + .arg(format!("--non-installed-overlay={}", sanitize_sh(&overlay))) + .arg(format!("--package-name={}-{}", name, target)) + .arg("--component-name=cargo") + .arg("--legacy-manifest-dirs=rustlib,cargo"); + build.run(&mut cmd); } /// Creates a combined installer for the specified target in the provided stage. @@ -580,10 +594,13 @@ pub fn extended(build: &Build, stage: u32, target: &str) { println!("Dist extended stage{} ({})", stage, target); let dist = distdir(build); + let cargo_vers = &build.crates["cargo"].version; let rustc_installer = dist.join(format!("{}-{}.tar.gz", pkgname(build, "rustc"), target)); - let cargo_installer = dist.join(format!("cargo-nightly-{}.tar.gz", target)); + let cargo_installer = dist.join(format!("cargo-{}-{}.tar.gz", + build.package_vers(&cargo_vers), + target)); let docs_installer = dist.join(format!("{}-{}.tar.gz", pkgname(build, "rust-docs"), target)); @@ -603,7 +620,7 @@ pub fn extended(build: &Build, stage: u32, target: &str) { install(&build.src.join("COPYRIGHT"), &overlay, 0o644); install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644); install(&build.src.join("LICENSE-MIT"), &overlay, 0o644); - let version = &build.version; + let version = build.rust_version(); t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes())); install(&etc.join("README.md"), &overlay, 0o644); @@ -876,16 +893,16 @@ pub fn extended(build: &Build, stage: u32, target: &str) { } fn add_env(build: &Build, cmd: &mut Command, target: &str) { - let mut parts = build.release_num.split('.'); - cmd.env("CFG_RELEASE_INFO", &build.version) - .env("CFG_RELEASE_NUM", &build.release_num) - .env("CFG_RELEASE", &build.release) - .env("CFG_PRERELEASE_VERSION", &build.prerelease_version) + let mut parts = channel::CFG_RELEASE_NUM.split('.'); + cmd.env("CFG_RELEASE_INFO", build.rust_version()) + .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM) + .env("CFG_RELEASE", build.rust_release()) + .env("CFG_PRERELEASE_VERSION", channel::CFG_PRERELEASE_VERSION) .env("CFG_VER_MAJOR", parts.next().unwrap()) .env("CFG_VER_MINOR", parts.next().unwrap()) .env("CFG_VER_PATCH", parts.next().unwrap()) .env("CFG_VER_BUILD", "0") // just needed to build - .env("CFG_PACKAGE_VERS", package_vers(build)) + .env("CFG_PACKAGE_VERS", build.rust_package_vers()) .env("CFG_PACKAGE_NAME", pkgname(build, "rust")) .env("CFG_BUILD", target) .env("CFG_CHANNEL", &build.config.channel); @@ -925,7 +942,8 @@ pub fn hash_and_sign(build: &Build) { cmd.arg(sign); cmd.arg(distdir(build)); cmd.arg(today.trim()); - cmd.arg(package_vers(build)); + cmd.arg(build.rust_package_vers()); + cmd.arg(build.cargo_info.version(build, &build.crates["cargo"].version)); cmd.arg(addr); t!(fs::create_dir_all(distdir(build))); diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 5fd6570e1610c..d19e5b1b88456 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -77,12 +77,9 @@ pub fn standalone(build: &Build, target: &str) { if !up_to_date(&version_input, &version_info) { let mut info = String::new(); t!(t!(File::open(&version_input)).read_to_string(&mut info)); - let blank = String::new(); - let short = build.short_ver_hash.as_ref().unwrap_or(&blank); - let hash = build.ver_hash.as_ref().unwrap_or(&blank); - let info = info.replace("VERSION", &build.release) - .replace("SHORT_HASH", short) - .replace("STAMP", hash); + let info = info.replace("VERSION", &build.rust_release()) + .replace("SHORT_HASH", build.rust_info.sha_short().unwrap_or("")) + .replace("STAMP", build.rust_info.sha().unwrap_or("")); t!(t!(File::create(&version_info)).write_all(info.as_bytes())); } diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index efc460f35838c..ba8442ebd8c37 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -19,7 +19,7 @@ use std::path::{Path, PathBuf, Component}; use std::process::Command; use Build; -use dist::{package_vers, sanitize_sh, tmpdir}; +use dist::{sanitize_sh, tmpdir}; /// Installs everything. pub fn install(build: &Build, stage: u32, host: &str) { @@ -59,7 +59,7 @@ pub fn install(build: &Build, stage: u32, host: &str) { fn install_sh(build: &Build, package: &str, name: &str, stage: u32, host: &str, prefix: &Path, docdir: &Path, libdir: &Path, mandir: &Path, empty_dir: &Path) { println!("Install {} stage{} ({})", package, stage, host); - let package_name = format!("{}-{}-{}", name, package_vers(build), host); + let package_name = format!("{}-{}-{}", name, build.rust_package_vers(), host); let mut cmd = Command::new("sh"); cmd.current_dir(empty_dir) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 2b34142b3b0f3..5215b281fac0b 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -84,7 +84,7 @@ use std::fs::{self, File}; use std::path::{Component, PathBuf, Path}; use std::process::Command; -use build_helper::{run_silent, output, mtime}; +use build_helper::{run_silent, run_suppressed, output, mtime}; use util::{exe, libdir, add_lib_path}; @@ -148,16 +148,9 @@ pub struct Build { rustc: PathBuf, src: PathBuf, out: PathBuf, - release: String, - unstable_features: bool, - ver_hash: Option, - short_ver_hash: Option, - ver_date: Option, - version: String, - package_vers: String, + rust_info: channel::GitInfo, + cargo_info: channel::GitInfo, local_rebuild: bool, - release_num: String, - prerelease_version: String, // Probed tools at runtime lldb_version: Option, @@ -173,6 +166,7 @@ pub struct Build { #[derive(Debug)] struct Crate { name: String, + version: String, deps: Vec, path: PathBuf, doc_step: String, @@ -236,6 +230,8 @@ impl Build { } None => false, }; + let rust_info = channel::GitInfo::new(&src); + let cargo_info = channel::GitInfo::new(&src.join("src/tools/cargo")); Build { flags: flags, @@ -245,22 +241,15 @@ impl Build { src: src, out: out, - release: String::new(), - unstable_features: false, - ver_hash: None, - short_ver_hash: None, - ver_date: None, - version: String::new(), + rust_info: rust_info, + cargo_info: cargo_info, local_rebuild: local_rebuild, - package_vers: String::new(), cc: HashMap::new(), cxx: HashMap::new(), crates: HashMap::new(), lldb_version: None, lldb_python_dir: None, is_sudo: is_sudo, - release_num: String::new(), - prerelease_version: String::new(), } } @@ -278,15 +267,14 @@ impl Build { cc::find(self); self.verbose("running sanity check"); sanity::check(self); - self.verbose("collecting channel variables"); - channel::collect(self); // If local-rust is the same major.minor as the current version, then force a local-rebuild let local_version_verbose = output( Command::new(&self.rustc).arg("--version").arg("--verbose")); let local_release = local_version_verbose .lines().filter(|x| x.starts_with("release:")) .next().unwrap().trim_left_matches("release:").trim(); - if local_release.split('.').take(2).eq(self.release.split('.').take(2)) { + let my_version = channel::CFG_RELEASE_NUM; + if local_release.split('.').take(2).eq(my_version.split('.').take(2)) { self.verbose(&format!("auto-detected local-rebuild {}", local_release)); self.local_rebuild = true; } @@ -478,10 +466,8 @@ impl Build { self.config.rust_codegen_units.to_string()) .env("RUSTC_DEBUG_ASSERTIONS", self.config.rust_debug_assertions.to_string()) - .env("RUSTC_SNAPSHOT", &self.rustc) .env("RUSTC_SYSROOT", self.sysroot(compiler)) .env("RUSTC_LIBDIR", self.rustc_libdir(compiler)) - .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()) .env("RUSTC_RPATH", self.config.rust_rpath.to_string()) .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc")) .env("RUSTDOC_REAL", self.rustdoc(compiler)) @@ -491,6 +477,27 @@ impl Build { cargo.env("RUSTC_BOOTSTRAP", "1"); self.add_rust_test_threads(&mut cargo); + // Almost all of the crates that we compile as part of the bootstrap may + // have a build script, including the standard library. To compile a + // build script, however, it itself needs a standard library! This + // introduces a bit of a pickle when we're compiling the standard + // library itself. + // + // To work around this we actually end up using the snapshot compiler + // (stage0) for compiling build scripts of the standard library itself. + // The stage0 compiler is guaranteed to have a libstd available for use. + // + // For other crates, however, we know that we've already got a standard + // library up and running, so we can use the normal compiler to compile + // build scripts in that situation. + if let Mode::Libstd = mode { + cargo.env("RUSTC_SNAPSHOT", &self.rustc) + .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()); + } else { + cargo.env("RUSTC_SNAPSHOT", self.compiler_path(compiler)) + .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler)); + } + // Ignore incremental modes except for stage0, since we're // not guaranteeing correctness acros builds if the compiler // is changing under your feet.` @@ -797,6 +804,12 @@ impl Build { run_silent(cmd) } + /// Runs a command, printing out nice contextual information if it fails. + fn run_quiet(&self, cmd: &mut Command) { + self.verbose(&format!("running: {:?}", cmd)); + run_suppressed(cmd) + } + /// Prints a message if this build is configured in verbose mode. fn verbose(&self, msg: &str) { if self.flags.verbose() || self.config.verbose() { @@ -914,6 +927,82 @@ impl Build { compiler.stage >= 2 && self.config.host.iter().any(|h| h == target) } + + /// Returns the directory that OpenSSL artifacts are compiled into if + /// configured to do so. + fn openssl_dir(&self, target: &str) -> Option { + // OpenSSL not used on Windows + if target.contains("windows") { + None + } else if self.config.openssl_static { + Some(self.out.join(target).join("openssl")) + } else { + None + } + } + + /// Returns the directory that OpenSSL artifacts are installed into if + /// configured as such. + fn openssl_install_dir(&self, target: &str) -> Option { + self.openssl_dir(target).map(|p| p.join("install")) + } + + /// Given `num` in the form "a.b.c" return a "release string" which + /// describes the release version number. + /// + /// For example on nightly this returns "a.b.c-nightly", on beta it returns + /// "a.b.c-beta.1" and on stable it just returns "a.b.c". + fn release(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => format!("{}-beta{}", num, channel::CFG_PRERELEASE_VERSION), + "nightly" => format!("{}-nightly", num), + _ => format!("{}-dev", num), + } + } + + /// Returns the value of `release` above for Rust itself. + fn rust_release(&self) -> String { + self.release(channel::CFG_RELEASE_NUM) + } + + /// Returns the "package version" for a component given the `num` release + /// number. + /// + /// The package version is typically what shows up in the names of tarballs. + /// For channels like beta/nightly it's just the channel name, otherwise + /// it's the `num` provided. + fn package_vers(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => "beta".to_string(), + "nightly" => "nightly".to_string(), + _ => format!("{}-dev", num), + } + } + + /// Returns the value of `package_vers` above for Rust itself. + fn rust_package_vers(&self) -> String { + self.package_vers(channel::CFG_RELEASE_NUM) + } + + /// Returns the `version` string associated with this compiler for Rust + /// itself. + /// + /// Note that this is a descriptive string which includes the commit date, + /// sha, version, etc. + fn rust_version(&self) -> String { + self.rust_info.version(self, channel::CFG_RELEASE_NUM) + } + + /// Returns whether unstable features should be enabled for the compiler + /// we're building. + fn unstable_features(&self) -> bool { + match &self.config.channel[..] { + "stable" | "beta" => false, + "nightly" | _ => true, + } + } } impl<'a> Compiler<'a> { diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/metadata.rs index 5ab542b6a2489..5918fe41e7c8b 100644 --- a/src/bootstrap/metadata.rs +++ b/src/bootstrap/metadata.rs @@ -27,6 +27,7 @@ struct Output { struct Package { id: String, name: String, + version: String, source: Option, manifest_path: String, } @@ -72,6 +73,7 @@ fn build_krate(build: &mut Build, krate: &str) { test_step: format!("test-crate-{}", package.name), bench_step: format!("bench-crate-{}", package.name), name: package.name, + version: package.version, deps: Vec::new(), path: path, }); diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index f16fc2092f616..90e1530308f4a 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -191,3 +191,96 @@ pub fn test_helpers(build: &Build, target: &str) { .file(build.src.join("src/rt/rust_test_helpers.c")) .compile("librust_test_helpers.a"); } +const OPENSSL_VERS: &'static str = "1.0.2k"; +const OPENSSL_SHA256: &'static str = + "6b3977c61f2aedf0f96367dcfb5c6e578cf37e7b8d913b4ecb6643c3cb88d8c0"; + +pub fn openssl(build: &Build, target: &str) { + let out = match build.openssl_dir(target) { + Some(dir) => dir, + None => return, + }; + + let stamp = out.join(".stamp"); + let mut contents = String::new(); + drop(File::open(&stamp).and_then(|mut f| f.read_to_string(&mut contents))); + if contents == OPENSSL_VERS { + return + } + t!(fs::create_dir_all(&out)); + + let name = format!("openssl-{}.tar.gz", OPENSSL_VERS); + let tarball = out.join(&name); + if !tarball.exists() { + let tmp = tarball.with_extension("tmp"); + build.run(Command::new("curl") + .arg("-o").arg(&tmp) + .arg(format!("https://www.openssl.org/source/{}", name))); + let mut shasum = if target.contains("apple") { + let mut cmd = Command::new("shasum"); + cmd.arg("-a").arg("256"); + cmd + } else { + Command::new("sha256sum") + }; + let output = output(&mut shasum.arg(&tmp)); + let found = output.split_whitespace().next().unwrap(); + if found != OPENSSL_SHA256 { + panic!("downloaded openssl sha256 different\n\ + expected: {}\n\ + found: {}\n", OPENSSL_SHA256, found); + } + t!(fs::rename(&tmp, &tarball)); + } + let obj = out.join(format!("openssl-{}", OPENSSL_VERS)); + let dst = build.openssl_install_dir(target).unwrap(); + drop(fs::remove_dir_all(&obj)); + drop(fs::remove_dir_all(&dst)); + build.run(Command::new("tar").arg("xf").arg(&tarball).current_dir(&out)); + + let mut configure = Command::new(obj.join("Configure")); + configure.arg(format!("--prefix={}", dst.display())); + configure.arg("no-dso"); + configure.arg("no-ssl2"); + configure.arg("no-ssl3"); + + let os = match target { + "aarch64-unknown-linux-gnu" => "linux-aarch64", + "arm-unknown-linux-gnueabi" => "linux-armv4", + "arm-unknown-linux-gnueabihf" => "linux-armv4", + "armv7-unknown-linux-gnueabihf" => "linux-armv4", + "i686-apple-darwin" => "darwin-i386-cc", + "i686-unknown-freebsd" => "BSD-x86-elf", + "i686-unknown-linux-gnu" => "linux-elf", + "i686-unknown-linux-musl" => "linux-elf", + "mips-unknown-linux-gnu" => "linux-mips32", + "mips64-unknown-linux-gnuabi64" => "linux64-mips64", + "mips64el-unknown-linux-gnuabi64" => "linux64-mips64", + "mipsel-unknown-linux-gnu" => "linux-mips32", + "powerpc-unknown-linux-gnu" => "linux-ppc", + "powerpc64-unknown-linux-gnu" => "linux-ppc64", + "powerpc64le-unknown-linux-gnu" => "linux-ppc64le", + "s390x-unknown-linux-gnu" => "linux64-s390x", + "x86_64-apple-darwin" => "darwin64-x86_64-cc", + "x86_64-unknown-freebsd" => "BSD-x86_64", + "x86_64-unknown-linux-gnu" => "linux-x86_64", + "x86_64-unknown-linux-musl" => "linux-x86_64", + "x86_64-unknown-netbsd" => "BSD-x86_64", + _ => panic!("don't know how to configure OpenSSL for {}", target), + }; + configure.arg(os); + configure.env("CC", build.cc(target)); + for flag in build.cflags(target) { + configure.arg(flag); + } + configure.current_dir(&obj); + println!("Configuring openssl for {}", target); + build.run_quiet(&mut configure); + println!("Building openssl for {}", target); + build.run_quiet(Command::new("make").current_dir(&obj)); + println!("Installing openssl for {}", target); + build.run_quiet(Command::new("make").arg("install").current_dir(&obj)); + + let mut f = t!(File::create(&stamp)); + t!(f.write_all(OPENSSL_VERS.as_bytes())); +} diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index dcd3407e174aa..6f8975309d112 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -278,9 +278,19 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.build(&krate.build_step, path) .dep(|s| s.name("libtest-link")) .dep(move |s| s.name("llvm").host(&build.config.build).stage(0)) + .dep(|s| s.name("may-run-build-script")) .run(move |s| compile::rustc(build, s.target, &s.compiler())); } + // Crates which have build scripts need to rely on this rule to ensure that + // the necessary prerequisites for a build script are linked and located in + // place. + rules.build("may-run-build-script", "path/to/nowhere") + .dep(move |s| { + s.name("libstd-link") + .host(&build.config.build) + .target(&build.config.build) + }); rules.build("startup-objects", "src/rtstartup") .dep(|s| s.name("create-sysroot").target(s.host)) .run(move |s| compile::build_startup_objects(build, &s.compiler(), s.target)); @@ -478,9 +488,10 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .dep(|s| s.name("dist-src")) .run(move |_| check::distcheck(build)); - rules.build("test-helpers", "src/rt/rust_test_helpers.c") .run(move |s| native::test_helpers(build, s.target)); + rules.build("openssl", "path/to/nowhere") + .run(move |s| native::openssl(build, s.target)); // Some test suites are run inside emulators, and most of our test binaries // are linked dynamically which means we need to ship the standard library @@ -547,6 +558,17 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.build("tool-qemu-test-client", "src/tools/qemu-test-client") .dep(|s| s.name("libstd")) .run(move |s| compile::tool(build, s.stage, s.target, "qemu-test-client")); + rules.build("tool-cargo", "src/tools/cargo") + .dep(|s| s.name("libstd")) + .dep(|s| s.stage(0).host(s.target).name("openssl")) + .dep(move |s| { + // Cargo depends on procedural macros, which requires a full host + // compiler to be available, so we need to depend on that. + s.name("librustc-link") + .target(&build.config.build) + .host(&build.config.build) + }) + .run(move |s| compile::tool(build, s.stage, s.target, "cargo")); // ======================================================================== // Documentation targets @@ -673,6 +695,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.dist("dist-cargo", "cargo") .host(true) .only_host_build(true) + .dep(|s| s.name("tool-cargo")) .run(move |s| dist::cargo(build, s.stage, s.target)); rules.dist("dist-extended", "extended") .default(build.config.extended) @@ -1180,6 +1203,7 @@ mod tests { build_step: "build-crate-std".to_string(), test_step: "test-std".to_string(), bench_step: "bench-std".to_string(), + version: String::new(), }); build.crates.insert("test".to_string(), ::Crate { name: "test".to_string(), @@ -1189,10 +1213,12 @@ mod tests { build_step: "build-crate-test".to_string(), test_step: "test-test".to_string(), bench_step: "bench-test".to_string(), + version: String::new(), }); build.crates.insert("rustc-main".to_string(), ::Crate { name: "rustc-main".to_string(), deps: Vec::new(), + version: String::new(), path: cwd.join("src/rustc-main"), doc_step: "doc-rustc-main".to_string(), build_step: "build-crate-rustc-main".to_string(), @@ -1378,7 +1404,7 @@ mod tests { let all = rules.expand(&plan); println!("all rules: {:#?}", all); assert!(!all.contains(&step.name("rustc"))); - assert!(!all.contains(&step.name("build-crate-std").stage(1))); + assert!(!all.contains(&step.name("build-crate-test").stage(1))); // all stage0 compiles should be for the build target, A for step in all.iter().filter(|s| s.stage == 0) { diff --git a/src/build_helper/lib.rs b/src/build_helper/lib.rs index 3dfd293808286..08f1f31c2d74b 100644 --- a/src/build_helper/lib.rs +++ b/src/build_helper/lib.rs @@ -53,6 +53,24 @@ pub fn run_silent(cmd: &mut Command) { } } +pub fn run_suppressed(cmd: &mut Command) { + let output = match cmd.output() { + Ok(status) => status, + Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", + cmd, e)), + }; + if !output.status.success() { + fail(&format!("command did not execute successfully: {:?}\n\ + expected success, got: {}\n\n\ + stdout ----\n{}\n\ + stderr ----\n{}\n", + cmd, + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr))); + } +} + pub fn gnu_target(target: &str) -> String { match target { "i686-pc-windows-msvc" => "i686-pc-win32".to_string(), diff --git a/src/ci/run.sh b/src/ci/run.sh index 53d16f6239e66..4c4836d7ca230 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -27,6 +27,7 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps" +RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-openssl-static" if [ "$DIST_SRC" = "" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src" diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index c2c91487b0c17..ceefcc9e0ec46 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -131,7 +131,8 @@ macro_rules! t { } struct Builder { - channel: String, + rust_release: String, + cargo_release: String, input: PathBuf, output: PathBuf, gpg_passphrase: String, @@ -147,13 +148,15 @@ fn main() { let input = PathBuf::from(args.next().unwrap()); let output = PathBuf::from(args.next().unwrap()); let date = args.next().unwrap(); - let channel = args.next().unwrap(); + let rust_release = args.next().unwrap(); + let cargo_release = args.next().unwrap(); let s3_address = args.next().unwrap(); let mut passphrase = String::new(); t!(io::stdin().read_to_string(&mut passphrase)); Builder { - channel: channel, + rust_release: rust_release, + cargo_release: cargo_release, input: input, output: output, gpg_passphrase: passphrase, @@ -184,10 +187,10 @@ impl Builder { manifest.insert("pkg".to_string(), toml::encode(&pkg)); let manifest = toml::Value::Table(manifest).to_string(); - let filename = format!("channel-rust-{}.toml", self.channel); + let filename = format!("channel-rust-{}.toml", self.rust_release); self.write_manifest(&manifest, &filename); - if self.channel != "beta" && self.channel != "nightly" { + if self.rust_release != "beta" && self.rust_release != "nightly" { self.write_manifest(&manifest, "channel-rust-stable.toml"); } } @@ -336,11 +339,11 @@ impl Builder { fn filename(&self, component: &str, target: &str) -> String { if component == "rust-src" { - format!("rust-src-{}.tar.gz", self.channel) + format!("rust-src-{}.tar.gz", self.rust_release) } else if component == "cargo" { - format!("cargo-nightly-{}.tar.gz", target) + format!("cargo-{}-{}.tar.gz", self.cargo_release, target) } else { - format!("{}-{}-{}.tar.gz", component, self.channel, target) + format!("{}-{}-{}.tar.gz", component, self.rust_release, target) } } diff --git a/src/tools/cargo b/src/tools/cargo index 6ed5a43bb566e..d17b61aa5a2ca 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit 6ed5a43bb566e0ee3fe7981de5aa5a36e2905ebd +Subproject commit d17b61aa5a2ca790f268a043bffdb0ffb04f0ec7 diff --git a/src/tools/tidy/src/cargo.rs b/src/tools/tidy/src/cargo.rs index 11acb64743a7a..053f0bbe3b81d 100644 --- a/src/tools/tidy/src/cargo.rs +++ b/src/tools/tidy/src/cargo.rs @@ -20,7 +20,7 @@ use std::fs::File; use std::path::Path; pub fn check(path: &Path, bad: &mut bool) { - if path.ends_with("vendor") { + if !super::filter_dirs(path) { return } for entry in t!(path.read_dir(), path).map(|e| t!(e)) { diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index e1c44a20e9756..3bf396db4d39d 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -16,6 +16,7 @@ use std::path::Path; static LICENSES: &'static [&'static str] = &[ "MIT/Apache-2.0", + "MIT / Apache-2.0", "Apache-2.0/MIT", "MIT OR Apache-2.0", "MIT", @@ -25,6 +26,7 @@ static LICENSES: &'static [&'static str] = &[ /// These MPL licensed projects are acceptable, but only these. static EXCEPTIONS: &'static [&'static str] = &[ "mdbook", + "openssl", "pest", "thread-id", ]; diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs index 1bb7919c1d35b..2af891b5b8562 100644 --- a/src/tools/tidy/src/main.rs +++ b/src/tools/tidy/src/main.rs @@ -70,6 +70,7 @@ fn filter_dirs(path: &Path) -> bool { "src/rustllvm", "src/rust-installer", "src/liblibc", + "src/tools/cargo", "src/vendor", ]; skip.iter().any(|p| path.ends_with(p))