From e652f88d726f9d47a66919f0d267239cc112048a Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 3 Feb 2021 11:22:35 -0500 Subject: [PATCH 1/2] Add track_caller to `builder_helper::output` If something goes wrong here, showing `fn output` is unhelpful. Show where the command is being run instead. --- src/build_helper/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/build_helper/lib.rs b/src/build_helper/lib.rs index 80f804174ed08..b1ec072f3f8aa 100644 --- a/src/build_helper/lib.rs +++ b/src/build_helper/lib.rs @@ -130,6 +130,7 @@ pub fn make(host: &str) -> PathBuf { } } +#[track_caller] pub fn output(cmd: &mut Command) -> String { let output = match cmd.stderr(Stdio::inherit()).output() { Ok(status) => status, From 0be4046bf91ea3726098e7714642697c43af7e98 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 31 Jan 2021 12:20:30 -0500 Subject: [PATCH 2/2] Move llvm submodule updates to rustbuild This enables better caching, since LLVM is only updated when needed, not whenever x.py is run. Before, bootstrap.py had to use heuristics to guess if LLVM would be needed, and updated the module more often than necessary as a result. This syncs the LLVM submodule only just before building the compiler, so people working on the standard library never have to worry about it. Example output: ``` Copying stage0 std from stage0 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu) Updating submodule src/llvm-project Submodule 'src/llvm-project' (https://github.com/rust-lang/llvm-project.git) registered for path 'src/llvm-project' Submodule path 'src/llvm-project': checked out 'f9a8d70b6e0365ac2172ca6b7f1de0341297458d' ``` - Don't try to update the LLVM submodule when using system LLVM Previously, this would try to update LLVM unconditionally. Now the submodule is only initialized if `llvm-config` is not set. - Don't update LLVM submodule in dry runs This prevents the following test failures: ``` running 17 tests fatal: invalid gitfile format: /checkout/src/llvm-project/.git test builder::tests::defaults::build_cross_compile ... FAILED ---- builder::tests::defaults::build_default stdout ---- thread 'main' panicked at 'command did not execute successfully: "git" "rev-parse" "HEAD" expected success, got: exit code: 128', src/build_helper/lib.rs:139:9 ``` - Try running git without --progress if it fails the first time This avoids having to do version detection to see if --progress is supported or not. - Don't try to update submodules when the source repository isn't managed by git - Update LLVM submodules that have already been checked out - Only check for whether the submodule should be updated in lib.rs; update it unconditionally in native.rs --- src/bootstrap/bootstrap.py | 16 ++------ src/bootstrap/lib.rs | 10 +++++ src/bootstrap/native.rs | 84 +++++++++++++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 13 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 149a899cef7a0..bd5b3797ea825 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -991,28 +991,20 @@ def update_submodules(self): ).decode(default_encoding).splitlines()] filtered_submodules = [] submodules_names = [] - llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git")) - external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm() - llvm_needed = not self.get_toml('codegen-backends', 'rust') \ - or "llvm" in self.get_toml('codegen-backends', 'rust') for module in submodules: + # This is handled by native::Llvm in rustbuild, not here if module.endswith("llvm-project"): - # Don't sync the llvm-project submodule if an external LLVM was - # provided, if we are downloading LLVM or if the LLVM backend is - # not being built. Also, if the submodule has been initialized - # already, sync it anyways so that it doesn't mess up contributor - # pull requests. - if external_llvm_provided or not llvm_needed: - if self.get_toml('lld') != 'true' and not llvm_checked_out: - continue + continue check = self.check_submodule(module, slow_submodules) filtered_submodules.append((module, check)) submodules_names.append(module) recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, cwd=self.rust_root, stdout=subprocess.PIPE) recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() + # { filename: hash } recorded_submodules = {} for data in recorded: + # [mode, kind, hash, filename] data = data.split() recorded_submodules[data[3]] = data[2] for module in filtered_submodules: diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 2960dd3df6bf4..a351290a4206f 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -472,12 +472,22 @@ impl Build { slice::from_ref(&self.build.triple) } + /// If the LLVM submodule has been initialized already, sync it unconditionally. This avoids + /// contributors checking in a submodule change by accident. + pub fn maybe_update_llvm_submodule(&self) { + if self.in_tree_llvm_info.is_git() { + native::update_llvm_submodule(self); + } + } + /// Executes the entire build, as configured by the flags and configuration. pub fn build(&mut self) { unsafe { job::setup(self); } + self.maybe_update_llvm_submodule(); + if let Subcommand::Format { check, paths } = &self.config.cmd { return format::format(self, *check, &paths); } diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index bde0a96f03013..44c281efe22be 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -21,7 +21,7 @@ use build_helper::{output, t}; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::config::TargetSelection; use crate::util::{self, exe}; -use crate::GitRepo; +use crate::{Build, GitRepo}; use build_helper::up_to_date; pub struct Meta { @@ -91,6 +91,85 @@ pub fn prebuilt_llvm_config( Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) } +// modified from `check_submodule` and `update_submodule` in bootstrap.py +pub(crate) fn update_llvm_submodule(build: &Build) { + let llvm_project = &Path::new("src").join("llvm-project"); + + fn dir_is_empty(dir: &Path) -> bool { + t!(std::fs::read_dir(dir)).next().is_none() + } + + // NOTE: The check for the empty directory is here because when running x.py + // the first time, the llvm submodule won't be checked out. Check it out + // now so we can build it. + if !build.in_tree_llvm_info.is_git() && !dir_is_empty(&build.config.src.join(llvm_project)) { + return; + } + + // check_submodule + let checked_out = if build.config.fast_submodules { + Some(output( + Command::new("git") + .args(&["rev-parse", "HEAD"]) + .current_dir(build.config.src.join(llvm_project)), + )) + } else { + None + }; + + // update_submodules + let recorded = output( + Command::new("git") + .args(&["ls-tree", "HEAD"]) + .arg(llvm_project) + .current_dir(&build.config.src), + ); + let hash = + recorded.split(' ').nth(2).unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); + + // update_submodule + if let Some(llvm_hash) = checked_out { + if hash == llvm_hash { + // already checked out + return; + } + } + + println!("Updating submodule {}", llvm_project.display()); + build.run( + Command::new("git") + .args(&["submodule", "-q", "sync"]) + .arg(llvm_project) + .current_dir(&build.config.src), + ); + + // Try passing `--progress` to start, then run git again without if that fails. + let update = |progress: bool| { + let mut git = Command::new("git"); + git.args(&["submodule", "update", "--init", "--recursive"]); + if progress { + git.arg("--progress"); + } + git.arg(llvm_project).current_dir(&build.config.src); + git + }; + // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails. + if !update(true).status().map_or(false, |status| status.success()) { + build.run(&mut update(false)); + } + + build.run( + Command::new("git") + .args(&["reset", "-q", "--hard"]) + .current_dir(build.config.src.join(llvm_project)), + ); + build.run( + Command::new("git") + .args(&["clean", "-qdfx"]) + .current_dir(build.config.src.join(llvm_project)), + ); +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Llvm { pub target: TargetSelection, @@ -128,6 +207,9 @@ impl Step for Llvm { Err(m) => m, }; + if !builder.config.dry_run { + update_llvm_submodule(builder); + } if builder.config.llvm_link_shared && (target.contains("windows") || target.contains("apple-darwin")) {