Move llvm submodule updates to rustbuild

This enables better caching, since LLVM is only updated when needed, not
whenever x.py is run. Before, bootstrap.py had to use heuristics to
guess if LLVM would be needed, and updated the module more often than
necessary as a result.

This syncs the LLVM submodule only just before building the compiler, so
people working on the standard library never have to worry about it.
Example output:

```
Copying stage0 std from stage0 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu)
Updating submodule src/llvm-project
Submodule 'src/llvm-project' (https:/rust-lang/llvm-project.git) registered for path 'src/llvm-project'
Submodule path 'src/llvm-project': checked out 'f9a8d70b6e0365ac2172ca6b7f1de0341297458d'
```

- Don't try to update the LLVM submodule when using system LLVM

  Previously, this would try to update LLVM unconditionally. Now the
  submodule is only initialized if `llvm-config` is not set.

- Don't update LLVM submodule in dry runs

  This prevents the following test failures:

  ```
  running 17 tests
  fatal: invalid gitfile format: /checkout/src/llvm-project/.git
  test builder::tests::defaults::build_cross_compile ... FAILED

  ---- builder::tests::defaults::build_default stdout ----
  thread 'main' panicked at 'command did not execute successfully: "git" "rev-parse" "HEAD"
  expected success, got: exit code: 128', src/build_helper/lib.rs:139:9
  ```

- Try running git without --progress if it fails the first time

  This avoids having to do version detection to see if --progress is
  supported or not.

- Don't try to update submodules when the source repository isn't managed by git

- Update LLVM submodules that have already been checked out

- Only check for whether the submodule should be updated in lib.rs; update
it unconditionally in native.rs

src/bootstrap/bootstrap.py

@@ -991,28 +991,20 @@ def update_submodules(self):
  ).decode(default_encoding).splitlines()]
  filtered_submodules = []
  submodules_names = []
- llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
- external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
- llvm_needed = not self.get_toml('codegen-backends', 'rust') \
- or "llvm" in self.get_toml('codegen-backends', 'rust')
  for module in submodules:
+ # This is handled by native::Llvm in rustbuild, not here
  if module.endswith("llvm-project"):
- # Don't sync the llvm-project submodule if an external LLVM was
- # provided, if we are downloading LLVM or if the LLVM backend is
- # not being built. Also, if the submodule has been initialized
- # already, sync it anyways so that it doesn't mess up contributor
- # pull requests.
- if external_llvm_provided or not llvm_needed:
- if self.get_toml('lld') != 'true' and not llvm_checked_out:
- continue
+ continue
  check = self.check_submodule(module, slow_submodules)
  filtered_submodules.append((module, check))
  submodules_names.append(module)
  recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
  cwd=self.rust_root, stdout=subprocess.PIPE)
  recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
+ # { filename: hash }
  recorded_submodules = {}
  for data in recorded:
+ # [mode, kind, hash, filename]
  data = data.split()
  recorded_submodules[data[3]] = data[2]
  for module in filtered_submodules:

src/bootstrap/lib.rs

@@ -472,12 +472,22 @@ impl Build {
  slice::from_ref(&self.build.triple)
  }
 
+ /// If the LLVM submodule has been initialized already, sync it unconditionally. This avoids
+ /// contributors checking in a submodule change by accident.
+ pub fn maybe_update_llvm_submodule(&self) {
+ if self.in_tree_llvm_info.is_git() {
+ native::update_llvm_submodule(self);
+ }
+ }
+
  /// Executes the entire build, as configured by the flags and configuration.
  pub fn build(&mut self) {
  unsafe {
  job::setup(self);
  }
 
+ self.maybe_update_llvm_submodule();
+
  if let Subcommand::Format { check, paths } = &self.config.cmd {
  return format::format(self, *check, &paths);
  }

src/bootstrap/native.rs

@@ -21,7 +21,7 @@ use build_helper::{output, t};
 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::config::TargetSelection;
 use crate::util::{self, exe};
-use crate::GitRepo;
+use crate::{Build, GitRepo};
 use build_helper::up_to_date;
 
 pub struct Meta {
@@ -91,6 +91,85 @@ pub fn prebuilt_llvm_config(
  Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() })
 }
 
+// modified from `check_submodule` and `update_submodule` in bootstrap.py
+pub(crate) fn update_llvm_submodule(build: &Build) {
+ let llvm_project = &Path::new("src").join("llvm-project");
+
+ fn dir_is_empty(dir: &Path) -> bool {
+ t!(std::fs::read_dir(dir)).next().is_none()
+ }
+
+ // NOTE: The check for the empty directory is here because when running x.py
+ // the first time, the llvm submodule won't be checked out. Check it out
+ // now so we can build it.
+ if !build.in_tree_llvm_info.is_git() && !dir_is_empty(&build.config.src.join(llvm_project)) {
+ return;
+ }
+
+ // check_submodule
+ let checked_out = if build.config.fast_submodules {
+ Some(output(
+ Command::new("git")
+ .args(&["rev-parse", "HEAD"])
+ .current_dir(build.config.src.join(llvm_project)),
+ ))
+ } else {
+ None
+ };
+
+ // update_submodules
+ let recorded = output(
+ Command::new("git")
+ .args(&["ls-tree", "HEAD"])
+ .arg(llvm_project)
+ .current_dir(&build.config.src),
+ );
+ let hash =
+ recorded.split(' ').nth(2).unwrap_or_else(|| panic!("unexpected output `{}`", recorded));
+
+ // update_submodule
+ if let Some(llvm_hash) = checked_out {
+ if hash == llvm_hash {
+ // already checked out
+ return;
+ }
+ }
+
+ println!("Updating submodule {}", llvm_project.display());
+ build.run(
+ Command::new("git")
+ .args(&["submodule", "-q", "sync"])
+ .arg(llvm_project)
+ .current_dir(&build.config.src),
+ );
+
+ // Try passing `--progress` to start, then run git again without if that fails.
+ let update = |progress: bool| {
+ let mut git = Command::new("git");
+ git.args(&["submodule", "update", "--init", "--recursive"]);
+ if progress {
+ git.arg("--progress");
+ }
+ git.arg(llvm_project).current_dir(&build.config.src);
+ git
+ };
+ // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails.
+ if !update(true).status().map_or(false, |status| status.success()) {
+ build.run(&mut update(false));
+ }
+
+ build.run(
+ Command::new("git")
+ .args(&["reset", "-q", "--hard"])
+ .current_dir(build.config.src.join(llvm_project)),
+ );
+ build.run(
+ Command::new("git")
+ .args(&["clean", "-qdfx"])
+ .current_dir(build.config.src.join(llvm_project)),
+ );
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Llvm {
  pub target: TargetSelection,
@@ -128,6 +207,9 @@ impl Step for Llvm {
  Err(m) => m,
  };
 
+ if !builder.config.dry_run {
+ update_llvm_submodule(builder);
+ }
  if builder.config.llvm_link_shared
  && (target.contains("windows") || target.contains("apple-darwin"))
  {