Skip to content

Commit

Permalink
Rollup merge of #104512 - jyn514:download-ci-llvm-default, r=Mark-Sim…
Browse files Browse the repository at this point in the history
…ulacrum

Set `download-ci-llvm = "if-available"` by default when `channel = dev`

See rust-lang/compiler-team#566. The motivation for changing the default is to avoid downloading and building LLVM when someone runs `x build` before running `x setup`. The motivation for only doing it on `channel = "dev"` is to avoid breaking distros or users installing from source. It works because `dev` is also the default channel.

The diff looks larger than it is; most of it is moving the `llvm` branch below the `rust` so `config.channel` is set.

r? `@Mark-Simulacrum` cc `@oli-obk` `@bjorn3` `@cuviper`
  • Loading branch information
matthiaskrgr authored Dec 10, 2022
2 parents 0f5d3ba + ac67262 commit 1ce18d2
Show file tree
Hide file tree
Showing 5 changed files with 141 additions and 100 deletions.
7 changes: 3 additions & 4 deletions config.toml.example
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,16 @@ changelog-seen = 2
# Unless you're developing for a target where Rust CI doesn't build a compiler
# toolchain or changing LLVM locally, you probably want to set this to true.
#
# This is false by default so that distributions don't unexpectedly download
# LLVM from the internet.
#
# All tier 1 targets are currently supported; set this to `"if-available"` if
# you are not sure whether you're on a tier 1 target.
#
# We also currently only support this when building LLVM for the build triple.
#
# Note that many of the LLVM options are not currently supported for
# downloading. Currently only the "assertions" option can be toggled.
#download-ci-llvm = false
#
# Defaults to "if-available" when `channel = "dev"` and "false" otherwise.
#download-ci-llvm = "if-available"

# Indicates whether LLVM rebuild should be skipped when running bootstrap. If
# this is `false` then the compiler's LLVM will be rebuilt whenever the built
Expand Down
202 changes: 108 additions & 94 deletions src/bootstrap/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
//! This module implements parsing `config.toml` configuration files to tweak
//! how the build runs.

#[cfg(test)]
mod tests;

use std::cell::{Cell, RefCell};
use std::cmp;
use std::collections::{HashMap, HashSet};
Expand Down Expand Up @@ -696,7 +699,7 @@ define_config! {
}
}

#[derive(Deserialize)]
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum StringOrBool {
String(String),
Expand Down Expand Up @@ -822,6 +825,29 @@ impl Config {
}

pub fn parse(args: &[String]) -> Config {
#[cfg(test)]
let get_toml = |_: &_| TomlConfig::default();
#[cfg(not(test))]
let get_toml = |file: &Path| {
let contents =
t!(fs::read_to_string(file), format!("config file {} not found", file.display()));
// Deserialize to Value and then TomlConfig to prevent the Deserialize impl of
// TomlConfig and sub types to be monomorphized 5x by toml.
match toml::from_str(&contents)
.and_then(|table: toml::Value| TomlConfig::deserialize(table))
{
Ok(table) => table,
Err(err) => {
eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err);
crate::detail_exit(2);
}
}
};

Self::parse_inner(args, get_toml)
}

fn parse_inner<'a>(args: &[String], get_toml: impl 'a + Fn(&Path) -> TomlConfig) -> Config {
let flags = Flags::parse(&args);
let mut config = Config::default_opts();

Expand Down Expand Up @@ -907,25 +933,6 @@ impl Config {

config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));

#[cfg(test)]
let get_toml = |_| TomlConfig::default();
#[cfg(not(test))]
let get_toml = |file: &Path| {
let contents =
t!(fs::read_to_string(file), format!("config file {} not found", file.display()));
// Deserialize to Value and then TomlConfig to prevent the Deserialize impl of
// TomlConfig and sub types to be monomorphized 5x by toml.
match toml::from_str(&contents)
.and_then(|table: toml::Value| TomlConfig::deserialize(table))
{
Ok(table) => table,
Err(err) => {
eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err);
crate::detail_exit(2);
}
}
};

// Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory.
let toml_path = flags
.config
Expand Down Expand Up @@ -1063,6 +1070,79 @@ impl Config {
let mut optimize = None;
let mut ignore_git = None;

if let Some(rust) = toml.rust {
debug = rust.debug;
debug_assertions = rust.debug_assertions;
debug_assertions_std = rust.debug_assertions_std;
overflow_checks = rust.overflow_checks;
overflow_checks_std = rust.overflow_checks_std;
debug_logging = rust.debug_logging;
debuginfo_level = rust.debuginfo_level;
debuginfo_level_rustc = rust.debuginfo_level_rustc;
debuginfo_level_std = rust.debuginfo_level_std;
debuginfo_level_tools = rust.debuginfo_level_tools;
debuginfo_level_tests = rust.debuginfo_level_tests;
config.rust_split_debuginfo = rust
.split_debuginfo
.as_deref()
.map(SplitDebuginfo::from_str)
.map(|v| v.expect("invalid value for rust.split_debuginfo"))
.unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple));
optimize = rust.optimize;
ignore_git = rust.ignore_git;
config.rust_new_symbol_mangling = rust.new_symbol_mangling;
set(&mut config.rust_optimize_tests, rust.optimize_tests);
set(&mut config.codegen_tests, rust.codegen_tests);
set(&mut config.rust_rpath, rust.rpath);
set(&mut config.jemalloc, rust.jemalloc);
set(&mut config.test_compare_mode, rust.test_compare_mode);
set(&mut config.backtrace, rust.backtrace);
set(&mut config.channel, rust.channel);
config.description = rust.description;
set(&mut config.rust_dist_src, rust.dist_src);
set(&mut config.verbose_tests, rust.verbose_tests);
// in the case "false" is set explicitly, do not overwrite the command line args
if let Some(true) = rust.incremental {
config.incremental = true;
}
set(&mut config.use_lld, rust.use_lld);
set(&mut config.lld_enabled, rust.lld);
set(&mut config.llvm_tools_enabled, rust.llvm_tools);
config.rustc_parallel = rust.parallel_compiler.unwrap_or(false);
config.rustc_default_linker = rust.default_linker;
config.musl_root = rust.musl_root.map(PathBuf::from);
config.save_toolstates = rust.save_toolstates.map(PathBuf::from);
set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings));
set(&mut config.backtrace_on_ice, rust.backtrace_on_ice);
set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir);
config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit;
set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo);
set(&mut config.control_flow_guard, rust.control_flow_guard);
config.llvm_libunwind_default = rust
.llvm_libunwind
.map(|v| v.parse().expect("failed to parse rust.llvm-libunwind"));

if let Some(ref backends) = rust.codegen_backends {
config.rust_codegen_backends =
backends.iter().map(|s| INTERNER.intern_str(s)).collect();
}

config.rust_codegen_units = rust.codegen_units.map(threads_from_config);
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc);

config.rust_lto = rust
.lto
.as_deref()
.map(|value| RustcLto::from_str(value).unwrap())
.unwrap_or_default();
} else {
config.rust_profile_use = flags.rust_profile_use;
config.rust_profile_generate = flags.rust_profile_generate;
}

if let Some(llvm) = toml.llvm {
match llvm.ccache {
Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()),
Expand Down Expand Up @@ -1099,13 +1179,17 @@ impl Config {
config.llvm_polly = llvm.polly.unwrap_or(false);
config.llvm_clang = llvm.clang.unwrap_or(false);
config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default());

let asserts = llvm_assertions.unwrap_or(false);
config.llvm_from_ci = match llvm.download_ci_llvm {
Some(StringOrBool::String(s)) => {
assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s);
crate::native::is_ci_llvm_available(&config, llvm_assertions.unwrap_or(false))
crate::native::is_ci_llvm_available(&config, asserts)
}
Some(StringOrBool::Bool(b)) => b,
None => false,
None => {
config.channel == "dev" && crate::native::is_ci_llvm_available(&config, asserts)
}
};

if config.llvm_from_ci {
Expand Down Expand Up @@ -1145,79 +1229,9 @@ impl Config {
// the link step) with each stage.
config.llvm_link_shared.set(Some(true));
}
}

if let Some(rust) = toml.rust {
debug = rust.debug;
debug_assertions = rust.debug_assertions;
debug_assertions_std = rust.debug_assertions_std;
overflow_checks = rust.overflow_checks;
overflow_checks_std = rust.overflow_checks_std;
debug_logging = rust.debug_logging;
debuginfo_level = rust.debuginfo_level;
debuginfo_level_rustc = rust.debuginfo_level_rustc;
debuginfo_level_std = rust.debuginfo_level_std;
debuginfo_level_tools = rust.debuginfo_level_tools;
debuginfo_level_tests = rust.debuginfo_level_tests;
config.rust_split_debuginfo = rust
.split_debuginfo
.as_deref()
.map(SplitDebuginfo::from_str)
.map(|v| v.expect("invalid value for rust.split_debuginfo"))
.unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple));
optimize = rust.optimize;
ignore_git = rust.ignore_git;
config.rust_new_symbol_mangling = rust.new_symbol_mangling;
set(&mut config.rust_optimize_tests, rust.optimize_tests);
set(&mut config.codegen_tests, rust.codegen_tests);
set(&mut config.rust_rpath, rust.rpath);
set(&mut config.jemalloc, rust.jemalloc);
set(&mut config.test_compare_mode, rust.test_compare_mode);
set(&mut config.backtrace, rust.backtrace);
set(&mut config.channel, rust.channel);
config.description = rust.description;
set(&mut config.rust_dist_src, rust.dist_src);
set(&mut config.verbose_tests, rust.verbose_tests);
// in the case "false" is set explicitly, do not overwrite the command line args
if let Some(true) = rust.incremental {
config.incremental = true;
}
set(&mut config.use_lld, rust.use_lld);
set(&mut config.lld_enabled, rust.lld);
set(&mut config.llvm_tools_enabled, rust.llvm_tools);
config.rustc_parallel = rust.parallel_compiler.unwrap_or(false);
config.rustc_default_linker = rust.default_linker;
config.musl_root = rust.musl_root.map(PathBuf::from);
config.save_toolstates = rust.save_toolstates.map(PathBuf::from);
set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings));
set(&mut config.backtrace_on_ice, rust.backtrace_on_ice);
set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir);
config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit;
set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo);
set(&mut config.control_flow_guard, rust.control_flow_guard);
config.llvm_libunwind_default = rust
.llvm_libunwind
.map(|v| v.parse().expect("failed to parse rust.llvm-libunwind"));

if let Some(ref backends) = rust.codegen_backends {
config.rust_codegen_backends =
backends.iter().map(|s| INTERNER.intern_str(s)).collect();
}

config.rust_codegen_units = rust.codegen_units.map(threads_from_config);
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc);

config.rust_lto = rust
.lto
.as_deref()
.map(|value| RustcLto::from_str(value).unwrap())
.unwrap_or_default();
} else {
config.rust_profile_use = flags.rust_profile_use;
config.rust_profile_generate = flags.rust_profile_generate;
config.llvm_from_ci =
config.channel == "dev" && crate::native::is_ci_llvm_available(&config, false);
}

if let Some(t) = toml.target {
Expand Down
24 changes: 24 additions & 0 deletions src/bootstrap/config/tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
use super::{Config, TomlConfig};
use std::path::Path;

fn toml(config: &str) -> impl '_ + Fn(&Path) -> TomlConfig {
|&_| toml::from_str(config).unwrap()
}

fn parse(config: &str) -> Config {
Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], toml(config))
}

#[test]
fn download_ci_llvm() {
let parse_llvm = |s| parse(s).llvm_from_ci;
let if_available = parse_llvm("llvm.download-ci-llvm = \"if-available\"");

assert!(parse_llvm("llvm.download-ci-llvm = true"));
assert!(!parse_llvm("llvm.download-ci-llvm = false"));
assert_eq!(parse_llvm(""), if_available);
assert_eq!(parse_llvm("rust.channel = \"dev\""), if_available);
assert!(!parse_llvm("rust.channel = \"stable\""));
}

// FIXME: add test for detecting `src` and `out`
4 changes: 4 additions & 0 deletions src/bootstrap/defaults/config.user.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,7 @@ test-stage = 2
doc-stage = 2
# When compiling from source, you usually want all tools.
extended = true

[llvm]
# Most users installing from source want to build all parts of the project from source, not just rustc itself.
download-ci-llvm = false
4 changes: 2 additions & 2 deletions src/bootstrap/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1639,10 +1639,10 @@ fn chmod(_path: &Path, _perms: u32) {}
/// If the test is running and code is an error code, it will cause a panic.
fn detail_exit(code: i32) -> ! {
// if in test and code is an error code, panic with status code provided
if cfg!(test) && code != 0 {
if cfg!(test) {
panic!("status code: {}", code);
} else {
//otherwise,exit with provided status code
// otherwise,exit with provided status code
std::process::exit(code);
}
}
Expand Down

0 comments on commit 1ce18d2

Please sign in to comment.