[7.1.0] Introduce --local_resources flag

src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java

@@ -16,6 +16,7 @@
 
 import com.google.devtools.build.lib.util.CpuResourceConverter;
 import com.google.devtools.build.lib.util.RegexFilter;
+import com.google.devtools.build.lib.util.ResourceConverter;
 import com.google.devtools.common.options.Option;
 import com.google.devtools.common.options.OptionDocumentationCategory;
 import com.google.devtools.common.options.OptionEffectTag;
@@ -106,24 +107,29 @@ public class AnalysisOptions extends OptionsBase {
 
  @Option(
  name = "experimental_skyframe_cpu_heavy_skykeys_thread_pool_size",
- defaultValue = "HOST_CPUS",
+ defaultValue = ResourceConverter.HOST_CPUS_KEYWORD,
  documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
  metadataTags = OptionMetadataTag.EXPERIMENTAL,
  effectTags = {
  OptionEffectTag.LOADING_AND_ANALYSIS,
  OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION
  },
  help =
- "If set to a positive value (e.g. \"HOST_CPUS*1.5\"), Skyframe will run the"
- + " loading/analysis phase with 2 separate thread pools: 1 with <value> threads"
- + " (ideally close to HOST_CPUS) reserved for CPU-heavy SkyKeys, and 1 \"standard\""
+ "If set to a positive value (e.g. \""
+ + ResourceConverter.HOST_CPUS_KEYWORD
+ + "*1.5\"),"
+ + " Skyframe will run the loading/analysis phase with 2 separate thread pools:"
+ + " 1 with <value> threads (ideally close to "
+ + ResourceConverter.HOST_CPUS_KEYWORD
+ + ")"
+ + " reserved for CPU-heavy SkyKeys, and 1 \"standard\""
  + " thread pool (whose size is controlled by --loading_phase_threads) for the rest.",
  converter = CpuResourceConverter.class)
  public int cpuHeavySkyKeysThreadPoolSize;
 
  @Option(
  name = "experimental_oom_sensitive_skyfunctions_semaphore_size",
- defaultValue = "HOST_CPUS",
+ defaultValue = ResourceConverter.HOST_CPUS_KEYWORD,
  documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
  metadataTags = OptionMetadataTag.EXPERIMENTAL,
  effectTags = {
@@ -133,7 +139,9 @@ public class AnalysisOptions extends OptionsBase {
  help =
  "Sets the size of the semaphore used to prevent SkyFunctions with large peak memory"
  + " requirement from OOM-ing blaze. A value of 0 indicates that no semaphore should"
- + " be used. Example value: \"HOST_CPUS*0.5\".",
+ + " be used. Example value: \""
+ + ResourceConverter.HOST_CPUS_KEYWORD
+ + "*0.5\".",
  converter = CpuResourceConverter.class)
  public int oomSensitiveSkyFunctionsSemaphoreSize;
 }

src/main/java/com/google/devtools/build/lib/analysis/BUILD

@@ -473,6 +473,7 @@ java_library(
  deps = [
  "//src/main/java/com/google/devtools/build/lib/util",
  "//src/main/java/com/google/devtools/build/lib/util:cpu_resource_converter",
+ "//src/main/java/com/google/devtools/build/lib/util:resource_converter",
  "//src/main/java/com/google/devtools/common/options",
  ],
 )

src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java

@@ -16,7 +16,6 @@
 import com.github.benmanes.caffeine.cache.CaffeineSpec;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.flogger.GoogleLogger;
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.util.OptionsUtils;
 import com.google.devtools.build.lib.util.ResourceConverter;
 import com.google.devtools.build.lib.vfs.PathFragment;
@@ -399,13 +398,9 @@ public String getSymlinkPrefix(String productName) {
  public int skymeldAnalysisOverlapPercentage;
 
  /** Converter for filesystem value checker threads. */
- public static class ThreadConverter extends ResourceConverter {
+ public static class ThreadConverter extends ResourceConverter.IntegerConverter {
  public ThreadConverter() {
- super(
- /* autoSupplier= */ () ->
- (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
- /* minValue= */ 1,
- /* maxValue= */ Integer.MAX_VALUE);
+ super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 1, /* maxValue= */ Integer.MAX_VALUE);
  }
  }
 
@@ -448,27 +443,24 @@ public ThreadConverter() {
  * Converter for jobs: Takes keyword ({@value #FLAG_SYNTAX}). Values must be between 1 and
  * MAX_JOBS.
  */
- public static class JobsConverter extends ResourceConverter {
+ public static class JobsConverter extends ResourceConverter.IntegerConverter {
  public JobsConverter() {
- super(
- () -> (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
- 1,
- MAX_JOBS);
+ super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 1, /* maxValue= */ MAX_JOBS);
  }
 
  @Override
- public int checkAndLimit(int value) throws OptionsParsingException {
- if (value < minValue) {
+ public Integer checkAndLimit(Integer value) throws OptionsParsingException {
+ if (value.doubleValue() < minValue) {
  throw new OptionsParsingException(
  String.format("Value '(%d)' must be at least %d.", value, minValue));
  }
- if (value > maxValue) {
+ if (value.doubleValue() > maxValue) {
  logger.atWarning().log(
  "Flag remoteWorker \"jobs\" ('%d') was set too high. "
  + "This is a result of passing large values to --local_resources or --jobs. "
  + "Using '%d' jobs",
  value, maxValue);
- value = maxValue;
+ return maxValue;
  }
  return value;
  }

src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java

@@ -24,6 +24,7 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import com.google.common.eventbus.Subscribe;
 import com.google.common.flogger.GoogleLogger;
 import com.google.devtools.build.lib.actions.Action;
@@ -115,7 +116,6 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Stream;
 import javax.annotation.Nullable;
 
 /**
@@ -996,11 +996,16 @@ public static void configureResourceManager(ResourceManager resourceMgr, BuildRe
  ImmutableMap<String, Double> cpuRam =
  ImmutableMap.of(
  ResourceSet.CPU,
+ // Replace with 1.0 * ResourceConverter.HOST_CPUS.get() after flag deprecation
  options.localCpuResources,
  ResourceSet.MEMORY,
+ // Replace with 0.67 * ResourceConverter.HOST_RAM.get() after flag deprecation
  options.localRamResources);
  ImmutableMap<String, Double> resources =
- Stream.concat(options.localExtraResources.stream(), cpuRam.entrySet().stream())
+ Streams.concat(
+ options.localExtraResources.stream(),
+ cpuRam.entrySet().stream(),
+ options.localResources.stream())
  .collect(
  ImmutableMap.toImmutableMap(
  Map.Entry::getKey, Map.Entry::getValue, (v1, v2) -> v2));

src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java

@@ -293,35 +293,54 @@ public boolean shouldMaterializeParamFiles() {
 
  @Option(
  name = "local_cpu_resources",
- defaultValue = "HOST_CPUS",
+ defaultValue = ResourceConverter.HOST_CPUS_KEYWORD,
+ deprecationWarning =
+ "--local_cpu_resources is deprecated, please use --local_resources=cpu= instead.",
  documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
  effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS},
  help =
  "Explicitly set the total number of local CPU cores available to Bazel to spend on build"
- + " actions executed locally. Takes an integer, or \"HOST_CPUS\", optionally followed"
- + " by [-|*]<float> (eg. HOST_CPUS*.5 to use half the available CPU cores).By"
- + " default, (\"HOST_CPUS\"), Bazel will query system configuration to estimate"
- + " the number of CPU cores available.",
+ + " actions executed locally. Takes an integer, or \""
+ + ResourceConverter.HOST_CPUS_KEYWORD
+ + "\", optionally followed"
+ + " by [-|*]<float> (eg. "
+ + ResourceConverter.HOST_CPUS_KEYWORD
+ + "*.5"
+ + " to use half the available CPU cores). By default, (\""
+ + ResourceConverter.HOST_CPUS_KEYWORD
+ + "\"), Bazel will query system"
+ + " configuration to estimate the number of CPU cores available.",
  converter = CpuResourceConverter.class)
  public double localCpuResources;
 
  @Option(
  name = "local_ram_resources",
- defaultValue = "HOST_RAM*.67",
+ defaultValue = ResourceConverter.HOST_RAM_KEYWORD + "*.67",
+ deprecationWarning =
+ "--local_ram_resources is deprecated, please use --local_resources=memory= instead.",
  documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
  effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS},
  help =
  "Explicitly set the total amount of local host RAM (in MB) available to Bazel to spend on"
- + " build actions executed locally. Takes an integer, or \"HOST_RAM\", optionally"
- + " followed by [-|*]<float> (eg. HOST_RAM*.5 to use half the available RAM). By"
- + " default, (\"HOST_RAM*.67\"), Bazel will query system configuration to estimate"
- + " the amount of RAM available and will use 67% of it.",
+ + " build actions executed locally. Takes an integer, or \""
+ + ResourceConverter.HOST_RAM_KEYWORD
+ + "\", optionally followed by [-|*]<float>"
+ + " (eg. "
+ + ResourceConverter.HOST_RAM_KEYWORD
+ + "*.5 to use half the available"
+ + " RAM). By default, (\""
+ + ResourceConverter.HOST_RAM_KEYWORD
+ + "*.67\"),"
+ + " Bazel will query system configuration to estimate the amount of RAM available"
+ + " and will use 67% of it.",
  converter = RamResourceConverter.class)
  public double localRamResources;
 
  @Option(
  name = "local_extra_resources",
  defaultValue = "null",
+ deprecationWarning =
+ "--local_extra_resources is deprecated, please use --local_resources instead.",
  documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
  effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS},
  allowMultiple = true,
@@ -336,6 +355,31 @@ public boolean shouldMaterializeParamFiles() {
  converter = Converters.StringToDoubleAssignmentConverter.class)
  public List<Map.Entry<String, Double>> localExtraResources;
 
+ @Option(
+ name = "local_resources",
+ defaultValue = "null",
+ documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
+ effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS},
+ allowMultiple = true,
+ help =
+ "Set the number of resources available to Bazel. "
+ + "Takes in an assignment to a float or "
+ + ResourceConverter.HOST_RAM_KEYWORD
+ + "/"
+ + ResourceConverter.HOST_CPUS_KEYWORD
+ + ", optionally "
+ + "followed by [-|*]<float> (eg. memory="
+ + ResourceConverter.HOST_RAM_KEYWORD
+ + "*.5 to use half the available RAM). "
+ + "Can be used multiple times to specify multiple "
+ + "types of resources. Bazel will limit concurrently running actions "
+ + "based on the available resources and the resources required. "
+ + "Tests can declare the amount of resources they need "
+ + "by using a tag of the \"resources:<resource name>:<amount>\" format. "
+ + "Overrides resources specified by --local_{cpu|ram|extra}_resources.",
+ converter = ResourceConverter.AssignmentConverter.class)
+ public List<Map.Entry<String, Double>> localResources;
+
  @Option(
  name = "local_test_jobs",
  defaultValue = "auto",
@@ -574,9 +618,9 @@ public String getTypeDescription() {
  }
 
  /** Converter for --local_test_jobs, which takes {@value FLAG_SYNTAX} */
- public static class LocalTestJobsConverter extends ResourceConverter {
+ public static class LocalTestJobsConverter extends ResourceConverter.IntegerConverter {
  public LocalTestJobsConverter() throws OptionsParsingException {
- super(/* autoSupplier= */ () -> 0, /* minValue= */ 0, /* maxValue= */ Integer.MAX_VALUE);
+ super(/* auto= */ () -> 0, /* minValue= */ 0, /* maxValue= */ Integer.MAX_VALUE);
  }
  }
 

src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningOptions.java

@@ -13,7 +13,6 @@
 // limitations under the License.
 package com.google.devtools.build.lib.includescanning;
 
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.util.ResourceConverter;
 import com.google.devtools.common.options.Option;
 import com.google.devtools.common.options.OptionDocumentationCategory;
@@ -30,13 +29,9 @@ public class IncludeScanningOptions extends OptionsBase {
  * Converter for scanning parallelism threads: Takes {@value #FLAG_SYNTAX} 0 disables scanning
  * parallelism.
  */
- public static class ParallelismConverter extends ResourceConverter {
+ public static class ParallelismConverter extends ResourceConverter.IntegerConverter {
  public ParallelismConverter() throws OptionsParsingException {
- super(
- /* autoSupplier= */ () ->
- (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
- /* minValue= */ 0,
- /* maxValue= */ Integer.MAX_VALUE);
+ super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 0, /* maxValue= */ Integer.MAX_VALUE);
  }
  }
 

src/main/java/com/google/devtools/build/lib/pkgcache/PackageOptions.java

@@ -19,7 +19,6 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
 import com.google.devtools.build.lib.cmdline.PackageIdentifier;
 import com.google.devtools.build.lib.packages.RuleVisibility;
@@ -59,13 +58,9 @@ public String getTypeDescription() {
  }
 
  /** Converter for globbing threads. */
- public static class ParallelismConverter extends ResourceConverter {
+ public static class ParallelismConverter extends ResourceConverter.IntegerConverter {
  public ParallelismConverter() throws OptionsParsingException {
- super(
- /* autoSupplier= */ () ->
- (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
- /* minValue= */ 1,
- /* maxValue= */ Integer.MAX_VALUE);
+ super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 1, /* maxValue= */ Integer.MAX_VALUE);
  }
  }
 

src/main/java/com/google/devtools/build/lib/runtime/LoadingPhaseThreadsOption.java

@@ -14,7 +14,6 @@
 package com.google.devtools.build.lib.runtime;
 
 import com.google.common.flogger.GoogleLogger;
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.util.ResourceConverter;
 import com.google.devtools.build.lib.util.TestType;
 import com.google.devtools.common.options.Option;
@@ -45,18 +44,19 @@ public class LoadingPhaseThreadsOption extends OptionsBase {
  /**
  * A converter for loading phase thread count. Takes {@value FLAG_SYNTAX}. Caps at 20 for tests.
  */
- public static final class LoadingPhaseThreadCountConverter extends ResourceConverter {
+ public static final class LoadingPhaseThreadCountConverter
+ extends ResourceConverter.IntegerConverter {
 
  public LoadingPhaseThreadCountConverter() {
  // TODO(jmmv): Using the number of cores has proven to yield reasonable analysis times on
  // Mac Pros and MacBook Pros but we should probably do better than this. (We haven't made
  // any guarantees that "auto" means number of cores precisely to leave us room to tune this
  // further in the future.)
- super(() -> (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()));
+ super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 1, /* maxValue= */ Integer.MAX_VALUE);
  }
 
  @Override
- public int checkAndLimit(int value) throws OptionsParsingException {
+ public Integer checkAndLimit(Integer value) throws OptionsParsingException {
  // Cap thread count while running tests. Test cases are typically small and large thread
  // pools vying for a relatively small number of CPU cores may induce non-optimal
  // performance.

src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java

@@ -18,7 +18,6 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.util.OptionsUtils;
 import com.google.devtools.build.lib.util.RamResourceConverter;
 import com.google.devtools.build.lib.util.ResourceConverter;
@@ -371,12 +370,9 @@ public ImmutableSet<Path> getInaccessiblePaths(FileSystem fs) {
  public int memoryLimitMb;
 
  /** Converter for the number of threads used for asynchronous tree deletion. */
- public static final class AsyncTreeDeletesConverter extends ResourceConverter {
+ public static final class AsyncTreeDeletesConverter extends ResourceConverter.IntegerConverter {
  public AsyncTreeDeletesConverter() {
- super(
- () -> (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()),
- 0,
- Integer.MAX_VALUE);
+ super(/* auto= */ HOST_CPUS_SUPPLIER, /* minValue= */ 0, /* maxValue= */ Integer.MAX_VALUE);
  }
  }
 }

src/main/java/com/google/devtools/build/lib/util/BUILD

@@ -127,6 +127,7 @@ java_library(
  deps = [
  "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
  "//src/main/java/com/google/devtools/common/options",
+ "//third_party:error_prone_annotations",
  "//third_party:guava",
  "//third_party:jsr305",
  ],
@@ -139,7 +140,6 @@ java_library(
  ],
  deps = [
  ":resource_converter",
- "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
  "//third_party:guava",
  ],
 )
@@ -151,7 +151,6 @@ java_library(
  ],
  deps = [
  ":resource_converter",
- "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
  "//third_party:guava",
  ],
 )