From ba5b2a7c9448a3681a0d86d80670447e338a06dc Mon Sep 17 00:00:00 2001 From: Johannes Abt Date: Wed, 16 Jun 2021 23:47:38 -0700 Subject: [PATCH] when writing to local disk cache, open files later in order to avoid "too many open files" Re-use the existing "LazyFileOutputStream" in DiskAndRemoteCacheClient.java in order to avoid "Too many open files". Resolves https://github.com/bazelbuild/bazel/issues/13435 Closes #13574. PiperOrigin-RevId: 379892227 --- .../build/lib/remote/RemoteCache.java | 52 +------------- .../remote/common/LazyFileOutputStream.java | 69 +++++++++++++++++++ .../remote/disk/DiskAndRemoteCacheClient.java | 7 +- 3 files changed, 72 insertions(+), 56 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/remote/common/LazyFileOutputStream.java diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java index 150f61881a9223..2ac802907e6fbb 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java @@ -56,6 +56,7 @@ import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.DirectoryMetadata; import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.FileMetadata; import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.SymlinkMetadata; +import com.google.devtools.build.lib.remote.common.LazyFileOutputStream; import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException; import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; import com.google.devtools.build.lib.remote.common.RemoteActionFileArtifactValue; @@ -1099,57 +1100,6 @@ private static FailureDetail createFailureDetail(String message, Code detailedCo .build(); } - /** - * Creates an {@link OutputStream} that isn't actually opened until the first data is written. - * This is useful to only have as many open file descriptors as necessary at a time to avoid - * running into system limits. - */ - private static class LazyFileOutputStream extends OutputStream { - - private final Path path; - private OutputStream out; - - public LazyFileOutputStream(Path path) { - this.path = path; - } - - @Override - public void write(byte[] b) throws IOException { - ensureOpen(); - out.write(b); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - ensureOpen(); - out.write(b, off, len); - } - - @Override - public void write(int b) throws IOException { - ensureOpen(); - out.write(b); - } - - @Override - public void flush() throws IOException { - ensureOpen(); - out.flush(); - } - - @Override - public void close() throws IOException { - ensureOpen(); - out.close(); - } - - private void ensureOpen() throws IOException { - if (out == null) { - out = path.getOutputStream(); - } - } - } - /** In-memory representation of action result metadata. */ static class ActionResultMetadata { diff --git a/src/main/java/com/google/devtools/build/lib/remote/common/LazyFileOutputStream.java b/src/main/java/com/google/devtools/build/lib/remote/common/LazyFileOutputStream.java new file mode 100644 index 00000000000000..dbe002628ccf33 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/common/LazyFileOutputStream.java @@ -0,0 +1,69 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.remote.common; + +import com.google.devtools.build.lib.vfs.Path; +import java.io.IOException; +import java.io.OutputStream; + +/** + * Creates an {@link OutputStream} that isn't actually opened until the first data is written. This + * is useful to only have as many open file descriptors as necessary at a time to avoid running into + * system limits. + */ +public class LazyFileOutputStream extends OutputStream { + + private final Path path; + private OutputStream out; + + public LazyFileOutputStream(Path path) { + this.path = path; + } + + @Override + public void write(byte[] b) throws IOException { + ensureOpen(); + out.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + ensureOpen(); + out.write(b, off, len); + } + + @Override + public void write(int b) throws IOException { + ensureOpen(); + out.write(b); + } + + @Override + public void flush() throws IOException { + ensureOpen(); + out.flush(); + } + + @Override + public void close() throws IOException { + ensureOpen(); + out.close(); + } + + private void ensureOpen() throws IOException { + if (out == null) { + out = path.getOutputStream(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/remote/disk/DiskAndRemoteCacheClient.java b/src/main/java/com/google/devtools/build/lib/remote/disk/DiskAndRemoteCacheClient.java index b34098556a8c7b..97a973e12961a1 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/disk/DiskAndRemoteCacheClient.java +++ b/src/main/java/com/google/devtools/build/lib/remote/disk/DiskAndRemoteCacheClient.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.devtools.build.lib.remote.common.LazyFileOutputStream; import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; import com.google.devtools.build.lib.remote.common.RemoteCacheClient; import com.google.devtools.build.lib.remote.options.RemoteOptions; @@ -142,11 +143,7 @@ public ListenableFuture downloadBlob( Path tempPath = newTempPath(); final OutputStream tempOut; - try { - tempOut = tempPath.getOutputStream(); - } catch (IOException e) { - return Futures.immediateFailedFuture(e); - } + tempOut = new LazyFileOutputStream(tempPath); if (!options.incompatibleRemoteResultsIgnoreDisk || options.remoteAcceptCached) { ListenableFuture download =