Skip to content

Commit

Permalink
[credentialhelper] Ignore failing to provide request (#17745)
Browse files Browse the repository at this point in the history 
This fixes a race condition where writing to stdin of the subprocess fails because the subprocess terminated quickly (e.g., because it did not need the request to provide credentials, or some other error occurred in the helper before it reads from stdin).

```
Starting local Bazel server and connecting to it...
INFO: Writing tracer profile to '/private/var/tmp/_bazel_yannic/bb6c3b02ccc98aaca966df445b5024eb/command.profile.gz'
INFO: Invocation ID: 86e227bc-187e-4d28-8f04-40b1ce9a36a4
ERROR: java.io.IOException: Stream closed
	at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source)
	at java.base/java.io.OutputStream.write(Unknown Source)
	at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source)
	at java.base/java.io.BufferedOutputStream.flush(Unknown Source)
	at java.base/java.io.FilterOutputStream.close(Unknown Source)
	at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source)
	at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source)
	at java.base/java.io.OutputStreamWriter.close(Unknown Source)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413)
	at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394)
	at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108)
	at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80)
	at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112)
	at com.google.auth.Credentials$1.run(Credentials.java:98)
	at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771)
	at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754)
	at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157)
	at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108)
	at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112)
	at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52)
	at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110)
	at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85)
	at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233)
	at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176)
	at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85)
	at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68)
	at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79)
	at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247)
	at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507)
	at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468)
	at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224)
	at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176)
	at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64)
	at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83)
	at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244)
	at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125)
	at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81)
	at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209)
	at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542)
	at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379)
	at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233)
	at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550)
	at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614)
	at io.grpc.Context$1.run(Context.java:566)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
	at java.base/java.lang.Thread.run(Unknown Source)

ERROR: Failed to query remote execution capabilities: Stream closed
java.io.IOException: Stream closed
	at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source)
	at java.base/java.io.OutputStream.write(Unknown Source)
	at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source)
	at java.base/java.io.BufferedOutputStream.flush(Unknown Source)
	at java.base/java.io.FilterOutputStream.close(Unknown Source)
	at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source)
	at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source)
	at java.base/java.io.OutputStreamWriter.close(Unknown Source)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413)
	at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394)
	at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108)
	at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80)
	at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112)
	at com.google.auth.Credentials$1.run(Credentials.java:98)
	at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771)
	at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754)
	at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157)
	at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108)
	at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112)
	at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52)
	at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110)
	at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85)
	at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233)
	at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176)
	at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85)
	at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68)
	at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79)
	at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247)
	at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507)
	at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468)
	at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224)
	at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176)
	at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64)
	at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83)
	at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244)
	at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125)
	at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81)
	at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209)
	at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542)
	at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379)
	at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233)
	at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550)
	at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614)
	at io.grpc.Context$1.run(Context.java:566)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
	at java.base/java.lang.Thread.run(Unknown Source)

INFO: Streaming build results to: https://opal.cluster.engflow.com/invocation/86e227bc-187e-4d28-8f04-40b1ce9a36a4
```

Closes #17666.

PiperOrigin-RevId: 514377908
Change-Id: I772a5e462146e3e9699b45090120634a057a5532

Co-authored-by: Yannic Bonenberger <[email protected]>
  • Loading branch information
@ShreeM01 @Yannic
ShreeM01 and Yannic authored Mar 13, 2023
1 parent 94575fd commit e362509
Showing 1 changed file with 8 additions and 0 deletions.
8 changes: 8 additions & 0 deletions ...main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,14 @@ public GetCredentialsResponse getCredentials(CredentialHelperEnvironment environ
Reader stderr = new InputStreamReader(process.getErrorStream(), UTF_8)) {
try (Writer stdin = new OutputStreamWriter(process.getOutputStream(), UTF_8)) {
GSON.toJson(GetCredentialsRequest.newBuilder().setUri(uri).build(), stdin);
} catch (IOException e) {
// This can happen if the helper prints a static set of credentials without reading from
// stdin (e.g., with a simple shell script running `echo "{...}"`). If the process is
// already finished even though we failed to write to its stdin, ignore the error and
// assume the process did not need the request payload.
if (!process.finished()) {
throw e;
}
}

try {
Expand Down

0 comments on commit e362509

Please sign in to comment.