diff --git a/src/main/java/com/google/devtools/build/lib/remote/ToplevelArtifactsDownloader.java b/src/main/java/com/google/devtools/build/lib/remote/ToplevelArtifactsDownloader.java
index b4fd000090286b..b8b11d02822174 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/ToplevelArtifactsDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/ToplevelArtifactsDownloader.java
@@ -184,9 +184,9 @@ private boolean shouldDownloadToplevelOutputs(ConfiguredTargetKey configuredTarg
           if (configuredTarget instanceof RuleConfiguredTarget) {
             var ruleConfiguredTarget = (RuleConfiguredTarget) configuredTarget;
             var isTestRule = isTestRuleName(ruleConfiguredTarget.getRuleClassString());
-            return !isTestRule;
+            return !isTestRule && downloadToplevel;
           }
-          return true;
+          return downloadToplevel;
         } catch (InterruptedException ignored) {
           Thread.currentThread().interrupt();
           return false;