Skip to content

Commit

Permalink
UrlRewriter should be able to load credentials from .netrc (bazelbuil…
Browse files Browse the repository at this point in the history
…d#14834)

Addresses bazelbuild#13111

Closes bazelbuild#14066.

PiperOrigin-RevId: 424854105
(cherry picked from commit 1e53b1f)

Co-authored-by: Denys Kurylenko <[email protected]>
  • Loading branch information
brentleyjones and Denys Kurylenko authored Feb 16, 2022
1 parent af34c45 commit 6990c02
Show file tree
Hide file tree
Showing 6 changed files with 344 additions and 69 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,10 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
try {
UrlRewriter rewriter =
UrlRewriter.getDownloaderUrlRewriter(
repoOptions == null ? null : repoOptions.downloaderConfig, env.getReporter());
repoOptions == null ? null : repoOptions.downloaderConfig,
env.getReporter(),
env.getClientEnv(),
env.getRuntime().getFileSystem());
downloadManager.setUrlRewriter(rewriter);
} catch (UrlRewriterParseException e) {
// It's important that the build stops ASAP, because this config file may be required for
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,20 @@ java_library(
srcs = glob(["*.java"]),
deps = [
"//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info",
"//src/main/java/com/google/devtools/build/lib/authandtls",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/cache:events",
"//src/main/java/com/google/devtools/build/lib/buildeventstream",
"//src/main/java/com/google/devtools/build/lib/clock",
"//src/main/java/com/google/devtools/build/lib/concurrent",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/util",
"//src/main/java/com/google/devtools/build/lib/util:os",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/net/starlark/java/syntax",
"//third_party:auth",
"//third_party:auto_value",
"//third_party:guava",
"//third_party:jsr305",
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package com.google.devtools.build.lib.bazel.repository.downloader;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableList.toImmutableList;

import com.google.common.base.MoreObjects;
import com.google.common.base.Optional;
Expand Down Expand Up @@ -108,12 +109,14 @@ public Path download(
throw new InterruptedException();
}

List<URL> rewrittenUrls = originalUrls;
ImmutableList<URL> rewrittenUrls = ImmutableList.copyOf(originalUrls);
Map<URI, Map<String, String>> rewrittenAuthHeaders = authHeaders;

if (rewriter != null) {
rewrittenUrls = rewriter.amend(originalUrls);
rewrittenAuthHeaders = rewriter.updateAuthHeaders(rewrittenUrls, authHeaders);
ImmutableList<UrlRewriter.RewrittenURL> rewrittenUrlMappings = rewriter.amend(originalUrls);
rewrittenUrls =
rewrittenUrlMappings.stream().map(url -> url.url()).collect(toImmutableList());
rewrittenAuthHeaders = rewriter.updateAuthHeaders(rewrittenUrlMappings, authHeaders);
}

URL mainUrl; // The "main" URL for this request
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,23 @@
import static com.google.common.collect.ImmutableList.toImmutableList;
import static java.nio.charset.StandardCharsets.ISO_8859_1;

import com.google.auth.Credentials;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ascii;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.authandtls.Netrc;
import com.google.devtools.build.lib.authandtls.NetrcCredentials;
import com.google.devtools.build.lib.authandtls.NetrcParser;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
Expand All @@ -38,14 +46,17 @@
import java.nio.file.Paths;
import java.util.Base64;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
import net.starlark.java.syntax.Location;

/**
* Helper class for taking URLs and converting them according to an optional config specified by
Expand All @@ -59,13 +70,19 @@ public class UrlRewriter {
private static final ImmutableSet<String> REWRITABLE_SCHEMES = ImmutableSet.of("http", "https");

private final UrlRewriterConfig config;
private final Function<URL, List<URL>> rewriter;
private final Function<URL, List<RewrittenURL>> rewriter;
@Nullable private final Credentials netrcCreds;

@VisibleForTesting
UrlRewriter(Consumer<String> log, String filePathForErrorReporting, Reader reader)
UrlRewriter(
Consumer<String> log,
String filePathForErrorReporting,
Reader reader,
@Nullable Credentials netrcCreds)
throws UrlRewriterParseException {
Preconditions.checkNotNull(reader, "UrlRewriterConfig source must be set");
this.config = new UrlRewriterConfig(filePathForErrorReporting, reader);
this.netrcCreds = netrcCreds;

this.rewriter = this::rewrite;
}
Expand All @@ -75,89 +92,124 @@ public class UrlRewriter {
*
* @param configPath Path to the config file to use. May be null.
* @param reporter Used for logging when URLs are rewritten.
* @param clientEnv a map of the current Bazel command's environment
* @param fileSystem the Blaze file system
*/
public static UrlRewriter getDownloaderUrlRewriter(String configPath, Reporter reporter)
public static UrlRewriter getDownloaderUrlRewriter(
String configPath,
Reporter reporter,
ImmutableMap<String, String> clientEnv,
FileSystem fileSystem)
throws UrlRewriterParseException {
Consumer<String> log = str -> reporter.handle(Event.info(str));

// "empty" UrlRewriter shouldn't alter auth headers
if (Strings.isNullOrEmpty(configPath)) {
return new UrlRewriter(log, "", new StringReader(""));
return new UrlRewriter(log, "", new StringReader(""), null);
}

Credentials creds = null;
try {
creds = newCredentialsFromNetrc(clientEnv, fileSystem);
} catch (UrlRewriterParseException e) {
// If the credentials extraction failed, we're letting bazel try without credentials.
}
try (BufferedReader reader = Files.newBufferedReader(Paths.get(configPath))) {
return new UrlRewriter(log, configPath, reader);
return new UrlRewriter(log, configPath, reader, creds);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}

/**
* Rewrites {@code urls} using the configuration provided to {@link
* #getDownloaderUrlRewriter(String, Reporter)}. The returned list of URLs may be empty if the
* configuration used blocks all the input URLs.
* #getDownloaderUrlRewriter(String, Reporter, ImmutableMap, FileSystem)}. The returned list of
* URLs may be empty if the configuration used blocks all the input URLs.
*
* @param urls The input list of {@link URL}s. May be empty.
* @return The amended lists of URLs.
*/
public List<URL> amend(List<URL> urls) {
public ImmutableList<RewrittenURL> amend(List<URL> urls) {
Objects.requireNonNull(urls, "URLS to check must be set but may be empty");

ImmutableList<URL> rewritten =
urls.stream().map(rewriter).flatMap(Collection::stream).collect(toImmutableList());

return rewritten;
return urls.stream().map(rewriter).flatMap(Collection::stream).collect(toImmutableList());
}

/**
* Updates {@code authHeaders} using the userInfo available in the provided {@code urls}.
* Updates {@code authHeaders} using the userInfo available in the provided {@code urls}. Note
* that if the same url is present in both {@code authHeaders} and <b>download config</b> then it
* will be overridden with the value from <b>download config</b>.
*
* @param urls The input list of {@link URL}s. May be empty.
* @param authHeaders A map of the URLs and their corresponding auth tokens.
* @return A map of the updated authentication headers.
*/
public Map<URI, Map<String, String>> updateAuthHeaders(
List<URL> urls, Map<URI, Map<String, String>> authHeaders) {
ImmutableMap.Builder<URI, Map<String, String>> authHeadersBuilder =
ImmutableMap.<URI, Map<String, String>>builder().putAll(authHeaders);
List<RewrittenURL> urls, Map<URI, Map<String, String>> authHeaders) {
Map<URI, Map<String, String>> updatedAuthHeaders = new HashMap<>(authHeaders);

for (URL url : urls) {
String userInfo = url.getUserInfo();
for (RewrittenURL url : urls) {
// if URL was not re-written by UrlRewriter in first place, we should not attach auth headers
// to it
if (!url.rewritten()) {
continue;
}

String userInfo = url.url().getUserInfo();
if (userInfo != null) {
try {
String token =
"Basic " + Base64.getEncoder().encodeToString(userInfo.getBytes(ISO_8859_1));
authHeadersBuilder.put(url.toURI(), ImmutableMap.of("Authorization", token));
updatedAuthHeaders.put(url.url().toURI(), ImmutableMap.of("Authorization", token));
} catch (URISyntaxException e) {
// If the credentials extraction failed, we're letting bazel try without credentials.
}
} else if (this.netrcCreds != null) {
try {
Map<String, List<String>> urlAuthHeaders =
this.netrcCreds.getRequestMetadata(url.url().toURI());
if (urlAuthHeaders == null || urlAuthHeaders.isEmpty()) {
continue;
}
// there could be multiple Auth headers, take the first one
Map.Entry<String, List<String>> firstAuthHeader =
urlAuthHeaders.entrySet().stream().findFirst().get();
if (firstAuthHeader.getValue() != null && !firstAuthHeader.getValue().isEmpty()) {
updatedAuthHeaders.put(
url.url().toURI(),
ImmutableMap.of(firstAuthHeader.getKey(), firstAuthHeader.getValue().get(0)));
}
} catch (URISyntaxException | IOException e) {
// If the credentials extraction failed, we're letting bazel try without credentials.
}
}
}

return authHeadersBuilder.build();
return ImmutableMap.copyOf(updatedAuthHeaders);
}

private ImmutableList<URL> rewrite(URL url) {
private ImmutableList<RewrittenURL> rewrite(URL url) {
Preconditions.checkNotNull(url);

// Cowardly refuse to rewrite non-HTTP(S) urls
if (REWRITABLE_SCHEMES.stream()
.noneMatch(scheme -> Ascii.equalsIgnoreCase(scheme, url.getProtocol()))) {
return ImmutableList.of(url);
return ImmutableList.of(RewrittenURL.create(url, false));
}

List<URL> rewrittenUrls = applyRewriteRules(url);
ImmutableList<RewrittenURL> rewrittenUrls = applyRewriteRules(url);

ImmutableList.Builder<URL> toReturn = ImmutableList.builder();
ImmutableList.Builder<RewrittenURL> toReturn = ImmutableList.builder();
// Now iterate over the URLs
for (URL consider : rewrittenUrls) {
for (RewrittenURL consider : rewrittenUrls) {
// If there's an allow entry, add it to the set to return and continue
if (isAllowMatched(consider)) {
if (isAllowMatched(consider.url())) {
toReturn.add(consider);
continue;
}

// If there's no block that matches the domain, add it to the set to return and continue
if (!isBlockMatched(consider)) {
if (!isBlockMatched(consider.url())) {
toReturn.add(consider);
}
}
Expand Down Expand Up @@ -192,7 +244,7 @@ private static boolean isMatchingHostName(URL url, String host) {
return host.equals(url.getHost()) || url.getHost().endsWith("." + host);
}

private ImmutableList<URL> applyRewriteRules(URL url) {
private ImmutableList<RewrittenURL> applyRewriteRules(URL url) {
String withoutScheme = url.toString().substring(url.getProtocol().length() + 3);

ImmutableSet.Builder<String> rewrittenUrls = ImmutableSet.builder();
Expand All @@ -210,11 +262,12 @@ private ImmutableList<URL> applyRewriteRules(URL url) {
}

if (!matchMade) {
return ImmutableList.of(url);
return ImmutableList.of(RewrittenURL.create(url, false));
}

return rewrittenUrls.build().stream()
.map(urlString -> prefixWithProtocol(urlString, url.getProtocol()))
.map(plainUrl -> RewrittenURL.create(plainUrl, true))
.collect(toImmutableList());
}

Expand All @@ -232,8 +285,64 @@ private static URL prefixWithProtocol(String url, String protocol) {
}
}

/**
* Create a new {@link Credentials} object by parsing the .netrc file with following order to
* search it:
*
* <ol>
* <li>If environment variable $NETRC exists, use it as the path to the .netrc file
* <li>Fallback to $HOME/.netrc or $USERPROFILE/.netrc
* </ol>
*
* @return the {@link Credentials} object or {@code null} if there is no .netrc file.
* @throws UrlRewriterParseException in case the credentials can't be constructed.
*/
// TODO : consider re-using RemoteModule.newCredentialsFromNetrc
@VisibleForTesting
static Credentials newCredentialsFromNetrc(Map<String, String> clientEnv, FileSystem fileSystem)
throws UrlRewriterParseException {
final Optional<String> homeDir;
if (OS.getCurrent() == OS.WINDOWS) {
homeDir = Optional.ofNullable(clientEnv.get("USERPROFILE"));
} else {
homeDir = Optional.ofNullable(clientEnv.get("HOME"));
}
String netrcFileString =
Optional.ofNullable(clientEnv.get("NETRC"))
.orElseGet(() -> homeDir.map(home -> home + "/.netrc").orElse(null));
if (netrcFileString == null) {
return null;
}
Location location = Location.fromFileLineColumn(netrcFileString, 0, 0);

Path netrcFile = fileSystem.getPath(netrcFileString);
if (netrcFile.exists()) {
try {
Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
return new NetrcCredentials(netrc);
} catch (IOException e) {
throw new UrlRewriterParseException(
"Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), location);
}
} else {
return null;
}
}

@Nullable
public String getAllBlockedMessage() {
return config.getAllBlockedMessage();
}

/** Holds the URL along with meta-info, such as whether URL was re-written or not. */
@AutoValue
public abstract static class RewrittenURL {
static RewrittenURL create(URL url, boolean rewritten) {
return new AutoValue_UrlRewriter_RewrittenURL(url, rewritten);
}

abstract URL url();

abstract boolean rewritten();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@ java_library(
name = "DownloaderTestSuite_lib",
srcs = glob(["*.java"]),
deps = [
"//src/main/java/com/google/devtools/build/lib/authandtls",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
"//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/util",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
"//src/main/java/net/starlark/java/syntax",
"//src/test/java/com/google/devtools/build/lib/testutil",
"//third_party:auth",
"//third_party:guava",
"//third_party:jsr305",
"//third_party:junit4",
Expand Down
Loading

0 comments on commit 6990c02

Please sign in to comment.