diff --git a/CHANGELOG.md b/CHANGELOG.md index b829a11e6f08b..2aa892f5f4b70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,6 +128,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735)) - Exclude 'benchmarks' from codecov report ([#8805](https://github.com/opensearch-project/OpenSearch/pull/8805)) - Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807)) +- Allow test clusters to run with TLS ([#8900](https://github.com/opensearch-project/OpenSearch/pull/8900)) - Replace the deprecated IndexReader APIs with new storedFields() & termVectors() ([#7792](https://github.com/opensearch-project/OpenSearch/pull/7792)) - [Remote Store] Add support to restore only unassigned shards of an index ([#8792](https://github.com/opensearch-project/OpenSearch/pull/8792)) - Add safeguard limits for file cache during node level allocation ([#8208](https://github.com/opensearch-project/OpenSearch/pull/8208)) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/http/WaitForHttpResource.java b/buildSrc/src/main/java/org/opensearch/gradle/http/WaitForHttpResource.java index 6b2ee31a964ae..54c544a299b84 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/http/WaitForHttpResource.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/http/WaitForHttpResource.java @@ -83,6 +83,24 @@ public WaitForHttpResource(String protocol, String host, int numberOfNodes) thro this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow")); } + public WaitForHttpResource(String protocol, String host, String username, String password, int numberOfNodes) + throws MalformedURLException { + this( + new URL( + protocol + + "://" + + username + + ":" + + password + + "@" + + host + + "/_cluster/health?wait_for_nodes=>=" + + numberOfNodes + + "&wait_for_status=yellow" + ) + ); + } + public WaitForHttpResource(URL url) { this.url = url; } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java index ffb3360e3cc55..505f773f6d9da 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java @@ -81,7 +81,6 @@ public class OpenSearchCluster implements TestClusterConfiguration, Named { private final FileSystemOperations fileSystemOperations; private final ArchiveOperations archiveOperations; private int nodeIndex = 0; - private int zoneCount = 1; public OpenSearchCluster( @@ -100,7 +99,6 @@ public OpenSearchCluster( this.archiveOperations = archiveOperations; this.workingDirBase = workingDirBase; this.nodes = project.container(OpenSearchNode.class); - // Always add the first node String zone = hasZoneProperty() ? "zone-1" : ""; addNode(clusterName + "-0", zone); @@ -265,6 +263,11 @@ public void keystorePassword(String password) { nodes.all(each -> each.keystorePassword(password)); } + @Override + public void setSecure(boolean secure) { + nodes.all(each -> each.setSecure(secure)); + } + @Override public void cliSetup(String binTool, CharSequence... args) { nodes.all(each -> each.cliSetup(binTool, args)); @@ -367,6 +370,7 @@ private void commonNodeConfig() { } else { nodeNames = nodes.stream().map(OpenSearchNode::getName).map(this::safeName).collect(Collectors.joining(",")); } + OpenSearchNode firstNode = null; for (OpenSearchNode node : nodes) { // Can only configure master nodes if we have node names defined @@ -554,12 +558,25 @@ public OpenSearchNode singleNode() { private void addWaitForClusterHealth() { waitConditions.put("cluster health yellow", (node) -> { try { - WaitForHttpResource wait = new WaitForHttpResource("http", getFirstNode().getHttpSocketURI(), nodes.size()); - - List> credentials = getFirstNode().getCredentials(); - if (getFirstNode().getCredentials().isEmpty() == false) { - wait.setUsername(credentials.get(0).get("useradd")); - wait.setPassword(credentials.get(0).get("-p")); + WaitForHttpResource wait; + if (!getFirstNode().isSecure()) { + wait = new WaitForHttpResource("http", getFirstNode().getHttpSocketURI(), nodes.size()); + List> credentials = getFirstNode().getCredentials(); + if (getFirstNode().getCredentials().isEmpty() == false) { + wait.setUsername(credentials.get(0).get("useradd")); + wait.setPassword(credentials.get(0).get("-p")); + } + } else { + wait = new WaitForHttpResource( + "https", + getFirstNode().getHttpSocketURI(), + getFirstNode().getCredentials().get(0).get("username"), + getFirstNode().getCredentials().get(0).get("password"), + nodes.size() + ); + wait.setUsername(getFirstNode().getCredentials().get(0).get("username")); + wait.setPassword(getFirstNode().getCredentials().get(0).get("password")); + wait.setCertificateAuthorities(getFirstNode().getExtraConfigFilesMap().get("root-ca.pem")); } return wait.wait(500); } catch (IOException e) { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java index e47cf86685f55..268de50340cbf 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java @@ -160,6 +160,7 @@ public class OpenSearchNode implements TestClusterConfiguration { private final Path httpPortsFile; private final Path tmpDir; + private boolean secure = false; private int currentDistro = 0; private TestDistribution testDistribution; private final List distributions = new ArrayList<>(); @@ -209,6 +210,7 @@ public class OpenSearchNode implements TestClusterConfiguration { setTestDistribution(TestDistribution.INTEG_TEST); setVersion(VersionProperties.getOpenSearch()); this.zone = zone; + this.credentials.add(new HashMap<>()); } @Input @@ -217,6 +219,11 @@ public String getName() { return nameCustomization.apply(name); } + @Internal + public boolean isSecure() { + return secure; + } + @Internal public Version getVersion() { return Version.fromString(distributions.get(currentDistro).getVersion()); @@ -452,6 +459,11 @@ public void setPreserveDataDir(boolean preserveDataDir) { this.preserveDataDir = preserveDataDir; } + @Override + public void setSecure(boolean secure) { + this.secure = secure; + } + @Override public void freeze() { requireNonNull(testDistribution, "null testDistribution passed when configuring test cluster `" + this + "`"); @@ -471,6 +483,18 @@ public Stream logLines() throws IOException { @Override public synchronized void start() { LOGGER.info("Starting `{}`", this); + if (System.getProperty("tests.opensearch.secure") != null + && System.getProperty("tests.opensearch.secure").equalsIgnoreCase("true")) { + secure = true; + } + if (System.getProperty("tests.opensearch.username") != null) { + this.credentials.get(0).put("username", System.getProperty("tests.opensearch.username")); + LOGGER.info("Overwriting username to: " + this.getCredentials().get(0).get("username")); + } + if (System.getProperty("tests.opensearch.password") != null) { + this.credentials.get(0).put("password", System.getProperty("tests.opensearch.password")); + LOGGER.info("Overwriting password to: " + this.getCredentials().get(0).get("password")); + } if (Files.exists(getExtractedDistributionDir()) == false) { throw new TestClustersException("Can not start " + this + ", missing: " + getExtractedDistributionDir()); } @@ -1349,6 +1373,11 @@ public List getExtraConfigFiles() { return extraConfigFiles.getNormalizedCollection(); } + @Internal + public Map getExtraConfigFilesMap() { + return extraConfigFiles; + } + @Override @Internal public boolean isProcessAlive() { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java index 70773884eb920..22c4185a39a98 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java @@ -108,6 +108,8 @@ public interface TestClusterConfiguration { void setPreserveDataDir(boolean preserveDataDir); + void setSecure(boolean secure); + void freeze(); void start();