Merge remote-tracking branch 'upstream/main' into decommission/init-m…

…erge

.github/CODEOWNERS

@@ -1,3 +1 @@
-# This should match the owning team set up in https:/orgs/opensearch-project/teams
-* @opensearch-project/opensearch-core @reta
-
+* @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @adnapibar @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @xuezhou25

.github/workflows/gradle-check.yml

@@ -8,8 +8,15 @@ on:
  pull_request_target:
  types: [opened, synchronize, reopened]
 
+permissions:
+ contents: read # to fetch code (actions/checkout)
+
 jobs:
  gradle-check:
+ permissions:
+ contents: read # to fetch code (actions/checkout)
+ pull-requests: write # to create or update comment (peter-evans/create-or-update-comment)
+
  runs-on: ubuntu-latest
  timeout-minutes: 130
  steps:

.github/workflows/links.yml

@@ -2,6 +2,8 @@ name: Link Checker
 on:
  schedule:
  - cron: '0 0 * * *'
+permissions:
+ contents: read # to fetch code (actions/checkout)
 jobs:
  linkchecker:
  if: github.repository == 'opensearch-project/OpenSearch'

.github/workflows/version.yml

@@ -5,6 +5,7 @@ on:
  tags:
  - '*.*.*'
 
+permissions: {}
 jobs:
  build:
  runs-on: ubuntu-latest

CHANGELOG.md

@@ -6,13 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 3.0]
 ### Added
+- Hardened token permissions in GitHub workflows ([#4587](https:/opensearch-project/OpenSearch/pull/4587))
 - Support for HTTP/2 (server-side) ([#3847](https:/opensearch-project/OpenSearch/pull/3847))
 - Add getter for path field in NestedQueryBuilder ([#4636](https:/opensearch-project/OpenSearch/pull/4636))
 - Apply reproducible builds configuration for OpenSearch plugins through gradle plugin ([#4746](https:/opensearch-project/OpenSearch/pull/4746))
 - Add project health badges to the README.md ([#4843](https:/opensearch-project/OpenSearch/pull/4843))
 - [Test] Add IAE test for deprecated edgeNGram analyzer name ([#5040](https:/opensearch-project/OpenSearch/pull/5040))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https:/opensearch-project/OpenSearch/pull/5151))
 - Add feature flag for extensions ([#5211](https:/opensearch-project/OpenSearch/pull/5211))
+- Added jackson dependency to server ([#5366] (https:/opensearch-project/OpenSearch/pull/5366))
+- Added experimental extensions to main ([#5347](https:/opensearch-project/OpenSearch/pull/5347))
 
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
@@ -55,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https:/opensearch-project/OpenSearch/pull/4459))
 - Support remote translog transfer for request level durability([#4480](https:/opensearch-project/OpenSearch/pull/4480))
 - Changed http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https:/opensearch-project/OpenSearch/pull/4773))
+- Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https:/opensearch-project/OpenSearch/pull/5283))
 
 ### Deprecated
 
@@ -77,12 +81,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https:/opensearch-project/OpenSearch/pull/4827))
 - Fixed compression support for h2c protocol ([#4944](https:/opensearch-project/OpenSearch/pull/4944))
 - Reject bulk requests with invalid actions ([#5299](https:/opensearch-project/OpenSearch/issues/5299))
+- Support OpenSSL Provider with default Netty allocator ([#5460](https:/opensearch-project/OpenSearch/pull/5460))
 
 ### Security
 
 ## [Unreleased 2.x]
 ### Added
 - Prevent deletion of snapshots that are backing searchable snapshot indexes ([#5069](https:/opensearch-project/OpenSearch/pull/5069))
+- Add max_shard_size parameter for shrink API ([#5229](https:/opensearch-project/OpenSearch/pull/5229))
 
 ### Dependencies
 - Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1
@@ -95,6 +101,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Deprecated
 ### Removed
 ### Fixed
+- Fix 1.x compatibility bug with stored Tasks ([#5412](https:/opensearch-project/OpenSearch/pull/5412))
+- Fix case sensitivity for wildcard queries ([#5462](https:/opensearch-project/OpenSearch/pull/5462))
 ### Security
 
 [Unreleased 3.0]: https:/opensearch-project/OpenSearch/compare/2.4...HEAD

MAINTAINERS.md

@@ -5,7 +5,6 @@
 
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
-| Abbas Hussain | [abbashus](https:/abbashus) | Amazon |
 | Anas Alkouz | [anasalkouz](https:/anasalkouz) | Amazon |
 | Andrew Ross | [andrross](https:/andrross)| Amazon |
 | Andriy Redko | [reta](https:/reta) | Aiven |
@@ -22,8 +21,8 @@
 | Rishikesh Pasham | [Rishikesh1159](https:/Rishikesh1159) | Amazon|
 | Ryan Bogan | [ryanbogan](https:/ryanbogan) | Amazon |
 | Sarat Vemulapalli | [saratvemulapalli](https:/saratvemulapalli) | Amazon |
-| Shweta Thareja |[shwetathareja](https:/shwetathareja) | Amazon |
-| Suraj Singh |[dreamer-89](https:/dreamer-89) | Amazon |
+| Shweta Thareja | [shwetathareja](https:/shwetathareja) | Amazon |
+| Suraj Singh | [dreamer-89](https:/dreamer-89) | Amazon |
 | Tianli Feng | [tlfeng](https:/tlfeng) | Amazon |
 | Vacha Shah | [VachaShah](https:/VachaShah) | Amazon |
 | Xue Zhou | [xuezhou25](https:/xuezhou25) | Amazon |
@@ -32,6 +31,7 @@
 
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
+| Abbas Hussain | [abbashus](https:/abbashus) | Amazon |
 | Megha Sai Kavikondala | [meghasaik](https:/meghasaik) | Amazon |
 
 [This document](https:/opensearch-project/.github/blob/main/MAINTAINERS.md) explains what maintainers do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md).

client/rest-high-level/src/main/java/org/opensearch/client/indices/ResizeRequest.java

@@ -39,6 +39,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.ToXContentObject;
 import org.opensearch.common.xcontent.XContentBuilder;
+import org.opensearch.common.unit.ByteSizeValue;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -58,6 +59,7 @@ public class ResizeRequest extends TimedRequest implements Validatable, ToXConte
  private final String targetIndex;
  private Settings settings = Settings.EMPTY;
  private Set<Alias> aliases = new HashSet<>();
+ private ByteSizeValue maxShardSize;
 
  /**
  * Creates a new resize request
@@ -155,6 +157,24 @@ public ActiveShardCount getWaitForActiveShards() {
  return waitForActiveShards;
  }
 
+ /**
+ * Sets the maximum size of a primary shard in the new shrunken index.
+ * This parameter can be used to calculate the lowest factor of the source index's shards number
+ * which satisfies the maximum shard size requirement.
+ *
+ * @param maxShardSize the maximum size of a primary shard in the new shrunken index
+ */
+ public void setMaxShardSize(ByteSizeValue maxShardSize) {
+ this.maxShardSize = maxShardSize;
+ }
+
+ /**
+ * Returns the maximum size of a primary shard in the new shrunken index.
+ */
+ public ByteSizeValue getMaxShardSize() {
+ return maxShardSize;
+ }
+
  @Override
  public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
  builder.startObject();

client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java

@@ -79,6 +79,7 @@
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Assert;
+import org.opensearch.common.unit.ByteSizeValue;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -701,6 +702,8 @@ private void resizeTest(ResizeType resizeType, CheckedFunction<ResizeRequest, Re
  RequestConvertersTests.setRandomWaitForActiveShards(resizeRequest::setWaitForActiveShards, expectedParams);
  if (resizeType == ResizeType.SPLIT) {
  resizeRequest.setSettings(Settings.builder().put("index.number_of_shards", 2).build());
+ } else if (resizeType == ResizeType.SHRINK) {
+ resizeRequest.setMaxShardSize(new ByteSizeValue(randomIntBetween(1, 1000)));
  }
 
  Request request = function.apply(resizeRequest);

modules/ingest-geoip/build.gradle

@@ -41,8 +41,6 @@ opensearchplugin {
 dependencies {
  api('com.maxmind.geoip2:geoip2:3.0.2')
  // geoip2 dependencies:
- api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
- api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
  api('com.maxmind.db:maxmind-db:2.1.0')
 
  testImplementation 'org.elasticsearch:geolite2-databases:20191119'

modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4NioServerSocketChannel.java

@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import io.netty.channel.socket.InternetProtocolFamily;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.util.internal.SocketUtils;
+import io.netty.util.internal.logging.InternalLogger;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.channels.spi.SelectorProvider;
+import java.util.List;
+
+public class Netty4NioServerSocketChannel extends NioServerSocketChannel {
+ private static final InternalLogger logger = InternalLoggerFactory.getInstance(Netty4NioServerSocketChannel.class);
+
+ public Netty4NioServerSocketChannel() {
+ super();
+ }
+
+ public Netty4NioServerSocketChannel(SelectorProvider provider) {
+ super(provider);
+ }
+
+ public Netty4NioServerSocketChannel(SelectorProvider provider, InternetProtocolFamily family) {
+ super(provider, family);
+ }
+
+ public Netty4NioServerSocketChannel(ServerSocketChannel channel) {
+ super(channel);
+ }
+
+ @Override
+ protected int doReadMessages(List<Object> buf) throws Exception {
+ SocketChannel ch = SocketUtils.accept(javaChannel());
+
+ try {
+ if (ch != null) {
+ buf.add(new Netty4NioSocketChannel(this, ch));
+ return 1;
+ }
+ } catch (Throwable t) {
+ logger.warn("Failed to create a new channel from an accepted socket.", t);
+
+ try {
+ ch.close();
+ } catch (Throwable t2) {
+ logger.warn("Failed to close a socket.", t2);
+ }
+ }
+
+ return 0;
+ }
+}

modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java

@@ -39,7 +39,6 @@
 import io.netty.buffer.UnpooledByteBufAllocator;
 import io.netty.channel.Channel;
 import io.netty.channel.ServerChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.common.Booleans;
@@ -181,7 +180,7 @@ public static Class<? extends ServerChannel> getServerChannelType() {
  if (ALLOCATOR instanceof NoDirectBuffers) {
  return CopyBytesServerSocketChannel.class;
  } else {
- return NioServerSocketChannel.class;
+ return Netty4NioServerSocketChannel.class;
  }
  }
 

plugins/discovery-ec2/build.gradle

@@ -46,8 +46,6 @@ dependencies {
  api "commons-logging:commons-logging:${versions.commonslogging}"
  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
  api "commons-codec:commons-codec:${versions.commonscodec}"
- api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
- api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
 restResources {

plugins/ingest-attachment/build.gradle

@@ -38,10 +38,10 @@ opensearchplugin {
 }
 
 versions << [
- 'tika' : '2.5.0',
- 'pdfbox': '2.0.25',
- 'poi' : '5.2.2',
- 'mime4j': '0.8.3'
+ 'tika' : '2.6.0',
+ 'pdfbox': '2.0.27',
+ 'poi' : '5.2.3',
+ 'mime4j': '0.8.8'
 ]
 
 dependencies {

plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.8.jar.sha1

@@ -0,0 +1 @@
+7330de23c52f71617cbec7f1d2760dae32e687cd

plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.8.jar.sha1

@@ -0,0 +1 @@
+e76715563a6bd150f84ccb0adb920aec8faf4779

plugins/ingest-attachment/licenses/fontbox-2.0.27.jar.sha1

@@ -0,0 +1 @@
+d08c064d18b2b149da937d15c0d1708cba03f29d

plugins/ingest-attachment/licenses/pdfbox-2.0.27.jar.sha1

@@ -0,0 +1 @@
+416a9dfce3714116bfdf793b15368df04266845f

plugins/ingest-attachment/licenses/poi-5.2.3.jar.sha1

@@ -0,0 +1 @@
+2fb22ae74ad5aea6af1a9c64b9542f2ccf348604

plugins/ingest-attachment/licenses/poi-ooxml-5.2.3.jar.sha1

@@ -0,0 +1 @@
+02efd11c940adb18c03eb9ce7ad88fc40ee6a196

plugins/ingest-attachment/licenses/poi-ooxml-lite-5.2.3.jar.sha1

@@ -0,0 +1 @@
+db113c8e9051b0ff967f4911fa20336c8325a7c5