Merge branch 'main' into support_flat_object_docvalue

Signed-off-by: kkewwei <[email protected]>

CHANGELOG.md

@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https:/opensearch-project/OpenSearch/pull/15978))
 - Add changes to block calls in cat shards, indices and segments based on dynamic limit settings ([#15986](https:/opensearch-project/OpenSearch/pull/15986))
 - New `phone` & `phone-search` analyzer + tokenizer ([#15915](https:/opensearch-project/OpenSearch/pull/15915))
+- Add _list/shards API as paginated alternate to _cat/shards ([#14641](https:/opensearch-project/OpenSearch/pull/14641))
 - Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https:/opensearch-project/OpenSearch/issues/14383))
 
 ### Dependencies
@@ -35,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https:/opensearch-project/OpenSearch/pull/16041))
 - Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https:/opensearch-project/OpenSearch/pull/16042))
 - Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https:/opensearch-project/OpenSearch/pull/16137))
+- Bump Apache lucene from 9.11.1 to 9.12.0 ([#15333](https:/opensearch-project/OpenSearch/pull/15333))
 - Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https:/opensearch-project/OpenSearch/pull/16133))
 - Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https:/opensearch-project/OpenSearch/pull/16135))
 - Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https:/opensearch-project/OpenSearch/pull/16182))

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch = 3.0.0
-lucene = 9.12.0-snapshot-847316d
+lucene = 9.12.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.4+7

libs/core/licenses/lucene-core-9.12.0.jar.sha1

@@ -0,0 +1 @@
+fdb055d569bb20bfce9618fe2b01c29bab7f290c

modules/lang-expression/licenses/lucene-expressions-9.12.0.jar.sha1

@@ -0,0 +1 @@
+5ba843374a0aab3dfe0b11cb28b251844d85bf5b

plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0.jar.sha1

@@ -0,0 +1 @@
+a9232b6a4882979118d3281b98dfdb6e0e1cb5ca

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0.jar.sha1

@@ -0,0 +1 @@
+a3a6950ffc22e76a082e1b3cefb022b9f7870d29

plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0.jar.sha1

@@ -0,0 +1 @@
+e71f85b72ed3939039ba8897b28b065dd11918b9

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0.jar.sha1

@@ -0,0 +1 @@
+6baa3ae7ab20d6e644cf0bedb271c50a44c0e259

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0.jar.sha1

@@ -0,0 +1 @@
+f183e1e8b1eaaa4dec444774a285bb8b66518522

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0.jar.sha1

@@ -0,0 +1 @@
+b81a609934e65d12ab9d2d84bc2ea6f56a360e57

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0.jar.sha1

@@ -0,0 +1 @@
+bec069f286b45f20b743c81e84202369cd0467e7

plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java

@@ -8,8 +8,8 @@
 
 package org.opensearch.plugin.correlation.core.index.codec;
 
+import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.plugin.correlation.core.index.codec.correlation990.CorrelationCodec;
 import org.opensearch.plugin.correlation.core.index.codec.correlation990.PerFieldCorrelationVectorsFormat;

server/licenses/lucene-analysis-common-9.12.0.jar.sha1

@@ -0,0 +1 @@
+4c2503cfaba37249e20ea877555cb52ee89d1ae1

server/licenses/lucene-backward-codecs-9.12.0.jar.sha1

@@ -0,0 +1 @@
+68fe98c94e9644a584ea1bf525e68d9406fc61ec

server/licenses/lucene-core-9.12.0.jar.sha1

@@ -0,0 +1 @@
+fdb055d569bb20bfce9618fe2b01c29bab7f290c

server/licenses/lucene-grouping-9.12.0.jar.sha1

@@ -0,0 +1 @@
+ccf99f8db57aa97b2c1f95c5cc2a11156a043921

server/licenses/lucene-highlighter-9.12.0.jar.sha1

@@ -0,0 +1 @@
+e93429f66fbcd3b58d81f01223d6ce5688047296

server/licenses/lucene-join-9.12.0.jar.sha1

@@ -0,0 +1 @@
+14c802d6955eaf11107375a2ada8fe8ec53b3e01

server/licenses/lucene-memory-9.12.0.jar.sha1

@@ -0,0 +1 @@
+ffe090339540876b40df792aee51a42af6b3f37f

server/licenses/lucene-misc-9.12.0.jar.sha1

@@ -0,0 +1 @@
+ad17704ee90eb926b6d3105f7027485cdadbecd9

server/licenses/lucene-queries-9.12.0.jar.sha1

@@ -0,0 +1 @@
+3ac2a62b0b55c5725bb65f0c5454f9f8a401cf43

server/licenses/lucene-queryparser-9.12.0.jar.sha1

@@ -0,0 +1 @@
+55959399373876f4c184944315458dc6b88fbd81

server/licenses/lucene-sandbox-9.12.0.jar.sha1

@@ -0,0 +1 @@
+f65882536d681c11a1cbc920e5679201101e3603

server/licenses/lucene-spatial-extras-9.12.0.jar.sha1

@@ -0,0 +1 @@
+9d00cc7cc2279822ef6740f0677cafacfb439fa8

server/licenses/lucene-spatial3d-9.12.0.jar.sha1

@@ -0,0 +1 @@
+e3092632ca1d4427d3ebb2c866ac89d90f5b61ec

server/licenses/lucene-suggest-9.12.0.jar.sha1

@@ -0,0 +1 @@
+e1c6636499317ebe498f3490a1ec8b86b8a363dd

server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java

@@ -110,7 +110,7 @@ public void testConcurrentSearchTaskTracking() {
  Map<Long, List<ThreadResourceInfo>> threadStats = getThreadStats(SearchAction.NAME + "[*]", taskInfo.getTaskId());
  // Concurrent search forks each slice of 5 segments to different thread (see please
  // https:/apache/lucene/issues/12498)
- assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0) + 1, threadStats.size());
+ assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0), threadStats.size());
 
  // assert that all task descriptions have non-zero length
  MatcherAssert.assertThat(taskInfo.getDescription().length(), greaterThan(0));

server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/shards/TransportCatShardsActionIT.java

@@ -8,7 +8,6 @@
 
 package org.opensearch.action.admin.cluster.shards;
 
-import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
 import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.routing.ShardRouting;
@@ -51,9 +50,9 @@ public void testCatShardsWithSuccessResponse() throws InterruptedException {
  client().execute(CatShardsAction.INSTANCE, shardsRequest, new ActionListener<CatShardsResponse>() {
  @Override
  public void onResponse(CatShardsResponse catShardsResponse) {
- ClusterStateResponse clusterStateResponse = catShardsResponse.getClusterStateResponse();
+ List<ShardRouting> shardRoutings = catShardsResponse.getResponseShards();
  IndicesStatsResponse indicesStatsResponse = catShardsResponse.getIndicesStatsResponse();
- for (ShardRouting shard : clusterStateResponse.getState().routingTable().allShards()) {
+ for (ShardRouting shard : shardRoutings) {
  assertEquals("test", shard.getIndexName());
  assertNotNull(indicesStatsResponse.asMap().get(shard));
  }

server/src/internalClusterTest/java/org/opensearch/recovery/TruncatedRecoveryIT.java

@@ -58,6 +58,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.opensearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_CHUNK_SIZE_SETTING;
@@ -184,7 +185,7 @@ public void testCancelRecoveryAndResume() throws Exception {
  )
  .get();
 
- latch.await();
+ latch.await(5, TimeUnit.MINUTES);
 
  // at this point we got some truncated left overs on the replica on the unlucky node
  // now we are allowing the recovery to allocate again and finish to see if we wipe the truncated files

server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java

@@ -479,6 +479,7 @@ public void testCloneSnapshotV2MasterSwitch() throws Exception {
  assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), csr2.getSnapshotInfo()));
  }
 
+ @AwaitsFix(bugUrl = "https:/opensearch-project/OpenSearch/pull/16191")
  public void testDeleteWhileV2CreateOngoing() throws Exception {
  final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
  internalCluster().startDataOnlyNode(pinnedTimestampSettings());
@@ -537,6 +538,7 @@ public void testDeleteWhileV2CreateOngoing() throws Exception {
  assertThat(snapInfo, contains(csr.getSnapshotInfo()));
  }
 
+ @AwaitsFix(bugUrl = "https:/opensearch-project/OpenSearch/pull/16191")
  public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception {
  final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
  internalCluster().startDataOnlyNode(pinnedTimestampSettings());

server/src/main/java/org/apache/lucene/misc/search/similarity/LegacyBM25Similarity.java

@@ -70,6 +70,7 @@ public LegacyBM25Similarity(float k1, float b) {
  * within the range {@code [0..1]}
  */
  public LegacyBM25Similarity(float k1, float b, boolean discountOverlaps) {
+ super(discountOverlaps);
  this.bm25Similarity = new BM25Similarity(k1, b, discountOverlaps);
  }
 
@@ -101,15 +102,6 @@ public final float getB() {
  return bm25Similarity.getB();
  }
 
- /**
- * Returns true if overlap tokens are discounted from the document's length.
- *
- * @see #LegacyBM25Similarity(float, float, boolean)
- */
- public boolean getDiscountOverlaps() {
- return bm25Similarity.getDiscountOverlaps();
- }
-
  @Override
  public String toString() {
  return bm25Similarity.toString();

server/src/main/java/org/apache/lucene/util/CombinedBitSet.java

@@ -87,15 +87,7 @@ public int prevSetBit(int index) {
 
  @Override
  public int nextSetBit(int index) {
- assert index >= 0 && index < length : "index=" + index + " numBits=" + length();
- int next = first.nextSetBit(index);
- while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
- if (next == length() - 1) {
- return DocIdSetIterator.NO_MORE_DOCS;
- }
- next = first.nextSetBit(next + 1);
- }
- return next;
+ return nextSetBit(index, length() - 1);
  }
 
  @Override
@@ -132,4 +124,20 @@ public void clear(int startIndex, int endIndex) {
  public boolean getAndSet(int i) {
  throw new UnsupportedOperationException("not implemented");
  }
+
+ @Override
+ public int nextSetBit(int start, int end) {
+ assert start >= 0 && start < length() : "start=" + start + " numBits=" + length();
+ assert end >= start && end < length() : "end=" + end + " numBits=" + length();
+
+ int next = first.nextSetBit(start);
+ while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
+ if (next >= end) {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+ next = first.nextSetBit(next + 1);
+ }
+ return next;
+
+ }
 }

server/src/main/java/org/opensearch/action/ActionModule.java

@@ -465,6 +465,7 @@
 import org.opensearch.rest.action.list.AbstractListAction;
 import org.opensearch.rest.action.list.RestIndicesListAction;
 import org.opensearch.rest.action.list.RestListAction;
+import org.opensearch.rest.action.list.RestShardsListAction;
 import org.opensearch.rest.action.search.RestClearScrollAction;
 import org.opensearch.rest.action.search.RestCountAction;
 import org.opensearch.rest.action.search.RestCreatePitAction;
@@ -993,6 +994,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
 
  // LIST API
  registerHandler.accept(new RestIndicesListAction(responseLimitSettings));
+ registerHandler.accept(new RestShardsListAction());
 
  // Point in time API
  registerHandler.accept(new RestCreatePitAction());

server/src/main/java/org/opensearch/action/admin/cluster/shards/CatShardsRequest.java

@@ -8,10 +8,13 @@
 
 package org.opensearch.action.admin.cluster.shards;
 
+import org.opensearch.Version;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.action.pagination.PageParams;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.tasks.TaskId;
 import org.opensearch.rest.action.admin.cluster.ClusterAdminTask;
 
@@ -27,13 +30,39 @@ public class CatShardsRequest extends ClusterManagerNodeReadRequest<CatShardsReq
 
  private String[] indices;
  private TimeValue cancelAfterTimeInterval;
+ private PageParams pageParams = null;
  private boolean requestLimitCheckSupported;
 
  public CatShardsRequest() {}
 
  public CatShardsRequest(StreamInput in) throws IOException {
  super(in);
- this.requestLimitCheckSupported = false;
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ indices = in.readStringArray();
+ cancelAfterTimeInterval = in.readOptionalTimeValue();
+ if (in.readBoolean()) {
+ pageParams = new PageParams(in);
+ }
+ requestLimitCheckSupported = in.readBoolean();
+ }
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ super.writeTo(out);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ if (indices == null) {
+ out.writeVInt(0);
+ } else {
+ out.writeStringArray(indices);
+ }
+ out.writeOptionalTimeValue(cancelAfterTimeInterval);
+ out.writeBoolean(pageParams != null);
+ if (pageParams != null) {
+ pageParams.writeTo(out);
+ }
+ out.writeBoolean(requestLimitCheckSupported);
+ }
  }
 
  @Override
@@ -57,6 +86,14 @@ public TimeValue getCancelAfterTimeInterval() {
  return this.cancelAfterTimeInterval;
  }
 
+ public void setPageParams(PageParams pageParams) {
+ this.pageParams = pageParams;
+ }
+
+ public PageParams getPageParams() {
+ return pageParams;
+ }
+
  public void setRequestLimitCheckSupported(final boolean requestLimitCheckSupported) {
  this.requestLimitCheckSupported = requestLimitCheckSupported;
  }