From b7d8013b0acb2d5a24f31c52deebc46afac543f7 Mon Sep 17 00:00:00 2001 From: Neetika Singhal Date: Fri, 10 Nov 2023 14:13:30 -0800 Subject: [PATCH] update index random function to fix the bogus documents deletion (#11142) Signed-off-by: Neetika Singhal --- .../basic/TransportTwoNodesSearchIT.java | 1 + .../highlight/HighlighterSearchIT.java | 16 ---------------- .../test/OpenSearchIntegTestCase.java | 19 +++++++++++++------ 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java index ce5f3f63faa66..edceb0cbc0d24 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/TransportTwoNodesSearchIT.java @@ -128,6 +128,7 @@ private Set prepareData(int numShards) throws Exception { fullExpectedIds.add(Integer.toString(i)); } refresh(); + indexRandomForConcurrentSearch("test"); return fullExpectedIds; } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index f7bc5eb75ad0f..2afa911223074 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -2013,10 +2013,6 @@ private static String randomStoreField() { } public void testHighlightNoMatchSize() throws IOException, InterruptedException { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10900", - internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) - ); assertAcked( prepareCreate("test").setMapping( "text", @@ -2128,10 +2124,6 @@ public void testHighlightNoMatchSize() throws IOException, InterruptedException } public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException, InterruptedException { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10900", - internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) - ); assertAcked( prepareCreate("test").setMapping( "text", @@ -2228,10 +2220,6 @@ public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException, } public void testHighlightNoMatchSizeNumberOfFragments() throws IOException, InterruptedException { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10900", - internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) - ); assertAcked( prepareCreate("test").setMapping( "text", @@ -3416,10 +3404,6 @@ public void testFiltersFunctionScoreQueryHighlight() throws Exception { } public void testHighlightQueryRewriteDatesWithNow() throws Exception { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10434", - internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) - ); assertAcked( client().admin() .indices() diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 0c6c81103922f..7614cd0e8f920 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -1667,10 +1667,6 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma } assertThat(actualErrors, emptyIterable()); - if (dummyDocuments) { - bogusIds.addAll(indexRandomForMultipleSlices(indicesArray)); - } - if (!bogusIds.isEmpty()) { // delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs! for (List doc : bogusIds) { @@ -1686,6 +1682,9 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma client().admin().indices().prepareRefresh(indicesArray).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get() ); } + if (dummyDocuments) { + indexRandomForMultipleSlices(indicesArray); + } } /* @@ -1694,7 +1693,7 @@ public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean ma * multiple slices based on segment count. * @param indices the indices in which bogus documents should be ingested * */ - protected Set> indexRandomForMultipleSlices(String... indices) throws InterruptedException { + protected void indexRandomForMultipleSlices(String... indices) throws InterruptedException { Set> bogusIds = new HashSet<>(); int refreshCount = randomIntBetween(2, 3); for (String index : indices) { @@ -1731,7 +1730,15 @@ protected Set> indexRandomForMultipleSlices(String... indices) thro refresh(index); } } - return bogusIds; + for (List doc : bogusIds) { + assertEquals( + "failed to delete a dummy doc [" + doc.get(0) + "][" + doc.get(1) + "]", + DocWriteResponse.Result.DELETED, + client().prepareDelete(doc.get(0), doc.get(1)).setRouting(doc.get(1)).get().getResult() + ); + } + // refresh is called to make sure the bogus docs doesn't affect the search results + refresh(); } private final AtomicInteger dummmyDocIdGenerator = new AtomicInteger();