Merge branch 'main' into test-integ-branch

Signed-off-by: Rajat Gupta <[email protected]>

Author: RajatGupta02

CHANGELOG-3.0.md

@@ -22,6 +22,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Arrow Flight RPC plugin with Flight server bootstrap logic and client for internode communication ([#16962](https://github.com/opensearch-project/OpenSearch/pull/16962))
 - Added offset management for the pull-based Ingestion ([#17354](https://github.com/opensearch-project/OpenSearch/pull/17354))
 - Added integ tests for systemd configs ([#17410](https://github.com/opensearch-project/OpenSearch/pull/17410))
+- Add filter function for AbstractQueryBuilder, BoolQueryBuilder, ConstantScoreQueryBuilder([#17409](https://github.com/opensearch-project/OpenSearch/pull/17409))
+- [Star Tree] [Search] Resolving keyword & numeric bucket aggregation with metric aggregation using star-tree ([#17165](https://github.com/opensearch-project/OpenSearch/pull/17165))
+
 
 ### Dependencies
 - Update Apache Lucene to 10.1.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366))

plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java

@@ -16,6 +16,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.index.query.RangeQueryBuilder;
+import org.opensearch.indices.pollingingest.PollingIngestStats;
 import org.opensearch.plugins.PluginInfo;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.junit.Assert;
@@ -75,6 +76,11 @@ public void testKafkaIngestion() {
             refresh("test");
             SearchResponse response = client().prepareSearch("test").setQuery(query).get();
             assertThat(response.getHits().getTotalHits().value(), is(1L));
+            PollingIngestStats stats = client().admin().indices().prepareStats("test").get().getIndex("test").getShards()[0]
+                .getPollingIngestStats();
+            assertNotNull(stats);
+            assertThat(stats.getMessageProcessorStats().getTotalProcessedCount(), is(2L));
+            assertThat(stats.getConsumerStats().getTotalPolledCount(), is(2L));
         });
     }
 

server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java

@@ -250,7 +250,7 @@ private void assertOnGoingRecoveryState(
         assertThat(state.getStage(), not(equalTo(Stage.DONE)));
     }
 
-    private void slowDownRecovery(ByteSizeValue shardSize) {
+    public void slowDownRecovery(ByteSizeValue shardSize) {
         long chunkSize = Math.max(1, shardSize.getBytes() / 10);
         assertTrue(
             client().admin()
@@ -528,7 +528,7 @@ public void testRerouteRecovery() throws Exception {
             assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(), equalTo(1));
             indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
             assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(), equalTo(1));
-        }, TimeValue.timeValueSeconds(10), TimeValue.timeValueMillis(500));
+        }, TimeValue.timeValueSeconds(60), TimeValue.timeValueMillis(500));
 
         logger.info("--> request recoveries");
         RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();

server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java

@@ -10,9 +10,12 @@
 
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.indices.recovery.IndexRecoveryIT;
+import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.hamcrest.Matcher;
@@ -22,6 +25,7 @@
 
 import java.nio.file.Path;
 
+import static org.opensearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_CHUNK_SIZE_SETTING;
 import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
@@ -54,6 +58,26 @@ public Settings indexSettings() {
             .build();
     }
 
+    @Override
+    public void slowDownRecovery(ByteSizeValue shardSize) {
+        logger.info("--> shardSize: " + shardSize);
+        long chunkSize = Math.max(1, shardSize.getBytes() / 50);
+        assertTrue(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(
+                    Settings.builder()
+                        // one chunk per sec..
+                        .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), chunkSize, ByteSizeUnit.BYTES)
+                        // small chunks
+                        .put(INDICES_RECOVERY_CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkSize, ByteSizeUnit.BYTES))
+                )
+                .get()
+                .isAcknowledged()
+        );
+    }
+
     @After
     public void teardown() {
         clusterAdmin().prepareCleanupRepository(REPOSITORY_NAME).get();

server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -56,6 +56,7 @@
 import org.opensearch.index.seqno.SeqNoStats;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.pollingingest.PollingIngestStats;
 import org.opensearch.node.NodeService;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportRequest;
@@ -210,15 +211,18 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
                         CommitStats commitStats;
                         SeqNoStats seqNoStats;
                         RetentionLeaseStats retentionLeaseStats;
+                        PollingIngestStats pollingIngestStats;
                         try {
                             commitStats = indexShard.commitStats();
                             seqNoStats = indexShard.seqNoStats();
                             retentionLeaseStats = indexShard.getRetentionLeaseStats();
+                            pollingIngestStats = indexShard.pollingIngestStats();
                         } catch (final AlreadyClosedException e) {
                             // shard is closed - no stats is fine
                             commitStats = null;
                             seqNoStats = null;
                             retentionLeaseStats = null;
+                            pollingIngestStats = null;
                         }
                         shardsStats.add(
                             new ShardStats(
@@ -227,7 +231,8 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
                                 new CommonStats(indicesService.getIndicesQueryCache(), indexShard, commonStatsFlags),
                                 commitStats,
                                 seqNoStats,
-                                retentionLeaseStats
+                                retentionLeaseStats,
+                                pollingIngestStats
                             )
                         );
                     }

server/src/main/java/org/opensearch/action/admin/indices/stats/ShardStats.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.indices.stats;
 
+import org.opensearch.Version;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.annotation.PublicApi;
@@ -44,6 +45,7 @@
 import org.opensearch.index.seqno.RetentionLeaseStats;
 import org.opensearch.index.seqno.SeqNoStats;
 import org.opensearch.index.shard.ShardPath;
+import org.opensearch.indices.pollingingest.PollingIngestStats;
 
 import java.io.IOException;
 
@@ -65,6 +67,9 @@ public class ShardStats implements Writeable, ToXContentFragment {
     @Nullable
     private RetentionLeaseStats retentionLeaseStats;
 
+    @Nullable
+    private PollingIngestStats pollingIngestStats;
+
     /**
      * Gets the current retention lease stats.
      *
@@ -87,6 +92,9 @@ public ShardStats(StreamInput in) throws IOException {
         isCustomDataPath = in.readBoolean();
         seqNoStats = in.readOptionalWriteable(SeqNoStats::new);
         retentionLeaseStats = in.readOptionalWriteable(RetentionLeaseStats::new);
+        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+            pollingIngestStats = in.readOptionalWriteable(PollingIngestStats::new);
+        }
     }
 
     public ShardStats(
@@ -95,7 +103,8 @@ public ShardStats(
         final CommonStats commonStats,
         final CommitStats commitStats,
         final SeqNoStats seqNoStats,
-        final RetentionLeaseStats retentionLeaseStats
+        final RetentionLeaseStats retentionLeaseStats,
+        final PollingIngestStats pollingIngestStats
     ) {
         this.shardRouting = routing;
         this.dataPath = shardPath.getRootDataPath().toString();
@@ -105,6 +114,7 @@ public ShardStats(
         this.commonStats = commonStats;
         this.seqNoStats = seqNoStats;
         this.retentionLeaseStats = retentionLeaseStats;
+        this.pollingIngestStats = pollingIngestStats;
     }
 
     /**
@@ -128,6 +138,11 @@ public SeqNoStats getSeqNoStats() {
         return this.seqNoStats;
     }
 
+    @Nullable
+    public PollingIngestStats getPollingIngestStats() {
+        return this.pollingIngestStats;
+    }
+
     public String getDataPath() {
         return dataPath;
     }
@@ -150,6 +165,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(isCustomDataPath);
         out.writeOptionalWriteable(seqNoStats);
         out.writeOptionalWriteable(retentionLeaseStats);
+        if (out.getVersion().onOrAfter((Version.V_3_0_0))) {
+            out.writeOptionalWriteable(pollingIngestStats);
+        }
     }
 
     @Override
@@ -171,6 +189,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (retentionLeaseStats != null) {
             retentionLeaseStats.toXContent(builder, params);
         }
+        if (pollingIngestStats != null) {
+            pollingIngestStats.toXContent(builder, params);
+        }
         builder.startObject(Fields.SHARD_PATH);
         builder.field(Fields.STATE_PATH, statePath);
         builder.field(Fields.DATA_PATH, dataPath);

server/src/main/java/org/opensearch/action/admin/indices/stats/TransportIndicesStatsAction.java

@@ -52,6 +52,7 @@
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.ShardNotFoundException;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.pollingingest.PollingIngestStats;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
@@ -141,16 +142,27 @@ protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting sh
         CommitStats commitStats;
         SeqNoStats seqNoStats;
         RetentionLeaseStats retentionLeaseStats;
+        PollingIngestStats pollingIngestStats;
         try {
             commitStats = indexShard.commitStats();
             seqNoStats = indexShard.seqNoStats();
             retentionLeaseStats = indexShard.getRetentionLeaseStats();
+            pollingIngestStats = indexShard.pollingIngestStats();
         } catch (final AlreadyClosedException e) {
             // shard is closed - no stats is fine
             commitStats = null;
             seqNoStats = null;
             retentionLeaseStats = null;
+            pollingIngestStats = null;
         }
-        return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), commonStats, commitStats, seqNoStats, retentionLeaseStats);
+        return new ShardStats(
+            indexShard.routingEntry(),
+            indexShard.shardPath(),
+            commonStats,
+            commitStats,
+            seqNoStats,
+            retentionLeaseStats,
+            pollingIngestStats
+        );
     }
 }

server/src/main/java/org/opensearch/index/engine/Engine.java

@@ -93,6 +93,7 @@
 import org.opensearch.index.translog.Translog;
 import org.opensearch.index.translog.TranslogDeletionPolicy;
 import org.opensearch.index.translog.TranslogManager;
+import org.opensearch.indices.pollingingest.PollingIngestStats;
 import org.opensearch.search.suggest.completion.CompletionStats;
 
 import java.io.Closeable;
@@ -946,6 +947,13 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl
         return stats;
     }
 
+    /**
+     * @return Stats for pull-based ingestion.
+     */
+    public PollingIngestStats pollingIngestStats() {
+        return null;
+    }
+
     protected TranslogDeletionPolicy getTranslogDeletionPolicy(EngineConfig engineConfig) {
         TranslogDeletionPolicy customTranslogDeletionPolicy = null;
         if (engineConfig.getCustomTranslogDeletionPolicyFactory() != null) {

server/src/main/java/org/opensearch/index/engine/IngestionEngine.java

@@ -29,6 +29,7 @@
 import org.opensearch.index.translog.TranslogStats;
 import org.opensearch.index.translog.listener.CompositeTranslogEventListener;
 import org.opensearch.indices.pollingingest.DefaultStreamPoller;
+import org.opensearch.indices.pollingingest.PollingIngestStats;
 import org.opensearch.indices.pollingingest.StreamPoller;
 
 import java.io.IOException;
@@ -288,4 +289,9 @@ protected TranslogManager createTranslogManager(
     protected Map<String, String> commitDataAsMap() {
         return commitDataAsMap(indexWriter);
     }
+
+    @Override
+    public PollingIngestStats pollingIngestStats() {
+        return streamPoller.getStats();
+    }
 }

server/src/main/java/org/opensearch/index/query/AbstractQueryBuilder.java

@@ -86,6 +86,30 @@ protected AbstractQueryBuilder(StreamInput in) throws IOException {
         queryName = in.readOptionalString();
     }
 
+    /**
+     * Check the input parameters of filter function.
+     * @param filter filter to combine with current query builder
+     * @return true if parameters are valid. Returns false when the filter is null.
+     */
+    public static boolean validateFilterParams(QueryBuilder filter) {
+        return filter != null;
+    }
+
+    /**
+     * Combine filter with current query builder
+     * @param filter filter to combine with current query builder
+     * @return query builder with filter combined
+     */
+    public QueryBuilder filter(QueryBuilder filter) {
+        if (validateFilterParams(filter) == false) {
+            return this;
+        }
+        final BoolQueryBuilder modifiedQB = new BoolQueryBuilder();
+        modifiedQB.must(this);
+        modifiedQB.filter(filter);
+        return modifiedQB;
+    }
+
     @Override
     public final void writeTo(StreamOutput out) throws IOException {
         out.writeFloat(boost);

server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java

@@ -135,13 +135,15 @@ public List<QueryBuilder> must() {
 
     /**
      * Adds a query that <b>must</b> appear in the matching documents but will
-     * not contribute to scoring. No {@code null} value allowed.
+     * not contribute to scoring. If null value passed, then do nothing and return.
+     * @param filter the filter to add to the current ConstantScoreQuery
+     * @return query builder with filter combined
      */
-    public BoolQueryBuilder filter(QueryBuilder queryBuilder) {
-        if (queryBuilder == null) {
-            throw new IllegalArgumentException("inner bool query clause cannot be null");
+    public BoolQueryBuilder filter(QueryBuilder filter) {
+        if (validateFilterParams(filter) == false) {
+            return this;
         }
-        filterClauses.add(queryBuilder);
+        filterClauses.add(filter);
         return this;
     }
 

server/src/main/java/org/opensearch/index/query/ConstantScoreQueryBuilder.java

@@ -101,6 +101,22 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.endObject();
     }
 
+    /**
+     * Adds a filter to the current ConstantScoreQuery.
+     * @param filter the filter to add to the current ConstantScoreQuery
+     * @return query builder with filter combined
+     */
+    public ConstantScoreQueryBuilder filter(QueryBuilder filter) {
+        if (validateFilterParams(filter) == false) {
+            return this;
+        }
+        QueryBuilder filteredFilterBuilder = filterBuilder.filter(filter);
+        if (filteredFilterBuilder != filterBuilder) {
+            return new ConstantScoreQueryBuilder(filteredFilterBuilder);
+        }
+        return this;
+    }
+
     public static ConstantScoreQueryBuilder fromXContent(XContentParser parser) throws IOException {
         QueryBuilder query = null;
         boolean queryFound = false;

server/src/main/java/org/opensearch/index/query/QueryBuilder.java

@@ -47,6 +47,18 @@
 @PublicApi(since = "1.0.0")
 public interface QueryBuilder extends NamedWriteable, ToXContentObject, Rewriteable<QueryBuilder> {
 
+    /**
+     * This function combines a filter with a query builder. If the query builder itself has
+     * a filter we will combine the filter and return the query builder itself.
+     * If not we will use a bool query builder to combine the query builder and
+     * the filter and then return the bool query builder.
+     * If the filter is null we simply return the query builder without any operation.
+     *
+     * @param filter The null filter to be added to the existing filter.
+     * @return A QueryBuilder with the filter added to the existing filter.
+     */
+    QueryBuilder filter(QueryBuilder filter);
+
     /**
      * Converts this QueryBuilder to a lucene {@link Query}.
      * Returns {@code null} if this query should be ignored in the context of