Skip to content

Commit d6fa5f3

Browse files
SwethaGupthaSwetha Guptha
SwethaGuptha
authored and
Swetha Guptha
committed
URI path filtering support in cluster stats API (opensearch-project#15938)
* URI path filtering support in cluster stats API Signed-off-by: Swetha Guptha <[email protected]> (cherry picked from commit 1982427) Signed-off-by: Swetha Guptha <[email protected]>
1 parent 4ed6a47 commit d6fa5f3

File tree

14 files changed

+1483
-115
lines changed

14 files changed

+1483
-115
lines changed

Diff for: ‎CHANGELOG.md

+2
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2727
- Add method to return dynamic SecureTransportParameters from SecureTransportSettingsProvider interface ([#16387](https://github.com/opensearch-project/OpenSearch/pull/16387)
2828
- Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641))
2929
- [Star Tree - Search] Add support for metric aggregations with/without term query ([15289](https://github.com/opensearch-project/OpenSearch/pull/15289))
30+
- URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938))
31+
3032

3133
### Dependencies
3234
- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858))

Diff for: ‎qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStatsIT.java

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.upgrades;
10+
11+
import org.opensearch.Version;
12+
import org.opensearch.client.Request;
13+
import org.opensearch.client.Response;
14+
15+
import java.io.IOException;
16+
import java.util.Collections;
17+
import java.util.List;
18+
import java.util.Map;
19+
20+
public class ClusterStatsIT extends AbstractRollingTestCase {
21+
22+
private final List<String> nodeStatsMetrics = List.of("os", "process", "jvm", "fs", "plugins", "ingest", "network_types", "discovery_types", "packaging_types");
23+
24+
private final List<String> indicesStatsMetrics = List.of("shards", "docs", "store", "fielddata", "query_cache", "completion", "segments", "analysis", "mappings");
25+
26+
public void testClusterStats() throws IOException {
27+
Response response = client().performRequest(new Request("GET", "/_cluster/stats"));
28+
validateClusterStatsWithFilterResponse(response, nodeStatsMetrics, indicesStatsMetrics);
29+
if (AbstractRollingTestCase.UPGRADE_FROM_VERSION.onOrAfter(Version.V_2_18_0) || (
30+
CLUSTER_TYPE == ClusterType.UPGRADED && Version.CURRENT.onOrAfter(Version.V_2_18_0))) {
31+
response = client().performRequest(new Request("GET", "/_cluster/stats/os/nodes/_all"));
32+
validateClusterStatsWithFilterResponse(response, List.of("os"), Collections.emptyList());
33+
response = client().performRequest(new Request("GET", "/_cluster/stats/indices/mappings/nodes/_all"));
34+
validateClusterStatsWithFilterResponse(response, Collections.emptyList(), List.of("mappings"));
35+
response = client().performRequest(new Request("GET", "/_cluster/stats/os,indices/mappings/nodes/_all"));
36+
validateClusterStatsWithFilterResponse(response, List.of("os"), List.of("mappings"));
37+
}
38+
}
39+
40+
private void validateClusterStatsWithFilterResponse(Response response, List<String> requestedNodesStatsMetrics, List<String> requestedIndicesStatsMetrics) throws IOException {
41+
assertEquals(200, response.getStatusLine().getStatusCode());
42+
Map<String, Object> entity = entityAsMap(response);
43+
if (requestedNodesStatsMetrics != null && !requestedNodesStatsMetrics.isEmpty()) {
44+
assertTrue(entity.containsKey("nodes"));
45+
Map<?, ?> nodesStats = (Map<?, ?>) entity.get("nodes");
46+
for (String metric : nodeStatsMetrics) {
47+
if (requestedNodesStatsMetrics.contains(metric)) {
48+
assertTrue(nodesStats.containsKey(metric));
49+
} else {
50+
assertFalse(nodesStats.containsKey(metric));
51+
}
52+
}
53+
}
54+
55+
if (requestedIndicesStatsMetrics != null && !requestedIndicesStatsMetrics.isEmpty()) {
56+
assertTrue(entity.containsKey("indices"));
57+
Map<?, ?> indicesStats = (Map<?, ?>) entity.get("indices");
58+
for (String metric : indicesStatsMetrics) {
59+
if (requestedIndicesStatsMetrics.contains(metric)) {
60+
assertTrue(indicesStats.containsKey(metric));
61+
} else {
62+
assertFalse(indicesStats.containsKey(metric));
63+
}
64+
}
65+
}
66+
}
67+
}

Diff for: ‎server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIT.java

+376
Large diffs are not rendered by default.

Diff for: ‎server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsIndices.java

+80-29
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
package org.opensearch.action.admin.cluster.stats;
3434

35+
import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.IndexMetric;
3536
import org.opensearch.action.admin.indices.stats.CommonStats;
3637
import org.opensearch.common.annotation.PublicApi;
3738
import org.opensearch.core.xcontent.ToXContentFragment;
@@ -47,6 +48,7 @@
4748
import java.util.HashMap;
4849
import java.util.List;
4950
import java.util.Map;
51+
import java.util.Set;
5052

5153
/**
5254
* Cluster Stats per index
@@ -68,14 +70,23 @@ public class ClusterStatsIndices implements ToXContentFragment {
6870
private MappingStats mappings;
6971

7072
public ClusterStatsIndices(List<ClusterStatsNodeResponse> nodeResponses, MappingStats mappingStats, AnalysisStats analysisStats) {
71-
Map<String, ShardStats> countsPerIndex = new HashMap<>();
73+
this(Set.of(IndexMetric.values()), nodeResponses, mappingStats, analysisStats);
74+
75+
}
7276

73-
this.docs = new DocsStats();
74-
this.store = new StoreStats();
75-
this.fieldData = new FieldDataStats();
76-
this.queryCache = new QueryCacheStats();
77-
this.completion = new CompletionStats();
78-
this.segments = new SegmentsStats();
77+
public ClusterStatsIndices(
78+
Set<IndexMetric> indicesMetrics,
79+
List<ClusterStatsNodeResponse> nodeResponses,
80+
MappingStats mappingStats,
81+
AnalysisStats analysisStats
82+
) {
83+
Map<String, ShardStats> countsPerIndex = new HashMap<>();
84+
this.docs = indicesMetrics.contains(IndexMetric.DOCS) ? new DocsStats() : null;
85+
this.store = indicesMetrics.contains(IndexMetric.STORE) ? new StoreStats() : null;
86+
this.fieldData = indicesMetrics.contains(IndexMetric.FIELDDATA) ? new FieldDataStats() : null;
87+
this.queryCache = indicesMetrics.contains(IndexMetric.QUERY_CACHE) ? new QueryCacheStats() : null;
88+
this.completion = indicesMetrics.contains(IndexMetric.COMPLETION) ? new CompletionStats() : null;
89+
this.segments = indicesMetrics.contains(IndexMetric.SEGMENTS) ? new SegmentsStats() : null;
7990

8091
for (ClusterStatsNodeResponse r : nodeResponses) {
8192
// Aggregated response from the node
@@ -92,12 +103,24 @@ public ClusterStatsIndices(List<ClusterStatsNodeResponse> nodeResponses, Mapping
92103
}
93104
}
94105

95-
docs.add(r.getAggregatedNodeLevelStats().commonStats.docs);
96-
store.add(r.getAggregatedNodeLevelStats().commonStats.store);
97-
fieldData.add(r.getAggregatedNodeLevelStats().commonStats.fieldData);
98-
queryCache.add(r.getAggregatedNodeLevelStats().commonStats.queryCache);
99-
completion.add(r.getAggregatedNodeLevelStats().commonStats.completion);
100-
segments.add(r.getAggregatedNodeLevelStats().commonStats.segments);
106+
if (indicesMetrics.contains(IndexMetric.DOCS)) {
107+
docs.add(r.getAggregatedNodeLevelStats().commonStats.docs);
108+
}
109+
if (indicesMetrics.contains(IndexMetric.STORE)) {
110+
store.add(r.getAggregatedNodeLevelStats().commonStats.store);
111+
}
112+
if (indicesMetrics.contains(IndexMetric.FIELDDATA)) {
113+
fieldData.add(r.getAggregatedNodeLevelStats().commonStats.fieldData);
114+
}
115+
if (indicesMetrics.contains(IndexMetric.QUERY_CACHE)) {
116+
queryCache.add(r.getAggregatedNodeLevelStats().commonStats.queryCache);
117+
}
118+
if (indicesMetrics.contains(IndexMetric.COMPLETION)) {
119+
completion.add(r.getAggregatedNodeLevelStats().commonStats.completion);
120+
}
121+
if (indicesMetrics.contains(IndexMetric.SEGMENTS)) {
122+
segments.add(r.getAggregatedNodeLevelStats().commonStats.segments);
123+
}
101124
} else {
102125
// Default response from the node
103126
for (org.opensearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) {
@@ -113,21 +136,35 @@ public ClusterStatsIndices(List<ClusterStatsNodeResponse> nodeResponses, Mapping
113136

114137
if (shardStats.getShardRouting().primary()) {
115138
indexShardStats.primaries++;
116-
docs.add(shardCommonStats.docs);
139+
if (indicesMetrics.contains(IndexMetric.DOCS)) {
140+
docs.add(shardCommonStats.docs);
141+
}
142+
}
143+
if (indicesMetrics.contains(IndexMetric.STORE)) {
144+
store.add(shardCommonStats.store);
145+
}
146+
if (indicesMetrics.contains(IndexMetric.FIELDDATA)) {
147+
fieldData.add(shardCommonStats.fieldData);
148+
}
149+
if (indicesMetrics.contains(IndexMetric.QUERY_CACHE)) {
150+
queryCache.add(shardCommonStats.queryCache);
151+
}
152+
if (indicesMetrics.contains(IndexMetric.COMPLETION)) {
153+
completion.add(shardCommonStats.completion);
154+
}
155+
if (indicesMetrics.contains(IndexMetric.SEGMENTS)) {
156+
segments.add(shardCommonStats.segments);
117157
}
118-
store.add(shardCommonStats.store);
119-
fieldData.add(shardCommonStats.fieldData);
120-
queryCache.add(shardCommonStats.queryCache);
121-
completion.add(shardCommonStats.completion);
122-
segments.add(shardCommonStats.segments);
123158
}
124159
}
125160
}
126161

127-
shards = new ShardStats();
128162
indexCount = countsPerIndex.size();
129-
for (final ShardStats indexCountsCursor : countsPerIndex.values()) {
130-
shards.addIndexShardCount(indexCountsCursor);
163+
if (indicesMetrics.contains(IndexMetric.SHARDS)) {
164+
shards = new ShardStats();
165+
for (final ShardStats indexCountsCursor : countsPerIndex.values()) {
166+
shards.addIndexShardCount(indexCountsCursor);
167+
}
131168
}
132169

133170
this.mappings = mappingStats;
@@ -186,13 +223,27 @@ static final class Fields {
186223
@Override
187224
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
188225
builder.field(Fields.COUNT, indexCount);
189-
shards.toXContent(builder, params);
190-
docs.toXContent(builder, params);
191-
store.toXContent(builder, params);
192-
fieldData.toXContent(builder, params);
193-
queryCache.toXContent(builder, params);
194-
completion.toXContent(builder, params);
195-
segments.toXContent(builder, params);
226+
if (shards != null) {
227+
shards.toXContent(builder, params);
228+
}
229+
if (docs != null) {
230+
docs.toXContent(builder, params);
231+
}
232+
if (store != null) {
233+
store.toXContent(builder, params);
234+
}
235+
if (fieldData != null) {
236+
fieldData.toXContent(builder, params);
237+
}
238+
if (queryCache != null) {
239+
queryCache.toXContent(builder, params);
240+
}
241+
if (completion != null) {
242+
completion.toXContent(builder, params);
243+
}
244+
if (segments != null) {
245+
segments.toXContent(builder, params);
246+
}
196247
if (mappings != null) {
197248
mappings.toXContent(builder, params);
198249
}

Diff for: ‎server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodes.java

+73-32
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.opensearch.action.admin.cluster.node.info.NodeInfo;
3737
import org.opensearch.action.admin.cluster.node.info.PluginsAndModules;
3838
import org.opensearch.action.admin.cluster.node.stats.NodeStats;
39+
import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest.Metric;
3940
import org.opensearch.cluster.node.DiscoveryNode;
4041
import org.opensearch.cluster.node.DiscoveryNodeRole;
4142
import org.opensearch.common.annotation.PublicApi;
@@ -89,10 +90,29 @@ public class ClusterStatsNodes implements ToXContentFragment {
8990
private final PackagingTypes packagingTypes;
9091
private final IngestStats ingestStats;
9192

93+
public static final Set<Metric> NODE_STATS_METRICS = Set.of(
94+
// Stats computed from node info and node stat
95+
Metric.OS,
96+
Metric.JVM,
97+
// Stats computed from node stat
98+
Metric.FS,
99+
Metric.PROCESS,
100+
Metric.INGEST,
101+
// Stats computed from node info
102+
Metric.PLUGINS,
103+
Metric.NETWORK_TYPES,
104+
Metric.DISCOVERY_TYPES,
105+
Metric.PACKAGING_TYPES
106+
);
107+
92108
ClusterStatsNodes(List<ClusterStatsNodeResponse> nodeResponses) {
109+
this(Set.of(Metric.values()), nodeResponses);
110+
}
111+
112+
ClusterStatsNodes(Set<Metric> requestedMetrics, List<ClusterStatsNodeResponse> nodeResponses) {
93113
this.versions = new HashSet<>();
94-
this.fs = new FsInfo.Path();
95-
this.plugins = new HashSet<>();
114+
this.fs = requestedMetrics.contains(ClusterStatsRequest.Metric.FS) ? new FsInfo.Path() : null;
115+
this.plugins = requestedMetrics.contains(ClusterStatsRequest.Metric.PLUGINS) ? new HashSet<>() : null;
96116

97117
Set<InetAddress> seenAddresses = new HashSet<>(nodeResponses.size());
98118
List<NodeInfo> nodeInfos = new ArrayList<>(nodeResponses.size());
@@ -101,26 +121,29 @@ public class ClusterStatsNodes implements ToXContentFragment {
101121
nodeInfos.add(nodeResponse.nodeInfo());
102122
nodeStats.add(nodeResponse.nodeStats());
103123
this.versions.add(nodeResponse.nodeInfo().getVersion());
104-
this.plugins.addAll(nodeResponse.nodeInfo().getInfo(PluginsAndModules.class).getPluginInfos());
124+
if (requestedMetrics.contains(ClusterStatsRequest.Metric.PLUGINS)) {
125+
this.plugins.addAll(nodeResponse.nodeInfo().getInfo(PluginsAndModules.class).getPluginInfos());
126+
}
105127

106128
// now do the stats that should be deduped by hardware (implemented by ip deduping)
107129
TransportAddress publishAddress = nodeResponse.nodeInfo().getInfo(TransportInfo.class).address().publishAddress();
108130
final InetAddress inetAddress = publishAddress.address().getAddress();
109131
if (!seenAddresses.add(inetAddress)) {
110132
continue;
111133
}
112-
if (nodeResponse.nodeStats().getFs() != null) {
134+
if (requestedMetrics.contains(ClusterStatsRequest.Metric.FS) && nodeResponse.nodeStats().getFs() != null) {
113135
this.fs.add(nodeResponse.nodeStats().getFs().getTotal());
114136
}
115137
}
138+
116139
this.counts = new Counts(nodeInfos);
117-
this.os = new OsStats(nodeInfos, nodeStats);
118-
this.process = new ProcessStats(nodeStats);
119-
this.jvm = new JvmStats(nodeInfos, nodeStats);
120-
this.networkTypes = new NetworkTypes(nodeInfos);
121-
this.discoveryTypes = new DiscoveryTypes(nodeInfos);
122-
this.packagingTypes = new PackagingTypes(nodeInfos);
123-
this.ingestStats = new IngestStats(nodeStats);
140+
this.networkTypes = requestedMetrics.contains(ClusterStatsRequest.Metric.NETWORK_TYPES) ? new NetworkTypes(nodeInfos) : null;
141+
this.discoveryTypes = requestedMetrics.contains(ClusterStatsRequest.Metric.DISCOVERY_TYPES) ? new DiscoveryTypes(nodeInfos) : null;
142+
this.packagingTypes = requestedMetrics.contains(ClusterStatsRequest.Metric.PACKAGING_TYPES) ? new PackagingTypes(nodeInfos) : null;
143+
this.ingestStats = requestedMetrics.contains(ClusterStatsRequest.Metric.INGEST) ? new IngestStats(nodeStats) : null;
144+
this.process = requestedMetrics.contains(ClusterStatsRequest.Metric.PROCESS) ? new ProcessStats(nodeStats) : null;
145+
this.os = requestedMetrics.contains(ClusterStatsRequest.Metric.OS) ? new OsStats(nodeInfos, nodeStats) : null;
146+
this.jvm = requestedMetrics.contains(ClusterStatsRequest.Metric.JVM) ? new JvmStats(nodeInfos, nodeStats) : null;
124147
}
125148

126149
public Counts getCounts() {
@@ -179,36 +202,54 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
179202
}
180203
builder.endArray();
181204

182-
builder.startObject(Fields.OS);
183-
os.toXContent(builder, params);
184-
builder.endObject();
205+
if (os != null) {
206+
builder.startObject(Fields.OS);
207+
os.toXContent(builder, params);
208+
builder.endObject();
209+
}
185210

186-
builder.startObject(Fields.PROCESS);
187-
process.toXContent(builder, params);
188-
builder.endObject();
211+
if (process != null) {
212+
builder.startObject(Fields.PROCESS);
213+
process.toXContent(builder, params);
214+
builder.endObject();
215+
}
189216

190-
builder.startObject(Fields.JVM);
191-
jvm.toXContent(builder, params);
192-
builder.endObject();
217+
if (jvm != null) {
218+
builder.startObject(Fields.JVM);
219+
jvm.toXContent(builder, params);
220+
builder.endObject();
221+
}
193222

194-
builder.field(Fields.FS);
195-
fs.toXContent(builder, params);
223+
if (fs != null) {
224+
builder.field(Fields.FS);
225+
fs.toXContent(builder, params);
226+
}
196227

197-
builder.startArray(Fields.PLUGINS);
198-
for (PluginInfo pluginInfo : plugins) {
199-
pluginInfo.toXContent(builder, params);
228+
if (plugins != null) {
229+
builder.startArray(Fields.PLUGINS);
230+
for (PluginInfo pluginInfo : plugins) {
231+
pluginInfo.toXContent(builder, params);
232+
}
233+
builder.endArray();
200234
}
201-
builder.endArray();
202235

203-
builder.startObject(Fields.NETWORK_TYPES);
204-
networkTypes.toXContent(builder, params);
205-
builder.endObject();
236+
if (networkTypes != null) {
237+
builder.startObject(Fields.NETWORK_TYPES);
238+
networkTypes.toXContent(builder, params);
239+
builder.endObject();
240+
}
206241

207-
discoveryTypes.toXContent(builder, params);
242+
if (discoveryTypes != null) {
243+
discoveryTypes.toXContent(builder, params);
244+
}
208245

209-
packagingTypes.toXContent(builder, params);
246+
if (packagingTypes != null) {
247+
packagingTypes.toXContent(builder, params);
248+
}
210249

211-
ingestStats.toXContent(builder, params);
250+
if (ingestStats != null) {
251+
ingestStats.toXContent(builder, params);
252+
}
212253

213254
return builder;
214255
}

0 commit comments

Comments
 (0)