Skip to content

Commit 0610f67

Browse files
lukas-vlceklukas-vlcek
committed
Bring indices level metrics back
Closes vvanholl#115
1 parent ea03684 commit 0610f67

29 files changed

+1802
-111
lines changed

‎.gitignore

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
# I do not know why this file is left in the <ROOT> after `gradle build` or `gradle check`
2-
# and why `gradle clean` does not delete it.
3-
# See https://discuss.elastic.co/t/leftovers-after-integtestrunner-in-root/152610
1+
# There is a file left in the <ROOT> after `gradle build` or `gradle check`. This is expected.
2+
# https://discuss.elastic.co/t/leftovers-after-integtestrunner-in-root/152610
43
\.local*-integTestRunner-execution-times.log
54

65
# Eclipse

‎checkstyle.xml

+8
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ page at http://checkstyle.sourceforge.net/config.html -->
5050
5151
-->
5252

53+
<module name="SuppressWarningsHolder" />
54+
<module name="SuppressWarnings">
55+
<property name="id" value="checkstyle:suppresswarnings"/>
56+
</module>
57+
5358
<module name="RedundantImport">
5459
<!-- Checks for redundant import statements. -->
5560
<property name="severity" value="warning"/>
@@ -322,4 +327,7 @@ page at http://checkstyle.sourceforge.net/config.html -->
322327
</module>
323328

324329
</module>
330+
331+
<module name="SuppressWarningsFilter"/>
332+
325333
</module>

‎src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java

+224-15
Large diffs are not rendered by default.

‎src/main/java/org/elasticsearch/action/NodePrometheusMetricsResponse.java

+29-2
Original file line numberDiff line numberDiff line change
@@ -19,26 +19,36 @@
1919

2020
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
2121
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
22+
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
23+
import org.elasticsearch.action.admin.indices.stats.PackageAccessHelper;
24+
import org.elasticsearch.action.admin.indices.stats.ShardStats;
25+
import org.elasticsearch.action.support.broadcast.BroadcastResponse;
26+
import org.elasticsearch.common.Nullable;
2227
import org.elasticsearch.common.io.stream.StreamInput;
2328
import org.elasticsearch.common.io.stream.StreamOutput;
2429
import java.io.IOException;
30+
import java.util.Arrays;
2531

2632
/**
2733
* Action response class for Prometheus Exporter plugin.
2834
*/
2935
public class NodePrometheusMetricsResponse extends ActionResponse {
3036
private ClusterHealthResponse clusterHealth;
3137
private NodeStats nodeStats;
38+
@Nullable
39+
private IndicesStatsResponse indicesStats;
3240

3341
public NodePrometheusMetricsResponse() {
3442
}
3543

36-
public NodePrometheusMetricsResponse(ClusterHealthResponse clusterHealth, NodeStats nodesStats) {
44+
public NodePrometheusMetricsResponse(ClusterHealthResponse clusterHealth, NodeStats nodesStats,
45+
@Nullable IndicesStatsResponse indicesStats) {
3746
this.clusterHealth = clusterHealth;
3847
this.nodeStats = nodesStats;
48+
this.indicesStats = indicesStats;
3949
}
4050

41-
public static NodePrometheusMetricsResponse readNodePrometheusMetrics(StreamInput in) throws IOException {
51+
public NodePrometheusMetricsResponse readNodePrometheusMetrics(StreamInput in) throws IOException {
4252
NodePrometheusMetricsResponse metrics = new NodePrometheusMetricsResponse();
4353
metrics.readFrom(in);
4454
return metrics;
@@ -52,17 +62,34 @@ public NodeStats getNodeStats() {
5262
return this.nodeStats;
5363
}
5464

65+
@Nullable
66+
public IndicesStatsResponse getIndicesStats() {
67+
return this.indicesStats;
68+
}
69+
5570
@Override
5671
public void readFrom(StreamInput in) throws IOException {
5772
super.readFrom(in);
5873
clusterHealth = ClusterHealthResponse.readResponseFrom(in);
5974
nodeStats = NodeStats.readNodeStats(in);
75+
BroadcastResponse br = new BroadcastResponse();
76+
br.readFrom(in);
77+
ShardStats[] ss = in.readArray(ShardStats::readShardStats, (size) -> new ShardStats[size]);
78+
indicesStats = PackageAccessHelper.createIndicesStatsResponse(
79+
ss, br.getTotalShards(), br.getSuccessfulShards(), br.getFailedShards(),
80+
Arrays.asList(br.getShardFailures())
81+
);
6082
}
6183

6284
@Override
6385
public void writeTo(StreamOutput out) throws IOException {
6486
super.writeTo(out);
6587
clusterHealth.writeTo(out);
6688
nodeStats.writeTo(out);
89+
if (indicesStats != null) {
90+
//indicesStats.writeTo(out);
91+
((BroadcastResponse) indicesStats).writeTo(out);
92+
out.writeArray(indicesStats.getShards());
93+
}
6794
}
6895
}

‎src/main/java/org/elasticsearch/action/TransportNodePrometheusMetricsAction.java

+46-9
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,21 @@
1717

1818
package org.elasticsearch.action;
1919

20+
import static org.compuscene.metrics.prometheus.PrometheusMetricsCollector.PROMETHEUS_INDICES;
21+
2022
import org.elasticsearch.ElasticsearchException;
2123
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
2224
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
2325
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
2426
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
27+
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
28+
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
2529
import org.elasticsearch.action.support.ActionFilters;
2630
import org.elasticsearch.action.support.HandledTransportAction;
2731
import org.elasticsearch.client.Client;
32+
import org.elasticsearch.client.Requests;
2833
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
34+
import org.elasticsearch.common.Nullable;
2935
import org.elasticsearch.common.inject.Inject;
3036
import org.elasticsearch.common.settings.Settings;
3137
import org.elasticsearch.threadpool.ThreadPool;
@@ -56,12 +62,48 @@ private class AsyncAction {
5662
private final ActionListener<NodePrometheusMetricsResponse> listener;
5763
private final ClusterHealthRequest healthRequest;
5864
private final NodesStatsRequest nodesStatsRequest;
65+
private final IndicesStatsRequest indicesStatsRequest;
5966
private ClusterHealthResponse clusterHealthResponse;
67+
private NodesStatsResponse nodesStatsResponse;
68+
69+
private AsyncAction(ActionListener<NodePrometheusMetricsResponse> listener) {
70+
this.listener = listener;
71+
72+
// Note: when using ClusterHealthRequest in Java, it pulls data at the shards level, according to ES source
73+
// code comment this is "so it is backward compatible with the transport client behaviour".
74+
// hence we are explicit about ClusterHealthRequest level and do not rely on defaults.
75+
// https://www.elastic.co/guide/en/elasticsearch/reference/6.4/cluster-health.html#request-params
76+
this.healthRequest = Requests.clusterHealthRequest().local(true);
77+
this.healthRequest.level(ClusterHealthRequest.Level.SHARDS);
78+
79+
this.nodesStatsRequest = Requests.nodesStatsRequest("_local").clear().all();
80+
// Note: this request is not "node-specific", it does not support any "_local" notion
81+
// it is broad-casted to all cluster nodes.
82+
this.indicesStatsRequest = new IndicesStatsRequest();
83+
}
84+
85+
private ActionListener<IndicesStatsResponse> indicesStatsResponseActionListener =
86+
new ActionListener<IndicesStatsResponse>() {
87+
@Override
88+
public void onResponse(IndicesStatsResponse indicesStatsResponse) {
89+
listener.onResponse(buildResponse(clusterHealthResponse, nodesStatsResponse, indicesStatsResponse));
90+
}
91+
92+
@Override
93+
public void onFailure(Exception e) {
94+
listener.onFailure(new ElasticsearchException("Indices stats request failed", e));
95+
}
96+
};
6097

6198
private ActionListener<NodesStatsResponse> nodesStatsResponseActionListener = new ActionListener<NodesStatsResponse>() {
6299
@Override
63100
public void onResponse(NodesStatsResponse nodeStats) {
64-
listener.onResponse(buildResponse(clusterHealthResponse, nodeStats));
101+
if (PROMETHEUS_INDICES.get(settings)) {
102+
nodesStatsResponse = nodeStats;
103+
client.admin().indices().stats(indicesStatsRequest, indicesStatsResponseActionListener);
104+
} else {
105+
listener.onResponse(buildResponse(clusterHealthResponse, nodeStats, null));
106+
}
65107
}
66108

67109
@Override
@@ -84,20 +126,15 @@ public void onFailure(Exception e) {
84126
}
85127
};
86128

87-
private AsyncAction(ActionListener<NodePrometheusMetricsResponse> listener) {
88-
this.listener = listener;
89-
this.healthRequest = new ClusterHealthRequest();
90-
this.nodesStatsRequest = new NodesStatsRequest("_local").all();
91-
}
92-
93129
private void start() {
94130
client.admin().cluster().health(healthRequest, clusterHealthResponseActionListener);
95131
}
96132

97133
protected NodePrometheusMetricsResponse buildResponse(ClusterHealthResponse clusterHealth,
98-
NodesStatsResponse nodesStats) {
134+
NodesStatsResponse nodesStats,
135+
@Nullable IndicesStatsResponse indicesStats) {
99136
NodePrometheusMetricsResponse response = new NodePrometheusMetricsResponse(clusterHealth,
100-
nodesStats.getNodes().get(0));
137+
nodesStats.getNodes().get(0), indicesStats);
101138
if (logger.isTraceEnabled()) {
102139
logger.trace("Return response: [{}]", response);
103140
}

‎src/main/java/org/elasticsearch/action/admin/indices/stats/PackageAccessHelper.java

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright [2016] [Vincent VAN HOLLEBEKE]
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
*/
17+
package org.elasticsearch.action.admin.indices.stats;
18+
19+
import org.elasticsearch.action.support.DefaultShardOperationFailedException;
20+
21+
import java.util.List;
22+
23+
/**
24+
* Utility methods.
25+
*/
26+
public class PackageAccessHelper {
27+
28+
/**
29+
* Shortcut to IndicesStatsResponse constructor which has package access restriction.
30+
*
31+
* @param shards The shards stats.
32+
* @param totalShards The total shards this request ran against.
33+
* @param successfulShards The successful shards this request was executed on.
34+
* @param failedShards The failed shards this request was executed on.
35+
* @param shardFailures The list of shard failures exception.
36+
* @return new instance of IndicesStatsResponse.
37+
*/
38+
public static IndicesStatsResponse createIndicesStatsResponse(ShardStats[] shards, int totalShards,
39+
int successfulShards, int failedShards,
40+
List<DefaultShardOperationFailedException> shardFailures) {
41+
return new IndicesStatsResponse(shards, totalShards, successfulShards, failedShards, shardFailures);
42+
}
43+
}

‎src/main/java/org/elasticsearch/action/admin/indices/stats/package-info.java

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright [2016] [Vincent VAN HOLLEBEKE]
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
*/
17+
18+
/**
19+
* Utility classes.
20+
*/
21+
package org.elasticsearch.action.admin.indices.stats;

‎src/main/java/org/elasticsearch/plugin/prometheus/PrometheusExporterPlugin.java

-8
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,8 @@
2525
import org.elasticsearch.action.TransportNodePrometheusMetricsAction;
2626
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2727
import org.elasticsearch.cluster.node.DiscoveryNodes;
28-
import org.elasticsearch.common.inject.Inject;
2928
import org.elasticsearch.common.logging.Loggers;
3029
import org.elasticsearch.common.settings.*;
31-
import org.elasticsearch.index.IndexModule;
3230
import org.elasticsearch.plugins.ActionPlugin;
3331
import org.elasticsearch.plugins.Plugin;
3432
import org.elasticsearch.rest.RestController;
@@ -44,16 +42,10 @@
4442
public class PrometheusExporterPlugin extends Plugin implements ActionPlugin {
4543
private static final Logger logger = Loggers.getLogger(PrometheusExporterPlugin.class);
4644

47-
@Inject
4845
public PrometheusExporterPlugin() {
4946
logger.info("starting Prometheus exporter plugin");
5047
}
5148

52-
@Override
53-
public void onIndexModule(IndexModule indexModule) {
54-
super.onIndexModule(indexModule);
55-
}
56-
5749
@Override
5850
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
5951
return Arrays.asList(

‎src/main/java/org/elasticsearch/rest/prometheus/RestPrometheusMetricsAction.java

+7-8
Original file line numberDiff line numberDiff line change
@@ -20,26 +20,22 @@
2020
import static org.elasticsearch.action.NodePrometheusMetricsAction.INSTANCE;
2121
import static org.elasticsearch.rest.RestRequest.Method.GET;
2222

23-
import org.apache.logging.log4j.Logger;
2423
import org.compuscene.metrics.prometheus.PrometheusMetricsCatalog;
2524
import org.compuscene.metrics.prometheus.PrometheusMetricsCollector;
2625
import org.elasticsearch.action.NodePrometheusMetricsRequest;
2726
import org.elasticsearch.action.NodePrometheusMetricsResponse;
2827
import org.elasticsearch.client.node.NodeClient;
2928
import org.elasticsearch.common.inject.Inject;
30-
import org.elasticsearch.common.logging.ESLoggerFactory;
3129
import org.elasticsearch.common.settings.Settings;
3230
import org.elasticsearch.rest.*;
3331
import org.elasticsearch.rest.action.RestResponseListener;
3432

35-
import java.io.IOException;
3633
import java.util.Locale;
3734

3835
/**
3936
* REST action class for Prometheus Exporter plugin.
4037
*/
4138
public class RestPrometheusMetricsAction extends BaseRestHandler {
42-
private static final Logger logger = ESLoggerFactory.getLogger(RestPrometheusMetricsAction.class.getSimpleName());
4339

4440
@Inject
4541
public RestPrometheusMetricsAction(Settings settings, RestController controller) {
@@ -52,8 +48,10 @@ public String getName() {
5248
return "prometheus_metrics_action";
5349
}
5450

51+
// This method does not throw any IOException because there are no request parameters to be parsed
52+
// and processed. This may change in the future.
5553
@Override
56-
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
54+
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
5755
if (logger.isTraceEnabled()) {
5856
logger.trace(String.format(Locale.ENGLISH, "Received request for Prometheus metrics from %s",
5957
request.getRemoteAddress().toString()));
@@ -63,8 +61,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
6361

6462
return channel -> client.execute(INSTANCE, metricsRequest,
6563
new RestResponseListener<NodePrometheusMetricsResponse>(channel) {
66-
@Override
67-
public RestResponse buildResponse(NodePrometheusMetricsResponse response) throws Exception {
64+
65+
@Override
66+
public RestResponse buildResponse(NodePrometheusMetricsResponse response) throws Exception {
6867
String clusterName = response.getClusterHealth().getClusterName();
6968
String nodeName = response.getNodeStats().getNode().getName();
7069
String nodeId = response.getNodeStats().getNode().getId();
@@ -75,7 +74,7 @@ public RestResponse buildResponse(NodePrometheusMetricsResponse response) throws
7574
PrometheusMetricsCatalog catalog = new PrometheusMetricsCatalog(clusterName, nodeName, nodeId, "es_");
7675
PrometheusMetricsCollector collector = new PrometheusMetricsCollector(settings, catalog);
7776
collector.registerMetrics();
78-
collector.updateMetrics(response.getClusterHealth(), response.getNodeStats());
77+
collector.updateMetrics(response.getClusterHealth(), response.getNodeStats(), response.getIndicesStats());
7978
return new BytesRestResponse(RestStatus.OK, collector.getCatalog().toTextFormat());
8079
}
8180
});

‎src/test/java/org/elasticsearch/rest/PrometheusRestHandlerClientYamlTestSuiteIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
2424

2525
/**
26-
* Needed class to enable esplugin rest api tests
26+
* Needed class to enable esplugin rest api tests.
2727
*/
2828
public class PrometheusRestHandlerClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
2929

‎src/test/resources/rest-api-spec/test/resthandler/10_basic.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
2-
# We sent number of cluster nodes in build.gradle file. See `integTestCluster.numNodes`
3-
"Verify we have cluster with 2 nodes":
2+
# We set number of cluster nodes in build.gradle file. See `integTestCluster.numNodes`
3+
"Verify we have cluster with expectd number of nodes":
44
- do:
55
cluster.health:
66
wait_for_nodes: 2

‎src/test/resources/rest-api-spec/test/resthandler/20_metrics.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# At this moment this is mostly smoke test only. The goal is the have a test to verify
22
# that metrics are relevant to specific ES node only but it seems that
3-
# current esplugin support does not easily allow for communication with specific ES nodes
3+
# current esplugin does not support for communication with specific ES nodes
44
# only (see below for more details).
55
#
66
# Going forward one possible solution would be to implement this in Java similarly

0 commit comments

Comments
 (0)