[#587] Support accessing cluster with Cloud ID

README.md

@@ -48,7 +48,8 @@ elasticsearch_exporter --help
 
 | Argument                | Introduced in Version | Description | Default     |
 | --------                | --------------------- | ----------- | ----------- |
-| es.uri                  | 1.0.2                 | Address (host and port) of the Elasticsearch node we should connect to. This could be a local node (`localhost:9200`, for instance), or the address of a remote Elasticsearch server. When basic auth is needed, specify as: `<proto>://<user>:<password>@<host>:<port>`. E.G., `http://admin:pass@localhost:9200`. Special characters in the user credentials need to be URL-encoded. | http://localhost:9200 |
+| es.uri                  | 1.0.2                 | Address (host and port) of the Elasticsearch node we should connect to. This could be a local node (`localhost:9200`, for instance), or the address of a remote Elasticsearch server. When basic auth is needed, specify as: `<proto>://<user>:<password>@<host>:<port>`. E.G., `http://admin:pass@localhost:9200`. Special characters in the user credentials need to be URL-encoded. Cannot be used at the same time with `es.cloudid`. | |
+| es.cloudid | | Cloud ID of an Elasticsearch Service cluster. Please visit [the official document](https://www.elastic.co/guide/en/cloud/current/ec-cloud-id.html) for more information about Cloud ID. Cannot be used at the same time with `es.uri`. | |
 | es.all                  | 1.0.2                 | If true, query stats for all nodes in the cluster, rather than just the node we connect to.                             | false |
 | es.cluster_settings     | 1.1.0rc1              | If true, query stats for cluster settings. | false |
 | es.indices              | 1.0.2                 | If true, query stats for all indices in the cluster. | false |
@@ -73,6 +74,8 @@ For versions greater than `1.1.0rc1`, commandline parameters are specified with
 
 The API key used to connect can be set with the `ES_API_KEY` environment variable.
 
+We provides 2 ways to specify the address of the Elasticsearch node/cluster, `es.uri` and `es.cloudid`. When both of them are not specified, default address (`http://localhost:9200`) will be used.
+
 #### Elasticsearch 7.x security privileges
 
 Username and password can be passed either directly in the URI or through the `ES_USERNAME` and `ES_PASSWORD` environment variables.

main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/prometheus-community/elasticsearch_exporter/collector"
 	"github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo"
+	esurl "github.com/prometheus-community/elasticsearch_exporter/pkg/url"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/version"
@@ -54,7 +55,10 @@ func main() {
 			Default("/metrics").String()
 		esURI = kingpin.Flag("es.uri",
 			"HTTP API address of an Elasticsearch node.").
-			Default("http://localhost:9200").String()
+			Default("").String()
+		esCloudID = kingpin.Flag("es.cloudid",
+			"Cloud ID of an Elasticsearch Service cluster").
+			Default("").String()
 		esTimeout = kingpin.Flag("es.timeout",
 			"Timeout for trying to get stats from Elasticsearch.").
 			Default("5s").Duration()
@@ -120,10 +124,10 @@ func main() {
 
 	logger := getLogger(*logLevel, *logOutput, *logFormat)
 
-	esURL, err := url.Parse(*esURI)
+	esURL, err := esurl.GetEsURL(*esURI, *esCloudID)
 	if err != nil {
 		_ = level.Error(logger).Log(
-			"msg", "failed to parse es.uri",
+			"msg", "error while parsing URL",
 			"err", err,
 		)
 		os.Exit(1)

pkg/url/url.go

@@ -0,0 +1,78 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package url
+
+import (
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+const defaultEsURI = "http://localhost:9200"
+
+func GetEsURL(esURI, cloudID string) (*url.URL, error) {
+	var uri string
+	var err error
+	if len(esURI) == 0 && len(cloudID) == 0 {
+		uri = defaultEsURI
+	} else {
+		if len(esURI) > 0 && len(cloudID) > 0 {
+			return nil, errors.New("cannot create client: both es.uri and ES_CLOUD_ID are set")
+		}
+
+		if len(esURI) > 0 {
+			uri = esURI
+		}
+
+		if len(cloudID) > 0 {
+			uri, err = addrFromCloudID(cloudID)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	esURL, err := url.Parse(uri)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse url: %v", err)
+	}
+
+	return esURL, nil
+}
+
+// addrFromCloudID extracts the Elasticsearch URL from CloudID.
+// See: https://www.elastic.co/guide/en/cloud/current/ec-cloud-id.html
+//
+// This function was originally copied from https://github.com/elastic/go-elasticsearch/blob/8134a159aafedf58af2780ebb3a30ec1938956f3/elasticsearch.go#L365-L383
+func addrFromCloudID(input string) (string, error) {
+	var scheme = "https://"
+
+	values := strings.Split(input, ":")
+	if len(values) != 2 {
+		return "", fmt.Errorf("unexpected format: %q", input)
+	}
+	data, err := base64.StdEncoding.DecodeString(values[1])
+	if err != nil {
+		return "", err
+	}
+	parts := strings.Split(string(data), "$")
+
+	if len(parts) < 2 {
+		return "", fmt.Errorf("invalid encoded value: %s", parts)
+	}
+
+	return fmt.Sprintf("%s%s.%s", scheme, parts[1], parts[0]), nil
+}

pkg/url/url_test.go

@@ -0,0 +1,128 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package url
+
+import (
+	"encoding/base64"
+	"fmt"
+	"regexp"
+	"testing"
+)
+
+func TestAddrFromCloudID(t *testing.T) {
+	t.Run("Parse", func(t *testing.T) {
+		var testdata = []struct {
+			in  string
+			out string
+		}{
+			{
+				in:  "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid$kibana_uuid")),
+				out: "https://es_uuid.host",
+			},
+			{
+				in:  "name:" + base64.StdEncoding.EncodeToString([]byte("host:9243$es_uuid$kibana_uuid")),
+				out: "https://es_uuid.host:9243",
+			},
+			{
+				in:  "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid$")),
+				out: "https://es_uuid.host",
+			},
+			{
+				in:  "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid")),
+				out: "https://es_uuid.host",
+			},
+		}
+
+		for _, tt := range testdata {
+			actual, err := addrFromCloudID(tt.in)
+			if err != nil {
+				t.Errorf("Unexpected error: %s", err)
+			}
+			if actual != tt.out {
+				t.Errorf("Unexpected output, want=%q, got=%q", tt.out, actual)
+			}
+		}
+
+	})
+
+	t.Run("Invalid format", func(t *testing.T) {
+		input := "foobar"
+		_, err := addrFromCloudID(input)
+		if err == nil {
+			t.Errorf("Expected error for input %q, got %v", input, err)
+		}
+		match, _ := regexp.MatchString("unexpected format", err.Error())
+		if !match {
+			t.Errorf("Unexpected error string: %s", err)
+		}
+	})
+
+	t.Run("Invalid base64 value", func(t *testing.T) {
+		input := "foobar:xxxxx"
+		_, err := addrFromCloudID(input)
+		if err == nil {
+			t.Errorf("Expected error for input %q, got %v", input, err)
+		}
+		match, _ := regexp.MatchString("illegal base64 data", err.Error())
+		if !match {
+			t.Errorf("Unexpected error string: %s", err)
+		}
+	})
+}
+
+func TestGetEsURL(t *testing.T) {
+	cases := map[string]struct {
+		esURI       string
+		cloudID     string
+		expectedUrl string
+		expectErr   bool
+	}{
+		"success - url = defaultURI": {
+			expectedUrl: defaultEsURI,
+		},
+		"success - url = esURI": {
+			esURI:       "http://example.com:9200",
+			expectedUrl: "http://example.com:9200",
+		},
+		"success - url = parsed from cloudID": {
+			cloudID:     "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid$kibana_uuid")),
+			expectedUrl: "https://es_uuid.host",
+		},
+		"error - both esURI and cloudID are specified": {
+			esURI:     "http://example.com:9200",
+			cloudID:   "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid$kibana_uuid")),
+			expectErr: true,
+		},
+	}
+
+	for name, tc := range cases {
+		tc := tc
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			got, err := GetEsURL(tc.esURI, tc.cloudID)
+
+			if err != nil {
+				if !tc.expectErr {
+					t.Fatalf("Failed to prepare URL: %s", err)
+				}
+				return
+			}
+
+			url := fmt.Sprintf("%s://%s", got.Scheme, got.Host)
+			if url != tc.expectedUrl {
+				t.Fatalf("Fatiled to parse URL: got:%s, want: %s", url, tc.expectedUrl)
+			}
+		})
+	}
+}