From a465f0a02741083441ca6600dfea85bd288d96c1 Mon Sep 17 00:00:00 2001 From: "pakiosann@gmail.com" Date: Tue, 21 Jun 2022 23:53:30 +0900 Subject: [PATCH 1/3] Support accessing cluster with CloudID Signed-off-by: Kazuma (Pakio) Arimura --- main.go | 7 +-- pkg/url/url.go | 78 +++++++++++++++++++++++++++ pkg/url/url_test.go | 128 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 pkg/url/url.go create mode 100644 pkg/url/url_test.go diff --git a/main.go b/main.go index 89236ceb..5bd7fdeb 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ import ( "github.com/go-kit/log/level" "github.com/prometheus-community/elasticsearch_exporter/collector" "github.com/prometheus-community/elasticsearch_exporter/pkg/clusterinfo" + esurl "github.com/prometheus-community/elasticsearch_exporter/pkg/url" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/version" @@ -54,7 +55,7 @@ func main() { Default("/metrics").String() esURI = kingpin.Flag("es.uri", "HTTP API address of an Elasticsearch node."). - Default("http://localhost:9200").String() + Default("").String() esTimeout = kingpin.Flag("es.timeout", "Timeout for trying to get stats from Elasticsearch."). Default("5s").Duration() @@ -120,10 +121,10 @@ func main() { logger := getLogger(*logLevel, *logOutput, *logFormat) - esURL, err := url.Parse(*esURI) + esURL, err := esurl.GetEsURL(*esURI, os.Getenv("ES_CLOUD_ID")) if err != nil { _ = level.Error(logger).Log( - "msg", "failed to parse es.uri", + "msg", "error while parsing URL", "err", err, ) os.Exit(1) diff --git a/pkg/url/url.go b/pkg/url/url.go new file mode 100644 index 00000000..11948e14 --- /dev/null +++ b/pkg/url/url.go @@ -0,0 +1,78 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package url + +import ( + "encoding/base64" + "errors" + "fmt" + "net/url" + "strings" +) + +const defaultEsURI = "http://localhost:9200" + +func GetEsURL(esURI, cloudID string) (*url.URL, error) { + var uri string + var err error + if len(esURI) == 0 && len(cloudID) == 0 { + uri = defaultEsURI + } else { + if len(esURI) > 0 && len(cloudID) > 0 { + return nil, errors.New("cannot create client: both es.uri and ES_CLOUD_ID are set") + } + + if len(esURI) > 0 { + uri = esURI + } + + if len(cloudID) > 0 { + uri, err = addrFromCloudID(cloudID) + if err != nil { + return nil, err + } + } + } + + esURL, err := url.Parse(uri) + if err != nil { + return nil, fmt.Errorf("failed to parse url: %v", err) + } + + return esURL, nil +} + +// addrFromCloudID extracts the Elasticsearch URL from CloudID. +// See: https://www.elastic.co/guide/en/cloud/current/ec-cloud-id.html +// +// This function was originally copied from https://github.com/elastic/go-elasticsearch/blob/8134a159aafedf58af2780ebb3a30ec1938956f3/elasticsearch.go#L365-L383 +func addrFromCloudID(input string) (string, error) { + var scheme = "https://" + + values := strings.Split(input, ":") + if len(values) != 2 { + return "", fmt.Errorf("unexpected format: %q", input) + } + data, err := base64.StdEncoding.DecodeString(values[1]) + if err != nil { + return "", err + } + parts := strings.Split(string(data), "$") + + if len(parts) < 2 { + return "", fmt.Errorf("invalid encoded value: %s", parts) + } + + return fmt.Sprintf("%s%s.%s", scheme, parts[1], parts[0]), nil +} diff --git a/pkg/url/url_test.go b/pkg/url/url_test.go new file mode 100644 index 00000000..ec488c2f --- /dev/null +++ b/pkg/url/url_test.go @@ -0,0 +1,128 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package url + +import ( + "encoding/base64" + "fmt" + "regexp" + "testing" +) + +func TestAddrFromCloudID(t *testing.T) { + t.Run("Parse", func(t *testing.T) { + var testdata = []struct { + in string + out string + }{ + { + in: "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid$kibana_uuid")), + out: "https://es_uuid.host", + }, + { + in: "name:" + base64.StdEncoding.EncodeToString([]byte("host:9243$es_uuid$kibana_uuid")), + out: "https://es_uuid.host:9243", + }, + { + in: "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid$")), + out: "https://es_uuid.host", + }, + { + in: "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid")), + out: "https://es_uuid.host", + }, + } + + for _, tt := range testdata { + actual, err := addrFromCloudID(tt.in) + if err != nil { + t.Errorf("Unexpected error: %s", err) + } + if actual != tt.out { + t.Errorf("Unexpected output, want=%q, got=%q", tt.out, actual) + } + } + + }) + + t.Run("Invalid format", func(t *testing.T) { + input := "foobar" + _, err := addrFromCloudID(input) + if err == nil { + t.Errorf("Expected error for input %q, got %v", input, err) + } + match, _ := regexp.MatchString("unexpected format", err.Error()) + if !match { + t.Errorf("Unexpected error string: %s", err) + } + }) + + t.Run("Invalid base64 value", func(t *testing.T) { + input := "foobar:xxxxx" + _, err := addrFromCloudID(input) + if err == nil { + t.Errorf("Expected error for input %q, got %v", input, err) + } + match, _ := regexp.MatchString("illegal base64 data", err.Error()) + if !match { + t.Errorf("Unexpected error string: %s", err) + } + }) +} + +func TestGetEsURL(t *testing.T) { + cases := map[string]struct { + esURI string + cloudID string + expectedUrl string + expectErr bool + }{ + "success - url = defaultURI": { + expectedUrl: defaultEsURI, + }, + "success - url = esURI": { + esURI: "http://example.com:9200", + expectedUrl: "http://example.com:9200", + }, + "success - url = parsed from cloudID": { + cloudID: "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid$kibana_uuid")), + expectedUrl: "https://es_uuid.host", + }, + "error - both esURI and cloudID are specified": { + esURI: "http://example.com:9200", + cloudID: "name:" + base64.StdEncoding.EncodeToString([]byte("host$es_uuid$kibana_uuid")), + expectErr: true, + }, + } + + for name, tc := range cases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + got, err := GetEsURL(tc.esURI, tc.cloudID) + + if err != nil { + if !tc.expectErr { + t.Fatalf("Failed to prepare URL: %s", err) + } + return + } + + url := fmt.Sprintf("%s://%s", got.Scheme, got.Host) + if url != tc.expectedUrl { + t.Fatalf("Fatiled to parse URL: got:%s, want: %s", url, tc.expectedUrl) + } + }) + } +} From d7ea55ab039f63922c181b059b44cbdd336f2d41 Mon Sep 17 00:00:00 2001 From: "pakiosann@gmail.com" Date: Wed, 22 Jun 2022 00:17:42 +0900 Subject: [PATCH 2/3] make esCloudID as an argument, not an env var Signed-off-by: Kazuma (Pakio) Arimura --- main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 5bd7fdeb..b1847306 100644 --- a/main.go +++ b/main.go @@ -56,6 +56,9 @@ func main() { esURI = kingpin.Flag("es.uri", "HTTP API address of an Elasticsearch node."). Default("").String() + esCloudID = kingpin.Flag("es.cloudid", + "Cloud ID of an Elasticsearch Service cluster"). + Default("").String() esTimeout = kingpin.Flag("es.timeout", "Timeout for trying to get stats from Elasticsearch."). Default("5s").Duration() @@ -121,7 +124,7 @@ func main() { logger := getLogger(*logLevel, *logOutput, *logFormat) - esURL, err := esurl.GetEsURL(*esURI, os.Getenv("ES_CLOUD_ID")) + esURL, err := esurl.GetEsURL(*esURI, *esCloudID) if err != nil { _ = level.Error(logger).Log( "msg", "error while parsing URL", From 55079b74fad977123737f33f27ce4c9d746b1355 Mon Sep 17 00:00:00 2001 From: "pakiosann@gmail.com" Date: Wed, 22 Jun 2022 00:18:00 +0900 Subject: [PATCH 3/3] Add documentation about Cloud ID Signed-off-by: Kazuma (Pakio) Arimura --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index abb57bdf..c2c42a49 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,8 @@ elasticsearch_exporter --help | Argument | Introduced in Version | Description | Default | | -------- | --------------------- | ----------- | ----------- | -| es.uri | 1.0.2 | Address (host and port) of the Elasticsearch node we should connect to. This could be a local node (`localhost:9200`, for instance), or the address of a remote Elasticsearch server. When basic auth is needed, specify as: `://:@:`. E.G., `http://admin:pass@localhost:9200`. Special characters in the user credentials need to be URL-encoded. | http://localhost:9200 | +| es.uri | 1.0.2 | Address (host and port) of the Elasticsearch node we should connect to. This could be a local node (`localhost:9200`, for instance), or the address of a remote Elasticsearch server. When basic auth is needed, specify as: `://:@:`. E.G., `http://admin:pass@localhost:9200`. Special characters in the user credentials need to be URL-encoded. Cannot be used at the same time with `es.cloudid`. | | +| es.cloudid | | Cloud ID of an Elasticsearch Service cluster. Please visit [the official document](https://www.elastic.co/guide/en/cloud/current/ec-cloud-id.html) for more information about Cloud ID. Cannot be used at the same time with `es.uri`. | | | es.all | 1.0.2 | If true, query stats for all nodes in the cluster, rather than just the node we connect to. | false | | es.cluster_settings | 1.1.0rc1 | If true, query stats for cluster settings. | false | | es.indices | 1.0.2 | If true, query stats for all indices in the cluster. | false | @@ -73,6 +74,8 @@ For versions greater than `1.1.0rc1`, commandline parameters are specified with The API key used to connect can be set with the `ES_API_KEY` environment variable. +We provides 2 ways to specify the address of the Elasticsearch node/cluster, `es.uri` and `es.cloudid`. When both of them are not specified, default address (`http://localhost:9200`) will be used. + #### Elasticsearch 7.x security privileges Username and password can be passed either directly in the URI or through the `ES_USERNAME` and `ES_PASSWORD` environment variables.