Skip to content

Commit a2be7e8

Browse files
jurgispodschris-brindley
jurgispods
and
chris-brindley
authored
Add Datadog site config (#48)
* add Datadog site config * Make linting and tests pass --------- Co-authored-by: chris-brindley <[email protected]>
1 parent d709ef9 commit a2be7e8

File tree

5 files changed

+54
-6
lines changed

5 files changed

+54
-6
lines changed

‎example-config.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
# Manager secret that contains the key can be made.
66
#
77
# datadog-api-key: ***REDACTED***
8+
# datadog-site: US
89
# datadog-api-key-file: /etc/
910
# datadog-api-key-secret-id: projects/my-project/secrets/my-datadog-api-key/version/latest
1011

‎pkg/config/config.go

+18
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ var Version = "0.0.0"
3838
// Config holds the configuration for the application
3939
type Config struct {
4040
DatadogAPIKey string `viper:"datadog-api-key"`
41+
DatadogSite string `viper:"datadog-site"`
4142
DatasetFilter string `viper:"dataset-filter"`
4243
GcpProject string `viper:"gcp-project-id"`
4344
MetricPrefix string `viper:"metric-prefix"`
@@ -137,12 +138,28 @@ func NormaliseConfig(c *Config) {
137138
}
138139
}
139140

141+
// DatadogSites is a mapping of valid Datadog site names and their respective URLs
142+
var DatadogSites = map[string]string{
143+
"US": "datadoghq.com",
144+
"US1": "datadoghq.com",
145+
"US3": "us3.datadoghq.com",
146+
"US5": "us5.datadoghq.com",
147+
"EU": "datadoghq.eu",
148+
"EU1": "datadoghq.eu",
149+
"US1-FED": "ddog-gov.com",
150+
"AP1": "ap1.datadoghq.com",
151+
}
152+
140153
// ValidateConfig will validate that all of the required config parameters are present
141154
func ValidateConfig(c *Config) error {
142155
if c.DatadogAPIKey == "" {
143156
return ErrMissingDatadogAPIKey
144157
}
145158

159+
if _, ok := DatadogSites[c.DatadogSite]; !ok {
160+
return ErrInvalidDatadogSite
161+
}
162+
146163
if c.GcpProject == "" {
147164
return ErrMissingGcpProject
148165
}
@@ -197,6 +214,7 @@ func configFlags(name string) *pflag.FlagSet {
197214
flags.String("dataset-filter", "", "BigQuery label to filter datasets for metric collection")
198215
flags.String("datadog-api-key-file", "", "File containing the Datadog API key")
199216
flags.String("datadog-api-key-secret-id", "", "Google Secret Manager Resource ID containing the Datadog API key")
217+
flags.String("datadog-site", "US", "Datadog site to use (see https://docs.datadoghq.com/getting_started/site/)")
200218
flags.String("gcp-project-id", "", "The GCP project to extract BigQuery metrics from")
201219
flags.String("metric-prefix", DefaultMetricPrefix, fmt.Sprintf("The prefix for the metrics names exported to Datadog (Default %s)", DefaultMetricPrefix))
202220
flags.Duration("metric-interval", defInterval, fmt.Sprintf("The interval between metrics submissions (Default %s)", DefaultMetricInterval))

‎pkg/config/config_test.go

+31-5
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,9 @@ func TestNewConfig(t *testing.T) {
4141
want *Config
4242
wantErr bool
4343
}{
44-
{"all via env", setup([]string{"DATADOG_API_KEY=abc123", "DATASET_FILTER=bqmetrics:enabled", "GCP_PROJECT_ID=my-project-id", "METRIC_PREFIX=custom.gcp.bigquery.stats", "METRIC_TAGS=env:prod", "METRIC_INTERVAL=2m", "HEALTHCHECK_ENABLED=true"}, nil, ""), args{"bqmetricstest"}, &Config{
44+
{"all via env", setup([]string{"DATADOG_API_KEY=abc123", "DATADOG_SITE=EU", "DATASET_FILTER=bqmetrics:enabled", "GCP_PROJECT_ID=my-project-id", "METRIC_PREFIX=custom.gcp.bigquery.stats", "METRIC_TAGS=env:prod", "METRIC_INTERVAL=2m", "HEALTHCHECK_ENABLED=true"}, nil, ""), args{"bqmetricstest"}, &Config{
4545
DatadogAPIKey: "abc123",
46+
DatadogSite: "EU",
4647
DatasetFilter: "bqmetrics:enabled",
4748
GcpProject: "my-project-id",
4849
MetricPrefix: "custom.gcp.bigquery.stats",
@@ -51,8 +52,9 @@ func TestNewConfig(t *testing.T) {
5152
Profiler: Profiler{false, 6060},
5253
HealthCheck: HealthCheck{true, 8080},
5354
}, false},
54-
{"all via cmd", setup(nil, []string{"--datadog-api-key-file=/tmp/dd.key", "--dataset-filter=bqmetrics:enabled", "--gcp-project-id=my-project-id", "--metric-prefix=custom.gcp.bigquery.stats", "--metric-tags=env:prod", "--metric-interval=2m", "--profiler.enabled"}, "abc123"), args{"bqmetricstest"}, &Config{
55+
{"all via cmd", setup(nil, []string{"--datadog-api-key-file=/tmp/dd.key", "--datadog-site=EU", "--dataset-filter=bqmetrics:enabled", "--gcp-project-id=my-project-id", "--metric-prefix=custom.gcp.bigquery.stats", "--metric-tags=env:prod", "--metric-interval=2m", "--profiler.enabled"}, "abc123"), args{"bqmetricstest"}, &Config{
5556
DatadogAPIKey: "abc123",
57+
DatadogSite: "EU",
5658
DatasetFilter: "bqmetrics:enabled",
5759
GcpProject: "my-project-id",
5860
MetricPrefix: "custom.gcp.bigquery.stats",
@@ -61,8 +63,9 @@ func TestNewConfig(t *testing.T) {
6163
Profiler: Profiler{true, 6060},
6264
HealthCheck: HealthCheck{false, 8080},
6365
}, false},
64-
{"mixture of sources", setup([]string{"DATADOG_API_KEY=abc123", "GCP_PROJECT_ID=my-project-id"}, []string{"--metric-prefix=custom.gcp.bigquery.stats", "--metric-tags=env:prod", "--metric-interval=2m"}, ""), args{"bqmetricstest"}, &Config{
66+
{"mixture of sources", setup([]string{"DATADOG_API_KEY=abc123", "DATADOG_SITE=US", "GCP_PROJECT_ID=my-project-id"}, []string{"--metric-prefix=custom.gcp.bigquery.stats", "--metric-tags=env:prod", "--metric-interval=2m"}, ""), args{"bqmetricstest"}, &Config{
6567
DatadogAPIKey: "abc123",
68+
DatadogSite: "US",
6669
GcpProject: "my-project-id",
6770
MetricPrefix: "custom.gcp.bigquery.stats",
6871
MetricTags: []string{"env:prod"},
@@ -72,6 +75,7 @@ func TestNewConfig(t *testing.T) {
7275
}, false},
7376
{"minimum required config", setup([]string{"DATADOG_API_KEY=abc123", "GCP_PROJECT_ID=my-project-id"}, nil, ""), args{"bqmetricstest"}, &Config{
7477
DatadogAPIKey: "abc123",
78+
DatadogSite: "US",
7579
GcpProject: "my-project-id",
7680
MetricPrefix: DefaultMetricPrefix,
7781
MetricTags: []string{},
@@ -81,6 +85,7 @@ func TestNewConfig(t *testing.T) {
8185
}, false},
8286
{"default credentials", setup([]string{"DATADOG_API_KEY=abc123", "GOOGLE_APPLICATION_CREDENTIALS=/tmp/dd.key"}, nil, "{\"type\": \"service_account\", \"project_id\": \"my-project-id\"}"), args{"bqmetricstest"}, &Config{
8387
DatadogAPIKey: "abc123",
88+
DatadogSite: "US",
8489
GcpProject: "my-project-id",
8590
MetricPrefix: DefaultMetricPrefix,
8691
MetricTags: []string{},
@@ -118,14 +123,15 @@ func TestNewConfig_configFile(t *testing.T) {
118123
_ = os.Remove(n)
119124
}()
120125

121-
data := []byte("{\"datadog-api-key\": \"abc123\", \"dataset-filter\": \"bqmetrics:enabled\", \"gcp-project-id\": \"my-project-id\", \"metric-prefix\": \"custom.gcp.bigquery.stats\", \"metric-tags\": \"env:prod,team:my-team\", \"metric-interval\": \"2m\", \"healthcheck\": {\"enabled\": true, \"port\": 8081}}")
126+
data := []byte("{\"datadog-api-key\": \"abc123\", \"datadog-site\": \"US\", \"dataset-filter\": \"bqmetrics:enabled\", \"gcp-project-id\": \"my-project-id\", \"metric-prefix\": \"custom.gcp.bigquery.stats\", \"metric-tags\": \"env:prod,team:my-team\", \"metric-interval\": \"2m\", \"healthcheck\": {\"enabled\": true, \"port\": 8081}}")
122127
if _, err = f.Write(data); err != nil {
123128
t.Fatalf("error when writing test config file: %s", err)
124129
}
125130

126131
os.Args = []string{"./bqmetricstest", "--config-file", f.Name()}
127132
want := &Config{
128133
DatadogAPIKey: "abc123",
134+
DatadogSite: "US",
129135
DatasetFilter: "bqmetrics:enabled",
130136
GcpProject: "my-project-id",
131137
MetricPrefix: "custom.gcp.bigquery.stats",
@@ -180,14 +186,15 @@ func TestNewConfig_configFileWithCustomQueries(t *testing.T) {
180186
_ = os.Remove(n)
181187
}()
182188

183-
data := []byte("{\"datadog-api-key\": \"abc123\", \"gcp-project-id\": \"my-project-id\", \"metric-prefix\": \"custom.gcp.bigquery.stats\", \"metric-tags\": \"env:prod,team:my-team\", \"metric-interval\": \"2m\", \"custom-metrics\": [{\"metric-name\": \"my_metric\", \"metric-tags\": [\"table_id:table\"], \"sql\": \"SELECT COUNT(DISTINCT *) FROM `table`\"}]}")
189+
data := []byte("{\"datadog-api-key\": \"abc123\", \"datadog-site\": \"US\", \"gcp-project-id\": \"my-project-id\", \"metric-prefix\": \"custom.gcp.bigquery.stats\", \"metric-tags\": \"env:prod,team:my-team\", \"metric-interval\": \"2m\", \"custom-metrics\": [{\"metric-name\": \"my_metric\", \"metric-tags\": [\"table_id:table\"], \"sql\": \"SELECT COUNT(DISTINCT *) FROM `table`\"}]}")
184190
if _, err = f.Write(data); err != nil {
185191
t.Fatalf("error when writing test config file: %s", err)
186192
}
187193

188194
os.Args = []string{"./bqmetricstest", "--config-file", f.Name()}
189195
want := &Config{
190196
DatadogAPIKey: "abc123",
197+
DatadogSite: "US",
191198
GcpProject: "my-project-id",
192199
MetricPrefix: "custom.gcp.bigquery.stats",
193200
MetricTags: []string{"env:prod", "team:my-team"},
@@ -223,48 +230,62 @@ func TestValidateConfig(t *testing.T) {
223230
}{
224231
{"valid", args{&Config{
225232
DatadogAPIKey: "abc123",
233+
DatadogSite: "US",
226234
GcpProject: "my-project-id",
227235
MetricPrefix: "custom.gcp.bigquery.stats",
228236
MetricTags: []string{"env:prod"},
229237
MetricInterval: time.Duration(30000),
230238
}}, false},
231239
{"missing datadog api key", args{&Config{
232240
DatadogAPIKey: "",
241+
DatadogSite: "US",
242+
GcpProject: "my-project-id",
243+
MetricPrefix: "custom.gcp.bigquery.stats",
244+
MetricTags: []string{"env:prod"},
245+
MetricInterval: time.Duration(30000),
246+
}}, true},
247+
{"missing datadog site", args{&Config{
248+
DatadogAPIKey: "abc123",
233249
GcpProject: "my-project-id",
234250
MetricPrefix: "custom.gcp.bigquery.stats",
235251
MetricTags: []string{"env:prod"},
236252
MetricInterval: time.Duration(30000),
237253
}}, true},
238254
{"missing gcp project id", args{&Config{
239255
DatadogAPIKey: "abc123",
256+
DatadogSite: "US",
240257
GcpProject: "",
241258
MetricPrefix: "custom.gcp.bigquery.stats",
242259
MetricTags: []string{"env:prod"},
243260
MetricInterval: time.Duration(30000),
244261
}}, true},
245262
{"missing metric prefix", args{&Config{
246263
DatadogAPIKey: "abc123",
264+
DatadogSite: "US",
247265
GcpProject: "my-project-id",
248266
MetricPrefix: "",
249267
MetricTags: []string{"env:prod"},
250268
MetricInterval: time.Duration(30000),
251269
}}, true},
252270
{"missing metric tags", args{&Config{
253271
DatadogAPIKey: "abc123",
272+
DatadogSite: "US",
254273
GcpProject: "my-project-id",
255274
MetricPrefix: "custom.gcp.bigquery.stats",
256275
MetricTags: nil,
257276
MetricInterval: time.Duration(30000),
258277
}}, false},
259278
{"missing metric interval", args{&Config{
260279
DatadogAPIKey: "abc123",
280+
DatadogSite: "US",
261281
GcpProject: "my-project-id",
262282
MetricPrefix: "custom.gcp.bigquery.stats",
263283
MetricTags: []string{"env:prod"},
264284
MetricInterval: time.Duration(0),
265285
}}, true},
266286
{"missing profiling", args{&Config{
267287
DatadogAPIKey: "abc123",
288+
DatadogSite: "US",
268289
GcpProject: "my-project-id",
269290
MetricPrefix: "custom.gcp.bigquery.stats",
270291
MetricTags: []string{"env:prod"},
@@ -273,6 +294,7 @@ func TestValidateConfig(t *testing.T) {
273294
}}, false},
274295
{"custom metrics okay", args{&Config{
275296
DatadogAPIKey: "abc123",
297+
DatadogSite: "US",
276298
GcpProject: "my-project-id",
277299
MetricPrefix: "custom.gcp.bigquery.stats",
278300
MetricTags: []string{"env:prod"},
@@ -287,6 +309,7 @@ func TestValidateConfig(t *testing.T) {
287309
}}, false},
288310
{"custom metrics missing name", args{&Config{
289311
DatadogAPIKey: "abc123",
312+
DatadogSite: "US",
290313
GcpProject: "my-project-id",
291314
MetricPrefix: "custom.gcp.bigquery.stats",
292315
MetricTags: []string{"env:prod"},
@@ -298,6 +321,7 @@ func TestValidateConfig(t *testing.T) {
298321
}}, true},
299322
{"custom metrics missing sql", args{&Config{
300323
DatadogAPIKey: "abc123",
324+
DatadogSite: "US",
301325
GcpProject: "my-project-id",
302326
MetricPrefix: "custom.gcp.bigquery.stats",
303327
MetricTags: []string{"env:prod"},
@@ -309,6 +333,7 @@ func TestValidateConfig(t *testing.T) {
309333
}}, true},
310334
{"health check disabled", args{&Config{
311335
DatadogAPIKey: "abc123",
336+
DatadogSite: "US",
312337
GcpProject: "my-project-id",
313338
MetricPrefix: "custom.gcp.bigquery.stats",
314339
MetricTags: []string{"env:prod"},
@@ -317,6 +342,7 @@ func TestValidateConfig(t *testing.T) {
317342
}}, false},
318343
{"health check port invalid", args{&Config{
319344
DatadogAPIKey: "abc123",
345+
DatadogSite: "US",
320346
GcpProject: "my-project-id",
321347
MetricPrefix: "custom.gcp.bigquery.stats",
322348
MetricTags: []string{"env:prod"},

‎pkg/config/errors.go

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import "errors"
55
var (
66
// ErrMissingDatadogAPIKey is the error returned when the Config is missing an API key for Datadog
77
ErrMissingDatadogAPIKey = errors.New("no Datadog API key configured")
8+
// ErrInvalidDatadogSite is the error returned when the Config contains an invalid value for the Datadog site
9+
ErrInvalidDatadogSite = errors.New("invalid Datadog site configured")
810

911
// ErrMissingGcpProject is the error returned when the Config is missing the Google project ID
1012
ErrMissingGcpProject = errors.New("no GCP project ID configured")

‎pkg/metrics/datadog.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ func (dp *DatadogPublisher) PublishMetricsSet(ctx context.Context, metrics []Met
4444
return NewUnrecoverableError(err)
4545
}
4646

47-
url := fmt.Sprintf("https://api.datadoghq.com/api/v1/series?api_key=%s", dp.cfg.DatadogAPIKey)
47+
ddSite := config.DatadogSites[dp.cfg.DatadogSite]
48+
url := fmt.Sprintf("https://api.%s/api/v1/series?api_key=%s", ddSite, dp.cfg.DatadogAPIKey)
4849
request, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(body))
4950
if err != nil {
5051
return NewUnrecoverableError(err)

0 commit comments

Comments
 (0)