Skip to content

Commit 5cdba47

Browse files
sky333999jefchien
sky333999
and
jefchien
authoredOct 9, 2024··
Extend OTLP receiver to publish via EMF logs (#1375)
Co-authored-by: Jeffrey Chien <chienjef@amazon.com>
·
v1.300054.0v1.300048.0
1 parent fce5a3c commit 5cdba47

33 files changed

+675
-226
lines changed
 

‎internal/util/collections/collections.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@ func NewPair[K any, V any](key K, value V) *Pair[K, V] {
6161

6262
// Set is a map with a comparable K key and no
6363
// meaningful value.
64-
type Set[K comparable] map[K]any
64+
type Set[K comparable] map[K]struct{}
6565

6666
// Add keys to the Set.
6767
func (s Set[K]) Add(keys ...K) {
6868
for _, key := range keys {
69-
s[key] = nil
69+
s[key] = struct{}{}
7070
}
7171
}
7272

‎translator/config/sampleSchema/validOTLPMetrics.json

+18
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,23 @@
1616
}
1717
]
1818
}
19+
},
20+
"logs": {
21+
"metrics_collected": {
22+
"otlp": [
23+
{
24+
"grpc_endpoint": "0.0.0.0:1234",
25+
"http_endpoint": "0.0.0.0:2345"
26+
},
27+
{
28+
"grpc_endpoint": "0.0.0.0:3456",
29+
"http_endpoint": "0.0.0.0:4567",
30+
"tls": {
31+
"cert_file": "/path/to/cert.pem",
32+
"key_file": "/path/to/key.pem"
33+
}
34+
}
35+
]
36+
}
1937
}
2038
}

‎translator/config/schema.json

+3
Original file line numberDiff line numberDiff line change
@@ -802,6 +802,9 @@
802802
}
803803
},
804804
"additionalProperties": false
805+
},
806+
"otlp": {
807+
"$ref": "#/definitions/otlpDefinitions"
805808
}
806809
},
807810
"additionalProperties": true

‎translator/tocwconfig/sampleConfig/amp_config_linux.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ service:
122122
- agenthealth/metrics
123123
- sigv4auth
124124
pipelines:
125-
metrics/host:
125+
metrics/host/cloudwatch:
126126
exporters:
127127
- awscloudwatch
128128
processors:

‎translator/tocwconfig/sampleConfig/complete_linux_config.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ processors:
107107
send_batch_max_size: 0
108108
send_batch_size: 8192
109109
timeout: 200ms
110-
cumulativetodelta/hostDeltaMetrics:
110+
cumulativetodelta/hostDeltaMetrics/cloudwatch:
111111
exclude:
112112
match_type: strict
113113
metrics:
@@ -362,7 +362,7 @@ service:
362362
- batch/emf_logs
363363
receivers:
364364
- udplog/emf_logs
365-
metrics/host:
365+
metrics/host/cloudwatch:
366366
exporters:
367367
- awscloudwatch
368368
processors:
@@ -378,11 +378,11 @@ service:
378378
- telegraf_netstat
379379
- telegraf_processes
380380
- telegraf_mem
381-
metrics/hostDeltaMetrics:
381+
metrics/hostDeltaMetrics/cloudwatch:
382382
exporters:
383383
- awscloudwatch
384384
processors:
385-
- cumulativetodelta/hostDeltaMetrics
385+
- cumulativetodelta/hostDeltaMetrics/cloudwatch
386386
- ec2tagger
387387
- transform
388388
receivers:
@@ -394,9 +394,9 @@ service:
394394
processors:
395395
- filter/jmx/0
396396
- resource/jmx
397-
- cumulativetodelta/jmx
398397
- transform/jmx/0
399398
- ec2tagger
399+
- cumulativetodelta/jmx
400400
receivers:
401401
- jmx/0
402402
metrics/jmx/cloudwatch/1:

‎translator/tocwconfig/sampleConfig/jmx_config_linux.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ service:
161161
- agenthealth/metrics
162162
- sigv4auth
163163
pipelines:
164-
metrics/host:
164+
metrics/host/cloudwatch:
165165
exporters:
166166
- awscloudwatch
167167
processors:
@@ -184,8 +184,8 @@ service:
184184
processors:
185185
- filter/jmx
186186
- resource/jmx
187-
- batch/jmx/amp
188187
- transform/jmx
188+
- batch/jmx/amp
189189
receivers:
190190
- jmx
191191
metrics/jmx/cloudwatch:

‎translator/tocwconfig/sampleConfig/otlp_metrics_cloudwatchlogs_config.conf

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
[agent]
2+
collection_jitter = "0s"
3+
debug = false
4+
flush_interval = "1s"
5+
flush_jitter = "0s"
6+
hostname = ""
7+
interval = "60s"
8+
logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log"
9+
logtarget = "lumberjack"
10+
metric_batch_size = 1000
11+
metric_buffer_limit = 10000
12+
omit_hostname = false
13+
precision = ""
14+
quiet = false
15+
round_interval = false
16+
17+
[inputs]
18+
19+
[outputs]
20+
21+
[[outputs.cloudwatchlogs]]
22+
force_flush_interval = "30s"
23+
log_stream_name = "i-UNKNOWN"
24+
region = "us-west-2"

‎translator/tocwconfig/sampleConfig/otlp_metrics_cloudwatchlogs_config.json

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"logs": {
3+
"force_flush_interval": 30,
4+
"metrics_collected": {
5+
"otlp": {
6+
"grpc_endpoint": "0.0.0.0:1234",
7+
"http_endpoint": "0.0.0.0:2345",
8+
"tls": {
9+
"cert_file": "/path/to/cert.pem",
10+
"key_file": "/path/to/key.pem"
11+
}
12+
}
13+
}
14+
}
15+
}

‎translator/tocwconfig/sampleConfig/otlp_metrics_cloudwatchlogs_config.yaml

+122
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
exporters:
2+
awsemf:
3+
certificate_file_path: ""
4+
detailed_metrics: false
5+
dimension_rollup_option: NoDimensionRollup
6+
disable_metric_extraction: false
7+
eks_fargate_container_insights_enabled: false
8+
endpoint: ""
9+
enhanced_container_insights: false
10+
imds_retries: 1
11+
local_mode: false
12+
log_group_name: /aws/cwagent
13+
log_retention: 0
14+
log_stream_name: ""
15+
max_retries: 2
16+
middleware: agenthealth/logs
17+
namespace: CWAgent
18+
no_verify_ssl: false
19+
num_workers: 8
20+
output_destination: cloudwatch
21+
profile: ""
22+
proxy_address: ""
23+
region: us-west-2
24+
request_timeout_seconds: 30
25+
resource_arn: ""
26+
resource_to_telemetry_conversion:
27+
enabled: true
28+
retain_initial_value_of_delta_metric: false
29+
role_arn: ""
30+
version: "0"
31+
extensions:
32+
agenthealth/logs:
33+
is_usage_data_enabled: true
34+
stats:
35+
operations:
36+
- PutLogEvents
37+
usage_flags:
38+
mode: EC2
39+
region_type: ACJ
40+
processors:
41+
batch/hostDeltaMetrics/cloudwatchlogs:
42+
metadata_cardinality_limit: 1000
43+
send_batch_max_size: 0
44+
send_batch_size: 8192
45+
timeout: 30s
46+
cumulativetodelta/hostDeltaMetrics/cloudwatchlogs:
47+
exclude:
48+
match_type: ""
49+
include:
50+
match_type: ""
51+
initial_value: 2
52+
max_staleness: 0s
53+
receivers:
54+
otlp/metrics:
55+
protocols:
56+
grpc:
57+
dialer:
58+
timeout: 0s
59+
endpoint: 0.0.0.0:1234
60+
include_metadata: false
61+
max_concurrent_streams: 0
62+
max_recv_msg_size_mib: 0
63+
read_buffer_size: 524288
64+
transport: tcp
65+
write_buffer_size: 0
66+
tls:
67+
ca_file: ""
68+
cert_file: /path/to/cert.pem
69+
client_ca_file: ""
70+
client_ca_file_reload: false
71+
include_system_ca_certs_pool: false
72+
key_file: /path/to/key.pem
73+
max_version: ""
74+
min_version: ""
75+
reload_interval: 0s
76+
http:
77+
endpoint: 0.0.0.0:2345
78+
include_metadata: false
79+
logs_url_path: /v1/logs
80+
max_request_body_size: 0
81+
metrics_url_path: /v1/metrics
82+
traces_url_path: /v1/traces
83+
tls:
84+
ca_file: ""
85+
cert_file: /path/to/cert.pem
86+
client_ca_file: ""
87+
client_ca_file_reload: false
88+
include_system_ca_certs_pool: false
89+
key_file: /path/to/key.pem
90+
max_version: ""
91+
min_version: ""
92+
reload_interval: 0s
93+
service:
94+
extensions:
95+
- agenthealth/logs
96+
pipelines:
97+
metrics/hostDeltaMetrics/cloudwatchlogs:
98+
exporters:
99+
- awsemf
100+
processors:
101+
- batch/hostDeltaMetrics/cloudwatchlogs
102+
- cumulativetodelta/hostDeltaMetrics/cloudwatchlogs
103+
receivers:
104+
- otlp/metrics
105+
telemetry:
106+
logs:
107+
development: false
108+
disable_caller: false
109+
disable_stacktrace: false
110+
encoding: console
111+
level: info
112+
output_paths:
113+
- /opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log
114+
sampling:
115+
enabled: true
116+
initial: 2
117+
thereafter: 500
118+
tick: 10s
119+
metrics:
120+
address: ""
121+
level: None
122+
traces: { }

‎translator/tocwconfig/tocwconfig_test.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,14 @@ func TestOtlpMetricsConfig(t *testing.T) {
226226
checkTranslation(t, "otlp_metrics_config", "windows", nil, "")
227227
}
228228

229+
func TestOtlpMetricsEmfConfig(t *testing.T) {
230+
resetContext(t)
231+
context.CurrentContext().SetMode(config.ModeEC2)
232+
checkTranslation(t, "otlp_metrics_cloudwatchlogs_config", "linux", nil, "")
233+
checkTranslation(t, "otlp_metrics_cloudwatchlogs_config", "darwin", nil, "")
234+
checkTranslation(t, "otlp_metrics_cloudwatchlogs_config", "windows", nil, "")
235+
}
236+
229237
func TestProcstatMemorySwapConfig(t *testing.T) {
230238
resetContext(t)
231239
context.CurrentContext().SetRunInContainer(false)
@@ -234,7 +242,6 @@ func TestProcstatMemorySwapConfig(t *testing.T) {
234242
t.Setenv(config.HOST_IP, "127.0.0.1")
235243
checkTranslation(t, "procstat_memory_swap_config", "linux", nil, "")
236244
checkTranslation(t, "procstat_memory_swap_config", "darwin", nil, "")
237-
238245
}
239246

240247
func TestWindowsEventOnlyConfig(t *testing.T) {

‎translator/translate/otel/common/common.go

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ const (
2929
ECSKey = "ecs"
3030
KubernetesKey = "kubernetes"
3131
CloudWatchKey = "cloudwatch"
32+
CloudWatchLogsKey = "cloudwatchlogs"
3233
PrometheusKey = "prometheus"
3334
AMPKey = "amp"
3435
WorkspaceIDKey = "workspace_id"

‎translator/translate/otel/common/destination.go

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package common
5+
6+
import "go.opentelemetry.io/collector/confmap"
7+
8+
const (
9+
DefaultDestination = ""
10+
)
11+
12+
var (
13+
metricsDestinationsKey = ConfigKey(MetricsKey, MetricsDestinationsKey)
14+
)
15+
16+
func GetMetricsDestinations(conf *confmap.Conf) []string {
17+
var destinations []string
18+
if conf.IsSet(ConfigKey(metricsDestinationsKey, CloudWatchKey)) {
19+
destinations = append(destinations, CloudWatchKey)
20+
}
21+
if conf.IsSet(ConfigKey(metricsDestinationsKey, AMPKey)) {
22+
destinations = append(destinations, AMPKey)
23+
}
24+
if conf.IsSet(MetricsKey) && len(destinations) == 0 {
25+
destinations = append(destinations, DefaultDestination)
26+
}
27+
return destinations
28+
}

0 commit comments

Comments
 (0)
Please sign in to comment.