Skip to content

Commit 2b7d4b3

Browse files
authored
Introduce typed pipeline (#2896)
* introduce more typing for pipelines * rock and a hard place * resolve tests * Fix tests * chlog * fix docs * thing * alphabet * remove chlog entry
1 parent 5ca66dd commit 2b7d4b3

19 files changed

+324
-32
lines changed

apis/v1beta1/config.go

+44-1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,18 @@ import (
2727
"gopkg.in/yaml.v3"
2828
)
2929

30+
type ComponentType int
31+
32+
const (
33+
ComponentTypeReceiver ComponentType = iota
34+
ComponentTypeExporter
35+
ComponentTypeProcessor
36+
)
37+
38+
func (c ComponentType) String() string {
39+
return [...]string{"receiver", "exporter", "processor"}[c]
40+
}
41+
3042
// AnyConfig represent parts of the config.
3143
type AnyConfig struct {
3244
Object map[string]interface{} `json:"-" yaml:",inline"`
@@ -75,6 +87,37 @@ func (c *AnyConfig) MarshalJSON() ([]byte, error) {
7587
return json.Marshal(c.Object)
7688
}
7789

90+
// Pipeline is a struct of component type to a list of component IDs.
91+
type Pipeline struct {
92+
Exporters []string `json:"exporters" yaml:"exporters"`
93+
Processors []string `json:"processors" yaml:"processors"`
94+
Receivers []string `json:"receivers" yaml:"receivers"`
95+
}
96+
97+
// GetEnabledComponents constructs a list of enabled components by component type.
98+
func (c *Config) GetEnabledComponents() map[ComponentType]map[string]interface{} {
99+
toReturn := map[ComponentType]map[string]interface{}{
100+
ComponentTypeReceiver: {},
101+
ComponentTypeProcessor: {},
102+
ComponentTypeExporter: {},
103+
}
104+
for _, pipeline := range c.Service.Pipelines {
105+
if pipeline == nil {
106+
continue
107+
}
108+
for _, componentId := range pipeline.Receivers {
109+
toReturn[ComponentTypeReceiver][componentId] = struct{}{}
110+
}
111+
for _, componentId := range pipeline.Exporters {
112+
toReturn[ComponentTypeExporter][componentId] = struct{}{}
113+
}
114+
for _, componentId := range pipeline.Processors {
115+
toReturn[ComponentTypeProcessor][componentId] = struct{}{}
116+
}
117+
}
118+
return toReturn
119+
}
120+
78121
// Config encapsulates collector config.
79122
type Config struct {
80123
// +kubebuilder:pruning:PreserveUnknownFields
@@ -135,7 +178,7 @@ type Service struct {
135178
// +kubebuilder:pruning:PreserveUnknownFields
136179
Telemetry *AnyConfig `json:"telemetry,omitempty" yaml:"telemetry,omitempty"`
137180
// +kubebuilder:pruning:PreserveUnknownFields
138-
Pipelines AnyConfig `json:"pipelines" yaml:"pipelines"`
181+
Pipelines map[string]*Pipeline `json:"pipelines" yaml:"pipelines"`
139182
}
140183

141184
// MetricsPort gets the port number for the metrics endpoint from the collector config if it has been set.

apis/v1beta1/config_test.go

+119-12
Original file line numberDiff line numberDiff line change
@@ -145,11 +145,11 @@ func TestConfigYaml(t *testing.T) {
145145
"insights": "yeah!",
146146
},
147147
},
148-
Pipelines: AnyConfig{
149-
Object: map[string]interface{}{
150-
"receivers": []string{"otlp"},
151-
"processors": []string{"modify_2000"},
152-
"exporters": []string{"otlp/exporter", "con"},
148+
Pipelines: map[string]*Pipeline{
149+
"traces": {
150+
Receivers: []string{"otlp"},
151+
Processors: []string{"modify_2000"},
152+
Exporters: []string{"otlp/exporter", "con"},
153153
},
154154
},
155155
},
@@ -173,13 +173,14 @@ service:
173173
telemetry:
174174
insights: yeah!
175175
pipelines:
176-
exporters:
177-
- otlp/exporter
178-
- con
179-
processors:
180-
- modify_2000
181-
receivers:
182-
- otlp
176+
traces:
177+
exporters:
178+
- otlp/exporter
179+
- con
180+
processors:
181+
- modify_2000
182+
receivers:
183+
- otlp
183184
`
184185

185186
assert.Equal(t, expected, yamlCollector)
@@ -278,3 +279,109 @@ func TestConfigToMetricsPort(t *testing.T) {
278279
})
279280
}
280281
}
282+
283+
func TestConfig_GetEnabledComponents(t *testing.T) {
284+
tests := []struct {
285+
name string
286+
file string
287+
want map[ComponentType]map[string]interface{}
288+
}{
289+
290+
{
291+
name: "connectors",
292+
file: "testdata/otelcol-connectors.yaml",
293+
want: map[ComponentType]map[string]interface{}{
294+
ComponentTypeReceiver: {
295+
"foo": struct{}{},
296+
"count": struct{}{},
297+
},
298+
ComponentTypeProcessor: {},
299+
ComponentTypeExporter: {
300+
"bar": struct{}{},
301+
"count": struct{}{},
302+
},
303+
},
304+
},
305+
{
306+
name: "couchbase",
307+
file: "testdata/otelcol-couchbase.yaml",
308+
want: map[ComponentType]map[string]interface{}{
309+
ComponentTypeReceiver: {
310+
"prometheus/couchbase": struct{}{},
311+
},
312+
ComponentTypeProcessor: {
313+
"filter/couchbase": struct{}{},
314+
"metricstransform/couchbase": struct{}{},
315+
"transform/couchbase": struct{}{},
316+
},
317+
ComponentTypeExporter: {
318+
"prometheus": struct{}{},
319+
},
320+
},
321+
},
322+
{
323+
name: "demo",
324+
file: "testdata/otelcol-demo.yaml",
325+
want: map[ComponentType]map[string]interface{}{
326+
ComponentTypeReceiver: {
327+
"otlp": struct{}{},
328+
},
329+
ComponentTypeProcessor: {
330+
"batch": struct{}{},
331+
},
332+
ComponentTypeExporter: {
333+
"debug": struct{}{},
334+
"zipkin": struct{}{},
335+
"otlp": struct{}{},
336+
"prometheus": struct{}{},
337+
},
338+
},
339+
},
340+
{
341+
name: "extensions",
342+
file: "testdata/otelcol-extensions.yaml",
343+
want: map[ComponentType]map[string]interface{}{
344+
ComponentTypeReceiver: {
345+
"otlp": struct{}{},
346+
},
347+
ComponentTypeProcessor: {},
348+
ComponentTypeExporter: {
349+
"otlp/auth": struct{}{},
350+
},
351+
},
352+
},
353+
{
354+
name: "filelog",
355+
file: "testdata/otelcol-filelog.yaml",
356+
want: map[ComponentType]map[string]interface{}{
357+
ComponentTypeReceiver: {
358+
"filelog": struct{}{},
359+
},
360+
ComponentTypeProcessor: {},
361+
ComponentTypeExporter: {
362+
"debug": struct{}{},
363+
},
364+
},
365+
},
366+
{
367+
name: "null",
368+
file: "testdata/otelcol-null-values.yaml",
369+
want: map[ComponentType]map[string]interface{}{
370+
ComponentTypeReceiver: {},
371+
ComponentTypeProcessor: {},
372+
ComponentTypeExporter: {},
373+
},
374+
},
375+
}
376+
for _, tt := range tests {
377+
t.Run(tt.name, func(t *testing.T) {
378+
collectorYaml, err := os.ReadFile(tt.file)
379+
require.NoError(t, err)
380+
381+
c := &Config{}
382+
err = go_yaml.Unmarshal(collectorYaml, c)
383+
require.NoError(t, err)
384+
assert.Equalf(t, tt.want, c.GetEnabledComponents(), "GetEnabledComponents()")
385+
})
386+
}
387+
}

apis/v1beta1/opentelemetrycollector_types.go

+1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ type OpenTelemetryCollector struct {
4949
Status OpenTelemetryCollectorStatus `json:"status,omitempty"`
5050
}
5151

52+
// Hub exists to allow for conversion.
5253
func (*OpenTelemetryCollector) Hub() {}
5354

5455
//+kubebuilder:object:root=true

apis/v1beta1/testdata/otelcol-connectors.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ service:
1717
pipelines:
1818
traces:
1919
receivers: [foo]
20+
processors: []
2021
exporters: [count]
2122
metrics:
2223
receivers: [count]
24+
processors: []
2325
exporters: [bar]

apis/v1beta1/testdata/otelcol-filelog.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -81,4 +81,5 @@ service:
8181
pipelines:
8282
logs:
8383
receivers: [filelog]
84+
processors: []
8485
exporters: [debug]

apis/v1beta1/zz_generated.deepcopy.go

+46-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -5522,6 +5522,25 @@ spec:
55225522
type: string
55235523
type: array
55245524
pipelines:
5525+
additionalProperties:
5526+
properties:
5527+
exporters:
5528+
items:
5529+
type: string
5530+
type: array
5531+
processors:
5532+
items:
5533+
type: string
5534+
type: array
5535+
receivers:
5536+
items:
5537+
type: string
5538+
type: array
5539+
required:
5540+
- exporters
5541+
- processors
5542+
- receivers
5543+
type: object
55255544
type: object
55265545
x-kubernetes-preserve-unknown-fields: true
55275546
telemetry:

config/crd/bases/opentelemetry.io_opentelemetrycollectors.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -5508,6 +5508,25 @@ spec:
55085508
type: string
55095509
type: array
55105510
pipelines:
5511+
additionalProperties:
5512+
properties:
5513+
exporters:
5514+
items:
5515+
type: string
5516+
type: array
5517+
processors:
5518+
items:
5519+
type: string
5520+
type: array
5521+
receivers:
5522+
items:
5523+
type: string
5524+
type: array
5525+
required:
5526+
- exporters
5527+
- processors
5528+
- receivers
5529+
type: object
55115530
type: object
55125531
x-kubernetes-preserve-unknown-fields: true
55135532
telemetry:

0 commit comments

Comments
 (0)