From 9fdba0e3862b12c7b9645557f83eb9281554f6cc Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 26 Oct 2021 17:58:59 +0200 Subject: [PATCH 01/13] Add some dimensions to kubernetes integration --- packages/kubernetes/changelog.yml | 5 +++++ .../data_stream/apiserver/fields/fields.yml | 13 +++++++++++++ .../data_stream/container/fields/base-fields.yml | 4 ++++ .../data_stream/pod/fields/base-fields.yml | 3 +++ .../state_container/fields/base-fields.yml | 4 ++++ .../data_stream/state_pod/fields/base-fields.yml | 3 +++ packages/kubernetes/manifest.yml | 2 +- 7 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/kubernetes/changelog.yml b/packages/kubernetes/changelog.yml index 718e8bb4143..3cfe296f113 100644 --- a/packages/kubernetes/changelog.yml +++ b/packages/kubernetes/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.4.2-next" + changes: + - description: Add dimension fields + type: enhancement + link: https://github.com/elastic/integrations/pull/2076 - version: "1.4.1" changes: - description: Remove overriding of index pattern on the Kubernetes overview dashboard diff --git a/packages/kubernetes/data_stream/apiserver/fields/fields.yml b/packages/kubernetes/data_stream/apiserver/fields/fields.yml index d53efadb1a4..8ec75318fbe 100644 --- a/packages/kubernetes/data_stream/apiserver/fields/fields.yml +++ b/packages/kubernetes/data_stream/apiserver/fields/fields.yml @@ -2,26 +2,32 @@ type: group fields: - name: request.client + dimension: true type: keyword description: | Client executing requests - name: request.resource + dimension: true type: keyword description: | Requested resource - name: request.subresource + dimension: true type: keyword description: | Requested subresource - name: request.scope + dimension: true type: keyword description: | Request scope (cluster, namespace, resource) - name: request.verb + dimension: true type: keyword description: | HTTP verb - name: request.code + dimension: true type: keyword description: | HTTP code @@ -34,30 +40,37 @@ description: | Wether the request uses dry run - name: request.kind + dimension: true type: keyword description: | Kind of request - name: request.component + dimension: true type: keyword description: | Component handling the request - name: request.group + dimension: true type: keyword description: | API group for the resource - name: request.version + dimension: true type: keyword description: | version for the group - name: request.handler + dimension: true type: keyword description: | Request handler - name: request.method + dimension: true type: keyword description: | HTTP method - name: request.host + dimension: true type: keyword description: | Request host diff --git a/packages/kubernetes/data_stream/container/fields/base-fields.yml b/packages/kubernetes/data_stream/container/fields/base-fields.yml index dd6d5804e22..52b8a84b44e 100644 --- a/packages/kubernetes/data_stream/container/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/container/fields/base-fields.yml @@ -14,11 +14,13 @@ type: group fields: - name: pod.name + dimension: true type: keyword description: > Kubernetes pod name - name: pod.uid + dimension: true type: keyword description: > Kubernetes pod UID @@ -32,6 +34,7 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name @@ -107,6 +110,7 @@ Kubernetes statefulset name - name: container.name + dimension: true type: keyword description: > Kubernetes container name diff --git a/packages/kubernetes/data_stream/pod/fields/base-fields.yml b/packages/kubernetes/data_stream/pod/fields/base-fields.yml index dd6d5804e22..1447ee08bb3 100644 --- a/packages/kubernetes/data_stream/pod/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/pod/fields/base-fields.yml @@ -14,11 +14,13 @@ type: group fields: - name: pod.name + dimension: true type: keyword description: > Kubernetes pod name - name: pod.uid + dimension: true type: keyword description: > Kubernetes pod UID @@ -32,6 +34,7 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name diff --git a/packages/kubernetes/data_stream/state_container/fields/base-fields.yml b/packages/kubernetes/data_stream/state_container/fields/base-fields.yml index dd6d5804e22..52b8a84b44e 100644 --- a/packages/kubernetes/data_stream/state_container/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_container/fields/base-fields.yml @@ -14,11 +14,13 @@ type: group fields: - name: pod.name + dimension: true type: keyword description: > Kubernetes pod name - name: pod.uid + dimension: true type: keyword description: > Kubernetes pod UID @@ -32,6 +34,7 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name @@ -107,6 +110,7 @@ Kubernetes statefulset name - name: container.name + dimension: true type: keyword description: > Kubernetes container name diff --git a/packages/kubernetes/data_stream/state_pod/fields/base-fields.yml b/packages/kubernetes/data_stream/state_pod/fields/base-fields.yml index 348874b18aa..4de9c425348 100644 --- a/packages/kubernetes/data_stream/state_pod/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_pod/fields/base-fields.yml @@ -14,11 +14,13 @@ type: group fields: - name: pod.name + dimension: true type: keyword description: > Kubernetes pod name - name: pod.uid + dimension: true type: keyword description: > Kubernetes pod UID @@ -32,6 +34,7 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name diff --git a/packages/kubernetes/manifest.yml b/packages/kubernetes/manifest.yml index 549b76c41ea..7d4e6b34204 100644 --- a/packages/kubernetes/manifest.yml +++ b/packages/kubernetes/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: kubernetes title: Kubernetes -version: 1.4.1 +version: 1.4.2-next license: basic description: Collect logs and metrics from Kubernetes clusters with Elastic Agent. type: integration From 61efffe94055cdf4135e75e74bb05596b8ab1999 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 9 Nov 2021 12:27:34 +0100 Subject: [PATCH 02/13] Use container.id as dimension too --- packages/kubernetes/data_stream/container/fields/agent.yml | 5 ----- packages/kubernetes/data_stream/container/fields/ecs.yml | 3 +++ .../kubernetes/data_stream/state_container/fields/agent.yml | 5 ----- .../kubernetes/data_stream/state_container/fields/ecs.yml | 3 +++ 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/kubernetes/data_stream/container/fields/agent.yml b/packages/kubernetes/data_stream/container/fields/agent.yml index da4e652c53b..9dfc8d1aebc 100644 --- a/packages/kubernetes/data_stream/container/fields/agent.yml +++ b/packages/kubernetes/data_stream/container/fields/agent.yml @@ -62,11 +62,6 @@ These fields help correlate data based containers from any runtime.' type: group fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - name: image.name level: extended type: keyword diff --git a/packages/kubernetes/data_stream/container/fields/ecs.yml b/packages/kubernetes/data_stream/container/fields/ecs.yml index cd4e3a89b1a..d0dec5a850b 100644 --- a/packages/kubernetes/data_stream/container/fields/ecs.yml +++ b/packages/kubernetes/data_stream/container/fields/ecs.yml @@ -8,3 +8,6 @@ name: orchestrator.cluster.name - external: ecs name: orchestrator.cluster.url +- external: ecs + name: container.id + dimension: true diff --git a/packages/kubernetes/data_stream/state_container/fields/agent.yml b/packages/kubernetes/data_stream/state_container/fields/agent.yml index da4e652c53b..9dfc8d1aebc 100644 --- a/packages/kubernetes/data_stream/state_container/fields/agent.yml +++ b/packages/kubernetes/data_stream/state_container/fields/agent.yml @@ -62,11 +62,6 @@ These fields help correlate data based containers from any runtime.' type: group fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - name: image.name level: extended type: keyword diff --git a/packages/kubernetes/data_stream/state_container/fields/ecs.yml b/packages/kubernetes/data_stream/state_container/fields/ecs.yml index 31cb2817b80..b83f49d9d58 100644 --- a/packages/kubernetes/data_stream/state_container/fields/ecs.yml +++ b/packages/kubernetes/data_stream/state_container/fields/ecs.yml @@ -10,3 +10,6 @@ name: orchestrator.cluster.name - external: ecs name: orchestrator.cluster.url +- external: ecs + name: container.id + dimension: true From 682f8896d944fb318d78586370b4c5f498f1bbb5 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 15 Nov 2021 16:07:21 +0100 Subject: [PATCH 03/13] container_logs --- .../data_stream/container_logs/fields/agent.yml | 10 ---------- .../data_stream/container_logs/fields/base-fields.yml | 5 +++++ .../data_stream/container_logs/fields/ecs.yml | 6 ++++++ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/kubernetes/data_stream/container_logs/fields/agent.yml b/packages/kubernetes/data_stream/container_logs/fields/agent.yml index da4e652c53b..308fb8882cf 100644 --- a/packages/kubernetes/data_stream/container_logs/fields/agent.yml +++ b/packages/kubernetes/data_stream/container_logs/fields/agent.yml @@ -62,11 +62,6 @@ These fields help correlate data based containers from any runtime.' type: group fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - name: image.name level: extended type: keyword @@ -77,11 +72,6 @@ type: object object_type: keyword description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 diff --git a/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml b/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml index 314d7c522d3..34e4564f970 100644 --- a/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml @@ -20,11 +20,13 @@ type: group fields: - name: pod.name + dimension: true type: keyword description: > Kubernetes pod name - name: pod.uid + dimension: true type: keyword description: > Kubernetes pod UID @@ -38,11 +40,13 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name - name: uid + dimension: true type: keyword description: > Kubernetes namespace uid @@ -113,6 +117,7 @@ Kubernetes statefulset name - name: container.name + dimension: true type: keyword description: > Kubernetes container name diff --git a/packages/kubernetes/data_stream/container_logs/fields/ecs.yml b/packages/kubernetes/data_stream/container_logs/fields/ecs.yml index f6818be260a..0751f613ab3 100644 --- a/packages/kubernetes/data_stream/container_logs/fields/ecs.yml +++ b/packages/kubernetes/data_stream/container_logs/fields/ecs.yml @@ -14,5 +14,11 @@ name: agent.ephemeral_id - external: ecs name: agent.version +- external: ecs + name: container.id + dimension: true +- external: ecs + name: container.name + dimension: true - external: ecs name: message From 7b669399956edfb4b5610dc9a3fe36efb2667e9f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 15 Nov 2021 16:17:09 +0100 Subject: [PATCH 04/13] event, controller manager --- .../data_stream/controllermanager/fields/fields.yml | 6 ++++++ packages/kubernetes/data_stream/event/fields/fields.yml | 9 +++++++++ packages/kubernetes/data_stream/event/sample_event.json | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/kubernetes/data_stream/controllermanager/fields/fields.yml b/packages/kubernetes/data_stream/controllermanager/fields/fields.yml index d3a4e449b3e..1ef76f93e8b 100644 --- a/packages/kubernetes/data_stream/controllermanager/fields/fields.yml +++ b/packages/kubernetes/data_stream/controllermanager/fields/fields.yml @@ -2,26 +2,32 @@ type: group fields: - name: handler + dimension: true type: keyword description: | Request handler - name: code + dimension: true type: keyword description: | HTTP code - name: method + dimension: true type: keyword description: | HTTP method - name: host + dimension: true type: keyword description: | Request host - name: name + dimension: true type: keyword description: | Name for the resource - name: zone + dimension: true type: keyword description: | Infrastructure zone diff --git a/packages/kubernetes/data_stream/event/fields/fields.yml b/packages/kubernetes/data_stream/event/fields/fields.yml index 5f156a6a2c7..9bcbf000165 100644 --- a/packages/kubernetes/data_stream/event/fields/fields.yml +++ b/packages/kubernetes/data_stream/event/fields/fields.yml @@ -22,10 +22,12 @@ description: | Message recorded for the given event - name: reason + dimension: true type: keyword description: | Reason recorded for the given event - name: type + dimension: true type: keyword description: | Type of the given event @@ -33,10 +35,12 @@ type: group fields: - name: component + dimension: true type: keyword description: | Component from which the event is generated - name: host + dimension: true type: keyword description: | Node name on which the event is generated @@ -51,18 +55,22 @@ description: | Timestamp of creation of the given event - name: generate_name + dimension: true type: keyword description: | Generate name of the event - name: name + dimension: true type: keyword description: | Name of the event - name: namespace + dimension: true type: keyword description: | Namespace in which event was generated - name: resource_version + dimension: true type: keyword description: | Version of the event resource @@ -94,6 +102,7 @@ description: | resource version of the object - name: uid + dimension: true type: keyword description: | uid version of the object diff --git a/packages/kubernetes/data_stream/event/sample_event.json b/packages/kubernetes/data_stream/event/sample_event.json index 01c0c93fcfd..74c433743c7 100644 --- a/packages/kubernetes/data_stream/event/sample_event.json +++ b/packages/kubernetes/data_stream/event/sample_event.json @@ -75,4 +75,4 @@ "codename": "Core" } } -} \ No newline at end of file +} From 61022f5c448e9ea3802b138576f303afb83b5fa7 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 15 Nov 2021 16:21:14 +0100 Subject: [PATCH 05/13] Node --- packages/kubernetes/data_stream/node/fields/base-fields.yml | 1 + packages/kubernetes/data_stream/node/fields/ecs.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/packages/kubernetes/data_stream/node/fields/base-fields.yml b/packages/kubernetes/data_stream/node/fields/base-fields.yml index 9bc71c2c7cf..6d6ff7e8594 100644 --- a/packages/kubernetes/data_stream/node/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/node/fields/base-fields.yml @@ -56,6 +56,7 @@ Kubernetes namespace annotations map - name: node.name + dimension: true type: keyword description: > Kubernetes node name diff --git a/packages/kubernetes/data_stream/node/fields/ecs.yml b/packages/kubernetes/data_stream/node/fields/ecs.yml index cd4e3a89b1a..93e8521180a 100644 --- a/packages/kubernetes/data_stream/node/fields/ecs.yml +++ b/packages/kubernetes/data_stream/node/fields/ecs.yml @@ -6,5 +6,7 @@ name: service.type - external: ecs name: orchestrator.cluster.name + dimension: true - external: ecs name: orchestrator.cluster.url + dimension: true From 88f18e45ed4d2419e59afc657b5064b4ad3989e3 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 15 Nov 2021 16:31:13 +0100 Subject: [PATCH 06/13] More dimensions --- .../kubernetes/data_stream/scheduler/fields/fields.yml | 7 +++++++ .../data_stream/state_cronjob/fields/base-fields.yml | 5 +++++ .../kubernetes/data_stream/state_cronjob/fields/fields.yml | 4 ++++ .../data_stream/state_daemonset/fields/base-fields.yml | 2 ++ .../data_stream/state_daemonset/fields/fields.yml | 1 + .../data_stream/state_deployment/fields/base-fields.yml | 2 ++ .../data_stream/state_job/fields/base-fields.yml | 2 ++ .../kubernetes/data_stream/state_job/fields/fields.yml | 1 + .../data_stream/state_node/fields/base-fields.yml | 1 + packages/kubernetes/data_stream/state_node/fields/ecs.yml | 2 ++ .../data_stream/state_persistentvolume/fields/fields.yml | 1 + .../state_persistentvolumeclaim/fields/base-fields.yml | 2 ++ .../state_persistentvolumeclaim/fields/fields.yml | 1 + .../data_stream/state_replicaset/fields/base-fields.yml | 4 ++++ .../data_stream/state_resourcequota/fields/base-fields.yml | 2 ++ .../data_stream/state_resourcequota/fields/fields.yml | 3 +++ .../data_stream/state_service/fields/base-fields.yml | 2 ++ .../kubernetes/data_stream/state_service/fields/fields.yml | 1 + .../data_stream/state_statefulset/fields/base-fields.yml | 3 +++ .../data_stream/state_storageclass/fields/fields.yml | 1 + .../kubernetes/data_stream/system/fields/base-fields.yml | 1 + packages/kubernetes/data_stream/system/fields/fields.yml | 1 + .../kubernetes/data_stream/volume/fields/base-fields.yml | 2 ++ packages/kubernetes/data_stream/volume/fields/fields.yml | 1 + 24 files changed, 52 insertions(+) diff --git a/packages/kubernetes/data_stream/scheduler/fields/fields.yml b/packages/kubernetes/data_stream/scheduler/fields/fields.yml index e6f7f48ee72..fa717504afa 100644 --- a/packages/kubernetes/data_stream/scheduler/fields/fields.yml +++ b/packages/kubernetes/data_stream/scheduler/fields/fields.yml @@ -2,30 +2,37 @@ type: group fields: - name: handler + dimension: true type: keyword description: | Request handler - name: code + dimension: true type: keyword description: | HTTP code - name: method + dimension: true type: keyword description: | HTTP method - name: host + dimension: true type: keyword description: | Request host - name: name + dimension: true type: keyword description: | Name for the resource - name: result + dimension: true type: keyword description: | Schedule attempt result - name: operation + dimension: true type: keyword description: | Scheduling operation diff --git a/packages/kubernetes/data_stream/state_cronjob/fields/base-fields.yml b/packages/kubernetes/data_stream/state_cronjob/fields/base-fields.yml index 9bc71c2c7cf..cd7725582fd 100644 --- a/packages/kubernetes/data_stream/state_cronjob/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_cronjob/fields/base-fields.yml @@ -14,11 +14,13 @@ type: group fields: - name: pod.name + dimension: true type: keyword description: > Kubernetes pod name - name: pod.uid + dimension: true type: keyword description: > Kubernetes pod UID @@ -32,11 +34,13 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name - name: uid + dimension: true type: keyword description: > Kubernetes namespace uid @@ -102,6 +106,7 @@ Kubernetes statefulset name - name: container.name + dimension: true type: keyword description: > Kubernetes container name diff --git a/packages/kubernetes/data_stream/state_cronjob/fields/fields.yml b/packages/kubernetes/data_stream/state_cronjob/fields/fields.yml index 53d086b4d13..5e3470530c7 100644 --- a/packages/kubernetes/data_stream/state_cronjob/fields/fields.yml +++ b/packages/kubernetes/data_stream/state_cronjob/fields/fields.yml @@ -2,12 +2,15 @@ type: group fields: - name: name + dimension: true type: keyword description: Cronjob name - name: schedule + dimension: true type: keyword description: Cronjob schedule - name: concurrency + dimension: true type: keyword description: Concurrency policy - name: active.count @@ -15,6 +18,7 @@ metric_type: gauge description: Number of active pods for the cronjob - name: is_suspended + dimension: true type: boolean description: Whether the cronjob is suspended - name: created.sec diff --git a/packages/kubernetes/data_stream/state_daemonset/fields/base-fields.yml b/packages/kubernetes/data_stream/state_daemonset/fields/base-fields.yml index 9bc71c2c7cf..1b4f7bd5375 100644 --- a/packages/kubernetes/data_stream/state_daemonset/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_daemonset/fields/base-fields.yml @@ -32,11 +32,13 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name - name: uid + dimension: true type: keyword description: > Kubernetes namespace uid diff --git a/packages/kubernetes/data_stream/state_daemonset/fields/fields.yml b/packages/kubernetes/data_stream/state_daemonset/fields/fields.yml index 0e06111729b..c763091832c 100644 --- a/packages/kubernetes/data_stream/state_daemonset/fields/fields.yml +++ b/packages/kubernetes/data_stream/state_daemonset/fields/fields.yml @@ -2,6 +2,7 @@ type: group fields: - name: name + dimension: true type: keyword - name: replicas type: group diff --git a/packages/kubernetes/data_stream/state_deployment/fields/base-fields.yml b/packages/kubernetes/data_stream/state_deployment/fields/base-fields.yml index 9bc71c2c7cf..07523e6231b 100644 --- a/packages/kubernetes/data_stream/state_deployment/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_deployment/fields/base-fields.yml @@ -87,11 +87,13 @@ Kubernetes Service selectors map - name: replicaset.name + dimension: true type: keyword description: > Kubernetes replicaset name - name: deployment.name + dimension: true type: keyword description: > Kubernetes deployment name diff --git a/packages/kubernetes/data_stream/state_job/fields/base-fields.yml b/packages/kubernetes/data_stream/state_job/fields/base-fields.yml index 9bc71c2c7cf..1b4f7bd5375 100644 --- a/packages/kubernetes/data_stream/state_job/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_job/fields/base-fields.yml @@ -32,11 +32,13 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name - name: uid + dimension: true type: keyword description: > Kubernetes namespace uid diff --git a/packages/kubernetes/data_stream/state_job/fields/fields.yml b/packages/kubernetes/data_stream/state_job/fields/fields.yml index cbf96e3177e..dd96148faec 100644 --- a/packages/kubernetes/data_stream/state_job/fields/fields.yml +++ b/packages/kubernetes/data_stream/state_job/fields/fields.yml @@ -2,6 +2,7 @@ type: group fields: - name: name + dimension: true type: keyword description: > The name of the job resource diff --git a/packages/kubernetes/data_stream/state_node/fields/base-fields.yml b/packages/kubernetes/data_stream/state_node/fields/base-fields.yml index 9bc71c2c7cf..6d6ff7e8594 100644 --- a/packages/kubernetes/data_stream/state_node/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_node/fields/base-fields.yml @@ -56,6 +56,7 @@ Kubernetes namespace annotations map - name: node.name + dimension: true type: keyword description: > Kubernetes node name diff --git a/packages/kubernetes/data_stream/state_node/fields/ecs.yml b/packages/kubernetes/data_stream/state_node/fields/ecs.yml index cd4e3a89b1a..b6e96032932 100644 --- a/packages/kubernetes/data_stream/state_node/fields/ecs.yml +++ b/packages/kubernetes/data_stream/state_node/fields/ecs.yml @@ -5,6 +5,8 @@ - external: ecs name: service.type - external: ecs + dimension: true name: orchestrator.cluster.name - external: ecs + dimension: true name: orchestrator.cluster.url diff --git a/packages/kubernetes/data_stream/state_persistentvolume/fields/fields.yml b/packages/kubernetes/data_stream/state_persistentvolume/fields/fields.yml index 883ddcbd220..e441ac8f9c4 100644 --- a/packages/kubernetes/data_stream/state_persistentvolume/fields/fields.yml +++ b/packages/kubernetes/data_stream/state_persistentvolume/fields/fields.yml @@ -2,6 +2,7 @@ type: group fields: - name: name + dimension: true type: keyword description: Volume name. - name: capacity.bytes diff --git a/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/base-fields.yml b/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/base-fields.yml index 9bc71c2c7cf..1b4f7bd5375 100644 --- a/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/base-fields.yml @@ -32,11 +32,13 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name - name: uid + dimension: true type: keyword description: > Kubernetes namespace uid diff --git a/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/fields.yml b/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/fields.yml index 3b5f813d07b..6f11ce66b78 100644 --- a/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/fields.yml +++ b/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/fields.yml @@ -2,6 +2,7 @@ type: group fields: - name: name + dimension: true type: keyword description: PVC name. - name: volume_name diff --git a/packages/kubernetes/data_stream/state_replicaset/fields/base-fields.yml b/packages/kubernetes/data_stream/state_replicaset/fields/base-fields.yml index 9bc71c2c7cf..4bcc999dc2f 100644 --- a/packages/kubernetes/data_stream/state_replicaset/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_replicaset/fields/base-fields.yml @@ -32,11 +32,13 @@ type: group fields: - name: name + dimensiont: true type: keyword description: > Kubernetes namespace name - name: uid + dimensiont: true type: keyword description: > Kubernetes namespace uid @@ -87,11 +89,13 @@ Kubernetes Service selectors map - name: replicaset.name + dimensiont: true type: keyword description: > Kubernetes replicaset name - name: deployment.name + dimensiont: true type: keyword description: > Kubernetes deployment name diff --git a/packages/kubernetes/data_stream/state_resourcequota/fields/base-fields.yml b/packages/kubernetes/data_stream/state_resourcequota/fields/base-fields.yml index 9bc71c2c7cf..1b4f7bd5375 100644 --- a/packages/kubernetes/data_stream/state_resourcequota/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_resourcequota/fields/base-fields.yml @@ -32,11 +32,13 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name - name: uid + dimension: true type: keyword description: > Kubernetes namespace uid diff --git a/packages/kubernetes/data_stream/state_resourcequota/fields/fields.yml b/packages/kubernetes/data_stream/state_resourcequota/fields/fields.yml index 015f40048ad..530619270cf 100644 --- a/packages/kubernetes/data_stream/state_resourcequota/fields/fields.yml +++ b/packages/kubernetes/data_stream/state_resourcequota/fields/fields.yml @@ -11,11 +11,14 @@ metric_type: gauge description: Quota informed (hard or used) for the resource - name: name + dimension: true type: keyword description: ResourceQuota name - name: type + dimension: true type: keyword description: Quota information type, `hard` or `used` - name: resource + dimension: true type: keyword description: Resource name the quota applies to diff --git a/packages/kubernetes/data_stream/state_service/fields/base-fields.yml b/packages/kubernetes/data_stream/state_service/fields/base-fields.yml index 9bc71c2c7cf..1b4f7bd5375 100644 --- a/packages/kubernetes/data_stream/state_service/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_service/fields/base-fields.yml @@ -32,11 +32,13 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name - name: uid + dimension: true type: keyword description: > Kubernetes namespace uid diff --git a/packages/kubernetes/data_stream/state_service/fields/fields.yml b/packages/kubernetes/data_stream/state_service/fields/fields.yml index 26bd2700e47..0bec4028605 100644 --- a/packages/kubernetes/data_stream/state_service/fields/fields.yml +++ b/packages/kubernetes/data_stream/state_service/fields/fields.yml @@ -2,6 +2,7 @@ type: group fields: - name: name + dimension: true type: keyword description: Service name. - name: cluster_ip diff --git a/packages/kubernetes/data_stream/state_statefulset/fields/base-fields.yml b/packages/kubernetes/data_stream/state_statefulset/fields/base-fields.yml index 9bc71c2c7cf..084aee9d794 100644 --- a/packages/kubernetes/data_stream/state_statefulset/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/state_statefulset/fields/base-fields.yml @@ -32,11 +32,13 @@ type: group fields: - name: name + dimensions: true type: keyword description: > Kubernetes namespace name - name: uid + dimensions: true type: keyword description: > Kubernetes namespace uid @@ -97,6 +99,7 @@ Kubernetes deployment name - name: statefulset.name + dimensions: true type: keyword description: > Kubernetes statefulset name diff --git a/packages/kubernetes/data_stream/state_storageclass/fields/fields.yml b/packages/kubernetes/data_stream/state_storageclass/fields/fields.yml index c190f1dbb1c..6a0d31a6274 100644 --- a/packages/kubernetes/data_stream/state_storageclass/fields/fields.yml +++ b/packages/kubernetes/data_stream/state_storageclass/fields/fields.yml @@ -2,6 +2,7 @@ type: group fields: - name: name + dimension: true type: keyword description: Storage class name. - name: provisioner diff --git a/packages/kubernetes/data_stream/system/fields/base-fields.yml b/packages/kubernetes/data_stream/system/fields/base-fields.yml index 9bc71c2c7cf..6d6ff7e8594 100644 --- a/packages/kubernetes/data_stream/system/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/system/fields/base-fields.yml @@ -56,6 +56,7 @@ Kubernetes namespace annotations map - name: node.name + dimension: true type: keyword description: > Kubernetes node name diff --git a/packages/kubernetes/data_stream/system/fields/fields.yml b/packages/kubernetes/data_stream/system/fields/fields.yml index 67f7f21604c..65fc48d0dd2 100644 --- a/packages/kubernetes/data_stream/system/fields/fields.yml +++ b/packages/kubernetes/data_stream/system/fields/fields.yml @@ -2,6 +2,7 @@ type: group fields: - name: container + dimension: true type: keyword description: | Container name diff --git a/packages/kubernetes/data_stream/volume/fields/base-fields.yml b/packages/kubernetes/data_stream/volume/fields/base-fields.yml index 9bc71c2c7cf..1b4f7bd5375 100644 --- a/packages/kubernetes/data_stream/volume/fields/base-fields.yml +++ b/packages/kubernetes/data_stream/volume/fields/base-fields.yml @@ -32,11 +32,13 @@ type: group fields: - name: name + dimension: true type: keyword description: > Kubernetes namespace name - name: uid + dimension: true type: keyword description: > Kubernetes namespace uid diff --git a/packages/kubernetes/data_stream/volume/fields/fields.yml b/packages/kubernetes/data_stream/volume/fields/fields.yml index 49e510f1b44..afebbf228d8 100644 --- a/packages/kubernetes/data_stream/volume/fields/fields.yml +++ b/packages/kubernetes/data_stream/volume/fields/fields.yml @@ -2,6 +2,7 @@ type: group fields: - name: name + dimension: true type: keyword description: | Volume name From 834d9cb2d1d713cccc039c2b96f20370554bb2f3 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 15 Nov 2021 17:02:42 +0100 Subject: [PATCH 07/13] Fix fmt --- packages/kubernetes/data_stream/event/sample_event.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kubernetes/data_stream/event/sample_event.json b/packages/kubernetes/data_stream/event/sample_event.json index 74c433743c7..01c0c93fcfd 100644 --- a/packages/kubernetes/data_stream/event/sample_event.json +++ b/packages/kubernetes/data_stream/event/sample_event.json @@ -75,4 +75,4 @@ "codename": "Core" } } -} +} \ No newline at end of file From b0289bc5edf7c6f4f5afddabdea6715067773bdb Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 16 Nov 2021 14:10:11 +0100 Subject: [PATCH 08/13] Don't move container fields to ecs.yml --- .../data_stream/container/fields/agent.yml | 6 ++++++ .../kubernetes/data_stream/container/fields/ecs.yml | 3 --- .../data_stream/container_logs/fields/agent.yml | 12 ++++++++++++ .../data_stream/container_logs/fields/ecs.yml | 6 ------ .../data_stream/state_container/fields/agent.yml | 6 ++++++ .../data_stream/state_container/fields/ecs.yml | 3 --- 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/kubernetes/data_stream/container/fields/agent.yml b/packages/kubernetes/data_stream/container/fields/agent.yml index 9dfc8d1aebc..d16c8825520 100644 --- a/packages/kubernetes/data_stream/container/fields/agent.yml +++ b/packages/kubernetes/data_stream/container/fields/agent.yml @@ -62,6 +62,12 @@ These fields help correlate data based containers from any runtime.' type: group fields: + - name: id + dimension: true + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. - name: image.name level: extended type: keyword diff --git a/packages/kubernetes/data_stream/container/fields/ecs.yml b/packages/kubernetes/data_stream/container/fields/ecs.yml index d0dec5a850b..cd4e3a89b1a 100644 --- a/packages/kubernetes/data_stream/container/fields/ecs.yml +++ b/packages/kubernetes/data_stream/container/fields/ecs.yml @@ -8,6 +8,3 @@ name: orchestrator.cluster.name - external: ecs name: orchestrator.cluster.url -- external: ecs - name: container.id - dimension: true diff --git a/packages/kubernetes/data_stream/container_logs/fields/agent.yml b/packages/kubernetes/data_stream/container_logs/fields/agent.yml index 308fb8882cf..dae6838c3ad 100644 --- a/packages/kubernetes/data_stream/container_logs/fields/agent.yml +++ b/packages/kubernetes/data_stream/container_logs/fields/agent.yml @@ -62,6 +62,12 @@ These fields help correlate data based containers from any runtime.' type: group fields: + - name: id + dimension: true + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. - name: image.name level: extended type: keyword @@ -72,6 +78,12 @@ type: object object_type: keyword description: Image labels. + - name: name + dimension: true + level: core + type: keyword + ignore_above: 1024 + description: Container name. - name: host title: Host group: 2 diff --git a/packages/kubernetes/data_stream/container_logs/fields/ecs.yml b/packages/kubernetes/data_stream/container_logs/fields/ecs.yml index 0751f613ab3..f6818be260a 100644 --- a/packages/kubernetes/data_stream/container_logs/fields/ecs.yml +++ b/packages/kubernetes/data_stream/container_logs/fields/ecs.yml @@ -14,11 +14,5 @@ name: agent.ephemeral_id - external: ecs name: agent.version -- external: ecs - name: container.id - dimension: true -- external: ecs - name: container.name - dimension: true - external: ecs name: message diff --git a/packages/kubernetes/data_stream/state_container/fields/agent.yml b/packages/kubernetes/data_stream/state_container/fields/agent.yml index 9dfc8d1aebc..d16c8825520 100644 --- a/packages/kubernetes/data_stream/state_container/fields/agent.yml +++ b/packages/kubernetes/data_stream/state_container/fields/agent.yml @@ -62,6 +62,12 @@ These fields help correlate data based containers from any runtime.' type: group fields: + - name: id + dimension: true + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. - name: image.name level: extended type: keyword diff --git a/packages/kubernetes/data_stream/state_container/fields/ecs.yml b/packages/kubernetes/data_stream/state_container/fields/ecs.yml index b83f49d9d58..31cb2817b80 100644 --- a/packages/kubernetes/data_stream/state_container/fields/ecs.yml +++ b/packages/kubernetes/data_stream/state_container/fields/ecs.yml @@ -10,6 +10,3 @@ name: orchestrator.cluster.name - external: ecs name: orchestrator.cluster.url -- external: ecs - name: container.id - dimension: true From 7645308a30eef0a1458aa4d1c75268fe1c6b970b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 16 Nov 2021 14:11:25 +0100 Subject: [PATCH 09/13] Use a stable version --- packages/kubernetes/changelog.yml | 2 +- packages/kubernetes/manifest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kubernetes/changelog.yml b/packages/kubernetes/changelog.yml index 3cfe296f113..caad0ad4142 100644 --- a/packages/kubernetes/changelog.yml +++ b/packages/kubernetes/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "1.4.2-next" +- version: "1.4.2" changes: - description: Add dimension fields type: enhancement diff --git a/packages/kubernetes/manifest.yml b/packages/kubernetes/manifest.yml index 7d4e6b34204..21cfeb9acf1 100644 --- a/packages/kubernetes/manifest.yml +++ b/packages/kubernetes/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: kubernetes title: Kubernetes -version: 1.4.2-next +version: 1.4.2 license: basic description: Collect logs and metrics from Kubernetes clusters with Elastic Agent. type: integration From fbdb974183bac97d9bba6c30a5302577c3dc6db9 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 16 Nov 2021 14:14:09 +0100 Subject: [PATCH 10/13] Add dimensions to proxy data stream --- packages/kubernetes/data_stream/proxy/fields/fields.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/kubernetes/data_stream/proxy/fields/fields.yml b/packages/kubernetes/data_stream/proxy/fields/fields.yml index 5b50c2299f8..7e37ae78934 100644 --- a/packages/kubernetes/data_stream/proxy/fields/fields.yml +++ b/packages/kubernetes/data_stream/proxy/fields/fields.yml @@ -2,18 +2,22 @@ type: group fields: - name: handler + dimension: true type: keyword description: | Request handler - name: code + dimension: true type: keyword description: | HTTP code - name: method + dimension: true type: keyword description: | HTTP method - name: host + dimension: true type: keyword description: | Request host From 51a374b5b8a1bfe2c9bd73f78d75a0d469ed5b9a Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 16 Nov 2021 14:15:57 +0100 Subject: [PATCH 11/13] Remove orchestrator fields as dimensions --- packages/kubernetes/data_stream/node/fields/ecs.yml | 2 -- packages/kubernetes/data_stream/state_node/fields/ecs.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/packages/kubernetes/data_stream/node/fields/ecs.yml b/packages/kubernetes/data_stream/node/fields/ecs.yml index 93e8521180a..cd4e3a89b1a 100644 --- a/packages/kubernetes/data_stream/node/fields/ecs.yml +++ b/packages/kubernetes/data_stream/node/fields/ecs.yml @@ -6,7 +6,5 @@ name: service.type - external: ecs name: orchestrator.cluster.name - dimension: true - external: ecs name: orchestrator.cluster.url - dimension: true diff --git a/packages/kubernetes/data_stream/state_node/fields/ecs.yml b/packages/kubernetes/data_stream/state_node/fields/ecs.yml index b6e96032932..f74927855bf 100644 --- a/packages/kubernetes/data_stream/state_node/fields/ecs.yml +++ b/packages/kubernetes/data_stream/state_node/fields/ecs.yml @@ -6,7 +6,5 @@ name: service.type - external: ecs dimension: true - name: orchestrator.cluster.name - external: ecs dimension: true - name: orchestrator.cluster.url From 7cdaac24c7ea1f480d5332004ee9c2a2fe447230 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 16 Nov 2021 14:20:41 +0100 Subject: [PATCH 12/13] Revert field level changed by mistake --- packages/kubernetes/data_stream/container_logs/fields/agent.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kubernetes/data_stream/container_logs/fields/agent.yml b/packages/kubernetes/data_stream/container_logs/fields/agent.yml index dae6838c3ad..5959b701dc1 100644 --- a/packages/kubernetes/data_stream/container_logs/fields/agent.yml +++ b/packages/kubernetes/data_stream/container_logs/fields/agent.yml @@ -80,7 +80,7 @@ description: Image labels. - name: name dimension: true - level: core + level: extended type: keyword ignore_above: 1024 description: Container name. From f753d85d8b03adb78a73833dea9a67e1fba614f8 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 16 Nov 2021 14:48:53 +0100 Subject: [PATCH 13/13] Recover wrong lines removed --- packages/kubernetes/data_stream/state_node/fields/ecs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kubernetes/data_stream/state_node/fields/ecs.yml b/packages/kubernetes/data_stream/state_node/fields/ecs.yml index f74927855bf..cd4e3a89b1a 100644 --- a/packages/kubernetes/data_stream/state_node/fields/ecs.yml +++ b/packages/kubernetes/data_stream/state_node/fields/ecs.yml @@ -5,6 +5,6 @@ - external: ecs name: service.type - external: ecs - dimension: true + name: orchestrator.cluster.name - external: ecs - dimension: true + name: orchestrator.cluster.url