Merge pull request #104 from splunk/update-cAdvisor-metric

Ignored cAdvisor metric with empty container name

Author: hvaghani221

lib/fluent/plugin/in_kubernetes_metrics.rb

@@ -614,6 +614,7 @@ def emit_cadvisor_metrics(metrics)
         metrics.each do |metric|
 
           next if metric[0] == '#' or not container_name = grep_using_regex(metric, /container(?:_name)?="([^"]*)"/)
+          next if container_name.empty?
 
           metric_str, metric_val = metric.split(' ')
           metric_val = metric_val.to_f if metric_val.is_a? String

test/metrics_cadvisor.txt

@@ -907,6 +907,7 @@ container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstab
 container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 1.786404787e+09
 container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 1.786399729e+09
 container_network_receive_bytes_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 1.786393994e+09
+container_network_receive_bytes_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad00container_network_receive_errors_total4a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 1.786399238e+09
 # HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving
 # TYPE container_network_receive_errors_total counter
 container_network_receive_errors_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0
@@ -918,6 +919,7 @@ container_network_receive_errors_total{container_name="POD",id="/kubepods/bursta
 container_network_receive_errors_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0
 container_network_receive_errors_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0
 container_network_receive_errors_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0
+container_network_receive_errors_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0
 # HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving
 # TYPE container_network_receive_packets_dropped_total counter
 container_network_receive_packets_dropped_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0
@@ -929,6 +931,7 @@ container_network_receive_packets_dropped_total{container_name="POD",id="/kubepo
 container_network_receive_packets_dropped_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0
 container_network_receive_packets_dropped_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0
 container_network_receive_packets_dropped_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0
+container_network_receive_packets_dropped_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0
 # HELP container_network_receive_packets_total Cumulative count of packets received
 # TYPE container_network_receive_packets_total counter
 container_network_receive_packets_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 7.074054e+06
@@ -940,6 +943,7 @@ container_network_receive_packets_total{container_name="POD",id="/kubepods/burst
 container_network_receive_packets_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 7.073992e+06
 container_network_receive_packets_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 7.073969e+06
 container_network_receive_packets_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 7.073917e+06
+container_network_receive_packets_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 7.073237e+06
 # HELP container_network_tcp_usage_total tcp connection usage statistic for container
 # TYPE container_network_tcp_usage_total gauge
 container_network_tcp_usage_total{container_name="",id="/",image="",name="",namespace="",pod_name="",tcp_state="close"} 0
@@ -1250,6 +1254,7 @@ container_network_transmit_bytes_total{container_name="POD",id="/kubepods/bursta
 container_network_transmit_bytes_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 3.253087401e+09
 container_network_transmit_bytes_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 3.253065965e+09
 container_network_transmit_bytes_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 3.253037394e+09
+container_network_transmit_bytes_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 3.253037394e+09
 # HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting
 # TYPE container_network_transmit_errors_total counter
 container_network_transmit_errors_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0
@@ -1261,6 +1266,7 @@ container_network_transmit_errors_total{container_name="POD",id="/kubepods/burst
 container_network_transmit_errors_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0
 container_network_transmit_errors_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0
 container_network_transmit_errors_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0
+container_network_transmit_errors_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0
 # HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting
 # TYPE container_network_transmit_packets_dropped_total counter
 container_network_transmit_packets_dropped_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 0
@@ -1272,6 +1278,7 @@ container_network_transmit_packets_dropped_total{container_name="POD",id="/kubep
 container_network_transmit_packets_dropped_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 0
 container_network_transmit_packets_dropped_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 0
 container_network_transmit_packets_dropped_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 0
+container_network_transmit_packets_dropped_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 0
 # HELP container_network_transmit_packets_total Cumulative count of packets transmitted
 # TYPE container_network_transmit_packets_total counter
 container_network_transmit_packets_total{container_name="",id="/",image="",interface="eth0",name="",namespace="",pod_name=""} 7.727546e+06
@@ -1283,6 +1290,7 @@ container_network_transmit_packets_total{container_name="POD",id="/kubepods/burs
 container_network_transmit_packets_total{container_name="POD",id="/kubepods/burstable/pod76a4641508128c1971204ecb3dedd60d/d06b05ca98150013ff4f403599bef93f6eb48a5dd52f3f6dc2e28fbbb207e429",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal_kube-system_76a4641508128c1971204ecb3dedd60d_0",namespace="kube-system",pod_name="kube-scheduler-ip-172-20-57-89.us-east-2.compute.internal"} 7.727485e+06
 container_network_transmit_packets_total{container_name="POD",id="/kubepods/burstable/podb6fbcc401119c99e5f1b51db3f9db506/1e2b5ffcf815cb12c797719af078f469e4a88da09e1f47138e977d77b2ab2e01",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal_kube-system_b6fbcc401119c99e5f1b51db3f9db506_0",namespace="kube-system",pod_name="etcd-server-events-ip-172-20-57-89.us-east-2.compute.internal"} 7.727451e+06
 container_network_transmit_packets_total{container_name="POD",id="/kubepods/burstable/poddc5e34d8b875a92be236483682c1e45b/12e31f62189efe783bc2bb56056bb653fe866ac8f04fd51d13a73984dea14a40",image="k8s.gcr.io/pause-amd64:3.0",interface="eth0",name="k8s_POD_kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal_kube-system_dc5e34d8b875a92be236483682c1e45b_0",namespace="kube-system",pod_name="kube-apiserver-ip-172-20-57-89.us-east-2.compute.internal"} 7.727389e+06
+container_network_transmit_packets_total{container_name="etcd-container",id="/kubepods/burstable/pod310ff556af68b46e5580c376563548b3/60adb3f0b9e38f68e93d3c7e5d3fa50341d868127c7f4b659e560ed41dad004a",image="k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",name="k8s_etcd-container_etcd-server-ip-172-20-57-89.us-east-2.compute.internal_kube-system_310ff556af68b46e5580c376563548b3_0",namespace="kube-system",pod_name="etcd-server-ip-172-20-57-89.us-east-2.compute.internal",tcp_state="close"} 7.727481e+06
 # HELP container_network_udp_usage_total udp connection usage statistic for container
 # TYPE container_network_udp_usage_total gauge
 container_network_udp_usage_total{container_name="",id="/",image="",name="",namespace="",pod_name="",udp_state="dropped"} 0

test/plugin/test_in_kubernetes_metrics.rb

@@ -61,6 +61,8 @@ class KubernetesMetricsInputTest < Test::Unit::TestCase
 
       container_name = metric.match(/container_name="\S*"/).to_s
       container_name = container_name.split('"')[1]
+      
+      next if container_name.empty?
 
       metric_str, metric_val = metric.split(' ')
       metric_val = metric_val.to_f if metric_val.is_a? String