fix Prometheus config and Grafana Dashboard for GPU support

Author: nicolaven

grafana/dashboards/gpu.json

@@ -18,8 +18,8 @@
   "editable": true,
   "gnetId": 11752,
   "graphTooltip": 0,
-  "id": 7,
-  "iteration": 1605631221337,
+  "id": 5,
+  "iteration": 1605896702545,
   "links": [
     {
       "icon": "external link",
@@ -951,7 +951,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.2",
+      "pluginVersion": "7.3.3",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1060,7 +1060,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.2",
+      "pluginVersion": "7.3.3",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1160,7 +1160,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.2",
+      "pluginVersion": "7.3.3",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1264,7 +1264,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.2",
+      "pluginVersion": "7.3.3",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1381,7 +1381,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.2",
+      "pluginVersion": "7.3.3",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1478,7 +1478,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.2",
+      "pluginVersion": "7.3.3",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1578,7 +1578,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.2",
+      "pluginVersion": "7.3.3",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1680,7 +1680,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.2",
+      "pluginVersion": "7.3.3",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1777,7 +1777,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.2",
+      "pluginVersion": "7.3.3",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1848,21 +1848,16 @@
     "list": [
       {
         "allValue": null,
-        "current": {
-          "selected": true,
-          "text": "i-002fd33e53d27b6dd",
-          "value": "i-002fd33e53d27b6dd"
-        },
         "datasource": "prometheus",
-        "definition": "label_values(node_uname_info{job=~\"ec2_instances\"}, instance_id)",
+        "definition": "label_values(node_uname_info{job=~\"ec2_instances\",instance_type=~\"g[3-4].*\"}, instance_id)",
         "error": null,
         "hide": 0,
         "includeAll": false,
         "label": "Instance ID",
         "multi": false,
         "name": "instance_id",
         "options": [],
-        "query": "label_values(node_uname_info{job=~\"ec2_instances\"}, instance_id)",
+        "query": "label_values(node_uname_info{job=~\"ec2_instances\",instance_type=~\"g[3-4].*\"}, instance_id)",
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
@@ -1907,5 +1902,5 @@
   "timezone": "browser",
   "title": "GPU Nodes",
   "uid": "hpcsyl6zhqk",
-  "version": 19
+  "version": 1
 }

prometheus/prometheus.yml

@@ -28,6 +28,12 @@ scrape_configs:
     ec2_sd_configs:
       - port: 9100
         refresh_interval: 10s
+      - port: 9400
+        refresh_interval: 10s
+        filters:
+          - name: instance-type
+            values:
+              - g* 
     relabel_configs:
       - source_labels: [__meta_ec2_tag_Name]
         target_label: instance_name
@@ -46,28 +52,4 @@ scrape_configs:
       - source_labels: [__meta_ec2_instance_type]
         target_label: instance_type
       - source_labels: [__meta_ec2_vpc_id]
-        target_label: instance_vpc
-  - job_name: 'ec2_gpu_instances'
-    scrape_interval: 5s
-    ec2_sd_configs:
-      - port: 9400
-        refresh_interval: 10s
-    relabel_configs:
-      - source_labels: [__meta_ec2_tag_Name]
-        target_label: gpu_instance_name
-      - source_labels: [__meta_ec2_tag_Application]
-        target_label: gpu_instance_grafana
-        regex: __Application__
-        action: keep
-      - source_labels: [__meta_ec2_instance_id]
-        target_label: gpu_instance_id
-      - source_labels: [__meta_ec2_availability_zone]
-        target_label: gpu_instance_az
-      - source_labels: [__meta_ec2_instance_state]
-        regex: running
-        action: keep
-        target_label: gpu_instance_state
-      - source_labels: [__meta_ec2_instance_type]
-        target_label: gpu_instance_type
-      - source_labels: [__meta_ec2_vpc_id]
-        target_label: gpu_instance_vpc
+        target_label: instance_vpc