Merge pull request rubrikinc#14 from rubrikinc/failedjobstshoot

Fixed client dumping on receiving no, or bad data back from Rubrik

Author: Draper1

quickstart.md

@@ -33,6 +33,8 @@ docker build -t rubrikinc/prometheus-client -f Dockerfile .
 
 The resulting docker image will be in the local repository on the server.
 
+This docker image is also available on the Docker Hub at [this link](https://hub.docker.com/repository/docker/rubrikinc/rubrik-prometheus-client).
+
 ## Using the Prometheus Agent
 
 Ensure that the following environment variables exist, and are defined: `rubrik_cdm_node_ip`, `rubrik_cdm_username`, `rubrik_cdm_password`.
@@ -53,3 +55,18 @@ docker run -d -t -e rubrik_cdm_node_ip=$rubrik_cdm_node_ip \
 ```
 
 This will map port 8080 inside the container, to port 8080 on the docker host. Metrics will then be browsable via `http://localhost:8080/metrics`.
+
+### Using an alternative HTTP port
+
+In order to use an alternative HTTP port (if default of 8080 is already in use, or using it is not desirable) we can set the below environment variable to override the port:
+
+```bash
+export RUBRIK_PROMETHEUS_PORT=9090
+```
+
+When the application starts, the log will show the port being used:
+
+```none
+2020/10/22 11:21:47 Cluster name: rubrik-1
+2020/10/22 11:21:47 Starting on HTTP port 9090
+```

src/golang/jobs/rubrik_failed_jobs.go

@@ -4,8 +4,9 @@ import (
 	"log"
 	"strconv"
 	"strings"
-	"github.com/rubrikinc/rubrik-sdk-for-go/rubrikcdm"
+
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/rubrikinc/rubrik-sdk-for-go/rubrikcdm"
 )
 
 var (
@@ -36,29 +37,34 @@ func init() {
 
 // GetMssqlFailedJobs ...
 func GetMssqlFailedJobs(rubrik *rubrikcdm.Credentials, clusterName string) {
-	clusterVersion,err := rubrik.ClusterVersion()
+	clusterVersion, err := rubrik.ClusterVersion()
 	if err != nil {
-		log.Println("Error from jobs.GetMssqlFailedJobs: ",err)
+		log.Printf("Error from jobs.GetMssqlFailedJobs: ", err)
+		return
 	}
-	clusterMajorVersion,err := strconv.ParseInt(strings.Split(clusterVersion,".")[0], 10, 64)
+	clusterMajorVersion, err := strconv.ParseInt(strings.Split(clusterVersion, ".")[0], 10, 64)
 	if err != nil {
-		log.Println("Error from jobs.GetMssqlFailedJobs: ",err)
+		log.Printf("Error from jobs.GetMssqlFailedJobs: ", err)
+		return
 	}
-	clusterMinorVersion,err := strconv.ParseInt(strings.Split(clusterVersion,".")[1], 10, 64)
+	clusterMinorVersion, err := strconv.ParseInt(strings.Split(clusterVersion, ".")[1], 10, 64)
 	if err != nil {
-		log.Println("Error from jobs.GetMssqlFailedJobs: ",err)
+		log.Printf("Error from jobs.GetMssqlFailedJobs: ", err)
+		return
 	}
 	if (clusterMajorVersion == 5 && clusterMinorVersion < 2) || clusterMajorVersion < 5 { // cluster version is older than 5.1
-		eventData,err := rubrik.Get("internal","/event_series?status=Failure&event_type=Backup&object_type=Mssql")
+		eventData, err := rubrik.Get("internal", "/event_series?status=Failure&event_type=Backup&object_type=Mssql", 60)
 		if err != nil {
-			log.Println("Error from jobs.GetMssqlFailedJobs: ",err)
+			log.Printf("Error from jobs.GetMssqlFailedJobs: ", err)
+			return
 		}
-		if (eventData.(map[string]interface{})["data"] != nil) {
+		if eventData != nil || eventData.(map[string]interface{})["data"] != nil {
 			for _, v := range eventData.(map[string]interface{})["data"].([]interface{}) {
 				thisEventSeriesID := v.(map[string]interface{})["eventSeriesId"]
-				eventSeriesData,err := rubrik.Get("internal","/event_series/"+thisEventSeriesID.(string))
+				eventSeriesData, err := rubrik.Get("internal", "/event_series/"+thisEventSeriesID.(string), 60)
 				if err != nil {
-					log.Println("Error from jobs.GetMssqlFailedJobs: ",err)
+					log.Printf("Error from jobs.GetMssqlFailedJobs: ", err)
+					return
 				}
 				hasFailedEvent := false
 				for _, w := range eventSeriesData.(map[string]interface{})["eventDetailList"].([]interface{}) {
@@ -110,16 +116,18 @@ func GetMssqlFailedJobs(rubrik *rubrikcdm.Credentials, clusterName string) {
 			}
 		}
 	} else { // cluster version is 5.2 or newer
-		eventData,err := rubrik.Get("v1","/event/latest?event_status=Failure&event_type=Backup&object_type=Mssql")
+		eventData, err := rubrik.Get("v1", "/event/latest?event_status=Failure&event_type=Backup&object_type=Mssql", 60)
 		if err != nil {
-			log.Println("Error from jobs.GetMssqlFailedJobs: ",err)
+			log.Printf("Error from jobs.GetMssqlFailedJobs: ", err)
+			return
 		}
-		if (eventData.(map[string]interface{})["data"] != nil) {
+		if eventData != nil || eventData.(map[string]interface{})["data"] != nil {
 			for _, v := range eventData.(map[string]interface{})["data"].([]interface{}) {
 				thisEventSeriesID := v.(map[string]interface{})["latestEvent"].(map[string]interface{})["eventSeriesId"]
-				eventSeriesData,err := rubrik.Get("v1","/event_series/"+thisEventSeriesID.(string))
+				eventSeriesData, err := rubrik.Get("v1", "/event_series/"+thisEventSeriesID.(string), 60)
 				if err != nil {
-					log.Println("Error from jobs.GetMssqlFailedJobs: ",err)
+					log.Printf("Error from jobs.GetMssqlFailedJobs: ", err)
+					return
 				}
 				hasFailedEvent := false
 				for _, w := range eventSeriesData.(map[string]interface{})["eventDetailList"].([]interface{}) {
@@ -169,4 +177,4 @@ func GetMssqlFailedJobs(rubrik *rubrikcdm.Credentials, clusterName string) {
 			}
 		}
 	}
-}
+}

src/golang/livemount/rubrik_live_mount.go

@@ -30,9 +30,10 @@ func init() {
 
 // GetMssqlLiveMountAges ...
 func GetMssqlLiveMountAges(rubrik *rubrikcdm.Credentials, clusterName string) {
-	mountData,err := rubrik.Get("v1","/mssql/db/mount") // get our mssql live mount summary
+	mountData,err := rubrik.Get("v1","/mssql/db/mount", 60) // get our mssql live mount summary
 	if err != nil {
-		log.Println("Error from livemount.GetMssqlLiveMountAges: ",err)
+		log.Printf("Error from livemount.GetMssqlLiveMountAges: ",err)
+		return
 	}
 	for _, v := range mountData.(map[string]interface{})["data"].([]interface{}) {
 		thisSourceDatabaseName := v.(map[string]interface{})["sourceDatabaseName"]

src/golang/main.go

@@ -12,30 +12,38 @@ Requirements:
 package main
 
 import (
-	"fmt"
 	"log"
 	"net/http"
 	"time"
-	"github.com/rubrikinc/rubrik-sdk-for-go/rubrikcdm"
+	"os"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/rubrikinc/rubrik-client-for-prometheus/src/golang/stats"
-	"github.com/rubrikinc/rubrik-client-for-prometheus/src/golang/livemount"
 	"github.com/rubrikinc/rubrik-client-for-prometheus/src/golang/jobs"
+	"github.com/rubrikinc/rubrik-client-for-prometheus/src/golang/livemount"
+	"github.com/rubrikinc/rubrik-client-for-prometheus/src/golang/stats"
+	"github.com/rubrikinc/rubrik-sdk-for-go/rubrikcdm"
 )
 
 func main() {
+	// set our Prometheus variables
+	httpPortEnv, _ := os.LookupEnv("RUBRIK_PROMETHEUS_PORT")
+	var httpPort string;
+	if httpPortEnv == "" {
+		httpPort = "8080"
+	} else {
+		httpPort = httpPortEnv
+	}
 	rubrik, err := rubrikcdm.ConnectEnv()
 	if err != nil {
-		log.Println("Error from main.go:")
+		log.Printf("Error from main.go:")
 		log.Fatal(err)
 	}
-	clusterDetails,err := rubrik.Get("v1","/cluster/me")
+	clusterDetails, err := rubrik.Get("v1", "/cluster/me", 60)
 	if err != nil {
-		log.Println("Error from main.go:")
+		log.Printf("Error from main.go:")
 		log.Fatal(err)
 	}
 	clusterName := clusterDetails.(map[string]interface{})["name"]
-	fmt.Println("Cluster name: "+clusterName.(string))
+	log.Printf("Cluster name: " + clusterName.(string))
 
 	// get storage summary
 	go func() {
@@ -105,5 +113,6 @@ func main() {
 	// The Handler function provides a default handler to expose metrics
 	// via an HTTP server. "/metrics" is the usual endpoint for that.
 	http.Handle("/metrics", promhttp.Handler())
-	log.Fatal(http.ListenAndServe(":8080" , nil))
+	log.Printf("Starting on HTTP port "+httpPort)
+	log.Fatal(http.ListenAndServe(":"+httpPort, nil))
 }

src/golang/stats/rubrik_compliance_stats.go

@@ -36,15 +36,16 @@ func init() {
 
 // Get ...
 func GetSlaComplianceStats(rubrik *rubrikcdm.Credentials, clusterName string) {
-	reportData,err := rubrik.Get("internal","/report?report_template=SlaComplianceSummary&report_type=Canned") // get our sla compliance summary report
+	reportData,err := rubrik.Get("internal","/report?report_template=SlaComplianceSummary&report_type=Canned", 60) // get our sla compliance summary report
 	if err != nil {
-		log.Println("Error from stats.GetSlaComplianceStats: ",err)
+		log.Printf("Error from stats.GetSlaComplianceStats: ",err)
 	}
 	reports := reportData.(map[string]interface{})["data"].([]interface{})
 	reportID := reports[0].(map[string]interface{})["id"]
 	chartData,err := rubrik.Get("internal","/report/"+reportID.(string)+"/chart?chart_id=chart0") // get our chart for the report
 	if err != nil {
-		log.Println("Error from stats.GetSlaComplianceStats: ",err)
+		log.Printf("Error from stats.GetSlaComplianceStats: ",err)
+		return
 	}
 	for _, v := range chartData.([]interface{}) {
 		dataColumns := v.(map[string]interface{})["dataColumns"]

src/golang/stats/rubrik_job_stats.go

@@ -46,15 +46,17 @@ func init() {
 
 // GetMssqlLiveMounts ...
 func Get24HJobStats(rubrik *rubrikcdm.Credentials, clusterName string) {
-	reportData,err := rubrik.Get("internal","/report?report_template=ProtectionTasksDetails&report_type=Canned") // get our protection tasks details report
+	reportData,err := rubrik.Get("internal","/report?report_template=ProtectionTasksDetails&report_type=Canned", 60) // get our protection tasks details report
 	if err != nil {
-		log.Println("Error from stats.Get24HJobStats: ",err)
+		log.Printf("Error from stats.Get24HJobStats: ",err)
+		return
 	}
 	reports := reportData.(map[string]interface{})["data"].([]interface{})
 	reportID := reports[0].(map[string]interface{})["id"]
-	chartData,err := rubrik.Get("internal","/report/"+reportID.(string)+"/chart?chart_id=chart0") // get our chart for the report
+	chartData,err := rubrik.Get("internal","/report/"+reportID.(string)+"/chart?chart_id=chart0", 60) // get our chart for the report
 	if err != nil {
-		log.Println("Error from stats.Get24HJobStats: ",err)
+		log.Printf("Error from stats.Get24HJobStats: ",err)
+		return
 	}
 	for _, v := range chartData.([]interface{}) {
 		dataColumns := v.(map[string]interface{})["dataColumns"]

src/golang/stats/rubrik_mssql_stats.go

@@ -43,9 +43,10 @@ func init() {
 
 // GetMssqlCapacityStats ...
 func GetMssqlCapacityStats(rubrik *rubrikcdm.Credentials, clusterName string) {
-	reportData,err := rubrik.Get("internal","/report?report_template=ObjectProtectionSummary&report_type=Canned") // get our object protection summary report
+	reportData,err := rubrik.Get("internal","/report?report_template=ObjectProtectionSummary&report_type=Canned", 60) // get our object protection summary report
 	if err != nil {
-		log.Println("Error from stats.GetMssqlCapacityStats: ",err)
+		log.Printf("Error from stats.GetMssqlCapacityStats: ",err)
+		return
 	}
 	reports := reportData.(map[string]interface{})["data"].([]interface{})
 	reportID := reports[0].(map[string]interface{})["id"]
@@ -57,9 +58,10 @@ func GetMssqlCapacityStats(rubrik *rubrikcdm.Credentials, clusterName string) {
 	}
 	for {
 		hasMore := true
-		tableData,err := rubrik.Post("internal","/report/"+reportID.(string)+"/table",body) // get our first page of data for the report
+		tableData,err := rubrik.Post("internal","/report/"+reportID.(string)+"/table",body, 60) // get our first page of data for the report
 		if err != nil {
-			log.Println("Error from stats.GetMssqlCapacityStats: ",err)
+			log.Printf("Error from stats.GetMssqlCapacityStats: ",err)
+			return
 		}
 		dataGrid := tableData.(map[string]interface{})["dataGrid"].([]interface{})
 		hasMore = tableData.(map[string]interface{})["hasMore"].(bool)
@@ -96,7 +98,7 @@ func GetMssqlCapacityStats(rubrik *rubrikcdm.Credentials, clusterName string) {
 				thisLocation).Set(thisArchiveStorage)
 		}
 		if !hasMore {
-			break
+			return
 		} else {
 			body = map[string]interface{}{
 				"limit": 1000,

src/golang/stats/rubrik_node_stats.go

@@ -60,15 +60,17 @@ func init() {
 
 // GetNodeStats ...
 func GetNodeStats(rubrik *rubrikcdm.Credentials, clusterName string) {
-	nodes,err := rubrik.Get("internal","/node")
+	nodes,err := rubrik.Get("internal","/node", 60)
 	if err != nil {
-		log.Println("Error from stats.GetNodeStats: ",err)
+		log.Printf("Error from stats.GetNodeStats: ",err)
+		return
 	}
 	for _, v := range nodes.(map[string]interface{})["data"].([]interface{}) {
 		thisNode := (v.(interface{}).(map[string]interface{})["id"])
-		nodeDetail,err := rubrik.Get("internal","/node/"+thisNode.(string))
+		nodeDetail,err := rubrik.Get("internal","/node/"+thisNode.(string), 60)
 		if err != nil {
-			log.Println("Error from stats.GetNodeStats: ",err)
+			log.Printf("Error from stats.GetNodeStats: ",err)
+			return
 		}
 		thisNodeStatus := nodeDetail.(map[string]interface{})["status"]
 		switch thisNodeStatus {
@@ -78,9 +80,10 @@ func GetNodeStats(rubrik *rubrikcdm.Credentials, clusterName string) {
 			rubrikNodeStatus.WithLabelValues(clusterName,thisNode.(string)).Set(0)
 		}
 
-		nodeStats,err := rubrik.Get("internal","/node/"+thisNode.(string)+"/stats?range=-6min")
+		nodeStats,err := rubrik.Get("internal","/node/"+thisNode.(string)+"/stats?range=-6min", 60)
 		if err != nil {
-			log.Println("Error from stats.GetNodeStats: ",err)
+			log.Printf("Error from stats.GetNodeStats: ",err)
+			return
 		}
 		// get cpu stat
 		cpuData := nodeStats.(map[string]interface{})["cpuStat"].([]interface{})

src/golang/stats/rubrik_oracle_stats.go

@@ -43,9 +43,10 @@ func init() {
 
 // GetOracleCapacityStats ...
 func GetOracleCapacityStats(rubrik *rubrikcdm.Credentials, clusterName string) {
-	reportData,err := rubrik.Get("internal","/report?report_template=ObjectProtectionSummary&report_type=Canned") // get our object protection summary report
+	reportData,err := rubrik.Get("internal","/report?report_template=ObjectProtectionSummary&report_type=Canned", 60) // get our object protection summary report
 	if err != nil {
 		log.Fatal(err)
+		return
 	}
 	reports := reportData.(map[string]interface{})["data"].([]interface{})
 	reportID := reports[0].(map[string]interface{})["id"]
@@ -57,9 +58,10 @@ func GetOracleCapacityStats(rubrik *rubrikcdm.Credentials, clusterName string) {
 	}
 	for {
 		hasMore := true
-		tableData,err := rubrik.Post("internal","/report/"+reportID.(string)+"/table",body) // get our first page of data for the report
+		tableData,err := rubrik.Post("internal","/report/"+reportID.(string)+"/table",body, 60) // get our first page of data for the report
 		if err != nil {
 			log.Fatal(err)
+			return
 		}
 		dataGrid := tableData.(map[string]interface{})["dataGrid"].([]interface{})
 		hasMore = tableData.(map[string]interface{})["hasMore"].(bool)
@@ -70,7 +72,7 @@ func GetOracleCapacityStats(rubrik *rubrikcdm.Credentials, clusterName string) {
 			thisLocalStorage, thisArchiveStorage := 0.0,0.0
 			for i := 0; i < len(columns); i++ {
 				switch columns[i] {
-				case "ObjectId":
+				case "ObjectId","ObjectLinkingId":
 					thisObjectID = v.([]interface{})[i].(string)
 				case "ObjectName":
 					thisObjectName = v.([]interface{})[i].(string)
@@ -94,7 +96,7 @@ func GetOracleCapacityStats(rubrik *rubrikcdm.Credentials, clusterName string) {
 				thisLocation).Set(thisArchiveStorage)
 		}
 		if !hasMore {
-			break
+			return
 		} else {
 			body = map[string]interface{}{
 				"limit": 1000,

src/golang/stats/rubrik_storage_stats.go

@@ -86,9 +86,10 @@ func init() {
 
 // GetStorageSummaryStats ...
 func GetStorageSummaryStats(rubrik *rubrikcdm.Credentials, clusterName string) {
-	storageStats,err := rubrik.Get("internal","/stats/system_storage")
+	storageStats,err := rubrik.Get("internal","/stats/system_storage", 60)
 	if err != nil {
-		log.Println("Error from stats.GetStorageSummaryStats: ",err)
+		log.Printf("Error from stats.GetStorageSummaryStats: ",err)
+		return
 	}
 	// get total storage stat
 	if total, ok := storageStats.(map[string]interface{})["total"].(float64); ok {
@@ -118,9 +119,10 @@ func GetStorageSummaryStats(rubrik *rubrikcdm.Credentials, clusterName string) {
 
 // GetRunwayRemaining ...q
 func GetRunwayRemaining(rubrik *rubrikcdm.Credentials, clusterName string) {
-	runwayRemaining,err := rubrik.Get("internal","/stats/runway_remaining")
+	runwayRemaining,err := rubrik.Get("internal","/stats/runway_remaining", 60)
 	if err != nil {
-		log.Println("Error from stats.GetRunwayRemaining: ",err)
+		log.Printf("Error from stats.GetRunwayRemaining: ",err)
+		return
 	}
 	// get runway remaining stat
 	if runway, ok := runwayRemaining.(map[string]interface{})["days"].(float64); ok {