Implement webhook call to reload proxy (#51)

Author: man90es

handlers/getStatus.go

@@ -12,7 +12,7 @@ import (
 )
 
 var initTime = time.Now()
-var version = "1.13.1"
+var version = "1.14.0"
 
 func getStatus(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(map[string]interface{}{

main.go

@@ -18,13 +18,14 @@ func main() {
 	flagAdminToken := flag.String("admintoken", "", "Admin token to allow certain endpoints")
 	flagCacheTTL := flag.Uint("cachettl", 180, "Cache TTL in minutes")
 	flagMaintenanceTTL := flag.Uint("maintenancettl", 5, "Allows to limit how frequently scraper can check for maintenance end in minutes")
+	flagMaxTasksPerClient := flag.Uint("maxtasksperclient", 5, "Maximum number of scraping tasks per client")
 	flagMongo := flag.String("mongo", "", "MongoDB connection string for loggig")
 	flagPort := flag.Uint("port", 8001, "Port to catch requests on")
 	flagProxy := flag.String("proxy", "", "Open proxy address to make requests to BDO servers")
+	flagProxyReloadWebhook := flag.String("proxyreloadwebhook", "", "Webhook address to request proxy reload")
 	flagRateLimit := flag.Uint64("ratelimit", 512, "Maximum number of requests per minute per IP")
-	flagVerbose := flag.Bool("verbose", false, "Print out additional logs into stdout")
 	flagTaskRetries := flag.Uint("taskretries", 3, "Number of retries for a scraping task")
-	flagMaxTasksPerClient := flag.Uint("maxtasksperclient", 5, "Maximum number of scraping tasks per client")
+	flagVerbose := flag.Bool("verbose", false, "Print out additional logs into stdout")
 	flag.Parse()
 
 	// Read port from flags and env
@@ -52,6 +53,7 @@ func main() {
 	viper.Set("maintenancettl", time.Duration(*flagMaintenanceTTL)*time.Minute)
 	viper.Set("maxtasksperclient", int(*flagMaxTasksPerClient))
 	viper.Set("mongo", *flagMongo)
+	viper.Set("proxyreloadwebhook", *flagProxyReloadWebhook)
 	viper.Set("ratelimit", int64(*flagRateLimit))
 	viper.Set("taskretries", int(*flagTaskRetries))
 	viper.Set("verbose", *flagVerbose)

scraper/scraper.go

@@ -82,7 +82,12 @@ func InitScraper() {
 		if imperva {
 			taskRetries, _ := strconv.Atoi(body.Request.Ctx.Get("taskRetries"))
 			logger.Error(fmt.Sprintf("Hit Imperva while loading %v, retries: %v", body.Request.URL.String(), taskRetries))
-			taskQueue.Pause(time.Duration(60-time.Now().Second()) * time.Second)
+			if proxyReloadWebhook := viper.GetString("proxyreloadwebhook"); proxyReloadWebhook != "" {
+				utils.SendDummyRequest(proxyReloadWebhook)
+				taskQueue.Pause(time.Second * 5)
+			} else {
+				taskQueue.Pause(time.Duration(60-time.Now().Second()) * time.Second)
+			}
 			taskQueue.ConfirmTaskCompletion(taskClient, taskHash)
 
 			if taskRetries < viper.GetInt("taskretries") {

utils/SendRequest.go

@@ -0,0 +1,18 @@
+package utils
+
+import "net/http"
+
+func SendDummyRequest(url string) error {
+	req, err := http.NewRequest("POST", url, nil)
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}