Add useragent & improve maintenance detection

Author: man90es

docs/openapi.json

@@ -36,6 +36,9 @@
 			"404": {
 				"description": "Not Found: try requesting something else."
 			},
+			"500": {
+				"description": "Something went wrong. Currently is returned when server's or proxy's IP address is banned on the BDO website."
+			},
 			"503": {
 				"description": "Service Unavailable: [https://naeu.playblackdesert.com](naeu.playblackdesert.com) is currently under maintenance and requested data is not cached."
 			}
@@ -269,6 +272,9 @@
 					"404": {
 						"$ref": "#/components/responses/404"
 					},
+					"500": {
+						"$ref": "#/components/responses/500"
+					},
 					"503": {
 						"$ref": "#/components/responses/503"
 					}
@@ -387,6 +393,9 @@
 					"404": {
 						"$ref": "#/components/responses/404"
 					},
+					"500": {
+						"$ref": "#/components/responses/500"
+					},
 					"503": {
 						"$ref": "#/components/responses/503"
 					}
@@ -485,6 +494,9 @@
 					"404": {
 						"$ref": "#/components/responses/404"
 					},
+					"500": {
+						"$ref": "#/components/responses/500"
+					},
 					"503": {
 						"$ref": "#/components/responses/503"
 					}
@@ -582,6 +594,9 @@
 					"404": {
 						"$ref": "#/components/responses/404"
 					},
+					"500": {
+						"$ref": "#/components/responses/500"
+					},
 					"503": {
 						"$ref": "#/components/responses/503"
 					}

handlers/GetAdventurer.go

@@ -30,6 +30,11 @@ func GetAdventurer(w http.ResponseWriter, r *http.Request) {
 	if !found {
 		data, status = scrapers.ScrapeAdventurer(region, profileTarget)
 
+		if status == http.StatusInternalServerError {
+			w.WriteHeader(status)
+			return
+		}
+
 		if ok := giveMaintenanceResponse(w, region); ok {
 			return
 		}

handlers/GetAdventurerSearch.go

@@ -33,6 +33,11 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) {
 	if !found {
 		data, status = scrapers.ScrapeAdventurerSearch(region, query, searchType, page)
 
+		if status == http.StatusInternalServerError {
+			w.WriteHeader(status)
+			return
+		}
+
 		if ok := giveMaintenanceResponse(w, region); ok {
 			return
 		}

handlers/GetGuild.go

@@ -30,6 +30,11 @@ func GetGuild(w http.ResponseWriter, r *http.Request) {
 	if !found {
 		data, status = scrapers.ScrapeGuild(region, name)
 
+		if status == http.StatusInternalServerError {
+			w.WriteHeader(status)
+			return
+		}
+
 		if ok := giveMaintenanceResponse(w, region); ok {
 			return
 		}

handlers/GetGuildSearch.go

@@ -32,6 +32,11 @@ func GetGuildSearch(w http.ResponseWriter, r *http.Request) {
 	if !found {
 		data, status = scrapers.ScrapeGuildSearch(region, name, page)
 
+		if status == http.StatusInternalServerError {
+			w.WriteHeader(status)
+			return
+		}
+
 		if ok := giveMaintenanceResponse(w, region); ok {
 			return
 		}

scrapers/ScrapeAdventurer.go

@@ -22,6 +22,11 @@ func ScrapeAdventurer(region string, profileTarget string) (profile models.Profi
 	profile.Region = region
 	status = http.StatusNotFound
 
+	// Detect being banned by IP address
+	c.OnHTML(`.type_2`, func(e *colly.HTMLElement) {
+		status = http.StatusInternalServerError
+	})
+
 	c.OnHTML(`.nick`, func(e *colly.HTMLElement) {
 		profile.FamilyName = e.Text
 		status = http.StatusOK

scrapers/ScrapeAdventurerSearch.go

@@ -16,6 +16,11 @@ func ScrapeAdventurerSearch(region string, query string, searchType uint8, page
 
 	status = http.StatusNotFound
 
+	// Detect being banned by IP address
+	c.OnHTML(`.type_2`, func(e *colly.HTMLElement) {
+		status = http.StatusInternalServerError
+	})
+
 	c.OnHTML(`.box_list_area li:not(.no_result)`, func(e *colly.HTMLElement) {
 		status = http.StatusOK
 		profile := models.Profile{

scrapers/ScrapeGuild.go

@@ -17,6 +17,11 @@ func ScrapeGuild(region, name string) (guildProfile models.GuildProfile, status
 	guildProfile.Region = region
 	status = http.StatusNotFound
 
+	// Detect being banned by IP address
+	c.OnHTML(`.type_2`, func(e *colly.HTMLElement) {
+		status = http.StatusInternalServerError
+	})
+
 	c.OnHTML(`.region_info`, func(e *colly.HTMLElement) {
 		guildProfile.Region = e.Text
 	})

scrapers/ScrapeGuildSearch.go

@@ -17,6 +17,11 @@ func ScrapeGuildSearch(region, query string, page uint16) (guildProfiles []model
 
 	status = http.StatusNotFound
 
+	// Detect being banned by IP address
+	c.OnHTML(`.type_2`, func(e *colly.HTMLElement) {
+		status = http.StatusInternalServerError
+	})
+
 	c.OnHTML(`.box_list_area li:not(.no_result)`, func(e *colly.HTMLElement) {
 		createdOn := utils.ParseDate(e.ChildText(".date"))
 		status = http.StatusOK

scrapers/scraper.go

@@ -16,8 +16,10 @@ type scraper struct {
 }
 
 func newScraper(region string) (s scraper) {
+	useragent := colly.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36")
+
 	s.region = region
-	s.c = colly.NewCollector()
+	s.c = colly.NewCollector(useragent)
 	s.c.SetRequestTimeout(time.Minute / 2)
 
 	if len(config.GetProxyList()) > 0 {
@@ -30,7 +32,8 @@ func newScraper(region string) (s scraper) {
 		}
 	})
 
-	s.OnHTML(`.closetime_wrap`, func(e *colly.HTMLElement) {
+	// Detect maintenance
+	s.OnHTML(`.type_3`, func(e *colly.HTMLElement) {
 		setCloseTime(region)
 	})