Update API to scrape profile professions instead of character professions

man90es · man90es · commit 41f35d7763c5 · 2024-05-02T18:10:41.000+02:00
diff --git a/docs/openapi.json b/docs/openapi.json
@@ -118,7 +118,7 @@
 		"/v1/adventurer": {
 			"get": {
 				"summary": "Retrieve player's profile.",
-				"description": "Retrieve the full profile of a single player by his or her profileTarget. Watch out for the \"privacy\" attribute that is added to the response when the player whose profile is requested turned on at least one of the privacy options BDO website has. It is a single number that you can decode using bitmasks. `0b0001` for private character levels, `0b0010` for private guild, `0b0100` for private contribution points and `0b1000` for private lifeskill levels. If the attribute is equal to `0b1111` then everything is set to private.",
+				"description": "Retrieve the full profile of a single player by his or her profileTarget. Watch out for the \"privacy\" attribute that is added to the response when the player whose profile is requested turned on at least one of the privacy options BDO website has. It is a single number that you can decode using bitmasks. `0b0001` for private character levels, `0b0010` for private guild, `0b0100` for private contribution points and `0b1000` for private lifeskill levels. If the attribute is equal to `0b1111` then everything is set to private.\n\n<b>Important!</b> Property \"specLevels\" used to be on a character level, but BDO has changed it so that all characters have shared profession levels. This API will continue to return \"specLevels\" on the character level alongside with \"specLevels\" on the profile level for compatibility reasons until 1 September 2024.",
 				"operationId": "getAdventurer",
 				"parameters": [
 					{
@@ -208,6 +208,7 @@
 														"example": 56
 													},
 													"specLevels": {
+														"deprecated": true,
 														"properties": {
 															"gathering": {
 																"type": "string",
@@ -257,6 +258,54 @@
 													}
 												}
 											}
+										},
+										"specLevels": {
+											"properties": {
+												"gathering": {
+													"type": "string",
+													"example": "Beginner 6"
+												},
+												"fishing": {
+													"type": "string",
+													"example": "Master 18"
+												},
+												"hunting": {
+													"type": "string",
+													"example": "Beginner 1"
+												},
+												"cooking": {
+													"type": "string",
+													"example": "Beginner 4"
+												},
+												"alchemy": {
+													"type": "string",
+													"example": "Beginner 1"
+												},
+												"processing": {
+													"type": "string",
+													"example": "Beginner 9"
+												},
+												"training": {
+													"type": "string",
+													"example": "Apprentice 1"
+												},
+												"trading": {
+													"type": "string",
+													"example": "Apprentice 3"
+												},
+												"farming": {
+													"type": "string",
+													"example": "Beginner 1"
+												},
+												"sailing": {
+													"type": "string",
+													"example": "Beginner 1"
+												},
+												"barter": {
+													"type": "string",
+													"example": "Beginner 1"
+												}
+											}
 										}
 									}
 								}
diff --git a/handlers/GetStatus.go b/handlers/GetStatus.go
@@ -28,6 +28,6 @@ func GetStatus(w http.ResponseWriter, r *http.Request) {
 		},
 		"proxies": len(config.GetProxyList()),
 		"uptime":  time.Since(initTime).Round(time.Second).String(),
-		"version": "1.5.5",
+		"version": "1.6.0",
 	})
 }
diff --git a/models/Character.go b/models/Character.go
@@ -5,5 +5,5 @@ type Character struct {
 	Class      string `json:"class"`
 	Main       bool   `json:"main,omitempty"`
 	Level      uint8  `json:"level,omitempty"`
-	SpecLevels *Specs `json:"specLevels,omitempty"`
+	SpecLevels *Specs `json:"specLevels,omitempty"` // Deprecated, will be removed on 1 September 2024
 }
diff --git a/models/Profile.go b/models/Profile.go
@@ -18,4 +18,5 @@ type Profile struct {
 	CreatedOn          *time.Time    `json:"createdOn,omitempty"`
 	Characters         []Character   `json:"characters,omitempty"`
 	Privacy            int8          `json:"privacy,omitempty"`
+	SpecLevels         *Specs        `json:"specLevels,omitempty"`
 }
diff --git a/scrapers/ScrapeAdventurer.go b/scrapers/ScrapeAdventurer.go
@@ -62,9 +62,42 @@ func ScrapeAdventurer(region string, profileTarget string) (profile models.Profi
 		}
 	})
 
+	c.OnHTML(`.character_spec:not(.lock)`, func(e *colly.HTMLElement) {
+		specLevels := [11]string{}
+
+		e.ForEach(".spec_level", func(ind int, el *colly.HTMLElement) {
+			// "Beginner1" → "Beginner 1"
+			i := regexp.MustCompile(`[0-9]`).FindStringIndex(el.Text)[0]
+			wordLevel := el.Text[:i]
+
+			if region != "EU" && region != "NA" {
+				translators.TranslateSpecLevel(&wordLevel)
+			}
+
+			specLevels[ind] = wordLevel + " " + el.Text[i:]
+		})
+
+		if len(specLevels[0]) > 0 {
+			profile.SpecLevels = &models.Specs{
+				Gathering:  specLevels[0],
+				Fishing:    specLevels[1],
+				Hunting:    specLevels[2],
+				Cooking:    specLevels[3],
+				Alchemy:    specLevels[4],
+				Processing: specLevels[5],
+				Training:   specLevels[6],
+				Trading:    specLevels[7],
+				Farming:    specLevels[8],
+				Sailing:    specLevels[9],
+				Barter:     specLevels[10],
+			}
+		}
+	})
+
 	c.OnHTML(`.character_desc_area`, func(e *colly.HTMLElement) {
 		character := models.Character{
-			Class: e.ChildText(".character_info .character_symbol em:last-child"),
+			Class:      e.ChildText(".character_info .character_symbol em:last-child"),
+			SpecLevels: profile.SpecLevels, // Deprecated, will be removed on 1 September 2024
 		}
 
 		if region != "EU" && region != "NA" {
@@ -91,36 +124,6 @@ func ScrapeAdventurer(region string, profileTarget string) (profile models.Profi
 			}
 		}
 
-		if specLevels := [11]string{}; true {
-			e.ForEach(".character_spec:not(.lock) .spec_level", func(ind int, el *colly.HTMLElement) {
-				// "Beginner1" → "Beginner 1"
-				i := regexp.MustCompile(`[0-9]`).FindStringIndex(el.Text)[0]
-				wordLevel := el.Text[:i]
-
-				if region != "EU" && region != "NA" {
-					translators.TranslateSpecLevel(&wordLevel)
-				}
-
-				specLevels[ind] = wordLevel + " " + el.Text[i:]
-			})
-
-			if len(specLevels[0]) > 0 {
-				character.SpecLevels = &models.Specs{
-					Gathering:  specLevels[0],
-					Fishing:    specLevels[1],
-					Hunting:    specLevels[2],
-					Cooking:    specLevels[3],
-					Alchemy:    specLevels[4],
-					Processing: specLevels[5],
-					Training:   specLevels[6],
-					Trading:    specLevels[7],
-					Farming:    specLevels[8],
-					Sailing:    specLevels[9],
-					Barter:     specLevels[10],
-				}
-			}
-		}
-
 		profile.Characters = append(profile.Characters, character)
 	})
 

Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,6 @@ func GetStatus(w http.ResponseWriter, r *http.Request) {`
`28`	`28`	`},`
`29`	`29`	`"proxies": len(config.GetProxyList()),`
`30`	`30`	`"uptime": time.Since(initTime).Round(time.Second).String(),`
`31`		`- "version": "1.5.5",`
	`31`	`+ "version": "1.6.0",`
`32`	`32`	`})`
`33`	`33`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,5 +5,5 @@ type Character struct {`
`5`	`5`	Class string `json:"class"`
`6`	`6`	Main bool `json:"main,omitempty"`
`7`	`7`	Level uint8 `json:"level,omitempty"`
`8`		- SpecLevels *Specs `json:"specLevels,omitempty"`
	`8`	+ SpecLevels *Specs `json:"specLevels,omitempty"` // Deprecated, will be removed on 1 September 2024
`9`	`9`	`}`
Original file line number	Diff line number	Diff line change
`@@ -18,4 +18,5 @@ type Profile struct {`
`18`	`18`	CreatedOn *time.Time `json:"createdOn,omitempty"`
`19`	`19`	Characters []Character `json:"characters,omitempty"`
`20`	`20`	Privacy int8 `json:"privacy,omitempty"`
	`21`	+ SpecLevels *Specs `json:"specLevels,omitempty"`
`21`	`22`	`}`