Skip to content

Commit 2457ea9

Browse files
committed
better cv content checking
1 parent 2b3009d commit 2457ea9

File tree

6 files changed

+159
-37
lines changed

6 files changed

+159
-37
lines changed

cv.go

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"errors"
6+
"strconv"
7+
"strings"
8+
)
9+
10+
// StrippedCVWithOriginal contains the stripped cv and the original bytes
11+
// Using this we can unmarshal the cv and then marshal it again without losing any data
12+
// Without this we will loos all info and only have the fields within the stripped cv
13+
type StrippedCVWithOriginal struct {
14+
StrippedCV
15+
JSONBytes []byte
16+
}
17+
18+
// UnmarshalJSON unmarshals the cv into StrippedCV and saves the bytes also to JSONBytes
19+
func (a *StrippedCVWithOriginal) UnmarshalJSON(b []byte) error {
20+
// Copy the bytes of the cv into a.JSONBytes
21+
a.JSONBytes = append([]byte{}, b...)
22+
return json.Unmarshal(b, &a.StrippedCV)
23+
}
24+
25+
// MarshalJSON returns the cv that was inputted to UnmarshalJSON
26+
func (a StrippedCVWithOriginal) MarshalJSON() ([]byte, error) {
27+
return a.JSONBytes, nil
28+
}
29+
30+
// StrippedCV is a stripped down version of the RT-CV CV model that only contains the fields we check in this tool
31+
type StrippedCV struct {
32+
ReferenceNumber string `json:"referenceNumber"`
33+
PersonalDetails StrippedPersonalDetails `json:"personalDetails"`
34+
}
35+
36+
// StrippedPersonalDetails contains a stripped version of the personal details of a RT-CV cv.
37+
// We only have the fields from the RT-CV we use for checking if the cv is valid
38+
type StrippedPersonalDetails struct {
39+
Zip string `json:"zip"`
40+
}
41+
42+
func (cv *StrippedCV) checkRefNr() error {
43+
if cv.ReferenceNumber == "" {
44+
return errors.New("referenceNumber cannot be empty")
45+
}
46+
return nil
47+
}
48+
49+
// ErrInvalidZip is returned when the zip code is not valid
50+
var ErrInvalidZip = errors.New("personalDetails.zip has a invalid zip code")
51+
52+
func (cv *StrippedCV) checkMustHaveValidZip() error {
53+
zip := strings.TrimSpace(cv.PersonalDetails.Zip)
54+
if zip == "" {
55+
return errors.New("personalDetails.zip required")
56+
}
57+
58+
zipLen := len(zip)
59+
if zipLen != 4 && zipLen != 6 {
60+
return ErrInvalidZip
61+
}
62+
_, err := strconv.Atoi(zip[:4])
63+
if err != nil {
64+
return ErrInvalidZip
65+
}
66+
return nil
67+
}

cv_test.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"strings"
7+
"testing"
8+
)
9+
10+
func checkErr(err error) {
11+
if err != nil {
12+
panic(err.Error())
13+
}
14+
}
15+
16+
func mustEq(a string, b string) {
17+
if a != b {
18+
panic(fmt.Sprintf("left != right : %s != %s", a, b))
19+
}
20+
}
21+
22+
func TestStrippedCVWithOriginal(t *testing.T) {
23+
testInput := []byte(`{"referenceNumber":"a","other":true,"personalDetails":{"zip":"1234"}}`)
24+
testOutput := StrippedCVWithOriginal{}
25+
err := json.Unmarshal(testInput, &testOutput)
26+
checkErr(err)
27+
mustEq("a", testOutput.ReferenceNumber)
28+
mustEq("1234", testOutput.PersonalDetails.Zip)
29+
30+
formattedOutput, err := json.Marshal(testOutput)
31+
checkErr(err)
32+
mustEq(string(testInput), string(formattedOutput))
33+
34+
testInput = []byte(strings.Join([]string{`[`,
35+
`{"referenceNumber":"a","other1":true,"personalDetails":{"zip":"1234"}},`,
36+
`{"referenceNumber":"b","personalDetails":{"zip":"4321","other2":true}}`,
37+
`]`}, ""))
38+
testOutputs := []StrippedCVWithOriginal{}
39+
err = json.Unmarshal(testInput, &testOutputs)
40+
checkErr(err)
41+
mustEq("a", testOutputs[0].ReferenceNumber)
42+
mustEq("1234", testOutputs[0].PersonalDetails.Zip)
43+
mustEq("b", testOutputs[1].ReferenceNumber)
44+
mustEq("4321", testOutputs[1].PersonalDetails.Zip)
45+
46+
formattedOutput, err = json.Marshal(testOutputs)
47+
checkErr(err)
48+
mustEq(string(testInput), string(formattedOutput))
49+
}

go.mod

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@ module github.com/script-development/rtcv_scraper_client/v2
22

33
go 1.18
44

5-
require github.com/valyala/fasthttp v1.37.0
5+
require github.com/valyala/fasthttp v1.40.0
66

77
require (
88
github.com/andybalholm/brotli v1.0.4 // indirect
9-
github.com/klauspost/compress v1.15.0 // indirect
9+
github.com/klauspost/compress v1.15.9 // indirect
1010
github.com/valyala/bytebufferpool v1.0.0 // indirect
11-
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8
12-
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
11+
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
12+
golang.org/x/sys v0.0.0-20220907062415-87db552b00fd // indirect
1313
)

go.sum

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,31 @@
11
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
22
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
3-
github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U=
43
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
4+
github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
5+
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
56
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
67
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
7-
github.com/valyala/fasthttp v1.37.0 h1:7WHCyI7EAkQMVmrfBhWTCOaeROb1aCBiTopx63LkMbE=
8-
github.com/valyala/fasthttp v1.37.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
8+
github.com/valyala/fasthttp v1.39.0 h1:lW8mGeM7yydOqZKmwyMTaz/PH/A+CLgtmmcjv+OORfU=
9+
github.com/valyala/fasthttp v1.39.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
10+
github.com/valyala/fasthttp v1.40.0 h1:CRq/00MfruPGFLTQKY8b+8SfdK60TxNztjRMnH0t1Yc=
11+
github.com/valyala/fasthttp v1.40.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
912
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
1013
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
1114
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c=
1215
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
16+
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
17+
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
1318
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
1419
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
1520
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
1621
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
1722
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1823
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
19-
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
2024
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
25+
golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8=
26+
golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
27+
golang.org/x/sys v0.0.0-20220907062415-87db552b00fd h1:AZeIEzg+8RCELJYq8w+ODLVxFgLMMigSwO/ffKPEd9U=
28+
golang.org/x/sys v0.0.0-20220907062415-87db552b00fd/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
2129
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
2230
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
2331
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

helpers.go

Lines changed: 0 additions & 17 deletions
This file was deleted.

webserver.go

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,20 @@ func startWebserver(env Env, api *API, loginUsers []EnvUser) string {
2323
body := ctx.Request.Body()
2424
switch path {
2525
case "/send_cv":
26-
cvContent := map[string]interface{}{}
27-
err := json.Unmarshal(body, &cvContent)
26+
cvForChecking := StrippedCV{}
27+
err := json.Unmarshal(body, &cvForChecking)
2828
if err != nil {
2929
errorResp(ctx, 400, "invalid CV")
3030
return
3131
}
3232

33-
referenceNr, err := checkIfCVHasReferenceNr(cvContent)
33+
err = cvForChecking.checkRefNr()
3434
if err != nil {
3535
errorResp(ctx, 400, err.Error())
3636
return
3737
}
3838

39-
cacheEntryExists := api.CacheEntryExists(referenceNr)
39+
cacheEntryExists := api.CacheEntryExists(cvForChecking.ReferenceNumber)
4040
if cacheEntryExists {
4141
// Cannot send the same cv twice
4242
ctx.Response.AppendBodyString("false")
@@ -45,7 +45,7 @@ func startWebserver(env Env, api *API, loginUsers []EnvUser) string {
4545

4646
hasMatch := false
4747
if api.MockMode {
48-
api.SetCacheEntry(referenceNr, time.Hour*72)
48+
api.SetCacheEntry(cvForChecking.ReferenceNumber, time.Hour*72)
4949
hasMatch = true
5050
} else {
5151
scanCVBody := json.RawMessage(append(append([]byte(`{"cv":`), body...), '}'))
@@ -65,33 +65,48 @@ func startWebserver(env Env, api *API, loginUsers []EnvUser) string {
6565
hasMatch = response.HasMatches
6666
if hasMatch {
6767
// Only cache the CVs that where matched to something
68-
api.SetCacheEntry(referenceNr, time.Hour*72) // 3 days
68+
api.SetCacheEntry(cvForChecking.ReferenceNumber, time.Hour*72) // 3 days
6969
}
7070
}
7171
}
7272
}
7373

7474
ctx.Response.AppendBodyString("true")
7575
case "/cvs_list":
76-
cvsContent := []map[string]interface{}{}
77-
err := json.Unmarshal(body, &cvsContent)
76+
cvs := []StrippedCVWithOriginal{}
77+
err := json.Unmarshal(body, &cvs)
7878
if err != nil {
7979
errorResp(ctx, 400, "invalid CV")
8080
return
8181
}
8282

83-
for idx, cv := range cvsContent {
84-
_, err := checkIfCVHasReferenceNr(cv)
83+
checkedRefNrs := map[string]struct{}{}
84+
for idx := 10; idx >= 0; idx-- {
85+
cv := cvs[idx]
86+
err := cv.checkRefNr()
8587
if err != nil {
8688
errorResp(ctx, 400, fmt.Sprintf("error in cv with index %d, error: %s", idx, err.Error()))
8789
return
8890
}
91+
92+
_, ok := checkedRefNrs[cv.ReferenceNumber]
93+
if ok {
94+
// Remove this cv as it's a duplicate of another one in the list
95+
cvs = append(cvs[:idx], cvs[idx+1:]...)
96+
continue
97+
}
98+
checkedRefNrs[cv.ReferenceNumber] = struct{}{}
99+
100+
if cv.checkMustHaveValidZip() != nil {
101+
// Remove this cv from the list as it does not have a valid zip code
102+
cvs = append(cvs[:idx], cvs[idx+1:]...)
103+
continue
104+
}
89105
}
90106

91107
if !api.MockMode {
92-
body := append(append([]byte(`{"cvs":`), body...), '}')
93108
for _, conn := range api.connections {
94-
err = conn.Post("/api/v1/scraper/allCVs", json.RawMessage(body), nil)
109+
err = conn.Post("/api/v1/scraper/allCVs", map[string]any{"cvs": cvs}, nil)
95110
if err != nil {
96111
errorResp(ctx, 500, err.Error())
97112
return

0 commit comments

Comments
 (0)