SYN-6770: preserve request redaction

syntheticsclientv2/common_models.go

@@ -827,6 +827,52 @@ type BrowserCheckV2Input struct {
 	} `json:"test"`
 }
 
+func (b BrowserCheckV2Input) MarshalJSON() ([]byte, error) {
+	type advancedSettingsJSON struct {
+		Advancedsettings
+		CertificateIDs *[]int `json:"certificateIds,omitempty"`
+	}
+
+	type browserCheckV2InputTestJSON struct {
+		Name               string               `json:"name"`
+		Transactions       []Transactions       `json:"transactions"`
+		Urlprotocol        string               `json:"urlProtocol"`
+		Starturl           string               `json:"startUrl"`
+		LocationIds        []string             `json:"locationIds"`
+		DeviceID           int                  `json:"deviceId"`
+		Frequency          int                  `json:"frequency"`
+		Schedulingstrategy string               `json:"schedulingStrategy"`
+		Active             bool                 `json:"active"`
+		Advancedsettings   advancedSettingsJSON `json:"advancedSettings,omitempty"`
+		Customproperties   []CustomProperties   `json:"customProperties"`
+		Automaticretries   int                  `json:"automaticRetries"`
+	}
+
+	advancedSettings := advancedSettingsJSON{Advancedsettings: b.Test.Advancedsettings}
+	if b.Test.CertificateIDs != nil {
+		advancedSettings.CertificateIDs = &b.Test.CertificateIDs
+	}
+
+	return json.Marshal(struct {
+		Test browserCheckV2InputTestJSON `json:"test"`
+	}{
+		Test: browserCheckV2InputTestJSON{
+			Name:               b.Test.Name,
+			Transactions:       b.Test.Transactions,
+			Urlprotocol:        b.Test.Urlprotocol,
+			Starturl:           b.Test.Starturl,
+			LocationIds:        b.Test.LocationIds,
+			DeviceID:           b.Test.DeviceID,
+			Frequency:          b.Test.Frequency,
+			Schedulingstrategy: b.Test.Schedulingstrategy,
+			Active:             b.Test.Active,
+			Advancedsettings:   advancedSettings,
+			Customproperties:   b.Test.Customproperties,
+			Automaticretries:   b.Test.Automaticretries,
+		},
+	})
+}
+
 type BrowserCheckV2Response struct {
 	Test BrowserCheckV2ResponseTest `json:"test"`
 }

syntheticsclientv2/create_apicheckv2_test.go

@@ -27,7 +27,7 @@ import (
 )
 
 var (
-	createApiV2Body = `{"test":{"automaticRetries": 1,"customProperties": [{"key": "Test_Key", "value": "Test Custom Properties"}], "active":true,"deviceId":1,"frequency":5,"location_ids":["aws-us-east-1"],"name":"boop-test","scheduling_strategy":"round_robin","requests":[{"configuration":{"name":"Get-Test","requestMethod":"GET","url":"https://api.us1.signalfx.com/v2/synthetics/v2/tests/api/489","certificateId":123,"headers":{"X-SF-TOKEN":"jinglebellsbatmanshells", "beep":"boop"},"body":null},"setup":[{"name":"Extract from response body","type":"extract_json","source":"{{response.body}}","extractor":"$.requests","variable":"custom-varz"}],"validations":[{"name":"Assert response code equals 200","type":"assert_numeric","actual":"{{response.code}}","expected":"200","comparator":"equals"}]}]}}`
+	createApiV2Body = `{"test":{"automaticRetries": 1,"customProperties": [{"key": "Test_Key", "value": "Test Custom Properties"}], "active":true,"deviceId":1,"frequency":5,"location_ids":["aws-us-east-1"],"name":"boop-test","scheduling_strategy":"round_robin","requests":[{"configuration":{"name":"Get-Test","requestMethod":"GET","url":"https://api.us1.signalfx.com/v2/synthetics/v2/tests/api/489","certificateId":123,"headers":{"X-SF-TOKEN":"jinglebellsbatmanshells", "beep":"plain-header-secret"},"body":null},"setup":[{"name":"Extract from response body","type":"extract_json","source":"{{response.body}}","extractor":"$.requests","variable":"custom-varz"}],"validations":[{"name":"Assert response code equals 200","type":"assert_numeric","actual":"{{response.code}}","expected":"200","comparator":"equals"}]}]}}`
 	inputData       = ApiCheckV2Input{}
 )
 
@@ -58,11 +58,24 @@ func TestCreateApiCheckV2(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	resp, _, err := testClient.CreateApiCheckV2(&inputData)
+	resp, details, err := testClient.CreateApiCheckV2(&inputData)
 
 	if err != nil {
 		t.Fatal(err)
 	}
+	if details == nil {
+		t.Fatal("expected request details")
+	}
+	for _, secret := range []string{"jinglebellsbatmanshells", "plain-header-secret"} {
+		if strings.Contains(details.RequestBody, secret) {
+			t.Fatalf("expected request details to redact API check header value %q, but saw: %s", secret, details.RequestBody)
+		}
+	}
+	for _, redacted := range []string{`"X-SF-TOKEN":"[REDACTED]"`, `"beep":"[REDACTED]"`} {
+		if !strings.Contains(details.RequestBody, redacted) {
+			t.Fatalf("expected request details to contain redacted header %q, but saw: %s", redacted, details.RequestBody)
+		}
+	}
 
 	if !reflect.DeepEqual(resp.Test.Name, inputData.Test.Name) {
 		t.Errorf("returned \n\n%#v want \n\n%#v", resp.Test.Name, inputData.Test.Name)

syntheticsclientv2/create_browsercheckv2_test.go

@@ -41,11 +41,15 @@ func TestCreateBrowserCheckV2(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		if !strings.Contains(string(requestBody), `"certificateIds":[123]`) {
-			t.Fatalf("request body missing certificateIds: %s", requestBody)
+		requestBodyString := string(requestBody)
+		testPayload := browserCheckV2RequestTestPayload(t, requestBodyString)
+		if got := testPayload["name"]; got != "browser-beep-test" {
+			t.Fatalf("expected request body to preserve browser test name, but saw %#v in body: %s", got, requestBodyString)
 		}
-		if strings.Contains(string(requestBody), "certificate_ids") {
-			t.Fatalf("request body contains internal certificate_ids field: %s", requestBody)
+		advancedSettings := browserCheckV2RequestAdvancedSettingsPayload(t, requestBodyString)
+		assertBrowserCheckV2RequestCertificateIDs(t, advancedSettings, []int{123})
+		if strings.Contains(requestBodyString, "certificate_ids") {
+			t.Fatalf("request body contains internal certificate_ids field: %s", requestBodyString)
 		}
 		_, err = w.Write([]byte(createBrowserCheckV2Body))
 		if err != nil {

syntheticsclientv2/synthetics.go

@@ -63,11 +63,23 @@ func (c Client) String() string {
 }
 
 var sensitiveJSONFieldNames = map[string]struct{}{
+	"body":     {},
 	"content":  {},
 	"password": {},
+	"username": {},
 	"value":    {},
 }
 
+var sensitiveHeaderNames = map[string]struct{}{
+	"authorization":       {},
+	"proxy-authorization": {},
+	"cookie":              {},
+	"set-cookie":          {},
+	"x-sf-token":          {},
+	"x-api-key":           {},
+	"api-key":             {},
+}
+
 func (c Client) makePublicAPICall(method string, endpoint string, requestBody io.Reader, queryParams map[string]string) (*RequestDetails, error) {
 	details := RequestDetails{}
 	// Create the request
@@ -136,9 +148,35 @@ func redactSensitiveValue(value string, sensitiveValue string) string {
 
 func sanitizeRequestDump(requestDump string, apiKey string, _ string) string {
 	sanitizedRequestDump := redactSensitiveValue(requestDump, apiKey)
+	sanitizedRequestDump = redactRequestDumpURLQuery(sanitizedRequestDump)
 	return redactSensitiveRequestBody(sanitizedRequestDump)
 }
 
+func redactRequestDumpURLQuery(requestDump string) string {
+	lineEnd := strings.Index(requestDump, "\n")
+	if lineEnd == -1 {
+		return redactRequestLineURLQuery(requestDump)
+	}
+
+	return redactRequestLineURLQuery(requestDump[:lineEnd]) + requestDump[lineEnd:]
+}
+
+func redactRequestLineURLQuery(requestLine string) string {
+	lineSuffix := ""
+	if strings.HasSuffix(requestLine, "\r") {
+		requestLine = strings.TrimSuffix(requestLine, "\r")
+		lineSuffix = "\r"
+	}
+
+	requestLineParts := strings.SplitN(requestLine, " ", 3)
+	if len(requestLineParts) != 3 {
+		return requestLine + lineSuffix
+	}
+
+	requestLineParts[1] = redactURLQueryValues(requestLineParts[1])
+	return strings.Join(requestLineParts, " ") + lineSuffix
+}
+
 func redactSensitiveRequestBody(requestDump string) string {
 	headers, body, separator, ok := splitRequestDump(requestDump)
 	if !ok || strings.TrimSpace(body) == "" {
@@ -183,6 +221,20 @@ func redactSensitiveJSONFields(value interface{}) {
 				typedValue[key] = "[REDACTED]"
 				continue
 			}
+			if isSensitiveHeaderName(key) {
+				typedValue[key] = "[REDACTED]"
+				continue
+			}
+			if isURLFieldName(key) {
+				if urlValue, ok := nestedValue.(string); ok {
+					typedValue[key] = redactURLQueryValues(urlValue)
+					continue
+				}
+			}
+			if strings.EqualFold(key, "headers") {
+				redactHeaderValues(nestedValue)
+				continue
+			}
 			redactSensitiveJSONFields(nestedValue)
 		}
 	case []interface{}:
@@ -192,6 +244,95 @@ func redactSensitiveJSONFields(value interface{}) {
 	}
 }
 
+func redactHeaderValues(value interface{}) {
+	switch typedValue := value.(type) {
+	case map[string]interface{}:
+		for key := range typedValue {
+			typedValue[key] = "[REDACTED]"
+		}
+	default:
+		redactSensitiveJSONFields(value)
+	}
+}
+
+func isSensitiveHeaderName(key string) bool {
+	lowerKey := strings.ToLower(key)
+	if _, ok := sensitiveHeaderNames[lowerKey]; ok {
+		return true
+	}
+
+	return strings.Contains(lowerKey, "token") ||
+		strings.Contains(lowerKey, "secret") ||
+		strings.Contains(lowerKey, "password")
+}
+
+func isURLFieldName(key string) bool {
+	lowerKey := strings.ToLower(key)
+	return lowerKey == "url" || strings.HasSuffix(lowerKey, "url")
+}
+
+func redactURLQueryValues(rawURL string) string {
+	redactedURL := redactURLUserinfo(rawURL)
+	queryStart := strings.Index(redactedURL, "?")
+	if queryStart == -1 || queryStart == len(redactedURL)-1 {
+		return redactedURL
+	}
+
+	prefix := redactedURL[:queryStart+1]
+	query := redactedURL[queryStart+1:]
+	fragment := ""
+	if fragmentStart := strings.Index(query, "#"); fragmentStart != -1 {
+		fragment = query[fragmentStart:]
+		query = query[:fragmentStart]
+	}
+	if query == "" {
+		return redactedURL
+	}
+
+	queryParts := strings.Split(query, "&")
+	for i, queryPart := range queryParts {
+		if queryPart == "" {
+			continue
+		}
+		key := queryPart
+		if equalSign := strings.Index(queryPart, "="); equalSign != -1 {
+			key = queryPart[:equalSign]
+		}
+		if key == "" {
+			queryParts[i] = "[REDACTED]"
+			continue
+		}
+		queryParts[i] = key + "=[REDACTED]"
+	}
+
+	return prefix + strings.Join(queryParts, "&") + fragment
+}
+
+func redactURLUserinfo(rawURL string) string {
+	authorityStart := strings.Index(rawURL, "://")
+	if authorityStart == -1 {
+		if !strings.HasPrefix(rawURL, "//") {
+			return rawURL
+		}
+		authorityStart = 2
+	} else {
+		authorityStart += len("://")
+	}
+
+	authorityEnd := len(rawURL)
+	for _, separator := range []string{"/", "?", "#"} {
+		if index := strings.Index(rawURL[authorityStart:], separator); index != -1 && authorityStart+index < authorityEnd {
+			authorityEnd = authorityStart + index
+		}
+	}
+
+	if atSign := strings.LastIndex(rawURL[authorityStart:authorityEnd], "@"); atSign != -1 {
+		return rawURL[:authorityStart] + "[REDACTED]@" + rawURL[authorityStart+atSign+1:]
+	}
+
+	return rawURL
+}
+
 func NewClientArgs(timeout int, baseUrl string) ClientArgs {
 	return ClientArgs{
 		timeoutSeconds: timeout,

syntheticsclientv2/synthetics_test.go

@@ -299,16 +299,79 @@ func TestSanitizeRequestDumpRedactsHeadersAndSecretJSONFields(t *testing.T) {
 
 func TestSanitizeRequestDumpRedactsHeaderCookieAndAuthenticationValues(t *testing.T) {
 	requestDump := "PUT /tests/browser/123 HTTP/1.1\r\nX-Sf-Token: token-abc\r\n\r\n" +
-		`{"test":{"advancedSettings":{"headers":[{"name":"Authorization","value":"Bearer secret"}],"cookies":[{"name":"session","value":"cookie-secret"}],"authentication":{"password":"auth-secret"}}}}`
+		`{"test":{"advancedSettings":{"headers":[{"name":"Authorization","value":"Bearer secret"}],"cookies":[{"name":"session","value":"cookie-secret"}],"authentication":{"username":"auth-user-secret","password":"auth-secret"}}}}`
 
 	sanitized := sanitizeRequestDump(requestDump, "token-abc", "/tests/browser/123")
 
-	for _, secret := range []string{"token-abc", "Bearer secret", "cookie-secret", "auth-secret"} {
+	for _, secret := range []string{"token-abc", "Bearer secret", "cookie-secret", "auth-user-secret", "auth-secret"} {
 		if strings.Contains(sanitized, secret) {
 			t.Fatalf("sanitized request dump leaked %q: %s", secret, sanitized)
 		}
 	}
-	for _, redacted := range []string{`"value":"[REDACTED]"`, `"password":"[REDACTED]"`, "X-Sf-Token: [REDACTED]"} {
+	for _, redacted := range []string{`"value":"[REDACTED]"`, `"username":"[REDACTED]"`, `"password":"[REDACTED]"`, "X-Sf-Token: [REDACTED]"} {
+		if !strings.Contains(sanitized, redacted) {
+			t.Fatalf("sanitized request dump missing %q: %s", redacted, sanitized)
+		}
+	}
+}
+
+func TestSanitizeRequestDumpRedactsAllAPIHeaderMapValues(t *testing.T) {
+	requestDump := "POST /v2/tests/api HTTP/1.1\r\nHost: example.com\r\nX-SF-TOKEN: client-api-key\r\n\r\n" +
+		`{"test":{"requests":[{"configuration":{"headers":{"X-SF-TOKEN":"request-token","Accept":"accept-json-secret","beep":"plain-header-secret"},"body":null}}]}}`
+
+	sanitized := sanitizeRequestDump(requestDump, "client-api-key", "/v2/tests/api")
+
+	for _, secret := range []string{"client-api-key", "request-token", "accept-json-secret", "plain-header-secret"} {
+		if strings.Contains(sanitized, secret) {
+			t.Fatalf("sanitized request dump leaked %q: %s", secret, sanitized)
+		}
+	}
+	for _, redacted := range []string{`"X-SF-TOKEN":"[REDACTED]"`, `"Accept":"[REDACTED]"`, `"beep":"[REDACTED]"`} {
+		if !strings.Contains(sanitized, redacted) {
+			t.Fatalf("sanitized request dump missing %q: %s", redacted, sanitized)
+		}
+	}
+}
+
+func TestSanitizeRequestDumpRedactsAPIAndHTTPCheckBodyFields(t *testing.T) {
+	requestDump := "POST /tests/api HTTP/1.1\r\nHost: example.com\r\nX-SF-TOKEN: client-api-key\r\n\r\n" +
+		`{"test":{"requests":[{"configuration":{"name":"login","body":"client_secret=api-body-secret&password=api-password-secret"}}],"body":"password=http-body-secret"}}`
+
+	sanitized := sanitizeRequestDump(requestDump, "client-api-key", "/tests/api")
+
+	for _, secret := range []string{"client-api-key", "api-body-secret", "api-password-secret", "http-body-secret"} {
+		if strings.Contains(sanitized, secret) {
+			t.Fatalf("sanitized request dump leaked %q: %s", secret, sanitized)
+		}
+	}
+	if got := strings.Count(sanitized, `"body":"[REDACTED]"`); got != 2 {
+		t.Fatalf("expected both API and HTTP check body fields to be redacted, saw %d redactions in: %s", got, sanitized)
+	}
+}
+
+func TestSanitizeRequestDumpRedactsURLQueryValues(t *testing.T) {
+	requestDump := "POST /tests/http?trace=client-query-secret&limit=25 HTTP/1.1\r\nHost: example.com\r\nX-SF-TOKEN: client-api-key\r\n\r\n" +
+		`{"test":{"startUrl":"https://browser-user:browser-password@browser.example/start?login_token=browser-start-token&continue=browser-start-continue","url":"https://target-user:target-password@target.example/login?token=url-token-secret&tenant=tenant-secret","callbackUrl":"https://callback-user:callback-password@callback.example/path","requests":[{"configuration":{"url":"https://api.example/search?api_key=api-url-secret&q=customer-secret"}}]}}`
+
+	sanitized := sanitizeRequestDump(requestDump, "client-api-key", "/tests/http")
+
+	for _, secret := range []string{"client-api-key", "client-query-secret", "25", "browser-user", "browser-password", "browser-start-token", "browser-start-continue", "target-user", "target-password", "url-token-secret", "tenant-secret", "callback-user", "callback-password", "api-url-secret", "customer-secret"} {
+		if strings.Contains(sanitized, secret) {
+			t.Fatalf("sanitized request dump leaked %q: %s", secret, sanitized)
+		}
+	}
+	for _, redacted := range []string{
+		"/tests/http?trace=[REDACTED]&limit=[REDACTED]",
+		"[REDACTED]@browser.example",
+		"[REDACTED]@target.example",
+		"[REDACTED]@callback.example",
+		"login_token=[REDACTED]",
+		"continue=[REDACTED]",
+		"token=[REDACTED]",
+		"tenant=[REDACTED]",
+		"api_key=[REDACTED]",
+		"q=[REDACTED]",
+	} {
 		if !strings.Contains(sanitized, redacted) {
 			t.Fatalf("sanitized request dump missing %q: %s", redacted, sanitized)
 		}