Skip to content

Commit 5b287e3

Browse files
jeanbisuttisteveraolaurittrask
authored
Redact query string values for http client spans (#13114)
Co-authored-by: Steve Rao <[email protected]> Co-authored-by: Lauri Tulmin <[email protected]> Co-authored-by: Lauri Tulmin <[email protected]> Co-authored-by: Trask Stalnaker <[email protected]>
1 parent c93eecb commit 5b287e3

File tree

9 files changed

+262
-1
lines changed

9 files changed

+262
-1
lines changed

‎instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/builder/internal/DefaultHttpClientInstrumenterBuilder.java

+15
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import io.opentelemetry.instrumentation.api.instrumenter.SpanKindExtractor;
2121
import io.opentelemetry.instrumentation.api.instrumenter.SpanNameExtractor;
2222
import io.opentelemetry.instrumentation.api.instrumenter.SpanStatusExtractor;
23+
import io.opentelemetry.instrumentation.api.internal.Experimental;
2324
import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesExtractor;
2425
import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesExtractorBuilder;
2526
import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesGetter;
@@ -177,6 +178,18 @@ public DefaultHttpClientInstrumenterBuilder<REQUEST, RESPONSE> setKnownMethods(
177178
return this;
178179
}
179180

181+
/**
182+
* Configures the instrumentation to redact sensitive URL parameters.
183+
*
184+
* @param redactQueryParameters {@code true} if the sensitive URL parameters have to be redacted.
185+
*/
186+
@CanIgnoreReturnValue
187+
public DefaultHttpClientInstrumenterBuilder<REQUEST, RESPONSE> setRedactQueryParameters(
188+
boolean redactQueryParameters) {
189+
Experimental.setRedactQueryParameters(httpAttributesExtractorBuilder, redactQueryParameters);
190+
return this;
191+
}
192+
180193
/** Sets custom {@link SpanNameExtractor} via transform function. */
181194
@CanIgnoreReturnValue
182195
public DefaultHttpClientInstrumenterBuilder<REQUEST, RESPONSE> setSpanNameExtractor(
@@ -225,6 +238,7 @@ public Instrumenter<REQUEST, RESPONSE> build() {
225238
.addAttributesExtractor(HttpExperimentalAttributesExtractor.create(attributesGetter))
226239
.addOperationMetrics(HttpClientExperimentalMetrics.get());
227240
}
241+
228242
builderCustomizer.accept(builder);
229243

230244
if (headerSetter != null) {
@@ -248,6 +262,7 @@ public DefaultHttpClientInstrumenterBuilder<REQUEST, RESPONSE> configure(CommonC
248262
set(
249263
config::shouldEmitExperimentalHttpClientTelemetry,
250264
this::setEmitExperimentalHttpClientMetrics);
265+
set(config::redactQueryParameters, this::setRedactQueryParameters);
251266
return this;
252267
}
253268

‎instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/config/internal/CommonConfig.java

+8
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ public final class CommonConfig {
3131
private final boolean statementSanitizationEnabled;
3232
private final boolean emitExperimentalHttpClientTelemetry;
3333
private final boolean emitExperimentalHttpServerTelemetry;
34+
private final boolean redactQueryParameters;
3435
private final String loggingTraceIdKey;
3536
private final String loggingSpanIdKey;
3637
private final String loggingTraceFlagsKey;
@@ -57,6 +58,9 @@ public CommonConfig(InstrumentationConfig config) {
5758
config.getBoolean("otel.instrumentation.common.db-statement-sanitizer.enabled", true);
5859
emitExperimentalHttpClientTelemetry =
5960
config.getBoolean("otel.instrumentation.http.client.emit-experimental-telemetry", false);
61+
redactQueryParameters =
62+
config.getBoolean(
63+
"otel.instrumentation.http.client.experimental.redact-query-parameters", true);
6064
emitExperimentalHttpServerTelemetry =
6165
config.getBoolean("otel.instrumentation.http.server.emit-experimental-telemetry", false);
6266
enduserConfig = new EnduserConfig(config);
@@ -111,6 +115,10 @@ public boolean shouldEmitExperimentalHttpServerTelemetry() {
111115
return emitExperimentalHttpServerTelemetry;
112116
}
113117

118+
public boolean redactQueryParameters() {
119+
return redactQueryParameters;
120+
}
121+
114122
public String getTraceIdKey() {
115123
return loggingTraceIdKey;
116124
}

‎instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/internal/Experimental.java

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.api.internal;
7+
8+
import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesExtractorBuilder;
9+
import java.util.function.BiConsumer;
10+
import javax.annotation.Nullable;
11+
12+
/**
13+
* This class is internal and is hence not for public use. Its APIs are unstable and can change at
14+
* any time.
15+
*/
16+
public final class Experimental {
17+
18+
@Nullable
19+
private static volatile BiConsumer<HttpClientAttributesExtractorBuilder<?, ?>, Boolean>
20+
redactHttpClientQueryParameters;
21+
22+
private Experimental() {}
23+
24+
public static void setRedactQueryParameters(
25+
HttpClientAttributesExtractorBuilder<?, ?> builder, boolean redactQueryParameters) {
26+
if (redactHttpClientQueryParameters != null) {
27+
redactHttpClientQueryParameters.accept(builder, redactQueryParameters);
28+
}
29+
}
30+
31+
public static void internalSetRedactHttpClientQueryParameters(
32+
BiConsumer<HttpClientAttributesExtractorBuilder<?, ?>, Boolean>
33+
redactHttpClientQueryParameters) {
34+
Experimental.redactHttpClientQueryParameters = redactHttpClientQueryParameters;
35+
}
36+
}

‎instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/semconv/http/HttpClientAttributesExtractor.java

+72-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
import io.opentelemetry.instrumentation.api.semconv.network.internal.InternalServerAttributesExtractor;
1818
import io.opentelemetry.semconv.HttpAttributes;
1919
import io.opentelemetry.semconv.UrlAttributes;
20+
import java.util.Arrays;
21+
import java.util.HashSet;
22+
import java.util.Set;
2023
import java.util.function.ToIntFunction;
2124
import javax.annotation.Nullable;
2225

@@ -32,6 +35,9 @@ public final class HttpClientAttributesExtractor<REQUEST, RESPONSE>
3235
REQUEST, RESPONSE, HttpClientAttributesGetter<REQUEST, RESPONSE>>
3336
implements SpanKeyProvider {
3437

38+
private static final Set<String> PARAMS_TO_REDACT =
39+
new HashSet<>(Arrays.asList("AWSAccessKeyId", "Signature", "sig", "X-Goog-Signature"));
40+
3541
/**
3642
* Creates the HTTP client attributes extractor with default configuration.
3743
*
@@ -54,6 +60,7 @@ public static <REQUEST, RESPONSE> HttpClientAttributesExtractorBuilder<REQUEST,
5460
private final InternalNetworkAttributesExtractor<REQUEST, RESPONSE> internalNetworkExtractor;
5561
private final InternalServerAttributesExtractor<REQUEST> internalServerExtractor;
5662
private final ToIntFunction<Context> resendCountIncrementer;
63+
private final boolean redactQueryParameters;
5764

5865
HttpClientAttributesExtractor(HttpClientAttributesExtractorBuilder<REQUEST, RESPONSE> builder) {
5966
super(
@@ -65,6 +72,7 @@ public static <REQUEST, RESPONSE> HttpClientAttributesExtractorBuilder<REQUEST,
6572
internalNetworkExtractor = builder.buildNetworkExtractor();
6673
internalServerExtractor = builder.buildServerExtractor();
6774
resendCountIncrementer = builder.resendCountIncrementer;
75+
redactQueryParameters = builder.redactQueryParameters;
6876
}
6977

7078
@Override
@@ -104,11 +112,21 @@ public SpanKey internalGetSpanKey() {
104112
}
105113

106114
@Nullable
107-
private static String stripSensitiveData(@Nullable String url) {
115+
private String stripSensitiveData(@Nullable String url) {
108116
if (url == null || url.isEmpty()) {
109117
return url;
110118
}
111119

120+
url = redactUserInfo(url);
121+
122+
if (redactQueryParameters) {
123+
url = redactQueryParameters(url);
124+
}
125+
126+
return url;
127+
}
128+
129+
private static String redactUserInfo(String url) {
112130
int schemeEndIndex = url.indexOf(':');
113131

114132
if (schemeEndIndex == -1) {
@@ -145,4 +163,57 @@ private static String stripSensitiveData(@Nullable String url) {
145163
}
146164
return url.substring(0, schemeEndIndex + 3) + "REDACTED:REDACTED" + url.substring(atIndex);
147165
}
166+
167+
private static String redactQueryParameters(String url) {
168+
int questionMarkIndex = url.indexOf('?');
169+
if (questionMarkIndex == -1 || !containsParamToRedact(url)) {
170+
return url;
171+
}
172+
173+
StringBuilder urlAfterQuestionMark = new StringBuilder();
174+
175+
// To build a parameter name until we reach the '=' character
176+
// If the parameter name is a one to redact, we will redact the value
177+
StringBuilder currentParamName = new StringBuilder();
178+
179+
for (int i = questionMarkIndex + 1; i < url.length(); i++) {
180+
char currentChar = url.charAt(i);
181+
182+
if (currentChar == '=') {
183+
urlAfterQuestionMark.append('=');
184+
if (PARAMS_TO_REDACT.contains(currentParamName.toString())) {
185+
urlAfterQuestionMark.append("REDACTED");
186+
// skip over parameter value
187+
for (; i + 1 < url.length(); i++) {
188+
char c = url.charAt(i + 1);
189+
if (c == '&' || c == '#') {
190+
break;
191+
}
192+
}
193+
}
194+
} else if (currentChar == '&') { // New parameter delimiter
195+
urlAfterQuestionMark.append(currentChar);
196+
// To avoid creating a new StringBuilder for each new parameter
197+
currentParamName.setLength(0);
198+
} else if (currentChar == '#') { // Reference delimiter
199+
urlAfterQuestionMark.append(url.substring(i));
200+
break;
201+
} else {
202+
// param values can be appended to currentParamName here but it's not an issue
203+
currentParamName.append(currentChar);
204+
urlAfterQuestionMark.append(currentChar);
205+
}
206+
}
207+
208+
return url.substring(0, questionMarkIndex) + "?" + urlAfterQuestionMark;
209+
}
210+
211+
private static boolean containsParamToRedact(String urlpart) {
212+
for (String param : PARAMS_TO_REDACT) {
213+
if (urlpart.contains(param)) {
214+
return true;
215+
}
216+
}
217+
return false;
218+
}
148219
}

‎instrumentation-api/src/main/java/io/opentelemetry/instrumentation/api/semconv/http/HttpClientAttributesExtractorBuilder.java

+7
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import io.opentelemetry.context.Context;
1212
import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor;
1313
import io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder;
14+
import io.opentelemetry.instrumentation.api.internal.Experimental;
1415
import io.opentelemetry.instrumentation.api.internal.HttpConstants;
1516
import io.opentelemetry.instrumentation.api.semconv.network.internal.AddressAndPortExtractor;
1617
import io.opentelemetry.instrumentation.api.semconv.network.internal.InternalNetworkAttributesExtractor;
@@ -37,6 +38,12 @@ public final class HttpClientAttributesExtractorBuilder<REQUEST, RESPONSE> {
3738
List<String> capturedResponseHeaders = emptyList();
3839
Set<String> knownMethods = HttpConstants.KNOWN_METHODS;
3940
ToIntFunction<Context> resendCountIncrementer = HttpClientRequestResendCount::getAndIncrement;
41+
boolean redactQueryParameters;
42+
43+
static {
44+
Experimental.internalSetRedactHttpClientQueryParameters(
45+
(builder, redact) -> builder.redactQueryParameters = redact);
46+
}
4047

4148
HttpClientAttributesExtractorBuilder(
4249
HttpClientAttributesGetter<REQUEST, RESPONSE> httpAttributesGetter) {

‎instrumentation-api/src/test/java/io/opentelemetry/instrumentation/api/semconv/http/HttpClientAttributesExtractorTest.java

+93
Original file line numberDiff line numberDiff line change
@@ -23,22 +23,28 @@
2323
import static java.util.Collections.emptyMap;
2424
import static java.util.Collections.singletonList;
2525
import static org.assertj.core.api.Assertions.entry;
26+
import static org.junit.jupiter.params.provider.Arguments.arguments;
2627

2728
import io.opentelemetry.api.common.AttributeKey;
2829
import io.opentelemetry.api.common.Attributes;
2930
import io.opentelemetry.api.common.AttributesBuilder;
3031
import io.opentelemetry.context.Context;
3132
import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor;
33+
import io.opentelemetry.instrumentation.api.internal.Experimental;
3234
import io.opentelemetry.instrumentation.api.internal.HttpConstants;
3335
import java.net.ConnectException;
3436
import java.util.HashMap;
3537
import java.util.HashSet;
3638
import java.util.List;
3739
import java.util.Map;
3840
import java.util.function.ToIntFunction;
41+
import java.util.stream.Stream;
3942
import javax.annotation.Nullable;
4043
import org.junit.jupiter.api.Test;
44+
import org.junit.jupiter.api.extension.ExtensionContext;
4145
import org.junit.jupiter.params.ParameterizedTest;
46+
import org.junit.jupiter.params.provider.Arguments;
47+
import org.junit.jupiter.params.provider.ArgumentsProvider;
4248
import org.junit.jupiter.params.provider.ArgumentsSource;
4349
import org.junit.jupiter.params.provider.ValueSource;
4450

@@ -200,6 +206,93 @@ void normal() {
200206
entry(NETWORK_PEER_PORT, 456L));
201207
}
202208

209+
@ParameterizedTest
210+
@ArgumentsSource(UrlSourceToRedact.class)
211+
void shouldRedactUserInfoAndQueryParameters(String url, String expectedResult) {
212+
Map<String, String> request = new HashMap<>();
213+
request.put("urlFull", url);
214+
215+
HttpClientAttributesExtractorBuilder<Map<String, String>, Map<String, String>> builder =
216+
HttpClientAttributesExtractor.builder(new TestHttpClientAttributesGetter());
217+
Experimental.setRedactQueryParameters(builder, true);
218+
AttributesExtractor<Map<String, String>, Map<String, String>> extractor = builder.build();
219+
220+
AttributesBuilder attributes = Attributes.builder();
221+
extractor.onStart(attributes, Context.root(), request);
222+
223+
assertThat(attributes.build()).containsOnly(entry(URL_FULL, expectedResult));
224+
}
225+
226+
static final class UrlSourceToRedact implements ArgumentsProvider {
227+
228+
@Override
229+
public Stream<? extends Arguments> provideArguments(ExtensionContext context) {
230+
return Stream.of(
231+
arguments("https://user1:[email protected]", "https://REDACTED:[email protected]"),
232+
arguments(
233+
"https://user1:[email protected]/path/",
234+
"https://REDACTED:[email protected]/path/"),
235+
arguments(
236+
"https://user1:[email protected]#test.html",
237+
"https://REDACTED:[email protected]#test.html"),
238+
arguments(
239+
"https://user1:[email protected]?foo=b@r",
240+
"https://REDACTED:[email protected]?foo=b@r"),
241+
arguments(
242+
"https://user1:[email protected]/p@th?foo=b@r",
243+
"https://REDACTED:[email protected]/p@th?foo=b@r"),
244+
arguments("https://github.com/p@th?foo=b@r", "https://github.com/p@th?foo=b@r"),
245+
arguments("https://github.com#[email protected]", "https://github.com#[email protected]"),
246+
arguments("user1:[email protected]", "user1:[email protected]"),
247+
arguments("https://github.com@", "https://github.com@"),
248+
arguments(
249+
"https://service.com?paramA=valA&paramB=valB",
250+
"https://service.com?paramA=valA&paramB=valB"),
251+
arguments(
252+
"https://service.com?AWSAccessKeyId=AKIAIOSFODNN7",
253+
"https://service.com?AWSAccessKeyId=REDACTED"),
254+
arguments(
255+
"https://service.com?Signature=39Up9jzHkxhuIhFE9594DJxe7w6cIRCg0V6ICGS0%3A377",
256+
"https://service.com?Signature=REDACTED"),
257+
arguments(
258+
"https://service.com?sig=39Up9jzHkxhuIhFE9594DJxe7w6cIRCg0V6ICGS0",
259+
"https://service.com?sig=REDACTED"),
260+
arguments(
261+
"https://service.com?X-Goog-Signature=39Up9jzHkxhuIhFE9594DJxe7w6cIRCg0V6ICGS0",
262+
"https://service.com?X-Goog-Signature=REDACTED"),
263+
arguments(
264+
"https://service.com?paramA=valA&AWSAccessKeyId=AKIAIOSFODNN7&paramB=valB",
265+
"https://service.com?paramA=valA&AWSAccessKeyId=REDACTED&paramB=valB"),
266+
arguments(
267+
"https://service.com?AWSAccessKeyId=AKIAIOSFODNN7&paramA=valA",
268+
"https://service.com?AWSAccessKeyId=REDACTED&paramA=valA"),
269+
arguments(
270+
"https://service.com?paramA=valA&AWSAccessKeyId=AKIAIOSFODNN7",
271+
"https://service.com?paramA=valA&AWSAccessKeyId=REDACTED"),
272+
arguments(
273+
"https://service.com?AWSAccessKeyId=AKIAIOSFODNN7&AWSAccessKeyId=ZGIAIOSFODNN7",
274+
"https://service.com?AWSAccessKeyId=REDACTED&AWSAccessKeyId=REDACTED"),
275+
arguments(
276+
"https://service.com?AWSAccessKeyId=AKIAIOSFODNN7#ref",
277+
"https://service.com?AWSAccessKeyId=REDACTED#ref"),
278+
arguments(
279+
"https://service.com?AWSAccessKeyId=AKIAIOSFODNN7&aa&bb",
280+
"https://service.com?AWSAccessKeyId=REDACTED&aa&bb"),
281+
arguments(
282+
"https://service.com?aa&bb&AWSAccessKeyId=AKIAIOSFODNN7",
283+
"https://service.com?aa&bb&AWSAccessKeyId=REDACTED"),
284+
arguments(
285+
"https://service.com?AWSAccessKeyId=AKIAIOSFODNN7&&",
286+
"https://service.com?AWSAccessKeyId=REDACTED&&"),
287+
arguments(
288+
"https://service.com?&&AWSAccessKeyId=AKIAIOSFODNN7",
289+
"https://service.com?&&AWSAccessKeyId=REDACTED"),
290+
arguments(
291+
"https://service.com?AWSAccessKeyId=AKIAIOSFODNN7&a&b#fragment",
292+
"https://service.com?AWSAccessKeyId=REDACTED&a&b#fragment"));
293+
}
294+
}
295+
203296
@ParameterizedTest
204297
@ArgumentsSource(ValidRequestMethodsProvider.class)
205298
void shouldExtractKnownMethods(String requestMethod) {

‎instrumentation/spring/spring-boot-autoconfigure/src/main/resources/META-INF/additional-spring-configuration-metadata.json

+6
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,12 @@
311311
"description": "Enable the capture of experimental HTTP client telemetry. Add the <code>http.request.body.size</code> and <code>http.response.body.size> attributes to spans, and record the <code>http.client.request.size</code> and <code>http.client.response.size</code> metrics.",
312312
"defaultValue": false
313313
},
314+
{
315+
"name": "otel.instrumentation.http.client.experimental.redact-query-parameters",
316+
"type": "java.lang.Boolean",
317+
"description": "Redact sensitive URL parameters. See https://opentelemetry.io/docs/specs/semconv/http/http-spans.",
318+
"defaultValue": true
319+
},
314320
{
315321
"name": "otel.instrumentation.http.known-methods",
316322
"type": "java.util.List<java.lang.String>",

0 commit comments

Comments
 (0)