Skip to content

Commit 6b65447

Browse files
johnbleylaurit
andauthored
Propagate otel context through custom aws client context for lambda direct calls (#11675)
Co-authored-by: Lauri Tulmin <[email protected]>
1 parent ec91735 commit 6b65447

File tree

14 files changed

+502
-1
lines changed

14 files changed

+502
-1
lines changed

instrumentation/aws-lambda/aws-lambda-core-1.0/library/src/main/java/io/opentelemetry/instrumentation/awslambdacore/v1_0/internal/AwsLambdaFunctionInstrumenter.java

+11-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
1212
import io.opentelemetry.instrumentation.api.internal.ContextPropagationDebug;
1313
import io.opentelemetry.instrumentation.awslambdacore.v1_0.AwsLambdaRequest;
14+
import java.util.HashMap;
1415
import java.util.Locale;
1516
import java.util.Map;
1617
import javax.annotation.Nullable;
@@ -48,11 +49,20 @@ public void end(
4849

4950
public Context extract(AwsLambdaRequest input) {
5051
ContextPropagationDebug.debugContextLeakIfEnabled();
52+
// Look in both the http headers and the custom client context
53+
Map<String, String> headers = input.getHeaders();
54+
if (input.getAwsContext() != null && input.getAwsContext().getClientContext() != null) {
55+
Map<String, String> customContext = input.getAwsContext().getClientContext().getCustom();
56+
if (customContext != null) {
57+
headers = new HashMap<>(headers);
58+
headers.putAll(customContext);
59+
}
60+
}
5161

5262
return openTelemetry
5363
.getPropagators()
5464
.getTextMapPropagator()
55-
.extract(Context.root(), input.getHeaders(), MapGetter.INSTANCE);
65+
.extract(Context.root(), headers, MapGetter.INSTANCE);
5666
}
5767

5868
private enum MapGetter implements TextMapGetter<Map<String, String>> {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.awslambdacore.v1_0.internal;
7+
8+
import static org.assertj.core.api.Assertions.assertThat;
9+
import static org.mockito.Mockito.mock;
10+
import static org.mockito.Mockito.when;
11+
12+
import com.amazonaws.services.lambda.runtime.ClientContext;
13+
import io.opentelemetry.api.OpenTelemetry;
14+
import io.opentelemetry.api.trace.Span;
15+
import io.opentelemetry.api.trace.SpanContext;
16+
import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
17+
import io.opentelemetry.context.Context;
18+
import io.opentelemetry.context.propagation.ContextPropagators;
19+
import io.opentelemetry.instrumentation.awslambdacore.v1_0.AwsLambdaRequest;
20+
import java.util.HashMap;
21+
import org.junit.jupiter.api.Test;
22+
23+
class InstrumenterExtractionTest {
24+
@Test
25+
public void useCustomContext() {
26+
AwsLambdaFunctionInstrumenter instr =
27+
AwsLambdaFunctionInstrumenterFactory.createInstrumenter(
28+
OpenTelemetry.propagating(
29+
ContextPropagators.create(W3CTraceContextPropagator.getInstance())));
30+
com.amazonaws.services.lambda.runtime.Context awsContext =
31+
mock(com.amazonaws.services.lambda.runtime.Context.class);
32+
ClientContext clientContext = mock(ClientContext.class);
33+
when(awsContext.getClientContext()).thenReturn(clientContext);
34+
HashMap<String, String> customMap = new HashMap<>();
35+
customMap.put("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01");
36+
when(clientContext.getCustom()).thenReturn(customMap);
37+
38+
AwsLambdaRequest input = AwsLambdaRequest.create(awsContext, new HashMap<>(), new HashMap<>());
39+
40+
Context extracted = instr.extract(input);
41+
SpanContext spanContext = Span.fromContext(extracted).getSpanContext();
42+
assertThat(spanContext.getTraceId()).isEqualTo("4bf92f3577b34da6a3ce929d0e0e4736");
43+
assertThat(spanContext.getSpanId()).isEqualTo("00f067aa0ba902b7");
44+
}
45+
}

instrumentation/aws-sdk/aws-sdk-2.2/javaagent/build.gradle.kts

+18
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ muzzle {
1313

1414
excludeInstrumentationName("aws-sdk-2.2-sqs")
1515
excludeInstrumentationName("aws-sdk-2.2-sns")
16+
excludeInstrumentationName("aws-sdk-2.2-lambda")
1617

1718
// several software.amazon.awssdk artifacts are missing for this version
1819
skip("2.17.200")
@@ -43,6 +44,7 @@ muzzle {
4344
extraDependency("software.amazon.awssdk:protocol-core")
4445

4546
excludeInstrumentationName("aws-sdk-2.2-sns")
47+
excludeInstrumentationName("aws-sdk-2.2-lambda")
4648

4749
// several software.amazon.awssdk artifacts are missing for this version
4850
skip("2.17.200")
@@ -57,6 +59,21 @@ muzzle {
5759
extraDependency("software.amazon.awssdk:protocol-core")
5860

5961
excludeInstrumentationName("aws-sdk-2.2-sqs")
62+
excludeInstrumentationName("aws-sdk-2.2-lambda")
63+
64+
// several software.amazon.awssdk artifacts are missing for this version
65+
skip("2.17.200")
66+
}
67+
pass {
68+
group.set("software.amazon.awssdk")
69+
module.set("lambda")
70+
versions.set("[2.17.0,)")
71+
// Used by all SDK services, the only case it isn't is an SDK extension such as a custom HTTP
72+
// client, which is not target of instrumentation anyways.
73+
extraDependency("software.amazon.awssdk:protocol-core")
74+
75+
excludeInstrumentationName("aws-sdk-2.2-sqs")
76+
excludeInstrumentationName("aws-sdk-2.2-sns")
6077

6178
// several software.amazon.awssdk artifacts are missing for this version
6279
skip("2.17.200")
@@ -81,6 +98,7 @@ dependencies {
8198
testLibrary("software.amazon.awssdk:dynamodb:2.2.0")
8299
testLibrary("software.amazon.awssdk:ec2:2.2.0")
83100
testLibrary("software.amazon.awssdk:kinesis:2.2.0")
101+
testLibrary("software.amazon.awssdk:lambda:2.2.0")
84102
testLibrary("software.amazon.awssdk:rds:2.2.0")
85103
testLibrary("software.amazon.awssdk:s3:2.2.0")
86104
testLibrary("software.amazon.awssdk:sqs:2.2.0")
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.awssdk.v2_2;
7+
8+
public final class LambdaAdviceBridge {
9+
private LambdaAdviceBridge() {}
10+
11+
public static void referenceForMuzzleOnly() {
12+
throw new UnsupportedOperationException(
13+
LambdaImpl.class.getName() + " referencing for muzzle, should never be actually called");
14+
}
15+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.javaagent.instrumentation.awssdk.v2_2;
7+
8+
import static io.opentelemetry.javaagent.extension.matcher.AgentElementMatchers.hasClassesNamed;
9+
import static net.bytebuddy.matcher.ElementMatchers.none;
10+
11+
import com.google.auto.service.AutoService;
12+
import io.opentelemetry.instrumentation.awssdk.v2_2.LambdaAdviceBridge;
13+
import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
14+
import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
15+
import net.bytebuddy.asm.Advice;
16+
import net.bytebuddy.matcher.ElementMatcher;
17+
18+
@AutoService(InstrumentationModule.class)
19+
public class LambdaInstrumentationModule extends AbstractAwsSdkInstrumentationModule {
20+
21+
public LambdaInstrumentationModule() {
22+
super("aws-sdk-2.2-lambda");
23+
}
24+
25+
@Override
26+
public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
27+
return hasClassesNamed(
28+
"software.amazon.awssdk.services.lambda.model.InvokeRequest",
29+
"software.amazon.awssdk.protocols.jsoncore.JsonNode");
30+
}
31+
32+
@Override
33+
public void doTransform(TypeTransformer transformer) {
34+
transformer.applyAdviceToMethod(
35+
none(), LambdaInstrumentationModule.class.getName() + "$RegisterAdvice");
36+
}
37+
38+
@SuppressWarnings("unused")
39+
public static class RegisterAdvice {
40+
@Advice.OnMethodExit(suppress = Throwable.class)
41+
public static void onExit() {
42+
// (indirectly) using LambdaImpl class here to make sure it is available from LambdaAccess
43+
// (injected into app classloader) and checked by Muzzle
44+
LambdaAdviceBridge.referenceForMuzzleOnly();
45+
}
46+
}
47+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.javaagent.instrumentation.awssdk.v2_2;
7+
8+
import io.opentelemetry.instrumentation.awssdk.v2_2.AbstractAws2LambdaTest;
9+
import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
10+
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
11+
import org.junit.jupiter.api.extension.RegisterExtension;
12+
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
13+
14+
class Aws2LambdaTest extends AbstractAws2LambdaTest {
15+
16+
@RegisterExtension
17+
private static final AgentInstrumentationExtension testing =
18+
AgentInstrumentationExtension.create();
19+
20+
@Override
21+
protected InstrumentationExtension getTesting() {
22+
return testing;
23+
}
24+
25+
@Override
26+
protected boolean canTestLambdaInvoke() {
27+
// only supported since 2.17.0
28+
return Boolean.getBoolean("testLatestDeps");
29+
}
30+
31+
@Override
32+
protected ClientOverrideConfiguration.Builder createOverrideConfigurationBuilder() {
33+
return ClientOverrideConfiguration.builder();
34+
}
35+
}

instrumentation/aws-sdk/aws-sdk-2.2/library-autoconfigure/build.gradle.kts

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ dependencies {
1717
testLibrary("software.amazon.awssdk:dynamodb:2.2.0")
1818
testLibrary("software.amazon.awssdk:ec2:2.2.0")
1919
testLibrary("software.amazon.awssdk:kinesis:2.2.0")
20+
testLibrary("software.amazon.awssdk:lambda:2.2.0")
2021
testLibrary("software.amazon.awssdk:rds:2.2.0")
2122
testLibrary("software.amazon.awssdk:s3:2.2.0")
2223
testLibrary("software.amazon.awssdk:sqs:2.2.0")

instrumentation/aws-sdk/aws-sdk-2.2/library/build.gradle.kts

+17
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,11 @@ dependencies {
77

88
library("software.amazon.awssdk:aws-core:2.2.0")
99
library("software.amazon.awssdk:sqs:2.2.0")
10+
library("software.amazon.awssdk:lambda:2.2.0")
1011
library("software.amazon.awssdk:sns:2.2.0")
1112
library("software.amazon.awssdk:aws-json-protocol:2.2.0")
13+
// json-utils was added in 2.17.0
14+
compileOnly("software.amazon.awssdk:json-utils:2.17.0")
1215
compileOnly(project(":muzzle")) // For @NoMuzzle
1316

1417
testImplementation(project(":instrumentation:aws-sdk:aws-sdk-2.2:testing"))
@@ -38,10 +41,24 @@ testing {
3841
implementation("software.amazon.awssdk:aws-core:+")
3942
implementation("software.amazon.awssdk:aws-json-protocol:+")
4043
implementation("software.amazon.awssdk:dynamodb:+")
44+
implementation("software.amazon.awssdk:lambda:+")
4145
} else {
4246
implementation("software.amazon.awssdk:aws-core:2.2.0")
4347
implementation("software.amazon.awssdk:aws-json-protocol:2.2.0")
4448
implementation("software.amazon.awssdk:dynamodb:2.2.0")
49+
implementation("software.amazon.awssdk:lambda:2.2.0")
50+
}
51+
}
52+
}
53+
54+
val testLambda by registering(JvmTestSuite::class) {
55+
dependencies {
56+
implementation(project())
57+
implementation(project(":instrumentation:aws-sdk:aws-sdk-2.2:testing"))
58+
if (findProperty("testLatestDeps") as Boolean) {
59+
implementation("software.amazon.awssdk:lambda:+")
60+
} else {
61+
implementation("software.amazon.awssdk:lambda:2.17.0")
4562
}
4663
}
4764
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.awssdk.v2_2;
7+
8+
import io.opentelemetry.context.Context;
9+
import io.opentelemetry.javaagent.tooling.muzzle.NoMuzzle;
10+
import software.amazon.awssdk.core.SdkRequest;
11+
12+
final class LambdaAccess {
13+
private LambdaAccess() {}
14+
15+
private static final boolean enabled = PluginImplUtil.isImplPresent("LambdaImpl");
16+
17+
@NoMuzzle
18+
public static SdkRequest modifyRequest(SdkRequest request, Context otelContext) {
19+
return enabled ? LambdaImpl.modifyRequest(request, otelContext) : null;
20+
}
21+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.awssdk.v2_2;
7+
8+
import io.opentelemetry.api.GlobalOpenTelemetry;
9+
import java.nio.charset.StandardCharsets;
10+
import java.util.Base64;
11+
import java.util.LinkedHashMap;
12+
import java.util.Map;
13+
import javax.annotation.Nullable;
14+
import software.amazon.awssdk.core.SdkRequest;
15+
import software.amazon.awssdk.protocols.jsoncore.JsonNode;
16+
import software.amazon.awssdk.protocols.jsoncore.internal.ObjectJsonNode;
17+
import software.amazon.awssdk.protocols.jsoncore.internal.StringJsonNode;
18+
import software.amazon.awssdk.services.lambda.model.InvokeRequest;
19+
20+
// this class is only used from LambdaAccess from method with @NoMuzzle annotation
21+
22+
// Direct lambda invocations (e.g., not through an api gateway) currently strip
23+
// away the otel propagation headers (but leave x-ray ones intact). Use the
24+
// custom client context header as an additional propagation mechanism for this
25+
// very specific scenario. For reference, the header is named "X-Amz-Client-Context" but the api to
26+
// manipulate it abstracts that away. The client context field is documented in
27+
// https://docs.aws.amazon.com/lambda/latest/api/API_Invoke.html#API_Invoke_RequestParameters
28+
29+
final class LambdaImpl {
30+
static {
31+
// Force loading of InvokeRequest; this ensures that an exception is thrown at this point when
32+
// the Lambda library is not present, which will cause DirectLambdaAccess to have
33+
// enabled=false in library mode.
34+
@SuppressWarnings("unused")
35+
String invokeRequestName = InvokeRequest.class.getName();
36+
// was added in 2.17.0
37+
@SuppressWarnings("unused")
38+
String jsonNodeName = JsonNode.class.getName();
39+
}
40+
41+
private static final String CLIENT_CONTEXT_CUSTOM_FIELDS_KEY = "custom";
42+
static final int MAX_CLIENT_CONTEXT_LENGTH = 3583; // visible for testing
43+
44+
private LambdaImpl() {}
45+
46+
@Nullable
47+
static SdkRequest modifyRequest(
48+
SdkRequest request, io.opentelemetry.context.Context otelContext) {
49+
if (isDirectLambdaInvocation(request)) {
50+
return modifyOrAddCustomContextHeader((InvokeRequest) request, otelContext);
51+
}
52+
return null;
53+
}
54+
55+
static boolean isDirectLambdaInvocation(SdkRequest request) {
56+
return request instanceof InvokeRequest;
57+
}
58+
59+
static SdkRequest modifyOrAddCustomContextHeader(
60+
InvokeRequest request, io.opentelemetry.context.Context otelContext) {
61+
InvokeRequest.Builder builder = request.toBuilder();
62+
// Unfortunately the value of this thing is a base64-encoded json with a character limit; also
63+
// therefore not comma-composable like many http headers
64+
String clientContextString = request.clientContext();
65+
String clientContextJsonString = "{}";
66+
if (clientContextString != null && !clientContextString.isEmpty()) {
67+
clientContextJsonString =
68+
new String(Base64.getDecoder().decode(clientContextString), StandardCharsets.UTF_8);
69+
}
70+
JsonNode jsonNode = JsonNode.parser().parse(clientContextJsonString);
71+
if (!jsonNode.isObject()) {
72+
return null;
73+
}
74+
JsonNode customNode =
75+
jsonNode
76+
.asObject()
77+
.computeIfAbsent(
78+
CLIENT_CONTEXT_CUSTOM_FIELDS_KEY, (k) -> new ObjectJsonNode(new LinkedHashMap<>()));
79+
if (!customNode.isObject()) {
80+
return null;
81+
}
82+
Map<String, JsonNode> map = customNode.asObject();
83+
GlobalOpenTelemetry.getPropagators()
84+
.getTextMapPropagator()
85+
.inject(otelContext, map, (nodes, key, value) -> nodes.put(key, new StringJsonNode(value)));
86+
if (map.isEmpty()) {
87+
return null;
88+
}
89+
90+
// turn it back into a string (json encode)
91+
String newJson = jsonNode.toString();
92+
93+
// turn it back into a base64 string
94+
String newJson64 = Base64.getEncoder().encodeToString(newJson.getBytes(StandardCharsets.UTF_8));
95+
// check it for length (err on the safe side with >=)
96+
if (newJson64.length() >= MAX_CLIENT_CONTEXT_LENGTH) {
97+
return null;
98+
}
99+
builder.clientContext(newJson64);
100+
return builder.build();
101+
}
102+
}

0 commit comments

Comments
 (0)