diff --git a/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiAttributesExtractor.java b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiAttributesExtractor.java index 2207e0c9f865..263f9c99fcb5 100644 --- a/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiAttributesExtractor.java +++ b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiAttributesExtractor.java @@ -29,16 +29,14 @@ public final class GenAiAttributesExtractor implements AttributesExtractor { // copied from GenAiIncubatingAttributes - private static final AttributeKey GEN_AI_OPERATION_NAME = - stringKey("gen_ai.operation.name"); + static final AttributeKey GEN_AI_OPERATION_NAME = stringKey("gen_ai.operation.name"); private static final AttributeKey> GEN_AI_REQUEST_ENCODING_FORMATS = stringArrayKey("gen_ai.request.encoding_formats"); private static final AttributeKey GEN_AI_REQUEST_FREQUENCY_PENALTY = doubleKey("gen_ai.request.frequency_penalty"); private static final AttributeKey GEN_AI_REQUEST_MAX_TOKENS = longKey("gen_ai.request.max_tokens"); - private static final AttributeKey GEN_AI_REQUEST_MODEL = - stringKey("gen_ai.request.model"); + static final AttributeKey GEN_AI_REQUEST_MODEL = stringKey("gen_ai.request.model"); private static final AttributeKey GEN_AI_REQUEST_PRESENCE_PENALTY = doubleKey("gen_ai.request.presence_penalty"); private static final AttributeKey GEN_AI_REQUEST_SEED = longKey("gen_ai.request.seed"); @@ -53,12 +51,10 @@ public final class GenAiAttributesExtractor private static final AttributeKey> GEN_AI_RESPONSE_FINISH_REASONS = stringArrayKey("gen_ai.response.finish_reasons"); private static final AttributeKey GEN_AI_RESPONSE_ID = stringKey("gen_ai.response.id"); - private static final AttributeKey GEN_AI_RESPONSE_MODEL = - stringKey("gen_ai.response.model"); - private static final AttributeKey GEN_AI_SYSTEM = stringKey("gen_ai.system"); - private static final AttributeKey GEN_AI_USAGE_INPUT_TOKENS = - longKey("gen_ai.usage.input_tokens"); - private static final AttributeKey GEN_AI_USAGE_OUTPUT_TOKENS = + static final AttributeKey GEN_AI_RESPONSE_MODEL = stringKey("gen_ai.response.model"); + static final AttributeKey GEN_AI_SYSTEM = stringKey("gen_ai.system"); + static final AttributeKey GEN_AI_USAGE_INPUT_TOKENS = longKey("gen_ai.usage.input_tokens"); + static final AttributeKey GEN_AI_USAGE_OUTPUT_TOKENS = longKey("gen_ai.usage.output_tokens"); /** Creates the GenAI attributes extractor. */ diff --git a/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiClientMetrics.java b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiClientMetrics.java new file mode 100644 index 000000000000..52abf0f65c00 --- /dev/null +++ b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiClientMetrics.java @@ -0,0 +1,123 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.instrumentation.api.incubator.semconv.genai; + +import static io.opentelemetry.api.common.AttributeKey.stringKey; +import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor.GEN_AI_USAGE_INPUT_TOKENS; +import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor.GEN_AI_USAGE_OUTPUT_TOKENS; +import static java.util.logging.Level.FINE; + +import com.google.auto.value.AutoValue; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.LongHistogramBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.instrumentation.api.incubator.semconv.db.DbClientMetrics; +import io.opentelemetry.instrumentation.api.instrumenter.OperationListener; +import io.opentelemetry.instrumentation.api.instrumenter.OperationMetrics; +import io.opentelemetry.instrumentation.api.internal.OperationMetricsUtil; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * {@link OperationListener} which keeps track of Generative + * AI Client Metrics. + */ +public final class GenAiClientMetrics implements OperationListener { + + private static final double NANOS_PER_S = TimeUnit.SECONDS.toNanos(1); + + private static final ContextKey GEN_AI_CLIENT_METRICS_STATE = + ContextKey.named("gen-ai-client-metrics-state"); + + private static final Logger logger = Logger.getLogger(DbClientMetrics.class.getName()); + + static final AttributeKey GEN_AI_TOKEN_TYPE = stringKey("gen_ai.token.type"); + + /** + * Returns an {@link OperationMetrics} instance which can be used to enable recording of {@link + * GenAiClientMetrics}. + * + * @see + * io.opentelemetry.instrumentation.api.instrumenter.InstrumenterBuilder#addOperationMetrics(OperationMetrics) + */ + public static OperationMetrics get() { + return OperationMetricsUtil.create("gen_ai client", GenAiClientMetrics::new); + } + + private final LongHistogram tokenUsage; + private final DoubleHistogram operationDuration; + + private GenAiClientMetrics(Meter meter) { + LongHistogramBuilder tokenUsageBuilder = + meter + .histogramBuilder("gen_ai.client.token.usage") + .ofLongs() + .setUnit("{token}") + .setDescription("Measures number of input and output tokens used") + .setExplicitBucketBoundariesAdvice(GenAiMetricsAdvice.CLIENT_TOKEN_USAGE_BUCKETS); + GenAiMetricsAdvice.applyClientTokenUsageAdvice(tokenUsageBuilder); + this.tokenUsage = tokenUsageBuilder.build(); + DoubleHistogramBuilder operationDurationBuilder = + meter + .histogramBuilder("gen_ai.client.operation.duration") + .setUnit("s") + .setDescription("GenAI operation duration") + .setExplicitBucketBoundariesAdvice( + GenAiMetricsAdvice.CLIENT_OPERATION_DURATION_BUCKETS); + GenAiMetricsAdvice.applyClientOperationDurationAdvice(operationDurationBuilder); + this.operationDuration = operationDurationBuilder.build(); + } + + @Override + public Context onStart(Context context, Attributes startAttributes, long startNanos) { + return context.with( + GEN_AI_CLIENT_METRICS_STATE, + new AutoValue_GenAiClientMetrics_State(startAttributes, startNanos)); + } + + @Override + public void onEnd(Context context, Attributes endAttributes, long endNanos) { + State state = context.get(GEN_AI_CLIENT_METRICS_STATE); + if (state == null) { + logger.log( + FINE, + "No state present when ending context {0}. Cannot record gen_ai operation metrics.", + context); + return; + } + + AttributesBuilder attributesBuilder = state.startAttributes().toBuilder().putAll(endAttributes); + + operationDuration.record( + (endNanos - state.startTimeNanos()) / NANOS_PER_S, attributesBuilder.build(), context); + + Long inputTokens = endAttributes.get(GEN_AI_USAGE_INPUT_TOKENS); + if (inputTokens != null) { + tokenUsage.record( + inputTokens, attributesBuilder.put(GEN_AI_TOKEN_TYPE, "input").build(), context); + } + Long outputTokens = endAttributes.get(GEN_AI_USAGE_OUTPUT_TOKENS); + if (outputTokens != null) { + tokenUsage.record( + outputTokens, attributesBuilder.put(GEN_AI_TOKEN_TYPE, "output").build(), context); + } + } + + @AutoValue + abstract static class State { + abstract Attributes startAttributes(); + + abstract long startTimeNanos(); + } +} diff --git a/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiMetricsAdvice.java b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiMetricsAdvice.java new file mode 100644 index 000000000000..0efa2b1beb6d --- /dev/null +++ b/instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/genai/GenAiMetricsAdvice.java @@ -0,0 +1,72 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.instrumentation.api.incubator.semconv.genai; + +import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor.GEN_AI_OPERATION_NAME; +import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor.GEN_AI_REQUEST_MODEL; +import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor.GEN_AI_RESPONSE_MODEL; +import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor.GEN_AI_SYSTEM; +import static io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiClientMetrics.GEN_AI_TOKEN_TYPE; +import static io.opentelemetry.semconv.ServerAttributes.SERVER_ADDRESS; +import static io.opentelemetry.semconv.ServerAttributes.SERVER_PORT; +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; + +import io.opentelemetry.api.incubator.metrics.ExtendedDoubleHistogramBuilder; +import io.opentelemetry.api.incubator.metrics.ExtendedLongHistogramBuilder; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.LongHistogramBuilder; +import io.opentelemetry.semconv.ErrorAttributes; +import java.util.List; + +final class GenAiMetricsAdvice { + + static final List CLIENT_OPERATION_DURATION_BUCKETS = + unmodifiableList( + asList( + 0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, + 81.92)); + + static final List CLIENT_TOKEN_USAGE_BUCKETS = + unmodifiableList( + asList( + 1L, 4L, 16L, 64L, 256L, 1024L, 4096L, 16384L, 65536L, 262144L, 1048576L, 4194304L, + 16777216L, 67108864L)); + + static void applyClientTokenUsageAdvice(LongHistogramBuilder builder) { + if (!(builder instanceof ExtendedLongHistogramBuilder)) { + return; + } + ((ExtendedLongHistogramBuilder) builder) + .setAttributesAdvice( + asList( + GEN_AI_OPERATION_NAME, + GEN_AI_SYSTEM, + GEN_AI_TOKEN_TYPE, + GEN_AI_REQUEST_MODEL, + SERVER_PORT, + GEN_AI_RESPONSE_MODEL, + SERVER_ADDRESS)); + } + + static void applyClientOperationDurationAdvice(DoubleHistogramBuilder builder) { + if (!(builder instanceof ExtendedDoubleHistogramBuilder)) { + return; + } + ((ExtendedDoubleHistogramBuilder) builder) + .setAttributesAdvice( + asList( + GEN_AI_OPERATION_NAME, + GEN_AI_SYSTEM, + ErrorAttributes.ERROR_TYPE, + GEN_AI_REQUEST_MODEL, + SERVER_PORT, + GEN_AI_RESPONSE_MODEL, + SERVER_ADDRESS)); + } + + private GenAiMetricsAdvice() {} +} diff --git a/instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/internal/AwsSdkInstrumenterFactory.java b/instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/internal/AwsSdkInstrumenterFactory.java index a61c17d519f1..b35a174b244a 100644 --- a/instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/internal/AwsSdkInstrumenterFactory.java +++ b/instrumentation/aws-sdk/aws-sdk-2.2/library/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/internal/AwsSdkInstrumenterFactory.java @@ -15,6 +15,7 @@ import io.opentelemetry.context.propagation.TextMapPropagator; import io.opentelemetry.instrumentation.api.incubator.semconv.db.DbClientMetrics; import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiAttributesExtractor; +import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiClientMetrics; import io.opentelemetry.instrumentation.api.incubator.semconv.genai.GenAiSpanNameExtractor; import io.opentelemetry.instrumentation.api.incubator.semconv.messaging.MessageOperation; import io.opentelemetry.instrumentation.api.incubator.semconv.messaging.MessagingAttributesExtractor; @@ -228,8 +229,10 @@ public Instrumenter bedrockRuntimeInstrumenter() SpanKindExtractor.alwaysClient(), attributesExtractors(), builder -> - builder.addAttributesExtractor( - GenAiAttributesExtractor.create(BedrockRuntimeAttributesGetter.INSTANCE)), + builder + .addAttributesExtractor( + GenAiAttributesExtractor.create(BedrockRuntimeAttributesGetter.INSTANCE)) + .addOperationMetrics(GenAiClientMetrics.get()), true); } diff --git a/instrumentation/aws-sdk/aws-sdk-2.2/testing/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AbstractAws2BedrockRuntimeTest.java b/instrumentation/aws-sdk/aws-sdk-2.2/testing/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AbstractAws2BedrockRuntimeTest.java index 862e3281d79c..4bca9b5e30df 100644 --- a/instrumentation/aws-sdk/aws-sdk-2.2/testing/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AbstractAws2BedrockRuntimeTest.java +++ b/instrumentation/aws-sdk/aws-sdk-2.2/testing/src/main/java/io/opentelemetry/instrumentation/awssdk/v2_2/AbstractAws2BedrockRuntimeTest.java @@ -15,6 +15,7 @@ import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_REQUEST_TOP_P; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_RESPONSE_FINISH_REASONS; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_SYSTEM; +import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_TOKEN_TYPE; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_USAGE_INPUT_TOKENS; import static io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.GEN_AI_USAGE_OUTPUT_TOKENS; import static java.util.Arrays.asList; @@ -42,6 +43,7 @@ import software.amazon.awssdk.services.bedrockruntime.model.Message; public abstract class AbstractAws2BedrockRuntimeTest { + private static final String INSTRUMENTATION_NAME = "io.opentelemetry.aws-sdk-2.2"; private static final String API_URL = "https://bedrock-runtime.us-east-1.amazonaws.com"; @@ -105,6 +107,75 @@ void testConverseBasic() { equalTo(GEN_AI_USAGE_INPUT_TOKENS, 8), equalTo(GEN_AI_USAGE_OUTPUT_TOKENS, 14), equalTo(GEN_AI_RESPONSE_FINISH_REASONS, asList("end_turn"))))); + + getTesting() + .waitAndAssertMetrics( + INSTRUMENTATION_NAME, + metric -> + metric + .hasName("gen_ai.client.token.usage") + .hasUnit("{token}") + .hasDescription("Measures number of input and output tokens used") + .hasHistogramSatisfying( + histogram -> + histogram.hasPointsSatisfying( + point -> + point + .hasSum(8) + .hasCount(1) + .hasAttributesSatisfyingExactly( + equalTo( + GEN_AI_SYSTEM, + GenAiIncubatingAttributes + .GenAiSystemIncubatingValues.AWS_BEDROCK), + equalTo( + GEN_AI_TOKEN_TYPE, + GenAiIncubatingAttributes + .GenAiTokenTypeIncubatingValues.INPUT), + equalTo( + GEN_AI_OPERATION_NAME, + GenAiIncubatingAttributes + .GenAiOperationNameIncubatingValues.CHAT), + equalTo(GEN_AI_REQUEST_MODEL, modelId)), + point -> + point + .hasSum(14) + .hasCount(1) + .hasAttributesSatisfyingExactly( + equalTo( + GEN_AI_SYSTEM, + GenAiIncubatingAttributes + .GenAiSystemIncubatingValues.AWS_BEDROCK), + equalTo( + GEN_AI_TOKEN_TYPE, + GenAiIncubatingAttributes + .GenAiTokenTypeIncubatingValues.COMPLETION), + equalTo( + GEN_AI_OPERATION_NAME, + GenAiIncubatingAttributes + .GenAiOperationNameIncubatingValues.CHAT), + equalTo(GEN_AI_REQUEST_MODEL, modelId)))), + metric -> + metric + .hasName("gen_ai.client.operation.duration") + .hasUnit("s") + .hasDescription("GenAI operation duration") + .hasHistogramSatisfying( + histogram -> + histogram.hasPointsSatisfying( + point -> + point + .hasSumGreaterThan(0.0) + .hasAttributesSatisfyingExactly( + equalTo( + GEN_AI_SYSTEM, + GenAiIncubatingAttributes + .GenAiSystemIncubatingValues.AWS_BEDROCK), + equalTo( + GEN_AI_OPERATION_NAME, + GenAiIncubatingAttributes + .GenAiOperationNameIncubatingValues.CHAT), + equalTo(GEN_AI_REQUEST_MODEL, modelId))))); } @Test @@ -161,5 +232,74 @@ void testConverseOptions() { equalTo(GEN_AI_USAGE_INPUT_TOKENS, 8), equalTo(GEN_AI_USAGE_OUTPUT_TOKENS, 10), equalTo(GEN_AI_RESPONSE_FINISH_REASONS, asList("max_tokens"))))); + + getTesting() + .waitAndAssertMetrics( + INSTRUMENTATION_NAME, + metric -> + metric + .hasName("gen_ai.client.token.usage") + .hasUnit("{token}") + .hasDescription("Measures number of input and output tokens used") + .hasHistogramSatisfying( + histogram -> + histogram.hasPointsSatisfying( + point -> + point + .hasSum(8) + .hasCount(1) + .hasAttributesSatisfyingExactly( + equalTo( + GEN_AI_SYSTEM, + GenAiIncubatingAttributes + .GenAiSystemIncubatingValues.AWS_BEDROCK), + equalTo( + GEN_AI_TOKEN_TYPE, + GenAiIncubatingAttributes + .GenAiTokenTypeIncubatingValues.INPUT), + equalTo( + GEN_AI_OPERATION_NAME, + GenAiIncubatingAttributes + .GenAiOperationNameIncubatingValues.CHAT), + equalTo(GEN_AI_REQUEST_MODEL, modelId)), + point -> + point + .hasSum(10) + .hasCount(1) + .hasAttributesSatisfyingExactly( + equalTo( + GEN_AI_SYSTEM, + GenAiIncubatingAttributes + .GenAiSystemIncubatingValues.AWS_BEDROCK), + equalTo( + GEN_AI_TOKEN_TYPE, + GenAiIncubatingAttributes + .GenAiTokenTypeIncubatingValues.COMPLETION), + equalTo( + GEN_AI_OPERATION_NAME, + GenAiIncubatingAttributes + .GenAiOperationNameIncubatingValues.CHAT), + equalTo(GEN_AI_REQUEST_MODEL, modelId)))), + metric -> + metric + .hasName("gen_ai.client.operation.duration") + .hasUnit("s") + .hasDescription("GenAI operation duration") + .hasHistogramSatisfying( + histogram -> + histogram.hasPointsSatisfying( + point -> + point + .hasSumGreaterThan(0.0) + .hasAttributesSatisfyingExactly( + equalTo( + GEN_AI_SYSTEM, + GenAiIncubatingAttributes + .GenAiSystemIncubatingValues.AWS_BEDROCK), + equalTo( + GEN_AI_OPERATION_NAME, + GenAiIncubatingAttributes + .GenAiOperationNameIncubatingValues.CHAT), + equalTo(GEN_AI_REQUEST_MODEL, modelId))))); } }