[.NET] Custom Metrics E2E Test

.github/workflows/dotnet-ec2-default-test.yml

@@ -38,7 +38,7 @@ env:
   E2E_TEST_ACCOUNT_ID: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ACCOUNT_ID }}
   E2E_TEST_ROLE_NAME: ${{ secrets.APPLICATION_SIGNALS_E2E_TEST_ROLE_NAME }}
   DOTNET_VERSION: ${{ inputs.dotnet-version }}
-  SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/dotnet-sample-app-${{ inputs.dotnet-version }}.zip
+  SAMPLE_APP_ZIP: s3://aws-appsignals-sample-app-prod-${{ inputs.aws-region }}/dotnet-sample-app-delete-me.zip
   METRIC_NAMESPACE: ApplicationSignals
   LOG_GROUP_NAME: /aws/application-signals/data
   ADOT_DISTRO_NAME: ${{ inputs.staging_distro_name }}
@@ -229,9 +229,26 @@ jobs:
           --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
           --rollup'
 
+      - name: Validate custom metrics
+        id: cwagent-metric-validation
+        if: (success() || steps.log-validation.outcome == 'failure' || steps.log-validation.outcome == 'failure') && !cancelled()
+        run: ./gradlew validator:run --args='-c dotnet/ec2/default/custom-metric-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
+          --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
+          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8081
+          --region ${{ env.E2E_TEST_AWS_REGION }}
+          --metric-namespace CWAgent
+          --log-group ${{ env.LOG_GROUP_NAME }}
+          --service-name dotnet-sample-application-${{ env.TESTING_ID }}
+          --remote-service-name dotnet-sample-remote-application-${{ env.TESTING_ID }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}
+          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
+          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
+          --rollup'
+
       - name: Validate generated traces
         id: trace-validation
-        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
+        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure' || steps.custom-metric-validation.outcome == 'failure') && !cancelled()
         run: ./gradlew validator:run --args='-c dotnet/ec2/default/trace-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
@@ -258,7 +275,7 @@ jobs:
         if: always()
         id: validation-result
         run: |
-          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
+          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.custom-metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
             echo "validation-result=success" >> $GITHUB_OUTPUT
           else
             echo "validation-result=failure" >> $GITHUB_OUTPUT

.github/workflows/test.yml

@@ -0,0 +1,23 @@
+## Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+## SPDX-License-Identifier: Apache-2.0
+
+# This is a reusable workflow for running the Enablement test for App Signals.
+# It is meant to be called from another workflow.
+# Read more about reusable workflows: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview
+name: Test
+on:
+  push:
+    branches:
+      - DOTNET_Custom_metrics
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  dotnet-ec2-default:
+    uses: ./.github/workflows/dotnet-ec2-default-test.yml
+    secrets: inherit
+    with:
+      caller-workflow-name: 'test'
+      aws-region: 'us-east-1'

sample-apps/dotnet/asp_frontend_service/Controllers/AppController.cs

@@ -9,6 +9,12 @@
 using Amazon.S3;
 using Microsoft.AspNetCore.Mvc;
 using Amazon.S3.Model;
+using System.Diagnostics.Metrics;
+using System.Collections.Generic;
+using OpenTelemetry;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Exporter;
 
 namespace asp_frontend_service.Controllers;
 
@@ -22,6 +28,57 @@ public class AppController : ControllerBase
     private static bool threadStarted = false;
     private readonly AmazonS3Client s3Client = new AmazonS3Client();
     private readonly HttpClient httpClient = new HttpClient();
+    private static readonly Meter meter = new Meter("myMeterSource");
+    private static readonly Counter<int> agentBasedCounter;
+    private static readonly Histogram<double> agentBasedHistogram;
+    private static readonly UpDownCounter<int> agentBasedGauge;
+
+    // Custom pipeline metrics - only create if specific env vars exist
+    private static readonly Meter pipelineMeter;
+    private static readonly Counter<int> customPipelineCounter;
+    private static readonly Histogram<double> customPipelineHistogram;
+    private static readonly UpDownCounter<int> customPipelineGauge;
+    private static readonly MeterProvider pipelineMeterProvider;
+
+    static AppController()
+    {
+
+        agentBasedCounter = meter.CreateCounter<int>("agent_based_counter");
+        agentBasedHistogram = meter.CreateHistogram<double>("agent_based_histogram");
+        agentBasedGauge = meter.CreateUpDownCounter<int>("agent_based_gauge");
+
+        var serviceName = Environment.GetEnvironmentVariable("SERVICE_NAME");
+        var deploymentEnv = Environment.GetEnvironmentVariable("DEPLOYMENT_ENVIRONMENT_NAME");
+
+        if (!string.IsNullOrEmpty(serviceName) && !string.IsNullOrEmpty(deploymentEnv))
+        {
+
+            pipelineMeterProvider = Sdk.CreateMeterProviderBuilder()
+                .SetResourceBuilder(ResourceBuilder.CreateEmpty().AddAttributes(new Dictionary<string, object>
+                {
+                    ["service.name"] = serviceName,
+                    ["deployment.environment.name"] = deploymentEnv
+                }))
+                .AddOtlpExporter(options =>
+                {
+                    options.Endpoint = new Uri("http://localhost:4318/v1/metrics");
+                    options.Protocol = OtlpExportProtocol.HttpProtobuf;
+                    options.ExportProcessorType = ExportProcessorType.Batch;
+                })
+                .AddReader(new PeriodicExportingMetricReader(new OtlpMetricExporter(new OtlpExporterOptions
+                {
+                    Endpoint = new Uri("http://localhost:4318/v1/metrics"),
+                    Protocol = OtlpExportProtocol.HttpProtobuf
+                }), 1000))
+                .AddMeter("myMeter")
+                .Build();
+
+            pipelineMeter = new Meter("myMeter");
+            customPipelineCounter = pipelineMeter.CreateCounter<int>("custom_pipeline_counter", "1", "pipeline export counter");
+            customPipelineHistogram = pipelineMeter.CreateHistogram<double>("custom_pipeline_histogram", "ms", "pipeline export histogram");
+            customPipelineGauge = pipelineMeter.CreateUpDownCounter<int>("custom_pipeline_gauge", "1", "pipeline export gauge");
+        }
+    }
 
     private static readonly Thread thread = new Thread(() =>
             {
@@ -50,7 +107,6 @@ public AppController()
     {
         if (!threadStarted)
         {
-            Console.WriteLine("Starting thread");
             threadStarted = true;
             thread.Start();
         }
@@ -69,7 +125,29 @@ public string OutgoingHttp()
     [Route("/aws-sdk-call")]
     public string AWSSDKCall([FromQuery] string testingId)
     {
-       var request = new GetBucketLocationRequest()
+        var random = new Random();
+
+        // Agent-based metrics
+        agentBasedCounter.Add(1, new KeyValuePair<string, object?>("Operation", "counter"));
+        agentBasedHistogram.Record(random.NextDouble() * 100, new KeyValuePair<string, object?>("Operation", "histogram"));
+        agentBasedGauge.Add(random.Next(-10, 11), new KeyValuePair<string, object?>("Operation", "gauge"));
+
+        // Pipeline metrics
+        if (customPipelineCounter != null)
+        {
+            customPipelineCounter.Add(1, new KeyValuePair<string, object?>("Operation", "pipeline_counter"));
+            customPipelineHistogram?.Record(random.Next(100, 1001), new KeyValuePair<string, object?>("Operation", "pipeline_histogram"));
+            customPipelineGauge?.Add(random.Next(-10, 11), new KeyValuePair<string, object?>("Operation", "pipeline_gauge"));
+        }
+
+
+        var bucketName = "e2e-test-bucket-name";
+        if (!string.IsNullOrEmpty(testingId))
+        {
+            bucketName += "-" + testingId;
+        }
+
+        var request = new GetBucketLocationRequest()
             {
                BucketName = testingId
             };

sample-apps/dotnet/asp_frontend_service/asp_frontend_service.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <TargetFramework>netcoreapp8.0</TargetFramework>
@@ -8,11 +8,15 @@
   <ItemGroup>
     <PackageReference Include="AWSSDK.S3" Version="3.7.310.7" />
     <PackageReference Include="AWSSDK.SecurityToken" Version="3.7.300.120" />
+    <PackageReference Include="AWS.Distro.OpenTelemetry.AutoInstrumentation" Version="1.9.1" />
+    <PackageReference Include="OpenTelemetry" Version="1.14.0" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.14.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.14.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <AdditionalFiles Include="./stylecop.json" Link="stylecop.json" />
   </ItemGroup>
 
-</Project>
+</Project>

terraform/dotnet/ec2/default/amazon-cloudwatch-agent.json

@@ -10,7 +10,11 @@
   },
   "logs": {
     "metrics_collected": {
-      "application_signals": {}
+      "application_signals": {},
+      "otlp": {
+        "grpc_endpoint": "0.0.0.0:4317",
+        "http_endpoint": "0.0.0.0:4318"
+      }
     }
   }
 }

terraform/dotnet/ec2/default/main.tf

@@ -119,6 +119,9 @@ resource "null_resource" "main_service_setup" {
       sudo dnf install -y dotnet-sdk-${var.language_version}
       sudo yum install unzip -y
 
+    # enable ec2 instance connect for debug
+      sudo yum install ec2-instance-connect -y
+
       # Copy in CW Agent configuration
       agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
       echo $agent_config > amazon-cloudwatch-agent.json
@@ -132,8 +135,8 @@ resource "null_resource" "main_service_setup" {
       ${var.get_adot_distro_command}
 
       # Get and run the sample application with configuration
-      aws s3 cp ${var.sample_app_zip} ./dotnet-sample-app.zip
-      unzip -o dotnet-sample-app.zip
+      aws s3 cp ${var.sample_app_zip} ./dotnet-sample-app-delete-me.zip
+      unzip -o dotnet-sample-app-delete-me.zip
 
       # Get Absolute Path
       current_dir=$(pwd)
@@ -150,14 +153,22 @@ resource "null_resource" "main_service_setup" {
       export DOTNET_STARTUP_HOOKS=$current_dir/dotnet-distro/net/OpenTelemetry.AutoInstrumentation.StartupHook.dll
       export OTEL_DOTNET_AUTO_HOME=$current_dir/dotnet-distro
       export OTEL_DOTNET_AUTO_PLUGINS="AWS.Distro.OpenTelemetry.AutoInstrumentation.Plugin, AWS.Distro.OpenTelemetry.AutoInstrumentation"
-      export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
-      export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4316
-      export OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://127.0.0.1:4316/v1/metrics
-      export OTEL_METRICS_EXPORTER=none
-      export OTEL_RESOURCE_ATTRIBUTES=service.name=dotnet-sample-application-${var.test_id}
+      export OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4316/v1/metrics
+      export OTEL_METRICS_EXPORTER=otlp
+      export OTEL_TRACES_EXPORTER=otlp
+      export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4316
+      export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4316/v1/traces
+      export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf
+      export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4318/v1/metrics
+      export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=http/protobuf
+      export OTEL_DOTNET_AUTO_METRICS_ADDITIONAL_SOURCES=myMeterSource
+      export SERVICE_NAME="dotnet-sample-application-${var.test_id}"
+      export DEPLOYMENT_ENVIRONMENT_NAME="ec2:default"
+      export OTEL_RESOURCE_ATTRIBUTES="service.name=$${SERVICE_NAME},deployment.environment=$${DEPLOYMENT_ENVIRONMENT_NAME}"
       export OTEL_AWS_APPLICATION_SIGNALS_ENABLED=true
       export OTEL_AWS_APPLICATION_SIGNALS_RUNTIME_ENABLED=false
       export OTEL_TRACES_SAMPLER=always_on
+      export AWS_REGION="${var.aws_region}"
       export ASPNETCORE_URLS=http://0.0.0.0:8080
       nohup dotnet bin/Debug/netcoreapp${var.language_version}/asp_frontend_service.dll &
 
@@ -240,8 +251,8 @@ resource "null_resource" "remote_service_setup" {
       ${var.get_adot_distro_command}
 
       # Get and run the sample application with configuration
-      aws s3 cp ${var.sample_app_zip} ./dotnet-sample-app.zip
-      unzip -o dotnet-sample-app.zip
+      aws s3 cp ${var.sample_app_zip} ./dotnet-sample-app-delete-me.zip
+      unzip -o dotnet-sample-app-delete-me.zip
 
       # Get Absolute Path
       current_dir=$(pwd)

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -315,6 +315,9 @@ public enum PredefinedExpectedTemplate implements FileConfig {
   DOTNET_EC2_WINDOWS_DEFAULT_AWS_SDK_CALL_METRIC("/expected-data-template/dotnet/ec2/windows/aws-sdk-call-metric.mustache"),
   DOTNET_EC2_WINDOWS_DEFAULT_AWS_SDK_CALL_TRACE("/expected-data-template/dotnet/ec2/windows/aws-sdk-call-trace.mustache"),
 
+    /** DOTNET EC2 Default Custom Metrics Test Case Validations */
+  DOTNET_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRIC("/expected-data-template/dotnet/ec2/default/aws-otel-custom-metrics.mustache"),
+
   DOTNET_EC2_WINDOWS_DEFAULT_REMOTE_SERVICE_LOG("/expected-data-template/dotnet/ec2/windows/remote-service-log.mustache"),
   DOTNET_EC2_WINDOWS_DEFAULT_REMOTE_SERVICE_METRIC("/expected-data-template/dotnet/ec2/windows/remote-service-metric.mustache"),
 //  Because of a time sync issue, block the remote service trace check for now