diff --git a/docs/preview/03-Features/01-Azure/01-service-bus.mdx b/docs/preview/03-Features/01-Azure/01-service-bus.mdx index 1506d1c0..8c2baa1b 100644 --- a/docs/preview/03-Features/01-Azure/01-service-bus.mdx +++ b/docs/preview/03-Features/01-Azure/01-service-bus.mdx @@ -184,6 +184,36 @@ Arcus Messaging makes it possible to make it visible in a logging system like Az Below, you will find the different options that are supported to enable Service Bus request tracking. When this is enabled, Arcus.Messaging will log a request operation for every message that is received from Service Bus and all traces and interactions to dependent systems that happen during the processing of that message, will be logged as children of this request operation. + + +```powershell +PS> Install-Package -Name Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry +``` + +Make sure to include the following line to your message pump registration: +```diff ++ ActivitySource applicationSource = new(""); + +services.AddServiceBusTopicMessagePump(...) ++ .UseServiceBusOpenTelemetryRequestTracking(applicationSource) + .WithServiceBusMessageHandler<...>() + .WithServiceBusMessageHandler<...>(); +``` + +Now Arcus will use the OpenTelemetry approach to track Azure Service Bus messages. Make sure that the `ActivitySource` that is passed, is also tracked by OpenTelemetry: +```csharp +IServiceCollection services = ... + +services.AddOpenTelemetry() + .AddTraces(traces => + { + traces.AddSource(""); + }); +``` + +> 🔗 [More info on OpenTelemetry on Azure](https://learn.microsoft.com/en-us/azure/azure-monitor/app/opentelemetry) + + ```powershell @@ -338,7 +368,9 @@ Both the recovery period after the circuit is open and the interval between mess } ``` -#### 🔔 Get notified on a circuit breaker state transition +
+**🔔 Get notified on a circuit breaker state transition** + To get notified on circuit-breaker state transitions, you can register one or more event handlers on a message pump. These event handlers should implement the `ICircuitBreakerEventHandler` interface: diff --git a/src/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry.csproj b/src/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry.csproj new file mode 100644 index 00000000..7ebb69c5 --- /dev/null +++ b/src/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry.csproj @@ -0,0 +1,33 @@ + + + + net8.0 + Arcus + Arcus + Arcus.Messaging + Provides capability to track message correlation information using OpenTelemetry for Azure Service Bus message pumps + Copyright (c) Arcus + https://messaging.arcus-azure.net/ + https://github.com/arcus-azure/arcus.messaging + LICENSE + icon.png + README.md + Git + Azure;Messaging;ServiceBus + true + true + true + S1133 + + + + + + + + + + + + + diff --git a/src/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry/Extensions/ServiceBusMessageHandlerCollectionExtensions.cs b/src/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry/Extensions/ServiceBusMessageHandlerCollectionExtensions.cs new file mode 100644 index 00000000..5f76f005 --- /dev/null +++ b/src/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry/Extensions/ServiceBusMessageHandlerCollectionExtensions.cs @@ -0,0 +1,39 @@ +using System; +using System.Diagnostics; +using Arcus.Messaging.Abstractions.ServiceBus.MessageHandling; +using Arcus.Messaging.Abstractions.ServiceBus.Telemetry; +using Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; + +// ReSharper disable once CheckNamespace +namespace Microsoft.Extensions.DependencyInjection +{ + /// + /// Extensions on the to register OpenTelemetry services for Azure Service Bus message pumps. + /// + public static class ServiceBusMessageHandlerCollectionExtensions + { + /// + /// Register OpenTelemetry as the correlation system to track Azure Service Bus message requests within the message pump. + /// + /// The collection of Azure Service Bus message handler collection. + /// The activity source to start instances from upon receiving Azure Service Bus messages. + /// Thrown when the or the is null. + public static ServiceBusMessageHandlerCollection UseServiceBusOpenTelemetryRequestTracking( + this ServiceBusMessageHandlerCollection handlers, + ActivitySource activitySource) + { + ArgumentNullException.ThrowIfNull(handlers); + ArgumentNullException.ThrowIfNull(activitySource); + + handlers.Services.TryAddSingleton(serviceProvider => + { + var logger = serviceProvider.GetService>(); + return new OpenTelemetryServiceBusMessageCorrelationScope(activitySource, logger); + }); + + return handlers; + } + } +} diff --git a/src/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry/OpenTelemetryServiceBusMessageCorrelationScope.cs b/src/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry/OpenTelemetryServiceBusMessageCorrelationScope.cs new file mode 100644 index 00000000..d7e3d460 --- /dev/null +++ b/src/Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry/OpenTelemetryServiceBusMessageCorrelationScope.cs @@ -0,0 +1,109 @@ +using System; +using System.Diagnostics; +using Arcus.Messaging.Abstractions.MessageHandling; +using Arcus.Messaging.Abstractions.ServiceBus.Telemetry; +using Arcus.Messaging.Abstractions.Telemetry; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry +{ + /// + /// Represents the OpenTelemetry implementation of the + /// to track the correlation information of a received Azure Service Bus message within a message pump. + /// + internal class OpenTelemetryServiceBusMessageCorrelationScope : IServiceBusMessageCorrelationScope + { + private readonly ActivitySource _activitySource; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + internal OpenTelemetryServiceBusMessageCorrelationScope(ActivitySource activitySource, ILogger logger) + { + ArgumentNullException.ThrowIfNull(activitySource); + _activitySource = activitySource; + _logger = logger ?? NullLogger.Instance; + } + + /// + /// Starts a new Azure Service bus request operation on the telemetry system. + /// + /// The message context for the currently received Azure Service bus message. + /// The user-configurable options to manipulate the telemetry. + public MessageOperationResult StartOperation(ServiceBusMessageContext messageContext, MessageTelemetryOptions options) + { + ArgumentNullException.ThrowIfNull(messageContext); + ArgumentNullException.ThrowIfNull(options); + + _logger.LogTrace("Start Azure Service Bus request '{OperationName}' operation...", options.OperationName); + (string transactionId, string operationParentId) = messageContext.Properties.GetTraceParent(); + + ActivityContext context = new( + ActivityTraceId.CreateFromString(transactionId), + ActivitySpanId.CreateFromString(operationParentId), + ActivityTraceFlags.None); + + Activity activity = _activitySource.CreateActivity( + name: options.OperationName, + kind: ActivityKind.Consumer, + context); + + activity?.Start(); + if (activity is null) + { + return new UnlinkedMessageOperationResult(transactionId, operationParentId); + } + + activity.SetTag("az.namespace", "Microsoft.ServiceBus"); + activity.SetTag("messaging.system", "servicebus"); + activity.SetTag("messaging.operation.type", "receive"); + activity.SetTag("messaging.destination.name", messageContext.EntityPath); + activity.SetTag("messaging.message.id", messageContext.MessageId); + activity.SetTag("network.protocol.name", "amqp"); + + activity.SetTag("ServiceBus-Endpoint", messageContext.FullyQualifiedNamespace); + activity.SetTag("ServiceBus-Entity", messageContext.EntityPath); + activity.SetTag("ServiceBus-EntityType", messageContext.EntityType.ToString()); + + return new OpenTelemetryMessageOperationResult(activity, _logger); + } + + private sealed class OpenTelemetryMessageOperationResult : MessageOperationResult + { + private readonly Activity _activity; + private readonly ILogger _logger; + + internal OpenTelemetryMessageOperationResult(Activity activity, ILogger logger) + : base(new MessageCorrelationInfo(activity.TraceId.ToString(), activity.SpanId.ToString(), activity.ParentSpanId.ToString())) + { + _activity = activity; + _logger = logger; + } + + protected override void StopOperation(bool isSuccessful, DateTimeOffset startTime, TimeSpan duration) + { + _logger.LogTrace("Stop Azure Service Bus request '{OperationName}' operation (isSuccessful={IsSuccessful})", _activity.OperationName, isSuccessful); + + _activity.SetStatus(isSuccessful ? ActivityStatusCode.Ok : ActivityStatusCode.Error); + _activity.SetTag("messaging.operation.name", isSuccessful ? "ack" : "nack"); + + _activity.SetEndTime(_activity.StartTimeUtc.Add(duration)); + _activity.Dispose(); + } + } + + private sealed class UnlinkedMessageOperationResult : MessageOperationResult + { + internal UnlinkedMessageOperationResult(string transactionId, string operationParentId) + : base(new MessageCorrelationInfo(Guid.NewGuid().ToString(), transactionId, operationParentId)) + { + } + + protected override void StopOperation(bool isSuccessful, DateTimeOffset startTime, TimeSpan duration) + { + } + } + } +} diff --git a/src/Arcus.Messaging.Tests.Core/ServiceBus/MessageHandlers/OrderWithAutoTrackingAzureServiceBusMessageHandler.cs b/src/Arcus.Messaging.Tests.Core/ServiceBus/MessageHandlers/OrderWithAutoTrackingAzureServiceBusMessageHandler.cs index 97d4ef60..1214ea9d 100644 --- a/src/Arcus.Messaging.Tests.Core/ServiceBus/MessageHandlers/OrderWithAutoTrackingAzureServiceBusMessageHandler.cs +++ b/src/Arcus.Messaging.Tests.Core/ServiceBus/MessageHandlers/OrderWithAutoTrackingAzureServiceBusMessageHandler.cs @@ -1,5 +1,5 @@ using System; -using System.Data.SqlClient; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Arcus.Messaging.Abstractions; @@ -13,17 +13,29 @@ namespace Arcus.Messaging.Tests.Workers.ServiceBus.MessageHandlers { public class OrderWithAutoTrackingAzureServiceBusMessageHandler : IAzureServiceBusMessageHandler { + private readonly bool _isSuccessful; private readonly ILogger _logger; + private static readonly HttpClient HttpClient = new(); + /// /// Initializes a new instance of the class. /// public OrderWithAutoTrackingAzureServiceBusMessageHandler(ILogger logger) + : this(isSuccessful: true, logger) + { + } + + /// + /// Initializes a new instance of the class. + /// + public OrderWithAutoTrackingAzureServiceBusMessageHandler(bool isSuccessful, ILogger logger) { _logger = logger; + _isSuccessful = isSuccessful; } - public Task ProcessMessageAsync( + public async Task ProcessMessageAsync( Order message, AzureServiceBusMessageContext messageContext, MessageCorrelationInfo correlationInfo, @@ -31,8 +43,13 @@ public Task ProcessMessageAsync( { _logger.LogAzureKeyVaultDependency("https://my-vault.azure.net", "Sql-connection-string", isSuccessful: true, DateTimeOffset.UtcNow, TimeSpan.FromSeconds(5)); SimulateSqlQueryWithMicrosoftTracking(); + await SimulateHttpClientWithMicrosoftTrackingAsync(); - return Task.CompletedTask; + if (!_isSuccessful) + { + throw new InvalidOperationException( + "[Test] Sabotage this message processing to let the message correlation system pick up an 'unsuccessful request'"); + } } private static void SimulateSqlQueryWithMicrosoftTracking() @@ -58,5 +75,10 @@ private static void SimulateSqlQueryWithMicrosoftTracking() // A failure will still result in a dependency telemetry instance that we can assert on. } } + + private static async Task SimulateHttpClientWithMicrosoftTrackingAsync() + { + string _ = await HttpClient.GetStringAsync("https://codit.eu"); + } } } diff --git a/src/Arcus.Messaging.Tests.Integration/Arcus.Messaging.Tests.Integration.csproj b/src/Arcus.Messaging.Tests.Integration/Arcus.Messaging.Tests.Integration.csproj index 284c3dfe..546894f2 100644 --- a/src/Arcus.Messaging.Tests.Integration/Arcus.Messaging.Tests.Integration.csproj +++ b/src/Arcus.Messaging.Tests.Integration/Arcus.Messaging.Tests.Integration.csproj @@ -11,11 +11,16 @@ + - + + + + + - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -24,6 +29,7 @@ + diff --git a/src/Arcus.Messaging.Tests.Integration/MessagePump/ServiceBusMessagePump.TelemetryTests.cs b/src/Arcus.Messaging.Tests.Integration/MessagePump/ServiceBusMessagePump.TelemetryTests.cs index 8e5ab41b..5ea2af5f 100644 --- a/src/Arcus.Messaging.Tests.Integration/MessagePump/ServiceBusMessagePump.TelemetryTests.cs +++ b/src/Arcus.Messaging.Tests.Integration/MessagePump/ServiceBusMessagePump.TelemetryTests.cs @@ -1,20 +1,27 @@ using System; using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using Arcus.Messaging.Tests.Core.Messages.v1; using Arcus.Messaging.Tests.Integration.Fixture; using Arcus.Messaging.Tests.Integration.Fixture.Logging; using Arcus.Messaging.Tests.Workers.ServiceBus.MessageHandlers; using Arcus.Testing; +using Azure.Messaging.ServiceBus; using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.DataContracts; using Microsoft.ApplicationInsights.Extensibility; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Hosting.Internal; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Trace; using Serilog; using Serilog.Sinks.ApplicationInsights.TelemetryConverters; using Xunit; +using Xunit.Sdk; using static Arcus.Observability.Telemetry.Core.ContextProperties.RequestTracking.ServiceBus; using static Microsoft.Extensions.Logging.ServiceBusEntityType; @@ -22,6 +29,154 @@ namespace Arcus.Messaging.Tests.Integration.MessagePump { public partial class ServiceBusMessagePumpTests { + private const string DefaultHttpOperationName = "System.Net.Http.HttpRequestOut"; + + private string CustomOperationName { get; } = $"operation-{Guid.NewGuid()}"; + private bool IsSuccessful { get; } = Bogus.Random.Bool(); + + [Fact] + public async Task ServiceBusMessagePump_WithW3CCorrelationFormatNewParentViaOpenTelemetry_AutomaticallyTracksMicrosoftDependencies() + { + // Arrange + using ActivitySource source = CreateActivitySource(); + await using var serviceBus = GivenServiceBus(); + + serviceBus.WhenServiceBusQueueMessagePump(pump => + { + pump.Telemetry.OperationName = CustomOperationName; + }).UseServiceBusOpenTelemetryRequestTracking(source) + .WithServiceBusMessageHandler(CreateAutoTrackingMessageHandler); + + var activities = new Collection(); + serviceBus.Services.AddOpenTelemetry() + .WithTracing(traces => + { + traces.AddSource(source.Name); + traces.AddInMemoryExporter(activities); + traces.AddHttpClientInstrumentation(); + traces.SetSampler(new AlwaysOnSampler()); + }); + + // Act + await serviceBus.WhenProducingMessagesAsync(msg => msg.WithoutTraceParent()); + + // Assert + Activity serviceBusRequest = await GetQueueRequestActivityAsync(activities, CustomOperationName); + Activity httpDependency = await GetDependencyActivityAsync(activities, DefaultHttpOperationName, a => a.ParentId == serviceBusRequest.Id); + + Assert.Equal(serviceBusRequest, httpDependency.Parent); + } + + [Fact] + public async Task ServiceBusMessagePump_WithW3CCorrelationFormatViaOpenTelemetry_AutomaticallyTracksMicrosoftDependencies() + { + // Arrange + using ActivitySource source = CreateActivitySource(); + + await using var serviceBus = GivenServiceBus(); + + serviceBus.WhenServiceBusQueueMessagePump(pump => + { + pump.Telemetry.OperationName = CustomOperationName; + + }).WithServiceBusMessageHandler(CreateAutoTrackingMessageHandler) + .UseServiceBusOpenTelemetryRequestTracking(source); + + var activities = new Collection(); + serviceBus.Services.AddOpenTelemetry() + .WithTracing(traces => + { + traces.AddSource(source.Name); + traces.AddInMemoryExporter(activities); + traces.AddHttpClientInstrumentation(); + traces.SetSampler(new AlwaysOnSampler()); + }); + + // Act + ServiceBusMessage message = await serviceBus.WhenProducingMessageAsync(); + + // Assert + (string transactionId, string operationParentId) = message.ApplicationProperties.GetTraceParent(); + + Activity serviceBusRequest = await GetQueueRequestActivityAsync(activities, CustomOperationName, a => a.TraceId.ToString() == transactionId && a.ParentSpanId.ToString() == operationParentId); + Activity httpDependency = await GetDependencyActivityAsync(activities, DefaultHttpOperationName, a => a.TraceId.ToString() == transactionId && a.ParentId == serviceBusRequest.Id); + + Assert.Equal(serviceBusRequest, httpDependency.Parent); + } + + private OrderWithAutoTrackingAzureServiceBusMessageHandler CreateAutoTrackingMessageHandler(IServiceProvider provider) + { + return new OrderWithAutoTrackingAzureServiceBusMessageHandler( + IsSuccessful, + provider.GetRequiredService>()); + } + + private static ActivitySource CreateActivitySource() + { + return new ActivitySource("Arcus.Messaging.Tests.Integration"); + } + + private async Task GetQueueRequestActivityAsync(IReadOnlyCollection activities, string operationName, Func filter = null) + { + return await Poll.Target(() => + { + var requestDependencies = activities.Where(a => a.OperationName == operationName).ToArray(); + Assert.True(requestDependencies.Length > 0, + $"no request activities found with operation name '{operationName}' in" + + $"[{string.Join(", ", activities.Select(a => a.OperationName))}]"); + + return AssertAny(requestDependencies, request => + { + Assert.True(IsSuccessful == request.Status is ActivityStatusCode.Ok, $"request for operation '{operationName}' did not match the expected status, expected '{(IsSuccessful ? ActivityStatusCode.Ok : ActivityStatusCode.Error)}' but got '{request.Status}'"); + Assert.Contains(request.Tags, tag => tag is { Key: "ServiceBus-EntityType", Value: "Queue" }); + Assert.True(filter is null || filter(request), $"request for operation '{operationName}' did not match the given custom filter assertion, please check whether the OpenTelemetry correlation system did add all the necessary properties"); + }); + + }).FailWith("cannot find request telemetry in spied-upon OpenTelemetry activities"); + } + + private static async Task GetDependencyActivityAsync(IReadOnlyCollection activities, string operationName, Func filter = null) + { + return await Poll.Target(() => + { + var dependencyActivities = activities.Where(a => a.OperationName == operationName).ToArray(); + Assert.True(dependencyActivities.Length > 0, + $"no dependency activities found with operation name '{operationName}' in " + + $"[{string.Join(", ", activities.Select(a => a.OperationName))}]"); + + return AssertAny(dependencyActivities, dependency => + { + Assert.True(filter is null || filter(dependency), $"dependency for operation '{operationName}' did not match the given custom filter assertion, please check whether the OpenTelemetry correlation system did add all the necessary properties"); + + }); + + }).FailWith("cannot find dependency telemetry in spied-upon OpenTelemetry activities"); + } + + public static T AssertAny(IEnumerable collection, Action action) + { + Stack<(int index, object item, Exception exception)> failures = new(); + T[] array = collection.ToArray(); + + for (int index = 0; index < array.Length; ++index) + { + T item = array[index]; + try + { + action(item); + return item; + } + catch (Exception ex) + { + failures.Push((index, item, ex)); + } + } + + throw new XunitException( + $"None of the {array.Length} item(s) matches against the given action: {Environment.NewLine}" + + $"{string.Join(Environment.NewLine, failures.Select(f => $"- [{f.index}] {f.item}: {f.exception}"))}"); + } + [Fact] public async Task ServiceBusMessagePump_WithW3CCorrelationFormatUsingSerilog_AutomaticallyTracksMicrosoftDependencies() { diff --git a/src/Arcus.Messaging.sln b/src/Arcus.Messaging.sln index 4a602aa1..730d5c90 100644 --- a/src/Arcus.Messaging.sln +++ b/src/Arcus.Messaging.sln @@ -26,6 +26,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Arcus.Messaging.ServiceBus.Telemetry.Serilog", "Arcus.Messaging.ServiceBus.Telemetry.Serilog\Arcus.Messaging.ServiceBus.Telemetry.Serilog.csproj", "{6F6E82EA-9A4A-4AB6-B48F-71C1B45EA3F5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry", "Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry\Arcus.Messaging.ServiceBus.Telemetry.OpenTelemetry.csproj", "{4C2DCD91-2D47-493C-82FC-03A76CFA2CB1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -60,6 +62,14 @@ Global {6F6E82EA-9A4A-4AB6-B48F-71C1B45EA3F5}.Debug|Any CPU.Build.0 = Debug|Any CPU {6F6E82EA-9A4A-4AB6-B48F-71C1B45EA3F5}.Release|Any CPU.ActiveCfg = Release|Any CPU {6F6E82EA-9A4A-4AB6-B48F-71C1B45EA3F5}.Release|Any CPU.Build.0 = Release|Any CPU + {864C12DF-DE3D-421F-8687-EC3918FFB8BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {864C12DF-DE3D-421F-8687-EC3918FFB8BE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {864C12DF-DE3D-421F-8687-EC3918FFB8BE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {864C12DF-DE3D-421F-8687-EC3918FFB8BE}.Release|Any CPU.Build.0 = Release|Any CPU + {4C2DCD91-2D47-493C-82FC-03A76CFA2CB1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4C2DCD91-2D47-493C-82FC-03A76CFA2CB1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4C2DCD91-2D47-493C-82FC-03A76CFA2CB1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4C2DCD91-2D47-493C-82FC-03A76CFA2CB1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -70,6 +80,8 @@ Global {9EED9AD7-B69D-45D5-870F-D4F63A1C3495} = {A1369CCD-42D1-43F6-98BC-D8EDA62C2B13} {55DE6D12-4C54-4570-BDFA-00B0FFDE5AB6} = {A1369CCD-42D1-43F6-98BC-D8EDA62C2B13} {6F6E82EA-9A4A-4AB6-B48F-71C1B45EA3F5} = {2CD090E7-7306-49A0-9680-6ED78CFECAE1} + {864C12DF-DE3D-421F-8687-EC3918FFB8BE} = {2CD090E7-7306-49A0-9680-6ED78CFECAE1} + {4C2DCD91-2D47-493C-82FC-03A76CFA2CB1} = {2CD090E7-7306-49A0-9680-6ED78CFECAE1} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {066FD85A-3DDE-4615-B550-BF67ACCDAA51}