use Aspire + blob storage to run multiple local replicas

Directory.Packages.props

@@ -5,21 +5,23 @@
     <AkkaHostingVersion>1.5.40</AkkaHostingVersion>
     <AkkaManagementVersion>1.5.37</AkkaManagementVersion>
     <PetabridgeCmdVersion>1.4.4</PetabridgeCmdVersion>
-    <AspireVersion>9.2.1</AspireVersion>
+    <AspireVersion>9.3.0</AspireVersion>
     <PlaywrightVersion>1.52.0</PlaywrightVersion>
   </PropertyGroup>
   <!-- Akka.NET Package Versions -->
   <ItemGroup>
     <PackageVersion Include="Akka" Version="$(AkkaVersion)" />
     <PackageVersion Include="Akka.Cluster.Hosting" Version="$(AkkaHostingVersion)" />
+    <PackageVersion Include="Akka.Discovery.Azure" Version="$(AkkaManagementVersion)" />
     <PackageVersion Include="Akka.Discovery.KubernetesApi" Version="$(AkkaManagementVersion)" />
     <PackageVersion Include="Akka.Hosting" Version="$(AkkaHostingVersion)" />
     <PackageVersion Include="Akka.Management" Version="$(AkkaManagementVersion)" />
     <PackageVersion Include="Akka.Persistence.Sql.Hosting" Version="1.5.40.1" />
     <PackageVersion Include="Akka.Streams" Version="$(AkkaVersion)" />
     <PackageVersion Include="Akka.Streams.TestKit" Version="$(AkkaVersion)" />
-    <PackageVersion Include="Aspire.Hosting.Docker" Version="9.2.1-preview.1.25222.1" />
-    <PackageVersion Include="Aspire.Hosting.Kubernetes" Version="9.2.1-preview.1.25222.1" />
+    <PackageVersion Include="Aspire.Hosting.Azure.Storage" Version="$(AspireVersion)" />
+    <PackageVersion Include="Aspire.Hosting.Docker" Version="9.3.0-preview.1.25265.20" />
+    <PackageVersion Include="Aspire.Hosting.Kubernetes" Version="9.3.0-preview.1.25265.20" />
     <PackageVersion Include="Grpc.Tools" Version="2.72.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/DrawTogether.AppHost/AkkaManagementExtensions.cs

@@ -0,0 +1,76 @@
+using System.Net.Sockets;
+using Aspire.Hosting.Azure;
+using Microsoft.Extensions.Hosting;
+
+namespace DrawTogether.AppHost;
+
+public static class AkkaManagementExtensions
+{
+    public static IResourceBuilder<ProjectResource> ConfigureAkkaManagementForApp(this IResourceBuilder<ProjectResource> appBuilder, DrawTogetherConfiguration config)
+    {
+        if (!config.UseAkkaManagement) return appBuilder;
+
+        var builder = appBuilder.ApplicationBuilder;
+
+        var azureStorage = builder.AddAzureStorage("storage")
+            .RunAsEmulator();
+
+        var tableStorage = azureStorage.AddTables("akka-discovery");
+
+        appBuilder.WaitFor(tableStorage)
+            .WithReference(tableStorage, "AkkaManagementAzure");
+
+        // Setup network endpoint ports
+        appBuilder
+            .WithEndpoint(name: "remote", protocol: ProtocolType.Tcp, env: "AkkaSettings__RemoteOptions__Port")
+            .WithEndpoint(name: "management", protocol: ProtocolType.Tcp, env: "AkkaSettings__AkkaManagementOptions__Port")
+            .WithEndpoint(name: "pbm", protocol: ProtocolType.Tcp, env: "AkkaSettings__PbmOptions__Port");
+
+        // need to populate some config for the hosts
+        appBuilder
+            .WithEnvironment("AkkaSettings__RemoteOptions__PublicHostName", "localhost")
+            .WithEnvironment("AkkaSettings__AkkaManagementOptions__Enabled", "true")
+            .WithEnvironment("AkkaSettings__AkkaManagementOptions__DiscoveryMethod", "AzureTableStorage")
+            .WithEnvironment("AkkaSettings__AkkaManagementOptions__FilterOnFallbackPort", "false");
+
+        return appBuilder;
+    }
+
+    private static IResourceBuilder<AzureTableStorageResource>? _tableStorage;
+    private static IResourceBuilder<AzureTableStorageResource> GetTableStorage(IDistributedApplicationBuilder builder)
+    {
+        if (_tableStorage != null) return _tableStorage;
+        var azureStorage = builder.AddAzureStorage("storage");
+        if(builder.Environment.IsDevelopment())
+            azureStorage.RunAsEmulator();
+
+        _tableStorage = azureStorage.AddTables("akka-discovery");
+        return _tableStorage;
+    }
+
+    public static IResourceBuilder<ContainerResource> ConfigureAkkaManagementForApp(this IResourceBuilder<ContainerResource> appBuilder, DrawTogetherConfiguration config)
+    {
+        if (!config.UseAkkaManagement) return appBuilder;
+
+        var tableStorage = GetTableStorage(appBuilder.ApplicationBuilder);
+
+        appBuilder.WaitFor(tableStorage)
+            .WithReference(tableStorage, "AkkaManagementAzure");
+
+        // Setup network endpoint ports
+        appBuilder
+            .WithHttpEndpoint(targetPort: 8080)
+            .WithEndpoint(name: "remote", protocol: ProtocolType.Tcp, targetPort: 14884, env: "AkkaSettings__RemoteOptions__Port")
+            .WithEndpoint(name: "management", protocol: ProtocolType.Tcp, targetPort: 8558, env: "AkkaSettings__AkkaManagementOptions__Port")
+            .WithEndpoint(name: "pbm", protocol: ProtocolType.Tcp, targetPort: 9110, env: "AkkaSettings__PbmOptions__Port");
+
+        // need to populate some config for the hosts
+        appBuilder
+            .WithEnvironment("AkkaSettings__AkkaManagementOptions__Enabled", "true")
+            .WithEnvironment("AkkaSettings__AkkaManagementOptions__DiscoveryMethod", "AzureTableStorage")
+            .WithEnvironment("AkkaSettings__AkkaManagementOptions__RequiredContactPointsNr", "1")
+            .WithEnvironment("AkkaSettings__AkkaManagementOptions__FilterOnFallbackPort", "false");
+
+        return appBuilder;
+    }
+}

src/DrawTogether.AppHost/Configuration.cs

@@ -9,4 +9,20 @@ public class DrawTogetherConfiguration
     /// Defaults to false so we don't accidentally persist data during testing.
     /// </remarks>
     public bool UseVolumes { get; set; } = false;
+
+    /// <summary>
+    /// When this is enabled, we will use Akka Management to manage our cluster.
+    ///
+    /// This means exposing an additional internal endpoint and spinning up Azure storage.
+    /// </summary>
+    public bool UseAkkaManagement { get; set; } = true;
+
+    /// <summary>
+    /// The total number of replicas we're going to run in our cluster.
+    ///
+    /// Only relevant when <see cref="UseAkkaManagement"/> is enabled.
+    /// </summary>
+    public int Replicas { get; set; } = 1;
+
+    public DeployEnvironment DeployEnvironment { get; set; } = DeployEnvironment.Local;
 }

src/DrawTogether.AppHost/DeployEnvironment.cs

@@ -0,0 +1,7 @@
+namespace DrawTogether.AppHost;
+
+public enum DeployEnvironment
+{
+    Local,
+    Docker
+}

src/DrawTogether.AppHost/DrawTogether.AppHost.csproj

@@ -9,6 +9,7 @@
 
   <ItemGroup>
     <PackageReference Include="Aspire.Hosting.AppHost" />
+    <PackageReference Include="Aspire.Hosting.Azure.Storage" />
     <PackageReference Include="Aspire.Hosting.Docker" />
     <PackageReference Include="Aspire.Hosting.Kubernetes" />
     <PackageReference Include="Aspire.Hosting.SqlServer" />

src/DrawTogether.AppHost/Program.cs

@@ -3,13 +3,9 @@
 
 var builder = DistributedApplication.CreateBuilder(args);
 
-
 var drawTogetherAspireConfig = builder.Configuration.GetSection("DrawTogether")
     .Get<DrawTogetherConfiguration>() ?? new DrawTogetherConfiguration();
 
-builder.AddDockerComposePublisher()
-    .AddKubernetesPublisher();
-
 // Adding a default password for ease of use - we can get rid of this but for a quick "git clone and run" it makes sense
 // have to add this when using data volumes otherwise Aspire will brick itself
 
@@ -32,21 +28,38 @@
     .WaitFor(db)
     .WithReference(db);
 
-var drawTogether = builder.AddProject<Projects.DrawTogether>("DrawTogether")
-    .WithReference(db, "DefaultConnection")
-    .WaitForCompletion(migrationService)
-    .WithEndpoint("pbm", annotation =>
+if (drawTogetherAspireConfig.DeployEnvironment == DeployEnvironment.Docker)
+{
+    var drawTogether = builder.AddDockerfile("DrawTogether-1", "../../", "./src/DrawTogether/DockerFile")
+        .WithImage("draw-together", "latest")
+        .WithReference(db, "DefaultConnection")
+        .ConfigureAkkaManagementForApp(drawTogetherAspireConfig)
+        .WaitForCompletion(migrationService);
+
+    foreach (var index in Enumerable.Range(2, drawTogetherAspireConfig.Replicas - 1))
     {
-        annotation.Port = 9110;
-
-        // not mean to be external, meant to be invoked via the pbm-sidecar
-        annotation.IsExternal = false;
-        annotation.IsProxied = false;
-    });
-
-// https://github.com/petabridge/pbm-sidecar - used to run `pbm` commands on the DrawTogether actor system
-var pbmSidecar = builder.AddContainer("pbm-sidecar", "petabridge/pbm:latest")
-    .WaitFor(drawTogether);
+        builder.AddContainer($"DrawTogether-{index}", "draw-together")
+            .WithReference(db, "DefaultConnection")
+            .ConfigureAkkaManagementForApp(drawTogetherAspireConfig)
+            .WaitFor(drawTogether);
+    }
+
+    // https://github.com/petabridge/pbm-sidecar - used to run `pbm` commands on the DrawTogether actor system
+    var pbmSidecar = builder.AddContainer("pbm-sidecar", "petabridge/pbm:latest")
+        .WaitFor(drawTogether);
+}
+else
+{
+    var drawTogether = builder.AddProject<Projects.DrawTogether>("DrawTogether")
+        .WithReplicas(drawTogetherAspireConfig.Replicas)
+        .WithReference(db, "DefaultConnection")
+        .WaitForCompletion(migrationService)
+        .ConfigureAkkaManagementForApp(drawTogetherAspireConfig);
+
+    // https://github.com/petabridge/pbm-sidecar - used to run `pbm` commands on the DrawTogether actor system
+    var pbmSidecar = builder.AddContainer("pbm-sidecar", "petabridge/pbm:latest")
+        .WaitFor(drawTogether);
+}
 
 builder
     .Build()

src/DrawTogether.AppHost/appsettings.Development.json

@@ -4,5 +4,8 @@
       "Default": "Information",
       "Microsoft.AspNetCore": "Warning"
     }
+  },
+  "DrawTogether": {
+    "DeployEnvironment": "Local"
   }
 }

src/DrawTogether.AppHost/appsettings.json

@@ -7,6 +7,9 @@
     }
   },
   "DrawTogether": {
-    "UseVolumes": true
+    "UseVolumes": true,
+    "UseAkkaManagement": true,
+    "Replicas": 3,
+    "DeployEnvironment": "Docker"
   }
 }

src/DrawTogether/Config/AkkaConfiguration.cs

@@ -1,4 +1,6 @@
-using Akka.Cluster.Hosting;
+using System.Diagnostics;
+using Akka.Cluster.Hosting;
+using Akka.Discovery.Azure;
 using Akka.Discovery.Config.Hosting;
 using Akka.Discovery.KubernetesApi;
 using Akka.Hosting;
@@ -62,6 +64,7 @@ public static AkkaConfigurationBuilder ConfigureNetwork(this AkkaConfigurationBu
         IServiceProvider serviceProvider)
     {
         var settings = serviceProvider.GetRequiredService<AkkaSettings>();
+        var configuration = serviceProvider.GetRequiredService<IConfiguration>();
 
         builder
             .WithRemoting(settings.RemoteOptions);
@@ -74,10 +77,22 @@ public static AkkaConfigurationBuilder ConfigureNetwork(this AkkaConfigurationBu
 
             builder
                 .WithClustering(clusterOptions)
-                .WithAkkaManagement(port: settings.AkkaManagementOptions.Port)
-                .WithClusterBootstrap(serviceName: settings.AkkaManagementOptions.ServiceName,
-                    portName: settings.AkkaManagementOptions.PortName,
-                    requiredContactPoints: settings.AkkaManagementOptions.RequiredContactPointsNr);
+                .WithAkkaManagement(setup =>
+                {
+                    setup.Http.HostName = settings.RemoteOptions.PublicHostName?.ToLower() ?? "localhost";
+                    setup.Http.Port = settings.AkkaManagementOptions.Port;
+                    setup.Http.BindHostName = "0.0.0.0";
+                    setup.Http.BindPort = settings.AkkaManagementOptions.Port;
+                })
+                .WithClusterBootstrap(options =>
+                {
+                    options.ContactPointDiscovery.ServiceName = settings.AkkaManagementOptions.ServiceName;
+                    options.ContactPointDiscovery.PortName = settings.AkkaManagementOptions.PortName;
+                    options.ContactPointDiscovery.RequiredContactPointsNr = settings.AkkaManagementOptions.RequiredContactPointsNr;
+                    options.ContactPointDiscovery.ContactWithAllContactPoints = true;
+
+                    options.ContactPoint.FilterOnFallbackPort = settings.AkkaManagementOptions.FilterOnFallbackPort;
+                }, autoStart: true);
 
             switch (settings.AkkaManagementOptions.DiscoveryMethod)
             {
@@ -90,16 +105,19 @@ public static AkkaConfigurationBuilder ConfigureNetwork(this AkkaConfigurationBu
                     break;
                 case DiscoveryMethod.AzureTableStorage:
                 {
-                    // var connectionStringName = configuration.GetSection("AzureStorageSettings")
-                    //     .Get<AzureStorageSettings>()?.ConnectionStringName;
-                    // Debug.Assert(connectionStringName != null, nameof(connectionStringName) + " != null");
-                    // var connectionString = configuration.GetConnectionString(connectionStringName);
-                    //
-                    // builder.WithAzureDiscovery(options =>
-                    // {
-                    //     options.ServiceName = settings.AkkaManagementOptions.ServiceName;
-                    //     options.ConnectionString = connectionString;
-                    // });
+                    var connectionString = configuration.GetConnectionString("AkkaManagementAzure");
+                    if (connectionString is null)
+                        throw new Exception("AkkaManagement table storage connection string [AkkaManagementAzure] is missing");
+
+                    builder
+                        .WithAzureDiscovery(options =>
+                        {
+                            options.ServiceName = settings.AkkaManagementOptions.ServiceName;
+                            options.ConnectionString = connectionString;
+                            options.HostName = settings.RemoteOptions.PublicHostName?.ToLower() ?? "localhost";
+                            options.Port = settings.AkkaManagementOptions.Port;
+                        })
+                        .AddHocon(AzureDiscovery.DefaultConfiguration(), HoconAddMode.Append);
                     break;
                 }
                 case DiscoveryMethod.Config:
@@ -110,10 +128,10 @@ public static AkkaConfigurationBuilder ConfigureNetwork(this AkkaConfigurationBu
                             options.Services.Add(new Service
                             {
                                 Name = settings.AkkaManagementOptions.ServiceName,
-                                Endpoints = new[]
-                                {
-                                    $"{settings.AkkaManagementOptions.Hostname}:{settings.AkkaManagementOptions.Port}",
-                                }
+                                Endpoints =
+                                [
+                                    $"{settings.AkkaManagementOptions.Hostname}:{settings.AkkaManagementOptions.Port}"
+                                ]
                             });
                         });
                     break;

src/DrawTogether/Config/AkkaManagementOptions.cs

@@ -20,6 +20,8 @@ public class AkkaManagementOptions
     public int RequiredContactPointsNr { get; set; } = 3;
 
     public DiscoveryMethod DiscoveryMethod { get; set; } = DiscoveryMethod.Config;
+
+    public bool FilterOnFallbackPort { get; set; } = true;
 }
 
 /// <summary>

src/DrawTogether/Config/AkkaSettings.cs

@@ -2,6 +2,7 @@
 using Akka.Cluster.Hosting;
 using Akka.Remote.Hosting;
 using DrawTogether.Actors;
+using Petabridge.Cmd.Host;
 
 namespace DrawTogether.Config;
 
@@ -15,6 +16,7 @@ public class AkkaSettings
     {
         // can be overridden via config, but is dynamic by default
         PublicHostName = Dns.GetHostName(),
+        HostName = "0.0.0.0",
         Port = 8081
     };
 
@@ -28,4 +30,10 @@ public class AkkaSettings
     public ShardOptions ShardOptions { get; set; } = new ShardOptions();
 
     public AkkaManagementOptions? AkkaManagementOptions { get; set; }
+
+    public PetabridgeCmdOptions PbmOptions { get; set; } = new()
+    {
+        Host = "0.0.0.0", 
+        Port = 9110
+    };
 }

src/DrawTogether/DockerFile

@@ -0,0 +1,28 @@
+# Use the official .NET SDK image to build
+FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
+WORKDIR ./
+
+# Copy the solution file and all csproj files first (to leverage Docker cache)
+COPY ["DrawTogether.sln", "./"]
+COPY ["Directory.Build.props", "./"]
+COPY ["Directory.Packages.props", "./"]
+COPY ["nuget.config", "./"]
+COPY ["global.json", "./"]
+
+# Copy all source
+COPY src/ ./src/
+COPY tests/ ./tests/
+
+# Publish the main application (DrawTogether) in Release mode
+WORKDIR ./src/DrawTogether
+RUN dotnet publish -c Release -o /app/out
+
+# Build runtime image
+FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS runtime
+WORKDIR /app
+COPY --from=build /app/out ./
+
+# Expose whichever port your service listens on (e.g., 80)
+EXPOSE 80
+
+ENTRYPOINT ["dotnet", "DrawTogether.dll"]

src/DrawTogether/DrawTogether.csproj

@@ -30,6 +30,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="Akka.Discovery.Azure" />
         <PackageReference Include="Akka.Discovery.KubernetesApi" />
         <PackageReference Include="Akka.Management" />
         <PackageReference Include="Akka.Persistence.Sql.Hosting" />