Skip to content

Commit e102d76

Browse files
AarononthewebArkatufus
Aaronontheweb
and
Arkatufus
authored
[WIP] use Aspire + blob storage to run multiple local replicas (#206)
* stash * need to keep configuration * working on getting `AzureTableStorage` for Akka.Discovery.Azure integrated * added extension methods for working with Akka.Management * disable volumes and Akka.Management during Aspire testing * WIP try to randomize port * Add Aspire support * Add missing DockerFile file --------- Co-authored-by: Gregorius Soedharmo <[email protected]>
1 parent 7ef0214 commit e102d76

16 files changed

+236
-47
lines changed

‎Directory.Packages.props

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,23 @@
55
<AkkaHostingVersion>1.5.40</AkkaHostingVersion>
66
<AkkaManagementVersion>1.5.37</AkkaManagementVersion>
77
<PetabridgeCmdVersion>1.4.4</PetabridgeCmdVersion>
8-
<AspireVersion>9.2.1</AspireVersion>
8+
<AspireVersion>9.3.0</AspireVersion>
99
<PlaywrightVersion>1.52.0</PlaywrightVersion>
1010
</PropertyGroup>
1111
<!-- Akka.NET Package Versions -->
1212
<ItemGroup>
1313
<PackageVersion Include="Akka" Version="$(AkkaVersion)" />
1414
<PackageVersion Include="Akka.Cluster.Hosting" Version="$(AkkaHostingVersion)" />
15+
<PackageVersion Include="Akka.Discovery.Azure" Version="$(AkkaManagementVersion)" />
1516
<PackageVersion Include="Akka.Discovery.KubernetesApi" Version="$(AkkaManagementVersion)" />
1617
<PackageVersion Include="Akka.Hosting" Version="$(AkkaHostingVersion)" />
1718
<PackageVersion Include="Akka.Management" Version="$(AkkaManagementVersion)" />
1819
<PackageVersion Include="Akka.Persistence.Sql.Hosting" Version="1.5.40.1" />
1920
<PackageVersion Include="Akka.Streams" Version="$(AkkaVersion)" />
2021
<PackageVersion Include="Akka.Streams.TestKit" Version="$(AkkaVersion)" />
21-
<PackageVersion Include="Aspire.Hosting.Docker" Version="9.2.1-preview.1.25222.1" />
22-
<PackageVersion Include="Aspire.Hosting.Kubernetes" Version="9.2.1-preview.1.25222.1" />
22+
<PackageVersion Include="Aspire.Hosting.Azure.Storage" Version="$(AspireVersion)" />
23+
<PackageVersion Include="Aspire.Hosting.Docker" Version="9.3.0-preview.1.25265.20" />
24+
<PackageVersion Include="Aspire.Hosting.Kubernetes" Version="9.3.0-preview.1.25265.20" />
2325
<PackageVersion Include="Grpc.Tools" Version="2.72.0">
2426
<PrivateAssets>all</PrivateAssets>
2527
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

‎src/DrawTogether.AppHost/AkkaManagementExtensions.cs

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
using System.Net.Sockets;
2+
using Aspire.Hosting.Azure;
3+
using Microsoft.Extensions.Hosting;
4+
5+
namespace DrawTogether.AppHost;
6+
7+
public static class AkkaManagementExtensions
8+
{
9+
public static IResourceBuilder<ProjectResource> ConfigureAkkaManagementForApp(this IResourceBuilder<ProjectResource> appBuilder, DrawTogetherConfiguration config)
10+
{
11+
if (!config.UseAkkaManagement) return appBuilder;
12+
13+
var builder = appBuilder.ApplicationBuilder;
14+
15+
var azureStorage = builder.AddAzureStorage("storage")
16+
.RunAsEmulator();
17+
18+
var tableStorage = azureStorage.AddTables("akka-discovery");
19+
20+
appBuilder.WaitFor(tableStorage)
21+
.WithReference(tableStorage, "AkkaManagementAzure");
22+
23+
// Setup network endpoint ports
24+
appBuilder
25+
.WithEndpoint(name: "remote", protocol: ProtocolType.Tcp, env: "AkkaSettings__RemoteOptions__Port")
26+
.WithEndpoint(name: "management", protocol: ProtocolType.Tcp, env: "AkkaSettings__AkkaManagementOptions__Port")
27+
.WithEndpoint(name: "pbm", protocol: ProtocolType.Tcp, env: "AkkaSettings__PbmOptions__Port");
28+
29+
// need to populate some config for the hosts
30+
appBuilder
31+
.WithEnvironment("AkkaSettings__RemoteOptions__PublicHostName", "localhost")
32+
.WithEnvironment("AkkaSettings__AkkaManagementOptions__Enabled", "true")
33+
.WithEnvironment("AkkaSettings__AkkaManagementOptions__DiscoveryMethod", "AzureTableStorage")
34+
.WithEnvironment("AkkaSettings__AkkaManagementOptions__FilterOnFallbackPort", "false");
35+
36+
return appBuilder;
37+
}
38+
39+
private static IResourceBuilder<AzureTableStorageResource>? _tableStorage;
40+
private static IResourceBuilder<AzureTableStorageResource> GetTableStorage(IDistributedApplicationBuilder builder)
41+
{
42+
if (_tableStorage != null) return _tableStorage;
43+
var azureStorage = builder.AddAzureStorage("storage");
44+
if(builder.Environment.IsDevelopment())
45+
azureStorage.RunAsEmulator();
46+
47+
_tableStorage = azureStorage.AddTables("akka-discovery");
48+
return _tableStorage;
49+
}
50+
51+
public static IResourceBuilder<ContainerResource> ConfigureAkkaManagementForApp(this IResourceBuilder<ContainerResource> appBuilder, DrawTogetherConfiguration config)
52+
{
53+
if (!config.UseAkkaManagement) return appBuilder;
54+
55+
var tableStorage = GetTableStorage(appBuilder.ApplicationBuilder);
56+
57+
appBuilder.WaitFor(tableStorage)
58+
.WithReference(tableStorage, "AkkaManagementAzure");
59+
60+
// Setup network endpoint ports
61+
appBuilder
62+
.WithHttpEndpoint(targetPort: 8080)
63+
.WithEndpoint(name: "remote", protocol: ProtocolType.Tcp, targetPort: 14884, env: "AkkaSettings__RemoteOptions__Port")
64+
.WithEndpoint(name: "management", protocol: ProtocolType.Tcp, targetPort: 8558, env: "AkkaSettings__AkkaManagementOptions__Port")
65+
.WithEndpoint(name: "pbm", protocol: ProtocolType.Tcp, targetPort: 9110, env: "AkkaSettings__PbmOptions__Port");
66+
67+
// need to populate some config for the hosts
68+
appBuilder
69+
.WithEnvironment("AkkaSettings__AkkaManagementOptions__Enabled", "true")
70+
.WithEnvironment("AkkaSettings__AkkaManagementOptions__DiscoveryMethod", "AzureTableStorage")
71+
.WithEnvironment("AkkaSettings__AkkaManagementOptions__RequiredContactPointsNr", "1")
72+
.WithEnvironment("AkkaSettings__AkkaManagementOptions__FilterOnFallbackPort", "false");
73+
74+
return appBuilder;
75+
}
76+
}

‎src/DrawTogether.AppHost/Configuration.cs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,20 @@ public class DrawTogetherConfiguration
99
/// Defaults to false so we don't accidentally persist data during testing.
1010
/// </remarks>
1111
public bool UseVolumes { get; set; } = false;
12+
13+
/// <summary>
14+
/// When this is enabled, we will use Akka Management to manage our cluster.
15+
///
16+
/// This means exposing an additional internal endpoint and spinning up Azure storage.
17+
/// </summary>
18+
public bool UseAkkaManagement { get; set; } = true;
19+
20+
/// <summary>
21+
/// The total number of replicas we're going to run in our cluster.
22+
///
23+
/// Only relevant when <see cref="UseAkkaManagement"/> is enabled.
24+
/// </summary>
25+
public int Replicas { get; set; } = 1;
26+
27+
public DeployEnvironment DeployEnvironment { get; set; } = DeployEnvironment.Local;
1228
}

‎src/DrawTogether.AppHost/DeployEnvironment.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
namespace DrawTogether.AppHost;
2+
3+
public enum DeployEnvironment
4+
{
5+
Local,
6+
Docker
7+
}

‎src/DrawTogether.AppHost/DrawTogether.AppHost.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
<ItemGroup>
1111
<PackageReference Include="Aspire.Hosting.AppHost" />
12+
<PackageReference Include="Aspire.Hosting.Azure.Storage" />
1213
<PackageReference Include="Aspire.Hosting.Docker" />
1314
<PackageReference Include="Aspire.Hosting.Kubernetes" />
1415
<PackageReference Include="Aspire.Hosting.SqlServer" />

‎src/DrawTogether.AppHost/Program.cs

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,9 @@
33

44
var builder = DistributedApplication.CreateBuilder(args);
55

6-
76
var drawTogetherAspireConfig = builder.Configuration.GetSection("DrawTogether")
87
.Get<DrawTogetherConfiguration>() ?? new DrawTogetherConfiguration();
98

10-
builder.AddDockerComposePublisher()
11-
.AddKubernetesPublisher();
12-
139
// Adding a default password for ease of use - we can get rid of this but for a quick "git clone and run" it makes sense
1410
// have to add this when using data volumes otherwise Aspire will brick itself
1511

@@ -32,21 +28,38 @@
3228
.WaitFor(db)
3329
.WithReference(db);
3430

35-
var drawTogether = builder.AddProject<Projects.DrawTogether>("DrawTogether")
36-
.WithReference(db, "DefaultConnection")
37-
.WaitForCompletion(migrationService)
38-
.WithEndpoint("pbm", annotation =>
31+
if (drawTogetherAspireConfig.DeployEnvironment == DeployEnvironment.Docker)
32+
{
33+
var drawTogether = builder.AddDockerfile("DrawTogether-1", "../../", "./src/DrawTogether/DockerFile")
34+
.WithImage("draw-together", "latest")
35+
.WithReference(db, "DefaultConnection")
36+
.ConfigureAkkaManagementForApp(drawTogetherAspireConfig)
37+
.WaitForCompletion(migrationService);
38+
39+
foreach (var index in Enumerable.Range(2, drawTogetherAspireConfig.Replicas - 1))
3940
{
40-
annotation.Port = 9110;
41-
42-
// not mean to be external, meant to be invoked via the pbm-sidecar
43-
annotation.IsExternal = false;
44-
annotation.IsProxied = false;
45-
});
46-
47-
// https://github.com/petabridge/pbm-sidecar - used to run `pbm` commands on the DrawTogether actor system
48-
var pbmSidecar = builder.AddContainer("pbm-sidecar", "petabridge/pbm:latest")
49-
.WaitFor(drawTogether);
41+
builder.AddContainer($"DrawTogether-{index}", "draw-together")
42+
.WithReference(db, "DefaultConnection")
43+
.ConfigureAkkaManagementForApp(drawTogetherAspireConfig)
44+
.WaitFor(drawTogether);
45+
}
46+
47+
// https://github.com/petabridge/pbm-sidecar - used to run `pbm` commands on the DrawTogether actor system
48+
var pbmSidecar = builder.AddContainer("pbm-sidecar", "petabridge/pbm:latest")
49+
.WaitFor(drawTogether);
50+
}
51+
else
52+
{
53+
var drawTogether = builder.AddProject<Projects.DrawTogether>("DrawTogether")
54+
.WithReplicas(drawTogetherAspireConfig.Replicas)
55+
.WithReference(db, "DefaultConnection")
56+
.WaitForCompletion(migrationService)
57+
.ConfigureAkkaManagementForApp(drawTogetherAspireConfig);
58+
59+
// https://github.com/petabridge/pbm-sidecar - used to run `pbm` commands on the DrawTogether actor system
60+
var pbmSidecar = builder.AddContainer("pbm-sidecar", "petabridge/pbm:latest")
61+
.WaitFor(drawTogether);
62+
}
5063

5164
builder
5265
.Build()

‎src/DrawTogether.AppHost/appsettings.Development.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,8 @@
44
"Default": "Information",
55
"Microsoft.AspNetCore": "Warning"
66
}
7+
},
8+
"DrawTogether": {
9+
"DeployEnvironment": "Local"
710
}
811
}

‎src/DrawTogether.AppHost/appsettings.json

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
}
88
},
99
"DrawTogether": {
10-
"UseVolumes": true
10+
"UseVolumes": true,
11+
"UseAkkaManagement": true,
12+
"Replicas": 3,
13+
"DeployEnvironment": "Docker"
1114
}
1215
}

‎src/DrawTogether/Config/AkkaConfiguration.cs

Lines changed: 37 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
using Akka.Cluster.Hosting;
1+
using System.Diagnostics;
2+
using Akka.Cluster.Hosting;
3+
using Akka.Discovery.Azure;
24
using Akka.Discovery.Config.Hosting;
35
using Akka.Discovery.KubernetesApi;
46
using Akka.Hosting;
@@ -62,6 +64,7 @@ public static AkkaConfigurationBuilder ConfigureNetwork(this AkkaConfigurationBu
6264
IServiceProvider serviceProvider)
6365
{
6466
var settings = serviceProvider.GetRequiredService<AkkaSettings>();
67+
var configuration = serviceProvider.GetRequiredService<IConfiguration>();
6568

6669
builder
6770
.WithRemoting(settings.RemoteOptions);
@@ -74,10 +77,22 @@ public static AkkaConfigurationBuilder ConfigureNetwork(this AkkaConfigurationBu
7477

7578
builder
7679
.WithClustering(clusterOptions)
77-
.WithAkkaManagement(port: settings.AkkaManagementOptions.Port)
78-
.WithClusterBootstrap(serviceName: settings.AkkaManagementOptions.ServiceName,
79-
portName: settings.AkkaManagementOptions.PortName,
80-
requiredContactPoints: settings.AkkaManagementOptions.RequiredContactPointsNr);
80+
.WithAkkaManagement(setup =>
81+
{
82+
setup.Http.HostName = settings.RemoteOptions.PublicHostName?.ToLower() ?? "localhost";
83+
setup.Http.Port = settings.AkkaManagementOptions.Port;
84+
setup.Http.BindHostName = "0.0.0.0";
85+
setup.Http.BindPort = settings.AkkaManagementOptions.Port;
86+
})
87+
.WithClusterBootstrap(options =>
88+
{
89+
options.ContactPointDiscovery.ServiceName = settings.AkkaManagementOptions.ServiceName;
90+
options.ContactPointDiscovery.PortName = settings.AkkaManagementOptions.PortName;
91+
options.ContactPointDiscovery.RequiredContactPointsNr = settings.AkkaManagementOptions.RequiredContactPointsNr;
92+
options.ContactPointDiscovery.ContactWithAllContactPoints = true;
93+
94+
options.ContactPoint.FilterOnFallbackPort = settings.AkkaManagementOptions.FilterOnFallbackPort;
95+
}, autoStart: true);
8196

8297
switch (settings.AkkaManagementOptions.DiscoveryMethod)
8398
{
@@ -90,16 +105,19 @@ public static AkkaConfigurationBuilder ConfigureNetwork(this AkkaConfigurationBu
90105
break;
91106
case DiscoveryMethod.AzureTableStorage:
92107
{
93-
// var connectionStringName = configuration.GetSection("AzureStorageSettings")
94-
// .Get<AzureStorageSettings>()?.ConnectionStringName;
95-
// Debug.Assert(connectionStringName != null, nameof(connectionStringName) + " != null");
96-
// var connectionString = configuration.GetConnectionString(connectionStringName);
97-
//
98-
// builder.WithAzureDiscovery(options =>
99-
// {
100-
// options.ServiceName = settings.AkkaManagementOptions.ServiceName;
101-
// options.ConnectionString = connectionString;
102-
// });
108+
var connectionString = configuration.GetConnectionString("AkkaManagementAzure");
109+
if (connectionString is null)
110+
throw new Exception("AkkaManagement table storage connection string [AkkaManagementAzure] is missing");
111+
112+
builder
113+
.WithAzureDiscovery(options =>
114+
{
115+
options.ServiceName = settings.AkkaManagementOptions.ServiceName;
116+
options.ConnectionString = connectionString;
117+
options.HostName = settings.RemoteOptions.PublicHostName?.ToLower() ?? "localhost";
118+
options.Port = settings.AkkaManagementOptions.Port;
119+
})
120+
.AddHocon(AzureDiscovery.DefaultConfiguration(), HoconAddMode.Append);
103121
break;
104122
}
105123
case DiscoveryMethod.Config:
@@ -110,10 +128,10 @@ public static AkkaConfigurationBuilder ConfigureNetwork(this AkkaConfigurationBu
110128
options.Services.Add(new Service
111129
{
112130
Name = settings.AkkaManagementOptions.ServiceName,
113-
Endpoints = new[]
114-
{
115-
$"{settings.AkkaManagementOptions.Hostname}:{settings.AkkaManagementOptions.Port}",
116-
}
131+
Endpoints =
132+
[
133+
$"{settings.AkkaManagementOptions.Hostname}:{settings.AkkaManagementOptions.Port}"
134+
]
117135
});
118136
});
119137
break;

‎src/DrawTogether/Config/AkkaManagementOptions.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ public class AkkaManagementOptions
2020
public int RequiredContactPointsNr { get; set; } = 3;
2121

2222
public DiscoveryMethod DiscoveryMethod { get; set; } = DiscoveryMethod.Config;
23+
24+
public bool FilterOnFallbackPort { get; set; } = true;
2325
}
2426

2527
/// <summary>

0 commit comments

Comments
 (0)