feat(doppler): connection gateway

.github/workspace-dep-closures.json

@@ -858,6 +858,8 @@
       "last_online_tracker",
       "macro_auth",
       "macro_aws_config",
+      "macro_config",
+      "macro_config_derive",
       "macro_db_client",
       "macro_entrypoint",
       "macro_env",

infra/stacks/connection-gateway/connection_gateway.ts

@@ -14,10 +14,11 @@ import { EcrImage } from '../../packages/service';
 import {
   BASE_DOMAIN,
   CLOUD_TRAIL_SNS_TOPIC_ARN,
+  DopplerEcsEnvironment,
   stack,
 } from '../../packages/shared';
 
-const BASE_NAME = 'connection-gateway';
+const BASE_NAME = pulumi.getProject();
 const BASE_PATH = '../../../rust/cloud-storage';
 
 export const SERVICE_DOMAIN_NAME = `connection-gateway${
@@ -175,6 +176,12 @@ export class ConnectionGateway extends pulumi.ComponentResource {
       { parent: this }
     );
 
+    const dopplerEcsEnvironment = new DopplerEcsEnvironment(
+      BASE_NAME,
+      { tags: this.tags },
+      { parent: this }
+    );
+
     // service
     const service = new awsx.ecs.FargateService(
       `${BASE_NAME}`,
@@ -194,6 +201,9 @@ export class ConnectionGateway extends pulumi.ComponentResource {
           taskRole: {
             roleArn: this.role.arn,
           },
+          executionRole: {
+            roleArn: dopplerEcsEnvironment.executionRole.arn,
+          },
           containers: {
             log_router: fargateLogRouterSidecarContainer,
             datadog_agent: datadogAgentContainer,
@@ -204,6 +214,7 @@ export class ConnectionGateway extends pulumi.ComponentResource {
               cpu: 4096,
               memory: 8192,
               environment: containerEnvVars,
+              secrets: [...dopplerEcsEnvironment.containerSecrets],
               logConfiguration: {
                 logDriver: 'awsfirelens',
                 options: {

infra/stacks/connection-gateway/index.ts

@@ -17,32 +17,11 @@ export const connectionGatewayTable: {
   policy: aws.iam.Policy;
 } = getConnectionGatewayTable();
 
-const LEGACY_JWT_SECRET = aws.secretsmanager
-  .getSecretVersionOutput({
-    secretId: config.get(`legacy_jwt_secret_key`) ?? '',
-  })
-  .apply((secret) => secret.secretString);
-
 const JWT_SECRET_KEY = config.require(`jwt_secret_key`);
 const jwtSecretKeyArn: pulumi.Output<string> = aws.secretsmanager
   .getSecretVersionOutput({ secretId: JWT_SECRET_KEY })
   .apply((secret) => secret.arn);
 
-const fusionauthClientIdSecretKey = config.require(`fusionauth_client_id`);
-
-const FUSIONAUTH_CLIENT_ID = aws.secretsmanager
-  .getSecretVersionOutput({
-    secretId: fusionauthClientIdSecretKey,
-  })
-  .apply((secret) => secret.secretString);
-const FUSIONAUTH_ISSUER = config.require(`fusionauth_issuer`);
-
-const INTERNAL_AUTH_KEY = aws.secretsmanager
-  .getSecretVersionOutput({
-    secretId: config.get(`internal_auth_key`) ?? '',
-  })
-  .apply((secret) => secret.secretString);
-
 export const coparse_api_vpc = get_coparse_api_vpc();
 
 const cloudStorageStack = new pulumi.StackReference('cloud-storage-stack', {
@@ -95,49 +74,6 @@ const connectionGateway = new ConnectionGateway(`connection-gateway-${stack}`, {
       name: 'ENVIRONMENT',
       value: stack,
     },
-    {
-      name: 'RUST_LOG',
-      value: `connection_gateway=${
-        stack === 'prod' ? 'info' : 'trace'
-      },tower_http=debug,frecency=warn`,
-    },
-    {
-      name: 'LEGACY_JWT_SECRET',
-      value: pulumi.interpolate`${LEGACY_JWT_SECRET}`,
-    },
-    {
-      name: 'INTERNAL_API_SECRET_KEY',
-      value: pulumi.interpolate`${INTERNAL_AUTH_KEY}`,
-    },
-    { name: 'ISSUER', value: pulumi.interpolate`${FUSIONAUTH_ISSUER}` },
-    {
-      name: 'JWT_SECRET_KEY',
-      value: pulumi.interpolate`${JWT_SECRET_KEY}`,
-    },
-    {
-      name: 'AUDIENCE',
-      value: pulumi.interpolate`${FUSIONAUTH_CLIENT_ID}`,
-    },
-    {
-      name: 'CONNECTION_GATEWAY_TABLE',
-      value: pulumi.interpolate`${connectionGatewayTable.table.name}`,
-    },
-    {
-      name: 'REDIS_HOST',
-      value: pulumi.interpolate`redis://${connectionGatewayRedis.endpoint}`,
-    },
-    {
-      name: 'MACRO_API_TOKEN_ISSUER',
-      value: pulumi.interpolate`${MACRO_API_TOKENS.macroApiTokenIssuer}`,
-    },
-    {
-      name: 'MACRO_API_TOKEN_PUBLIC_KEY',
-      value: pulumi.interpolate`${MACRO_API_TOKENS.macroApiTokenPublicKey}`,
-    },
-    {
-      name: 'MACRO_DB_URL',
-      value: pulumi.interpolate`${MACRO_DB_URL}`,
-    },
     // OpenTelemetry / Datadog tracing configuration
     {
       name: 'DD_SERVICE',

infra/stacks/doppler-projects/index.ts

@@ -10,6 +10,7 @@ const SERVICE_NAMES = [
   'unfurl-service',
   'static-file-service',
   'image-proxy-service',
+  'connection-gateway',
 ];
 
 for (const service_name of SERVICE_NAMES) {

rust/cloud-storage/connection_gateway/Cargo.toml

@@ -12,6 +12,11 @@ path = "src/lib.rs"
 name = "connection_gateway_service"
 path = "src/main.rs"
 
+[[bin]]
+name = "connection_gateway_doppler_config"
+path = "src/doppler_config.rs"
+
+
 [[bin]]
 name = "stale_connections"
 path = "scripts/stale_connections.rs"
@@ -44,6 +49,7 @@ last_online_tracker = { path = "../last_online_tracker" }
 macro_auth = { path = "../macro_auth" }
 macro_user_id = { path = "../macro_user_id" }
 macro_aws_config = { path = "../macro_aws_config" }
+macro_config = { path = "../macro_config" }
 macro_entrypoint = { path = "../macro_entrypoint" }
 macro_env = { path = "../macro_env" }
 macro_env_var = { path = "../macro_env_var" }
@@ -75,6 +81,7 @@ utoipa = { workspace = true, features = ["axum_extras"] }
 utoipa-swagger-ui = { workspace = true, features = ["axum"] }
 uuid = { workspace = true }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
+doppler-config = { workspace = true }
 
 [dev-dependencies]
 cool_asserts = "2"

rust/cloud-storage/connection_gateway/src/config.rs

@@ -1,6 +1,12 @@
-use anyhow::Result;
+use anyhow::Context;
 pub use macro_env::Environment;
-use macro_env_var::env_var;
+use macro_env_var::env_vars;
+use secretsmanager_client::LocalOrRemoteSecret;
+
+env_vars! {
+    pub struct MacroDbUrl;
+    pub struct RedisHost;
+}
 
 /// The configuration parameters for the application.
 ///
@@ -9,39 +15,22 @@ use macro_env_var::env_var;
 /// populate the Docker container
 ///
 /// See `.env.sample` in cognitive-workspace root for details.
+#[derive(macro_config::MacroConfig)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
 pub struct Config {
     /// The port the service is running on
+    #[macro_config_default(8080)]
     pub port: usize,
     /// The environment we are in
+    #[macro_config_default(Environment::new_or_prod())]
     pub environment: Environment,
     pub redis_host: RedisHost,
+    pub macro_db_url: LocalOrRemoteSecret<MacroDbUrl>,
 }
 
-env_var!(
-    pub struct EnvVars {
-        pub RedisHost,
-    }
-);
-
-env_var!(
-    struct Port;
-);
-
 impl Config {
-    pub fn from_env(env_vars: EnvVars) -> Self {
-        let port: usize = Port::new()
-            .ok()
-            .and_then(|v| v.parse().ok())
-            .unwrap_or(8080);
-
-        let environment = Environment::new_or_prod();
-
-        let EnvVars { redis_host } = env_vars;
-
-        Config {
-            port,
-            environment,
-            redis_host,
-        }
+    pub fn from_env() -> anyhow::Result<Self> {
+        macro_config::ConfigLoader::load::<Config>()
+            .context("failed to load connection gateway config")
     }
 }

rust/cloud-storage/connection_gateway/src/doppler_config.rs

@@ -0,0 +1,31 @@
+#![allow(unused)]
+#![recursion_limit = "256"]
+
+use macro_env::Environment;
+
+mod config;
+
+const DOPPLER_PROJECT: &str = "connection-gateway";
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let dev = doppler_config::DopplerConfig::builder()
+        .token_from_env("DOPPLER_TOKEN")
+        .config(Environment::Develop.to_doppler_slug())
+        .project(DOPPLER_PROJECT)
+        .build()
+        .expect("able to grab doppler project");
+
+    dev.load::<config::Config>().await?;
+
+    let prd = doppler_config::DopplerConfig::builder()
+        .token_from_env("DOPPLER_TOKEN")
+        .config(Environment::Production.to_doppler_slug())
+        .project(DOPPLER_PROJECT)
+        .build()
+        .expect("able to grab doppler project");
+
+    prd.load::<config::Config>().await?;
+
+    Ok(())
+}

rust/cloud-storage/connection_gateway/src/main.rs

@@ -7,7 +7,7 @@ mod model;
 mod service;
 use std::{sync::Arc, time::Duration};
 
-use crate::{api::router, config::EnvVars, context::AppState};
+use crate::{api::router, context::AppState};
 use anyhow::{Context, Result};
 use axum::http::{
     Method,
@@ -30,7 +30,7 @@ use last_online_tracker::{
 };
 use macro_auth::middleware::decode_jwt::JwtValidationArgs;
 use macro_entrypoint::MacroEntrypoint;
-use macro_env_var::env_var;
+use macro_env::Environment;
 use macro_middleware::auth::internal_access::InternalApiSecretKey;
 use secretsmanager_client::LocalOrRemoteSecret;
 use service::dynamodb::create_dynamo_db_connection_manager;
@@ -39,23 +39,22 @@ use sqlx::postgres::PgPoolOptions;
 use stream::outbound::redis_pg::{RedisPostgresStreamManager, RedisPostgresStreamRepo};
 use tower_http::cors::CorsLayer;
 
-env_var!(
-    struct MacroDbUrl;
-);
-
 #[tokio::main]
 #[tracing::instrument(ret, err)]
 async fn main() -> Result<()> {
     MacroEntrypoint::default().init();
 
-    // Parse our configuration from the environment.
-    let config = Arc::new(Config::from_env(EnvVars::unwrap_new()));
-
     let aws_config = macro_aws_config::get_macro_aws_config().await;
 
     let secretsmanager_client = secretsmanager_client::SecretsManager::new(
         aws_sdk_secretsmanager::Client::new(&aws_config),
     );
+
+    // Parse our configuration from the environment.
+    let config = Config::from_env()?
+        .resolve_remote_secrets(Environment::new_or_prod(), &secretsmanager_client)
+        .await?;
+
     let jwt_args =
         JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
             .await?;
@@ -103,14 +102,7 @@ async fn main() -> Result<()> {
     let pgpool = PgPoolOptions::new()
         .min_connections(3)
         .max_connections(20)
-        .connect(
-            LocalOrRemoteSecret::new_from_secret_manager(
-                MacroDbUrl::new()?,
-                &secretsmanager_client,
-            )
-            .await?
-            .as_ref(),
-        )
+        .connect(config.macro_db_url.as_ref())
         .await?;
 
     let stream_service = RedisPostgresStreamRepo::new((*redis_client).clone(), pgpool.clone());
@@ -127,6 +119,8 @@ async fn main() -> Result<()> {
 
     tokio::spawn(poll_messages(context.clone()));
 
+    let config = Arc::new(config);
+
     let app = router(AppState {
         context,
         config: Arc::clone(&config),