Use AWS RDS IAM secrets from duckdb-aws

This PR builds on top of the following PR in duckdb-aws:

 - duckdb/duckdb-aws#144

 With the RDS IAM token generation done in `duckdb-aws`, now two secrets
 are required for IAM auth:

 1. secret of type `rds` that contains all the details used to generate
   the IAM token
 2. secret of type `postgres` that contains all other connection details
   except the IAM token; the `rds` secret name is specified in this
   secret

Example:

```sql
LOAD aws;
LOAD postgres;

CREATE SECRET aws_rds_secret1 (
  TYPE rds,
  PROVIDER credential_chain,
  PROFILE 'DatabaseAdministrator-<account_id>',
  CHAIN 'env;sso;',
  REGION 'eu-west-1',
  RDS_USER 'postgres',
  RDS_HOST 'database-1-instance-1.xxxxxxxxxxxx.eu-west-1.rds.amazonaws.com',
  RDS_PORT '5432'
);

CREATE SECRET pg_rds_secret1 (
  TYPE postgres,
  HOST 'database-1-instance-1.xxxxxxxxxxxx.eu-west-1.rds.amazonaws.com',
  PORT '5432',
  USER 'postgres',
  DATABASE 'postgres',
  SSLMODE require,
  AWS_RDS_SECRET aws_rds_secret1
);
```

Testing: tested locally with Aurora using both `sso` and `env`
providers.

Ref: #464

Author: staticlibs

CMakeLists.txt

@@ -11,7 +11,6 @@ add_definitions(
 
 find_package(OpenSSL REQUIRED)
 find_package(PostgreSQL REQUIRED)
-find_package(AWSSDK REQUIRED COMPONENTS rds)
 
 if(NOT MSVC)
     add_compile_options(
@@ -55,7 +54,6 @@ target_link_libraries(${LOADABLE_EXTENSION_NAME}
     OpenSSL::SSL
     OpenSSL::Crypto
     PostgreSQL::PostgreSQL
-    ${AWSSDK_LINK_LIBRARIES}
     ${CURL_LIBRARIES})
 set_property(TARGET ${EXTENSION_NAME} PROPERTY C_STANDARD 99)
 set_property(TARGET ${LOADABLE_EXTENSION_NAME} PROPERTY C_STANDARD 99)

extension_config.cmake

@@ -10,6 +10,7 @@ duckdb_extension_load(postgres_scanner
 duckdb_extension_load(tpch)
 duckdb_extension_load(tpcds)
 duckdb_extension_load(json)
+# duckdb_extension_load(aws)
 
 # Any extra extensions that should be built
 # e.g.: duckdb_extension_load(json)

src/include/postgres_aws.hpp

@@ -17,17 +17,18 @@
 namespace duckdb {
 
 struct PostgresAwsRdsTokenConfig {
-	bool enabled = false;
-	std::string hostname;
-	std::string port;
-	std::string username;
-	std::string region;
-	std::uint64_t expiration_seconds = 900; // 15 min;
+	static const int64_t expiration_seconds = 900; // 15 min, this value is fixed, also used in duckdb-aws
+
+	std::string rds_secret_name;
 	std::string base_connection_string;
+
+	bool Enabled();
+	int64_t MaxAgeSeconds();
 };
 
 struct PostgresAws {
-	static std::string GenerateRdsAuthToken(const PostgresAwsRdsTokenConfig &token_config);
+	static std::string GenerateRdsAuthToken(AttachedDatabase &attached_db,
+	                                        const PostgresAwsRdsTokenConfig &token_config);
 
 	static PostgresAwsRdsTokenConfig ExtractTokenConfigFromSecret(optional_ptr<SecretEntry> secret_entry);
 };

src/include/postgres_secrets.hpp

@@ -16,6 +16,10 @@ namespace duckdb {
 struct PostgresSecrets {
 	static const std::vector<std::string> &ConnectionOptionNames();
 
+	static SecretType CreateType();
+
+	static SecretType CreateRdsType();
+
 	static unique_ptr<BaseSecret> CreateFunction(ClientContext &context, CreateSecretInput &input);
 
 	static void SetSecretParameters(CreateSecretFunction &function);

src/postgres_aws.cpp

@@ -3,49 +3,89 @@
 #include <mutex>
 #include <stdexcept>
 
-#include <aws/core/Aws.h>
-#include <aws/rds/RDSClient.h>
+#include "dbconnector/defer.hpp"
 
+#include "duckdb/parser/keyword_helper.hpp"
 #include "postgres_secrets.hpp"
 #include "postgres_utils.hpp"
 
 namespace duckdb {
 
-static std::mutex sdk_init_mutex;
-static bool sdk_initialized = false;
-static Aws::SDKOptions sdk_options;
+bool PostgresAwsRdsTokenConfig::Enabled() {
+	return !rds_secret_name.empty();
+}
+
+int64_t PostgresAwsRdsTokenConfig::MaxAgeSeconds() {
+	return expiration_seconds - 60;
+}
+
+static std::string MakeCreateSecretQuery(const std::string &template_secret_name, const std::string &secret_name,
+                                         const KeyValueSecret &kv_secret) {
+	std::string query("CREATE OR REPLACE TEMPORARY SECRET \"");
+	query += secret_name;
+	query += "\" (\n";
+	query += "  TYPE rds,\n";
+	query += "  PROVIDER credential_chain,\n";
+	for (auto &en : kv_secret.secret_map) {
+		query += "  " + en.first + " " + en.second.ToSQLString() + ",\n";
+	}
+	query += "  RDS_TEMPLATE_SECRET_NAME " + KeywordHelper::WriteQuoted(template_secret_name, '\'') + "\n";
+	query += ")";
+	return query;
+}
+
+static void RunQuery(Connection &conn, const std::string &query, const std::string &err_msg = std::string()) {
+	auto res = conn.Query(query);
+	if (res->HasError()) {
+		if (err_msg.empty()) {
+			throw InvalidConfigurationException("Error generating RDS IAM token: %s", res->GetError());
+		} else {
+			std::string error_type = EnumUtil::ToString(res->GetErrorType());
+			throw InvalidConfigurationException("Error generating RDS IAM token, type: %s, %s", error_type, err_msg);
+		}
+	}
+}
 
-static void EnsureAwsSdkInitialized() {
-	std::lock_guard<std::mutex> lock(sdk_init_mutex);
-	if (!sdk_initialized) {
-		Aws::InitAPI(sdk_options);
-		sdk_initialized = true;
+static unique_ptr<SecretEntry> GetRdsSecret(SecretManager &secret_manager, Connection &conn, const std::string &name) {
+	auto transaction = CatalogTransaction::GetSystemCatalogTransaction(*conn.context);
+	auto secret_entry = secret_manager.GetSecretByName(transaction, name);
+	if (!secret_entry) {
+		throw InvalidConfigurationException("Specified RDS secret with name: \"%s\" not found", name);
 	}
+	return secret_entry;
 }
 
-std::string PostgresAws::GenerateRdsAuthToken(const PostgresAwsRdsTokenConfig &token_config) {
-	EnsureAwsSdkInitialized();
-
-	Aws::Client::ClientConfiguration config;
-	config.region = token_config.region;
-	Aws::RDS::RDSClient rds_client(config);
-
-	// https://github.com/aws/aws-sdk-cpp/issues/861#issuecomment-386643571
-	// Aws::String token = rdsClient.GenerateConnectAuthToken(hostname.c_str(), aws_region.c_str(),
-	// static_cast<unsigned>(port_int), username.c_str());
-
-	std::string host_and_port = token_config.hostname + ":" + token_config.port;
-	std::string host_and_port_with_prefix = "http://" + host_and_port;
-	std::string host_and_port_with_suffix = host_and_port + "/";
-	Aws::Http::URI uri(host_and_port_with_prefix.c_str());
-	uri.AddQueryStringParameter("Action", "connect");
-	uri.AddQueryStringParameter("DBUser", token_config.username.c_str());
-	auto token = rds_client.GeneratePresignedUrl(uri, Aws::Http::HttpMethod::HTTP_GET, token_config.region.c_str(),
-	                                             "rds-db", static_cast<long long>(token_config.expiration_seconds));
-	Aws::Utils::StringUtils::Replace(token, host_and_port_with_prefix.c_str(), host_and_port_with_suffix.c_str());
-
-	std::string token_str = token.c_str();
-	return token_str;
+std::string PostgresAws::GenerateRdsAuthToken(AttachedDatabase &attached_db,
+                                              const PostgresAwsRdsTokenConfig &token_config) {
+	DatabaseInstance &db = attached_db.GetDatabase();
+	Connection conn(db);
+	RunQuery(conn, "LOAD aws");
+	RunQuery(conn, "BEGIN TRANSACTION");
+	auto deferred_rollback = dbconnector::Defer([&conn] { conn.Query("ROLLBACK"); });
+
+	SecretManager &secret_manager = SecretManager::Get(db);
+	auto template_secret = GetRdsSecret(secret_manager, conn, token_config.rds_secret_name);
+	const auto &kv_template_secret = dynamic_cast<const KeyValueSecret &>(*template_secret->secret);
+
+	std::string secret_name = token_config.rds_secret_name + "_" + attached_db.GetName() + "_postgres_temp";
+	std::string create_secret_query =
+	    MakeCreateSecretQuery(token_config.rds_secret_name, secret_name, kv_template_secret);
+
+	std::string quoted_secret_name = KeywordHelper::WriteQuoted(secret_name, '"');
+	RunQuery(conn, create_secret_query, "error creating RDS secret from template: " + token_config.rds_secret_name);
+	auto deferred_drop_secret =
+	    dbconnector::Defer([&conn, quoted_secret_name] { conn.Query("DROP SECRET " + quoted_secret_name); });
+
+	auto secret = GetRdsSecret(secret_manager, conn, secret_name);
+	const auto &kv_secret = dynamic_cast<const KeyValueSecret &>(*secret->secret);
+	std::string token = kv_secret.TryGetValue("session_token").ToString();
+
+	if (token.find("X-Amz-Algorithm") == std::string::npos) {
+		throw InvalidConfigurationException("Unable to generate RDS IAM token for secret with name: \"%s\"",
+		                                    token_config.rds_secret_name);
+	}
+
+	return token;
 }
 
 static std::string ExtractString(const KeyValueSecret &kv, const std::string name) {
@@ -64,30 +104,16 @@ PostgresAwsRdsTokenConfig PostgresAws::ExtractTokenConfigFromSecret(optional_ptr
 		return config;
 	}
 	const auto &kv = dynamic_cast<const KeyValueSecret &>(*secret_entry->secret);
-	Value enabled_val = kv.TryGetValue("aws_rds_iam_auth_enabled");
-	if (enabled_val.IsNull() || !BooleanValue::Get(enabled_val)) {
-		return config;
-	}
 
-	config.enabled = true;
-	config.hostname = ExtractString(kv, "host");
-	config.port = ExtractString(kv, "port");
-	config.username = ExtractString(kv, "user");
-	config.region = ExtractString(kv, "aws_region");
-	if (config.hostname.empty() || config.port.empty() || config.username.empty()) {
-		throw BinderException("Invalid AWS RDS IAM auth secret configuration: 'HOST', 'PORT', 'USER' and 'AWS_REGION' "
-		                      "parameters must be specified");
+	config.rds_secret_name = ExtractString(kv, "aws_rds_secret");
+	if (config.rds_secret_name.empty()) {
+		return config;
 	}
 
 	std::string password = ExtractString(kv, "password");
 	if (!password.empty()) {
 		throw BinderException("Invalid AWS RDS IAM auth secret configuration: 'PASSWORD' parameters must not be "
-		                      "specified - IAM token will be used instead of the password");
-	}
-
-	Value expiration_seconds_val = kv.TryGetValue("aws_rds_iam_token_expiration_seconds");
-	if (!expiration_seconds_val.IsNull()) {
-		config.expiration_seconds = UBigIntValue::Get(expiration_seconds_val);
+		                      "specified - generated IAM token will be used instead of the password");
 	}
 
 	// Build the base connection string (without password)

src/postgres_extension.cpp

@@ -157,12 +157,8 @@ static void LoadInternal(ExtensionLoader &loader) {
 	RegisterHstoreFunctions(loader);
 
 	// Register the new type
-	SecretType secret_type;
-	secret_type.name = "postgres";
-	secret_type.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
-	secret_type.default_provider = "config";
-
-	loader.RegisterSecretType(secret_type);
+	loader.RegisterSecretType(PostgresSecrets::CreateType());
+	loader.RegisterSecretType(PostgresSecrets::CreateRdsType());
 
 	CreateSecretFunction postgres_secret_function = {"postgres", "config", PostgresSecrets::CreateFunction};
 	PostgresSecrets::SetSecretParameters(postgres_secret_function);

src/postgres_secrets.cpp

@@ -69,9 +69,7 @@ static const std::unordered_map<std::string, std::string> connection_option_alia
 
 static const std::vector<std::string> other_option_names = {
   "uri",
-  "aws_rds_iam_auth_enabled",
-  "aws_rds_iam_token_expiration_seconds",
-  "aws_region"
+  "aws_rds_secret"
 };
 // clang-format on
 
@@ -87,6 +85,22 @@ const std::vector<std::string> &PostgresSecrets::ConnectionOptionNames() {
 	return connection_option_names;
 }
 
+SecretType PostgresSecrets::CreateType() {
+	SecretType secret_type;
+	secret_type.name = "postgres";
+	secret_type.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
+	secret_type.default_provider = "config";
+	return secret_type;
+}
+
+SecretType PostgresSecrets::CreateRdsType() {
+	SecretType secret_type;
+	secret_type.name = "rds";
+	secret_type.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
+	secret_type.default_provider = "credential_chain";
+	return secret_type;
+}
+
 unique_ptr<BaseSecret> PostgresSecrets::CreateFunction(ClientContext &context, CreateSecretInput &input) {
 	vector<string> prefix_paths;
 	auto result = make_uniq<KeyValueSecret>(prefix_paths, "postgres", "config", input.name);
@@ -113,9 +127,7 @@ void PostgresSecrets::SetSecretParameters(CreateSecretFunction &function) {
 	}
 	// other options
 	function.named_parameters["uri"] = LogicalType::VARCHAR;
-	function.named_parameters["aws_rds_iam_auth_enabled"] = LogicalType::BOOLEAN;
-	function.named_parameters["aws_rds_iam_token_expiration_seconds"] = LogicalType::UBIGINT;
-	function.named_parameters["aws_region"] = LogicalType::VARCHAR;
+	function.named_parameters["aws_rds_secret"] = LogicalType::VARCHAR;
 }
 
 } // namespace duckdb

src/storage/postgres_catalog.cpp

@@ -17,8 +17,7 @@ namespace duckdb {
 unique_ptr<SecretEntry> GetSecret(ClientContext &context, const string &secret_name) {
 	auto &secret_manager = SecretManager::Get(context);
 	auto transaction = CatalogTransaction::GetSystemCatalogTransaction(context);
-	// FIXME: this should be adjusted once the `GetSecretByName` API supports this use case
-	auto secret_entry = secret_manager.GetSecretByName(transaction, secret_name, "memory");
+	auto secret_entry = secret_manager.GetSecretByName(transaction, secret_name);
 	if (secret_entry) {
 		return secret_entry;
 	}
@@ -37,7 +36,7 @@ PostgresCatalog::PostgresCatalog(ClientContext &ctx, AttachedDatabase &db_p, str
       connection_pool(make_shared_ptr<PostgresConnectionPool>(*this, ctx)), default_schema(schema_to_load) {
 	auto secret_entry = GetSecretEntry(ctx, secret_name);
 	this->rds_token_config = PostgresAws::ExtractTokenConfigFromSecret(secret_entry);
-	if (!rds_token_config.enabled) {
+	if (!rds_token_config.Enabled()) {
 		this->connection_string = CreateConnectionString(secret_entry, attach_path);
 	}
 
@@ -103,7 +102,7 @@ string PostgresCatalog::CreateConnectionString(optional_ptr<SecretEntry> secret_
 }
 
 string PostgresCatalog::GetConnectionString() {
-	if (!rds_token_config.enabled) {
+	if (!rds_token_config.Enabled()) {
 		return connection_string;
 	}
 
@@ -116,12 +115,11 @@ string PostgresCatalog::GetConnectionString() {
 		expired = true;
 	} else {
 		int64_t age_seconds = std::chrono::duration_cast<std::chrono::seconds>(now - rds_token_last_refreshed).count();
-		int64_t max_age = static_cast<int64_t>(rds_token_config.expiration_seconds) - 60;
-		expired = age_seconds > max_age;
+		expired = age_seconds > rds_token_config.MaxAgeSeconds();
 	}
 
 	if (expired) {
-		this->rds_token = PostgresAws::GenerateRdsAuthToken(rds_token_config);
+		this->rds_token = PostgresAws::GenerateRdsAuthToken(GetAttached(), rds_token_config);
 		this->rds_token_last_refreshed = now;
 		this->connection_string = rds_token_config.base_connection_string +
 		                          "password=" + PostgresUtils::EscapeConnectionString(rds_token) + " " + attach_path;

test/sql/storage/attach_rds_iam.test

@@ -10,15 +10,32 @@ statement ok
 PRAGMA enable_verification
 
 statement ok
-CREATE TEMPORARY SECRET rds_secret (
-	TYPE POSTGRES,
-	HOST 'database-1-instance-1.xxxxxxxxxxx.eu-west-1.rds.amazonaws.com',
+LOAD postgres
+
+statement ok
+LOAD aws
+
+statement ok
+CREATE SECRET rds_template_secret (
+  TYPE rds,
+  PROVIDER credential_chain,
+  CHAIN 'env;sso;',
+  PROFILE 'DatabaseAdministrator-xxx',
+  REGION 'eu-west-1',
+  RDS_USER 'postgres',
+  RDS_HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com',
+  RDS_PORT '5432'
+);
+
+statement ok
+CREATE SECRET rds_secret (
+  TYPE postgres,
+  HOST 'database-1-instance-1.xxx.eu-west-1.rds.amazonaws.com',
   PORT '5432',
   USER 'postgres',
   DATABASE 'postgres',
-  SSLMODE 'require',
-  AWS_RDS_IAM_AUTH_ENABLED TRUE,
-  AWS_REGION 'eu-west-1'
+  SSLMODE require,
+  AWS_RDS_SECRET rds_template_secret
 );
 
 statement ok

vcpkg.json

@@ -1,15 +1,7 @@
 {
   "dependencies": [
     "openssl",
-    "libpq",
-    {
-      "name": "aws-sdk-cpp",
-      "features": [
-        "rds",
-        "sso",
-        "sts"
-      ]
-    }
+    "libpq"
   ],
   "vcpkg-configuration": {
     "overlay-ports": [