From 9a3907a90e3963a29ab2d03d0c58b8bcf2a005b5 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 9 Dec 2025 15:08:47 -0500 Subject: [PATCH 1/9] [SVLS-7934] feat: Support SSL certificate for trace/stats flusher --- bottlecap/Cargo.lock | 12 ++++ bottlecap/Cargo.toml | 3 + bottlecap/src/config/env.rs | 8 +++ bottlecap/src/config/mod.rs | 2 + bottlecap/src/config/yaml.rs | 6 +- bottlecap/src/traces/stats_flusher.rs | 4 +- bottlecap/src/traces/trace_flusher.rs | 92 ++++++++++++++++++++++++--- 7 files changed, 114 insertions(+), 13 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index a38c4babe..60dbad6e5 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -484,6 +484,7 @@ dependencies = [ "httpmock", "hyper 1.8.1", "hyper-http-proxy", + "hyper-rustls", "hyper-util", "indexmap 2.12.1", "itertools 0.14.0", @@ -509,6 +510,8 @@ dependencies = [ "reqwest", "rustls", "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", "serde", "serde-aux", "serde_html_form", @@ -2891,6 +2894,15 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.13.1" diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index bba3d162b..6faf879fb 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -38,6 +38,9 @@ sha2 = { version = "0.10", default-features = false } hex = { version = "0.4", default-features = false, features = ["std"] } base64 = { version = "0.22", default-features = false } rustls = { version = "0.23.18", default-features = false, features = ["aws-lc-rs"] } +rustls-pemfile = { version = "2.0", default-features = false, features = ["std"] } +rustls-pki-types = { version = "1.0", default-features = false } +hyper-rustls = { version = "0.27.7", default-features = false } rand = { version = "0.8", default-features = false } prost = { version = "0.13", default-features = false } zstd = { version = "0.13.3", default-features = false } diff --git a/bottlecap/src/config/env.rs b/bottlecap/src/config/env.rs index c4ed5cd44..51e25fc44 100644 --- a/bottlecap/src/config/env.rs +++ b/bottlecap/src/config/env.rs @@ -75,6 +75,11 @@ pub struct EnvConfig { /// The transport type to use for sending logs. Possible values are "auto" or "http1". #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, + /// @env `DD_SSL_CA_CERT` + /// The SSL CA certificate path to use for the Datadog Agent. + /// Example: `/opt/ca-cert.pem` + #[serde(deserialize_with = "deserialize_optional_string")] + pub ssl_ca_cert: Option, // Metrics /// @env `DD_DD_URL` @@ -466,6 +471,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option!(config, env_config, proxy_https); merge_vec!(config, env_config, proxy_no_proxy); merge_option!(config, env_config, http_protocol); + merge_option!(config, env_config, ssl_ca_cert); // Endpoints merge_string!(config, env_config, dd_url); @@ -695,6 +701,7 @@ mod tests { jail.set_env("DD_PROXY_HTTPS", "https://proxy.example.com"); jail.set_env("DD_PROXY_NO_PROXY", "localhost,127.0.0.1"); jail.set_env("DD_HTTP_PROTOCOL", "http1"); + jail.set_env("DD_SSL_CA_CERT", "/opt/ca-cert.pem"); // Metrics jail.set_env("DD_DD_URL", "https://metrics.datadoghq.com"); @@ -850,6 +857,7 @@ mod tests { proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), + ssl_ca_cert: Some("/opt/ca-cert.pem".to_string()), dd_url: "https://metrics.datadoghq.com".to_string(), url: "https://app.datadoghq.com".to_string(), additional_endpoints: HashMap::from([ diff --git a/bottlecap/src/config/mod.rs b/bottlecap/src/config/mod.rs index 336fcd192..1e9edfcec 100644 --- a/bottlecap/src/config/mod.rs +++ b/bottlecap/src/config/mod.rs @@ -252,6 +252,7 @@ pub struct Config { pub proxy_https: Option, pub proxy_no_proxy: Vec, pub http_protocol: Option, + pub ssl_ca_cert: Option, // Endpoints pub dd_url: String, @@ -366,6 +367,7 @@ impl Default for Config { proxy_https: None, proxy_no_proxy: vec![], http_protocol: None, + ssl_ca_cert: None, // Endpoints dd_url: String::default(), diff --git a/bottlecap/src/config/yaml.rs b/bottlecap/src/config/yaml.rs index cddaeb32a..fd2476d60 100644 --- a/bottlecap/src/config/yaml.rs +++ b/bottlecap/src/config/yaml.rs @@ -53,6 +53,8 @@ pub struct YamlConfig { pub dd_url: Option, #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, + #[serde(deserialize_with = "deserialize_optional_string")] + pub ssl_ca_cert: Option, // Endpoints #[serde(deserialize_with = "deserialize_additional_endpoints")] @@ -417,6 +419,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option!(config, proxy_https, yaml_config.proxy, https); merge_option_to_value!(config, proxy_no_proxy, yaml_config.proxy, no_proxy); merge_option!(config, yaml_config, http_protocol); + merge_option!(config, yaml_config, ssl_ca_cert); // Endpoints merge_hashmap!(config, yaml_config, additional_endpoints); @@ -746,7 +749,7 @@ proxy: https: "https://proxy.example.com" no_proxy: ["localhost", "127.0.0.1"] dd_url: "https://metrics.datadoghq.com" -http_protocol: "http1" +ssl_ca_cert: "/opt/ca-cert.pem" # Endpoints additional_endpoints: @@ -882,6 +885,7 @@ api_security_sample_delay: 60 # Seconds proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), + ssl_ca_cert: Some("/opt/ca-cert.pem".to_string()), dd_url: "https://metrics.datadoghq.com".to_string(), url: String::new(), // doesnt exist in yaml additional_endpoints: HashMap::from([ diff --git a/bottlecap/src/traces/stats_flusher.rs b/bottlecap/src/traces/stats_flusher.rs index 404f57b6b..5fb63693f 100644 --- a/bottlecap/src/traces/stats_flusher.rs +++ b/bottlecap/src/traces/stats_flusher.rs @@ -103,7 +103,7 @@ impl StatsFlusher for ServerlessStatsFlusher { let start = std::time::Instant::now(); let Ok(http_client) = - ServerlessTraceFlusher::get_http_client(self.config.proxy_https.as_ref()) + ServerlessTraceFlusher::get_http_client(self.config.proxy_https.as_ref(), self.config.ssl_ca_cert.as_ref()) else { error!("STATS_FLUSHER | Failed to create HTTP client"); return; @@ -140,4 +140,4 @@ impl StatsFlusher for ServerlessStatsFlusher { stats = guard.get_batch(force_flush).await; } } -} +} \ No newline at end of file diff --git a/bottlecap/src/traces/trace_flusher.rs b/bottlecap/src/traces/trace_flusher.rs index 81cfa10de..fd0f979b0 100644 --- a/bottlecap/src/traces/trace_flusher.rs +++ b/bottlecap/src/traces/trace_flusher.rs @@ -10,7 +10,13 @@ use libdd_trace_utils::{ send_data::SendDataBuilder, trace_utils::{self, SendData}, }; +use rustls_pki_types::CertificateDer; +use hyper_rustls::HttpsConnectorBuilder; +use rustls::{RootCertStore}; +use std::sync::LazyLock; +use std::fs::File; use std::error::Error; +use std::io::BufReader; use std::str::FromStr; use std::sync::Arc; use tokio::task::JoinSet; @@ -35,6 +41,7 @@ pub trait TraceFlusher { traces: Vec, endpoint: Option<&Endpoint>, proxy_https: &Option, + ssl_ca_cert: &Option, ) -> Option>; /// Flushes traces by getting every available batch on the aggregator. @@ -104,7 +111,7 @@ impl TraceFlusher for ServerlessTraceFlusher { "TRACES | Retrying to send {} previously failed batches", traces.len() ); - let retry_result = Self::send(traces, None, &self.config.proxy_https).await; + let retry_result = Self::send(traces, None, &self.config.proxy_https, &self.config.ssl_ca_cert).await; if retry_result.is_some() { // Still failed, return to retry later return retry_result; @@ -131,13 +138,15 @@ impl TraceFlusher for ServerlessTraceFlusher { let traces_clone = traces.clone(); let proxy_https = self.config.proxy_https.clone(); - batch_tasks.spawn(async move { Self::send(traces_clone, None, &proxy_https).await }); + let ssl_ca_cert = self.config.ssl_ca_cert.clone(); + batch_tasks.spawn(async move { Self::send(traces_clone, None, &proxy_https, &ssl_ca_cert).await }); for endpoint in self.additional_endpoints.clone() { let traces_clone = traces.clone(); let proxy_https = self.config.proxy_https.clone(); + let ssl_ca_cert = self.config.ssl_ca_cert.clone(); batch_tasks.spawn(async move { - Self::send(traces_clone, Some(&endpoint), &proxy_https).await + Self::send(traces_clone, Some(&endpoint), &proxy_https, &ssl_ca_cert).await }); } } @@ -158,6 +167,7 @@ impl TraceFlusher for ServerlessTraceFlusher { traces: Vec, endpoint: Option<&Endpoint>, proxy_https: &Option, + ssl_ca_cert: &Option, ) -> Option> { if traces.is_empty() { return None; @@ -167,7 +177,7 @@ impl TraceFlusher for ServerlessTraceFlusher { tokio::task::yield_now().await; debug!("TRACES | Flushing {} traces", coalesced_traces.len()); - let Ok(http_client) = ServerlessTraceFlusher::get_http_client(proxy_https.as_ref()) else { + let Ok(http_client) = ServerlessTraceFlusher::get_http_client(proxy_https.as_ref(), ssl_ca_cert.as_ref()) else { error!("TRACES | Failed to create HTTP client"); return None; }; @@ -195,23 +205,85 @@ impl TraceFlusher for ServerlessTraceFlusher { impl ServerlessTraceFlusher { pub fn get_http_client( proxy_https: Option<&String>, + ssl_ca_cert: Option<&String>, ) -> Result< GenericHttpClient>, Box, > { + // Create the base connector with optional custom TLS config + let connector = if let Some(ca_cert_path) = ssl_ca_cert { + // Ensure crypto provider is initialized before creating TLS config + ServerlessTraceFlusher::ensure_crypto_provider_initialized(); + + // Load the custom certificate + debug!("TRACES | Loading custom certificate from {}", ca_cert_path); + let cert_file = File::open(ca_cert_path)?; + let mut reader = BufReader::new(cert_file); + let certs: Vec = rustls_pemfile::certs(&mut reader) + .collect::, _>>()?; + + // Create a root certificate store and add custom certs + debug!("TRACES | Creating root certificate store"); + let mut root_store = RootCertStore::empty(); + for cert in certs { + root_store.add(cert)?; + } + + // Build the TLS config with custom root certificates + debug!("TRACES | Building TLS config"); + let tls_config = rustls::ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + + // Build the HTTPS connector with custom config + debug!("TRACES | Building HTTPS connector"); + let https_connector = HttpsConnectorBuilder::new() + .with_tls_config(tls_config) + .https_or_http() + .enable_http1() + .build(); + + debug!("TRACES | Added root certificate from {}", ca_cert_path); + + // Construct the Connector::Https variant directly + debug!("TRACES | Constructing Connector::Https variant"); + libdd_common::connector::Connector::Https(https_connector) + } else { + // Use default connector + libdd_common::connector::Connector::default() + }; + if let Some(proxy) = proxy_https { let proxy = hyper_http_proxy::Proxy::new(hyper_http_proxy::Intercept::Https, proxy.parse()?); let proxy_connector = hyper_http_proxy::ProxyConnector::from_proxy( - libdd_common::connector::Connector::default(), + connector, proxy, )?; - Ok(hyper_migration::client_builder().build(proxy_connector)) + debug!("TRACES | Proxy connector created with proxy: {:?}", proxy_https); + // Force HTTP/1.1 when using a proxy - many proxies don't support HTTP/2 CONNECT + let client = hyper_migration::client_builder() + .http2_only(false) // Disable HTTP/2-only mode to allow HTTP/1.1 + .build(proxy_connector); + debug!("TRACES | Client configured to allow HTTP/1.1 for proxy compatibility"); + Ok(client) } else { - let proxy_connector = hyper_http_proxy::ProxyConnector::new( - libdd_common::connector::Connector::default(), - )?; + let proxy_connector = hyper_http_proxy::ProxyConnector::new(connector)?; + debug!("TRACES | Proxy connector created without proxy"); Ok(hyper_migration::client_builder().build(proxy_connector)) } } -} + + /// Ensure the rustls crypto provider is initialized. + /// This is required before creating custom TLS configurations. + fn ensure_crypto_provider_initialized() { + static INIT_CRYPTO_PROVIDER: LazyLock<()> = LazyLock::new(|| { + #[cfg(unix)] + rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .expect("Failed to install default CryptoProvider"); + }); + + let () = &*INIT_CRYPTO_PROVIDER; + } +} \ No newline at end of file From c70697f261e455af41bfd8017d5e30703e8d5053 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 9 Dec 2025 16:48:53 -0500 Subject: [PATCH 2/9] Move ensure_crypto_provider_initialized() outside --- bottlecap/src/traces/trace_flusher.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/bottlecap/src/traces/trace_flusher.rs b/bottlecap/src/traces/trace_flusher.rs index fd0f979b0..97cea8d10 100644 --- a/bottlecap/src/traces/trace_flusher.rs +++ b/bottlecap/src/traces/trace_flusher.rs @@ -202,6 +202,17 @@ impl TraceFlusher for ServerlessTraceFlusher { } } +fn ensure_crypto_provider_initialized() { + static INIT_CRYPTO_PROVIDER: LazyLock<()> = LazyLock::new(|| { + #[cfg(unix)] + rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .expect("Failed to install default CryptoProvider"); + }); + + let () = &*INIT_CRYPTO_PROVIDER; +} + impl ServerlessTraceFlusher { pub fn get_http_client( proxy_https: Option<&String>, @@ -213,7 +224,7 @@ impl ServerlessTraceFlusher { // Create the base connector with optional custom TLS config let connector = if let Some(ca_cert_path) = ssl_ca_cert { // Ensure crypto provider is initialized before creating TLS config - ServerlessTraceFlusher::ensure_crypto_provider_initialized(); + ensure_crypto_provider_initialized(); // Load the custom certificate debug!("TRACES | Loading custom certificate from {}", ca_cert_path); @@ -273,17 +284,4 @@ impl ServerlessTraceFlusher { Ok(hyper_migration::client_builder().build(proxy_connector)) } } - - /// Ensure the rustls crypto provider is initialized. - /// This is required before creating custom TLS configurations. - fn ensure_crypto_provider_initialized() { - static INIT_CRYPTO_PROVIDER: LazyLock<()> = LazyLock::new(|| { - #[cfg(unix)] - rustls::crypto::aws_lc_rs::default_provider() - .install_default() - .expect("Failed to install default CryptoProvider"); - }); - - let () = &*INIT_CRYPTO_PROVIDER; - } } \ No newline at end of file From f20ade8d1c1a5bc14a7cdbf699edbe0656a59a7f Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 9 Dec 2025 17:12:20 -0500 Subject: [PATCH 3/9] Remove http2_only=false --- bottlecap/src/config/yaml.rs | 1 + bottlecap/src/traces/trace_flusher.rs | 11 ++--------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/bottlecap/src/config/yaml.rs b/bottlecap/src/config/yaml.rs index fd2476d60..53f1d2102 100644 --- a/bottlecap/src/config/yaml.rs +++ b/bottlecap/src/config/yaml.rs @@ -749,6 +749,7 @@ proxy: https: "https://proxy.example.com" no_proxy: ["localhost", "127.0.0.1"] dd_url: "https://metrics.datadoghq.com" +http_protocol: "http1" ssl_ca_cert: "/opt/ca-cert.pem" # Endpoints diff --git a/bottlecap/src/traces/trace_flusher.rs b/bottlecap/src/traces/trace_flusher.rs index 97cea8d10..72ac9d724 100644 --- a/bottlecap/src/traces/trace_flusher.rs +++ b/bottlecap/src/traces/trace_flusher.rs @@ -227,27 +227,23 @@ impl ServerlessTraceFlusher { ensure_crypto_provider_initialized(); // Load the custom certificate - debug!("TRACES | Loading custom certificate from {}", ca_cert_path); let cert_file = File::open(ca_cert_path)?; let mut reader = BufReader::new(cert_file); let certs: Vec = rustls_pemfile::certs(&mut reader) .collect::, _>>()?; // Create a root certificate store and add custom certs - debug!("TRACES | Creating root certificate store"); let mut root_store = RootCertStore::empty(); for cert in certs { root_store.add(cert)?; } // Build the TLS config with custom root certificates - debug!("TRACES | Building TLS config"); let tls_config = rustls::ClientConfig::builder() .with_root_certificates(root_store) .with_no_client_auth(); // Build the HTTPS connector with custom config - debug!("TRACES | Building HTTPS connector"); let https_connector = HttpsConnectorBuilder::new() .with_tls_config(tls_config) .https_or_http() @@ -257,7 +253,6 @@ impl ServerlessTraceFlusher { debug!("TRACES | Added root certificate from {}", ca_cert_path); // Construct the Connector::Https variant directly - debug!("TRACES | Constructing Connector::Https variant"); libdd_common::connector::Connector::Https(https_connector) } else { // Use default connector @@ -271,16 +266,14 @@ impl ServerlessTraceFlusher { connector, proxy, )?; - debug!("TRACES | Proxy connector created with proxy: {:?}", proxy_https); // Force HTTP/1.1 when using a proxy - many proxies don't support HTTP/2 CONNECT let client = hyper_migration::client_builder() - .http2_only(false) // Disable HTTP/2-only mode to allow HTTP/1.1 + // .http2_only(false) // Disable HTTP/2-only mode to allow HTTP/1.1 .build(proxy_connector); - debug!("TRACES | Client configured to allow HTTP/1.1 for proxy compatibility"); + debug!("TRACES | Proxy connector created with proxy: {:?}", proxy_https); Ok(client) } else { let proxy_connector = hyper_http_proxy::ProxyConnector::new(connector)?; - debug!("TRACES | Proxy connector created without proxy"); Ok(hyper_migration::client_builder().build(proxy_connector)) } } From d5c6a8e50cf34ae1c5ada97fa719fa032feb4700 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 9 Dec 2025 17:15:07 -0500 Subject: [PATCH 4/9] fmt --- bottlecap/src/traces/stats_flusher.rs | 9 +++--- bottlecap/src/traces/trace_flusher.rs | 45 ++++++++++++++++----------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/bottlecap/src/traces/stats_flusher.rs b/bottlecap/src/traces/stats_flusher.rs index 5fb63693f..232ef2348 100644 --- a/bottlecap/src/traces/stats_flusher.rs +++ b/bottlecap/src/traces/stats_flusher.rs @@ -102,9 +102,10 @@ impl StatsFlusher for ServerlessStatsFlusher { let start = std::time::Instant::now(); - let Ok(http_client) = - ServerlessTraceFlusher::get_http_client(self.config.proxy_https.as_ref(), self.config.ssl_ca_cert.as_ref()) - else { + let Ok(http_client) = ServerlessTraceFlusher::get_http_client( + self.config.proxy_https.as_ref(), + self.config.ssl_ca_cert.as_ref(), + ) else { error!("STATS_FLUSHER | Failed to create HTTP client"); return; }; @@ -140,4 +141,4 @@ impl StatsFlusher for ServerlessStatsFlusher { stats = guard.get_batch(force_flush).await; } } -} \ No newline at end of file +} diff --git a/bottlecap/src/traces/trace_flusher.rs b/bottlecap/src/traces/trace_flusher.rs index 72ac9d724..fe175c2e3 100644 --- a/bottlecap/src/traces/trace_flusher.rs +++ b/bottlecap/src/traces/trace_flusher.rs @@ -4,21 +4,21 @@ use async_trait::async_trait; use dogstatsd::api_key::ApiKeyFactory; use hyper_http_proxy; +use hyper_rustls::HttpsConnectorBuilder; use libdd_common::{Endpoint, GenericHttpClient, hyper_migration}; use libdd_trace_utils::{ config_utils::trace_intake_url_prefixed, send_data::SendDataBuilder, trace_utils::{self, SendData}, }; +use rustls::RootCertStore; use rustls_pki_types::CertificateDer; -use hyper_rustls::HttpsConnectorBuilder; -use rustls::{RootCertStore}; -use std::sync::LazyLock; -use std::fs::File; use std::error::Error; +use std::fs::File; use std::io::BufReader; use std::str::FromStr; use std::sync::Arc; +use std::sync::LazyLock; use tokio::task::JoinSet; use tracing::{debug, error}; @@ -111,7 +111,13 @@ impl TraceFlusher for ServerlessTraceFlusher { "TRACES | Retrying to send {} previously failed batches", traces.len() ); - let retry_result = Self::send(traces, None, &self.config.proxy_https, &self.config.ssl_ca_cert).await; + let retry_result = Self::send( + traces, + None, + &self.config.proxy_https, + &self.config.ssl_ca_cert, + ) + .await; if retry_result.is_some() { // Still failed, return to retry later return retry_result; @@ -139,7 +145,9 @@ impl TraceFlusher for ServerlessTraceFlusher { let traces_clone = traces.clone(); let proxy_https = self.config.proxy_https.clone(); let ssl_ca_cert = self.config.ssl_ca_cert.clone(); - batch_tasks.spawn(async move { Self::send(traces_clone, None, &proxy_https, &ssl_ca_cert).await }); + batch_tasks.spawn(async move { + Self::send(traces_clone, None, &proxy_https, &ssl_ca_cert).await + }); for endpoint in self.additional_endpoints.clone() { let traces_clone = traces.clone(); @@ -177,7 +185,9 @@ impl TraceFlusher for ServerlessTraceFlusher { tokio::task::yield_now().await; debug!("TRACES | Flushing {} traces", coalesced_traces.len()); - let Ok(http_client) = ServerlessTraceFlusher::get_http_client(proxy_https.as_ref(), ssl_ca_cert.as_ref()) else { + let Ok(http_client) = + ServerlessTraceFlusher::get_http_client(proxy_https.as_ref(), ssl_ca_cert.as_ref()) + else { error!("TRACES | Failed to create HTTP client"); return None; }; @@ -229,8 +239,8 @@ impl ServerlessTraceFlusher { // Load the custom certificate let cert_file = File::open(ca_cert_path)?; let mut reader = BufReader::new(cert_file); - let certs: Vec = rustls_pemfile::certs(&mut reader) - .collect::, _>>()?; + let certs: Vec = + rustls_pemfile::certs(&mut reader).collect::, _>>()?; // Create a root certificate store and add custom certs let mut root_store = RootCertStore::empty(); @@ -262,19 +272,16 @@ impl ServerlessTraceFlusher { if let Some(proxy) = proxy_https { let proxy = hyper_http_proxy::Proxy::new(hyper_http_proxy::Intercept::Https, proxy.parse()?); - let proxy_connector = hyper_http_proxy::ProxyConnector::from_proxy( - connector, - proxy, - )?; - // Force HTTP/1.1 when using a proxy - many proxies don't support HTTP/2 CONNECT - let client = hyper_migration::client_builder() - // .http2_only(false) // Disable HTTP/2-only mode to allow HTTP/1.1 - .build(proxy_connector); - debug!("TRACES | Proxy connector created with proxy: {:?}", proxy_https); + let proxy_connector = hyper_http_proxy::ProxyConnector::from_proxy(connector, proxy)?; + let client = hyper_migration::client_builder().build(proxy_connector); + debug!( + "TRACES | Proxy connector created with proxy: {:?}", + proxy_https + ); Ok(client) } else { let proxy_connector = hyper_http_proxy::ProxyConnector::new(connector)?; Ok(hyper_migration::client_builder().build(proxy_connector)) } } -} \ No newline at end of file +} From e3ab60e5860a892885d840441642a5d6e0c12618 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 9 Dec 2025 17:21:04 -0500 Subject: [PATCH 5/9] Change comments --- bottlecap/src/traces/trace_flusher.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bottlecap/src/traces/trace_flusher.rs b/bottlecap/src/traces/trace_flusher.rs index fe175c2e3..6cfc2a7d5 100644 --- a/bottlecap/src/traces/trace_flusher.rs +++ b/bottlecap/src/traces/trace_flusher.rs @@ -212,6 +212,7 @@ impl TraceFlusher for ServerlessTraceFlusher { } } +// Initialize the crypto provider needed for setting custom root certificates fn ensure_crypto_provider_initialized() { static INIT_CRYPTO_PROVIDER: LazyLock<()> = LazyLock::new(|| { #[cfg(unix)] @@ -260,7 +261,7 @@ impl ServerlessTraceFlusher { .enable_http1() .build(); - debug!("TRACES | Added root certificate from {}", ca_cert_path); + debug!("TRACES | GET_HTTP_CLIENT | Added root certificate from {}", ca_cert_path); // Construct the Connector::Https variant directly libdd_common::connector::Connector::Https(https_connector) @@ -275,7 +276,7 @@ impl ServerlessTraceFlusher { let proxy_connector = hyper_http_proxy::ProxyConnector::from_proxy(connector, proxy)?; let client = hyper_migration::client_builder().build(proxy_connector); debug!( - "TRACES | Proxy connector created with proxy: {:?}", + "TRACES | GET_HTTP_CLIENT | Proxy connector created with proxy: {:?}", proxy_https ); Ok(client) From 26e9b5722a832951aa071bfffc78919af33cdafc Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 9 Dec 2025 17:22:15 -0500 Subject: [PATCH 6/9] fmt --- bottlecap/src/traces/trace_flusher.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bottlecap/src/traces/trace_flusher.rs b/bottlecap/src/traces/trace_flusher.rs index 6cfc2a7d5..4ffb9f633 100644 --- a/bottlecap/src/traces/trace_flusher.rs +++ b/bottlecap/src/traces/trace_flusher.rs @@ -261,7 +261,10 @@ impl ServerlessTraceFlusher { .enable_http1() .build(); - debug!("TRACES | GET_HTTP_CLIENT | Added root certificate from {}", ca_cert_path); + debug!( + "TRACES | GET_HTTP_CLIENT | Added root certificate from {}", + ca_cert_path + ); // Construct the Connector::Https variant directly libdd_common::connector::Connector::Https(https_connector) From fcf63ac24313779b96f3ef7c38c68e7a345ada44 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 9 Dec 2025 21:01:00 -0500 Subject: [PATCH 7/9] Update license --- bottlecap/LICENSE-3rdparty.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/bottlecap/LICENSE-3rdparty.csv b/bottlecap/LICENSE-3rdparty.csv index 1e02a120c..f4fc046d8 100644 --- a/bottlecap/LICENSE-3rdparty.csv +++ b/bottlecap/LICENSE-3rdparty.csv @@ -173,6 +173,7 @@ rustc-hash,https://github.com/rust-lang/rustc-hash,Apache-2.0 OR MIT,The Rust Pr rustix,https://github.com/bytecodealliance/rustix,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,"Dan Gohman , Jakub Konka " rustls,https://github.com/rustls/rustls,Apache-2.0 OR ISC OR MIT,The rustls Authors rustls-native-certs,https://github.com/rustls/rustls-native-certs,Apache-2.0 OR ISC OR MIT,The rustls-native-certs Authors +rustls-pemfile,https://github.com/rustls/pemfile,Apache-2.0 OR ISC OR MIT,The rustls-pemfile Authors rustls-pki-types,https://github.com/rustls/pki-types,MIT OR Apache-2.0,The rustls-pki-types Authors rustls-webpki,https://github.com/rustls/webpki,ISC,The rustls-webpki Authors ryu,https://github.com/dtolnay/ryu,Apache-2.0 OR BSL-1.0,David Tolnay From ec33d198289f94a01e0a3d39b67be25574431460 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 10 Dec 2025 14:45:53 -0500 Subject: [PATCH 8/9] Rename: ssl_ca_cert -> tls_ca_cert --- bottlecap/src/config/env.rs | 12 ++++++------ bottlecap/src/config/mod.rs | 4 ++-- bottlecap/src/config/yaml.rs | 8 ++++---- bottlecap/src/traces/stats_flusher.rs | 2 +- bottlecap/src/traces/trace_flusher.rs | 20 ++++++++++---------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/bottlecap/src/config/env.rs b/bottlecap/src/config/env.rs index 51e25fc44..bfe2b6400 100644 --- a/bottlecap/src/config/env.rs +++ b/bottlecap/src/config/env.rs @@ -75,11 +75,11 @@ pub struct EnvConfig { /// The transport type to use for sending logs. Possible values are "auto" or "http1". #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, - /// @env `DD_SSL_CA_CERT` - /// The SSL CA certificate path to use for the Datadog Agent. + /// @env `DD_CLS_CA_CERT` + /// The path to a file of concatenated CA certificates in PEM format. /// Example: `/opt/ca-cert.pem` #[serde(deserialize_with = "deserialize_optional_string")] - pub ssl_ca_cert: Option, + pub cls_ca_cert: Option, // Metrics /// @env `DD_DD_URL` @@ -471,7 +471,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option!(config, env_config, proxy_https); merge_vec!(config, env_config, proxy_no_proxy); merge_option!(config, env_config, http_protocol); - merge_option!(config, env_config, ssl_ca_cert); + merge_option!(config, env_config, cls_ca_cert); // Endpoints merge_string!(config, env_config, dd_url); @@ -701,7 +701,7 @@ mod tests { jail.set_env("DD_PROXY_HTTPS", "https://proxy.example.com"); jail.set_env("DD_PROXY_NO_PROXY", "localhost,127.0.0.1"); jail.set_env("DD_HTTP_PROTOCOL", "http1"); - jail.set_env("DD_SSL_CA_CERT", "/opt/ca-cert.pem"); + jail.set_env("DD_CLS_CA_CERT", "/opt/ca-cert.pem"); // Metrics jail.set_env("DD_DD_URL", "https://metrics.datadoghq.com"); @@ -857,7 +857,7 @@ mod tests { proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), - ssl_ca_cert: Some("/opt/ca-cert.pem".to_string()), + cls_ca_cert: Some("/opt/ca-cert.pem".to_string()), dd_url: "https://metrics.datadoghq.com".to_string(), url: "https://app.datadoghq.com".to_string(), additional_endpoints: HashMap::from([ diff --git a/bottlecap/src/config/mod.rs b/bottlecap/src/config/mod.rs index 1e9edfcec..f302a3f8d 100644 --- a/bottlecap/src/config/mod.rs +++ b/bottlecap/src/config/mod.rs @@ -252,7 +252,7 @@ pub struct Config { pub proxy_https: Option, pub proxy_no_proxy: Vec, pub http_protocol: Option, - pub ssl_ca_cert: Option, + pub cls_ca_cert: Option, // Endpoints pub dd_url: String, @@ -367,7 +367,7 @@ impl Default for Config { proxy_https: None, proxy_no_proxy: vec![], http_protocol: None, - ssl_ca_cert: None, + cls_ca_cert: None, // Endpoints dd_url: String::default(), diff --git a/bottlecap/src/config/yaml.rs b/bottlecap/src/config/yaml.rs index 53f1d2102..74124c59e 100644 --- a/bottlecap/src/config/yaml.rs +++ b/bottlecap/src/config/yaml.rs @@ -54,7 +54,7 @@ pub struct YamlConfig { #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, #[serde(deserialize_with = "deserialize_optional_string")] - pub ssl_ca_cert: Option, + pub cls_ca_cert: Option, // Endpoints #[serde(deserialize_with = "deserialize_additional_endpoints")] @@ -419,7 +419,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option!(config, proxy_https, yaml_config.proxy, https); merge_option_to_value!(config, proxy_no_proxy, yaml_config.proxy, no_proxy); merge_option!(config, yaml_config, http_protocol); - merge_option!(config, yaml_config, ssl_ca_cert); + merge_option!(config, yaml_config, cls_ca_cert); // Endpoints merge_hashmap!(config, yaml_config, additional_endpoints); @@ -750,7 +750,7 @@ proxy: no_proxy: ["localhost", "127.0.0.1"] dd_url: "https://metrics.datadoghq.com" http_protocol: "http1" -ssl_ca_cert: "/opt/ca-cert.pem" +cls_ca_cert: "/opt/ca-cert.pem" # Endpoints additional_endpoints: @@ -886,7 +886,7 @@ api_security_sample_delay: 60 # Seconds proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), - ssl_ca_cert: Some("/opt/ca-cert.pem".to_string()), + cls_ca_cert: Some("/opt/ca-cert.pem".to_string()), dd_url: "https://metrics.datadoghq.com".to_string(), url: String::new(), // doesnt exist in yaml additional_endpoints: HashMap::from([ diff --git a/bottlecap/src/traces/stats_flusher.rs b/bottlecap/src/traces/stats_flusher.rs index 232ef2348..6a569d6f1 100644 --- a/bottlecap/src/traces/stats_flusher.rs +++ b/bottlecap/src/traces/stats_flusher.rs @@ -104,7 +104,7 @@ impl StatsFlusher for ServerlessStatsFlusher { let Ok(http_client) = ServerlessTraceFlusher::get_http_client( self.config.proxy_https.as_ref(), - self.config.ssl_ca_cert.as_ref(), + self.config.cls_ca_cert.as_ref(), ) else { error!("STATS_FLUSHER | Failed to create HTTP client"); return; diff --git a/bottlecap/src/traces/trace_flusher.rs b/bottlecap/src/traces/trace_flusher.rs index 4ffb9f633..bec703e69 100644 --- a/bottlecap/src/traces/trace_flusher.rs +++ b/bottlecap/src/traces/trace_flusher.rs @@ -41,7 +41,7 @@ pub trait TraceFlusher { traces: Vec, endpoint: Option<&Endpoint>, proxy_https: &Option, - ssl_ca_cert: &Option, + cls_ca_cert: &Option, ) -> Option>; /// Flushes traces by getting every available batch on the aggregator. @@ -115,7 +115,7 @@ impl TraceFlusher for ServerlessTraceFlusher { traces, None, &self.config.proxy_https, - &self.config.ssl_ca_cert, + &self.config.cls_ca_cert, ) .await; if retry_result.is_some() { @@ -144,17 +144,17 @@ impl TraceFlusher for ServerlessTraceFlusher { let traces_clone = traces.clone(); let proxy_https = self.config.proxy_https.clone(); - let ssl_ca_cert = self.config.ssl_ca_cert.clone(); + let cls_ca_cert = self.config.cls_ca_cert.clone(); batch_tasks.spawn(async move { - Self::send(traces_clone, None, &proxy_https, &ssl_ca_cert).await + Self::send(traces_clone, None, &proxy_https, &cls_ca_cert).await }); for endpoint in self.additional_endpoints.clone() { let traces_clone = traces.clone(); let proxy_https = self.config.proxy_https.clone(); - let ssl_ca_cert = self.config.ssl_ca_cert.clone(); + let cls_ca_cert = self.config.cls_ca_cert.clone(); batch_tasks.spawn(async move { - Self::send(traces_clone, Some(&endpoint), &proxy_https, &ssl_ca_cert).await + Self::send(traces_clone, Some(&endpoint), &proxy_https, &cls_ca_cert).await }); } } @@ -175,7 +175,7 @@ impl TraceFlusher for ServerlessTraceFlusher { traces: Vec, endpoint: Option<&Endpoint>, proxy_https: &Option, - ssl_ca_cert: &Option, + cls_ca_cert: &Option, ) -> Option> { if traces.is_empty() { return None; @@ -186,7 +186,7 @@ impl TraceFlusher for ServerlessTraceFlusher { debug!("TRACES | Flushing {} traces", coalesced_traces.len()); let Ok(http_client) = - ServerlessTraceFlusher::get_http_client(proxy_https.as_ref(), ssl_ca_cert.as_ref()) + ServerlessTraceFlusher::get_http_client(proxy_https.as_ref(), cls_ca_cert.as_ref()) else { error!("TRACES | Failed to create HTTP client"); return None; @@ -227,13 +227,13 @@ fn ensure_crypto_provider_initialized() { impl ServerlessTraceFlusher { pub fn get_http_client( proxy_https: Option<&String>, - ssl_ca_cert: Option<&String>, + cls_ca_cert: Option<&String>, ) -> Result< GenericHttpClient>, Box, > { // Create the base connector with optional custom TLS config - let connector = if let Some(ca_cert_path) = ssl_ca_cert { + let connector = if let Some(ca_cert_path) = cls_ca_cert { // Ensure crypto provider is initialized before creating TLS config ensure_crypto_provider_initialized(); From ae688e07e8c840f52631231f7147f21ac90ec2a8 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 10 Dec 2025 15:26:25 -0500 Subject: [PATCH 9/9] Rename as DD_TLS_CERT_FILE --- bottlecap/src/config/env.rs | 10 +++++----- bottlecap/src/config/mod.rs | 4 ++-- bottlecap/src/config/yaml.rs | 8 ++++---- bottlecap/src/traces/stats_flusher.rs | 2 +- bottlecap/src/traces/trace_flusher.rs | 20 ++++++++++---------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/bottlecap/src/config/env.rs b/bottlecap/src/config/env.rs index bfe2b6400..04d4e27e7 100644 --- a/bottlecap/src/config/env.rs +++ b/bottlecap/src/config/env.rs @@ -75,11 +75,11 @@ pub struct EnvConfig { /// The transport type to use for sending logs. Possible values are "auto" or "http1". #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, - /// @env `DD_CLS_CA_CERT` + /// @env `DD_TLS_CERT_FILE` /// The path to a file of concatenated CA certificates in PEM format. /// Example: `/opt/ca-cert.pem` #[serde(deserialize_with = "deserialize_optional_string")] - pub cls_ca_cert: Option, + pub tls_cert_file: Option, // Metrics /// @env `DD_DD_URL` @@ -471,7 +471,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option!(config, env_config, proxy_https); merge_vec!(config, env_config, proxy_no_proxy); merge_option!(config, env_config, http_protocol); - merge_option!(config, env_config, cls_ca_cert); + merge_option!(config, env_config, tls_cert_file); // Endpoints merge_string!(config, env_config, dd_url); @@ -701,7 +701,7 @@ mod tests { jail.set_env("DD_PROXY_HTTPS", "https://proxy.example.com"); jail.set_env("DD_PROXY_NO_PROXY", "localhost,127.0.0.1"); jail.set_env("DD_HTTP_PROTOCOL", "http1"); - jail.set_env("DD_CLS_CA_CERT", "/opt/ca-cert.pem"); + jail.set_env("DD_TLS_CERT_FILE", "/opt/ca-cert.pem"); // Metrics jail.set_env("DD_DD_URL", "https://metrics.datadoghq.com"); @@ -857,7 +857,7 @@ mod tests { proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), - cls_ca_cert: Some("/opt/ca-cert.pem".to_string()), + tls_cert_file: Some("/opt/ca-cert.pem".to_string()), dd_url: "https://metrics.datadoghq.com".to_string(), url: "https://app.datadoghq.com".to_string(), additional_endpoints: HashMap::from([ diff --git a/bottlecap/src/config/mod.rs b/bottlecap/src/config/mod.rs index f302a3f8d..9c1ecdd4e 100644 --- a/bottlecap/src/config/mod.rs +++ b/bottlecap/src/config/mod.rs @@ -252,7 +252,7 @@ pub struct Config { pub proxy_https: Option, pub proxy_no_proxy: Vec, pub http_protocol: Option, - pub cls_ca_cert: Option, + pub tls_cert_file: Option, // Endpoints pub dd_url: String, @@ -367,7 +367,7 @@ impl Default for Config { proxy_https: None, proxy_no_proxy: vec![], http_protocol: None, - cls_ca_cert: None, + tls_cert_file: None, // Endpoints dd_url: String::default(), diff --git a/bottlecap/src/config/yaml.rs b/bottlecap/src/config/yaml.rs index 74124c59e..5dbc95367 100644 --- a/bottlecap/src/config/yaml.rs +++ b/bottlecap/src/config/yaml.rs @@ -54,7 +54,7 @@ pub struct YamlConfig { #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, #[serde(deserialize_with = "deserialize_optional_string")] - pub cls_ca_cert: Option, + pub tls_cert_file: Option, // Endpoints #[serde(deserialize_with = "deserialize_additional_endpoints")] @@ -419,7 +419,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option!(config, proxy_https, yaml_config.proxy, https); merge_option_to_value!(config, proxy_no_proxy, yaml_config.proxy, no_proxy); merge_option!(config, yaml_config, http_protocol); - merge_option!(config, yaml_config, cls_ca_cert); + merge_option!(config, yaml_config, tls_cert_file); // Endpoints merge_hashmap!(config, yaml_config, additional_endpoints); @@ -750,7 +750,7 @@ proxy: no_proxy: ["localhost", "127.0.0.1"] dd_url: "https://metrics.datadoghq.com" http_protocol: "http1" -cls_ca_cert: "/opt/ca-cert.pem" +tls_cert_file: "/opt/ca-cert.pem" # Endpoints additional_endpoints: @@ -886,7 +886,7 @@ api_security_sample_delay: 60 # Seconds proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), - cls_ca_cert: Some("/opt/ca-cert.pem".to_string()), + tls_cert_file: Some("/opt/ca-cert.pem".to_string()), dd_url: "https://metrics.datadoghq.com".to_string(), url: String::new(), // doesnt exist in yaml additional_endpoints: HashMap::from([ diff --git a/bottlecap/src/traces/stats_flusher.rs b/bottlecap/src/traces/stats_flusher.rs index 6a569d6f1..7f2a0d998 100644 --- a/bottlecap/src/traces/stats_flusher.rs +++ b/bottlecap/src/traces/stats_flusher.rs @@ -104,7 +104,7 @@ impl StatsFlusher for ServerlessStatsFlusher { let Ok(http_client) = ServerlessTraceFlusher::get_http_client( self.config.proxy_https.as_ref(), - self.config.cls_ca_cert.as_ref(), + self.config.tls_cert_file.as_ref(), ) else { error!("STATS_FLUSHER | Failed to create HTTP client"); return; diff --git a/bottlecap/src/traces/trace_flusher.rs b/bottlecap/src/traces/trace_flusher.rs index bec703e69..6d88a12d2 100644 --- a/bottlecap/src/traces/trace_flusher.rs +++ b/bottlecap/src/traces/trace_flusher.rs @@ -41,7 +41,7 @@ pub trait TraceFlusher { traces: Vec, endpoint: Option<&Endpoint>, proxy_https: &Option, - cls_ca_cert: &Option, + tls_cert_file: &Option, ) -> Option>; /// Flushes traces by getting every available batch on the aggregator. @@ -115,7 +115,7 @@ impl TraceFlusher for ServerlessTraceFlusher { traces, None, &self.config.proxy_https, - &self.config.cls_ca_cert, + &self.config.tls_cert_file, ) .await; if retry_result.is_some() { @@ -144,17 +144,17 @@ impl TraceFlusher for ServerlessTraceFlusher { let traces_clone = traces.clone(); let proxy_https = self.config.proxy_https.clone(); - let cls_ca_cert = self.config.cls_ca_cert.clone(); + let tls_cert_file = self.config.tls_cert_file.clone(); batch_tasks.spawn(async move { - Self::send(traces_clone, None, &proxy_https, &cls_ca_cert).await + Self::send(traces_clone, None, &proxy_https, &tls_cert_file).await }); for endpoint in self.additional_endpoints.clone() { let traces_clone = traces.clone(); let proxy_https = self.config.proxy_https.clone(); - let cls_ca_cert = self.config.cls_ca_cert.clone(); + let tls_cert_file = self.config.tls_cert_file.clone(); batch_tasks.spawn(async move { - Self::send(traces_clone, Some(&endpoint), &proxy_https, &cls_ca_cert).await + Self::send(traces_clone, Some(&endpoint), &proxy_https, &tls_cert_file).await }); } } @@ -175,7 +175,7 @@ impl TraceFlusher for ServerlessTraceFlusher { traces: Vec, endpoint: Option<&Endpoint>, proxy_https: &Option, - cls_ca_cert: &Option, + tls_cert_file: &Option, ) -> Option> { if traces.is_empty() { return None; @@ -186,7 +186,7 @@ impl TraceFlusher for ServerlessTraceFlusher { debug!("TRACES | Flushing {} traces", coalesced_traces.len()); let Ok(http_client) = - ServerlessTraceFlusher::get_http_client(proxy_https.as_ref(), cls_ca_cert.as_ref()) + ServerlessTraceFlusher::get_http_client(proxy_https.as_ref(), tls_cert_file.as_ref()) else { error!("TRACES | Failed to create HTTP client"); return None; @@ -227,13 +227,13 @@ fn ensure_crypto_provider_initialized() { impl ServerlessTraceFlusher { pub fn get_http_client( proxy_https: Option<&String>, - cls_ca_cert: Option<&String>, + tls_cert_file: Option<&String>, ) -> Result< GenericHttpClient>, Box, > { // Create the base connector with optional custom TLS config - let connector = if let Some(ca_cert_path) = cls_ca_cert { + let connector = if let Some(ca_cert_path) = tls_cert_file { // Ensure crypto provider is initialized before creating TLS config ensure_crypto_provider_initialized();