Ignoring Unwanted Resources in APM (#794)

jchrostek-dd · web-flow · commit 38e8e60f05c5 · 2025-08-29T13:14:21.000-04:00
## Task https://datadoghq.atlassian.net/browse/SVLS-6846 ## Overview We want to allow users to set filter tags which drops traces with root spans that match specified span tags. Specifically, users can set `DD_APM_FILTER_TAGS_REQUIRE` or `DD_APM_FILTER_TAGS_REJECT`. More info [here](https://docs.datadoghq.com/tracing/guide/ignoring_apm_resources/?tab=datadogyaml#trace-agent-configuration-options). ## Testing Deployed changes to Lambda. Invoked Lambda directly and through API Gateway to check with different root spans. Set the tags to either be REQUIRE or REJECT with value `name:aws.lambda`. Confirmed in logs and UI that we were dropping spans.
diff --git a/bottlecap/src/config/env.rs b/bottlecap/src/config/env.rs
@@ -10,8 +10,9 @@ use crate::{
         Config, ConfigError, ConfigSource,
         additional_endpoints::deserialize_additional_endpoints,
         apm_replace_rule::deserialize_apm_replace_rules,
-        deserialize_array_from_comma_separated_string, deserialize_key_value_pairs,
-        deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds,
+        deserialize_apm_filter_tags, deserialize_array_from_comma_separated_string,
+        deserialize_key_value_pairs, deserialize_optional_bool_from_anything,
+        deserialize_optional_duration_from_microseconds,
         deserialize_optional_duration_from_seconds, deserialize_string_or_int,
         flush_strategy::FlushStrategy,
         log_level::LogLevel,
@@ -180,6 +181,34 @@ pub struct EnvConfig {
     /// <https://docs.datadoghq.com/agent/configuration/dual-shipping/?tab=helm#environment-variable-configuration-1>
     #[serde(deserialize_with = "deserialize_additional_endpoints")]
     pub apm_additional_endpoints: HashMap<String, Vec<String>>,
+    /// @env `DD_APM_FILTER_TAGS_REQUIRE`
+    ///
+    /// Space-separated list of key:value tag pairs that spans must match to be kept.
+    /// Only spans matching at least one of these tags will be sent to Datadog.
+    /// Example: "env:production service:api-gateway"
+    #[serde(deserialize_with = "deserialize_apm_filter_tags")]
+    pub apm_filter_tags_require: Option<Vec<String>>,
+    /// @env `DD_APM_FILTER_TAGS_REJECT`
+    ///
+    /// Space-separated list of key:value tag pairs that will cause spans to be filtered out.
+    /// Spans matching any of these tags will be dropped.
+    /// Example: "env:development debug:true name:health.check"
+    #[serde(deserialize_with = "deserialize_apm_filter_tags")]
+    pub apm_filter_tags_reject: Option<Vec<String>>,
+    /// @env `DD_APM_FILTER_TAGS_REGEX_REQUIRE`
+    ///
+    /// Space-separated list of key:value tag pairs with regex values that spans must match to be kept.
+    /// Only spans matching at least one of these regex patterns will be sent to Datadog.
+    /// Example: "env:^prod.*$ service:^api-.*$"
+    #[serde(deserialize_with = "deserialize_apm_filter_tags")]
+    pub apm_filter_tags_regex_require: Option<Vec<String>>,
+    /// @env `DD_APM_FILTER_TAGS_REGEX_REJECT`
+    ///
+    /// Space-separated list of key:value tag pairs with regex values that will cause spans to be filtered out.
+    /// Spans matching any of these regex patterns will be dropped.
+    /// Example: "env:^test.*$ debug:^true$"
+    #[serde(deserialize_with = "deserialize_apm_filter_tags")]
+    pub apm_filter_tags_regex_reject: Option<Vec<String>>,
     /// @env `DD_TRACE_AWS_SERVICE_REPRESENTATION_ENABLED`
     ///
     /// Enable the new AWS-resource naming logic in the tracer.
@@ -388,6 +417,10 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
     merge_option_to_value!(config, env_config, apm_config_compression_level);
     merge_vec!(config, env_config, apm_features);
     merge_hashmap!(config, env_config, apm_additional_endpoints);
+    merge_option!(config, env_config, apm_filter_tags_require);
+    merge_option!(config, env_config, apm_filter_tags_reject);
+    merge_option!(config, env_config, apm_filter_tags_regex_require);
+    merge_option!(config, env_config, apm_filter_tags_regex_reject);
     merge_option_to_value!(config, env_config, trace_aws_service_representation_enabled);
 
     // Trace Propagation
@@ -588,6 +621,16 @@ mod tests {
                 "enable_otlp_compute_top_level_by_span_kind,enable_stats_by_span_kind",
             );
             jail.set_env("DD_APM_ADDITIONAL_ENDPOINTS", "{\"https://trace.agent.datadoghq.com\": [\"apikey2\", \"apikey3\"], \"https://trace.agent.datadoghq.eu\": [\"apikey4\"]}");
+            jail.set_env("DD_APM_FILTER_TAGS_REQUIRE", "env:production service:api");
+            jail.set_env("DD_APM_FILTER_TAGS_REJECT", "debug:true env:test");
+            jail.set_env(
+                "DD_APM_FILTER_TAGS_REGEX_REQUIRE",
+                "env:^test.*$ debug:^true$",
+            );
+            jail.set_env(
+                "DD_APM_FILTER_TAGS_REGEX_REJECT",
+                "env:^test.*$ debug:^true$",
+            );
 
             // Trace Propagation
             jail.set_env("DD_TRACE_PROPAGATION_STYLE", "datadog");
@@ -744,6 +787,22 @@ mod tests {
                         vec!["apikey4".to_string()],
                     ),
                 ]),
+                apm_filter_tags_require: Some(vec![
+                    "env:production".to_string(),
+                    "service:api".to_string(),
+                ]),
+                apm_filter_tags_reject: Some(vec![
+                    "debug:true".to_string(),
+                    "env:test".to_string(),
+                ]),
+                apm_filter_tags_regex_require: Some(vec![
+                    "env:^test.*$".to_string(),
+                    "debug:^true$".to_string(),
+                ]),
+                apm_filter_tags_regex_reject: Some(vec![
+                    "env:^test.*$".to_string(),
+                    "debug:^true$".to_string(),
+                ]),
                 trace_propagation_style: vec![TracePropagationStyle::Datadog],
                 trace_propagation_style_extract: vec![TracePropagationStyle::B3],
                 trace_propagation_extract_first: true,
diff --git a/bottlecap/src/config/mod.rs b/bottlecap/src/config/mod.rs
@@ -279,6 +279,10 @@ pub struct Config {
     pub apm_config_compression_level: i32,
     pub apm_features: Vec<String>,
     pub apm_additional_endpoints: HashMap<String, Vec<String>>,
+    pub apm_filter_tags_require: Option<Vec<String>>,
+    pub apm_filter_tags_reject: Option<Vec<String>>,
+    pub apm_filter_tags_regex_require: Option<Vec<String>>,
+    pub apm_filter_tags_regex_reject: Option<Vec<String>>,
     //
     // Trace Propagation
     pub trace_propagation_style: Vec<TracePropagationStyle>,
@@ -380,6 +384,10 @@ impl Default for Config {
             apm_config_compression_level: 6,
             apm_features: vec![],
             apm_additional_endpoints: HashMap::new(),
+            apm_filter_tags_require: None,
+            apm_filter_tags_reject: None,
+            apm_filter_tags_regex_require: None,
+            apm_filter_tags_regex_reject: None,
             trace_aws_service_representation_enabled: true,
             trace_propagation_style: vec![
                 TracePropagationStyle::Datadog,
@@ -623,6 +631,53 @@ where
     Ok(map)
 }
 
+/// Deserialize APM filter tags from space-separated "key:value" pairs, also support key-only tags
+pub fn deserialize_apm_filter_tags<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let opt: Option<String> = Option::deserialize(deserializer)?;
+
+    match opt {
+        None => Ok(None),
+        Some(s) if s.trim().is_empty() => Ok(None),
+        Some(s) => {
+            let tags: Vec<String> = s
+                .split_whitespace()
+                .filter_map(|pair| {
+                    let parts: Vec<&str> = pair.splitn(2, ':').collect();
+                    if parts.len() == 2 {
+                        let key = parts[0].trim();
+                        let value = parts[1].trim();
+                        if key.is_empty() {
+                            None
+                        } else if value.is_empty() {
+                            Some(key.to_string())
+                        } else {
+                            Some(format!("{key}:{value}"))
+                        }
+                    } else if parts.len() == 1 {
+                        let key = parts[0].trim();
+                        if key.is_empty() {
+                            None
+                        } else {
+                            Some(key.to_string())
+                        }
+                    } else {
+                        None
+                    }
+                })
+                .collect();
+
+            if tags.is_empty() {
+                Ok(None)
+            } else {
+                Ok(Some(tags))
+            }
+        }
+    }
+}
+
 pub fn deserialize_optional_duration_from_microseconds<'de, D: Deserializer<'de>>(
     deserializer: D,
 ) -> Result<Option<Duration>, D::Error> {
diff --git a/bottlecap/src/config/yaml.rs b/bottlecap/src/config/yaml.rs
@@ -915,6 +915,10 @@ extension_version: "compatibility"
                 api_security_sample_delay: Duration::from_secs(60),
 
                 extension_version: Some("compatibility".to_string()),
+                apm_filter_tags_require: None,
+                apm_filter_tags_reject: None,
+                apm_filter_tags_regex_require: None,
+                apm_filter_tags_regex_reject: None,
             };
 
             // Assert that
diff --git a/bottlecap/src/traces/trace_processor.rs b/bottlecap/src/traces/trace_processor.rs
