Skip to content

feat: [Trace Stats] Use the concentrator in libdatadog #887

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Oct 16, 2025
Merged

feat: [Trace Stats] Use the concentrator in libdatadog #887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
4acefba
Use the stats concentrator in libdatadog
lym953 Oct 1, 2025
f4a3b66
Delete stats_concentrator.rs
lym953 Oct 1, 2025
2760507
Make StatsConcentratorService store tracer metadata
lym953 Oct 1, 2025
43469ff
Pass in config and tags_provider again
lym953 Oct 1, 2025
0ed5ccb
fmt
lym953 Oct 1, 2025
43bc5d5
Make concentrator service return None if no stats
lym953 Oct 1, 2025
ed44d75
Upgrade libdatadog
lym953 Oct 3, 2025
08c08a7
Organize code
lym953 Oct 3, 2025
e7bc0da
Fix tests
lym953 Oct 3, 2025
9123cd1
Update license
lym953 Oct 3, 2025
1998ba8
Update stats aggregator test
lym953 Oct 3, 2025
c4b7fb3
Merge branch 'main' into yiming.luo/trace-stats-14
lym953 Oct 3, 2025
8a483c5
Merge branch 'main' into yiming.luo/trace-stats-14
lym953 Oct 16, 2025
dd44d9b
Fix package name
lym953 Oct 16, 2025
5396cbb
Update LICENSE-3rdparty.csv
lym953 Oct 16, 2025
50bd2c2
Add comment for box
lym953 Oct 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 54 additions & 26 deletions bottlecap/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 6 additions & 5 deletions bottlecap/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,12 @@ ustr = { version = "1.0.0", default-features = false }
# be found in the clippy.toml file adjacent to this Cargo.toml.
datadog-protos = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222"}
ddsketch-agent = { version = "0.1.0", default-features = false, git = "https://github.com/DataDog/saluki/", rev = "c89b58e5784b985819baf11f13f7d35876741222"}
ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "9405db9cb4ef733f3954c3ee77ce71a502e98e50" }
datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "9405db9cb4ef733f3954c3ee77ce71a502e98e50" }
datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "9405db9cb4ef733f3954c3ee77ce71a502e98e50" , features = ["mini_agent"] }
datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "9405db9cb4ef733f3954c3ee77ce71a502e98e50" }
datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "9405db9cb4ef733f3954c3ee77ce71a502e98e50" }
ddcommon = { git = "https://github.com/DataDog/libdatadog", rev = "ba8955394cf35cf24a1a508fbe6264ad84702567" }
datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "ba8955394cf35cf24a1a508fbe6264ad84702567" }
datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "ba8955394cf35cf24a1a508fbe6264ad84702567" , features = ["mini_agent"] }
datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev = "ba8955394cf35cf24a1a508fbe6264ad84702567" }
datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "ba8955394cf35cf24a1a508fbe6264ad84702567" }
datadog-trace-stats = { git = "https://github.com/DataDog/libdatadog", rev = "ba8955394cf35cf24a1a508fbe6264ad84702567" }
dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "936b3440a1ffc3dd68d040354b721a3042aad47d", default-features = false }
datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "936b3440a1ffc3dd68d040354b721a3042aad47d", default-features = false }
libddwaf = { version = "1.28.1", git = "https://github.com/DataDog/libddwaf-rust", rev = "d1534a158d976bd4f747bf9fcc58e0712d2d17fc", default-features = false, features = ["serde"] }
Expand Down
4 changes: 4 additions & 0 deletions bottlecap/LICENSE-3rdparty.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ addr2line,https://github.com/gimli-rs/addr2line,Apache-2.0 OR MIT,The addr2line
adler2,https://github.com/oyvindln/adler2,0BSD OR MIT OR Apache-2.0,"Jonas Schievink <[email protected]>, oyvindln <[email protected]>"
ahash,https://github.com/tkaitchuck/ahash,MIT OR Apache-2.0,Tom Kaitchuck <[email protected]>
aho-corasick,https://github.com/BurntSushi/aho-corasick,Unlicense OR MIT,Andrew Gallant <[email protected]>
allocator-api2,https://github.com/zakarumych/allocator-api2,MIT OR Apache-2.0,Zakarum <[email protected]>
anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay <[email protected]>
async-trait,https://github.com/dtolnay/async-trait,MIT OR Apache-2.0,David Tolnay <[email protected]>
atomic,https://github.com/Amanieu/atomic-rs,Apache-2.0 OR MIT,Amanieu d'Antras <[email protected]>
Expand Down Expand Up @@ -31,11 +32,13 @@ core-foundation-sys,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.
cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers
crc32fast,https://github.com/srijs/rust-crc32fast,MIT OR Apache-2.0,"Sam Rijs <[email protected]>, Alex Crichton <[email protected]>"
crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers
datadog-ddsketch,https://github.com/DataDog/libdatadog,Apache-2.0,The datadog-ddsketch Authors
datadog-fips,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-fips Authors
datadog-protos,https://github.com/DataDog/saluki,Apache-2.0,The datadog-protos Authors
datadog-trace-normalization,https://github.com/DataDog/libdatadog,Apache-2.0,David Lee <[email protected]>
datadog-trace-obfuscation,https://github.com/DataDog/libdatadog,Apache-2.0,David Lee <[email protected]>
datadog-trace-protobuf,https://github.com/DataDog/libdatadog,Apache-2.0,David Lee <[email protected]>
datadog-trace-stats,https://github.com/DataDog/libdatadog,Apache-2.0,The datadog-trace-stats Authors
datadog-trace-utils,https://github.com/DataDog/libdatadog,Apache-2.0,The datadog-trace-utils Authors
ddcommon,https://github.com/DataDog/libdatadog,Apache-2.0,The ddcommon Authors
ddsketch-agent,https://github.com/DataDog/saluki,Apache-2.0,The ddsketch-agent Authors
Expand All @@ -53,6 +56,7 @@ figment,https://github.com/SergioBenitez/Figment,MIT OR Apache-2.0,Sergio Benite
flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton <[email protected]>, Josh Triplett <[email protected]>"
float-cmp,https://github.com/mikedilger/float-cmp,MIT,Mike Dilger <[email protected]>
fnv,https://github.com/servo/rust-fnv,Apache-2.0 OR MIT,Alex Crichton <[email protected]>
foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters <[email protected]>
form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers
futures,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures Authors
futures-channel,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-channel Authors
Expand Down
1 change: 1 addition & 0 deletions bottlecap/src/otlp/transform.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,6 +861,7 @@ pub fn otel_span_to_dd_span(
metrics: HashMap::new(),
meta_struct: HashMap::new(),
span_links: Vec::new(),
span_events: Vec::new(),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New field resulted from upgrading libdatadog

};

// Set error status
Expand Down
1 change: 0 additions & 1 deletion bottlecap/src/traces/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ pub mod proxy_aggregator;
pub mod proxy_flusher;
pub mod span_pointers;
pub mod stats_aggregator;
pub mod stats_concentrator;
pub mod stats_concentrator_service;
pub mod stats_flusher;
pub mod stats_generator;
Expand Down
14 changes: 11 additions & 3 deletions bottlecap/src/traces/stats_aggregator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@ impl StatsAggregator {
// Pull stats data from concentrator
match self.concentrator.flush(force_flush).await {
Ok(stats) => {
self.queue.extend(stats);
if let Some(stats) = stats {
self.queue.push_back(stats);
}
}
Err(e) => {
error!("Error getting stats from the stats concentrator: {e:?}");
Expand Down Expand Up @@ -121,6 +123,8 @@ mod tests {
tags: vec![],
git_commit_sha: "git_commit_sha".to_string(),
image_tag: "image_tag".to_string(),
process_tags: "process_tags".to_string(),
process_tags_hash: 0,
};

aggregator.add(payload.clone());
Expand Down Expand Up @@ -153,6 +157,8 @@ mod tests {
tags: vec![],
git_commit_sha: "git_commit_sha".to_string(),
image_tag: "image_tag".to_string(),
process_tags: "process_tags".to_string(),
process_tags_hash: 0,
};
aggregator.add(payload.clone());
assert_eq!(aggregator.queue.len(), 1);
Expand All @@ -169,8 +175,8 @@ mod tests {
&HashMap::new(),
));
let (_, concentrator) = StatsConcentratorService::new(config, tags_provider);
let mut aggregator = StatsAggregator::new(640, concentrator);
// Payload below is 115 bytes
let mut aggregator = StatsAggregator::new(704, concentrator);
// Payload below is 352 bytes
let payload = ClientStatsPayload {
hostname: "hostname".to_string(),
env: "dev".to_string(),
Expand All @@ -186,6 +192,8 @@ mod tests {
tags: vec![],
git_commit_sha: "git_commit_sha".to_string(),
image_tag: "image_tag".to_string(),
process_tags: "process_tags".to_string(),
process_tags_hash: 0,
Comment on lines +195 to +196
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New fields resulted from upgrading libdatadog

};

// Add 3 payloads
Expand Down
Loading
Loading