Skip to content

Commit 289410a

Browse files
abhinawkrabhinawkr
authored
GCP-226: expose Lustre start_time as per-target metrics (#154)
1 parent 48dc875 commit 289410a

72 files changed

Lines changed: 2350 additions & 1049 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎lustre-collector/src/brw_stats_parser.rs‎

Lines changed: 53 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
use crate::{
66
base_parsers::{digits, string_to, till_newline, word},
7-
time::time_triple,
7+
time::{StatsHeader, time_triple},
88
types::{BrwStats, BrwStatsBucket},
99
};
1010
use combine::{
@@ -120,17 +120,19 @@ where
120120
})
121121
}
122122

123-
pub(crate) fn brw_stats<I>() -> impl Parser<I, Output = Vec<BrwStats>>
123+
pub(crate) fn brw_stats<I>() -> impl Parser<I, Output = (StatsHeader, Vec<BrwStats>)>
124124
where
125125
I: Stream<Token = char>,
126126
I::Error: ParseError<I::Token, I::Range, I::Position>,
127127
{
128-
(newline().with(time_triple()), spaces(), many1(section())).map(|(_, _, y)| y)
128+
(newline().with(time_triple()), spaces(), many1(section()))
129+
.map(|(header, _, sections)| (header, sections))
129130
}
130131

131132
#[cfg(test)]
132133
mod tests {
133134
use super::*;
135+
use crate::time::StatsHeader;
134136
use insta::assert_debug_snapshot;
135137

136138
#[test]
@@ -307,48 +309,54 @@ pages per bulk r/w rpcs % cum % | rpcs % cum %
307309
assert_eq!(
308310
result,
309311
Ok((
310-
vec![
311-
BrwStats {
312-
name: "pages".to_string(),
313-
unit: "rpcs".to_string(),
314-
buckets: vec![],
312+
(
313+
StatsHeader {
314+
snapshot_time: "1534429278.185762481".to_string(),
315+
start_time: None,
315316
},
316-
BrwStats {
317-
name: "discont_pages".to_string(),
318-
unit: "rpcs".to_string(),
319-
buckets: vec![],
320-
},
321-
BrwStats {
322-
name: "discont_blocks".to_string(),
323-
unit: "rpcs".to_string(),
324-
buckets: vec![],
325-
},
326-
BrwStats {
327-
name: "dio_frags".to_string(),
328-
unit: "ios".to_string(),
329-
buckets: vec![],
330-
},
331-
BrwStats {
332-
name: "rpc_hist".to_string(),
333-
unit: "ios".to_string(),
334-
buckets: vec![],
335-
},
336-
BrwStats {
337-
name: "io_time".to_string(),
338-
unit: "ios".to_string(),
339-
buckets: vec![],
340-
},
341-
BrwStats {
342-
name: "disk_iosize".to_string(),
343-
unit: "ios".to_string(),
344-
buckets: vec![],
345-
},
346-
BrwStats {
347-
name: "block_maps_msec".to_string(),
348-
unit: "maps".to_string(),
349-
buckets: vec![],
350-
},
351-
],
317+
vec![
318+
BrwStats {
319+
name: "pages".to_string(),
320+
unit: "rpcs".to_string(),
321+
buckets: vec![],
322+
},
323+
BrwStats {
324+
name: "discont_pages".to_string(),
325+
unit: "rpcs".to_string(),
326+
buckets: vec![],
327+
},
328+
BrwStats {
329+
name: "discont_blocks".to_string(),
330+
unit: "rpcs".to_string(),
331+
buckets: vec![],
332+
},
333+
BrwStats {
334+
name: "dio_frags".to_string(),
335+
unit: "ios".to_string(),
336+
buckets: vec![],
337+
},
338+
BrwStats {
339+
name: "rpc_hist".to_string(),
340+
unit: "ios".to_string(),
341+
buckets: vec![],
342+
},
343+
BrwStats {
344+
name: "io_time".to_string(),
345+
unit: "ios".to_string(),
346+
buckets: vec![],
347+
},
348+
BrwStats {
349+
name: "disk_iosize".to_string(),
350+
unit: "ios".to_string(),
351+
buckets: vec![],
352+
},
353+
BrwStats {
354+
name: "block_maps_msec".to_string(),
355+
unit: "maps".to_string(),
356+
buckets: vec![],
357+
},
358+
],
359+
),
352360
""
353361
))
354362
);
@@ -358,7 +366,7 @@ pages per bulk r/w rpcs % cum % | rpcs % cum %
358366
fn test_brw_stats() {
359367
let x = include_str!("fixtures/brw_stats_with_data.txt");
360368

361-
let result: (Vec<_>, _) = brw_stats().parse(x).unwrap();
369+
let result = brw_stats().parse(x).unwrap();
362370

363371
assert_debug_snapshot!(result);
364372
}

‎lustre-collector/src/exports_parser.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ where
4242
string("stats").skip(equals()),
4343
stats(),
4444
))
45-
.map(|(nid, _, stats)| ExportStats { nid, stats })
45+
.map(|(nid, _, (_, stats))| ExportStats { nid, stats })
4646
.message("while parsing export_stats")
4747
}
4848

‎lustre-collector/src/io_latency_stats_parser.rs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
66

77
use crate::{
88
base_parsers::digits,
9-
time::time_triple,
9+
time::{StatsHeader, time_triple},
1010
types::{BrwStats, BrwStatsBucket},
1111
};
1212
use combine::{
@@ -65,7 +65,7 @@ where
6565

6666
/// Parses the full io_latency_stats output into Vec<BrwStats>.
6767
/// Groups rd/wr lines by opsize into BrwStats entries with name "io_time_{opsize}".
68-
pub(crate) fn io_latency_stats<I>() -> impl Parser<I, Output = Vec<BrwStats>>
68+
pub(crate) fn io_latency_stats<I>() -> impl Parser<I, Output = (StatsHeader, Vec<BrwStats>)>
6969
where
7070
I: Stream<Token = char>,
7171
I::Error: ParseError<I::Token, I::Range, I::Position>,
@@ -76,7 +76,7 @@ where
7676
time_triple(),
7777
many::<Vec<_>, _, _>(latency_line()),
7878
)
79-
.map(|(_, _, _, lines)| {
79+
.map(|(_, _, header, lines)| {
8080
let mut map: BTreeMap<String, BTreeMap<u64, (u64, u64)>> = BTreeMap::new();
8181

8282
for (operation, size, buckets) in lines {
@@ -93,7 +93,8 @@ where
9393
}
9494
}
9595

96-
map.into_iter()
96+
let states = map
97+
.into_iter()
9798
.map(|(size, buckets)| BrwStats {
9899
name: format!("io_time_{size}"),
99100
unit: "ios".to_string(),
@@ -106,7 +107,8 @@ where
106107
})
107108
.collect(),
108109
})
109-
.collect()
110+
.collect();
111+
(header, states)
110112
})
111113
}
112114

@@ -191,7 +193,7 @@ elapsed_time: 3011.066966653
191193

192194
let result = io_latency_stats().easy_parse(input).unwrap();
193195

194-
assert_eq!(result.0, vec![]);
196+
assert_eq!(result.0.1, vec![]);
195197
}
196198

197199
#[test]

‎lustre-collector/src/ldlm/ldlm_service_parser.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ where
4141
{
4242
attempt((string(LDLM_CANCELD), period(), param(STATS)))
4343
.with(stats())
44-
.map(LustreServiceStats::LdlmCanceld)
44+
.map(|(header, v)| LustreServiceStats::LdlmCanceld(header, v))
4545
.message("While parsing ldlm_canceld.stats")
4646
}
4747

@@ -52,6 +52,6 @@ where
5252
{
5353
(string(LDLM_CBD), period(), param(STATS))
5454
.with(stats())
55-
.map(LustreServiceStats::LdlmCbd)
55+
.map(|(header, v)| LustreServiceStats::LdlmCbd(header, v))
5656
.message("While parsing ldlm_cbd.stats")
5757
}

‎lustre-collector/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ mod top_level_parser;
2727
pub mod types;
2828

2929
pub use crate::error::LustreCollectorError;
30+
pub use crate::time::StatsHeader;
3031
use combine::parser::EasyParser;
3132
pub use lnetctl_parser::{
3233
parse as parse_lnetctl_output, parse_lnetctl_global_show, parse_lnetctl_stats,

‎lustre-collector/src/llite/mod.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use crate::{
66
Param, Record, Stat, Target, TargetStats,
77
base_parsers::{param, period, target},
88
stats_parser::stats,
9+
time::StatsHeader,
910
};
1011
use combine::{ParseError, Parser, Stream, parser::char::string};
1112

@@ -30,15 +31,15 @@ where
3031
}
3132

3233
enum LliteStat {
33-
Stats(Vec<Stat>),
34+
Stats(StatsHeader, Vec<Stat>),
3435
}
3536

3637
fn llite_stat<I>() -> impl Parser<I, Output = (Param, LliteStat)>
3738
where
3839
I: Stream<Token = char>,
3940
I::Error: ParseError<I::Token, I::Range, I::Position>,
4041
{
41-
(param(STATS), stats().map(LliteStat::Stats)).message("while parsing llite_stat")
42+
(param(STATS), stats().map(|(h, v)| LliteStat::Stats(h, v))).message("while parsing llite_stat")
4243
}
4344

4445
pub(crate) fn parse<I>() -> impl Parser<I, Output = Record>
@@ -48,10 +49,11 @@ where
4849
{
4950
(target_name(), llite_stat())
5051
.map(|(target, (param, value))| match value {
51-
LliteStat::Stats(stats) => TargetStats::Llite(crate::types::LliteStat {
52+
LliteStat::Stats(header, stats) => TargetStats::Llite(crate::types::LliteStat {
5253
target,
5354
param,
5455
stats,
56+
header,
5557
}),
5658
})
5759
.map(Record::Target)

‎lustre-collector/src/llite/snapshots/lustre_collector__llite__tests__parse.snap‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
source: src/llite/mod.rs
2+
source: lustre-collector/src/llite/mod.rs
33
expression: result
44
---
55
(
@@ -228,6 +228,10 @@ expression: result
228228
),
229229
},
230230
],
231+
header: StatsHeader {
232+
snapshot_time: "1689697369.331040915",
233+
start_time: None,
234+
},
231235
},
232236
),
233237
),

‎lustre-collector/src/mds/mds_parser.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
// license that can be found in the LICENSE file.
44

55
use crate::{
6-
MdsStat,
6+
MdsStat, StatsHeader,
77
base_parsers::{equals, period, target},
88
stats_parser::stats,
99
types::{Param, Record, Stat, Target, TargetStats},
@@ -64,7 +64,7 @@ where
6464
.message("while parsing `mds_suffix`")
6565
}
6666

67-
fn mds_stat<I>() -> impl Parser<I, Output = (Param, Vec<Stat>)>
67+
fn mds_stat<I>() -> impl Parser<I, Output = (Param, (StatsHeader, Vec<Stat>))>
6868
where
6969
I: Stream<Token = char>,
7070
I::Error: ParseError<I::Token, I::Range, I::Position>,
@@ -92,7 +92,7 @@ where
9292
{
9393
mds_prefix()
9494
.with(mds_stat())
95-
.map(|(param, stats)| TargetStats::Mds(MdsStat { param, stats }))
95+
.map(|(param, (_, stats))| TargetStats::Mds(MdsStat { param, stats }))
9696
.map(Record::Target)
9797
.message("while parsing mds")
9898
}

‎lustre-collector/src/mds/mdt_parser.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
// license that can be found in the LICENSE file.
44

55
use crate::{
6-
ExportStats,
6+
ExportStats, StatsHeader, TimedTargetStat,
77
base_parsers::{digits, param, param_period, period, target},
88
exports_parser::exports_stats,
99
oss::obdfilter_parser::{EXPORTS, EXPORTS_PARAMS},
@@ -21,7 +21,7 @@ pub(crate) const STATS: &str = "md_stats";
2121
pub(crate) const NUM_EXPORTS: &str = "num_exports";
2222

2323
enum MdtStat {
24-
Stats(Vec<Stat>),
24+
Stats(StatsHeader, Vec<Stat>),
2525
NumExports(u64),
2626
ExportStats(Vec<ExportStats>),
2727
}
@@ -36,7 +36,8 @@ where
3636
param(NUM_EXPORTS),
3737
digits().skip(newline()).map(MdtStat::NumExports),
3838
),
39-
(param(STATS), stats().map(MdtStat::Stats)).message("while parsing mdt_stat"),
39+
(param(STATS), stats().map(|(h, v)| MdtStat::Stats(h, v)))
40+
.message("while parsing mdt_stat"),
4041
(
4142
param_period(EXPORTS),
4243
exports_stats().map(MdtStat::ExportStats),
@@ -74,11 +75,12 @@ where
7475
{
7576
(target_name(), mdt_stat())
7677
.map(|(target, (param, value))| match value {
77-
MdtStat::Stats(value) => TargetStats::Stats(TargetStat {
78+
MdtStat::Stats(header, value) => TargetStats::Stats(TimedTargetStat {
7879
kind: TargetVariant::Mdt,
7980
target,
8081
param,
8182
value,
83+
header,
8284
}),
8385
MdtStat::NumExports(value) => TargetStats::NumExports(TargetStat {
8486
kind: TargetVariant::Mdt,

‎lustre-collector/src/mds/snapshots/lustre_collector__mds__client_count_parser__test__client_count_parser_multiple_fs.snap‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
---
2-
source: src/mds/client_count_parser.rs
2+
source: lustre-collector/src/mds/client_count_parser.rs
33
expression: result
4-
54
---
65
(
76
[

0 commit comments

Comments
 (0)