Skip to content

Commit bf45465

Browse files
palash3flei-ddn
palash3
and
flei-ddn
committed
EX-10319: Add health_value and health_sensitivity
Add health_value metrics for each inteface. Add health_sensitiviy metrics from 'lnetctl global show'. Co-authored-by: Feng Lei <flei@ddn.com>
1 parent 9bfe3df commit bf45465

13 files changed

Lines changed: 277 additions & 15 deletions

‎lustre-collector/src/lib.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@ pub mod types;
2727

2828
pub use crate::error::LustreCollectorError;
2929
use combine::parser::EasyParser;
30-
pub use lnetctl_parser::{parse as parse_lnetctl_output, parse_lnetctl_stats};
30+
pub use lnetctl_parser::{
31+
parse as parse_lnetctl_output, parse_lnetctl_global_show, parse_lnetctl_stats,
32+
};
3133
pub use node_stats_parsers::{parse_cpustats_output, parse_meminfo_output};
3234
use std::{io, str};
3335
pub use types::*;

‎lustre-collector/src/lnetctl_parser.rs‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
use crate::{
66
LNetStatGlobal, LustreCollectorError,
7-
lnet_exports::LNetStatsStatistics,
7+
lnet_exports::{LNetGlobal, LNetStatsStatistics},
88
types::{LNetStat, LNetStats, Param, Record, lnet_exports::Net},
99
};
1010

@@ -33,6 +33,11 @@ pub(crate) fn build_lnet_stats(x: &Net) -> Vec<Record> {
3333
param: Param("drop_count".to_string()),
3434
value: y.statistics.drop_count,
3535
}),
36+
LNetStats::HealthValue(LNetStat {
37+
nid: y.nid.to_string(),
38+
param: Param("health_value".to_string()),
39+
value: y.health_stats.health_value,
40+
}),
3641
]
3742
})
3843
.map(Record::LNetStat)
@@ -65,6 +70,11 @@ struct LnetStats {
6570
statistics: Option<LNetStatsStatistics>,
6671
}
6772

73+
#[derive(serde::Serialize, serde::Deserialize)]
74+
struct LnetGlobalShow {
75+
global: Option<LNetGlobal>,
76+
}
77+
6878
pub(crate) fn build_lnetctl_stats(x: &LNetStatsStatistics) -> Vec<Record> {
6979
vec![
7080
Record::LNetStat(LNetStats::SendLength(LNetStatGlobal {
@@ -82,6 +92,15 @@ pub(crate) fn build_lnetctl_stats(x: &LNetStatsStatistics) -> Vec<Record> {
8292
]
8393
}
8494

95+
pub(crate) fn build_lnetctl_global_show(x: &LNetGlobal) -> Vec<Record> {
96+
vec![Record::LNetStat(LNetStats::HealthSensitiveValue(
97+
LNetStatGlobal {
98+
param: Param("health_sensitivity".to_string()),
99+
value: x.health_sensitivity,
100+
},
101+
))]
102+
}
103+
85104
pub fn parse_lnetctl_stats(xs: &[u8]) -> Result<Vec<Record>, LustreCollectorError> {
86105
let xs = xs.trim_ascii();
87106

@@ -96,6 +115,20 @@ pub fn parse_lnetctl_stats(xs: &[u8]) -> Result<Vec<Record>, LustreCollectorErro
96115
.unwrap_or_default())
97116
}
98117

118+
pub fn parse_lnetctl_global_show(xs: &[u8]) -> Result<Vec<Record>, LustreCollectorError> {
119+
let xs = xs.trim_ascii();
120+
121+
if xs.is_empty() {
122+
return Ok(vec![]);
123+
}
124+
125+
let y: LnetGlobalShow = yaml_serde::from_slice(xs)?;
126+
127+
Ok(y.global
128+
.map(|x| build_lnetctl_global_show(&x))
129+
.unwrap_or_default())
130+
}
131+
99132
#[cfg(test)]
100133
mod tests {
101134
use super::*;
@@ -501,4 +534,28 @@ mod tests {
501534

502535
assert_debug_snapshot!(x);
503536
}
537+
538+
#[test]
539+
fn test_lnet_global_show() {
540+
let x = parse_lnetctl_global_show(
541+
br#"global:
542+
numa_range: 0
543+
max_intf: 200
544+
discovery: 1
545+
drop_asym_route: 0
546+
retry_count: 2
547+
transaction_timeout: 150
548+
health_sensitivity: 100
549+
recovery_interval: 1
550+
router_sensitivity: 100
551+
lnd_timeout: 49
552+
response_tracking: 3
553+
recovery_limit: 0
554+
max_recovery_ping_interval: 900
555+
"#,
556+
)
557+
.unwrap();
558+
559+
assert_debug_snapshot!(x);
560+
}
504561
}

‎lustre-collector/src/main.rs‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@
44

55
use clap::{Arg, ValueEnum, value_parser};
66
use lustre_collector::{
7-
error::LustreCollectorError, mgs::mgs_fs_parser, parse_lctl_output, parse_lnetctl_output,
8-
parse_lnetctl_stats, parse_mgs_fs_output, parse_recovery_status_output, parser,
9-
recovery_status_parser, types::Record,
7+
error::LustreCollectorError, mgs::mgs_fs_parser, parse_lctl_output, parse_lnetctl_global_show,
8+
parse_lnetctl_output, parse_lnetctl_stats, parse_mgs_fs_output, parse_recovery_status_output,
9+
parser, recovery_status_parser, types::Record,
1010
};
1111
use std::{
1212
fmt, panic,
@@ -81,6 +81,12 @@ fn get_lnetctl_stats_output() -> Result<Vec<u8>, LustreCollectorError> {
8181
Ok(r.stdout)
8282
}
8383

84+
fn get_lnetctl_global_show_output() -> Result<Vec<u8>, LustreCollectorError> {
85+
let r = Command::new("lnetctl").arg("global").arg("show").output()?;
86+
87+
Ok(r.stdout)
88+
}
89+
8490
fn main() -> ExitCode {
8591
match run() {
8692
Ok(()) => ExitCode::SUCCESS,
@@ -136,6 +142,14 @@ fn run() -> Result<(), LustreCollectorError> {
136142
Ok(lnetctl_stats_record)
137143
});
138144

145+
let lnetctl_global_show =
146+
thread::spawn(move || -> Result<Vec<Record>, LustreCollectorError> {
147+
let lnetctl_global_output = get_lnetctl_global_show_output()?;
148+
let lnetctl_global_record = parse_lnetctl_global_show(&lnetctl_global_output)?;
149+
150+
Ok(lnetctl_global_record)
151+
});
152+
139153
let recovery_status_handle =
140154
thread::spawn(move || -> Result<Vec<Record>, LustreCollectorError> {
141155
let recovery_status_output = get_recovery_status_output()?;
@@ -172,10 +186,16 @@ fn run() -> Result<(), LustreCollectorError> {
172186
Err(e) => panic::resume_unwind(e),
173187
};
174188

189+
let mut lnetctl_global_show_record = match lnetctl_global_show.join() {
190+
Ok(r) => r.unwrap_or_default(),
191+
Err(e) => panic::resume_unwind(e),
192+
};
193+
175194
lctl_record.append(&mut lnet_record);
176195
lctl_record.append(&mut mgs_fs_record);
177196
lctl_record.append(&mut recovery_status_records);
178197
lctl_record.append(&mut lnetctl_stats_record);
198+
lctl_record.append(&mut lnetctl_global_show_record);
179199

180200
let x = match format {
181201
Format::Json => serde_json::to_string(&lctl_record)?,

‎lustre-collector/src/snapshots/lustre_collector__lnetctl_parser__tests__lnet_export_parse_no_bonding.snap‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
---
2-
source: src/lnetctl_parser.rs
2+
source: lustre-collector/src/lnetctl_parser.rs
33
expression: x
4-
54
---
65
[
76
LNetStat(
@@ -37,6 +36,17 @@ expression: x
3736
},
3837
),
3938
),
39+
LNetStat(
40+
HealthValue(
41+
LNetStat {
42+
nid: "0@lo",
43+
param: Param(
44+
"health_value",
45+
),
46+
value: 800,
47+
},
48+
),
49+
),
4050
LNetStat(
4151
SendCount(
4252
LNetStat {
@@ -70,4 +80,15 @@ expression: x
7080
},
7181
),
7282
),
83+
LNetStat(
84+
HealthValue(
85+
LNetStat {
86+
nid: "10.36.4.130@tcp",
87+
param: Param(
88+
"health_value",
89+
),
90+
value: 1000,
91+
},
92+
),
93+
),
7394
]

‎lustre-collector/src/snapshots/lustre_collector__lnetctl_parser__tests__lnet_global_show.snap‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
source: lustre-collector/src/lnetctl_parser.rs
3+
expression: x
4+
---
5+
[
6+
LNetStat(
7+
HealthSensitiveValue(
8+
LNetStatGlobal {
9+
param: Param(
10+
"health_sensitivity",
11+
),
12+
value: 100,
13+
},
14+
),
15+
),
16+
]

‎lustre-collector/src/snapshots/lustre_collector__lnetctl_parser__tests__lnet_net_parse.snap‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
---
2-
source: src/lnetctl_parser.rs
2+
source: lustre-collector/src/lnetctl_parser.rs
33
expression: x
4-
54
---
65
[
76
LNetStat(
@@ -37,6 +36,17 @@ expression: x
3736
},
3837
),
3938
),
39+
LNetStat(
40+
HealthValue(
41+
LNetStat {
42+
nid: "0@lo",
43+
param: Param(
44+
"health_value",
45+
),
46+
value: 942,
47+
},
48+
),
49+
),
4050
LNetStat(
4151
SendCount(
4252
LNetStat {
@@ -70,4 +80,15 @@ expression: x
7080
},
7181
),
7282
),
83+
LNetStat(
84+
HealthValue(
85+
LNetStat {
86+
nid: "10.73.20.11@tcp",
87+
param: Param(
88+
"health_value",
89+
),
90+
value: 1000,
91+
},
92+
),
93+
),
7394
]

‎lustre-collector/src/snapshots/lustre_collector__lnetctl_parser__tests__lnet_parse2.snap‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
---
2-
source: src/lnetctl_parser.rs
2+
source: lustre-collector/src/lnetctl_parser.rs
33
expression: x
4-
54
---
65
[
76
LNetStat(
@@ -37,6 +36,17 @@ expression: x
3736
},
3837
),
3938
),
39+
LNetStat(
40+
HealthValue(
41+
LNetStat {
42+
nid: "0@lo",
43+
param: Param(
44+
"health_value",
45+
),
46+
value: 0,
47+
},
48+
),
49+
),
4050
LNetStat(
4151
SendCount(
4252
LNetStat {
@@ -70,6 +80,17 @@ expression: x
7080
},
7181
),
7282
),
83+
LNetStat(
84+
HealthValue(
85+
LNetStat {
86+
nid: "172.16.0.24@o2ib",
87+
param: Param(
88+
"health_value",
89+
),
90+
value: 1000,
91+
},
92+
),
93+
),
7394
LNetStat(
7495
SendCount(
7596
LNetStat {
@@ -103,4 +124,15 @@ expression: x
103124
},
104125
),
105126
),
127+
LNetStat(
128+
HealthValue(
129+
LNetStat {
130+
nid: "172.16.0.28@o2ib",
131+
param: Param(
132+
"health_value",
133+
),
134+
value: 1000,
135+
},
136+
),
137+
),
106138
]

‎lustre-collector/src/types.rs‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ pub mod lnet_exports {
167167
#[derive(serde::Serialize, serde::Deserialize)]
168168
pub struct HealthStats {
169169
#[serde(rename = "health value")]
170-
health_value: i64,
170+
pub health_value: i64,
171171
interrupts: i64,
172172
dropped: i64,
173173
aborted: i64,
@@ -268,6 +268,23 @@ pub mod lnet_exports {
268268
pub drop_length: i64,
269269
}
270270

271+
#[derive(serde::Serialize, serde::Deserialize)]
272+
pub struct LNetGlobal {
273+
pub numa_range: i64,
274+
pub max_intf: i64,
275+
pub discovery: i64,
276+
pub drop_asym_route: i64,
277+
pub retry_count: i64,
278+
pub transaction_timeout: i64,
279+
pub health_sensitivity: i64,
280+
pub recovery_interval: i64,
281+
pub router_sensitivity: i64,
282+
pub lnd_timeout: i64,
283+
pub response_tracking: i64,
284+
pub recovery_limit: i64,
285+
pub max_recovery_ping_interval: i64,
286+
}
287+
271288
#[derive(serde::Serialize, serde::Deserialize)]
272289
pub struct Tunables {
273290
pub peer_timeout: i64,
@@ -537,6 +554,8 @@ pub enum LNetStats {
537554
SendCount(LNetStat<i64>),
538555
RecvCount(LNetStat<i64>),
539556
DropCount(LNetStat<i64>),
557+
HealthValue(LNetStat<i64>),
558+
HealthSensitiveValue(LNetStatGlobal<i64>),
540559
SendLength(LNetStatGlobal<i64>),
541560
RecvLength(LNetStatGlobal<i64>),
542561
DropLength(LNetStatGlobal<i64>),

0 commit comments

Comments
 (0)