Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JMX Scraper: YAML file and integration test hadoop #1675

Merged
merged 18 commits into from
Feb 3, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
changed unit names
magda-woj committed Nov 25, 2024
commit 675cd7b5f83f510fa8211aed67e174d946299ca5
Original file line number Diff line number Diff line change
@@ -60,49 +60,49 @@ void endToEnd() {
metric,
"hadoop.name_node.block.count",
"The total number of blocks on the name node.",
"{blocks}",
"{block}",
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.block.missing",
"The number of blocks reported as missing to the name node.",
"{blocks}",
"{block}",
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.block.corrupt",
"The number of blocks reported as corrupt to the name node.",
"{blocks}",
"{block}",
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.volume.failed",
"The number of failed volumes reported to the name node.",
"{volumes}",
"{volume}",
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.file.count",
"The total number of files being tracked by the name node.",
"{files}",
"{file}",
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.file.load",
"The current number of concurrent file accesses.",
"{operations}",
"{operation}",
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.data_node.count",
"The number of data nodes reporting to the name node.",
"{nodes}",
"{node}",
attrs ->
attrs.containsOnly(entry("node_name", "test-host"), entry("state", "live")),
attrs ->
14 changes: 7 additions & 7 deletions jmx-metrics/src/main/resources/target-systems/hadoop.groovy
Original file line number Diff line number Diff line change
@@ -21,25 +21,25 @@ otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.capacity.usage", "The cu
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.capacity.limit", "The total capacity allotted to data nodes reporting to the name node.", "by",
["node_name" : { mbean -> mbean.getProperty("tag.Hostname") }],
"CapacityTotal", otel.&longUpDownCounterCallback)
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.block.count", "The total number of blocks on the name node.", "{blocks}",
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.block.count", "The total number of blocks on the name node.", "{block}",
["node_name" : { mbean -> mbean.getProperty("tag.Hostname") }],
"BlocksTotal", otel.&longUpDownCounterCallback)
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.block.missing", "The number of blocks reported as missing to the name node.", "{blocks}",
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.block.missing", "The number of blocks reported as missing to the name node.", "{block}",
["node_name" : { mbean -> mbean.getProperty("tag.Hostname") }],
"MissingBlocks", otel.&longUpDownCounterCallback)
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.block.corrupt", "The number of blocks reported as corrupt to the name node.", "{blocks}",
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.block.corrupt", "The number of blocks reported as corrupt to the name node.", "{block}",
["node_name" : { mbean -> mbean.getProperty("tag.Hostname") }],
"CorruptBlocks", otel.&longUpDownCounterCallback)
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.volume.failed", "The number of failed volumes reported to the name node.", "{volumes}",
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.volume.failed", "The number of failed volumes reported to the name node.", "{volume}",
["node_name" : { mbean -> mbean.getProperty("tag.Hostname") }],
"VolumeFailuresTotal", otel.&longUpDownCounterCallback)
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.file.count", "The total number of files being tracked by the name node.", "{files}",
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.file.count", "The total number of files being tracked by the name node.", "{file}",
["node_name" : { mbean -> mbean.getProperty("tag.Hostname") }],
"FilesTotal", otel.&longUpDownCounterCallback)
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.file.load", "The current number of concurrent file accesses.", "{operations}",
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.file.load", "The current number of concurrent file accesses.", "{operation}",
["node_name" : { mbean -> mbean.getProperty("tag.Hostname") }],
"TotalLoad", otel.&longUpDownCounterCallback)
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.data_node.count", "The number of data nodes reporting to the name node.", "{nodes}",
otel.instrument(beanHadoopNameNodeFS, "hadoop.name_node.data_node.count", "The number of data nodes reporting to the name node.", "{node}",
["node_name" : { mbean -> mbean.getProperty("tag.Hostname") }],
["NumLiveDataNodes":["state":{"live"}], "NumDeadDataNodes": ["state":{"dead"}]],
otel.&longUpDownCounterCallback)
Original file line number Diff line number Diff line change
@@ -57,55 +57,55 @@ protected void verifyMetrics() {
metric,
"hadoop.name_node.block.count",
"The total number of blocks on the name node.",
"{blocks}",
"{block}",
/* isMonotonic= */false,
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.block.missing",
"The number of blocks reported as missing to the name node.",
"{blocks}",
"{block}",
/* isMonotonic= */false,
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.block.corrupt",
"The number of blocks reported as corrupt to the name node.",
"{blocks}",
"{block}",
/* isMonotonic= */false,
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.volume.failed",
"The number of failed volumes reported to the name node.",
"{volumes}",
"{volume}",
/* isMonotonic= */false,
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.file.count",
"The total number of files being tracked by the name node.",
"{files}",
"{file}",
/* isMonotonic= */false,
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.file.load",
"The current number of concurrent file accesses.",
"{operations}",
"{operation}",
/* isMonotonic= */false,
attrs -> attrs.containsOnly(entry("node_name", "test-host"))),
metric ->
assertSumWithAttributes(
metric,
"hadoop.name_node.data_node.count",
"The number of data nodes reporting to the name node.",
"{nodes}",
"{node}",
/* isMonotonic= */false,
attrs ->
attrs.containsOnly(entry("node_name", "test-host"), entry("state", "live")),
14 changes: 7 additions & 7 deletions jmx-scraper/src/main/resources/hadoop.yaml
Original file line number Diff line number Diff line change
@@ -19,37 +19,37 @@ rules:
BlocksTotal:
metric: block.count
type: updowncounter
unit: "{blocks}"
unit: "{block}"
desc: The total number of blocks on the name node.
MissingBlocks:
metric: block.missing
type: updowncounter
unit: "{blocks}"
unit: "{block}"
desc: The number of blocks reported as missing to the name node.
CorruptBlocks:
metric: block.corrupt
type: updowncounter
unit: "{blocks}"
unit: "{block}"
desc: The number of blocks reported as corrupt to the name node.
VolumeFailuresTotal:
metric: volume.failed
type: updowncounter
unit: "{volumes}"
unit: "{volume}"
desc: The number of failed volumes reported to the name node.
FilesTotal:
metric: file.count
type: updowncounter
unit: "{files}"
unit: "{file}"
desc: The total number of files being tracked by the name node.
TotalLoad:
metric: file.load
type: updowncounter
unit: "{operations}"
unit: "{operation}"
desc: The current number of concurrent file accesses.
NumLiveDataNodes:
metric: &metric data_node.count
type: updowncounter
unit: &unit "{nodes}"
unit: &unit "{node}"
desc: &desc The number of data nodes reporting to the name node.
metricAttribute:
state: const(live)