Skip to content

Set extract_tags call prior exclude_fields #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Set extract_tags call prior exclude_fields #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4076813
Set extract_tags call prior exclude_fields
berghauz Sep 17, 2016
98fab10
Add mutate to test pipeline and delete host to avoid travis-ci problem
berghauz Sep 17, 2016
b33821f
Add logstash-filter-mutate to dev. dep.
berghauz Sep 17, 2016
a1b09cb
Update to new logstash API
berghauz May 11, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions lib/logstash/outputs/influxdb.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,6 @@ def receive(event)
end
end

exclude_fields!(point)
coerce_values!(point)

tags, point = extract_tags(point)

Expand All @@ -158,6 +156,10 @@ def receive(event)
"time" => time,
"fields" => point
}

exclude_fields!(point)
coerce_values!(point)

event_hash["tags"] = tags unless tags.empty?

buffer_receive(event_hash, event.sprintf(@db))
Expand Down Expand Up @@ -295,19 +297,21 @@ def exclude_fields!(event_data)
# original_fields = {"foo" => 1, "bar" => 2, "tags" => ["tag"]}
# tags, fields = extract_tags(original_fields)
# # tags: {"bar" => 2, "tag" => "true"} and fields: {"foo" => 1}

def extract_tags(fields)
remainder = fields.dup

tags = if remainder.has_key?("tags") && remainder["tags"].respond_to?(:inject)
remainder.delete("tags").inject({}) { |tags, tag| tags[tag] = "true"; tags }
remainder.delete("tags").inject({}) { |tags, tag| tags[tag] = if remainder.has_key?(tag) then fields[tag] else "true" end; tags }
else
{}
end

@send_as_tags.each { |key| (tags[key] = remainder.delete(key)) if remainder.has_key?(key) }

tags.delete_if { |key,value| value.nil? || value == "" }
remainder.delete_if { |key,value| value.nil? || value == "" }
remainder.delete_if { |key,value| tags.has_key?(key) }

[tags, remainder]
end
Expand Down
1 change: 1 addition & 0 deletions logstash-output-influxdb.gemspec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,6 @@ Gem::Specification.new do |s|
s.add_development_dependency 'logstash-devutils'
s.add_development_dependency 'logstash-input-generator'
s.add_development_dependency 'logstash-filter-kv'
s.add_development_dependency 'logstash-filter-mutate'
end

55 changes: 44 additions & 11 deletions spec/outputs/influxdb_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@

filter {
kv { }
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -66,7 +69,7 @@
measurement => "my_series"
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"]
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"]
}
}
CONFIG
Expand All @@ -93,6 +96,9 @@

filter {
kv { }
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -101,7 +107,7 @@
measurement => "my_series"
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"]
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"]
send_as_tags => ["bar", "baz", "qux"]
}
}
Expand Down Expand Up @@ -135,6 +141,9 @@
"feild space" => "pink dog"
}
}
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -143,7 +152,7 @@
measurement => "my series"
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"]
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"]
send_as_tags => ["bar", "baz", "test1", "test space"]
}
}
Expand Down Expand Up @@ -177,6 +186,9 @@
"feild, space" => "pink, dog"
}
}
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -185,7 +197,7 @@
measurement => "my, series"
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"]
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"]
send_as_tags => ["bar", "baz", "test1", "test, space"]
}
}
Expand Down Expand Up @@ -219,6 +231,9 @@
"feild= space" => "pink= dog"
}
}
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -227,7 +242,7 @@
measurement => "my=series"
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"]
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"]
send_as_tags => ["bar", "baz", "test1", "test=space"]
}
}
Expand Down Expand Up @@ -261,6 +276,9 @@
"feildspace" => 'C:\\Griffo'
}
}
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -269,7 +287,7 @@
measurement => 'my\\series'
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"]
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"]
send_as_tags => ['bar', "baz", "test1", "test=space"]
}
}
Expand Down Expand Up @@ -298,6 +316,9 @@

filter {
kv { add_tag => [ "tagged" ] }
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -306,7 +327,7 @@
measurement => "my_series"
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"]
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"]
}
}
CONFIG
Expand All @@ -333,6 +354,9 @@

filter {
kv { }
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -341,7 +365,7 @@
measurement => "my_series"
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type", "host"]
exclude_fields => ["@version", "@timestamp", "sequence", "message", "type"]
coerce_values => { "foo" => "integer" "bar" => "float" }
}
}
Expand Down Expand Up @@ -374,6 +398,9 @@

filter {
kv { }
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -384,7 +411,7 @@
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => [ "@version", "@timestamp", "sequence",
"message", "type", "host" ]
"message", "type"]
}
}
CONFIG
Expand Down Expand Up @@ -420,6 +447,9 @@

filter {
kv { }
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -430,7 +460,7 @@
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => [ "@version", "@timestamp", "sequence",
"message", "type", "host" ]
"message", "type"]
}
}
CONFIG
Expand Down Expand Up @@ -474,6 +504,9 @@

filter {
kv { }
mutate {
remove_field => ["host"]
}
}

output {
Expand All @@ -484,7 +517,7 @@
allow_time_override => true
use_event_fields_for_data_points => true
exclude_fields => [ "@version", "@timestamp", "sequence",
"message", "type", "host" ]
"message", "type"]
}
}
CONFIG
Expand Down