Skip to content

Uninstall lru_redux and install sin_lru_redux #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Rakefile
Original file line number Diff line number Diff line change
@@ -1 +1 @@
require "logstash/devutils/rake"
require 'logstash/devutils/rake'
46 changes: 21 additions & 25 deletions lib/logstash/filters/memoize.rb
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
require "lru_redux"
require 'logstash/filters/base'
require 'logstash/namespace'
require 'lru_redux'

# This filter provides https://en.wikipedia.org/wiki/Memoization[memoization] to wrapped filter.
# Internally, It based on https://en.wikipedia.org/wiki/Cache_replacement_policies#LRU[LRU] cache algorithm.
Expand All @@ -12,26 +11,25 @@
# filter {
# memoize {
# key => "%{host}" <1>
# fields => ["host_owner", "host_location"] <2>
# filter_name => "elasticsearch" <3>
# fields => ['host_owner', 'host_location'] <2>
# filter_name => 'elasticsearch' <3>
# filter_options => { <4>
# query => "host:%{host}"
# index => "known_host"
# index => 'known_host'
# fields => {
# "host_owner" => "host_owner"
# "host_location" => "host_location"
# 'host_owner' => 'host_owner'
# 'host_location' => 'host_location'
# }
# }
# }
# }
# --------------------------------------------------
#
#
# * When an event with a new <1> key comes in, execute wrapped <3> <4> filter and caches the <2> fields value.
# * When an event with a same <1> key comes in, sets cached value to target <2> fields without wrapped <3> <4> filter execution.
#
class LogStash::Filters::Memoize < LogStash::Filters::Base

config_name "memoize"
config_name 'memoize'

# The key to use caching and retrieving values. It can be dynamic and include parts of the event using the %{field}.
config :key, :validate => :string, :required => true
Expand All @@ -50,30 +48,28 @@ class LogStash::Filters::Memoize < LogStash::Filters::Base

# The TTL(Time To Live) in second of cached value.
config :ttl, :validate => :number

public

def register
@filter = LogStash::Plugin.lookup("filter", @filter_name).new(@filter_options)
@filter = LogStash::Plugin.lookup('filter', @filter_name).new(@filter_options)
@filter.register
@cache = ::LruRedux::TTL::ThreadSafeCache.new(@cache_size, @ttl)
end # def register
@cache = ::LruRedux::TTL::ThreadSafeCache.new(@cache_size, @ttl, true)
end

public
def filter(event)
formattedKey = event.sprintf(@key);
result = @cache[formattedKey]
formatted_key = event.sprintf(@key)
result = @cache[formatted_key]

if !result.nil?
@logger.debug("Cached value found.", :key => formattedKey, :value => result) if @logger.debug?
@logger.debug('Cached value found.', :key => formatted_key, :value => result) if @logger.debug?
@fields.each { |field| event.set(field, result[field]) }
else
@logger.debug("Cached value not found. Do filter.", :key => formattedKey, :filter => @filter) if @logger.debug?
@logger.debug('Cached value not found. Do filter.', :key => formatted_key, :filter => @filter) if @logger.debug?
@filter.filter(event)
@fields.each { |field| (result ||= {})[field] = event.get(field) }
@cache[formattedKey] = result
@cache[formatted_key] = result
end

# filter_matched should go in the last line of our successful code
filter_matched(event)
end # def filter
end # class LogStash::Filters::Memoize
end
end
10 changes: 5 additions & 5 deletions logstash-filter-memoize.gemspec
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@ Gem::Specification.new do |s|
s.require_paths = ['lib']

# Files
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
# Tests
s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
# Tests
s.test_files = s.files.grep(%r{^(test|spec|features)/})

# Special flag to let us know this is actually a logstash plugin
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'filter' }

# Gem dependencies
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
s.add_runtime_dependency 'lru_redux'
s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
s.add_runtime_dependency 'sin_lru_redux'
s.add_development_dependency 'logstash-devutils'
s.add_development_dependency 'logstash-filter-ruby'
s.add_development_dependency 'logstash-filter-sleep'
Expand Down
117 changes: 59 additions & 58 deletions spec/filters/memoize_spec.rb
Original file line number Diff line number Diff line change
@@ -1,45 +1,45 @@
# encoding: utf-8
require_relative '../spec_helper'
require "logstash/filters/memoize"
require 'logstash/filters/memoize'

describe LogStash::Filters::Memoize do
describe "basic work" do
let(:config) do <<-CONFIG
filter {
ruby {
code => "event.set('time', Time.now.to_f)"
}
RSpec.describe LogStash::Filters::Memoize do
describe 'basic work' do
let(:config) do
<<-CONFIG
filter {
ruby {
code => "event.set('time', Time.now.to_f)"
}

memoize {
key => "%{key}"
fields => ["result"]
filter_name => "ruby"
filter_options => {
code => "event.set('result', event.get('time'))"
memoize {
key => "%{key}"
fields => ['result']
filter_name => 'ruby'
filter_options => {
code => "event.set('result', event.get('time'))"
}
}
}
}
CONFIG
CONFIG
end

sample([
{"key" => "cache_key"},
{"key" => "cache_key2"},
{"key" => "cache_key"}
{ 'key' => 'cache_key' },
{ 'key' => 'cache_key2' },
{ 'key' => 'cache_key' }
]) do
# If key is different, result different also.
expect(subject[0].get("time")).to eq(subject[0].get("result"))
expect(subject[1].get("time")).to eq(subject[1].get("result"))
expect(subject[0].get("time")).not_to eq(subject[1].get("result"))
expect(subject[0].get('time')).to eq(subject[0].get('result'))
expect(subject[1].get('time')).to eq(subject[1].get('result'))
expect(subject[0].get('time')).not_to eq(subject[1].get('result'))

# If key is same, cache works
expect(subject[0].get("time")).to eq(subject[0].get("result"))
expect(subject[2].get("time")).not_to eq(subject[2].get("result"))
expect(subject[0].get("time")).to eq(subject[2].get("result"))
expect(subject[0].get('time')).to eq(subject[0].get('result'))
expect(subject[2].get('time')).not_to eq(subject[2].get('result'))
expect(subject[0].get('time')).to eq(subject[2].get('result'))
end
end

describe "ttl work" do
describe 'ttl work' do
let(:config) do <<-CONFIG
filter {
ruby {
Expand All @@ -48,62 +48,63 @@

memoize {
key => "%{key}"
fields => ["result"]
filter_name => "ruby"
fields => ['result']
filter_name => 'ruby'
filter_options => {
code => "event.set('result', event.get('time'))"
}
ttl => 1
}

sleep {
time => "2"
time => '2'
}
}
CONFIG
end

sample([
{"key" => "cache_key"},
{"key" => "cache_key"}
{ 'key' => 'cache_key' },
{ 'key' => 'cache_key' }
]) do
# After ttl seconds, cached value are delete
expect(subject[0].get("time")).to eq(subject[0].get("result"))
expect(subject[1].get("time")).to eq(subject[1].get("result"))
expect(subject[0].get("time")).not_to eq(subject[1].get("result"))
expect(subject[0].get('time')).to eq(subject[0].get('result'))
expect(subject[1].get('time')).to eq(subject[1].get('result'))
expect(subject[0].get('time')).not_to eq(subject[1].get('result'))
end
end

describe "cache_size work" do
let(:config) do <<-CONFIG
filter {
ruby {
code => "event.set('time', Time.now.to_f)"
}
describe 'cache_size work' do
let(:config) do
<<-CONFIG
filter {
ruby {
code => "event.set('time', Time.now.to_f)"
}

memoize {
key => "%{key}"
fields => ["result"]
filter_name => "ruby"
filter_options => {
code => "event.set('result', event.get('time'))"
memoize {
key => "%{key}"
fields => ['result']
filter_name => 'ruby'
filter_options => {
code => "event.set('result', event.get('time'))"
}
cache_size => 1
}
cache_size => 1
}
}
CONFIG
CONFIG
end

sample([
{"key" => "cache_key"},
{"key" => "cache_key2"},
{"key" => "cache_key"}
{ 'key' => 'cache_key' },
{ 'key' => 'cache_key2' },
{ 'key' => 'cache_key' }
]) do
# If cache over cache size, old cached value are delete first
expect(subject[0].get("time")).to eq(subject[0].get("result"))
expect(subject[1].get("time")).to eq(subject[1].get("result"))
expect(subject[2].get("time")).to eq(subject[2].get("result"))
expect(subject[0].get("time")).not_to eq(subject[2].get("result"))
expect(subject[0].get('time')).to eq(subject[0].get('result'))
expect(subject[1].get('time')).to eq(subject[1].get('result'))
expect(subject[2].get('time')).to eq(subject[2].get('result'))
expect(subject[0].get('time')).not_to eq(subject[2].get('result'))
end
end
end
3 changes: 1 addition & 2 deletions spec/spec_helper.rb
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
# encoding: utf-8
require "logstash/devutils/rspec/spec_helper"
require 'logstash/devutils/rspec/spec_helper'