diff --git a/.travis.yml b/.travis.yml index 97f26977..8adde4d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ rvm: - 2.0.0 - 2.1.2 - 2.2.5 - - 2.3.1 + - 2.3.0 gemfile: - gemfiles/rails_4.gemfile - gemfiles/rails_4.1.gemfile @@ -14,6 +14,7 @@ services: env: - LIT_STORAGE=hash - LIT_STORAGE=redis + - LIT_STORAGE=hybrid before_script: - cp test/dummy/config/database.yml.travis test/dummy/config/database.yml - psql -c 'create database lit_test;' -U postgres diff --git a/lib/lit.rb b/lib/lit.rb index e10d6448..b79feea7 100644 --- a/lib/lit.rb +++ b/lib/lit.rb @@ -46,6 +46,9 @@ def self.get_key_value_engine when 'redis' require 'lit/adapters/redis_storage' return RedisStorage.new + when 'hybrid' + require 'lit/adapters/hybrid_storage' + return HybridStorage.new else require 'lit/adapters/hash_storage' return HashStorage.new diff --git a/lib/lit/adapters/hybrid_storage.rb b/lib/lit/adapters/hybrid_storage.rb new file mode 100644 index 00000000..c2a4b90f --- /dev/null +++ b/lib/lit/adapters/hybrid_storage.rb @@ -0,0 +1,180 @@ +require 'redis' +require 'concurrent' + +ActionController::Base.class_eval do + after_action :clear_saved_redis_snapshot + + def clear_saved_redis_snapshot + Lit.saved_redis_snapshot = nil + end +end + +module Lit + extend self + def redis + $redis = Redis.new(url: determine_redis_provider) unless $redis + $redis + end + + def determine_redis_provider + ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL'] + end + + def _hash + $_hash ||= ::Concurrent::Hash.new + end + + def reset_hash + $_hash = nil + end + + def hash_dirty? + # Hash is considered dirty if hash snapshot is older + # than Redis snapshot. + Lit.hash_snapshot < Lit.redis_snapshot + end + + def hash_snapshot + $_hash_snapshot ||= DateTime.new.to_f.to_d + end + + def hash_snapshot= (timestamp) + $_hash_snapshot = timestamp + end + + def redis_snapshot + return Lit.saved_redis_snapshot unless Lit.saved_redis_snapshot.nil? + timestamp = Lit.redis.get(Lit.prefix + '_snapshot') + if timestamp.nil? + timestamp = DateTime.now.to_f.to_s + Lit.redis_snapshot = timestamp + end + Lit.saved_redis_snapshot = timestamp.to_f + end + + def redis_snapshot= (timestamp) + Lit.redis.set(Lit.prefix + '_snapshot', timestamp) + end + + def saved_redis_snapshot + $saved_redis_snapshot ||= Concurrent::MVar.new(nil) + $saved_redis_snapshot.value + end + + def saved_redis_snapshot= (snap) + $saved_redis_snapshot ||= Concurrent::MVar.new(nil) + $saved_redis_snapshot.set!(snap) + end + + def now_timestamp + DateTime.now.to_f.to_d + end + + def determine_redis_provider + ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL'] + end + + def prefix + pfx = 'lit:' + if Lit.storage_options.is_a?(Hash) + pfx += "#{Lit.storage_options[:prefix]}:" if Lit.storage_options.key?(:prefix) + end + pfx + end + + class HybridStorage + def initialize + Lit.redis + Lit._hash + end + + def [](key) + if Lit.hash_dirty? + Lit.hash_snapshot = Lit.now_timestamp + Lit._hash.clear + end + if Lit._hash.key? key + return Lit._hash[key] + else + redis_val = get_from_redis(key) + Lit._hash[key] = redis_val + end + end + + def get_from_redis(key) + if Lit.redis.exists(_prefixed_key_for_array(key)) + Lit.redis.lrange(_prefixed_key(key), 0, -1) + elsif Lit.redis.exists(_prefixed_key_for_nil(key)) + nil + else + Lit.redis.get(_prefixed_key(key)) + end + end + + def []=(k, v) + delete(k) + Lit._hash[k] = v + if v.is_a?(Array) + Lit.redis.set(_prefixed_key_for_array(k), '1') + v.each do |ve| + Lit.redis.rpush(_prefixed_key(k), ve.to_s) + end + elsif v.nil? + Lit.redis.set(_prefixed_key_for_nil(k), '1') + Lit.redis.set(_prefixed_key(k), '') + else + Lit.redis.set(_prefixed_key(k), v) + end + end + + def delete(k) + Lit.redis_snapshot = Lit.now_timestamp + Lit._hash.delete(k) + Lit.redis.del(_prefixed_key_for_array(k)) + Lit.redis.del(_prefixed_key_for_nil(k)) + Lit.redis.del(_prefixed_key(k)) + end + + def clear + Lit.redis_snapshot = Lit.now_timestamp + Lit._hash.clear + Lit.redis.del(keys) if keys.length > 0 + end + + def keys + Lit.redis.keys(_prefixed_key + '*') + end + + def has_key?(key) + Lit._hash.has_key?(key) || Lit.redis.exists(_prefixed_key(key)) # This is a derp + end + + def incr(key) + Lit.redis.incr(_prefixed_key(key)) + end + + def sort + Lit.redis.keys.sort.map do |k| + [k, self.[](k)] + end + end + + private + + def _prefix + Lit.prefix + end + + def _prefixed_key(key = '') + _prefix + key.to_s + end + + def _prefixed_key_for_array(key = '') + _prefix + 'array_flags:' + key.to_s + end + + def _prefixed_key_for_nil(key = '') + _prefix + 'nil_flags:' + key.to_s + end + end +end diff --git a/lib/lit/i18n_backend.rb b/lib/lit/i18n_backend.rb index a9056a07..745d3561 100644 --- a/lib/lit/i18n_backend.rb +++ b/lib/lit/i18n_backend.rb @@ -85,11 +85,9 @@ def lookup(locale, key, scope = [], options = {}) def store_item(locale, data, scope = [], unless_changed = false) if data.respond_to?(:to_hash) - # ActiveRecord::Base.transaction do - data.to_hash.each do |key, value| - store_item(locale, value, scope + [key], unless_changed) - end - # end + data.to_hash.each do |key, value| + store_item(locale, value, scope + [key], unless_changed) + end elsif data.respond_to?(:to_str) key = ([locale] + scope).join('.') @cache.update_locale(key, data, false, unless_changed) @@ -141,12 +139,17 @@ def is_ignored_key(key_without_locale) end def should_cache?(key_with_locale) - return false if @cache.has_key?(key_with_locale) + return false if @cache[key_with_locale] != nil _, key_without_locale = ::Lit::Cache.split_key(key_with_locale) return false if is_ignored_key(key_without_locale) true end + + def extract_non_symbol_default!(options) + defaults = [options[:default]].flatten + defaults.detect{|default| !default.is_a?(Symbol)} + end end end diff --git a/lit.gemspec b/lit.gemspec index 6abde74a..0e2fcef9 100644 --- a/lit.gemspec +++ b/lit.gemspec @@ -20,6 +20,7 @@ Gem::Specification.new do |s| s.add_dependency 'rails', '> 3.1.0' s.add_dependency 'i18n', '~> 0.7.0' s.add_dependency 'jquery-rails' + s.add_dependency 'concurrent-ruby' s.add_development_dependency 'pg' s.add_development_dependency 'devise' diff --git a/test/dummy/config/database.yml b/test/dummy/config/database.yml index 47455e7d..98a2ab84 100644 --- a/test/dummy/config/database.yml +++ b/test/dummy/config/database.yml @@ -15,8 +15,8 @@ development: test: adapter: postgresql database: lit_test - username: lit - password: lit + username: ebin + password: ebin host: localhost pool: 5 timeout: 5000 diff --git a/test/support/clear_snapshots.rb b/test/support/clear_snapshots.rb new file mode 100644 index 00000000..26428371 --- /dev/null +++ b/test/support/clear_snapshots.rb @@ -0,0 +1,6 @@ +require 'lit/adapters/hybrid_storage' +def clear_snapshots + Lit.reset_hash if defined?($_hash) + Lit.hash_snapshot = nil if defined?($_hash_snapshot) + Lit.redis.del(Lit.prefix + '_snapshot') if defined?($redis) +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 9f819809..2ea0f759 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -33,6 +33,7 @@ def load_sample_yml(fname) class ActiveSupport::TestCase self.use_transactional_fixtures = true setup do + clear_snapshots clear_redis Lit.init.cache.reset end diff --git a/test/unit/hybrid_storage_test.rb b/test/unit/hybrid_storage_test.rb new file mode 100644 index 00000000..3077d3b0 --- /dev/null +++ b/test/unit/hybrid_storage_test.rb @@ -0,0 +1,88 @@ +require 'test_helper' + +# Applicable only for LIT_STORAGE=hybrid +class HybridStorageTest < ActiveSupport::TestCase + if ENV['LIT_STORAGE'] == 'hybrid' + class Backend < Lit::I18nBackend + end + + fixtures :all + + def setup + Lit.init + Lit::Localization.delete_all + Lit::LocalizationKey.delete_all + Lit::LocalizationVersion.delete_all + @old_humanize_key = Lit.humanize_key + Lit.humanize_key = false + @old_load_path = I18n.load_path + Lit.reset_hash + I18n.backend.cache.clear + @locale = Lit::Locale.find_by_locale(I18n.locale) + super + end + + def teardown + Lit.loader = @old_loader + Lit.humanize_key = @old_humanize_key + I18n.backend = @old_backend + I18n.load_path = @old_load_path + super + end + + test 'it should update translation both in hash and in redis' do + # assertions to ensure that storage has been properly cleared + assert_nil Lit._hash['en.fizz'] + assert_nil Lit.redis.get(Lit.prefix + 'en.fizz') + I18n.t('fizz', default: 'buzz') + assert_equal 'buzz', Lit._hash['en.fizz'] + assert_equal 'buzz', Lit.redis.get(Lit.prefix + 'en.fizz') + end + + test 'it should clear hash when loading from redis something not yet in hash' do + # let's do something that creates a hash snapshot timestamp + assert_nil Lit._hash['en.fizz'] + old_hash_snapshot = Lit.hash_snapshot + I18n.t('fizz', default: 'buzz') + assert_operator Lit.hash_snapshot, :>, old_hash_snapshot + + # in the meantime let's create some new translation + # simulate as if it were created and redis snapshot has been updated + lk = Lit::LocalizationKey.create(localization_key: 'abcd') + l = lk.localizations.create!(locale: @locale, default_value: 'efgh') + + Lit.redis.set(Lit.prefix + 'en.abcd', 'efgh') + Lit.saved_redis_snapshot = Lit.now_timestamp + Lit.redis_snapshot = Lit.saved_redis_snapshot + # TODO: consider if this is not too implementation-specific + + # assert that the newly created localization has been fetched into hash + assert_equal 'efgh', I18n.t('abcd') + assert_equal 'efgh', Lit._hash['en.abcd'] + assert_equal 'efgh', Lit.redis.get(Lit.prefix + 'en.abcd') + + # assert that hash cache has been cleared + assert_nil Lit._hash['en.fizz'] + I18n.t('fizz') + + # assert that the value then gets loaded into hash again + assert_equal 'buzz', Lit._hash['en.fizz'] + end + + test 'local cache is used even when redis is cleared' do + # define a translation by specifying default value + assert_nil Lit._hash['en.fizz'] + I18n.t('fizz', default: 'buzz') + assert_equal 'buzz', Lit._hash['en.fizz'] + + # clear redis + I18n.backend.cache.clear + + # modify local cache and then see if it's used for loading translation + Lit._hash['en.fizz'] = 'fizzbuzz' + assert_equal 'fizzbuzz', I18n.t('fizz') + end + else + puts 'Skipping hybrid storage test' + end +end