Hybrid hash/redis storage

.travis.yml

@@ -4,7 +4,7 @@ rvm:
   - 2.0.0
   - 2.1.2
   - 2.2.5
-  - 2.3.1
+  - 2.3.0
 gemfile:
   - gemfiles/rails_4.gemfile
   - gemfiles/rails_4.1.gemfile
@@ -14,6 +14,7 @@ services:
 env:
   - LIT_STORAGE=hash
   - LIT_STORAGE=redis
+  - LIT_STORAGE=hybrid
 before_script:
   - cp test/dummy/config/database.yml.travis test/dummy/config/database.yml
   - psql -c 'create database lit_test;' -U postgres

lib/lit.rb

@@ -46,6 +46,9 @@ def self.get_key_value_engine
         when 'redis'
           require 'lit/adapters/redis_storage'
           return RedisStorage.new
+        when 'hybrid'
+          require 'lit/adapters/hybrid_storage'
+          return HybridStorage.new
         else
           require 'lit/adapters/hash_storage'
           return HashStorage.new

lib/lit/adapters/hybrid_storage.rb

@@ -0,0 +1,180 @@
+require 'redis'
+require 'concurrent'
+
+ActionController::Base.class_eval do
+  after_action :clear_saved_redis_snapshot
+
+  def clear_saved_redis_snapshot
+    Lit.saved_redis_snapshot = nil
+  end
+end
+
+module Lit
+  extend self
+  def redis
+    $redis = Redis.new(url: determine_redis_provider) unless $redis
+    $redis
+  end
+
+  def determine_redis_provider
+    ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+  end
+
+  def _hash
+    $_hash ||= ::Concurrent::Hash.new
+  end
+
+  def reset_hash
+    $_hash = nil
+  end
+
+  def hash_dirty?
+    # Hash is considered dirty if hash snapshot is older
+    # than Redis snapshot.
+    Lit.hash_snapshot < Lit.redis_snapshot
+  end
+
+  def hash_snapshot
+    $_hash_snapshot ||= DateTime.new.to_f.to_d
+  end
+
+  def hash_snapshot= (timestamp)
+    $_hash_snapshot = timestamp
+  end
+
+  def redis_snapshot
+    return Lit.saved_redis_snapshot unless Lit.saved_redis_snapshot.nil?
+    timestamp = Lit.redis.get(Lit.prefix + '_snapshot')
+    if timestamp.nil?
+      timestamp = DateTime.now.to_f.to_s
+      Lit.redis_snapshot = timestamp
+    end
+    Lit.saved_redis_snapshot = timestamp.to_f
+  end
+
+  def redis_snapshot= (timestamp)
+    Lit.redis.set(Lit.prefix + '_snapshot', timestamp)
+  end
+
+  def saved_redis_snapshot
+    $saved_redis_snapshot ||= Concurrent::MVar.new(nil)
+    $saved_redis_snapshot.value
+  end
+
+  def saved_redis_snapshot= (snap)
+    $saved_redis_snapshot ||= Concurrent::MVar.new(nil)
+    $saved_redis_snapshot.set!(snap)
+  end
+
+  def now_timestamp
+    DateTime.now.to_f.to_d
+  end
+
+  def determine_redis_provider
+    ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+  end
+
+  def prefix
+    pfx = 'lit:'
+    if Lit.storage_options.is_a?(Hash)
+      pfx += "#{Lit.storage_options[:prefix]}:" if Lit.storage_options.key?(:prefix)
+    end
+    pfx
+  end
+
+  class HybridStorage
+    def initialize
+      Lit.redis
+      Lit._hash
+    end
+
+    def [](key)
+      if Lit.hash_dirty?
+        Lit.hash_snapshot = Lit.now_timestamp
+        Lit._hash.clear
+      end
+      if Lit._hash.key? key
+        return Lit._hash[key]
+      else
+        redis_val = get_from_redis(key)
+        Lit._hash[key] = redis_val
+      end
+    end
+
+    def get_from_redis(key)
+      if Lit.redis.exists(_prefixed_key_for_array(key))
+        Lit.redis.lrange(_prefixed_key(key), 0, -1)
+      elsif Lit.redis.exists(_prefixed_key_for_nil(key))
+        nil
+      else
+        Lit.redis.get(_prefixed_key(key))
+      end
+    end
+
+    def []=(k, v)
+      delete(k)
+      Lit._hash[k] = v
+      if v.is_a?(Array)
+        Lit.redis.set(_prefixed_key_for_array(k), '1')
+        v.each do |ve|
+          Lit.redis.rpush(_prefixed_key(k), ve.to_s)
+        end
+      elsif v.nil?
+        Lit.redis.set(_prefixed_key_for_nil(k), '1')
+        Lit.redis.set(_prefixed_key(k), '')
+      else
+        Lit.redis.set(_prefixed_key(k), v)
+      end
+    end
+
+    def delete(k)
+      Lit.redis_snapshot = Lit.now_timestamp
+      Lit._hash.delete(k)
+      Lit.redis.del(_prefixed_key_for_array(k))
+      Lit.redis.del(_prefixed_key_for_nil(k))
+      Lit.redis.del(_prefixed_key(k))
+    end
+
+    def clear
+      Lit.redis_snapshot = Lit.now_timestamp
+      Lit._hash.clear
+      Lit.redis.del(keys) if keys.length > 0
+    end
+
+    def keys
+      Lit.redis.keys(_prefixed_key + '*')
+    end
+
+    def has_key?(key)
+      Lit._hash.has_key?(key) || Lit.redis.exists(_prefixed_key(key)) # This is a derp
+    end
+
+    def incr(key)
+      Lit.redis.incr(_prefixed_key(key))
+    end
+
+    def sort
+      Lit.redis.keys.sort.map do |k|
+        [k, self.[](k)]
+      end
+    end
+
+    private
+
+    def _prefix
+      Lit.prefix
+    end
+
+    def _prefixed_key(key = '')
+      _prefix + key.to_s
+    end
+
+    def _prefixed_key_for_array(key = '')
+      _prefix + 'array_flags:' + key.to_s
+    end
+
+    def _prefixed_key_for_nil(key = '')
+      _prefix + 'nil_flags:' + key.to_s
+    end
+  end
+end

lib/lit/i18n_backend.rb

@@ -85,11 +85,9 @@ def lookup(locale, key, scope = [], options = {})
 
     def store_item(locale, data, scope = [], unless_changed = false)
       if data.respond_to?(:to_hash)
-        # ActiveRecord::Base.transaction do
-          data.to_hash.each do |key, value|
-            store_item(locale, value, scope + [key], unless_changed)
-          end
-        # end
+        data.to_hash.each do |key, value|
+          store_item(locale, value, scope + [key], unless_changed)
+        end
       elsif data.respond_to?(:to_str)
         key = ([locale] + scope).join('.')
         @cache.update_locale(key, data, false, unless_changed)
@@ -141,12 +139,17 @@ def is_ignored_key(key_without_locale)
     end
 
     def should_cache?(key_with_locale)
-      return false if @cache.has_key?(key_with_locale)
+      return false if @cache[key_with_locale] != nil
 
       _, key_without_locale = ::Lit::Cache.split_key(key_with_locale)
       return false if is_ignored_key(key_without_locale)
 
       true
     end
+
+    def extract_non_symbol_default!(options)
+      defaults = [options[:default]].flatten
+      defaults.detect{|default| !default.is_a?(Symbol)}
+    end
   end
 end

lit.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'rails', '> 3.1.0'
   s.add_dependency 'i18n', '~> 0.7.0'
   s.add_dependency 'jquery-rails'
+  s.add_dependency 'concurrent-ruby'
 
   s.add_development_dependency 'pg'
   s.add_development_dependency 'devise'

test/dummy/config/database.yml

@@ -15,8 +15,8 @@ development:
 test:
   adapter: postgresql
   database: lit_test
-  username: lit
-  password: lit
+  username: ebin
+  password: ebin
   host: localhost
   pool: 5
   timeout: 5000

test/support/clear_snapshots.rb

@@ -0,0 +1,6 @@
+require 'lit/adapters/hybrid_storage'
+def clear_snapshots
+  Lit.reset_hash if defined?($_hash)
+  Lit.hash_snapshot = nil if defined?($_hash_snapshot)
+  Lit.redis.del(Lit.prefix + '_snapshot') if defined?($redis)
+end

test/test_helper.rb

@@ -33,6 +33,7 @@ def load_sample_yml(fname)
 class ActiveSupport::TestCase
   self.use_transactional_fixtures = true
   setup do
+    clear_snapshots
     clear_redis
     Lit.init.cache.reset
   end

test/unit/hybrid_storage_test.rb

@@ -0,0 +1,88 @@
+require 'test_helper'
+
+# Applicable only for LIT_STORAGE=hybrid
+class HybridStorageTest < ActiveSupport::TestCase
+  if ENV['LIT_STORAGE'] == 'hybrid'
+    class Backend < Lit::I18nBackend
+    end
+
+    fixtures :all
+
+    def setup
+      Lit.init
+      Lit::Localization.delete_all
+      Lit::LocalizationKey.delete_all
+      Lit::LocalizationVersion.delete_all
+      @old_humanize_key = Lit.humanize_key
+      Lit.humanize_key = false
+      @old_load_path = I18n.load_path
+      Lit.reset_hash
+      I18n.backend.cache.clear
+      @locale = Lit::Locale.find_by_locale(I18n.locale)
+      super
+    end
+
+    def teardown
+      Lit.loader = @old_loader
+      Lit.humanize_key = @old_humanize_key
+      I18n.backend = @old_backend
+      I18n.load_path = @old_load_path
+      super
+    end
+
+    test 'it should update translation both in hash and in redis' do
+      # assertions to ensure that storage has been properly cleared
+      assert_nil Lit._hash['en.fizz']
+      assert_nil Lit.redis.get(Lit.prefix + 'en.fizz')
+      I18n.t('fizz', default: 'buzz')
+      assert_equal 'buzz', Lit._hash['en.fizz']
+      assert_equal 'buzz', Lit.redis.get(Lit.prefix + 'en.fizz')
+    end
+
+    test 'it should clear hash when loading from redis something not yet in hash' do
+      # let's do something that creates a hash snapshot timestamp
+      assert_nil Lit._hash['en.fizz']
+      old_hash_snapshot = Lit.hash_snapshot
+      I18n.t('fizz', default: 'buzz')
+      assert_operator Lit.hash_snapshot, :>, old_hash_snapshot
+
+      # in the meantime let's create some new translation
+      # simulate as if it were created and redis snapshot has been updated
+      lk = Lit::LocalizationKey.create(localization_key: 'abcd')
+      l = lk.localizations.create!(locale: @locale, default_value: 'efgh')
+
+      Lit.redis.set(Lit.prefix + 'en.abcd', 'efgh')
+      Lit.saved_redis_snapshot = Lit.now_timestamp
+      Lit.redis_snapshot = Lit.saved_redis_snapshot
+      # TODO: consider if this is not too implementation-specific
+
+      # assert that the newly created localization has been fetched into hash
+      assert_equal 'efgh', I18n.t('abcd')
+      assert_equal 'efgh', Lit._hash['en.abcd']
+      assert_equal 'efgh', Lit.redis.get(Lit.prefix + 'en.abcd')
+
+      # assert that hash cache has been cleared
+      assert_nil Lit._hash['en.fizz']
+      I18n.t('fizz')
+
+      # assert that the value then gets loaded into hash again
+      assert_equal 'buzz', Lit._hash['en.fizz']
+    end
+
+    test 'local cache is used even when redis is cleared' do
+      # define a translation by specifying default value
+      assert_nil Lit._hash['en.fizz']
+      I18n.t('fizz', default: 'buzz')
+      assert_equal 'buzz', Lit._hash['en.fizz']
+
+      # clear redis
+      I18n.backend.cache.clear
+
+      # modify local cache and then see if it's used for loading translation
+      Lit._hash['en.fizz'] = 'fizzbuzz'
+      assert_equal 'fizzbuzz', I18n.t('fizz')
+    end
+  else
+    puts 'Skipping hybrid storage test'
+  end
+end