Create hybrid hash/redis storage

Author: vincentvanbush

.travis.yml

@@ -1,21 +1,22 @@
 language: ruby
 sudo: false
 rvm:
-  - 2.0.0
-  - 2.1.2
-  - 2.2.5
-  - 2.3.1
+#  - 2.0.0
+#  - 2.1.2
+#  - 2.2.5
+   - 2.3.0
 gemfile:
-  - gemfiles/rails_4.gemfile
-  - gemfiles/rails_4.1.gemfile
+#  - gemfiles/rails_4.gemfile
+#  - gemfiles/rails_4.1.gemfile
   - gemfiles/rails_4.2.gemfile
 services:
   - redis-server
 env:
-  - LIT_STORAGE=hash
-  - LIT_STORAGE=redis
+  # - LIT_STORAGE=hash
+  # - LIT_STORAGE=redis
+  - LIT_STORAGE=hybrid
 before_script:
   - cp test/dummy/config/database.yml.travis test/dummy/config/database.yml
   - psql -c 'create database lit_test;' -U postgres
   - RAILS_ENV=test bundle exec rake db:migrate
-script: "bundle exec rake test"
+script: "bundle exec rake test" # TEST=test/unit/lit_behaviour_test.rb"

lib/lit.rb

@@ -46,6 +46,9 @@ def self.get_key_value_engine
         when 'redis'
           require 'lit/adapters/redis_storage'
           return RedisStorage.new
+        when 'hybrid'
+          require 'lit/adapters/hybrid_storage'
+          return HybridStorage.new
         else
           require 'lit/adapters/hash_storage'
           return HashStorage.new

lib/lit/adapters/hybrid_storage.rb

@@ -0,0 +1,153 @@
+require 'redis'
+require 'concurrent'
+
+module Lit
+  extend self
+  def redis
+    $redis = Redis.new(url: determine_redis_provider) unless $redis
+    $redis
+  end
+
+  def determine_redis_provider
+    ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+  end
+
+  def _hash
+    $_hash ||= ::Concurrent::Hash.new
+  end
+
+  def reset_hash
+    $_hash = nil
+  end
+
+  def hash_dirty?
+    # Hash is considered dirty if hash snapshot is older
+    # than Redis snapshot.
+    Lit.hash_snapshot < Lit.redis_snapshot
+  end
+
+  def hash_snapshot
+    $_hash_snapshot ||= DateTime.new
+  end
+
+  def hash_snapshot= (timestamp)
+    $_hash_snapshot = timestamp
+  end
+
+  def redis_snapshot
+    timestamp = Lit.redis.get('lit:_snapshot')
+    if timestamp.nil?
+      timestamp = Time.current.to_s
+      Lit.redis_snapshot = timestamp
+    end
+    DateTime.parse(timestamp)
+  end
+
+  def redis_snapshot= (timestamp)
+    Lit.redis.set('lit:_snapshot', timestamp)
+  end
+
+  def determine_redis_provider
+    ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
+  end
+
+  class HybridStorage
+    def initialize
+      Lit.redis
+      Lit._hash
+    end
+
+    def [](key)
+      if Lit.hash_dirty?
+        Lit.hash_snapshot = DateTime.current
+        Lit._hash.clear
+      end
+      if Lit._hash.key? key
+        return Lit._hash[key]
+      else
+        redis_val = get_from_redis(key)
+        Lit._hash[key] = redis_val
+      end
+    end
+
+    def get_from_redis(key)
+      if Lit.redis.exists(_prefixed_key_for_array(key))
+        Lit.redis.lrange(_prefixed_key(key), 0, -1)
+      elsif Lit.redis.exists(_prefixed_key_for_nil(key))
+        nil
+      else
+        Lit.redis.get(_prefixed_key(key))
+      end
+    end
+
+    def []=(k, v)
+      delete(k)
+      Lit._hash[k] = v
+      if v.is_a?(Array)
+        Lit.redis.set(_prefixed_key_for_array(k), '1')
+        v.each do |ve|
+          Lit.redis.rpush(_prefixed_key(k), ve.to_s)
+        end
+      elsif v.nil?
+        Lit.redis.set(_prefixed_key_for_nil(k), '1')
+        Lit.redis.set(_prefixed_key(k), '')
+      else
+        Lit.redis.set(_prefixed_key(k), v)
+      end
+    end
+
+    def delete(k)
+      Lit.redis_snapshot = Time.current
+      Lit._hash.delete(k)
+      Lit.redis.del(_prefixed_key_for_array(k))
+      Lit.redis.del(_prefixed_key_for_nil(k))
+      Lit.redis.del(_prefixed_key(k))
+    end
+
+    def clear
+      Lit.redis_snapshot = Time.current
+      Lit._hash.clear
+      Lit.redis.del(keys) if keys.length > 0
+    end
+
+    def keys
+      Lit.redis.keys(_prefixed_key + '*')
+    end
+
+    def has_key?(key)
+      Lit._hash.has_key?(key) || Lit.redis.exists(_prefixed_key(key)) # This is a derp
+    end
+
+    def incr(key)
+      Lit.redis.incr(_prefixed_key(key))
+    end
+
+    def sort
+      Lit.redis.keys.sort.map do |k|
+        [k, self.[](k)]
+      end
+    end
+
+    private
+
+    def _prefix
+      prefix = 'lit:'
+      if Lit.storage_options.is_a?(Hash)
+        prefix += "#{Lit.storage_options[:prefix]}:" if Lit.storage_options.key?(:prefix)
+      end
+      prefix
+    end
+
+    def _prefixed_key(key = '')
+      _prefix + key.to_s
+    end
+
+    def _prefixed_key_for_array(key = '')
+      _prefix + 'array_flags:' + key.to_s
+    end
+
+    def _prefixed_key_for_nil(key = '')
+      _prefix + 'nil_flags:' + key.to_s
+    end
+  end
+end

lib/lit/i18n_backend.rb

@@ -59,6 +59,7 @@ def lookup(locale, key, scope = [], options = {})
       return content if parts.size <= 1
 
       if should_cache?(key_with_locale)
+        puts "SHOULD CACHE #{key_with_locale}"
         new_content = @cache.init_key_with_value(key_with_locale, content)
         content = new_content if content.nil? # Content can change when Lit.humanize is true for example
 
@@ -141,12 +142,17 @@ def is_ignored_key(key_without_locale)
     end
 
     def should_cache?(key_with_locale)
-      return false if @cache.has_key?(key_with_locale)
+      return false if @cache[key_with_locale] != nil
 
       _, key_without_locale = ::Lit::Cache.split_key(key_with_locale)
       return false if is_ignored_key(key_without_locale)
 
       true
     end
+
+    def extract_non_symbol_default!(options)
+      defaults = [options[:default]].flatten
+      defaults.detect{|default| !default.is_a?(Symbol)}
+    end
   end
 end

lit.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'rails', '> 3.1.0'
   s.add_dependency 'i18n', '~> 0.7.0'
   s.add_dependency 'jquery-rails'
+  s.add_dependency 'concurrent-ruby'
 
   s.add_development_dependency 'pg'
   s.add_development_dependency 'devise'

test/dummy/config/database.yml

@@ -15,8 +15,8 @@ development:
 test:
   adapter: postgresql
   database: lit_test
-  username: lit
-  password: lit
+  username: ebin
+  password: ebin
   host: localhost
   pool: 5
   timeout: 5000

test/support/clear_snapshots.rb

@@ -0,0 +1,7 @@
+require 'lit/adapters/hybrid_storage'
+def clear_snapshots
+  trace_var :$_hash, proc { |h| print 'hash is now', v }
+  Lit.reset_hash if defined?($_hash)
+  Lit.hash_snapshot = nil if defined?($_hash_snapshot)
+  Lit.redis.del('lit:_snapshot') if defined?($redis)
+end

test/test_helper.rb

@@ -28,6 +28,7 @@
 class ActiveSupport::TestCase
   self.use_transactional_fixtures = true
   setup do
+    clear_snapshots
     clear_redis
     Lit.init.cache.reset
   end