Statsd logger

Gemfile

@@ -2,3 +2,18 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in goliath-contrib.gemspec
 gemspec
+
+gem 'goliath', :path => '../goliath'
+
+group :development do
+  gem 'rake'
+end
+
+# Gems for testing and coverage
+group :test do
+  gem 'pry'
+  gem 'rspec'
+  gem 'guard'
+  gem 'guard-rspec'
+  gem 'guard-yard'
+end

Guardfile

@@ -0,0 +1,20 @@
+# -*- ruby -*-
+
+format = 'progress' # 'doc' for more verbose, 'progress' for less
+tags   = %w[ ]
+guard 'rspec', :version => 2, :cli => "--format #{format} #{ tags.map{|tag| "--tag #{tag}"}.join(' ')  }" do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})           { |m| "spec/#{m[1]}_spec.rb"  }
+  watch(%r{^examples/(.+)\.rb$})      { |m| "spec/integration/#{m[1]}_spec.rb" }
+
+  watch('spec/spec_helper.rb')         { 'spec' }
+  watch(/spec\/support\/(.+)\.rb/)     { 'spec' }
+end
+
+group :docs do
+  guard 'yard' do
+    watch(%r{app/.+\.rb})
+    watch(%r{lib/.+\.rb})
+    watch(%r{ext/.+\.c})
+  end
+end

Rakefile

@@ -1,2 +1,27 @@
-#!/usr/bin/env rake
-require "bundler/gem_tasks"
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'yard'
+require 'rspec/core/rake_task'
+require 'rake/testtask'
+
+task :default => [:test]
+task :test => [:spec, :unit]
+
+desc "run the unit test"
+Rake::TestTask.new(:unit) do |t|
+   t.libs << "test"
+   t.test_files = FileList['test/**/*_test.rb']
+   t.verbose = true
+end
+
+desc "run spec tests"
+RSpec::Core::RakeTask.new('spec') do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+desc 'Generate documentation'
+YARD::Rake::YardocTask.new do |t|
+  t.files   = ['lib/**/*.rb', '-', 'LICENSE']
+  t.options = ['--main', 'README.md', '--no-private']
+end

examples/statsd_demo.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+# See notes in examples/test_rig for preconditions and usage
+
+require File.expand_path('test_rig', File.dirname(__FILE__))
+require 'goliath/contrib/statsd_agent'
+
+class StatsdDemo < TestRig
+  statsd_agent = Goliath::Contrib::StatsdAgent.new('statsd_demo', '33.33.33.30')
+  plugin Goliath::Contrib::Plugin::StatsdPlugin, statsd_agent
+  use    Goliath::Contrib::Rack::StatsdLogger,   statsd_agent
+
+  self.set_middleware!
+
+end

examples/test_rig.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+# $:.unshift File.expand_path('../lib', File.dirname(__FILE__))
+
+require 'goliath'
+require 'goliath/contrib/rack/configurator'
+require 'goliath/contrib/rack/diagnostics'
+require 'goliath/contrib/rack/force_delay'
+require 'goliath/contrib/rack/force_drop'
+require 'goliath/contrib/rack/force_fault'
+require 'goliath/contrib/rack/force_response'
+require 'goliath/contrib/rack/force_timeout'
+require 'goliath/contrib/rack/handle_exceptions'
+
+#
+# A test endpoint allowing fault injection, variable delay, or a response forced
+# by the client.  Besides being a nice demo of those middlewares, it's a useful
+# test dummy for seeing how your SOA apps handle downstream failures.
+#
+# Launch with
+#
+#     bundle exec ./examples/test_rig.rb       -s -p 9000 -e development &
+#
+# If using it as a test rig, launch with `-e production`. The test rig acts on
+# the following URL parameters:
+#
+# * `_force_timeout`                    -- raise an error if response takes longer than given time
+# * `_force_delay`                      -- delay the given length of time before responding
+# * `_force_drop`/`_force_drop_after`   -- drop connection immediately with no response
+# * `_force_fail`/`_force_fail_after`   -- raise an error of the given type (eg `_force_fail_pre=400` causes a BadRequestError)
+# * `_force_status`, `_force_headers`, or `_force_body' --  replace the given component directly.
+#
+# @example delay for 2 seconds:
+#   curl -v 'http://127.0.0.1:9000/?_force_delay=2'
+#   => Headers: X-Resp-Delay: 2.0 / X-Resp-Randelay: 0.0 / X-Resp-Actual: 2.003681182861328
+#
+# @example drop connection:
+#   curl -v 'http://127.0.0.1:9000/?_force_drop=true'
+#
+# @example delay for 2 seconds, then drop the connection:
+#   curl -v 'http://127.0.0.1:9000/?_force_delay=2&_force_drop_after=true'
+#
+# @example force timeout; first call is 200 OK, second will error with 408 RequestTimeoutError:
+#   curl -v 'http://127.0.0.1:9000/?_force_timeout=1.0&_force_delay=0.5'
+#   => Headers: X-Resp-Delay: 0.5 / X-Resp-Randelay: 0.0 / X-Resp-Actual: 0.513401985168457 / X-Resp-Timeout: 1.0
+#   curl -v 'http://127.0.0.1:9000/?_force_timeout=1.0&_force_delay=2.0'
+#   => {"status":408,"error":"RequestTimeoutError","message":"Request exceeded 1.0 seconds"}
+#
+# @example simulate a 503:
+#   curl -v 'http://127.0.0.1:9000/?_force_fault=503'
+#   => {"status":503,"error":"ServiceUnavailableError","message":"Injected middleware fault 503"}
+#
+# @example force-set headers and body:
+#   curl -v -H "Content-Type: application/json" --data-ascii '{"_force_headers":{"X-Question":"What is brown and sticky"},"_force_body":{"answer":"a stick"}}' 'http://127.0.0.1:9001/'
+#   => {"answer":"a stick"}
+#
+class TestRig < Goliath::API
+  include Goliath::Contrib::CaptureHeaders
+
+  def self.set_middleware!
+    use Goliath::Rack::Heartbeat                 # respond to /status with 200, OK (monitoring, etc)
+    use Goliath::Rack::Tracer                    # log trace statistics
+    use Goliath::Rack::DefaultMimeType           # cleanup accepted media types
+    use Goliath::Rack::Render, 'json'            # auto-negotiate response format
+    use Goliath::Contrib::Rack::HandleExceptions # turn raised errors into HTTP responses
+    use Goliath::Rack::Params                    # parse & merge query and body parameters
+
+    # turn params like '_force_delay' into env vars :force_delay
+    use(Goliath::Contrib::Rack::ConfigurateFromParams,
+      [ :force_timeout, :force_drop, :force_drop_after, :force_fault, :force_fault_after,
+        :force_status, :force_headers, :force_body, :force_delay, :force_randelay, ],)
+
+    use Goliath::Contrib::Rack::ForceTimeout     # raise an error if response takes longer than given time
+    use Goliath::Contrib::Rack::ForceDrop        # drop connection immediately with no response
+    use Goliath::Contrib::Rack::ForceFault       # raise an error of the given type (eg `_force_fault=400` causes a BadRequestError)
+    use Goliath::Contrib::Rack::ForceResponse    # replace as given by '_force_status', '_force_headers' or '_force_body'
+    use Goliath::Contrib::Rack::ForceDelay       # force response to take at least (_force_delay + rand*_force_randelay) seconds
+    use Goliath::Contrib::Rack::Diagnostics      # summarize the request in the response headers
+  end
+  self.set_middleware!
+
+  def response(env)
+    [200, { 'X-API' => self.class.name }, {}]
+  end
+end

goliath-contrib.gemspec

@@ -1,23 +1,52 @@
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require 'goliath/contrib/version'
 
-require File.expand_path('../lib/goliath/contrib', __FILE__)
+Gem::Specification.new do |s|
+  s.name        = "goliath-contrib"
+  s.version     = Goliath::Contrib::VERSION
 
-# require './lib/goliath/contrib'
+  s.authors     = ["goliath-io"]
+  s.email       = ["[email protected]"]
 
-Gem::Specification.new do |gem|
-  gem.authors       = ["goliath-io"]
-  gem.email         = ["[email protected]"]
+  s.homepage    = "https://github.com/postrank-labs/goliath-contrib"
+  s.summary     = "Contributed Goliath middleware, plugins, and utilities"
+  s.description = s.summary
 
-  gem.homepage      = "https://github.com/postrank-labs/goliath-contrib"
-  gem.description   = "Contributed Goliath middleware, plugins, and utilities"
-  gem.summary       = gem.description
+  s.required_ruby_version = '>=1.9.2'
 
-  gem.files         = `git ls-files`.split($\)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.name          = "goliath-contrib"
-  gem.require_paths = ["lib"]
-  gem.version       = Goliath::Contrib::VERSION
+  s.add_dependency 'goliath'
 
-  gem.add_dependency 'goliath'  
+  s.add_development_dependency 'rspec', '>2.0'
+
+  s.add_development_dependency 'em-http-request', '>=1.0.0'
+  s.add_development_dependency 'postrank-uri'
+
+  s.add_development_dependency 'guard'
+  s.add_development_dependency 'guard-rspec'
+  if RUBY_PLATFORM.include?('darwin')
+    s.add_development_dependency 'growl', '~> 1.0.3'
+    s.add_development_dependency 'rb-fsevent'
+  end
+
+  if RUBY_PLATFORM != 'java'
+    s.add_development_dependency 'yajl-ruby'
+    s.add_development_dependency 'bluecloth'
+    s.add_development_dependency 'bson_ext'
+  else
+    s.add_development_dependency 'json-jruby'
+    s.add_development_dependency 'maruku'
+  end
+
+  ignores = File.readlines(".gitignore").grep(/\S+/).map {|i| i.chomp }
+  dotfiles = [".gemtest", ".gitignore", ".rspec", ".yardopts"]
+
+  # s.files         = `git ls-files`.split($\)
+  # s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  # s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  # s.require_paths = ["lib"]
+
+  s.files = Dir["**/*"].reject {|f| File.directory?(f) || ignores.any? {|i| File.fnmatch(i, f) } } + dotfiles
+  s.test_files = s.files.grep(/^spec\//)
+  s.require_paths = ['lib']
 end

lib/goliath/contrib.rb

@@ -1,11 +1,9 @@
+require 'goliath/contrib'
+
 # TODO: tries to start server :-)
 # require 'goliath'
 
 module Goliath
-  module Contrib
-  	VERSION = "1.0.0.beta1"
-  end
-
   # autoload :MiddlewareName, "goliath/contrib/middleware_name"
   # ...
 end

lib/goliath/contrib/plugin/statsd_plugin.rb

@@ -0,0 +1,63 @@
+module Goliath
+  module Contrib
+    module Plugin
+
+      # Initializes the statsd agent and dispatches regular metrics about this server
+      #
+      # Often enjoyed in the company of the Goliath::Contrib::Rack::StatsdLogger middleware.
+      #
+      # @example
+      #   plugin Goliath::Contrib::Plugin::StatsdPlugin, Goliath::Contrib::StatsdAgent.new('my_app', '33.33.33.30')
+      #
+      # A URL something like this will show you the state of the reactor latency:
+      #
+      #     http://33.33.33.30:5100/render/?from=-12minutes
+      #       &width=960&height=720
+      #       &yMin=&yMax=
+      #       &colorList=67A9CF,91CF60,1A9850,FC8D59,D73027
+      #       &bgcolor=FFFFF0
+      #       &fgcolor=808080
+      #       &target=stats.timers.statsd_demo.reactor.latency.lower
+      #       &target=stats.timers.statsd_demo.reactor.latency.mean_90
+      #       &target=stats.timers.statsd_demo.reactor.latency.upper_90
+      #       &target=stats.timers.statsd_demo.reactor.latency.upper
+      #
+      class StatsdPlugin
+        attr_reader :agent
+
+        # Called by the framework to initialize the plugin
+        #
+        # @param port [Integer] Unused
+        # @param global_config [Hash] The server configuration data
+        # @param status [Hash] A status hash
+        # @param logger [Log4R::Logger] The logger
+        # @return [Goliath::Contrib::Plugins::StatsdPlugin] An instance of the Goliath::Contrib::Plugins::StatsdPlugin plugin
+        def initialize(port, global_config, status, logger)
+          @logger = logger
+          @config = global_config
+        end
+
+        # Called automatically to start the plugin
+        #
+        # @example
+        #   plugin Goliath::Contrib::Plugin::StatsdPlugin, Goliath::Contrib::StatsdAgent.new('my_app')
+        def run(agent)
+          @agent = agent
+          agent.logger ||= @logger
+          register_latency_timer
+        end
+
+        # Send canary packets to the statsd reporting on this server's latency every 1 second
+        def register_latency_timer
+          @logger.info{ "#{self.class} registering timer for reactor latency" }
+          @last   = Time.now.to_f
+          #
+          EM.add_periodic_timer(1.0) do
+            agent.timing 'reactor.latency', (Time.now.to_f - @last)
+            @last = Time.now.to_f
+          end
+        end
+      end
+    end
+  end
+end

lib/goliath/contrib/rack/configurator.rb

@@ -0,0 +1,48 @@
+module Goliath
+  module Contrib
+    module Rack
+
+      # Place static values fron initialize into the env on each request
+      class StaticConfigurator
+        include Goliath::Rack::AsyncMiddleware
+
+        def initialize(app, env_vars)
+          @extra_env_vars = env_vars
+          super(app)
+        end
+
+        def call(env,*)
+          env.merge!(@extra_env_vars)
+          super
+        end
+      end
+
+      #
+      #
+      # @example imposes a timeout if 'rapid_timeout' param is present
+      #   class RapidServiceOrYour408Back < Goliath::API
+      #     use Goliath::Rack::Params
+      #     use ConfigurateFromParams, [:timeout], 'rapid'
+      #     use Goliath::Contrib::Rack::ForceTimeout
+      #   end
+      #
+      class ConfigurateFromParams
+        include Goliath::Rack::AsyncMiddleware
+
+        def initialize(app, param_keys, slug='')
+          @extra_env_vars = param_keys.inject({}){|acc,el| acc[el.to_sym] = [slug, el].join("_") ; acc }
+          super(app)
+        end
+
+        def call(env,*)
+          @extra_env_vars.each do |env_key, param_key|
+            # env.logger.info [env_key, param_key, env.params[param_key]]
+            env[env_key] ||= env.params.delete(param_key)
+          end
+          super
+        end
+      end
+
+    end
+  end
+end