Added HTTP protocol

.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+.idea/

README.md

@@ -25,12 +25,72 @@ Add the following to your collectd config **or** use the included kairosdb.conf.
         Import "kairosdb_writer"
 
         <Module "kairosdb_writer">
-            KairosDBHost "localhost"
-            KairosDBPort 4242
-            KairosDBProtocol "tcp"
+            KairosDBURI "telnet://localhost:4242"
             LowercaseMetricNames true
             TypesDB "/usr/share/collectd/types.db" "/etc/collectd/types/custom.db"
+            ConvertToRate "interface", "cpu"
         </Module>
     </Plugin>
+
+
+### Properties
+*AddHostTag* - adds a host tag if true. True by default.   
+
+*Formatter* - the full path to a formatter.  See the Formatters section below.
+
+*PluginFormatterPath* - the path to the directory where plugin formatters exist. Optional. See the Formatters section below. 
+
+*HostSeparator* - separator character used between host name parts. Defaults to underscore("_").  
+
+*LowercaseMetricNames* - lower cases the metric name if true. Defaults to false.  
+
+*KairosDBURI* - URI for the Kairos host, must be in the form <protocol>://<host>[:<port>].  Protocol may be one of (telnet, http, https). Required.   
+
+*TypesDB* - Data-set specifications. See [Types.db](https://collectd.org/documentation/manpages/types.db.5.shtml). 
+
+*MetricName* - the name of the metric. This is built using pre-defined variables. See [Naming Schema](https://collectd.org/wiki/index.php/Naming_schema) for information about these variables. 
+  For example, if the metric name is set to "collectd.%(plugin)s.%(plugin_instance)s.%(type)s.otherstuff", this will produce a metric name that looks like this 
+  "collectd.processes.ps_state.blocked.value.otherstuff". The pre-defined variables are *host*, *plugin*, *plugin_instance*, *type*, and *type_instance*. The default is "collectd.%(plugin)s.%(plugin_instance)s.%(type)s.%(type_instance)s".
+
+*MetricSeparator* - separator character used between metric name parts. Defaults to a period(".").
+
+*ConvertToRate* - converts COUNTER and DERIVE values to rates for the listed plugins. This is a comma delimited list of plugin names. The counter values are suppressed and a new metric containing the rate is sent to KairosDB. The name for rates will contain "_rate" on the end of the name.
+
+*Tags* - KairosDB tags to send. At least one tag is required. The host name is added as a tag by default unless AddHostTag is set to false. For example, "customer=acme"
+
+*ThrowawaySampleAge* - Discards metrics if they have not been sent to Kairos in a certain period of time. The time specified is in seconds. Some plugins collect such a large number of metrics that the pushing of metrics may take so much time that not all metrics are sent to Kairos before the next reporting cycle. This option will discard any metrics that are not sent after the specified period of time.
+
+*VerboseLogging* - Add additional logging information.
+
+
+### Formatters
+Formatters provide a way to customize the metric name or tags. A formatter is a python script that has a format function. See the ./examples/defaultFormatter.py for an example of a default formatter.
+See the ./examples/cpuFormatter.py for an example of a plugin formatter.
+There are two ways to create formatters.
+
+ 1. Default Formatter - The "Formatter" property let's you specify the location of a single formatter that will be called for all plugins.  
+ 2. Plugin Formatters - Formatters that applies to a specific list of plugins can be created. These formatters are loaded from the "formatters" subdirectory where kairosdb_writer.py exists. 
+
+Formatters are used in the following way.
+
+ When the writer is first loaded the default formatter and the plugin formatters are loaded. If the "Formatter" property exists, the default formatter is loaded from the specified location. 
+ All plugin formatter files that have a .py extension found in the "formatters" directory are loaded.
+
+ When the writer is called with values for a plugin then
+
+ 1. If a plugin formatter exists for the plugin, its format function is called and the default formatter is not called.
+ 2. If no plugin formatter is found and the default formatter exists then its format function is called. 
+
+#### Formatter Functions
+Both the default formatter and specific plugin formatters must have a format function with the following signature. It returns the metric name and a dictionary of tags (tag name/value).
+
+    format(metric_name_template, tags, hostname, plugin, plugin_instance, type, type_instance)
+
+Plugin formatters must also include a plugins function which is called when the writer is loaded. This returns a list of plugin names that this formatter applies to.
+
+    plugins()
+
+
+
 
 

examples/cassandraFormatter.py

@@ -0,0 +1,14 @@
+# Adds column family names as tags and removes them from the metric name.
+# This was written for use with the FastJMX Collectd plugin that reads JMX metrics from Cassandra.
+# NOTE: The code looks for "column_family" as the plugin name and replaces it with "cassandra". In the FastJMX configuration a value can have a "PluginName" property
+# which is set to "cassandra" for all metrics except those that have metrics for all column families. These I set to "column_family" as a way to easily
+#       identify metrics that this plugin will modify.
+# class Formatter:
+
+# For column families, remove plugin_instance from name and change plugin name to "cassandra" and add column_families as a tag
+def format_metric(metric_name_template, tags, hostname, plugin, plugin_instance, type, type_instance):
+    if plugin == 'column_family':
+        metric_name_template = metric_name_template.replace('%(plugin_instance)s.', '')
+        tags['column_family'] = plugin_instance
+        plugin = 'cassandra'
+    return metric_name_template % {'host': hostname, 'plugin': plugin, 'plugin_instance': plugin_instance, 'type': type, 'type_instance': type_instance}, tags

examples/cpuFormatter.py

@@ -0,0 +1,15 @@
+# Sample formatter for the cpu plugin.
+# cpu number and type are set as tags
+
+def plugins():
+    return ["cpu"]
+
+
+def format_metric(metric_name_template, tags, hostname, plugin, plugin_instance, type, type_instance):
+    metric_name_template = metric_name_template.replace('%(plugin_instance)s.', '')
+    metric_name_template = metric_name_template.replace('%(type)s.', '')
+    tags['cpu'] = plugin_instance
+    tags['type'] = type_instance
+    type_instance = 'utilization'
+
+    return metric_name_template % {'host': hostname, 'plugin': plugin, 'plugin_instance': plugin_instance, 'type': type, 'type_instance': type_instance}, tags

examples/defaultFormatter.py

@@ -0,0 +1,13 @@
+# Sample default formatter
+# Disk and interface (network) plugin metrics names are set along with tags
+
+def format_metric(metric_name_template, tags, hostname, plugin, plugin_instance, type, type_instance):
+    if plugin == 'disk':
+        metric_name_template = metric_name_template.replace('%(plugin_instance)s.', '')
+        tags['disk'] = plugin_instance
+
+    if plugin == 'interface':
+        metric_name_template = metric_name_template.replace('%(type_instance)s', '')
+        tags['interface'] = type_instance
+
+    return metric_name_template % {'host': hostname, 'plugin': plugin, 'plugin_instance': plugin_instance, 'type': type, 'type_instance': type_instance}, tags

kairosdb.conf

@@ -3,20 +3,22 @@
 </LoadPlugin>
 
 <Plugin "python">
-    ModulePath "/usr/lib64/collectd/"
+ModulePath "/usr/lib64/collectd/"
 
     Import "kairosdb_writer"
 
     <Module "kairosdb_writer">
-        #HostPostfix "myhost"
+KairosDBURI "telnet://localhost:4242"
+        #KairosDBURI "http://localhost:8080"
+        #KairosDBURI "https://localhost:443"
+        LowercaseMetricNames true
+TypesDB "/usr/share/collectd/types.db" #"/etc/collectd/types/custom.db"
+#Formatter "/opt/proofpoint/.../Formatter.py"
+        #AddHostTag false
         #HostSeparator "_"
-        KairosDBHost "localhost"
-        KairosDBPort 4242
-        KairosDBProtocol "tcp"
-        #MetricPrefix "collectd"
+        #MetricName "collectd.%(plugin)s.%(plugin_instance)s.%(type)s.%(type_instance)s"
         #MetricSeparator "."
-        LowercaseMetricNames true
-        Tags "host=web01.company.local" "role=web01"
-        TypesDB "/usr/share/collectd/types.db" "/etc/collectd/types/custom.db"
+        #Tags "role=web01"
+ConvertToRate "interface", "cpu"
     </Module>
 </Plugin>