Merged pull request #4 from temujin9/master.

Place statsd & graphite work in site-cookbooks

site-cookbooks/graphite/README.rdoc

@@ -0,0 +1,27 @@
+= DESCRIPTION:
+
+Installs and configures Graphite http://graphite.wikidot.com/
+
+= REQUIREMENTS:
+
+Ubuntu 10.04 (Lucid)
+
+= ATTRIBUTES:
+
+= USAGE:
+
+recipe[graphite] should build a stand-alone Graphite installation.
+
+recipe[graphite::ganglia] integrates with Ganglia. You'll want at
+least one monitor node (i.e. recipe[ganglia]) node to be running
+to use it.
+
+= CAVEATS:
+
+I didn't see an easy way to set Graphite's credentials easily.
+I set those to root/root with an email address going no where.
+
+Ships with two default schemas, stats.* (for Etsy's statsd) and a
+catchall that matches anything. The catchall retains minutely data for
+13 months, as in the default config. stats retains data every 10 seconds
+for 6 hours, every minute for a week, and every 10 minutes for 5 years.

site-cookbooks/graphite/attributes/graphite.rb

@@ -0,0 +1,15 @@
+default[:graphite][:carbon][:line_receiver_interface] =   "127.0.0.1"
+default[:graphite][:carbon][:pickle_receiver_interface] = "127.0.0.1"
+default[:graphite][:carbon][:cache_query_interface] =     "127.0.0.1"
+
+default[:graphite][:carbon][:version] = "0.9.7"
+default[:graphite][:carbon][:uri] = "http://launchpadlibrarian.net/61904798/carbon-0.9.7.tar.gz"
+default[:graphite][:carbon][:checksum] = "ba698aca"
+
+default[:graphite][:whisper][:version] = "0.9.7"
+default[:graphite][:whisper][:uri] = "http://launchpadlibrarian.net/61904764/whisper-0.9.7.tar.gz"
+default[:graphite][:whisper][:checksum] = "c6272ad6"
+
+default[:graphite][:graphite_web][:version] = "0.9.7c"
+default[:graphite][:graphite_web][:uri] = "http://launchpadlibrarian.net/62379635/graphite-web-0.9.7c.tar.gz"
+default[:graphite][:graphite_web][:checksum] = "a3e16265"

site-cookbooks/graphite/files/default/init.d_carbon-cache

@@ -0,0 +1,2 @@
+#!/bin/bash
+/opt/graphite/bin/carbon-cache.py $1

site-cookbooks/graphite/metadata.json

@@ -0,0 +1,41 @@
+{
+  "conflicting": {
+  },
+  "platforms": {
+    "ubuntu": [
+
+    ]
+  },
+  "license": "Apache 2.0",
+  "long_description": "= DESCRIPTION:\n\nInstalls and configures Graphite http://graphite.wikidot.com/\n\n= REQUIREMENTS:\n\nUbuntu 10.04 (Lucid)\n\n= ATTRIBUTES:\n\n= USAGE:\n\nrecipe[graphite] should build a stand-alone Graphite installation.\n\nrecipe[graphite::ganglia] integrates with Ganglia. You'll want at\nleast one monitor node (i.e. recipe[ganglia]) node to be running\nto use it.\n\n= CAVEATS:\n\nI didn't see an easy way to set Graphite's credentials easily.\nI set those to root/root with an email address going no where.\n\nShips with two default schemas, stats.* (for Etsy's statsd) and a\ncatchall that matches anything. The catchall retains minutely data for\n13 months, as in the default config. stats retains data every 10 seconds\nfor 6 hours, every minute for a week, and every 10 minutes for 5 years.\n",
+  "providing": {
+  },
+  "maintainer": "Heavy Water Software Inc.",
+  "replacing": {
+  },
+  "recipes": {
+  },
+  "maintainer_email": "[email protected]",
+  "description": "Installs/Configures graphite",
+  "version": "0.0.5",
+  "groupings": {
+  },
+  "recommendations": {
+  },
+  "attributes": {
+  },
+  "name": "graphite",
+  "suggestions": {
+  },
+  "dependencies": {
+    "python": [
+
+    ],
+    "ganglia": [
+
+    ],
+    "apache2": [
+
+    ]
+  }
+}

site-cookbooks/graphite/metadata.rb

@@ -0,0 +1,11 @@
+maintainer       "Heavy Water Software Inc."
+maintainer_email "[email protected]"
+license          "Apache 2.0"
+description      "Installs/Configures graphite"
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.rdoc'))
+version          "0.0.5"
+
+depends  "python"
+depends  "apache2"
+depends  "ganglia"
+supports "ubuntu"

site-cookbooks/graphite/recipes/carbon.rb

@@ -0,0 +1,45 @@
+package "python-twisted"
+
+remote_file "/usr/src/carbon-#{node.graphite.carbon.version}.tar.gz" do
+  source node.graphite.carbon.uri
+  checksum node.graphite.carbon.checksum
+end
+
+execute "untar carbon" do
+  command "tar xzf carbon-#{node.graphite.carbon.version}.tar.gz"
+  creates "/usr/src/carbon-#{node.graphite.carbon.version}"
+  cwd "/usr/src"
+end
+
+execute "install carbon" do
+  command "python setup.py install"
+  creates "/opt/graphite/lib/carbon-#{node.graphite.carbon.version}-py2.6.egg-info"
+  cwd "/usr/src/carbon-#{node.graphite.carbon.version}"
+end
+
+template "/opt/graphite/conf/carbon.conf" do
+  variables( :line_receiver_interface => node[:graphite][:carbon][:line_receiver_interface],
+             :pickle_receiver_interface => node[:graphite][:carbon][:pickle_receiver_interface],
+             :cache_query_interface => node[:graphite][:carbon][:cache_query_interface] )
+  notifies :restart, "service[carbon-cache]"
+end
+
+template "/opt/graphite/conf/storage-schemas.conf"
+
+cookbook_file "/etc/init.d/carbon-cache" do
+  source "init.d_carbon-cache"
+  mode 0755
+end
+
+# execute "setup carbon sysvinit script" do
+#   command "ln -nsf /opt/graphite/bin/carbon-cache.py /etc/init.d/carbon-cache"
+#   creates "/etc/init.d/carbon-cache"
+# end
+
+service "carbon-cache" do
+#   running true
+#   start_command "/opt/graphite/bin/carbon-cache.py start"
+#   stop_command "/opt/graphite/bin/carbon-cache.py stop"
+  action [ :enable, :start ]
+#   action :start
+end

site-cookbooks/graphite/recipes/default.rb

@@ -0,0 +1,24 @@
+#
+# Cookbook Name:: graphite
+# Recipe:: default
+#
+# Copyright 2011, Heavy Water Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "python"
+
+include_recipe "graphite::whisper"
+include_recipe "graphite::carbon"
+include_recipe "graphite::web"

site-cookbooks/graphite/recipes/ganglia.rb

@@ -0,0 +1,15 @@
+include_recipe "graphite"
+include_recipe "ganglia::gmetad"
+
+target = "/opt/graphite/storage/rrd/#{node.ganglia.cluster_name}"
+
+directory target do
+  mode "755"
+end
+
+Dir.glob("/var/lib/ganglia/rrds/#{node.ganglia.cluster_name}/*.*").each do |path|
+  source = File.basename(path).gsub(".", "_")
+  link "#{target}/#{source}" do
+    to path
+  end
+end

site-cookbooks/graphite/recipes/web.rb

@@ -0,0 +1,60 @@
+include_recipe "apache2::mod_python"
+
+package "python-cairo-dev"
+package "python-django"
+package "python-memcache"
+package "python-rrdtool"
+
+remote_file "/usr/src/graphite-web-#{node.graphite.graphite_web.version}.tar.gz" do
+  source node.graphite.graphite_web.uri
+  checksum node.graphite.graphite_web.checksum
+end
+
+execute "untar graphite-web" do
+  command "tar xzf graphite-web-#{node.graphite.graphite_web.version}.tar.gz"
+  creates "/usr/src/graphite-web-#{node.graphite.graphite_web.version}"
+  cwd "/usr/src"
+end
+
+remote_file "/usr/src/graphite-web-#{node.graphite.graphite_web.version}/webapp/graphite/storage.py.patch" do
+  source "http://launchpadlibrarian.net/65094495/storage.py.patch"
+  checksum "8bf57821"
+end
+
+execute "patch graphite-web" do
+  command "patch storage.py storage.py.patch"
+  creates "/opt/graphite/webapp/graphite_web-#{node.graphite.graphite_web.version}-py2.6.egg-info"
+  cwd "/usr/src/graphite-web-#{node.graphite.graphite_web.version}/webapp/graphite"
+end
+
+execute "install graphite-web" do
+  command "python setup.py install"
+  creates "/opt/graphite/webapp/graphite_web-#{node.graphite.graphite_web.version}-py2.6.egg-info"
+  cwd "/usr/src/graphite-web-#{node.graphite.graphite_web.version}"
+end
+
+template "/etc/apache2/sites-available/graphite" do
+  source "graphite-vhost.conf.erb"
+end
+
+apache_site "000-default" do
+  enable false
+end
+
+apache_site "graphite"
+
+directory "/opt/graphite/storage/log/webapp" do
+  owner "www-data"
+  group "www-data"
+end
+
+directory "/opt/graphite/storage" do
+  owner "www-data"
+  group "www-data"
+end
+
+cookbook_file "/opt/graphite/storage/graphite.db" do
+  owner "www-data"
+  group "www-data"
+  action :create_if_missing
+end

site-cookbooks/graphite/recipes/whisper.rb

@@ -0,0 +1,16 @@
+remote_file "/usr/src/whisper-#{node.graphite.whisper.version}.tar.gz" do
+  source node.graphite.whisper.uri
+  checksum node.graphite.whisper.checksum
+end
+
+execute "untar whisper" do
+  command "tar xzf whisper-#{node.graphite.whisper.version}.tar.gz"
+  creates "/usr/src/whisper-#{node.graphite.whisper.version}"
+  cwd "/usr/src"
+end
+
+execute "install whisper" do
+  command "python setup.py install"
+  creates "/usr/local/lib/python2.6/dist-packages/whisper-#{node.graphite.whisper.version}.egg-info"
+  cwd "/usr/src/whisper-#{node.graphite.whisper.version}"
+end

site-cookbooks/graphite/templates/default/carbon.conf.erb

@@ -0,0 +1,79 @@
+[cache]
+LOCAL_DATA_DIR = /opt/graphite/storage/whisper/
+
+# Specify the user to drop privileges to
+# If this is blank carbon runs as the user that invokes it
+# This user must have write access to the local data directory
+USER = 
+
+# Limit the size of the cache to avoid swapping or becoming CPU bound.
+# Sorts and serving cache queries gets more expensive as the cache grows.
+# Use the value "inf" (infinity) for an unlimited cache size.
+MAX_CACHE_SIZE = inf
+
+# Limits the number of whisper update_many() calls per second, which effectively
+# means the number of write requests sent to the disk. This is intended to
+# prevent over-utilizing the disk and thus starving the rest of the system.
+# When the rate of required updates exceeds this, then carbon's caching will
+# take effect and increase the overall throughput accordingly.
+MAX_UPDATES_PER_SECOND = 1000
+
+# Softly limits the number of whisper files that get created each minute.
+# Setting this value low (like at 50) is a good way to ensure your graphite
+# system will not be adversely impacted when a bunch of new metrics are
+# sent to it. The trade off is that it will take much longer for those metrics'
+# database files to all get created and thus longer until the data becomes usable.
+# Setting this value high (like "inf" for infinity) will cause graphite to create
+# the files quickly but at the risk of slowing I/O down considerably for a while.
+MAX_CREATES_PER_MINUTE = inf
+
+LINE_RECEIVER_INTERFACE = <%= @line_receiver_interface %>
+LINE_RECEIVER_PORT = 2003
+
+PICKLE_RECEIVER_INTERFACE = <%= @pickle_receiver_interface %>
+PICKLE_RECEIVER_PORT = 2004
+
+CACHE_QUERY_INTERFACE = <%= @cache_query_interface %>
+CACHE_QUERY_PORT = 7002
+
+# By default, carbon-cache will log every whisper update. This can be excessive and
+# degrade performance if logging on the same volume as the whisper data is stored.
+LOG_UPDATES = True
+
+
+# Enable AMQP if you want to receve metrics using an amqp broker
+# ENABLE_AMQP = False
+
+# Verbose means a line will be logged for every metric received
+# useful for testing
+# AMQP_VERBOSE = False
+
+# AMQP_HOST = localhost
+# AMQP_PORT = 5672
+# AMQP_VHOST = /
+# AMQP_USER = guest
+# AMQP_PASSWORD = guest
+# AMQP_EXCHANGE = graphite
+
+# Patterns for all of the metrics this machine will store. Read more at
+# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings
+#
+# Example: store all sales, linux servers, and utilization metrics
+# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization
+#
+# Example: store everything
+# BIND_PATTERNS = #
+
+# NOTE: you cannot run both a cache and a relay on the same server
+# with the default configuration, you have to specify a distinict
+# interfaces and ports for the listeners.
+
+#[relay]
+#LINE_RECEIVER_INTERFACE = 0.0.0.0
+#LINE_RECEIVER_PORT = 2003
+
+#PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+#PICKLE_RECEIVER_PORT = 2004
+
+#CACHE_SERVERS = server1, server2, server3
+#MAX_QUEUE_SIZE = 10000

site-cookbooks/graphite/templates/default/graphite-vhost.conf.erb

@@ -0,0 +1,38 @@
+NameVirtualHost *:80
+
+# You may need to manually edit this file to fit your needs.
+# This configuration assumes the default installation prefix
+# of /opt/graphite/, if you installed graphite somewhere else
+# you will need to change all the occurances of /opt/graphite/
+# in this file to your chosen install location.
+
+<VirtualHost *:80>
+        ServerName graphite
+        DocumentRoot "/opt/graphite/webapp"
+        ErrorLog /opt/graphite/storage/log/webapp/error.log
+        CustomLog /opt/graphite/storage/log/webapp/access.log common
+
+        <Location "/">
+                SetHandler python-program
+                PythonPath "['/opt/graphite/webapp'] + sys.path"
+                PythonHandler django.core.handlers.modpython
+                SetEnv DJANGO_SETTINGS_MODULE graphite.settings
+                PythonDebug Off
+                PythonAutoReload Off
+        </Location>
+
+        <Location "/content/">
+                SetHandler None
+        </Location>
+
+        <Location "/media/">
+                SetHandler None
+        </Location>
+
+	# NOTE: In order for the django admin site media to work you
+	# must change @DJANGO_ROOT@ to be the path to your django
+	# installation, which is probably something like:
+	# /usr/lib/python2.6/site-packages/django
+        Alias /media/ "@DJANGO_ROOT@/contrib/admin/media/"
+
+</VirtualHost>

site-cookbooks/graphite/templates/default/storage-schemas.conf.erb

@@ -0,0 +1,9 @@
+[stats]
+priority = 100
+pattern = ^stats\..*
+retentions = 10:2160,60:10080,600:262974
+
+[catchall]
+priority = 0
+pattern = ^.*
+retentions = 60:565920