Migration of elastic search and couch db checks to checks.d interface

.travis.yml

@@ -5,37 +5,44 @@ branches:
 services:
   - mysql
   - elasticsearch
+  - memcache
 before_script:
   - mysql -e "create user 'dog'@'localhost' identified by 'dog'"
 language: python
 python:
-  - "2.5"
   - "2.6"
   - "2.7"
-install:
-  - pip install -r requirements.txt --use-mirrors
-  - pip install . --use-mirrors
-script: nosetests -A 'not windows' tests
-before_script:
+before_install:
+  - sudo apt-get update
+  - sudo apt-get install openjdk-6-jre-headless
   - sudo apt-get install sysstat
   - sudo apt-get install haproxy
   - sudo apt-get install python-mysqldb
-  - curl -L https://raw.github.com/DataDog/dd-agent/master/tests/haproxy.cfg > /tmp/haproxy.cfg
-  - curl -L http://mirror.sdunix.com/apache/tomcat/tomcat-6/v6.0.36/bin/apache-tomcat-6.0.36.tar.gz | tar -C /tmp -xzf - && mv /tmp/apache-tomcat-6.0.36 /tmp/apache-tomcat-6 && echo 'export CATALINA_OPTS="-Dcom.sun.management.jmxremote.port=8090 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" export CATALINA_OUT="/tmp/apache-tomcat-6/catalina.out"' > /tmp/apache-tomcat-6/bin/setenv.sh
-  - curl -L http://mirrors.ibiblio.org/apache/tomcat/tomcat-7/v7.0.34/bin/apache-tomcat-7.0.34.tar.gz | tar -C /tmp -xzf - && mv /tmp/apache-tomcat-7.0.34/ /tmp/apache-tomcat-7 && echo 'export CATALINA_OPTS="-Dcom.sun.management.jmxremote.port=8091 -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.password.file=/tmp/apache-tomcat-7/conf/jmxremote.password -Dcom.sun.management.jmxremote.access.file=/tmp/apache-tomcat-7/conf/jmxremote.access -Dcom.sun.management.jmxremote.ssl=false" export CATALINA_OUT="/tmp/apache-tomcat-7/catalina.out"' > /tmp/apache-tomcat-7/bin/setenv.sh && echo 'monitorRole readonly' > /tmp/apache-tomcat-7/conf/jmxremote.access && echo 'monitorRole tomcat' > /tmp/apache-tomcat-7/conf/jmxremote.password && chmod 400 /tmp/apache-tomcat-7/conf/jmxremote.password
-  - curl -L https://raw.github.com/DataDog/dd-agent/master/tests/tomcat_cfg.xml > /tmp/apache-tomcat-6/conf/server.xml
-  - curl -L http://mirror.cc.columbia.edu/pub/software/apache/lucene/solr/3.6.1/apache-solr-3.6.1.tgz > /tmp/solr.tgz && tar -C /tmp -xzf /tmp/solr.tgz && mv /tmp/apache-solr-3.6.1 /tmp/apache-solr-3 && echo 'monitorRole readonly' > /tmp/apache-solr-3/example/jmxremote.access && echo 'monitorRole solr' > /tmp/apache-solr-3/example/jmxremote.password && chmod 400 /tmp/apache-solr-3/example/jmxremote.password
+  - sudo apt-get install tomcat6
+  - sudo apt-get install solr-tomcat
   - sudo apt-get install nginx
-  - curl -L https://raw.github.com/DataDog/dd-agent/master/tests/nginx.conf > /tmp/default.conf
-  - sudo cp /tmp/default.conf /etc/nginx/conf.d/default.conf
-  - sudo /etc/init.d/nginx restart
   - sudo apt-get install apache2
+  - sudo apt-get install couchdb
+install:
+  - pip install -r requirements.txt --use-mirrors
+  - pip install . --use-mirrors
+before_script:
+  - curl -L https://raw.github.com/DataDog/dd-agent/master/tests/haproxy.cfg > /tmp/haproxy.cfg
+  - sudo service haproxy restart
+  - sudo bash -c "curl -L https://raw.github.com/DataDog/dd-agent/master/tests/tomcat_cfg.xml > /etc/tomcat6/server.xml"
+  - sudo bash -c "curl -L https://raw.github.com/DataDog/dd-agent/master/tests/tomcat6 >> /etc/default/tomcat6"
+  - sudo service nginx stop
+  - sudo bash -c "curl -L https://raw.github.com/DataDog/dd-agent/master/tests/nginx.conf > /etc/nginx/conf.d/default.conf"
+  - sudo service apache2 stop
   - sudo bash -c "curl -L https://raw.github.com/DataDog/dd-agent/master/tests/apache/ports.conf > /etc/apache2/ports.conf"
   - sudo bash -c "curl -L https://raw.github.com/DataDog/dd-agent/master/tests/apache/apache.conf > /etc/apache2/apache.conf"
-  - sudo /etc/init.d/apache2 restart
-  - sudo apt-get remove memcached
-  - sudo apt-get install memcached
   - sudo mkdir -p /etc/dd-agent/
+  - sudo install -d -o "$(id -u)" /var/log/datadog
   - sudo bash -c "curl -L https://raw.github.com/DataDog/dd-agent/master/datadog.conf.example > /etc/dd-agent/datadog.conf"
+  - sudo service apache2 start
+  - sudo service nginx start
+  - sudo service tomcat6 restart
+  - sudo service couchdb start
 env:
   - DB=redis
+script: nosetests -A 'not windows' tests

checks.d/couch.py

@@ -0,0 +1,88 @@
+import urllib2
+from util import json, headers
+
+from checks import AgentCheck
+
+class CouchDb(AgentCheck):
+    """Extracts stats from CouchDB via its REST API
+    http://wiki.apache.org/couchdb/Runtime_Statistics
+    """
+    def _create_metric(self, data, tags=None):
+        overall_stats = data.get('stats', {})
+        for key, stats in overall_stats.items():
+            for metric, val in stats.items():
+                if val['current'] is not None:
+                    metric_name = '.'.join(['couchdb', key, metric])
+                    self.gauge(metric_name, val['current'], tags=tags)
+
+        for db_name, db_stats in data.get('databases', {}).items():
+            for name, val in db_stats.items():
+                if name in ['doc_count', 'disk_size'] and val is not None:
+                    metric_name = '.'.join(['couchdb', 'by_db', name])
+                    metric_tags = list(tags)
+                    metric_tags.append('db:%s' % db_name)
+                    self.gauge(metric_name, val, tags=metric_tags, device_name=db_name)
+
+
+    def _get_stats(self, url):
+        "Hit a given URL and return the parsed json"
+        self.log.debug('Fetching Couchdb stats at url: %s' % url)
+        req = urllib2.Request(url, None, headers(self.agentConfig))
+
+        # Do the request, log any errors
+        request = urllib2.urlopen(req)
+        response = request.read()
+        return json.loads(response)
+
+    def check(self, instance):
+        server = instance.get('server', None)
+        if server is None:
+            return False
+        data = self.get_data(server)
+        self._create_metric(data, tags=['instance:%s' % server])
+
+    def get_data(self, server):
+        # The dictionary to be returned.
+        couchdb = {'stats': None, 'databases': {}}
+
+        # First, get overall statistics.
+        endpoint = '/_stats/'
+
+        url = '%s%s' % (server, endpoint)
+        overall_stats = self._get_stats(url)
+
+        # No overall stats? bail out now
+        if overall_stats is None:
+            return False
+        else:
+            couchdb['stats'] = overall_stats
+
+        # Next, get all database names.
+        endpoint = '/_all_dbs/'
+
+        url = '%s%s' % (server, endpoint)
+        databases = self._get_stats(url)
+
+        if databases is not None:
+            for dbName in databases:
+                endpoint = '/%s/' % dbName
+
+                url = '%s%s' % (server, endpoint)
+                db_stats = self._get_stats(url)
+                if db_stats is not None:
+                    couchdb['databases'][dbName] = db_stats
+
+        return couchdb
+
+    @staticmethod
+    def parse_agent_config(agentConfig):
+        if not agentConfig.get('haproxy_url'):
+            return False
+
+        return {
+            'instances': [{
+                'server': agentConfig.get('couchdb_server'),
+            }]
+        }
+
+