From d767032b4542ec904fec4a7f96e181b77e5cd2d0 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Sat, 22 Jun 2013 07:55:22 -0400 Subject: [PATCH 1/6] add HDFS check using snakebite --- checks.d/hdfs.py | 45 ++++++++++++++++++++++++++++++++++++++++ conf.d/hdfs.yaml.example | 6 ++++++ 2 files changed, 51 insertions(+) create mode 100644 checks.d/hdfs.py create mode 100644 conf.d/hdfs.yaml.example diff --git a/checks.d/hdfs.py b/checks.d/hdfs.py new file mode 100644 index 0000000000..7d19527300 --- /dev/null +++ b/checks.d/hdfs.py @@ -0,0 +1,45 @@ +from checks import AgentCheck + +import snakebite.client + +class HDFSCheck(AgentCheck): + """Report on free space and space used in HDFS. + """ + + def check(self, instance): + if 'namenode' not in instance: + self.log.info('Missing key \'namenode\' in HDFSCheck config') + return + + hostport = instance['namenode'] + if ':' in hostport: + host, _, port = hostport.partition(':') + port = int(port) + else: + host = hostport + port = 8020 + + hdfs = snakebite.client.Client(host, port) + stats = hdfs.df() + # {'used': 2190859321781L, + # 'capacity': 76890897326080L, + # 'under_replicated': 0L, + # 'missing_blocks': 0L, + # 'filesystem': 'hdfs://hostname:port', + # 'remaining': 71186818453504L, + # 'corrupt_blocks': 0L} + + self.gauge('hdfs.used', stats['used']) + self.gauge('hdfs.free', stats['remaining']) + self.gauge('hdfs.capacity', stats['capacity']) + self.gauge('hdfs.under_replicated', stats['under_replicated']) + self.gauge('hdfs.missing_blocks', stats['missing_blocks']) + self.gauge('hdfs.corrupt_blocks', stats['corrupt_blocks']) + +if __name__ == '__main__': + check, instances = HDFSCheck.from_yaml('./hdfs.yaml') + for instance in instances: + check.check(instance) + print "Events: %r" % check.get_events() + print "Metrics: %r" % check.get_metrics() + diff --git a/conf.d/hdfs.yaml.example b/conf.d/hdfs.yaml.example new file mode 100644 index 0000000000..53536fe72e --- /dev/null +++ b/conf.d/hdfs.yaml.example @@ -0,0 +1,6 @@ +init_config: +# HDFS check does not require any init_config + +instances: +# Each instance requires a namenode hostname:port +- namenode: namenode.example.com:8020 From 6ff3a9491f34c5d4f24099af8a258e60104643c7 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Sat, 22 Jun 2013 10:15:29 -0400 Subject: [PATCH 2/6] add 'in_use' (percent used) gauge --- checks.d/hdfs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/checks.d/hdfs.py b/checks.d/hdfs.py index 7d19527300..24818eb4da 100644 --- a/checks.d/hdfs.py +++ b/checks.d/hdfs.py @@ -32,6 +32,7 @@ def check(self, instance): self.gauge('hdfs.used', stats['used']) self.gauge('hdfs.free', stats['remaining']) self.gauge('hdfs.capacity', stats['capacity']) + self.gauge('hdfs.in_use', float(stats['used']) / float(stats['capacity'])) self.gauge('hdfs.under_replicated', stats['under_replicated']) self.gauge('hdfs.missing_blocks', stats['missing_blocks']) self.gauge('hdfs.corrupt_blocks', stats['corrupt_blocks']) From d2ae1e20dce5477582cf88deb63a78d5df1cd364 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Tue, 2 Jul 2013 08:52:45 -0400 Subject: [PATCH 3/6] HDFSCheck: safely import snakebite.client --- checks.d/hdfs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/checks.d/hdfs.py b/checks.d/hdfs.py index 24818eb4da..e67de59744 100644 --- a/checks.d/hdfs.py +++ b/checks.d/hdfs.py @@ -1,12 +1,16 @@ from checks import AgentCheck -import snakebite.client class HDFSCheck(AgentCheck): """Report on free space and space used in HDFS. """ def check(self, instance): + try: + import snakebite.client + except ImportError: + raise ImportError('HDFSCheck requires the snakebite module') + if 'namenode' not in instance: self.log.info('Missing key \'namenode\' in HDFSCheck config') return From bc904f3ab7cc9d697dc56058ac9cb578055c401f Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Tue, 2 Jul 2013 08:53:13 -0400 Subject: [PATCH 4/6] HDFSCheck: raise exception rather than logging and returning --- checks.d/hdfs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/checks.d/hdfs.py b/checks.d/hdfs.py index e67de59744..2d8d1e8688 100644 --- a/checks.d/hdfs.py +++ b/checks.d/hdfs.py @@ -12,8 +12,7 @@ def check(self, instance): raise ImportError('HDFSCheck requires the snakebite module') if 'namenode' not in instance: - self.log.info('Missing key \'namenode\' in HDFSCheck config') - return + raise ValueError('Missing key \'namenode\' in HDFSCheck config') hostport = instance['namenode'] if ':' in hostport: From 1e40685203099f86a68d34eb9541c74ee604cb22 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Tue, 2 Jul 2013 08:53:54 -0400 Subject: [PATCH 5/6] HDFSCheck: split hostport into 2 configuration items --- checks.d/hdfs.py | 11 ++++------- conf.d/hdfs.yaml.example | 6 ++++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/checks.d/hdfs.py b/checks.d/hdfs.py index 2d8d1e8688..68aa093d94 100644 --- a/checks.d/hdfs.py +++ b/checks.d/hdfs.py @@ -14,13 +14,10 @@ def check(self, instance): if 'namenode' not in instance: raise ValueError('Missing key \'namenode\' in HDFSCheck config') - hostport = instance['namenode'] - if ':' in hostport: - host, _, port = hostport.partition(':') - port = int(port) - else: - host = hostport - port = 8020 + host, port = instance['namenode'], instance.get('port', 8020) + if not isinstance(port, int): + # PyYAML converts the number to an int for us + raise ValueError('Port %r is not an integer' % port) hdfs = snakebite.client.Client(host, port) stats = hdfs.df() diff --git a/conf.d/hdfs.yaml.example b/conf.d/hdfs.yaml.example index 53536fe72e..6afc6d8cff 100644 --- a/conf.d/hdfs.yaml.example +++ b/conf.d/hdfs.yaml.example @@ -2,5 +2,7 @@ init_config: # HDFS check does not require any init_config instances: -# Each instance requires a namenode hostname:port -- namenode: namenode.example.com:8020 +# Each instance requires a namenode hostname. +# Port defaults to 8020. +- namenode: namenode.example.com + port: 8020 From 1165b07c3710653730617a81bb1e4000872f8efc Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Tue, 2 Jul 2013 08:54:06 -0400 Subject: [PATCH 6/6] whitespace --- checks.d/hdfs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/checks.d/hdfs.py b/checks.d/hdfs.py index 68aa093d94..55a51dfef0 100644 --- a/checks.d/hdfs.py +++ b/checks.d/hdfs.py @@ -43,4 +43,3 @@ def check(self, instance): check.check(instance) print "Events: %r" % check.get_events() print "Metrics: %r" % check.get_metrics() -