Option --raw for sending raw metrics

logster/parsers/MetricLogster.py

@@ -57,10 +57,13 @@ def __init__(self, option_string=None):
         optparser = optparse.OptionParser()
         optparser.add_option('--percentiles', '-p', dest='percentiles', default='90',
                             help='Comma-separated list of integer percentiles to track: (default: "90")')
+        optparser.add_option('--raw', action="store_true", dest='raw',
+                            help='Send raw metric to graphite, do not take time duration into metric calculation')
 
         opts, args = optparser.parse_args(args=options)
 
         self.percentiles = opts.percentiles.split(',')
+        self.raw_data = opts.raw
 
         # General regular expressions, expecting the metric name to be included in the log file.
 
@@ -92,8 +95,10 @@ def get_state(self, duration):
         and return a list of metric objects.'''
         duration = float(duration)
         metrics = []
-        if duration > 0:
+        if duration > 0 and not self.raw_data:
             metrics += [MetricObject(counter, self.counts[counter]/duration) for counter in self.counts]
+        elif self.raw_data:
+            metrics += [MetricObject(counter, self.counts[counter]) for counter in self.counts]
         for time_name in self.times:
             values = self.times[time_name]['values']
             unit = self.times[time_name]['unit']

logster/parsers/OccurrenceLogster.py

@@ -0,0 +1,71 @@
+import re
+import optparse
+
+from logster.logster_helper import MetricObject, LogsterParser, LogsterParsingException
+
+
+class OccurrenceLogster(LogsterParser):
+
+    def __init__(self, option_string=None):
+        '''Initialize any data structures or variables needed for keeping track
+        of the tasty bits we find in the log we are parsing.'''
+        self.occurrence = []
+        self.pattern_list = []
+        self.error_type = {}
+
+        if option_string:
+            options = option_string.split(' ')
+        else:
+            options = []
+
+        optparser = optparse.OptionParser()
+        optparser.add_option('--pattern-file', '-p', dest='pattern_file',
+                            help='Path to file with pattern list. File struct: "pattern-name";"pattern";')
+        optparser.add_option('--raw', action="store_true", dest='raw',
+                            help='Send raw metric to graphite, do not take time duration into metric calculation')
+
+        opts, args = optparser.parse_args(args=options)
+
+        if opts.pattern_file:
+            for line in open(opts.pattern_file):
+                if len(line.strip()) > 0:
+                    pattern_line = self.read_pattern_line(line)
+                    self.pattern_list.append([pattern_line[0], re.compile(pattern_line[1], flags=re.IGNORECASE)])
+        else:
+            optparser.error('File with patterns not given')
+
+        self.raw_data = opts.raw
+
+    def read_pattern_line(self, line):
+        line_split = line.split(" : ")
+        return [self.remove_quotation_marks(line_split[0]), self.remove_quotation_marks(line_split[1])]
+
+    def remove_quotation_marks(self, word):
+        return word.replace("\"", "").strip()
+
+    def parse_line(self, line):
+        '''This function should digest the contents of one line at a time, updating
+        object's state variables. Takes a single argument, the line to be parsed.'''
+
+        for pattern in self.pattern_list:
+            try:
+                search = pattern[1].search(line)
+                if search:
+                    self.error_type.update({pattern[0]: self.error_type.get(pattern[0], 0)+1})
+            except Exception as e:
+                raise LogsterParsingException("Pattern search failed with %s" % e)
+
+    def get_state(self, duration):
+        '''Run any necessary calculations on the data collected from the logs
+        and return a list of metric objects.'''
+        self.duration = float(duration)
+
+        # Return a list of metrics objects
+        result = []
+        for key, occ in self.error_type.iteritems():
+            if self.raw_data:
+                metric = MetricObject(key, occ, "Responses")
+            else:
+                metric = MetricObject(key, (occ / self.duration), "Responses per sec")
+            result.append(metric)
+        return result

tests/resources/patterns.txt

@@ -0,0 +1,2 @@
+"errors" : "error"
+"hour_pattern" : "(([0-9]{2}:){2}[0-9]{2})"

tests/test_occureencelogster.py

@@ -0,0 +1,72 @@
+from logster.parsers.OccurrenceLogster import OccurrenceLogster
+import unittest, os, time
+
+TEST_PATTERN_FILE = os.path.join(os.path.dirname(__file__), 'resources/patterns.txt')
+
+
+class TestOccurrenceLogster(unittest.TestCase):
+
+    def setUp(self):
+        self.logster_raw = OccurrenceLogster("-p {} --raw".format(TEST_PATTERN_FILE))
+        self.logster = OccurrenceLogster("-p {}".format(TEST_PATTERN_FILE))
+
+    def test_valid_lines_raw(self):
+        access_log_tmpl = "{} [%s], something goes in some way.".format(time.ctime())
+        self.logster_raw.parse_line(access_log_tmpl % 'ERROR')
+        self.logster_raw.parse_line(access_log_tmpl % 'error')
+        self.logster_raw.parse_line(access_log_tmpl % 'WARN')
+        self.assertEqual(2, self.logster_raw.error_type['errors'])
+        self.assertEqual(3, self.logster_raw.error_type['hour_pattern'])
+        self.assertEqual(2, len(self.logster_raw.error_type))
+
+    def test_metrics_1sec_raw(self):
+        self.test_valid_lines_raw()
+        metrics = self.logster_raw.get_state(1)
+        self.assertEqual(2, len(metrics))
+
+        expected = {"errors": 2,
+                    "hour_pattern": 3}
+        for m in metrics:
+            self.assertEqual(expected[m.name], m.value)
+
+    def test_metrics_2sec_raw(self):
+        self.test_valid_lines_raw()
+        metrics = self.logster_raw.get_state(2)
+        self.assertEqual(2, len(metrics))
+
+        expected = {"errors": 2,
+                    "hour_pattern": 3}
+        for m in metrics:
+            self.assertEqual(expected[m.name], m.value)
+
+    def test_valid_lines(self):
+        access_log_tmpl = "{} [%s], something goes in some way.".format(time.ctime())
+        self.logster.parse_line(access_log_tmpl % 'ERROR')
+        self.logster.parse_line(access_log_tmpl % 'error')
+        self.logster.parse_line(access_log_tmpl % 'WARN')
+        self.assertEqual(2, self.logster.error_type['errors'])
+        self.assertEqual(3, self.logster.error_type['hour_pattern'])
+        self.assertEqual(2, len(self.logster.error_type))
+
+    def test_metrics_1sec(self):
+        self.test_valid_lines()
+        metrics = self.logster.get_state(1)
+        self.assertEqual(2, len(metrics))
+
+        expected = {"errors": 2,
+                    "hour_pattern": 3}
+        for m in metrics:
+            self.assertEqual(expected[m.name], m.value)
+
+    def test_metrics_2sec(self):
+        self.test_valid_lines()
+        metrics = self.logster.get_state(2)
+        self.assertEqual(2, len(metrics))
+
+        expected = {"errors": 1.0,
+                    "hour_pattern": 1.5}
+        for m in metrics:
+            self.assertEqual(expected[m.name], m.value)
+
+if __name__ == '__main__':
+    unittest.main()