Merge pull request #1427 from brettlangdon/fail2ban-check

Fail2ban check

checks.d/fail2ban.py

@@ -0,0 +1,216 @@
+# stdlib
+import subprocess
+
+# project
+from checks import AgentCheck
+
+
+class Fail2Ban(AgentCheck):
+    """ Fail2Ban agent check
+
+    This check is used to run the `fail2ban-client` command to get the status
+    of active fail2ban jails. The following is an example of the metrics retrieved
+    for a single jail:
+
+    fail2ban.filter.currently_failed = 2       {'type': 'gauge', 'tags': ['jail:ssh']}
+    fail2ban.action.total_banned     = 4955    {'type': 'gauge', 'tags': ['jail:ssh']}
+    fail2ban.action.currently_banned = 1       {'type': 'gauge', 'tags': ['jail:ssh']}
+    fail2ban.filter.total_failed     = 61645   {'type': 'gauge', 'tags': ['jail:ssh']}
+    """
+
+    STATS = dict(
+        filter=["currently_failed", "total_failed"],
+        action=["currently_banned", "total_banned"],
+    )
+    SERVICE_CHECK_NAME = "fail2ban.can_connect"
+
+    def check(self, instance):
+        """ Run the check
+
+        Each instance accepts the following paramaters:
+
+        sudo                Optional: Boolean - whether or not to prepend "sudo" to the fail2ban-client calls
+        jail_blacklist      Optional: List - any fail2ban jails to omit, e.g. "ssh-ddos"
+        tags                Optional: List - any additional tags to send with each metric for the instance
+        """
+        sudo = instance.get("sudo", False)
+        jail_blacklist = instance.get("jail_blacklist", [])
+        instance_tags = instance.get('tags', [])
+
+        if self.can_ping_fail2ban(sudo=sudo):
+            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.OK, tags=instance_tags)
+            stats = self.get_jail_stats(sudo=sudo, jail_blacklist=jail_blacklist)
+            for jail_name, metric, value in stats:
+                tags = instance_tags + ["jail:%s" % (jail_name, )]
+                self.gauge(metric, value, tags=tags)
+        else:
+            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, tags=instance_tags)
+
+    def execute_command(self, args, sudo=False):
+        """ Helper method used to execute the given command
+
+        :param args: Command arguments to execute, e.g. ["fail2ban-client", "status"]
+        :type args: list
+        :param sudo: Whether or not to prepend "sudo" to the argument list [default: False]
+        :type sudo: bool
+        :rtype: generator
+        :returns: a generator which will yield for every line in the commands stdout
+        """
+        if sudo:
+            args.insert(0, "sudo")
+        process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        output, err = process.communicate()
+        if output and not err and process.returncode == 0:
+            for line in output.split("\n"):
+                yield line
+
+    def get_jails(self, sudo=False, jail_blacklist=None):
+        """ Get a list of enabled jails
+
+        :param sudo: Whether or not sudo is needed [default: False]
+        :type sudo: bool
+        :param jail_blacklist: A list of jails to omit [default: None]
+        :type jail_blacklist: None|list
+        :rtype: generator
+        :returns: a generator which will yield for every jail name found (minus those
+            provided in ``jail_blacklist``)
+        """
+        for line in self.execute_command(["fail2ban-client", "status"], sudo=sudo):
+            if "list" in line:
+                _, _, jails = line.rpartition(":")
+                for jail in jails.split(","):
+                    jail = jail.strip(" \t\r\n")
+                    if not jail_blacklist or jail not in jail_blacklist:
+                        yield jail
+
+    def get_jail_status(self, jail, sudo=False):
+        """ Parse the status output for a given jail
+
+        The output will look like
+
+        .. code:: python
+
+            {
+              "filter": {
+                "total_failed": 456,
+                "currently_failed": 0
+              },
+              "action": {
+                "currently_banned": 2,
+                "total_banned": 345
+              }
+            }
+
+
+        Parsing these metrics is a little messy since the output of fail2ban-client looks like:
+
+        .. code:: none
+
+            Status for the jail: ssh
+            |- filter
+            |  |- File list:/var/log/auth.log
+            |  |- Currently failed:0
+            |  `- Total failed:62170
+            `- action
+               |- Currently banned:1
+               |  `- IP list:222.186.56.43
+               `- Total banned:4978
+
+
+        The main idea of this method is that we want to parse "filter" and "action" as
+        top level "buckets" and parse everything under them as metrics, so we end
+        up with the above example output
+
+        :param jail: The name of the jail to get the status of
+        :type jail: str
+        :param sudo: Whether or not sudo is needed to execute [default: False]
+        :type sudo: bool
+        :rtype: dict
+        :returns: A dictionary mapping jail filter or action metrics
+        """
+        contents = self.execute_command(["fail2ban-client", "status", jail], sudo=sudo)
+
+        # these two generator expresstions are used to "peel back" one level of the output,
+        # so that anything starting with "-" is our top level "bucket"
+        #
+        # [
+        #   "- filter",
+        #   "  |- File list:/var/log/auth.log",
+        #   "  |- Currently failed:0",
+        #   "  `- Total failed:62170",
+        #   "- action",
+        #   "  |- Currently banned:1",
+        #   "  |  `- IP list:222.186.56.43",
+        #   "  `- Total banned:4978"
+        # ]
+        contents = (l.strip("\r\n") for l in contents
+                    if len(l) and l[0] in (" ", "|", "-", "`"))
+        contents = (l.strip("|`") for l in contents)
+
+        last = None
+        tree = dict()
+        for line in contents:
+            # "- filter" or "- action"
+            if line.startswith("-"):
+                last = line.strip("- ")
+                tree[last] = dict()
+            elif last in tree:
+                # this is a metric under "filter" or "action", do some serious
+                # cleanup of excess characters, partition on ":", then do some more cleanup
+                part = line[2:].strip(" |`-")
+                key, _, value = part.partition(":")
+                key = key.strip(" \t").replace(" ", "_").lower()
+                value = value.strip(" \t")
+                if value != '':
+                    tree[last][key] = value
+
+        return tree
+
+    def get_jail_stats(self, sudo=False, jail_blacklist=None):
+        """ Parse out all the available stats from fail2ban-client
+
+        The output will be a generator which emits tuples used for emitting
+        metrics, the tuples will look like the following
+
+        .. code:: python
+
+            [('ssh', 'fail2ban.action.currently_banned', '1'),
+             ('ssh', 'fail2ban.action.total_banned', '4979'),
+             ('ssh', 'fail2ban.filter.currently_failed', '1'),
+             ('ssh', 'fail2ban.filter.total_failed', '62177'),
+             ('ssh_ddos', 'fail2ban.action.currently_banned', '0'),
+             ('ssh_ddos', 'fail2ban.action.total_banned', '19'),
+             ('ssh_ddos', 'fail2ban.filter.currently_failed', '0'),
+             ('ssh_ddos', 'fail2ban.filter.total_failed', '1425')]
+
+
+        :param sudo: Whether or not sudo is needed to execute [default: False]
+        :type sudo: bool
+        :param jail_blacklist: A list of jails to omit [default: None]
+        :type jail_blacklist: None|list
+        :rtype: generator
+        :returns: A generator which emits tuples ``(<jail_name>, <metric_name>, <metric_value>)``
+        """
+        for jail in self.get_jails(sudo=sudo, jail_blacklist=jail_blacklist):
+            jail_status = self.get_jail_status(jail, sudo=sudo)
+            jail_name = jail.replace("-", "_")
+            for stat, substats in self.STATS.iteritems():
+                for substat in substats:
+                    value = jail_status.get(stat, {}).get(substat, 0)
+                    yield (jail_name, "fail2ban.%s.%s" % (stat, substat), value)
+
+    def can_ping_fail2ban(self, sudo=False):
+        """ Check if we can ping fail2ban server
+
+        Simply executes ``fail2ban-client ping``, expects the response "pong"
+
+        :param sudo: Whether or not sudo is needed to execute [default: False]
+        :type sudo: bool
+        :rtype: bool
+        :returns: Whether or not we got a "pong" response back
+        """
+        output = self.execute_command(["fail2ban-client", "ping"], sudo=sudo)
+        for line in output:
+            if "pong" in line:
+                return True
+        return False

conf.d/fail2ban.yaml.example

@@ -0,0 +1,22 @@
+init_config:
+
+# this check uses fail2ban-client to get the status from fail2ban, usually you need root
+# in order to access the fail2ban-server
+#
+# since this check runs as the dd-agent user you will probably need to set sudo: True
+# as well as granting the dd-agent user rights to execute the fail2ban-client command
+# using sudo, you can do so by adding the following to your /etc/sudoers file
+#
+#   dd-agent ALL=(ALL) NOPASSWD: /usr/bin/fail2ban-client
+
+instances:
+  - # sudo: True
+    # this probably needs to be set to True, but lets leave commented out, just in case
+
+    # tags:
+    #   - optional:tag
+    # additional tags to emit along with the metrics
+
+    # jail_blacklist:
+    #   - optional_blacklist
+    # use this blacklist to omit fetching and emiting status/metrics for a given jail, e.g. "ssh"

tests/checks/mock/test_fail2ban.py

@@ -0,0 +1,151 @@
+# stdlib
+import subprocess  # noqa
+
+# third party
+import mock
+
+# project
+from tests.checks.common import AgentCheckTest
+
+
+class Fail2BanTestCase(AgentCheckTest):
+    CHECK_NAME = 'fail2ban'
+
+    def setUp(self):
+        self.config = {
+            "instances": [
+                {
+                    "sudo": True,
+                },
+            ],
+        }
+        self.load_check(self.config)
+
+    def test_execute_command(self):
+        with mock.patch("subprocess.Popen") as popen:
+            popen.return_value = mock.Mock()
+            popen.return_value.returncode = 0
+            popen.return_value.communicate.return_value = ("Output\nHere", None)
+            output = self.check.execute_command(["some", "args"])
+            self.assertEquals(["Output", "Here"], list(output))
+            args = list(popen.call_args)
+            self.assertTrue(["some", "args"] in args[0])
+
+    def test_execute_command_sudo(self):
+        with mock.patch("subprocess.Popen") as popen:
+            popen.return_value = mock.Mock()
+            popen.return_value.returncode = 0
+            popen.return_value.communicate.return_value = ("Output\nHere", None)
+            output = self.check.execute_command(["some", "args"], sudo=True)
+            self.assertEquals(["Output", "Here"], list(output))
+            args = list(popen.call_args)
+            self.assertTrue(["sudo", "some", "args"] in args[0])
+
+    def test_get_jails(self):
+        with mock.patch.object(self.check, "execute_command") as execute_command:
+            execute_command.return_value = [
+                "Status",
+                "|- Number of jail:\t2"
+                "`- Jail list:\tssh, ssh-ddos"
+            ]
+
+            self.assertEquals(["ssh", "ssh-ddos"], list(self.check.get_jails()))
+            self.assertEquals(["ssh"], list(self.check.get_jails(jail_blacklist=["ssh-ddos"])))
+            self.assertEquals([], list(self.check.get_jails(jail_blacklist=["ssh-ddos", "ssh"])))
+
+    def test_get_jail_status(self):
+        status_output = [
+            "Status for the jail: ssh",
+            "|- filter",
+            "|  |- File list:\t/var/log/auth.log",
+            "|  |- Currently failed:\t2",
+            "|  `- Total failed:\t62219",
+            "`- action",
+            "   |- Currently banned:\t2",
+            "   |  `- IP list:\t104.217.154.54 222.186.56.43",
+            "   `- Total banned:\t4985"
+        ]
+        expected = {
+            "filter": {
+                "file_list": "/var/log/auth.log",
+                "currently_failed": "2",
+                "total_failed": "62219"
+            },
+            "action": {
+                "currently_banned": "2",
+                "ip_list": "104.217.154.54 222.186.56.43",
+                "total_banned": "4985"
+            }
+        }
+        with mock.patch.object(self.check, "execute_command") as execute_command:
+            execute_command.return_value = status_output
+            status = self.check.get_jail_status("ssh")
+            self.assertEqual(expected, status)
+
+    def test_get_jail_stats(self):
+        jails = ["ssh"]
+        jail_status = {
+            "filter": {
+                "file_list": "/var/log/auth.log",
+                "currently_failed": "2",
+                "total_failed": "62219"
+            },
+            "action": {
+                "currently_banned": "2",
+                "ip_list": "104.217.154.54 222.186.56.43",
+                "total_banned": "4985"
+            }
+        }
+        expected = [
+            ("ssh", "fail2ban.action.currently_banned", "2"),
+            ("ssh", "fail2ban.action.total_banned", "4985"),
+            ("ssh", "fail2ban.filter.currently_failed", "2"),
+            ("ssh", "fail2ban.filter.total_failed", "62219")
+        ]
+        with mock.patch.object(self.check, "get_jail_status") as get_jail_status:
+            with mock.patch.object(self.check, "get_jails") as get_jails:
+                get_jails.return_value = jails
+                get_jail_status.return_value = jail_status
+                stats = self.check.get_jail_stats()
+                self.assertEquals(expected, list(stats))
+
+    def test_can_ping_fail2ban_pong(self):
+        with mock.patch.object(self.check, "execute_command") as execute_command:
+            execute_command.return_value = ["Server replied: pong"]
+            self.assertTrue(self.check.can_ping_fail2ban())
+            execute_command.assert_called_with(["fail2ban-client", "ping"], sudo=False)
+
+    def test_can_ping_fail2ban_fail(self):
+        with mock.patch.object(self.check, "execute_command") as execute_command:
+            # if it cannot connect we will get a subprocess.CalledProcessError
+            # which means execute_command will return []
+            execute_command.return_value = []
+            self.assertFalse(self.check.can_ping_fail2ban())
+            execute_command.assert_called_with(["fail2ban-client", "ping"], sudo=False)
+
+    def test_check(self):
+        def mock_can_ping_fail2ban(sudo=False):
+            return True
+
+        def mock_get_jail_stats(sudo=False, jail_blacklist=None):
+            return [
+                ("ssh", "fail2ban.action.currently_banned", "2"),
+                ("ssh", "fail2ban.action.total_banned", "4985"),
+                ("ssh", "fail2ban.filter.currently_failed", "2"),
+                ("ssh", "fail2ban.filter.total_failed", "62219")
+            ]
+
+        mocks = {
+            "can_ping_fail2ban": mock_can_ping_fail2ban,
+            "get_jail_stats": mock_get_jail_stats,
+        }
+
+        self.run_check(self.config, mocks=mocks)
+        expected_metrics = [
+            ('fail2ban.filter.total_failed', '62219', ['jail:ssh']),
+            ('fail2ban.action.total_banned', '4985', ['jail:ssh']),
+            ('fail2ban.action.currently_banned', '2', ['jail:ssh']),
+            ('fail2ban.filter.currently_failed', '2', ['jail:ssh'])
+        ]
+        for metric, value, tags in expected_metrics:
+            self.assertMetric(metric, value=value, tags=tags)