DBMON-2051: Adding a deadlock metric to MySQL databases

mysql/changelog.d/16904.added

@@ -0,0 +1 @@
+DBMON-2051: Adding a deadlock metric to Maria DB

mysql/datadog_checks/mysql/mysql.py

@@ -56,6 +56,7 @@
 from .innodb_metrics import InnoDBMetrics
 from .metadata import MySQLMetadata
 from .queries import (
+    QUERY_DEADLOCKS,
     QUERY_USER_CONNECTIONS,
     SQL_95TH_PERCENTILE,
     SQL_AVG_QUERY_RUN_TIME,
@@ -140,6 +141,8 @@ def __init__(self, name, init_config, instances):
         # go through the agent internal metrics submission processing those tags
         self._non_internal_tags = copy.deepcopy(self.tags)
         self.set_resource_tags()
+        with self._connect() as db:
+            self._is_innodb_engine_enabled = self._is_innodb_engine_enabled(db)
 
     def execute_query_raw(self, query):
         with closing(self._conn.cursor(pymysql.cursors.SSCursor)) as cursor:
@@ -345,6 +348,9 @@ def runtime_queries(self):
 
         queries = []
 
+        if self._is_innodb_engine_enabled:
+            queries.extend([QUERY_DEADLOCKS])
+
         if self.performance_schema_enabled:
             queries.extend([QUERY_USER_CONNECTIONS])
 
@@ -445,9 +451,10 @@ def _collect_metrics(self, db, tags):
         with tracked_query(self, operation="variables_metrics"):
             results.update(self._get_stats_from_variables(db))
 
-        if not is_affirmative(
-            self._config.options.get('disable_innodb_metrics', False)
-        ) and self._is_innodb_engine_enabled(db):
+        if (
+            not is_affirmative(self._config.options.get('disable_innodb_metrics', False))
+            and self._is_innodb_engine_enabled
+        ):
             with tracked_query(self, operation="innodb_metrics"):
                 results.update(self.innodb_stats.get_stats_from_innodb_status(db))
             self.innodb_stats.process_innodb_stats(results, self._config.options, metrics)

mysql/datadog_checks/mysql/queries.py

@@ -74,6 +74,19 @@
 SELECT plugin_status
 FROM information_schema.plugins WHERE plugin_name='group_replication'"""
 
+QUERY_DEADLOCKS = {
+    'name': 'information_schema.INNODB_METRICS.lock_deadlocks',
+    'query': """
+        SELECT
+            count as deadlocks
+        FROM
+            information_schema.INNODB_METRICS
+        WHERE
+            NAME='lock_deadlocks'
+    """.strip(),
+    'columns': [{'name': 'mysql.innodb.deadlocks', 'type': 'gauge'}],
+}
+
 QUERY_USER_CONNECTIONS = {
     'name': 'performance_schema.threads',
     'query': """

mysql/metadata.csv

@@ -1,3 +1,3 @@
 metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric
 mysql.info.schema.size,gauge,,mebibyte,,"Size of schemas in MiB",0,mysql,mysql schema size,
 mysql.info.table.rows.read,count,,row,,"Total number of rows read per table (Percona userstat only)",0,mysql,mysql table rows read,
@@ -12,6 +12,7 @@
 mysql.innodb.current_row_locks,gauge,,lock,,The number of current row locks.,-1,mysql,current row locks,
 mysql.innodb.data_reads,gauge,,read,second,The rate of data reads.,0,mysql,data reads,
 mysql.innodb.data_writes,gauge,,write,second,The rate of data writes.,0,mysql,data writes,
+mysql.innodb.deadlocks,monotonic_count,,lock,,"The number of deadlocks.",0,mysql,number of deadlocks,
 mysql.innodb.mutex_os_waits,gauge,,event,second,The rate of mutex OS waits.,0,mysql,mutex os waits,
 mysql.innodb.mutex_spin_rounds,gauge,,event,second,The rate of mutex spin rounds.,0,mysql,mutex spin rounds,
 mysql.innodb.mutex_spin_waits,gauge,,event,second,The rate of mutex spin waits.,0,mysql,mutex spin waits,

mysql/tests/test_query_activity.py

@@ -11,6 +11,7 @@
 from copy import copy
 from datetime import datetime
 from os import environ
+from threading import Event
 
 import mock
 import pymysql
@@ -478,6 +479,87 @@ def _expected_dbm_job_err_tags(dbm_instance):
     ]
 
 
+@pytest.mark.integration
+@pytest.mark.usefixtures('dd_environment')
+def test_if_deadlock_metric_is_collected(aggregator, dd_run_check, dbm_instance):
+    check = MySql(CHECK_NAME, {}, [dbm_instance])
+    dd_run_check(check)
+    deadlock_metric = aggregator.metrics("mysql.innodb.deadlocks")
+
+    assert len(deadlock_metric) == 1, "there should be one deadlock metric"
+
+
+@pytest.mark.skipif(environ.get('MYSQL_FLAVOR') == 'mariadb', reason='Deadock count is not updated in MariaDB')
+@pytest.mark.integration
+@pytest.mark.usefixtures('dd_environment')
+def test_deadlocks(aggregator, dd_run_check, dbm_instance):
+    check = MySql(CHECK_NAME, {}, [dbm_instance])
+    dd_run_check(check)
+    deadlocks_start = 0
+    deadlock_metric_start = aggregator.metrics("mysql.innodb.deadlocks")
+
+    assert len(deadlock_metric_start) == 1, "there should be one deadlock metric"
+
+    deadlocks_start = deadlock_metric_start[0].value
+
+    first_query = "SELECT * FROM testdb.users WHERE id = 1 FOR UPDATE;"
+    second_query = "SELECT * FROM testdb.users WHERE id = 2 FOR UPDATE;"
+
+    def run_first_deadlock_query(conn, event1, event2):
+        conn.begin()
+        try:
+            conn.cursor().execute("START TRANSACTION;")
+            conn.cursor().execute(first_query)
+            event1.set()
+            event2.wait()
+            conn.cursor().execute(second_query)
+            conn.cursor().execute("COMMIT;")
+        except Exception:
+            # Exception is expected due to a deadlock
+            pass
+        conn.commit()
+
+    def run_second_deadlock_query(conn, event1, event2):
+        conn.begin()
+        try:
+            event1.wait()
+            conn.cursor().execute("START TRANSACTION;")
+            conn.cursor().execute(second_query)
+            event2.set()
+            conn.cursor().execute(first_query)
+            conn.cursor().execute("COMMIT;")
+        except Exception:
+            # Exception is expected due to a deadlock
+            pass
+        conn.commit()
+
+    bob_conn = _get_conn_for_user('bob')
+    fred_conn = _get_conn_for_user('fred')
+
+    executor = concurrent.futures.thread.ThreadPoolExecutor(2)
+
+    event1 = Event()
+    event2 = Event()
+
+    futures_first_query = executor.submit(run_first_deadlock_query, bob_conn, event1, event2)
+    futures_second_query = executor.submit(run_second_deadlock_query, fred_conn, event1, event2)
+    futures_first_query.result()
+    futures_second_query.result()
+    # Make sure innodb is updated.
+    time.sleep(0.3)
+    bob_conn.close()
+    fred_conn.close()
+    executor.shutdown()
+
+    dd_run_check(check)
+
+    deadlock_metric_end = aggregator.metrics("mysql.innodb.deadlocks")
+
+    assert (
+        len(deadlock_metric_end) == 2 and deadlock_metric_end[1].value - deadlocks_start == 1
+    ), "there should be one new deadlock"
+
+
 def _get_conn_for_user(user, _autocommit=False):
     return pymysql.connect(host=HOST, port=PORT, user=user, password=user, autocommit=_autocommit)
 

mysql/tests/variables.py

@@ -90,6 +90,7 @@
     'mysql.innodb.buffer_pool_read_requests',
     'mysql.innodb.buffer_pool_reads',
     'mysql.innodb.buffer_pool_utilization',
+    'mysql.innodb.deadlocks',
 ]
 
 COMPLEX_INNODB_VARS = [