Add a repair issue when using MariaDB is affected by MDEV-25020 (#87040)

closes #83787

homeassistant/components/recorder/strings.json

@@ -7,5 +7,11 @@
       "database_engine": "Database Engine",
       "database_version": "Database Version"
     }
+  },
+  "issues": {
+    "maria_db_range_index_regression": {
+      "title": "Update MariaDB to {min_version} or later resolve a significant performance issue",
+      "description": "Older versions of MariaDB suffer from a significant performance regression when retrieving history data or purging the database. Update to MariaDB version {min_version} or later and restart Home Assistant. If you are using the MariaDB core add-on, make sure to update it to the latest version."
+    }
   }
 }

homeassistant/components/recorder/translations/en.json

@@ -1,4 +1,10 @@
 {
+    "issues": {
+        "maria_db_range_index_regression": {
+            "description": "Older versions of MariaDB suffer from a significant performance regression when retrieving history data or purging the database. Update to MariaDB version {min_version} or later and restart Home Assistant. If you are using the MariaDB core add-on, make sure to update it to the latest version.",
+            "title": "Update MariaDB to {min_version} or later resolve a significant performance issue"
+        }
+    },
     "system_health": {
         "info": {
             "current_recorder_run": "Current Run Start Time",

homeassistant/components/recorder/util.py

@@ -25,11 +25,11 @@
 from sqlalchemy.sql.lambdas import StatementLambdaElement
 import voluptuous as vol
 
-from homeassistant.core import HomeAssistant
-from homeassistant.helpers import config_validation as cv
+from homeassistant.core import HomeAssistant, callback
+from homeassistant.helpers import config_validation as cv, issue_registry as ir
 import homeassistant.util.dt as dt_util
 
-from .const import DATA_INSTANCE, SQLITE_URL_PREFIX, SupportedDialect
+from .const import DATA_INSTANCE, DOMAIN, SQLITE_URL_PREFIX, SupportedDialect
 from .db_schema import (
     TABLE_RECORDER_RUNS,
     TABLE_SCHEMA_CHANGES,
@@ -51,9 +51,35 @@
 SQLITE3_POSTFIXES = ["", "-wal", "-shm"]
 DEFAULT_YIELD_STATES_ROWS = 32768
 
+# Our minimum versions for each database
+#
+# Older MariaDB suffers https://jira.mariadb.org/browse/MDEV-25020
+# which is fixed in 10.5.17, 10.6.9, 10.7.5, 10.8.4
+#
 MIN_VERSION_MARIA_DB = AwesomeVersion(
     "10.3.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER
 )
+RECOMMENDED_MIN_VERSION_MARIA_DB = AwesomeVersion(
+    "10.5.17", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER
+)
+MARIA_DB_106 = AwesomeVersion(
+    "10.6.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER
+)
+RECOMMENDED_MIN_VERSION_MARIA_DB_106 = AwesomeVersion(
+    "10.6.9", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER
+)
+MARIA_DB_107 = AwesomeVersion(
+    "10.7.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER
+)
+RECOMMENDED_MIN_VERSION_MARIA_DB_107 = AwesomeVersion(
+    "10.7.5", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER
+)
+MARIA_DB_108 = AwesomeVersion(
+    "10.8.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER
+)
+RECOMMENDED_MIN_VERSION_MARIA_DB_108 = AwesomeVersion(
+    "10.8.4", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER
+)
 MIN_VERSION_MYSQL = AwesomeVersion(
     "8.0.0", ensure_strategy=AwesomeVersionStrategy.SIMPLEVER
 )
@@ -410,6 +436,34 @@ def build_mysqldb_conv() -> dict:
     return {**conversions, FIELD_TYPE.DATETIME: _datetime_or_none}
 
 
+@callback
+def _async_create_mariadb_range_index_regression_issue(
+    hass: HomeAssistant, version: AwesomeVersion
+) -> None:
+    """Create an issue for the index range regression in older MariaDB.
+
+    The range scan issue was fixed in MariaDB 10.5.17, 10.6.9, 10.7.5, 10.8.4 and later.
+    """
+    if version >= MARIA_DB_108:
+        min_version = RECOMMENDED_MIN_VERSION_MARIA_DB_108
+    elif version >= MARIA_DB_107:
+        min_version = RECOMMENDED_MIN_VERSION_MARIA_DB_107
+    elif version >= MARIA_DB_106:
+        min_version = RECOMMENDED_MIN_VERSION_MARIA_DB_106
+    else:
+        min_version = RECOMMENDED_MIN_VERSION_MARIA_DB
+    ir.async_create_issue(
+        hass,
+        DOMAIN,
+        "maria_db_range_index_regression",
+        is_fixable=False,
+        severity=ir.IssueSeverity.CRITICAL,
+        learn_more_url="https://jira.mariadb.org/browse/MDEV-25020",
+        translation_key="maria_db_range_index_regression",
+        translation_placeholders={"min_version": str(min_version)},
+    )
+
+
 def setup_connection_for_dialect(
     instance: Recorder,
     dialect_name: str,
@@ -466,6 +520,18 @@ def setup_connection_for_dialect(
                     _fail_unsupported_version(
                         version or version_string, "MariaDB", MIN_VERSION_MARIA_DB
                     )
+                if version and (
+                    (version < RECOMMENDED_MIN_VERSION_MARIA_DB)
+                    or (MARIA_DB_106 <= version < RECOMMENDED_MIN_VERSION_MARIA_DB_106)
+                    or (MARIA_DB_107 <= version < RECOMMENDED_MIN_VERSION_MARIA_DB_107)
+                    or (MARIA_DB_108 <= version < RECOMMENDED_MIN_VERSION_MARIA_DB_108)
+                ):
+                    instance.hass.add_job(
+                        _async_create_mariadb_range_index_regression_issue,
+                        instance.hass,
+                        version,
+                    )
+
             else:
                 if not version or version < MIN_VERSION_MYSQL:
                     _fail_unsupported_version(

tests/components/recorder/test_util.py

@@ -14,7 +14,7 @@
 
 from homeassistant.components import recorder
 from homeassistant.components.recorder import history, util
-from homeassistant.components.recorder.const import SQLITE_URL_PREFIX
+from homeassistant.components.recorder.const import DOMAIN, SQLITE_URL_PREFIX
 from homeassistant.components.recorder.db_schema import RecorderRuns
 from homeassistant.components.recorder.models import UnsupportedDialect
 from homeassistant.components.recorder.util import (
@@ -25,6 +25,7 @@
 )
 from homeassistant.const import EVENT_HOMEASSISTANT_STOP
 from homeassistant.core import HomeAssistant
+from homeassistant.helpers.issue_registry import async_get as async_get_issue_registry
 from homeassistant.util import dt as dt_util
 
 from .common import corrupt_db_file, run_information_with_session, wait_recording_done
@@ -549,6 +550,118 @@ def test_warn_unsupported_dialect(caplog, dialect, message):
     assert message in caplog.text
 
 
+@pytest.mark.parametrize(
+    "mysql_version,min_version",
+    [
+        (
+            "10.5.16-MariaDB",
+            "10.5.17",
+        ),
+        (
+            "10.6.8-MariaDB",
+            "10.6.9",
+        ),
+        (
+            "10.7.1-MariaDB",
+            "10.7.5",
+        ),
+        (
+            "10.8.0-MariaDB",
+            "10.8.4",
+        ),
+    ],
+)
+async def test_issue_for_mariadb_with_MDEV_25020(
+    hass, caplog, mysql_version, min_version
+):
+    """Test we create an issue for MariaDB versions affected.
+
+    See https://jira.mariadb.org/browse/MDEV-25020.
+    """
+    instance_mock = MagicMock()
+    instance_mock.hass = hass
+    execute_args = []
+    close_mock = MagicMock()
+
+    def execute_mock(statement):
+        nonlocal execute_args
+        execute_args.append(statement)
+
+    def fetchall_mock():
+        nonlocal execute_args
+        if execute_args[-1] == "SELECT VERSION()":
+            return [[mysql_version]]
+        return None
+
+    def _make_cursor_mock(*_):
+        return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock)
+
+    dbapi_connection = MagicMock(cursor=_make_cursor_mock)
+
+    await hass.async_add_executor_job(
+        util.setup_connection_for_dialect,
+        instance_mock,
+        "mysql",
+        dbapi_connection,
+        True,
+    )
+    await hass.async_block_till_done()
+
+    registry = async_get_issue_registry(hass)
+    issue = registry.async_get_issue(DOMAIN, "maria_db_range_index_regression")
+    assert issue is not None
+    assert issue.translation_placeholders == {"min_version": min_version}
+
+
+@pytest.mark.parametrize(
+    "mysql_version",
+    [
+        "10.5.17-MariaDB",
+        "10.6.9-MariaDB",
+        "10.7.5-MariaDB",
+        "10.8.4-MariaDB",
+        "10.9.1-MariaDB",
+    ],
+)
+async def test_no_issue_for_mariadb_with_MDEV_25020(hass, caplog, mysql_version):
+    """Test we do not create an issue for MariaDB versions not affected.
+
+    See https://jira.mariadb.org/browse/MDEV-25020.
+    """
+    instance_mock = MagicMock()
+    instance_mock.hass = hass
+    execute_args = []
+    close_mock = MagicMock()
+
+    def execute_mock(statement):
+        nonlocal execute_args
+        execute_args.append(statement)
+
+    def fetchall_mock():
+        nonlocal execute_args
+        if execute_args[-1] == "SELECT VERSION()":
+            return [[mysql_version]]
+        return None
+
+    def _make_cursor_mock(*_):
+        return MagicMock(execute=execute_mock, close=close_mock, fetchall=fetchall_mock)
+
+    dbapi_connection = MagicMock(cursor=_make_cursor_mock)
+
+    await hass.async_add_executor_job(
+        util.setup_connection_for_dialect,
+        instance_mock,
+        "mysql",
+        dbapi_connection,
+        True,
+    )
+    await hass.async_block_till_done()
+
+    registry = async_get_issue_registry(hass)
+    issue = registry.async_get_issue(DOMAIN, "maria_db_range_index_regression")
+    assert issue is None
+
+
 def test_basic_sanity_check(hass_recorder, recorder_db_url):
     """Test the basic sanity checks with a missing table."""
     if recorder_db_url.startswith("mysql://"):