From a0a5d4da429476b4ec1541a8212067f0427fff81 Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 27 Nov 2019 16:44:55 +1300 Subject: [PATCH] Add utf8 to utfmb8 conversion api command This extracts from https://github.com/civicrm/civicrm-core/pull/13633 the portion that deals with the conversion and switches it from an upgrade script (which is not agreed at this stage as there are concerns about imposing this change) to an api command (which can be added with relatively little discussion as it only affects those who choose to use it). We don't expect this command to outlive apiv3 so adding there & not creating insta-tech-debt on apiv4. In order to add a unit test I had to alter to support reverting - doesn't seem like a bad thing all round. As an aside - I like the way it changes the DB level charset & collation. I think I've seen these fail to be set in the wild --- CRM/Core/BAO/SchemaHandler.php | 79 +++++++++++++++++++++++++++++ api/v3/System.php | 32 ++++++++++++ tests/phpunit/api/v3/SystemTest.php | 15 ++++++ 3 files changed, 126 insertions(+) diff --git a/CRM/Core/BAO/SchemaHandler.php b/CRM/Core/BAO/SchemaHandler.php index b135036d645b..6079af2e0a02 100644 --- a/CRM/Core/BAO/SchemaHandler.php +++ b/CRM/Core/BAO/SchemaHandler.php @@ -791,4 +791,83 @@ public static function getFieldAlterSQL($params, $indexExist) { return $sql; } + /** + * Performs the utf8mb4 migration. + * + * @param bool $revert + * Being able to revert if primarily for unit testing. + * + * @return bool + */ + public static function migrateUtf8mb4($revert = FALSE) { + $newCharSet = $revert ? 'utf8' : 'utf8mb4'; + $newCollation = $revert ? 'utf8_unicode_ci' : 'utf8mb4_unicode_ci'; + $newBinaryCollation = $revert ? 'utf8_bin' : 'utf8mb4_bin'; + $tables = []; + $dao = new CRM_Core_DAO(); + $database = $dao->_database; + CRM_Core_DAO::executeQuery("ALTER DATABASE $database CHARACTER SET = $newCharSet COLLATE = $newCollation"); + $dao = CRM_Core_DAO::executeQuery("SHOW TABLE STATUS WHERE Engine = 'InnoDB' AND Name LIKE 'civicrm\_%'"); + while ($dao->fetch()) { + $tables[$dao->Name] = [ + 'Engine' => $dao->Engine, + ]; + } + $dsn = defined('CIVICRM_LOGGING_DSN') ? DB::parseDSN(CIVICRM_LOGGING_DSN) : DB::parseDSN(CIVICRM_DSN); + $logging_database = $dsn['database']; + $dao = CRM_Core_DAO::executeQuery("SHOW TABLE STATUS FROM `$logging_database` WHERE Engine <> 'MyISAM' AND Name LIKE 'log\_civicrm\_%'"); + while ($dao->fetch()) { + $tables["$logging_database.{$dao->Name}"] = [ + 'Engine' => $dao->Engine, + ]; + } + foreach ($tables as $table => $param) { + $query = "ALTER TABLE $table"; + $dao = CRM_Core_DAO::executeQuery("SHOW FULL COLUMNS FROM $table", [], TRUE, NULL, FALSE, FALSE); + $index = 0; + $params = []; + $tableCollation = $newCollation; + while ($dao->fetch()) { + if (!$dao->Collation || $dao->Collation === $newCollation || $dao->Collation === $newBinaryCollation) { + continue; + } + if (strpos($dao->Collation, 'utf8') !== 0) { + continue; + } + + if (strpos($dao->Collation, '_bin') !== FALSE) { + $tableCollation = $newBinaryCollation; + } + else { + $tableCollation = $newCollation; + } + if ($dao->Null === 'YES') { + $null = 'NULL'; + } + else { + $null = 'NOT NULL'; + } + $default = ''; + if ($dao->Default !== NULL) { + $index++; + $default = "DEFAULT %$index"; + $params[$index] = [$dao->Default, 'String']; + } + elseif ($dao->Null === 'YES') { + $default = 'DEFAULT NULL'; + } + $index++; + $params[$index] = [$dao->Comment, 'String']; + $query .= " MODIFY `{$dao->Field}` {$dao->Type} CHARACTER SET $newCharSet COLLATE $tableCollation $null $default {$dao->Extra} COMMENT %$index,"; + } + $query .= " CHARACTER SET = $newCharSet COLLATE = $tableCollation"; + if ($param['Engine'] === 'InnoDB') { + $query .= ' ROW_FORMAT = Dynamic'; + } + // Disable i18n rewrite. + CRM_Core_DAO::executeQuery($query, $params, TRUE, NULL, FALSE, FALSE); + } + return TRUE; + } + } diff --git a/api/v3/System.php b/api/v3/System.php index 6b593546a661..827ea409e000 100644 --- a/api/v3/System.php +++ b/api/v3/System.php @@ -384,6 +384,38 @@ function civicrm_api3_system_updatelogtables($params) { return civicrm_api3_create_success($updatedTablesCount); } +/** + * Update log table structures. + * + * This updates the engine type if defined in the hook and changes the field type + * for log_conn_id to reflect CRM-18193. + * + * @param array $params + * + * @return array + * + * @throws \API_Exception + */ +function civicrm_api3_system_utf8conversion($params) { + if (CRM_Core_BAO_SchemaHandler::migrateUtf8mb4($params['is_revert'])) { + return civicrm_api3_create_success(1); + } + throw new API_Exception('Conversion failed'); +} + +/** + * Metadata for conversion function. + * + * @param array $params + */ +function _civicrm_api3_system_utf8conversion_spec(&$params) { + $params['is_revert'] = [ + 'title' => ts('Revert back from UTF8MB4 to UTF8?'), + 'type' => CRM_Utils_Type::T_BOOLEAN, + 'api.default' => FALSE, + ]; +} + /** * Adjust Metadata for Flush action. * diff --git a/tests/phpunit/api/v3/SystemTest.php b/tests/phpunit/api/v3/SystemTest.php index 66fdb27d0877..ba6e77407fc1 100644 --- a/tests/phpunit/api/v3/SystemTest.php +++ b/tests/phpunit/api/v3/SystemTest.php @@ -83,4 +83,19 @@ public function testSystemGet() { $this->assertEquals('UnitTests', $result['values'][0]['uf']); } + /** + * @throws \CRM_Core_Exception + */ + public function testSystemUTFMB8Conversion() { + $this->callAPISuccess('System', 'utf8conversion', []); + $table = CRM_Core_DAO::executeQuery('SHOW CREATE TABLE civicrm_contact'); + $table->fetch(); + $this->assertStringEndsWith('DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC', $table->Create_Table); + + $this->callAPISuccess('System', 'utf8conversion', ['is_revert' => 1]); + $table = CRM_Core_DAO::executeQuery('SHOW CREATE TABLE civicrm_contact'); + $table->fetch(); + $this->assertStringEndsWith('DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci ROW_FORMAT=DYNAMIC', $table->Create_Table); + } + }