Add utf8 to utfmb8 conversion api command

This extracts from #13633 the portion that deals with the conversion and switches it from an upgrade script (which is not agreed at this stage as there are concerns about imposing this change) to an api command (which can be added with relatively little discussion as it only affects those who choose to use it). We don't expect this command to outlive apiv3 so adding there & not creating insta-tech-debt on apiv4. In order to add a unit test I had to alter to support reverting - doesn't seem like a bad thing all round. As an aside - I like the way it changes the DB level charset & collation. I think I've seen these fail to be set in the wild
civicrm · Dec 10, 2019 · a0a5d4d · a0a5d4d
1 parent b8e9143
commit a0a5d4d
Show file tree

Hide file tree

Showing 3 changed files with 126 additions and 0 deletions.
diff --git a/CRM/Core/BAO/SchemaHandler.php b/CRM/Core/BAO/SchemaHandler.php
@@ -791,4 +791,83 @@ public static function getFieldAlterSQL($params, $indexExist) {
     return $sql;
   }
 
+  /**
+   * Performs the utf8mb4 migration.
+   *
+   * @param bool $revert
+   *   Being able to revert if primarily for unit testing.
+   *
+   * @return bool
+   */
+  public static function migrateUtf8mb4($revert = FALSE) {
+    $newCharSet = $revert ? 'utf8' : 'utf8mb4';
+    $newCollation = $revert ? 'utf8_unicode_ci' : 'utf8mb4_unicode_ci';
+    $newBinaryCollation = $revert ? 'utf8_bin' : 'utf8mb4_bin';
+    $tables = [];
+    $dao = new CRM_Core_DAO();
+    $database = $dao->_database;
+    CRM_Core_DAO::executeQuery("ALTER DATABASE $database CHARACTER SET = $newCharSet COLLATE = $newCollation");
+    $dao = CRM_Core_DAO::executeQuery("SHOW TABLE STATUS WHERE Engine = 'InnoDB' AND Name LIKE 'civicrm\_%'");
+    while ($dao->fetch()) {
+      $tables[$dao->Name] = [
+        'Engine' => $dao->Engine,
+      ];
+    }
+    $dsn = defined('CIVICRM_LOGGING_DSN') ? DB::parseDSN(CIVICRM_LOGGING_DSN) : DB::parseDSN(CIVICRM_DSN);
+    $logging_database = $dsn['database'];
+    $dao = CRM_Core_DAO::executeQuery("SHOW TABLE STATUS FROM `$logging_database` WHERE Engine <> 'MyISAM' AND Name LIKE 'log\_civicrm\_%'");
+    while ($dao->fetch()) {
+      $tables["$logging_database.{$dao->Name}"] = [
+        'Engine' => $dao->Engine,
+      ];
+    }
+    foreach ($tables as $table => $param) {
+      $query = "ALTER TABLE $table";
+      $dao = CRM_Core_DAO::executeQuery("SHOW FULL COLUMNS FROM $table", [], TRUE, NULL, FALSE, FALSE);
+      $index = 0;
+      $params = [];
+      $tableCollation = $newCollation;
+      while ($dao->fetch()) {
+        if (!$dao->Collation || $dao->Collation === $newCollation || $dao->Collation === $newBinaryCollation) {
+          continue;
+        }
+        if (strpos($dao->Collation, 'utf8') !== 0) {
+          continue;
+        }
+
+        if (strpos($dao->Collation, '_bin') !== FALSE) {
+          $tableCollation = $newBinaryCollation;
+        }
+        else {
+          $tableCollation = $newCollation;
+        }
+        if ($dao->Null === 'YES') {
+          $null = 'NULL';
+        }
+        else {
+          $null = 'NOT NULL';
+        }
+        $default = '';
+        if ($dao->Default !== NULL) {
+          $index++;
+          $default = "DEFAULT %$index";
+          $params[$index] = [$dao->Default, 'String'];
+        }
+        elseif ($dao->Null === 'YES') {
+          $default = 'DEFAULT NULL';
+        }
+        $index++;
+        $params[$index] = [$dao->Comment, 'String'];
+        $query .= " MODIFY `{$dao->Field}` {$dao->Type} CHARACTER SET $newCharSet COLLATE $tableCollation $null $default {$dao->Extra} COMMENT %$index,";
+      }
+      $query .= " CHARACTER SET = $newCharSet COLLATE = $tableCollation";
+      if ($param['Engine'] === 'InnoDB') {
+        $query .= ' ROW_FORMAT = Dynamic';
+      }
+      // Disable i18n rewrite.
+      CRM_Core_DAO::executeQuery($query, $params, TRUE, NULL, FALSE, FALSE);
+    }
+    return TRUE;
+  }
+
 }
diff --git a/api/v3/System.php b/api/v3/System.php
@@ -384,6 +384,38 @@ function civicrm_api3_system_updatelogtables($params) {
   return civicrm_api3_create_success($updatedTablesCount);
 }
 
+/**
+ * Update log table structures.
+ *
+ * This updates the engine type if defined in the hook and changes the field type
+ * for log_conn_id to reflect CRM-18193.
+ *
+ * @param array $params
+ *
+ * @return array
+ *
+ * @throws \API_Exception
+ */
+function civicrm_api3_system_utf8conversion($params) {
+  if (CRM_Core_BAO_SchemaHandler::migrateUtf8mb4($params['is_revert'])) {
+    return civicrm_api3_create_success(1);
+  }
+  throw new API_Exception('Conversion failed');
+}
+
+/**
+ * Metadata for conversion function.
+ *
+ * @param array $params
+ */
+function _civicrm_api3_system_utf8conversion_spec(&$params) {
+  $params['is_revert'] = [
+    'title' => ts('Revert back from UTF8MB4 to UTF8?'),
+    'type' => CRM_Utils_Type::T_BOOLEAN,
+    'api.default' => FALSE,
+  ];
+}
+
 /**
  * Adjust Metadata for Flush action.
  *

diff --git a/tests/phpunit/api/v3/SystemTest.php b/tests/phpunit/api/v3/SystemTest.php
@@ -83,4 +83,19 @@ public function testSystemGet() {
     $this->assertEquals('UnitTests', $result['values'][0]['uf']);
   }
 
+  /**
+   * @throws \CRM_Core_Exception
+   */
+  public function testSystemUTFMB8Conversion() {
+    $this->callAPISuccess('System', 'utf8conversion', []);
+    $table = CRM_Core_DAO::executeQuery('SHOW CREATE TABLE civicrm_contact');
+    $table->fetch();
+    $this->assertStringEndsWith('DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC', $table->Create_Table);
+
+    $this->callAPISuccess('System', 'utf8conversion', ['is_revert' => 1]);
+    $table = CRM_Core_DAO::executeQuery('SHOW CREATE TABLE civicrm_contact');
+    $table->fetch();
+    $this->assertStringEndsWith('DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci ROW_FORMAT=DYNAMIC', $table->Create_Table);
+  }
+
 }