perf: split the export select query to avoid running out of memory

antares-sql · Nov 1, 2021 · 409ed54 · 409ed54
1 parent d9d3bf2
commit 409ed54
Showing 1 changed file with 53 additions and 49 deletions.
diff --git a/src/main/libs/exporters/sql/MysqlExporter.js b/src/main/libs/exporters/sql/MysqlExporter.js
@@ -52,13 +52,15 @@ ${footer}
    async getTableInsert (tableName) {
       let rowCount = 0;
       let sqlStr = '';
+      const pageSize = 1000;
 
       const countResults = await this._client.raw(
          `SELECT COUNT(1) as count FROM \`${this.schemaName}\`.\`${tableName}\``
       );
       if (countResults.rows.length === 1) rowCount = countResults.rows[0].count;
 
       if (rowCount > 0) {
+         const totalPages = Math.ceil(rowCount / pageSize);
          let queryLength = 0;
          let rowsWritten = 0;
          const { sqlInsertDivider, sqlInsertAfter } = this._options;
@@ -71,65 +73,67 @@ ${footer}
             ', '
          )}) VALUES`;
 
-         const tableResult = await this._client.raw(
-            `SELECT ${columnNames.join(', ')} FROM \`${
-               this.schemaName
-            }\`.\`${tableName}\``
-         );
-
          sqlStr += `LOCK TABLES \`${tableName}\` WRITE;\n`;
          sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` DISABLE KEYS */;`;
          sqlStr += '\n\n';
 
-         sqlStr += insertStmt;
-
-         for (const rowIndex in tableResult.rows) {
-            const row = tableResult.rows[rowIndex];
-            let sqlInsertString = '';
-
-            if (
-               (sqlInsertDivider === 'bytes' &&
-                  queryLength >= sqlInsertAfter * 1024) ||
-               (sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
-            ) {
-               sqlInsertString += `;\n${insertStmt}\n\t(`;
-
-               queryLength = 0;
-               rowsWritten = 0;
-            }
-            else if (parseInt(rowIndex) === 0) sqlInsertString += '\n\t(';
-            else sqlInsertString += ',\n\t(';
-
-            for (const i in columns) {
-               const column = columns[i];
-               const val = row[column.name];
-
-               if (val === null) sqlInsertString += 'NULL';
-               else if (BIT.includes(column.type)) {
-                  sqlInsertString += `b'${hexToBinary(
-                     Buffer.from(val).toString('hex')
-                  )}'`;
+         for (let pageNumber = 0; pageNumber < totalPages; pageNumber++) {
+            const tableResult = await this._client.raw(
+               `SELECT ${columnNames.join(', ')} FROM \`${
+                  this.schemaName
+               }\`.\`${tableName}\`
+               LIMIT ${pageSize} OFFSET ${pageSize * pageNumber}`
+            );
+
+            sqlStr += insertStmt;
+
+            for (const rowIndex in tableResult.rows) {
+               const row = tableResult.rows[rowIndex];
+               let sqlInsertString = '';
+
+               if (
+                  (sqlInsertDivider === 'bytes' &&
+                     queryLength >= sqlInsertAfter * 1024) ||
+                  (sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
+               ) {
+                  sqlInsertString += `;\n${insertStmt}\n\t(`;
+
+                  queryLength = 0;
+                  rowsWritten = 0;
                }
-               else if (BLOB.includes(column.type))
-                  sqlInsertString += `X'${val.toString('hex').toUpperCase()}'`;
-               else if (val === '') sqlInsertString += '\'\'';
-               else {
-                  sqlInsertString +=
-                     typeof val === 'string' ? this.escapeAndQuote(val) : val;
+               else if (parseInt(rowIndex) === 0) sqlInsertString += '\n\t(';
+               else sqlInsertString += ',\n\t(';
+
+               for (const i in columns) {
+                  const column = columns[i];
+                  const val = row[column.name];
+
+                  if (val === null) sqlInsertString += 'NULL';
+                  else if (BIT.includes(column.type)) {
+                     sqlInsertString += `b'${hexToBinary(
+                        Buffer.from(val).toString('hex')
+                     )}'`;
+                  }
+                  else if (BLOB.includes(column.type))
+                     sqlInsertString += `X'${val.toString('hex').toUpperCase()}'`;
+                  else if (val === '') sqlInsertString += '\'\'';
+                  else {
+                     sqlInsertString +=
+                        typeof val === 'string' ? this.escapeAndQuote(val) : val;
+                  }
+
+                  if (parseInt(i) !== columns.length - 1) sqlInsertString += ', ';
                }
 
-               if (parseInt(i) !== columns.length - 1) sqlInsertString += ', ';
-            }
-
-            sqlInsertString += ')';
-            sqlStr += sqlInsertString;
+               sqlInsertString += ')';
+               sqlStr += sqlInsertString;
 
-            queryLength += sqlInsertString.length;
-            rowsWritten++;
+               queryLength += sqlInsertString.length;
+               rowsWritten++;
+            }
+            sqlStr += ';\n\n';
          }
 
-         sqlStr += ';\n\n';
-
          sqlStr += `/*!40000 ALTER TABLE \`${tableName}\` ENABLE KEYS */;\n`;
          sqlStr += 'UNLOCK TABLES;';
       }