Integration tests for databases and GitHub actions

.github/workflows/damengDatabaseTests.yml

@@ -0,0 +1,48 @@
+name: Dameng database tests
+on:
+  push:
+    paths:
+      - 'DocService/sources/databaseConnectors/baseConnector.js'
+      - 'DocService/sources/databaseConnectors/damengConnector.js'
+jobs:
+  dameng-tests:
+    name: Dameng
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Run dameng DB docker container
+        run: docker run --name dameng -p 5236:5236 -e PAGE_SIZE=16 -e LD_LIBRARY_PATH=/opt/dmdbms/bin -e INSTANCE_NAME=dm8_01 -d danilaworker/damengdb:8.1.2
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Caching dependencies
+        uses: actions/setup-node@v3
+        with:
+          node-version: '14'
+          cache: 'npm'
+          cache-dependency-path: |
+            ./npm-shrinkwrap.json
+            ./Common/npm-shrinkwrap.json
+            ./DocService/npm-shrinkwrap.json
+
+      - name: Install modules
+        run: |
+          npm ci
+          npm --prefix Common ci
+          npm --prefix DocService ci
+
+      - name: Await database service to finish startup
+        run: sleep 15
+
+      - name: Creating service DB configuration
+        run: |
+          echo '{"services": {"CoAuthoring": {"sql": {"type": "dameng", "dbHost": "127.0.0.1", "dbPort": 5236, "dbUser": "SYSDBA", "dbPass": "SYSDBA001"}}}}' >> Common/config/local.json
+
+      - name: Creating schema
+        run: |
+          docker cp ./schema/dameng/createdb.sql dameng:/
+          docker exec dameng bash -c "cat /createdb.sql | /opt/dmdbms/bin/disql SYSDBA/SYSDBA001:5236"
+
+      - name: Run Jest
+        run: npm run "integration database tests"

.github/workflows/mssqlDatabaseTests.yml

@@ -0,0 +1,48 @@
+name: MSSQL database tests
+on:
+  push:
+    paths:
+      - 'DocService/sources/databaseConnectors/baseConnector.js'
+      - 'DocService/sources/databaseConnectors/mssqlConnector.js'
+jobs:
+  mssql-tests:
+    name: MSSQL
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Run MSSQL DB docker container
+        run: docker run --name mssql -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=onlYoff1ce" -p 8080:1433 -d mcr.microsoft.com/mssql/server:2022-latest
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Caching dependencies
+        uses: actions/setup-node@v3
+        with:
+          node-version: '14'
+          cache: 'npm'
+          cache-dependency-path: |
+            ./npm-shrinkwrap.json
+            ./Common/npm-shrinkwrap.json
+            ./DocService/npm-shrinkwrap.json
+
+      - name: Install modules
+        run: |
+          npm ci
+          npm --prefix Common ci
+          npm --prefix DocService ci
+
+      - name: Creating service DB configuration
+        run: |
+          echo '{"services": {"CoAuthoring": {"sql": {"type": "mssql", "dbHost": "localhost", "dbPort": 8080, "dbUser": "sa", "dbPass": "onlYoff1ce"}}}}' >> Common/config/local.json
+
+      - name: Await database service to finish startup
+        run: sleep 5
+
+      - name: Creating schema
+        run: |
+          docker cp ./schema/mssql/createdb.sql mssql:/
+          docker exec mssql /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P onlYoff1ce -i /createdb.sql
+
+      - name: Run Jest
+        run: npm run "integration database tests"

.github/workflows/mysqlDatabaseTests.yml

@@ -0,0 +1,45 @@
+name: MYSQL database tests
+on:
+  push:
+    paths:
+      - 'DocService/sources/databaseConnectors/baseConnector.js'
+      - 'DocService/sources/databaseConnectors/mysqlConnector.js'
+jobs:
+  mysql-tests:
+    name: MYSQL
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Run Mysql DB docker container
+        run: docker run --name mysql -p 8080:3306 -p 8081:33060 -e MYSQL_HOST=127.0.0.1 -e MYSQL_ROOT_PASSWORD=onlyoffice -e MYSQL_DATABASE=onlyoffice -d mysql:latest
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Caching dependencies
+        uses: actions/setup-node@v3
+        with:
+          node-version: '14'
+          cache: 'npm'
+          cache-dependency-path: |
+            ./npm-shrinkwrap.json
+            ./Common/npm-shrinkwrap.json
+            ./DocService/npm-shrinkwrap.json
+
+      - name: Install modules
+        run: |
+          npm ci
+          npm --prefix Common ci
+          npm --prefix DocService ci
+
+      - name: Creating service DB configuration
+        run: |
+          echo '{"services": {"CoAuthoring": {"sql": {"type": "mysql", "dbHost": "127.0.0.1", "dbPort": "8080", "dbUser": "root", "dbPass": "onlyoffice"}}}}' >> Common/config/local.json
+
+      - name : Creating schema
+        run: |
+          docker cp ./schema/mysql/createdb.sql mysql:/
+          docker exec mysql mysql -h 127.0.0.1 -u root --password=onlyoffice -D onlyoffice -e 'source /createdb.sql'
+
+      - name: Run Jest
+        run: npm run "integration database tests"

.github/workflows/oracleDatabaseTests.yml

@@ -0,0 +1,48 @@
+name: Oracle database tests
+on:
+  push:
+    paths:
+      - 'DocService/sources/databaseConnectors/baseConnector.js'
+      - 'DocService/sources/databaseConnectors/oracleConnector.js'
+jobs:
+  oracle-tests:
+    name: Oracle
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Run Oracle DB docker container
+        run: docker run --name oracle -p 8080:1521 -p 8081:5500 -e ORACLE_PASSWORD=admin -e APP_USER=onlyoffice -e APP_USER_PASSWORD=onlyoffice -d gvenzl/oracle-xe:21-slim
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Caching dependencies
+        uses: actions/setup-node@v3
+        with:
+          node-version: '14'
+          cache: 'npm'
+          cache-dependency-path: |
+            ./npm-shrinkwrap.json
+            ./Common/npm-shrinkwrap.json
+            ./DocService/npm-shrinkwrap.json
+
+      - name: Install modules
+        run: |
+          npm ci
+          npm --prefix Common ci
+          npm --prefix DocService ci
+
+      - name: Creating service DB configuration
+        run: |
+          echo '{"services": {"CoAuthoring": {"sql": {"type": "oracle", "dbHost": "127.0.0.1", "dbPort": "8080", "dbUser": "onlyoffice", "dbPass": "onlyoffice", "dbName": "xepdb1"}}}}' >> Common/config/local.json
+
+      - name: Await database service to finish startup
+        run: sleep 15
+
+      - name: Creating schema
+        run: |
+          docker cp ./schema/oracle/createdb.sql oracle:/
+          docker exec oracle sqlplus -s onlyoffice/onlyoffice@//localhost/xepdb1 @/createdb.sql
+
+      - name: Run Jest
+        run: npm run "integration database tests"

.github/workflows/postgreDatabaseTests.yml

@@ -0,0 +1,45 @@
+name: Postgre database tests
+on:
+  push:
+    paths:
+      - 'DocService/sources/databaseConnectors/baseConnector.js'
+      - 'DocService/sources/databaseConnectors/postgreConnector.js'
+jobs:
+  postgres-tests:
+    name: Postgres
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Run Postgres DB docker container
+        run: docker run --name postgres -p 5432:5432 -e POSTGRES_PASSWORD=onlyoffice -e POSTGRES_USER=onlyoffice -e POSTGRES_DB=onlyoffice -d postgres:latest
+
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Caching dependencies
+        uses: actions/setup-node@v3
+        with:
+          node-version: '14'
+          cache: 'npm'
+          cache-dependency-path: |
+            ./npm-shrinkwrap.json
+            ./Common/npm-shrinkwrap.json
+            ./DocService/npm-shrinkwrap.json
+
+      - name: Install modules
+        run: |
+          npm ci
+          npm --prefix Common ci
+          npm --prefix DocService ci
+
+      - name: Creating service DB configuration
+        run: |
+          echo '{"services": {"CoAuthoring": {"sql": {"dbHost": "127.0.0.1"}}}}' >> Common/config/local.json
+
+      - name: Creating schema
+        run: |
+          docker cp ./schema/postgresql/createdb.sql postgres:/
+          docker exec postgres psql -d onlyoffice -U onlyoffice -a -f /createdb.sql
+
+      - name: Run Jest
+        run: npm run "integration database tests"

.github/workflows/unitTests.yml

@@ -0,0 +1,34 @@
+name: Service unit tests
+on:
+  push:
+    paths:
+      - '**.js'
+      - '!tests/integration/**'
+      - '!DocService/sources/databaseConnectors/**'
+jobs:
+  unit-tests:
+    name: Service unit tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Caching dependencies
+        uses: actions/setup-node@v3
+        with:
+          node-version: '14'
+          cache: 'npm'
+          cache-dependency-path: |
+            ./npm-shrinkwrap.json
+            ./Common/npm-shrinkwrap.json
+            ./DocService/npm-shrinkwrap.json
+
+      - name: Install modules
+        run: |
+          npm ci
+          npm --prefix Common ci
+          npm --prefix DocService ci
+
+      - name: Run Jest
+        run: npm run "unit tests"

Common/config/default.json

@@ -177,7 +177,12 @@
 				"connectionlimit": 10,
 				"max_allowed_packet": 1048575,
 				"pgPoolExtraOptions": {},
-				"damengExtraOptions": {},
+				"damengExtraOptions": {
+					"columnNameUpperCase": false,
+					"columnNameCase": "lower",
+					"loginEncrypt": false,
+					"localTimezone": 0
+				},
 				"oracleExtraOptions": {},
 				"msSqlExtraOptions": {
 					"options": {

DocService/sources/DocsCoServer.js

@@ -90,7 +90,7 @@ const utils = require('./../../Common/sources/utils');
 const commonDefines = require('./../../Common/sources/commondefines');
 const statsDClient = require('./../../Common/sources/statsdclient');
 const config = require('config');
-const sqlBase = require('./baseConnector');
+const sqlBase = require('./databaseConnectors/baseConnector');
 const canvasService = require('./canvasservice');
 const converterService = require('./converterservice');
 const taskResult = require('./taskresult');
@@ -1284,7 +1284,7 @@ let unlockWopiDoc = co.wrap(function*(ctx, docId, opt_userIndex) {
     yield wopiClient.unlock(ctx, getRes.wopiParams);
     let unlockInfo = wopiClient.getWopiUnlockMarker(getRes.wopiParams);
     if (unlockInfo) {
-      yield canvasService.commandOpenStartPromise(ctx, docId, undefined, true, unlockInfo);
+      yield canvasService.commandOpenStartPromise(ctx, docId, undefined, unlockInfo);
     }
   }
 });
@@ -2494,10 +2494,9 @@ exports.install = function(server, callbackFunction) {
           }
         }
         let format = data.openCmd && data.openCmd.format;
-        upsertRes = yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByConnection(ctx, conn), true, data.documentCallbackUrl, format);
-        let isInserted = upsertRes.affectedRows == 1;
-        curIndexUser = isInserted ? 1 : upsertRes.insertId;
-        if (isInserted && undefined !== data.timezoneOffset) {
+        upsertRes = yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByConnection(ctx, conn), data.documentCallbackUrl, format);
+        curIndexUser = upsertRes.insertId;
+        if (upsertRes.isInsert && undefined !== data.timezoneOffset) {
           //todo insert in commandOpenStartPromise. insert here for database compatibility
           if (false === canvasService.hasAdditionalCol) {
             let selectRes = yield taskResult.select(ctx, docId);

DocService/sources/canvasservice.js

@@ -38,7 +38,7 @@ var co = require('co');
 const ms = require('ms');
 const retry = require('retry');
 const MultiRange = require('multi-integer-range').MultiRange;
-var sqlBase = require('./baseConnector');
+var sqlBase = require('./databaseConnectors/baseConnector');
 const utilsDocService = require('./utilsDocService');
 var docsCoServer = require('./DocsCoServer');
 var taskResult = require('./taskresult');
@@ -437,7 +437,7 @@ var cleanupCacheIf = co.wrap(function* (ctx, mask) {
   return res;
 });
 
-function commandOpenStartPromise(ctx, docId, baseUrl, opt_updateUserIndex, opt_documentCallbackUrl, opt_format) {
+function commandOpenStartPromise(ctx, docId, baseUrl, opt_documentCallbackUrl, opt_format) {
   var task = new taskResult.TaskResultData();
   task.tenant = ctx.tenant;
   task.key = docId;
@@ -451,7 +451,7 @@ function commandOpenStartPromise(ctx, docId, baseUrl, opt_updateUserIndex, opt_d
   if (opt_format) {
     task.changeId = formatChecker.getFormatFromString(opt_format);
   }
-  return taskResult.upsert(ctx, task, opt_updateUserIndex);
+  return taskResult.upsert(ctx, task);
 }
 function* commandOpen(ctx, conn, cmd, outputData, opt_upsertRes, opt_bIsRestore) {
   const tenForgottenFiles = ctx.getCfg('services.CoAuthoring.server.forgottenfiles', cfgForgottenFiles);
@@ -462,9 +462,7 @@ function* commandOpen(ctx, conn, cmd, outputData, opt_upsertRes, opt_bIsRestore)
   } else {
     upsertRes = yield commandOpenStartPromise(ctx, cmd.getDocId(), utils.getBaseUrlByConnection(ctx, conn));
   }
-  //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values
-  //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html
-  let bCreate = upsertRes.affectedRows == 1;
+  let bCreate = upsertRes.isInsert;
   let needAddTask = bCreate;
   if (!bCreate) {
     needAddTask = yield* commandOpenFillOutput(ctx, conn, cmd, outputData, opt_bIsRestore);
@@ -1210,7 +1208,7 @@ function* processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChange
         //collabora nexcloud connector
         if (body.LastModifiedTime) {
           let lastModifiedTimeInfo = wopiClient.getWopiModifiedMarker(wopiParams, body.LastModifiedTime);
-          yield commandOpenStartPromise(ctx, docId, undefined, true, lastModifiedTimeInfo);
+          yield commandOpenStartPromise(ctx, docId, undefined, lastModifiedTimeInfo);
         }
       } catch (e) {
         ctx.logger.debug('processWopiPutFile error: %s', e.stack);

DocService/sources/changes2forgotten.js

@@ -41,7 +41,7 @@ var utils = require('./../../Common/sources/utils');
 const storage = require('./../../Common/sources/storage-base');
 const queueService = require('./../../Common/sources/taskqueueRabbitMQ');
 const operationContext = require('./../../Common/sources/operationContext');
-const sqlBase = require('./baseConnector');
+const sqlBase = require('./databaseConnectors/baseConnector');
 const docsCoServer = require('./DocsCoServer');
 const taskResult = require('./taskresult');
 const editorDataStorage = require('./' + config.get('services.CoAuthoring.server.editorDataStorage'));
@@ -56,7 +56,7 @@ var WAIT_TIMEOUT = 30000;
 var LOOP_TIMEOUT = 1000;
 var EXEC_TIMEOUT = WAIT_TIMEOUT + utils.getConvertionTimeout(undefined);
 
-let addSqlParam = sqlBase.baseConnector.addSqlParameter;
+let addSqlParam = sqlBase.addSqlParameter;
 
 function updateDoc(ctx, docId, status, callback) {
   return new Promise(function(resolve, reject) {
@@ -66,7 +66,7 @@ function updateDoc(ctx, docId, status, callback) {
     let p3 = addSqlParam(ctx.tenant, values);
     let p4 = addSqlParam(docId, values);
     let sqlCommand = `UPDATE ${cfgTableResult} SET status=${p1},callback=${p2} WHERE tenant=${p3} AND id=${p4};`;
-    sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) {
+    sqlBase.sqlQuery(ctx, sqlCommand, function(error, result) {
       if (error) {
         reject(error);
       } else {