diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 000000000..42003aedf --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,49 @@ +name: "CodeQL" + +on: + push: + branches: + - 'master' + - 'hotfix/**' + - 'release/**' + paths-ignore: + - '**/README.md' + - '**/LICENSE' + - '.github/**' + + schedule: + - cron: '0 0 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'javascript-typescript' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/damengDatabaseTests.yml b/.github/workflows/damengDatabaseTests.yml new file mode 100644 index 000000000..ed465672c --- /dev/null +++ b/.github/workflows/damengDatabaseTests.yml @@ -0,0 +1,50 @@ +name: Dameng database tests +on: + push: + branches: + - '**' + paths: + - 'DocService/sources/databaseConnectors/baseConnector.js' + - 'DocService/sources/databaseConnectors/damengConnector.js' +jobs: + dameng-tests: + name: Dameng + runs-on: ubuntu-latest + + steps: + - name: Run dameng DB docker container + run: docker run --name dameng -p 5236:5236 -e PAGE_SIZE=16 -e LD_LIBRARY_PATH=/opt/dmdbms/bin -e INSTANCE_NAME=dm8_01 -d danilaworker/damengdb:8.1.2 + + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Caching dependencies + uses: actions/setup-node@v3 + with: + node-version: '14' + cache: 'npm' + cache-dependency-path: | + ./npm-shrinkwrap.json + ./Common/npm-shrinkwrap.json + ./DocService/npm-shrinkwrap.json + + - name: Install modules + run: | + npm ci + npm --prefix Common ci + npm --prefix DocService ci + + - name: Await database service to finish startup + run: sleep 15 + + - name: Creating service DB configuration + run: | + echo '{"services": {"CoAuthoring": {"sql": {"type": "dameng", "dbHost": "127.0.0.1", "dbPort": 5236, "dbUser": "SYSDBA", "dbPass": "SYSDBA001"}}}}' >> Common/config/local.json + + - name: Creating schema + run: | + docker cp ./schema/dameng/createdb.sql dameng:/ + docker exec dameng bash -c "cat /createdb.sql | /opt/dmdbms/bin/disql SYSDBA/SYSDBA001:5236" + + - name: Run Jest + run: npm run "integration database tests" diff --git a/.github/workflows/mssqlDatabaseTests.yml b/.github/workflows/mssqlDatabaseTests.yml new file mode 100644 index 000000000..0529401ff --- /dev/null +++ b/.github/workflows/mssqlDatabaseTests.yml @@ -0,0 +1,50 @@ +name: MSSQL database tests +on: + push: + branches: + - '**' + paths: + - 'DocService/sources/databaseConnectors/baseConnector.js' + - 'DocService/sources/databaseConnectors/mssqlConnector.js' +jobs: + mssql-tests: + name: MSSQL + runs-on: ubuntu-latest + + steps: + - name: Run MSSQL DB docker container + run: docker run --name mssql -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=onlYoff1ce" -p 8080:1433 -d mcr.microsoft.com/mssql/server:2022-latest + + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Caching dependencies + uses: actions/setup-node@v3 + with: + node-version: '14' + cache: 'npm' + cache-dependency-path: | + ./npm-shrinkwrap.json + ./Common/npm-shrinkwrap.json + ./DocService/npm-shrinkwrap.json + + - name: Install modules + run: | + npm ci + npm --prefix Common ci + npm --prefix DocService ci + + - name: Creating service DB configuration + run: | + echo '{"services": {"CoAuthoring": {"sql": {"type": "mssql", "dbHost": "localhost", "dbPort": 8080, "dbUser": "sa", "dbPass": "onlYoff1ce"}}}}' >> Common/config/local.json + + - name: Await database service to finish startup + run: sleep 5 + + - name: Creating schema + run: | + docker cp ./schema/mssql/createdb.sql mssql:/ + docker exec mssql /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P onlYoff1ce -i /createdb.sql + + - name: Run Jest + run: npm run "integration database tests" diff --git a/.github/workflows/mysqlDatabaseTests.yml b/.github/workflows/mysqlDatabaseTests.yml new file mode 100644 index 000000000..c8e424e63 --- /dev/null +++ b/.github/workflows/mysqlDatabaseTests.yml @@ -0,0 +1,47 @@ +name: MYSQL database tests +on: + push: + branches: + - '**' + paths: + - 'DocService/sources/databaseConnectors/baseConnector.js' + - 'DocService/sources/databaseConnectors/mysqlConnector.js' +jobs: + mysql-tests: + name: MYSQL + runs-on: ubuntu-latest + + steps: + - name: Run Mysql DB docker container + run: docker run --name mysql -p 8080:3306 -p 8081:33060 -e MYSQL_HOST=127.0.0.1 -e MYSQL_ROOT_PASSWORD=onlyoffice -e MYSQL_DATABASE=onlyoffice -d mysql:latest + + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Caching dependencies + uses: actions/setup-node@v3 + with: + node-version: '16' + cache: 'npm' + cache-dependency-path: | + ./npm-shrinkwrap.json + ./Common/npm-shrinkwrap.json + ./DocService/npm-shrinkwrap.json + + - name: Install modules + run: | + npm ci + npm --prefix Common ci + npm --prefix DocService ci + + - name: Creating service DB configuration + run: | + echo '{"services": {"CoAuthoring": {"sql": {"type": "mysql", "dbHost": "127.0.0.1", "dbPort": "8080", "dbUser": "root", "dbPass": "onlyoffice"}}}}' >> Common/config/local.json + + - name : Creating schema + run: | + docker cp ./schema/mysql/createdb.sql mysql:/ + docker exec mysql mysql -h 127.0.0.1 -u root --password=onlyoffice -D onlyoffice -e 'source /createdb.sql' + + - name: Run Jest + run: npm run "integration database tests" diff --git a/.github/workflows/oracleDatabaseTests.yml b/.github/workflows/oracleDatabaseTests.yml new file mode 100644 index 000000000..be6a37a47 --- /dev/null +++ b/.github/workflows/oracleDatabaseTests.yml @@ -0,0 +1,50 @@ +name: Oracle database tests +on: + push: + branches: + - '**' + paths: + - 'DocService/sources/databaseConnectors/baseConnector.js' + - 'DocService/sources/databaseConnectors/oracleConnector.js' +jobs: + oracle-tests: + name: Oracle + runs-on: ubuntu-latest + + steps: + - name: Run Oracle DB docker container + run: docker run --name oracle -p 8080:1521 -p 8081:5500 -e ORACLE_PASSWORD=admin -e APP_USER=onlyoffice -e APP_USER_PASSWORD=onlyoffice -d gvenzl/oracle-xe:21-slim + + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Caching dependencies + uses: actions/setup-node@v3 + with: + node-version: '14' + cache: 'npm' + cache-dependency-path: | + ./npm-shrinkwrap.json + ./Common/npm-shrinkwrap.json + ./DocService/npm-shrinkwrap.json + + - name: Install modules + run: | + npm ci + npm --prefix Common ci + npm --prefix DocService ci + + - name: Creating service DB configuration + run: | + echo '{"services": {"CoAuthoring": {"sql": {"type": "oracle", "dbHost": "127.0.0.1", "dbPort": "8080", "dbUser": "onlyoffice", "dbPass": "onlyoffice", "dbName": "xepdb1"}}}}' >> Common/config/local.json + + - name: Await database service to finish startup + run: sleep 15 + + - name: Creating schema + run: | + docker cp ./schema/oracle/createdb.sql oracle:/ + docker exec oracle sqlplus -s onlyoffice/onlyoffice@//localhost/xepdb1 @/createdb.sql + + - name: Run Jest + run: npm run "integration database tests" diff --git a/.github/workflows/postgreDatabaseTests.yml b/.github/workflows/postgreDatabaseTests.yml new file mode 100644 index 000000000..5df3d7c5e --- /dev/null +++ b/.github/workflows/postgreDatabaseTests.yml @@ -0,0 +1,47 @@ +name: Postgre database tests +on: + push: + branches: + - '**' + paths: + - 'DocService/sources/databaseConnectors/baseConnector.js' + - 'DocService/sources/databaseConnectors/postgreConnector.js' +jobs: + postgres-tests: + name: Postgres + runs-on: ubuntu-latest + + steps: + - name: Run Postgres DB docker container + run: docker run --name postgres -p 5432:5432 -e POSTGRES_PASSWORD=onlyoffice -e POSTGRES_USER=onlyoffice -e POSTGRES_DB=onlyoffice -d postgres:latest + + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Caching dependencies + uses: actions/setup-node@v3 + with: + node-version: '14' + cache: 'npm' + cache-dependency-path: | + ./npm-shrinkwrap.json + ./Common/npm-shrinkwrap.json + ./DocService/npm-shrinkwrap.json + + - name: Install modules + run: | + npm ci + npm --prefix Common ci + npm --prefix DocService ci + + - name: Creating service DB configuration + run: | + echo '{"services": {"CoAuthoring": {"sql": {"dbHost": "127.0.0.1"}}}}' >> Common/config/local.json + + - name: Creating schema + run: | + docker cp ./schema/postgresql/createdb.sql postgres:/ + docker exec postgres psql -d onlyoffice -U onlyoffice -a -f /createdb.sql + + - name: Run Jest + run: npm run "integration database tests" diff --git a/.github/workflows/unitTests.yml b/.github/workflows/unitTests.yml new file mode 100644 index 000000000..e1d0d2b67 --- /dev/null +++ b/.github/workflows/unitTests.yml @@ -0,0 +1,36 @@ +name: Service unit tests +on: + push: + branches: + - '**' + paths: + - '**.js' + - '!tests/integration/**' + - '!DocService/sources/databaseConnectors/**' +jobs: + unit-tests: + name: Service unit tests + runs-on: ubuntu-latest + + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Caching dependencies + uses: actions/setup-node@v3 + with: + node-version: '14' + cache: 'npm' + cache-dependency-path: | + ./npm-shrinkwrap.json + ./Common/npm-shrinkwrap.json + ./DocService/npm-shrinkwrap.json + + - name: Install modules + run: | + npm ci + npm --prefix Common ci + npm --prefix DocService ci + + - name: Run Jest + run: npm run "unit tests" diff --git a/Common/config/default.json b/Common/config/default.json index 4341e1c28..57d1c12b0 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -35,6 +35,8 @@ "s3ForcePathStyle": true, "externalHost": "" }, + "persistentStorage": { + }, "rabbitmq": { "url": "amqp://guest:guest@localhost:5672", "socketOptions": {}, @@ -84,25 +86,28 @@ "favIconUrlWord" : "/web-apps/apps/documenteditor/main/resources/img/favicon.ico", "favIconUrlCell" : "/web-apps/apps/spreadsheeteditor/main/resources/img/favicon.ico", "favIconUrlSlide" : "/web-apps/apps/presentationeditor/main/resources/img/favicon.ico", + "favIconUrlPdf" : "/web-apps/apps/pdfeditor/main/resources/img/favicon.ico", "fileInfoBlockList" : ["FileUrl"], - "pdfView": ["pdf", "djvu", "xps", "oxps"], - "wordView": ["doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "mhtml", "html", "htm", "xml", "epub", "fb2", "sxw", "stw", "wps", "wpt"], - "wordEdit": ["docx", "docm", "docxf", "oform", "odt", "txt"], + "pdfView": ["djvu", "xps", "oxps"], + "pdfEdit": ["pdf"], + "forms": ["pdf"], + "wordView": ["doc", "dotx", "dotm", "dot", "fodt", "ott", "rtf", "mht", "mhtml", "html", "htm", "xml", "epub", "fb2", "sxw", "stw", "wps", "wpt", "docxf", "oform"], + "wordEdit": ["docx", "docm", "odt", "txt"], "cellView": ["xls", "xlsb", "xltx", "xltm", "xlt", "fods", "ots", "sxc", "xml", "et", "ett"], "cellEdit": ["xlsx", "xlsm", "ods", "csv"], "slideView": ["ppt", "ppsx", "ppsm", "pps", "potx", "potm", "pot", "fodp", "otp", "sxi", "dps", "dpt"], "slideEdit": ["pptx", "pptm", "odp"], - "publicKey": "BgIAAACkAABSU0ExAAgAAAEAAQD/NVqekFNi8X3p6Bvdlaxm0GGuggW5kKfVEQzPGuOkGVrz6DrOMNR+k7Pq8tONY+1NHgS6Z+v3959em78qclVDuQX77Tkml0xMHAQHN4sAHF9iQJS8gOBUKSVKaHD7Z8YXch6F212YSUSc8QphpDSHWVShU7rcUeLQsd/0pkflh5+um4YKEZhm4Mou3vstp5p12NeffyK1WFZF7q4jB7jclAslYKQsP82YY3DcRwu5Tl/+W0ifVcXze0mI7v1reJ12pKn8ifRiq+0q5oJST3TRSrvmjLg9Gt3ozhVIt2HUi3La7Qh40YOAUXm0g/hUq2BepeOp1C7WSvaOFHXe6Hqq", - "modulus": "qnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/w==", - "exponent": "AQAB", - "privateKey": "MIIEowIBAAKCAQEAqnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/wIDAQABAoIBAQCKtUSBs8tNYrGTQTlBHXrwpkDg+u7WSZt5sEcfnkxA39BLtlHU8gGO0E9Ihr8GAL+oWjUsEltJ9GTtN8CJ9lFdPVS8sTiCZR/YQOggmFRZTJyVzMrkXgF7Uwwiu3+KxLiTOZx9eRhfDBlTD8W9fXaegX2i2Xp2ohUhBHthEBLdaZTWFi5Sid/Y0dDzBeP6UIJorZ5D+1ybaeIVHjndpwNsIEUGUxPFLrkeiU8Rm4MJ9ahxfywcP7DjQoPGY9Ge5cBhpxfzERWf732wUD6o3+L9tvOBU00CLVjULbGZKTVE2FJMyXK9jr6Zor9Mkhomp6/8Agkr9rp+TPyelFGYEz8hAoGBAOEc09CrL3eYBkhNEcaMQzxBLvOGpg8kaDX5SaArHfl9+U9yzRqss4ARECanp9HuHfjMQo7iejao0ngDjL7BNMSaH74QlSsPOY2iOm8Qvx8/zb7g4h9r1zLjFZb3mpSA4snRZvvdiZ9ugbuVPmhXnDzRRMg45MibJeeOTJNylofRAoGBAMHfF/WutqKDoX25qZo9m74W4bttOj6oIDk1N4/c6M1Z1v/aptYSE06bkWngj9P46kqjaay4hgMtzyGruc5aojPx5MHHf5bo14+Jv4NzYtR2llrUxO+UJX7aCfUYXI7RC93GUmhpeQ414j7SNAXec58d7e+ETw+6cHiAWO4uOSTPAoGATPq5qDLR4Zi4FUNdn8LZPyKfNqHF6YmupT5hIgd8kZO1jKiaYNPL8jBjkIRmjBBcaXcYD5p85nImvumf2J9jNxPpZOpwyC/Fo5xlVROp97qu1eY7DTmodntXJ6/2SXAlnZQhHmHsrPtyG752f+HtyJJbbgiem8cKWDu+DfHybfECgYBbSLo1WiBwgN4nHqZ3E48jgA6le5azLeKOTTpuKKwNFMIhEkj//t7MYn+jhLL0Mf3PSwZU50Vidc1To1IHkbFSGBGIFHFFEzl8QnXEZS4hr/y3o/teezj0c6HAn8nlDRUzRVBEDXWMdV6kCcGpCccTIrqHzpqTY0vV0UkOTQFnDQKBgAxSEhm/gtCYJIMCBe+KBJT9uECV5xDQopTTjsGOkd4306EN2dyPOIlAfwM6K/0qWisa0Ei5i8TbRRuBeTTdLEYLqXCJ7fj5tdD1begBdSVtHQ2WHqzPJSuImTkFi9NXxd1XUyZFM3y6YQvlssSuL7QSxUIEtZHnrJTt3QDd10dj", - "publicKeyOld": "BgIAAACkAABSU0ExAAgAAAEAAQD/NVqekFNi8X3p6Bvdlaxm0GGuggW5kKfVEQzPGuOkGVrz6DrOMNR+k7Pq8tONY+1NHgS6Z+v3959em78qclVDuQX77Tkml0xMHAQHN4sAHF9iQJS8gOBUKSVKaHD7Z8YXch6F212YSUSc8QphpDSHWVShU7rcUeLQsd/0pkflh5+um4YKEZhm4Mou3vstp5p12NeffyK1WFZF7q4jB7jclAslYKQsP82YY3DcRwu5Tl/+W0ifVcXze0mI7v1reJ12pKn8ifRiq+0q5oJST3TRSrvmjLg9Gt3ozhVIt2HUi3La7Qh40YOAUXm0g/hUq2BepeOp1C7WSvaOFHXe6Hqq", - "modulusOld": "qnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/w==", - "exponentOld": "AQAB", - "privateKeyOld": "MIIEowIBAAKCAQEAqnro3nUUjvZK1i7UqeOlXmCrVPiDtHlRgIPReAjt2nKL1GG3SBXO6N0aPbiM5rtK0XRPUoLmKu2rYvSJ/Kmkdp14a/3uiEl788VVn0hb/l9OuQtH3HBjmM0/LKRgJQuU3LgHI67uRVZYtSJ/n9fYdZqnLfveLsrgZpgRCoabrp+H5Uem9N+x0OJR3LpToVRZhzSkYQrxnERJmF3bhR5yF8Zn+3BoSiUpVOCAvJRAYl8cAIs3BwQcTEyXJjnt+wW5Q1VyKr+bXp/39+tnugQeTe1jjdPy6rOTftQwzjro81oZpOMazwwR1aeQuQWCrmHQZqyV3Rvo6X3xYlOQnlo1/wIDAQABAoIBAQCKtUSBs8tNYrGTQTlBHXrwpkDg+u7WSZt5sEcfnkxA39BLtlHU8gGO0E9Ihr8GAL+oWjUsEltJ9GTtN8CJ9lFdPVS8sTiCZR/YQOggmFRZTJyVzMrkXgF7Uwwiu3+KxLiTOZx9eRhfDBlTD8W9fXaegX2i2Xp2ohUhBHthEBLdaZTWFi5Sid/Y0dDzBeP6UIJorZ5D+1ybaeIVHjndpwNsIEUGUxPFLrkeiU8Rm4MJ9ahxfywcP7DjQoPGY9Ge5cBhpxfzERWf732wUD6o3+L9tvOBU00CLVjULbGZKTVE2FJMyXK9jr6Zor9Mkhomp6/8Agkr9rp+TPyelFGYEz8hAoGBAOEc09CrL3eYBkhNEcaMQzxBLvOGpg8kaDX5SaArHfl9+U9yzRqss4ARECanp9HuHfjMQo7iejao0ngDjL7BNMSaH74QlSsPOY2iOm8Qvx8/zb7g4h9r1zLjFZb3mpSA4snRZvvdiZ9ugbuVPmhXnDzRRMg45MibJeeOTJNylofRAoGBAMHfF/WutqKDoX25qZo9m74W4bttOj6oIDk1N4/c6M1Z1v/aptYSE06bkWngj9P46kqjaay4hgMtzyGruc5aojPx5MHHf5bo14+Jv4NzYtR2llrUxO+UJX7aCfUYXI7RC93GUmhpeQ414j7SNAXec58d7e+ETw+6cHiAWO4uOSTPAoGATPq5qDLR4Zi4FUNdn8LZPyKfNqHF6YmupT5hIgd8kZO1jKiaYNPL8jBjkIRmjBBcaXcYD5p85nImvumf2J9jNxPpZOpwyC/Fo5xlVROp97qu1eY7DTmodntXJ6/2SXAlnZQhHmHsrPtyG752f+HtyJJbbgiem8cKWDu+DfHybfECgYBbSLo1WiBwgN4nHqZ3E48jgA6le5azLeKOTTpuKKwNFMIhEkj//t7MYn+jhLL0Mf3PSwZU50Vidc1To1IHkbFSGBGIFHFFEzl8QnXEZS4hr/y3o/teezj0c6HAn8nlDRUzRVBEDXWMdV6kCcGpCccTIrqHzpqTY0vV0UkOTQFnDQKBgAxSEhm/gtCYJIMCBe+KBJT9uECV5xDQopTTjsGOkd4306EN2dyPOIlAfwM6K/0qWisa0Ei5i8TbRRuBeTTdLEYLqXCJ7fj5tdD1begBdSVtHQ2WHqzPJSuImTkFi9NXxd1XUyZFM3y6YQvlssSuL7QSxUIEtZHnrJTt3QDd10dj", + "publicKey": "BgIAAACkAABSU0ExAAgAAAEAAQBpTpiJQ2hD8plpGTfEEmcq4IKyr31HikXpuVSBraMfqyodn2PGXBJ3daNSmdPOc0Nz4HO9Auljn8YYXDPBdpiABptSKvEDPF23Q+Qytg0+vCRyondyBcW91w7KLzXce3fnk8ZfJ8QtbZPL9m11wJIWZueQF+l0HKYx4lty+nccbCanytFTADkGQ3SnmExGEF3rBz6I9+OcrDDK9NKPJgEmCiuyei/d4XbPgKls3EIG0h38X5mVF2VytfWm2Yu850B6z3N4MYhj4b4vsYT62zEC4pMRUeb8dIBy4Jsmr3avtmeO00MUH6DVyPC8nirixj2YIOPKk13CdVqGDSXA3cvl", + "modulus": "E5CBDDC0250D865A75C25D93CAE320983DC6E22A9EBCF0C8D5A01F1443D38E67B6AF76AF269BE0728074FCE6511193E20231DBFA84B12FBEE16388317873CF7A40E7BC8BD9A6F5B572651795995FFC1DD20642DC6CA980CF76E1DD2F7AB22B0A2601268FD2F4CA30AC9CE3F7883E07EB5D10464C98A7744306390053D1CAA7266C1C77FA725BE231A61C74E91790E7661692C0756DF6CB936D2DC4275FC693E7777BDC352FCA0ED7BDC5057277A27224BC3E0DB632E443B75D3C03F12A529B06809876C1335C18C69F63E902BD73E0734373CED39952A37577125CC6639F1D2AAB1FA3AD8154B9E9458A477DAFB282E02A6712C437196999F243684389984E69", + "exponent": 65537, + "privateKey": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDly93AJQ2GWnXC\nXZPK4yCYPcbiKp688MjVoB8UQ9OOZ7avdq8mm+BygHT85lERk+ICMdv6hLEvvuFj\niDF4c896QOe8i9mm9bVyZReVmV/8HdIGQtxsqYDPduHdL3qyKwomASaP0vTKMKyc\n4/eIPgfrXRBGTJindEMGOQBT0cqnJmwcd/pyW+Ixphx06ReQ52YWksB1bfbLk20t\nxCdfxpPnd3vcNS/KDte9xQVyd6JyJLw+DbYy5EO3XTwD8SpSmwaAmHbBM1wYxp9j\n6QK9c+BzQ3PO05lSo3V3ElzGY58dKqsfo62BVLnpRYpHfa+yguAqZxLENxlpmfJD\naEOJmE5pAgMBAAECggEALiL+RKOr0Xu8BOgQ0j1DwA03LxVrhXe6etmJI+JySTcd\ngKENjWziZVrRIi2DvUm5qMMl7WhSwslKK1eexxZJY7xASqSxcEoIwgz17T07/jxm\nfIdUBiUKDZ1Kv8PWmIr3oKW+fkXWi/m1zlIe0qXRpTmsGNEsHQLEqi0rmaiXTXOR\n/2Ldwi6kZR3sWFx97YS4Mx/pueGJTXEai6AVEZzN5Gog6xD8HXR1Rvq+hhd+MocG\nfnU4HgilKRfoJlWd9FOscgSufKG0L3ViO4fSKU46l5aullDYUk5ECMWiwuKSqSE7\nqD45jI3mbOre7S4u3S3TWdD3lzwiXL49LdwKlEC4mQKBgQD0sLr0GH4Wr+QX2xJE\nuA/Cb8QW41l8iSCBTRZZR/sJOd+o3rbcVidlzO/EbZblXG4ZPDmRjgBCGKIP5EZi\n0DsL+Wv32WOo44LpxJGhqExbm0H1iZ1zZ97l0P8fvIhHE42gmaLToOIGDhPSXGvv\nzlqOHbGbq4jsERc1jp1bej5q6wKBgQDwaueIc4pRchH98QYidcyr8Vwg9KhbnfYX\ny3W4RPlZtBdF34iJaio+ASzugo/zy1RTcVrsCskYWXyKDUQz1yu0iCng+fDCUnTm\nXGmEoEGNhk4vTJOt7hBav1/Ja/dUipGf6mXUuanwJ0e+1/Et/B0ah5X1Um5AyNZI\nM+SyRz3u+wKBgQCjvtUNXoqaghCBCmB6TjZ1prexnWkYFugCv2SSUMIk1W7gIlJ6\ntsjcrj1R1Qii6qzfBFd+GWoA0V06h0e2/qRVCg//p6GytrW33IycgvS+ZPLJ7tLI\nFR2r66WfRlpoPiSL8eRt/P7kkG0hXCn7K7ub2TEu/Ka/W1yNwad6PR8iCwKBgQC8\nXcZSrtQsxAc8w99emJVoEo9wcsCGJ9ltA0iUu9XyZpvlbyJ3J+s48YrWxQ0sop7L\nUgE+96Rfo51kPMi3JVtk81p8ntf4KMrWwokaFMXHsPcJMCJ1IBVIRLE0C5eZcYhv\nlyN57I4tT1lzOZYJxYK4Cot/zrn7oF/j6mTBGfh4iQKBgQCiJMUxRz01/czH/XSX\ngo3dVbHQ4FEOufWnE3Eb93S8r0/eq1RM118rb0TqzuiadW2xYDU4nucWQlrlmq0d\nFY/m+Hy97pqyk6jmoU5I/D+ssBIoYHWLnH9/xfvDEk2JGSJSHtzu0D4EDC/rgQ49\nMbYsO5oUrF8tPlhj5vzbf3GKLA==\n-----END PRIVATE KEY-----\n", + "publicKeyOld": "BgIAAACkAABSU0ExAAgAAAEAAQBpTpiJQ2hD8plpGTfEEmcq4IKyr31HikXpuVSBraMfqyodn2PGXBJ3daNSmdPOc0Nz4HO9Auljn8YYXDPBdpiABptSKvEDPF23Q+Qytg0+vCRyondyBcW91w7KLzXce3fnk8ZfJ8QtbZPL9m11wJIWZueQF+l0HKYx4lty+nccbCanytFTADkGQ3SnmExGEF3rBz6I9+OcrDDK9NKPJgEmCiuyei/d4XbPgKls3EIG0h38X5mVF2VytfWm2Yu850B6z3N4MYhj4b4vsYT62zEC4pMRUeb8dIBy4Jsmr3avtmeO00MUH6DVyPC8nirixj2YIOPKk13CdVqGDSXA3cvl", + "modulusOld": "E5CBDDC0250D865A75C25D93CAE320983DC6E22A9EBCF0C8D5A01F1443D38E67B6AF76AF269BE0728074FCE6511193E20231DBFA84B12FBEE16388317873CF7A40E7BC8BD9A6F5B572651795995FFC1DD20642DC6CA980CF76E1DD2F7AB22B0A2601268FD2F4CA30AC9CE3F7883E07EB5D10464C98A7744306390053D1CAA7266C1C77FA725BE231A61C74E91790E7661692C0756DF6CB936D2DC4275FC693E7777BDC352FCA0ED7BDC5057277A27224BC3E0DB632E443B75D3C03F12A529B06809876C1335C18C69F63E902BD73E0734373CED39952A37577125CC6639F1D2AAB1FA3AD8154B9E9458A477DAFB282E02A6712C437196999F243684389984E69", + "exponentOld": 65537, + "privateKeyOld": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDly93AJQ2GWnXC\nXZPK4yCYPcbiKp688MjVoB8UQ9OOZ7avdq8mm+BygHT85lERk+ICMdv6hLEvvuFj\niDF4c896QOe8i9mm9bVyZReVmV/8HdIGQtxsqYDPduHdL3qyKwomASaP0vTKMKyc\n4/eIPgfrXRBGTJindEMGOQBT0cqnJmwcd/pyW+Ixphx06ReQ52YWksB1bfbLk20t\nxCdfxpPnd3vcNS/KDte9xQVyd6JyJLw+DbYy5EO3XTwD8SpSmwaAmHbBM1wYxp9j\n6QK9c+BzQ3PO05lSo3V3ElzGY58dKqsfo62BVLnpRYpHfa+yguAqZxLENxlpmfJD\naEOJmE5pAgMBAAECggEALiL+RKOr0Xu8BOgQ0j1DwA03LxVrhXe6etmJI+JySTcd\ngKENjWziZVrRIi2DvUm5qMMl7WhSwslKK1eexxZJY7xASqSxcEoIwgz17T07/jxm\nfIdUBiUKDZ1Kv8PWmIr3oKW+fkXWi/m1zlIe0qXRpTmsGNEsHQLEqi0rmaiXTXOR\n/2Ldwi6kZR3sWFx97YS4Mx/pueGJTXEai6AVEZzN5Gog6xD8HXR1Rvq+hhd+MocG\nfnU4HgilKRfoJlWd9FOscgSufKG0L3ViO4fSKU46l5aullDYUk5ECMWiwuKSqSE7\nqD45jI3mbOre7S4u3S3TWdD3lzwiXL49LdwKlEC4mQKBgQD0sLr0GH4Wr+QX2xJE\nuA/Cb8QW41l8iSCBTRZZR/sJOd+o3rbcVidlzO/EbZblXG4ZPDmRjgBCGKIP5EZi\n0DsL+Wv32WOo44LpxJGhqExbm0H1iZ1zZ97l0P8fvIhHE42gmaLToOIGDhPSXGvv\nzlqOHbGbq4jsERc1jp1bej5q6wKBgQDwaueIc4pRchH98QYidcyr8Vwg9KhbnfYX\ny3W4RPlZtBdF34iJaio+ASzugo/zy1RTcVrsCskYWXyKDUQz1yu0iCng+fDCUnTm\nXGmEoEGNhk4vTJOt7hBav1/Ja/dUipGf6mXUuanwJ0e+1/Et/B0ah5X1Um5AyNZI\nM+SyRz3u+wKBgQCjvtUNXoqaghCBCmB6TjZ1prexnWkYFugCv2SSUMIk1W7gIlJ6\ntsjcrj1R1Qii6qzfBFd+GWoA0V06h0e2/qRVCg//p6GytrW33IycgvS+ZPLJ7tLI\nFR2r66WfRlpoPiSL8eRt/P7kkG0hXCn7K7ub2TEu/Ka/W1yNwad6PR8iCwKBgQC8\nXcZSrtQsxAc8w99emJVoEo9wcsCGJ9ltA0iUu9XyZpvlbyJ3J+s48YrWxQ0sop7L\nUgE+96Rfo51kPMi3JVtk81p8ntf4KMrWwokaFMXHsPcJMCJ1IBVIRLE0C5eZcYhv\nlyN57I4tT1lzOZYJxYK4Cot/zrn7oF/j6mTBGfh4iQKBgQCiJMUxRz01/czH/XSX\ngo3dVbHQ4FEOufWnE3Eb93S8r0/eq1RM118rb0TqzuiadW2xYDU4nucWQlrlmq0d\nFY/m+Hy97pqyk6jmoU5I/D+ssBIoYHWLnH9/xfvDEk2JGSJSHtzu0D4EDC/rgQ49\nMbYsO5oUrF8tPlhj5vzbf3GKLA==\n-----END PRIVATE KEY-----\n", "refreshLockInterval": "10m", "dummy" : { - "enable": true, + "enable": false, "sampleFilePath": "" } }, @@ -119,6 +124,23 @@ "useClones": false } }, + "externalRequest": { + "directIfIn" : { + "allowList": [], + "jwtToken": true + }, + "action": { + "allow": true, + "blockPrivateIP": true, + "proxyUrl": "", + "proxyUser": { + "username": "", + "password": "" + }, + "proxyHeaders": { + } + } + }, "services": { "CoAuthoring": { "server": { @@ -144,11 +166,11 @@ "openProtectedFile": true, "isAnonymousSupport": true, "editorDataStorage": "editorDataMemory", + "editorStatStorage": "", "assemblyFormatAsOrigin": true, "newFileTemplate" : "../../document-templates/new", "downloadFileAllowExt": ["pdf", "xlsx"], - "tokenRequiredParams": true, - "allowPrivateIPAddressForSignedRequests": true + "tokenRequiredParams": true }, "requestDefaults": { "headers": { @@ -180,15 +202,36 @@ "charset": "utf8", "connectionlimit": 10, "max_allowed_packet": 1048575, - "pgPoolExtraOptions": {}, - "damengExtraOptions": {}, - "oracleExtraOptions": {}, + "pgPoolExtraOptions": { + "idleTimeoutMillis": 30000, + "maxLifetimeSeconds ": 60000, + "statement_timeout ": 60000, + "query_timeout ": 60000, + "connectionTimeoutMillis": 60000 + }, + "damengExtraOptions": { + "columnNameUpperCase": false, + "columnNameCase": "lower", + "connectTimeout": 60000, + "loginEncrypt": false, + "localTimezone": 0, + "poolTimeout": 60, + "socketTimeout": 60000, + "queueTimeout": 60000 + }, + "oracleExtraOptions": { + "connectTimeout": 60 + }, "msSqlExtraOptions": { "options": { "encrypt": false, "trustServerCertificate": true + }, + "pool": { + "idleTimeoutMillis": 30000 } - } + }, + "mysqlExtraOptions": {} }, "redis": { "name": "redis", @@ -197,10 +240,14 @@ "port": 6379, "options": {}, "optionsCluster": {}, - "iooptions": {}, + "iooptions": { + "lazyConnect": true + }, "iooptionsClusterNodes": [ ], - "iooptionsClusterOptions": {} + "iooptionsClusterOptions": { + "lazyConnect": true + } }, "pubsub": { "maxChanges": 1000 diff --git a/Common/npm-shrinkwrap.json b/Common/npm-shrinkwrap.json index 5173fdfc5..94110ddb8 100644 --- a/Common/npm-shrinkwrap.json +++ b/Common/npm-shrinkwrap.json @@ -1472,16 +1472,6 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, - "fs-extra": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.0.tgz", - "integrity": "sha512-EglNDLRpmaTWiD/qraZn6HREAEAHJcJOmxNEYwq6xeMKnVMAy3GUcFB+wXt2C6k4CNvB/mP1y/U3dzvKKj5OtQ==", - "requires": { - "graceful-fs": "^4.1.2", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - } - }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -1577,14 +1567,6 @@ "minimist": "^1.2.0" } }, - "jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", - "requires": { - "graceful-fs": "^4.1.6" - } - }, "jsonwebtoken": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz", @@ -1722,21 +1704,6 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha512-SknJC52obPfGQPnjIkXbmA6+5H15E+fR+E4iR2oQ3zzCLbd7/ONua69R/Gw7AgkTLsRG+r5fzksYwWe1AgTyWA==", - "requires": { - "minimist": "0.0.8" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha512-miQKw5Hv4NS1Psg2517mV4e4dYNaO3++hjAvLOAzKqZ61rH8NS1SK+vbfBWZ5PY/Me/bEWhUwqMghEW5Fb9T7Q==" - } - } - }, "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", @@ -2050,11 +2017,6 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, - "universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" - }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", diff --git a/Common/package.json b/Common/package.json index 4d9971ba6..8fdcf927f 100644 --- a/Common/package.json +++ b/Common/package.json @@ -13,12 +13,10 @@ "dnscache": "1.0.1", "escape-string-regexp": "1.0.5", "forwarded": "0.1.2", - "fs-extra": "7.0.0", "ipaddr.js": "1.8.1", "jsonwebtoken": "9.0.0", "log4js": "6.4.1", "mime": "2.3.1", - "mkdirp": "0.5.1", "ms": "2.1.1", "node-cache": "4.2.1", "node-statsd": "0.1.1", diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 83fe464cc..fc8756b90 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -32,6 +32,7 @@ 'use strict'; +const config = require("config"); const constants = require('./constants'); function InputCommand(data, copyExplicit) { @@ -108,6 +109,7 @@ function InputCommand(data, copyExplicit) { this['status_info_in'] = data['status_info_in']; this['attempt'] = data['attempt']; this['convertToOrigin'] = data['convertToOrigin']; + this['isSaveAs'] = data['isSaveAs']; if (copyExplicit) { this['withAuthorization'] = data['withAuthorization']; this['externalChangeInfo'] = data['externalChangeInfo']; @@ -171,6 +173,7 @@ function InputCommand(data, copyExplicit) { this['attempt'] = undefined; this['convertToOrigin'] = undefined; this['originformat'] = undefined; + this['isSaveAs'] = undefined; } } InputCommand.prototype = { @@ -363,8 +366,12 @@ InputCommand.prototype = { getJsonParams: function() { return this['jsonparams']; }, - setJsonParams: function(data) { - this['jsonparams'] = data; + appendJsonParams: function (data) { + if (this['jsonparams']) { + config.util.extendDeep(this['jsonparams'], data); + } else { + this['jsonparams'] = data; + } }, getLCID: function() { return this['lcid']; @@ -503,6 +510,12 @@ InputCommand.prototype = { }, setConvertToOrigin: function(data) { this['convertToOrigin'] = data; + }, + getIsSaveAs: function() { + return this['isSaveAs']; + }, + setIsSaveAs: function(data) { + this['isSaveAs'] = data; } }; diff --git a/Common/sources/constants.js b/Common/sources/constants.js index 36471cad7..ffaabaf78 100644 --- a/Common/sources/constants.js +++ b/Common/sources/constants.js @@ -50,7 +50,8 @@ exports.VIEWER_ONLY = /^(?:(pdf|djvu|xps|oxps))$/; exports.DEFAULT_DOC_ID = 'docId'; exports.DEFAULT_USER_ID = 'userId'; exports.ALLOWED_PROTO = /^https?$/i; -exports.SHARED_KEY_NAME = 'WOPISrc'; +exports.SHARD_KEY_WOPI_NAME = 'WOPISrc'; +exports.SHARD_KEY_API_NAME = 'shardkey'; exports.RIGHTS = { None : 0, @@ -281,3 +282,12 @@ exports.FILE_STATUS_UPDATE_VERSION = 'updateversion'; exports.ACTIVEMQ_QUEUE_PREFIX = 'queue://'; exports.ACTIVEMQ_TOPIC_PREFIX = 'topic://'; + +exports.TEMPLATES_DEFAULT_LOCALE = 'en-US'; +exports.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP = { + 'en': 'en-US', + 'pt': 'pt-BR', + 'zh': 'zh-CH', + 'pt-PT': 'pt-PT', + 'zh-TW': 'zh-TW' +}; \ No newline at end of file diff --git a/Common/sources/operationContext.js b/Common/sources/operationContext.js index 0eeb11a5f..b9f9aaca7 100644 --- a/Common/sources/operationContext.js +++ b/Common/sources/operationContext.js @@ -41,11 +41,12 @@ function Context(){ this.logger = logger.getLogger('nodeJS'); this.initDefault(); } -Context.prototype.init = function(tenant, docId, userId, opt_shardKey) { +Context.prototype.init = function(tenant, docId, userId, opt_shardKey, opt_WopiSrc) { this.setTenant(tenant); this.setDocId(docId); this.setUserId(userId); this.setShardKey(opt_shardKey); + this.setWopiSrc(opt_WopiSrc); this.config = null; this.secret = null; @@ -65,21 +66,23 @@ Context.prototype.initFromConnection = function(conn) { } } let userId = conn.user?.id; - let shardKey = utils.getShardByConnection(this, conn); - this.init(tenant, docId || this.docId, userId || this.userId, shardKey); + let shardKey = utils.getShardKeyByConnection(this, conn); + let wopiSrc = utils.getWopiSrcByConnection(this, conn); + this.init(tenant, docId || this.docId, userId || this.userId, shardKey, wopiSrc); }; Context.prototype.initFromRequest = function(req) { let tenant = tenantManager.getTenantByRequest(this, req); let shardKey = utils.getShardKeyByRequest(this, req); - this.init(tenant, this.docId, this.userId, shardKey); + let wopiSrc = utils.getWopiSrcByRequest(this, req); + this.init(tenant, this.docId, this.userId, shardKey, wopiSrc); }; Context.prototype.initFromTaskQueueData = function(task) { let ctx = task.getCtx(); - this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey, ctx.wopiSrc); }; Context.prototype.initFromPubSub = function(data) { let ctx = data.ctx; - this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey); + this.init(ctx.tenant, ctx.docId, ctx.userId, ctx.shardKey, ctx.wopiSrc); }; Context.prototype.initTenantCache = async function() { this.config = await tenantManager.getTenantConfig(this); @@ -101,12 +104,16 @@ Context.prototype.setUserId = function(userId) { Context.prototype.setShardKey = function(shardKey) { this.shardKey = shardKey; }; +Context.prototype.setWopiSrc = function(wopiSrc) { + this.wopiSrc = wopiSrc; +}; Context.prototype.toJSON = function() { return { tenant: this.tenant, docId: this.docId, userId: this.userId, - shardKey: this.shardKey + shardKey: this.shardKey, + wopiSrc: this.wopiSrc } }; Context.prototype.getCfg = function(property, defaultValue) { diff --git a/Common/sources/storage-base.js b/Common/sources/storage-base.js index 1a85bf313..9f2228b8d 100644 --- a/Common/sources/storage-base.js +++ b/Common/sources/storage-base.js @@ -31,111 +31,185 @@ */ 'use strict'; +const os = require('os'); +const cluster = require('cluster'); var config = require('config'); var utils = require('./utils'); -var storage = require('./' + config.get('storage.name')); -var tenantManager = require('./tenantManager'); +const cfgCacheStorage = config.get('storage'); +const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); -const cfgCacheFolderName = config.get('storage.cacheFolderName'); +const cacheStorage = require('./' + cfgCacheStorage.name); +const persistentStorage = require('./' + cfgPersistentStorage.name); +const tenantManager = require('./tenantManager'); + +const HEALTH_CHECK_KEY_MAX = 10000; function getStoragePath(ctx, strPath, opt_specialDir) { - opt_specialDir = opt_specialDir || cfgCacheFolderName; - return opt_specialDir + '/' + tenantManager.getTenantPathPrefix(ctx) + strPath.replace(/\\/g, '/') + opt_specialDir = opt_specialDir || cfgCacheStorage.cacheFolderName; + return opt_specialDir + '/' + tenantManager.getTenantPathPrefix(ctx) + strPath.replace(/\\/g, '/'); +} +function getStorage(opt_specialDir) { + return opt_specialDir ? persistentStorage : cacheStorage; +} +function getStorageCfg(ctx, opt_specialDir) { + return opt_specialDir ? cfgPersistentStorage : cfgCacheStorage; +} +function canCopyBetweenStorage(storageCfgSrc, storageCfgDst) { + return storageCfgSrc.name === storageCfgDst.name && storageCfgSrc.endpoint === storageCfgDst.endpoint; +} +function isDiffrentPersistentStorage() { + return !canCopyBetweenStorage(cacheStorage, cfgPersistentStorage); } -exports.headObject = function(ctx, strPath, opt_specialDir) { - return storage.headObject(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.getObject = function(ctx, strPath, opt_specialDir) { - return storage.getObject(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.createReadStream = function(ctx, strPath, opt_specialDir) { - return storage.createReadStream(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.putObject = function(ctx, strPath, buffer, contentLength, opt_specialDir) { - return storage.putObject(getStoragePath(ctx, strPath, opt_specialDir), buffer, contentLength); -}; -exports.uploadObject = function(ctx, strPath, filePath, opt_specialDir) { - return storage.uploadObject(getStoragePath(ctx, strPath, opt_specialDir), filePath); -}; -exports.copyObject = function(ctx, sourceKey, destinationKey, opt_specialDirSrc, opt_specialDirDst) { - let storageSrc = getStoragePath(ctx, sourceKey, opt_specialDirSrc); - let storageDst = getStoragePath(ctx, destinationKey, opt_specialDirDst); - return storage.copyObject(storageSrc, storageDst); -}; -exports.copyPath = function(ctx, sourcePath, destinationPath, opt_specialDirSrc, opt_specialDirDst) { - let storageSrc = getStoragePath(ctx, sourcePath, opt_specialDirSrc); - let storageDst = getStoragePath(ctx, destinationPath, opt_specialDirDst); - return storage.listObjects(storageSrc).then(function(list) { - return Promise.all(list.map(function(curValue) { - return storage.copyObject(curValue, storageDst + '/' + exports.getRelativePath(storageSrc, curValue)); - })); - }); -}; -exports.listObjects = function(ctx, strPath, opt_specialDir) { +async function headObject(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.headObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); +} +async function getObject(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.getObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); +} +async function createReadStream(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.createReadStream(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); +} +async function putObject(ctx, strPath, buffer, contentLength, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.putObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), buffer, contentLength); +} +async function uploadObject(ctx, strPath, filePath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.uploadObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir), filePath); +} +async function copyObject(ctx, sourceKey, destinationKey, opt_specialDirSrc, opt_specialDirDst) { + let storageSrc = getStorage(opt_specialDirSrc); + let storagePathSrc = getStoragePath(ctx, sourceKey, opt_specialDirSrc); + let storagePathDst = getStoragePath(ctx, destinationKey, opt_specialDirDst); + let storageCfgSrc = getStorageCfg(ctx, opt_specialDirSrc); + let storageCfgDst = getStorageCfg(ctx, opt_specialDirDst); + if (canCopyBetweenStorage(storageCfgSrc, storageCfgDst)){ + return await storageSrc.copyObject(storageCfgSrc, storageCfgDst, storagePathSrc, storagePathDst); + } else { + let storageDst = getStorage(opt_specialDirDst); + //todo stream + let buffer = await storageSrc.getObject(storageCfgSrc, storagePathSrc); + return await storageDst.putObject(storageCfgDst, storagePathDst, buffer, buffer.length); + } +} +async function copyPath(ctx, sourcePath, destinationPath, opt_specialDirSrc, opt_specialDirDst) { + let list = await listObjects(ctx, sourcePath, opt_specialDirSrc); + await Promise.all(list.map(function(curValue) { + return copyObject(ctx, curValue, destinationPath + '/' + getRelativePath(sourcePath, curValue), opt_specialDirSrc, opt_specialDirDst); + })); +} +async function listObjects(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); let prefix = getStoragePath(ctx, "", opt_specialDir); - return storage.listObjects(getStoragePath(ctx, strPath, opt_specialDir)).then(function(list) { + try { + let list = await storage.listObjects(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); return list.map((currentValue) => { return currentValue.substring(prefix.length); }); - }).catch(function(e) { + } catch (e) { ctx.logger.error('storage.listObjects: %s', e.stack); return []; - }); -}; -exports.deleteObject = function(ctx, strPath, opt_specialDir) { - return storage.deleteObject(getStoragePath(ctx, strPath, opt_specialDir)); -}; -exports.deleteObjects = function(ctx, strPaths, opt_specialDir) { - var StoragePaths = strPaths.map(function(curValue) { - return getStoragePath(ctx, curValue, opt_specialDir); - }); - return storage.deleteObjects(StoragePaths); -}; -exports.deletePath = function(ctx, strPath, opt_specialDir) { - let storageSrc = getStoragePath(ctx, strPath, opt_specialDir); - return storage.listObjects(storageSrc).then(function(list) { - return storage.deleteObjects(list); - }); -}; -exports.getSignedUrl = function(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir) { - return storage.getSignedUrl(ctx, baseUrl, getStoragePath(ctx, strPath, opt_specialDir), urlType, optFilename, opt_creationDate); -}; -exports.getSignedUrls = function(ctx, baseUrl, strPath, urlType, opt_creationDate, opt_specialDir) { - let storageSrc = getStoragePath(ctx, strPath, opt_specialDir); - return storage.listObjects(storageSrc).then(function(list) { - return Promise.all(list.map(function(curValue) { - return storage.getSignedUrl(ctx, baseUrl, curValue, urlType, undefined, opt_creationDate); - })).then(function(urls) { - var outputMap = {}; - for (var i = 0; i < list.length && i < urls.length; ++i) { - outputMap[exports.getRelativePath(storageSrc, list[i])] = urls[i]; - } - return outputMap; - }); - }); -}; -exports.getSignedUrlsArrayByArray = function(ctx, baseUrl, list, urlType, opt_specialDir) { - return Promise.all(list.map(function(curValue) { - let storageSrc = getStoragePath(ctx, curValue, opt_specialDir); - return storage.getSignedUrl(ctx, baseUrl, storageSrc, urlType, undefined); + } +} +async function deleteObject(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.deleteObject(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); +} +async function deletePath(ctx, strPath, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.deletePath(storageCfg, getStoragePath(ctx, strPath, opt_specialDir)); +} +async function getSignedUrl(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate, opt_specialDir) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + return await storage.getSignedUrl(ctx, storageCfg, baseUrl, getStoragePath(ctx, strPath, opt_specialDir), urlType, optFilename, opt_creationDate); +} +async function getSignedUrls(ctx, baseUrl, strPath, urlType, opt_creationDate, opt_specialDir) { + let storagePathSrc = getStoragePath(ctx, strPath, opt_specialDir); + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + let list = await storage.listObjects(storageCfg, storagePathSrc, storageCfg); + let urls = await Promise.all(list.map(function(curValue) { + return storage.getSignedUrl(ctx, storageCfg, baseUrl, curValue, urlType, undefined, opt_creationDate); })); -}; -exports.getSignedUrlsByArray = function(ctx, baseUrl, list, optPath, urlType, opt_specialDir) { - return exports.getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir).then(function(urls) { - var outputMap = {}; - for (var i = 0; i < list.length && i < urls.length; ++i) { - if (optPath) { - let storageSrc = getStoragePath(ctx, optPath, opt_specialDir); - outputMap[exports.getRelativePath(storageSrc, list[i])] = urls[i]; - } else { - outputMap[list[i]] = urls[i]; - } + let outputMap = {}; + for (let i = 0; i < list.length && i < urls.length; ++i) { + outputMap[getRelativePath(storagePathSrc, list[i])] = urls[i]; + } + return outputMap; +} +async function getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir) { + return await Promise.all(list.map(function (curValue) { + let storage = getStorage(opt_specialDir); + let storageCfg = getStorageCfg(ctx, opt_specialDir); + let storagePathSrc = getStoragePath(ctx, curValue, opt_specialDir); + return storage.getSignedUrl(ctx, storageCfg, baseUrl, storagePathSrc, urlType, undefined); + })); +} +async function getSignedUrlsByArray(ctx, baseUrl, list, optPath, urlType, opt_specialDir) { + let urls = await getSignedUrlsArrayByArray(ctx, baseUrl, list, urlType, opt_specialDir); + var outputMap = {}; + for (var i = 0; i < list.length && i < urls.length; ++i) { + if (optPath) { + let storagePathSrc = getStoragePath(ctx, optPath, opt_specialDir); + outputMap[getRelativePath(storagePathSrc, list[i])] = urls[i]; + } else { + outputMap[list[i]] = urls[i]; } - return outputMap; - }); -}; -exports.getRelativePath = function(strBase, strPath) { + } + return outputMap; +} +function getRelativePath(strBase, strPath) { return strPath.substring(strBase.length + 1); +} +async function healthCheck(ctx, opt_specialDir) { + const clusterId = cluster.isWorker ? cluster.worker.id : ''; + const tempName = 'hc_' + os.hostname() + '_' + clusterId + '_' + Math.round(Math.random() * HEALTH_CHECK_KEY_MAX); + const tempBuffer = Buffer.from([1, 2, 3, 4, 5]); + try { + //It's proper to putObject one tempName + await putObject(ctx, tempName, tempBuffer, tempBuffer.length, opt_specialDir); + //try to prevent case, when another process can remove same tempName + await deleteObject(ctx, tempName, opt_specialDir); + } catch (err) { + ctx.logger.warn('healthCheck storage(%s) error %s', opt_specialDir, err.stack); + } +} +function needServeStatic(opt_specialDir) { + let storage = getStorage(opt_specialDir); + return storage.needServeStatic(); +} + +module.exports = { + headObject, + getObject, + createReadStream, + putObject, + uploadObject, + copyObject, + copyPath, + listObjects, + deleteObject, + deletePath, + getSignedUrl, + getSignedUrls, + getSignedUrlsArrayByArray, + getSignedUrlsByArray, + getRelativePath, + isDiffrentPersistentStorage, + healthCheck, + needServeStatic }; diff --git a/Common/sources/storage-fs.js b/Common/sources/storage-fs.js index e4feee43a..5c0a299f6 100644 --- a/Common/sources/storage-fs.js +++ b/Common/sources/storage-fs.js @@ -32,181 +32,148 @@ 'use strict'; -var fs = require('fs'); -const fse = require('fs-extra') +const { cp, rm, mkdir } = require('fs/promises'); +const { stat, readFile, writeFile } = require('fs/promises'); var path = require('path'); -var mkdirp = require('mkdirp'); var utils = require("./utils"); var crypto = require('crypto'); const ms = require('ms'); +const config = require('config'); const commonDefines = require('./../../Common/sources/commondefines'); const constants = require('./../../Common/sources/constants'); -var config = require('config'); -var configStorage = config.get('storage'); -var cfgBucketName = configStorage.get('bucketName'); -var cfgStorageFolderName = configStorage.get('storageFolderName'); -var configFs = configStorage.get('fs'); -var cfgStorageFolderPath = configFs.get('folderPath'); -var cfgStorageSecretString = configFs.get('secretString'); -var cfgStorageUrlExpires = configFs.get('urlExpires'); const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.sessionabsolute')); -function getFilePath(strPath) { - return path.join(cfgStorageFolderPath, strPath); +//Stubs are needed until integrators pass these parameters to all requests +let shardKeyCached; +let wopiSrcCached; + +function getFilePath(storageCfg, strPath) { + const storageFolderPath = storageCfg.fs.folderPath; + return path.join(storageFolderPath, strPath); } function getOutputPath(strPath) { return strPath.replace(/\\/g, '/'); } -function removeEmptyParent(strPath, done) { - if (cfgStorageFolderPath.length + 1 >= strPath.length) { - done(); + +async function headObject(storageCfg, strPath) { + let fsPath = getFilePath(storageCfg, strPath); + let stats = await stat(fsPath); + return {ContentLength: stats.size}; +} + +async function getObject(storageCfg, strPath) { + let fsPath = getFilePath(storageCfg, strPath); + return await readFile(fsPath); +} + +async function createReadStream(storageCfg, strPath) { + let fsPath = getFilePath(storageCfg, strPath); + let stats = await stat(fsPath); + let contentLength = stats.size; + let readStream = await utils.promiseCreateReadStream(fsPath); + return { + contentLength: contentLength, + readStream: readStream + }; +} + +async function putObject(storageCfg, strPath, buffer, contentLength) { + var fsPath = getFilePath(storageCfg, strPath); + await mkdir(path.dirname(fsPath), {recursive: true}); + + if (Buffer.isBuffer(buffer)) { + await writeFile(fsPath, buffer); } else { - fs.readdir(strPath, function(err, list) { - if (err) { - //we do not react to the error, because most likely this folder was deleted in a neighboring thread - done(); - } else { - if (list.length > 0) { - done(); - } else { - fs.rmdir(strPath, function(err) { - if (err) { - //we do not react to the error, because most likely this folder was deleted in a neighboring thread - done(); - } else { - removeEmptyParent(path.dirname(strPath), function(err) { - done(err); - }); - } - }); - } - } - }); + let writable = await utils.promiseCreateWriteStream(fsPath); + await utils.pipeStreams(buffer, writable, true); } } -exports.headObject = function(strPath) { - return utils.fsStat(getFilePath(strPath)).then(function(stats) { - return {ContentLength: stats.size}; - }); -}; -exports.getObject = function(strPath) { - return utils.readFile(getFilePath(strPath)); -}; -exports.createReadStream = function(strPath) { - let fsPath = getFilePath(strPath); - let contentLength; - return new Promise(function(resolve, reject) { - fs.stat(fsPath, function(err, stats) { - if (err) { - reject(err); - } else { - resolve(stats); - } - }); - }).then(function(stats){ - contentLength = stats.size; - return utils.promiseCreateReadStream(fsPath); - }).then(function(readStream, stats){ - return { - contentLength: contentLength, - readStream: readStream - }; - }); -}; +async function uploadObject(storageCfg, strPath, filePath) { + let fsPath = getFilePath(storageCfg, strPath); + await cp(filePath, fsPath, {force: true, recursive: true}); +} -exports.putObject = function(strPath, buffer, contentLength) { - return new Promise(function(resolve, reject) { - var fsPath = getFilePath(strPath); - mkdirp(path.dirname(fsPath), function(err) { - if (err) { - reject(err); - } else { - //todo 0666 - if (Buffer.isBuffer(buffer)) { - fs.writeFile(fsPath, buffer, function(err) { - if (err) { - reject(err); - } else { - resolve(); - } - }); - } else { - utils.promiseCreateWriteStream(fsPath).then(function(writable) { - buffer.pipe(writable); - }).catch(function(err) { - reject(err); - }); - } - } - }); - }); -}; -exports.uploadObject = function(strPath, filePath) { - let fsPath = getFilePath(strPath); - return fse.copy(filePath, fsPath); -}; -exports.copyObject = function(sourceKey, destinationKey) { - let fsPathSource = getFilePath(sourceKey); - let fsPathSestination = getFilePath(destinationKey); - return fse.copy(fsPathSource, fsPathSestination); -}; -exports.listObjects = function(strPath) { - return utils.listObjects(getFilePath(strPath)).then(function(values) { - return values.map(function(curvalue) { - return getOutputPath(curvalue.substring(cfgStorageFolderPath.length + 1)); - }); - }); -}; -exports.deleteObject = function(strPath) { - return new Promise(function(resolve, reject) { - const fsPath = getFilePath(strPath); - fs.unlink(fsPath, function(err) { - if (err) { - reject(err); - } else { - //resolve(); - removeEmptyParent(path.dirname(fsPath), function(err) { - if (err) { - reject(err); - } else { - resolve(); - } - }); - } - }); +async function copyObject(storageCfgSrc, storageCfgDst, sourceKey, destinationKey) { + let fsPathSource = getFilePath(storageCfgSrc, sourceKey); + let fsPathDestination = getFilePath(storageCfgDst, destinationKey); + await cp(fsPathSource, fsPathDestination, {force: true, recursive: true}); +} + +async function listObjects(storageCfg, strPath) { + const storageFolderPath = storageCfg.fs.folderPath; + let fsPath = getFilePath(storageCfg, strPath); + let values = await utils.listObjects(fsPath); + return values.map(function(curvalue) { + return getOutputPath(curvalue.substring(storageFolderPath.length + 1)); }); -}; -exports.deleteObjects = function(strPaths) { - return Promise.all(strPaths.map(exports.deleteObject)); -}; -exports.getSignedUrl = function(ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { - return new Promise(function(resolve, reject) { - //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path - var userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath); - var uri = '/' + cfgBucketName + '/' + cfgStorageFolderName + '/' + strPath + '/' + userFriendlyName; - //RFC 1123 does not allow underscores https://stackoverflow.com/questions/2180465/can-domain-name-subdomains-have-an-underscore-in-it - var url = utils.checkBaseUrl(ctx, baseUrl).replace(/_/g, "%5f"); - url += uri; - - var date = Date.now(); - let creationDate = opt_creationDate || date; - let expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : cfgStorageUrlExpires) || 31536000; - //todo creationDate can be greater because mysql CURRENT_TIMESTAMP uses local time, not UTC - var expires = creationDate + Math.ceil(Math.abs(date - creationDate)/expiredAfter) * expiredAfter; - expires = Math.ceil(expires / 1000); - expires += expiredAfter; - - var md5 = crypto.createHash('md5').update(expires + decodeURIComponent(uri) + cfgStorageSecretString).digest("base64"); - md5 = md5.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, ""); - - url += '?md5=' + encodeURIComponent(md5); - url += '&expires=' + encodeURIComponent(expires); +} + +async function deleteObject(storageCfg, strPath) { + const fsPath = getFilePath(storageCfg, strPath); + return rm(fsPath, {force: true, recursive: true}); +} + +async function deletePath(storageCfg, strPath) { + const fsPath = getFilePath(storageCfg, strPath); + return rm(fsPath, {force: true, recursive: true, maxRetries: 3}); +} + +async function getSignedUrl(ctx, storageCfg, baseUrl, strPath, urlType, optFilename, opt_creationDate) { + const storageSecretString = storageCfg.fs.secretString; + const storageUrlExpires = storageCfg.fs.urlExpires; + const bucketName = storageCfg.bucketName; + const storageFolderName = storageCfg.storageFolderName; + //replace '/' with %2f before encodeURIComponent becase nginx determine %2f as '/' and get wrong system path + const userFriendlyName = optFilename ? encodeURIComponent(optFilename.replace(/\//g, "%2f")) : path.basename(strPath); + var uri = '/' + bucketName + '/' + storageFolderName + '/' + strPath + '/' + userFriendlyName; + //RFC 1123 does not allow underscores https://stackoverflow.com/questions/2180465/can-domain-name-subdomains-have-an-underscore-in-it + var url = utils.checkBaseUrl(ctx, baseUrl, storageCfg).replace(/_/g, "%5f"); + url += uri; + + var date = Date.now(); + let creationDate = opt_creationDate || date; + let expiredAfter = (commonDefines.c_oAscUrlTypes.Session === urlType ? (cfgExpSessionAbsolute / 1000) : storageUrlExpires) || 31536000; + //todo creationDate can be greater because mysql CURRENT_TIMESTAMP uses local time, not UTC + var expires = creationDate + Math.ceil(Math.abs(date - creationDate) / expiredAfter) * expiredAfter; + expires = Math.ceil(expires / 1000); + expires += expiredAfter; + + var md5 = crypto.createHash('md5').update(expires + decodeURIComponent(uri) + storageSecretString).digest("base64"); + md5 = md5.replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, ""); + + url += '?md5=' + encodeURIComponent(md5); + url += '&expires=' + encodeURIComponent(expires); if (ctx.shardKey) { - url += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + shardKeyCached = ctx.shardKey; + url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } else if (ctx.wopiSrc) { + wopiSrcCached = ctx.wopiSrc; + url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(ctx.wopiSrc)}`; + } else if (shardKeyCached) { + url += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(shardKeyCached)}`; + } else if (wopiSrcCached) { + url += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(wopiSrcCached)}`; } - url += '&filename=' + userFriendlyName; - resolve(url); - }); + url += '&filename=' + userFriendlyName; + return url; +} + +function needServeStatic() { + return true; +} + +module.exports = { + headObject, + getObject, + createReadStream, + putObject, + uploadObject, + copyObject, + listObjects, + deleteObject, + deletePath, + getSignedUrl, + needServeStatic }; diff --git a/Common/sources/storage-s3.js b/Common/sources/storage-s3.js index 66db0d5d0..458b3b807 100644 --- a/Common/sources/storage-s3.js +++ b/Common/sources/storage-s3.js @@ -31,200 +31,225 @@ */ 'use strict'; -var fs = require('fs'); -var url = require('url'); -var path = require('path'); +const fs = require('fs'); +const url = require('url'); +const path = require('path'); const { S3Client, ListObjectsCommand, HeadObjectCommand} = require("@aws-sdk/client-s3"); const { GetObjectCommand, PutObjectCommand, CopyObjectCommand} = require("@aws-sdk/client-s3"); const { DeleteObjectsCommand, DeleteObjectCommand } = require("@aws-sdk/client-s3"); const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); -var mime = require('mime'); -var utils = require('./utils'); +const mime = require('mime'); +const config = require('config'); +const utils = require('./utils'); const ms = require('ms'); const commonDefines = require('./../../Common/sources/commondefines'); -var config = require('config'); -var configStorage = require('config').get('storage'); -var cfgRegion = configStorage.get('region'); -var cfgEndpoint = configStorage.get('endpoint'); -var cfgBucketName = configStorage.get('bucketName'); -var cfgStorageFolderName = configStorage.get('storageFolderName'); -var cfgAccessKeyId = configStorage.get('accessKeyId'); -var cfgSecretAccessKey = configStorage.get('secretAccessKey'); -var cfgSslEnabled = configStorage.get('sslEnabled'); -var cfgS3ForcePathStyle = configStorage.get('s3ForcePathStyle'); -var configFs = configStorage.get('fs'); -var cfgStorageUrlExpires = configFs.get('urlExpires'); const cfgExpSessionAbsolute = ms(config.get('services.CoAuthoring.expire.sessionabsolute')); -/** - * Don't hard-code your credentials! - * Export the following environment variables instead: - * - * export AWS_ACCESS_KEY_ID='AKID' - * export AWS_SECRET_ACCESS_KEY='SECRET' - */ -var configS3 = { - region: cfgRegion, - endpoint: cfgEndpoint, - credentials : { - accessKeyId: cfgAccessKeyId, - secretAccessKey: cfgSecretAccessKey - } -}; +//This operation enables you to delete multiple objects from a bucket using a single HTTP request. You may specify up to 1000 keys. +const MAX_DELETE_OBJECTS = 1000; +let clients = {}; -if (configS3.endpoint) { - configS3.tls = cfgSslEnabled; - configS3.forcePathStyle = cfgS3ForcePathStyle; -} -const client = new S3Client(configS3); +function getS3Client(storageCfg) { + /** + * Don't hard-code your credentials! + * Export the following environment variables instead: + * + * export AWS_ACCESS_KEY_ID='AKID' + * export AWS_SECRET_ACCESS_KEY='SECRET' + */ + let configS3 = { + region: storageCfg.region, + endpoint: storageCfg.endpoint, + credentials : { + accessKeyId: storageCfg.accessKeyId, + secretAccessKey: storageCfg.secretAccessKey + } + }; -//This operation enables you to delete multiple objects from a bucket using a single HTTP request. You may specify up to 1000 keys. -var MAX_DELETE_OBJECTS = 1000; + if (configS3.endpoint) { + configS3.tls = storageCfg.sslEnabled; + configS3.forcePathStyle = storageCfg.s3ForcePathStyle; + } + let configJson = JSON.stringify(configS3); + let client = clients[configJson]; + if (!client) { + client = new S3Client(configS3); + clients[configJson] = client; + } + return client; +} -function getFilePath(strPath) { - //todo - return cfgStorageFolderName + '/' + strPath; +function getFilePath(storageCfg, strPath) { + const storageFolderName = storageCfg.storageFolderName; + return storageFolderName + '/' + strPath; } -function joinListObjects(inputArray, outputArray) { +function joinListObjects(storageCfg, inputArray, outputArray) { if (!inputArray) { return; } - var length = inputArray.length; - for (var i = 0; i < length; i++) { - outputArray.push(inputArray[i].Key.substring((cfgStorageFolderName + '/').length)); + const storageFolderName = storageCfg.storageFolderName; + let length = inputArray.length; + for (let i = 0; i < length; i++) { + outputArray.push(inputArray[i].Key.substring((storageFolderName + '/').length)); } } -async function listObjectsExec(output, params) { - const data = await client.send(new ListObjectsCommand(params)); - joinListObjects(data.Contents, output); +async function listObjectsExec(storageCfg, output, params) { + const data = await getS3Client(storageCfg).send(new ListObjectsCommand(params)); + joinListObjects(storageCfg, data.Contents, output); if (data.IsTruncated && (data.NextMarker || (data.Contents && data.Contents.length > 0))) { - params.Marker = data.NextMarker || data.Contents[data.Contents.length - 1].Key; - return await listObjectsExec(output, params); - } else { + params.Marker = data.NextMarker || data.Contents[data.Contents.length - 1].Key; + return await listObjectsExec(storageCfg, output, params); + } else { return output; - } + } } -async function deleteObjectsHelp(aKeys) { +async function deleteObjectsHelp(storageCfg, aKeys) { //By default, the operation uses verbose mode in which the response includes the result of deletion of each key in your request. //In quiet mode the response includes only keys where the delete operation encountered an error. const input = { - Bucket: cfgBucketName, + Bucket: storageCfg.bucketName, Delete: { Objects: aKeys, Quiet: true } }; const command = new DeleteObjectsCommand(input); - return await client.send(command); + await getS3Client(storageCfg).send(command); } -exports.headObject = async function(strPath) { +async function headObject(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new HeadObjectCommand(input); - return await client.send(command); -}; -exports.getObject = async function(strPath) { + let output = await getS3Client(storageCfg).send(command); + return {ContentLength: output.ContentLength}; +} +async function getObject(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new GetObjectCommand(input); - const output = await client.send(command); + const output = await getS3Client(storageCfg).send(command); return await utils.stream2Buffer(output.Body); -}; -exports.createReadStream = async function(strPath) { +} +async function createReadStream(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new GetObjectCommand(input); - const output = await client.send(command); + const output = await getS3Client(storageCfg).send(command); return { contentLength: output.ContentLength, readStream: output.Body }; -}; -exports.putObject = async function(strPath, buffer, contentLength) { +} +async function putObject(storageCfg, strPath, buffer, contentLength) { //todo consider Expires const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath), + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath), Body: buffer, ContentLength: contentLength, ContentType: mime.getType(strPath) }; const command = new PutObjectCommand(input); - return await client.send(command); -}; -exports.uploadObject = async function(strPath, filePath) { + await getS3Client(storageCfg).send(command); +} +async function uploadObject(storageCfg, strPath, filePath) { const file = fs.createReadStream(filePath); //todo рассмотреть Expires const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath), + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath), Body: file, ContentType: mime.getType(strPath) }; const command = new PutObjectCommand(input); - return await client.send(command); -}; -exports.copyObject = function(sourceKey, destinationKey) { + await getS3Client(storageCfg).send(command); +} +async function copyObject(storageCfgSrc, storageCfgDst, sourceKey, destinationKey) { //todo source bucket const input = { - Bucket: cfgBucketName, - Key: getFilePath(destinationKey), - CopySource: `/${cfgBucketName}/${getFilePath(sourceKey)}` + Bucket: storageCfgDst.bucketName, + Key: getFilePath(storageCfgDst, destinationKey), + CopySource: `/${storageCfgSrc.bucketName}/${getFilePath(storageCfgSrc, sourceKey)}` }; const command = new CopyObjectCommand(input); - return client.send(command); -}; -exports.listObjects = async function(strPath) { - var params = {Bucket: cfgBucketName, Prefix: getFilePath(strPath)}; - var output = []; - return await listObjectsExec(output, params); -}; -exports.deleteObject = function(strPath) { + await getS3Client(storageCfgDst).send(command); +} +async function listObjects(storageCfg, strPath) { + let params = { + Bucket: storageCfg.bucketName, + Prefix: getFilePath(storageCfg, strPath) + }; + let output = []; + await listObjectsExec(storageCfg, output, params); + return output; +} +async function deleteObject(storageCfg, strPath) { const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath) + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath) }; const command = new DeleteObjectCommand(input); - return client.send(command); + await getS3Client(storageCfg).send(command); }; -exports.deleteObjects = function(strPaths) { - var aKeys = strPaths.map(function (currentValue) { - return {Key: getFilePath(currentValue)}; +async function deleteObjects(storageCfg, strPaths) { + let aKeys = strPaths.map(function (currentValue) { + return {Key: getFilePath(storageCfg, currentValue)}; }); - var deletePromises = []; - for (var i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) { - deletePromises.push(deleteObjectsHelp(aKeys.slice(i, i + MAX_DELETE_OBJECTS))); + for (let i = 0; i < aKeys.length; i += MAX_DELETE_OBJECTS) { + await deleteObjectsHelp(storageCfg, aKeys.slice(i, i + MAX_DELETE_OBJECTS)); } - return Promise.all(deletePromises); -}; -exports.getSignedUrl = async function (ctx, baseUrl, strPath, urlType, optFilename, opt_creationDate) { - var expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : cfgStorageUrlExpires) || 31536000; +} +async function deletePath(storageCfg, strPath) { + let list = await listObjects(storageCfg, strPath); + await deleteObjects(storageCfg, list); +} +async function getSignedUrlWrapper(ctx, storageCfg, baseUrl, strPath, urlType, optFilename, opt_creationDate) { + const storageUrlExpires = storageCfg.fs.urlExpires; + let expires = (commonDefines.c_oAscUrlTypes.Session === urlType ? cfgExpSessionAbsolute / 1000 : storageUrlExpires) || 31536000; // Signature version 4 presigned URLs must have an expiration date less than one week in the future expires = Math.min(expires, 604800); - var userFriendlyName = optFilename ? optFilename.replace(/\//g, "%2f") : path.basename(strPath); - var contentDisposition = utils.getContentDisposition(userFriendlyName, null, null); + let userFriendlyName = optFilename ? optFilename.replace(/\//g, "%2f") : path.basename(strPath); + let contentDisposition = utils.getContentDisposition(userFriendlyName, null, null); const input = { - Bucket: cfgBucketName, - Key: getFilePath(strPath), + Bucket: storageCfg.bucketName, + Key: getFilePath(storageCfg, strPath), ResponseContentDisposition: contentDisposition }; const command = new GetObjectCommand(input); //default Expires 900 seconds - var options = { + let options = { expiresIn: expires }; - return await getSignedUrl(client, command, options); + return await getSignedUrl(getS3Client(storageCfg), command, options); //extra query params cause SignatureDoesNotMatch //https://stackoverflow.com/questions/55503009/amazon-s3-signature-does-not-match-when-extra-query-params-ga-added-in-url // return utils.changeOnlyOfficeUrl(url, strPath, optFilename); +} + +function needServeStatic() { + return false; +} + +module.exports = { + headObject, + getObject, + createReadStream, + putObject, + uploadObject, + copyObject, + listObjects, + deleteObject, + deletePath, + getSignedUrl: getSignedUrlWrapper, + needServeStatic }; diff --git a/Common/sources/tenantManager.js b/Common/sources/tenantManager.js index 9ea972a6d..51eb5aeda 100644 --- a/Common/sources/tenantManager.js +++ b/Common/sources/tenantManager.js @@ -262,8 +262,13 @@ function getTenantLicense(ctx) { function getServerLicense(ctx) { return licenseInfo; } +let hasBaseDir = !!cfgTenantsBaseDir; function isMultitenantMode(ctx) { - return !!cfgTenantsBaseDir; + return hasBaseDir; +} +function setMultitenantMode(val) { + //for tests only!! + return hasBaseDir = val; } function isDefaultTenant(ctx) { return ctx.tenant === cfgTenantsDefaultTenant; @@ -304,11 +309,6 @@ async function readLicenseTenant(ctx, licenseFile, baseVerifiedLicense) { if (true === oLicense['developer']) { res.mode |= c_LM.Developer; } - // ToDo delete mode - if (oLicense.hasOwnProperty('mode')) { - res.mode |= ('developer' === oLicense['mode'] ? c_LM.Developer : ('trial' === oLicense['mode'] ? c_LM.Trial : c_LM.None)); - } - if (oLicense.hasOwnProperty('light')) { res.light = (true === oLicense['light'] || 'true' === oLicense['light'] || 'True' === oLicense['light']); // Someone who likes to put json string instead of bool } @@ -324,9 +324,6 @@ async function readLicenseTenant(ctx, licenseFile, baseVerifiedLicense) { if (oLicense.hasOwnProperty('advanced_api')) { res.advancedApi = !!oLicense['advanced_api']; } - if (oLicense.hasOwnProperty('process')) { - res.connections = Math.max(res.count, oLicense['process'] >> 0) * 75; - } if (oLicense.hasOwnProperty('connections')) { res.connections = oLicense['connections'] >> 0; } @@ -349,7 +346,7 @@ async function readLicenseTenant(ctx, licenseFile, baseVerifiedLicense) { const checkDate = ((res.mode & c_LM.Trial) || timeLimited) ? new Date() : licenseInfo.buildDate; //Calendar check of start_date allows to issue a license for old versions const checkStartDate = new Date(); - if (startDate <= checkStartDate && checkDate <= endDate && (!oLicense.hasOwnProperty('version') || 2 <= oLicense['version'])) { + if (startDate <= checkStartDate && checkDate <= endDate) { res.type = c_LR.Success; } else if (startDate > checkStartDate) { res.type = c_LR.NotBefore; @@ -413,4 +410,5 @@ exports.getTenantLicense = getTenantLicense; exports.getServerLicense = getServerLicense; exports.setDefLicense = setDefLicense; exports.isMultitenantMode = isMultitenantMode; +exports.setMultitenantMode = setMultitenantMode; exports.isDefaultTenant = isDefaultTenant; diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 3b02a5f9f..909d77333 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -35,6 +35,7 @@ //Fix EPROTO error in node 8.x at some web sites(https://github.com/nodejs/node/issues/21513) require("tls").DEFAULT_ECDH_CURVE = "auto"; +const { pipeline } = require('node:stream/promises'); var config = require('config'); var fs = require('fs'); var path = require('path'); @@ -51,7 +52,6 @@ const NodeCache = require( "node-cache" ); const ms = require('ms'); const constants = require('./constants'); const commonDefines = require('./commondefines'); -const logger = require('./logger'); const forwarded = require('forwarded'); const { RequestFilteringHttpAgent, RequestFilteringHttpsAgent } = require("request-filtering-agent"); const openpgp = require('openpgp'); @@ -85,6 +85,8 @@ const cfgPasswordDecrypt = config.get('openpgpjs.decrypt'); const cfgPasswordConfig = config.get('openpgpjs.config'); const cfgRequesFilteringAgent = config.get('services.CoAuthoring.request-filtering-agent'); const cfgStorageExternalHost = config.get('storage.externalHost'); +const cfgExternalRequestDirectIfIn = config.get('externalRequest.directIfIn'); +const cfgExternalRequestAction = config.get('externalRequest.action'); const dnscache = getDnsCache(cfgDnsCache); @@ -266,6 +268,53 @@ function raiseErrorObj(ro, error) { function isRedirectResponse(response) { return response && response.statusCode >= 300 && response.statusCode < 400 && response.caseless.has('location'); } + +function isAllowDirectRequest(ctx, uri, isInJwtToken) { + let res = false; + const tenExternalRequestDirectIfIn = ctx.getCfg('externalRequest.directIfIn', cfgExternalRequestDirectIfIn); + let allowList = tenExternalRequestDirectIfIn.allowList; + if (allowList.length > 0) { + let allowIndex = allowList.findIndex((allowPrefix) => { + return uri.startsWith(allowPrefix); + }, uri); + res = -1 !== allowIndex; + ctx.logger.debug("isAllowDirectRequest check allow list res=%s", res); + } else if (tenExternalRequestDirectIfIn.jwtToken) { + res = isInJwtToken; + ctx.logger.debug("isAllowDirectRequest url in jwt token res=%s", res); + } + return res; +} +function addExternalRequestOptions(ctx, uri, isInJwtToken, options) { + let res = false; + const tenExternalRequestAction = ctx.getCfg('externalRequest.action', cfgExternalRequestAction); + const tenRequesFilteringAgent = ctx.getCfg('services.CoAuthoring.request-filtering-agent', cfgRequesFilteringAgent); + if (isAllowDirectRequest(ctx, uri, isInJwtToken)) { + res = true; + } else if (tenExternalRequestAction.allow) { + res = true; + if (tenExternalRequestAction.blockPrivateIP) { + const agentOptions = Object.assign({}, https.globalAgent.options, tenRequesFilteringAgent); + options.agent = getRequestFilterAgent(uri, agentOptions); + } + if (tenExternalRequestAction.proxyUrl) { + options.proxy = tenExternalRequestAction.proxyUrl; + } + if (tenExternalRequestAction.proxyUser?.username) { + let user = tenExternalRequestAction.proxyUser.username; + let pass = tenExternalRequestAction.proxyUser.password; + options.headers = {'proxy-authorization': `${user}:${pass}`}; + } + if (tenExternalRequestAction.proxyHeaders) { + if (!options.headers) { + options.headers = {}; + } + Object.assign(options.headers, tenExternalRequestAction.proxyHeaders); + } + } + return res; +} + function downloadUrlPromise(ctx, uri, optTimeout, optLimit, opt_Authorization, opt_filterPrivate, opt_headers, opt_streamWriter) { //todo replace deprecated request module const tenTenantRequestDefaults = ctx.getCfg('services.CoAuthoring.requestDefaults', cfgRequestDefaults); @@ -298,21 +347,22 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A const tenTenantRequestDefaults = ctx.getCfg('services.CoAuthoring.requestDefaults', cfgRequestDefaults); const tenTokenOutboxHeader = ctx.getCfg('services.CoAuthoring.token.outbox.header', cfgTokenOutboxHeader); const tenTokenOutboxPrefix = ctx.getCfg('services.CoAuthoring.token.outbox.prefix', cfgTokenOutboxPrefix); - const tenRequesFilteringAgent = ctx.getCfg('services.CoAuthoring.request-filtering-agent', cfgRequesFilteringAgent); //IRI to URI uri = URI.serialize(URI.parse(uri)); var urlParsed = url.parse(uri); let sizeLimit = optLimit || Number.MAX_VALUE; - let bufferLength = 0; + let bufferLength = 0, timeoutId; let hash = crypto.createHash('sha256'); //if you expect binary data, you should set encoding: null let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); let options = config.util.extendDeep({}, tenTenantRequestDefaults); Object.assign(options, {uri: urlParsed, encoding: null, timeout: connectionAndInactivity, followRedirect: false}); - if (opt_filterPrivate) { - const agentOptions = Object.assign({}, https.globalAgent.options, tenRequesFilteringAgent); - options.agent = getRequestFilterAgent(uri, agentOptions); - } else { + if (!addExternalRequestOptions(ctx, uri, opt_filterPrivate, options)) { + reject(new Error('Block external request. See externalRequest config options')); + return; + } + + if (!options.agent) { //baseRequest creates new agent(win-ca injects in globalAgent) options.agentOptions = https.globalAgent.options; } @@ -326,6 +376,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A Object.assign(options.headers, opt_headers); } let fError = function(err) { + clearTimeout(timeoutId); reject(err); } if (!opt_streamWriter) { @@ -337,6 +388,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A } executed = true; if (err) { + clearTimeout(timeoutId); reject(err); } else { var contentLength = response.caseless.get('content-length'); @@ -344,6 +396,7 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A ctx.logger.warn('downloadUrlPromise body size mismatch: uri=%s; content-length=%s; body.length=%d', uri, contentLength, body.length); } let sha256 = hash.digest('hex'); + clearTimeout(timeoutId); resolve({response: response, body: body, sha256: sha256}); } }; @@ -365,13 +418,21 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A error.response = response; if (opt_streamWriter && !isRedirectResponse(response)) { this.off('error', fError); - resolve(pipeStreams(this, opt_streamWriter, true)); + pipeline(this, opt_streamWriter) + .then(resolve, reject) + .finally(() => { + clearTimeout(timeoutId); + }); } else { raiseErrorObj(this, error); } } else if (opt_streamWriter) { this.off('error', fError); - resolve(pipeStreams(this, opt_streamWriter, true)); + pipeline(this, opt_streamWriter) + .then(resolve, reject) + .finally(() => { + clearTimeout(timeoutId); + }); } }; let fData = function(chunk) { @@ -387,13 +448,13 @@ function downloadUrlPromiseWithoutRedirect(ctx, uri, optTimeout, optLimit, opt_A .on('data', fData) .on('error', fError); if (optTimeout && optTimeout.wholeCycle) { - setTimeout(function() { + timeoutId = setTimeout(function() { raiseError(ro, 'ETIMEDOUT', 'Error: whole request cycle timeout'); }, ms(optTimeout.wholeCycle)); } }); } -function postRequestPromise(ctx, uri, postData, postDataStream, postDataSize, optTimeout, opt_Authorization, opt_header) { +function postRequestPromise(ctx, uri, postData, postDataStream, postDataSize, optTimeout, opt_Authorization, opt_headers) { return new Promise(function(resolve, reject) { const tenTenantRequestDefaults = ctx.getCfg('services.CoAuthoring.requestDefaults', cfgRequestDefaults); const tenTokenOutboxHeader = ctx.getCfg('services.CoAuthoring.token.outbox.header', cfgTokenOutboxHeader); @@ -401,29 +462,32 @@ function postRequestPromise(ctx, uri, postData, postDataStream, postDataSize, op //IRI to URI uri = URI.serialize(URI.parse(uri)); var urlParsed = url.parse(uri); - var headers = {'Content-Type': 'application/json'}; + let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); + let options = config.util.extendDeep({}, tenTenantRequestDefaults); + Object.assign(options, {uri: urlParsed, encoding: 'utf8', timeout: connectionAndInactivity}); + //baseRequest creates new agent(win-ca injects in globalAgent) + options.agentOptions = https.globalAgent.options; + if (postData) { + options.body = postData; + } + if (!options.headers) { + options.headers = {}; + } if (opt_Authorization) { //todo ctx.getCfg - headers[tenTokenOutboxHeader] = tenTokenOutboxPrefix + opt_Authorization; + options.headers[tenTokenOutboxHeader] = tenTokenOutboxPrefix + opt_Authorization; + } + if (opt_headers) { + Object.assign(options.headers, opt_headers); } - headers = opt_header || headers; if (undefined !== postDataSize) { //If no Content-Length is set, data will automatically be encoded in HTTP Chunked transfer encoding, //so that server knows when the data ends. The Transfer-Encoding: chunked header is added. //https://nodejs.org/api/http.html#requestwritechunk-encoding-callback //issue with Transfer-Encoding: chunked wopi and sharepoint 2019 //https://community.alteryx.com/t5/Dev-Space/Download-Tool-amp-Microsoft-SharePoint-Chunked-Request-Error/td-p/735824 - headers['Content-Length'] = postDataSize; + options.headers['Content-Length'] = postDataSize; } - let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); - let options = config.util.extendDeep({}, tenTenantRequestDefaults); - Object.assign(options, {uri: urlParsed, encoding: 'utf8', headers: headers, timeout: connectionAndInactivity}); - //baseRequest creates new agent(win-ca injects in globalAgent) - options.agentOptions = https.globalAgent.options; - if (postData) { - options.body = postData; - } - let executed = false; let ro = request.post(options, function(err, response, body) { if (executed) { @@ -483,6 +547,8 @@ exports.mapAscServerErrorToOldError = function(error) { res = -7; break; case constants.CONVERT_LIMITS : + res = -10; + break; case constants.CONVERT_NEED_PARAMS : case constants.CONVERT_LIBREOFFICE : case constants.CONVERT_CORRUPTED : @@ -731,14 +797,22 @@ function getDomainByRequest(ctx, req) { } exports.getDomainByConnection = getDomainByConnection; exports.getDomainByRequest = getDomainByRequest; -function getShardByConnection(ctx, conn) { - return conn?.handshake?.query?.[constants.SHARED_KEY_NAME]; +function getShardKeyByConnection(ctx, conn) { + return conn?.handshake?.query?.[constants.SHARD_KEY_API_NAME]; +} +function getWopiSrcByConnection(ctx, conn) { + return conn?.handshake?.query?.[constants.SHARD_KEY_WOPI_NAME]; } function getShardKeyByRequest(ctx, req) { - return req.query[constants.SHARED_KEY_NAME]; + return req.query?.[constants.SHARD_KEY_API_NAME]; } -exports.getShardByConnection = getShardByConnection; +function getWopiSrcByRequest(ctx, req) { + return req.query?.[constants.SHARD_KEY_WOPI_NAME]; +} +exports.getShardKeyByConnection = getShardKeyByConnection; +exports.getWopiSrcByConnection = getWopiSrcByConnection; exports.getShardKeyByRequest = getShardKeyByRequest; +exports.getWopiSrcByRequest = getWopiSrcByRequest; function stream2Buffer(stream) { return new Promise(function(resolve, reject) { if (!stream.readable) { @@ -1036,8 +1110,9 @@ exports.convertLicenseInfoToServerParams = function(licenseInfo) { license.buildNumber = commonDefines.buildNumber; return license; }; -exports.checkBaseUrl = function(ctx, baseUrl) { - const tenStorageExternalHost = ctx.getCfg('storage.externalHost', cfgStorageExternalHost); +exports.checkBaseUrl = function(ctx, baseUrl, opt_storageCfg) { + let storageExternalHost = opt_storageCfg ? opt_storageCfg.externalHost : cfgStorageExternalHost + const tenStorageExternalHost = ctx.getCfg('storage.externalHost', storageExternalHost); return tenStorageExternalHost ? tenStorageExternalHost : baseUrl; }; exports.resolvePath = function(object, path, defaultValue) { @@ -1111,3 +1186,32 @@ exports.checksumFile = function(hashName, path) { stream.on('end', () => resolve(hash.digest('hex'))); }); }; + +function isObject(item) { + return (item && typeof item === 'object' && !Array.isArray(item)); +} + +function deepMergeObjects(target, ...sources) { + if (!sources.length) { + return target; + } + + const source = sources.shift(); + if (isObject(target) && isObject(source)) { + for (const key in source) { + if (isObject(source[key])) { + if (!target[key]) { + Object.assign(target, { [key]: {} }); + } + + deepMergeObjects(target[key], source[key]); + } else { + Object.assign(target, { [key]: source[key] }); + } + } + } + + return deepMergeObjects(target, ...sources); +} +exports.isObject = isObject; +exports.deepMergeObjects = deepMergeObjects; \ No newline at end of file diff --git a/DocService/npm-shrinkwrap.json b/DocService/npm-shrinkwrap.json index 9c818d38a..34ee90889 100644 --- a/DocService/npm-shrinkwrap.json +++ b/DocService/npm-shrinkwrap.json @@ -843,9 +843,9 @@ } }, "async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" }, "asynckit": { "version": "0.4.0", @@ -1148,9 +1148,9 @@ "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" }, "cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" }, "cookie-signature": { "version": "1.0.6", @@ -1243,9 +1243,9 @@ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, "denque": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.0.1.tgz", - "integrity": "sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" }, "depd": { "version": "1.1.2", @@ -1332,9 +1332,9 @@ "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "ejs": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.8.tgz", - "integrity": "sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==", + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", "requires": { "jake": "^10.8.5" } @@ -1519,16 +1519,16 @@ "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==" }, "express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -1556,15 +1556,30 @@ "vary": "~1.1.2" }, "dependencies": { - "accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "body-parser": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "requires": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" } }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1583,11 +1598,6 @@ "mime-db": "1.52.0" } }, - "negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" - }, "on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -1596,6 +1606,17 @@ "ee-first": "1.1.1" } }, + "raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "requires": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2117,7 +2138,7 @@ "is-property": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", - "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==" }, "is-regex": { "version": "1.1.4", @@ -2191,14 +2212,14 @@ } }, "jake": { - "version": "10.8.5", - "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz", - "integrity": "sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==", + "version": "10.8.7", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz", + "integrity": "sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==", "requires": { "async": "^3.2.3", "chalk": "^4.0.2", - "filelist": "^1.0.1", - "minimatch": "^3.0.4" + "filelist": "^1.0.4", + "minimatch": "^3.1.2" } }, "jimp": { @@ -2347,9 +2368,9 @@ "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, "long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" }, "lru-cache": { "version": "6.0.0", @@ -2531,16 +2552,16 @@ } }, "mysql2": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-2.3.3.tgz", - "integrity": "sha512-wxJUev6LgMSgACDkb/InIFxDprRa6T95+VEoR+xPvtngtccNH2dGjEB/fVZ8yg1gWv1510c9CvXuJHi5zUm0ZA==", + "version": "3.9.8", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.8.tgz", + "integrity": "sha512-+5JKNjPuks1FNMoy9TYpl77f+5frbTklz7eb3XDwbpsERRLEeXiW2PDEkakYF50UuKU2qwfGnyXpKYvukv8mGA==", "requires": { - "denque": "^2.0.1", + "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.6.3", - "long": "^4.0.0", - "lru-cache": "^6.0.0", - "named-placeholders": "^1.1.2", + "long": "^5.2.1", + "lru-cache": "^8.0.0", + "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" }, @@ -2553,34 +2574,25 @@ "safer-buffer": ">= 2.1.2 < 3.0.0" } }, - "sqlstring": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", - "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==" + "lru-cache": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", + "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==" } } }, "named-placeholders": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.2.tgz", - "integrity": "sha512-wiFWqxoLL3PGVReSZpjLVxyJ1bRqe+KKJVbr4hGs1KWfTZTQyezHFBbuKj9hsizHyGV2ne7EMjHdxEGAybD5SA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz", + "integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==", "requires": { - "lru-cache": "^4.1.3" + "lru-cache": "^7.14.1" }, "dependencies": { "lru-cache": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", - "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "requires": { - "pseudomap": "^1.0.2", - "yallist": "^2.1.2" - } - }, - "yallist": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==" } } }, @@ -2716,19 +2728,26 @@ "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, "pg": { - "version": "8.8.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.8.0.tgz", - "integrity": "sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==", + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", "requires": { "buffer-writer": "2.0.0", "packet-reader": "1.0.0", - "pg-connection-string": "^2.5.0", - "pg-pool": "^3.5.2", - "pg-protocol": "^1.5.0", + "pg-cloudflare": "^1.1.1", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", "pg-types": "^2.1.0", "pgpass": "1.x" } }, + "pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, "pg-connection-string": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", @@ -2829,11 +2848,6 @@ "ipaddr.js": "1.9.1" } }, - "pseudomap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" - }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", @@ -2877,7 +2891,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "requires": { "core-util-is": "~1.0.0", @@ -3079,7 +3093,7 @@ "seq-queue": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", - "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4=" + "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==" }, "serve-static": { "version": "1.15.0", @@ -3208,6 +3222,11 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" }, + "sqlstring": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", + "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==" + }, "standard-as-callback": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", diff --git a/DocService/package.json b/DocService/package.json index 92fad295d..f42a1f6e5 100644 --- a/DocService/package.json +++ b/DocService/package.json @@ -20,9 +20,9 @@ "cron": "1.5.0", "deep-equal": "1.0.1", "dmdb": "1.0.14280", - "ejs": "3.1.8", + "ejs": "3.1.10", "exif-parser": "0.1.12", - "express": "4.18.2", + "express": "4.19.2", "fakeredis": "2.0.0", "ioredis": "5.3.1", "jimp": "0.22.10", @@ -35,9 +35,9 @@ "multer": "1.4.3", "multi-integer-range": "4.0.7", "multiparty": "4.2.1", - "mysql2": "2.3.3", + "mysql2": "3.9.8", "oracledb": "6.3.0", - "pg": "8.8.0", + "pg": "8.11.3", "redis": "4.6.11", "retry": "0.12.0", "socket.io": "4.7.1", diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 8eedeb3d1..548754064 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -90,7 +90,7 @@ const utils = require('./../../Common/sources/utils'); const commonDefines = require('./../../Common/sources/commondefines'); const statsDClient = require('./../../Common/sources/statsdclient'); const config = require('config'); -const sqlBase = require('./baseConnector'); +const sqlBase = require('./databaseConnectors/baseConnector'); const canvasService = require('./canvasservice'); const converterService = require('./converterservice'); const taskResult = require('./taskresult'); @@ -102,7 +102,11 @@ const queueService = require('./../../Common/sources/taskqueueRabbitMQ'); const operationContext = require('./../../Common/sources/operationContext'); const tenantManager = require('./../../Common/sources/tenantManager'); -const editorDataStorage = require('./' + config.get('services.CoAuthoring.server.editorDataStorage')); +const cfgEditorDataStorage = config.get('services.CoAuthoring.server.editorDataStorage'); +const cfgEditorStatStorage = config.get('services.CoAuthoring.server.editorStatStorage'); +const editorDataStorage = require('./' + cfgEditorDataStorage); +const editorStatStorage = require('./' + (cfgEditorStatStorage || cfgEditorDataStorage)); +const utilsDocService = require("./utilsDocService"); const cfgEditSingleton = config.get('services.CoAuthoring.server.edit_singleton'); const cfgEditor = config.get('services.CoAuthoring.editor'); @@ -152,7 +156,9 @@ const EditorTypes = { }; const defaultHttpPort = 80, defaultHttpsPort = 443; // Default ports (for http and https) -const editorData = new editorDataStorage(); +//todo remove editorDataStorage constructor usage after 8.1 +const editorData = editorDataStorage.EditorData ? new editorDataStorage.EditorData() : new editorDataStorage(); +const editorStat = editorStatStorage.EditorStat ? new editorStatStorage.EditorStat() : new editorDataStorage(); const clientStatsD = statsDClient.getClient(); let connections = []; // Active connections let lockDocumentsTimerId = {};//to drop connection that can't unlockDocument @@ -435,30 +441,30 @@ function updatePresenceCounters(ctx, conn, val) { //yield ctx.initTenantCache(); //no need.only global config } if (utils.isLiveViewer(conn)) { - yield editorData.incrLiveViewerConnectionsCountByShard(ctx, SHARD_ID, val); + yield editorStat.incrLiveViewerConnectionsCountByShard(ctx, SHARD_ID, val); if (aggregationCtx) { - yield editorData.incrLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); + yield editorStat.incrLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); } if (clientStatsD) { - let countLiveView = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + let countLiveView = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.liveview', countLiveView); } } else if (conn.isCloseCoAuthoring || (conn.user && conn.user.view)) { - yield editorData.incrViewerConnectionsCountByShard(ctx, SHARD_ID, val); + yield editorStat.incrViewerConnectionsCountByShard(ctx, SHARD_ID, val); if (aggregationCtx) { - yield editorData.incrViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); + yield editorStat.incrViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, val); } if (clientStatsD) { - let countView = yield editorData.getViewerConnectionsCount(ctx, connections); + let countView = yield editorStat.getViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.view', countView); } } else { - yield editorData.incrEditorConnectionsCountByShard(ctx, SHARD_ID, val); + yield editorStat.incrEditorConnectionsCountByShard(ctx, SHARD_ID, val); if (aggregationCtx) { - yield editorData.incrEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, val); + yield editorStat.incrEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, val); } if (clientStatsD) { - let countEditors = yield editorData.getEditorConnectionsCount(ctx, connections); + let countEditors = yield editorStat.getEditorConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.edit', countEditors); } } @@ -489,7 +495,7 @@ function removePresence(ctx, conn) { let changeConnectionInfo = co.wrap(function*(ctx, conn, cmd) { if (!conn.denyChangeName && conn.user) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.changeConnecitonInfo, ctx: ctx, docId: conn.docId, useridoriginal: conn.user.idOriginal, cmd: cmd}); + yield publish(ctx, {type: commonDefines.c_oPublishType.changeConnecitonInfo, ctx: ctx, docId: conn.docId, useridoriginal: conn.user.idOriginal, cmd: cmd}); return true; } return false; @@ -625,11 +631,11 @@ function* updateEditUsers(ctx, licenseInfo, userId, anonym, isLiveViewer) { licenseInfo.usersExpire - 1; let period = utils.getLicensePeriod(licenseInfo.startDate, now); if (isLiveViewer) { - yield editorData.addPresenceUniqueViewUser(ctx, userId, expireAt, {anonym: anonym}); - yield editorData.addPresenceUniqueViewUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); + yield editorStat.addPresenceUniqueViewUser(ctx, userId, expireAt, {anonym: anonym}); + yield editorStat.addPresenceUniqueViewUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); } else { - yield editorData.addPresenceUniqueUser(ctx, userId, expireAt, {anonym: anonym}); - yield editorData.addPresenceUniqueUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); + yield editorStat.addPresenceUniqueUser(ctx, userId, expireAt, {anonym: anonym}); + yield editorStat.addPresenceUniqueUsersOfMonth(ctx, userId, period, {anonym: anonym, firstOpenDate: now.toISOString()}); } } function* getEditorsCount(ctx, docId, opt_hvals) { @@ -666,12 +672,12 @@ function* isUserReconnect(ctx, docId, userId, connectionId) { } let pubsubOnMessage = null;//todo move function -function* publish(ctx, data, optDocId, optUserId, opt_pubsub) { +async function publish(ctx, data, optDocId, optUserId, opt_pubsub) { var needPublish = true; let hvals; if (optDocId && optUserId) { needPublish = false; - hvals = yield editorData.getPresence(ctx, optDocId, connections); + hvals = await editorData.getPresence(ctx, optDocId, connections); for (var i = 0; i < hvals.length; ++i) { var elem = JSON.parse(hvals[i]); if (optUserId != elem.id) { @@ -689,7 +695,7 @@ function* publish(ctx, data, optDocId, optUserId, opt_pubsub) { //todo send connections from getLocalConnectionCount to pubsubOnMessage pubsubOnMessage(msg); } else if(realPubsub) { - yield realPubsub.publish(msg); + await realPubsub.publish(msg); } } return needPublish; @@ -741,7 +747,8 @@ async function sendServerRequest(ctx, uri, dataObject, opt_checkAndFixAuthorizat } dataObject.setToken(bodyToken); } - let postRes = await utils.postRequestPromise(ctx, uri, JSON.stringify(dataObject), undefined, undefined, tenCallbackRequestTimeout, auth); + let headers = {'Content-Type': 'application/json'}; + let postRes = await utils.postRequestPromise(ctx, uri, JSON.stringify(dataObject), undefined, undefined, tenCallbackRequestTimeout, auth, headers); ctx.logger.debug('postData response: data = %s', postRes.body); return postRes.body; } @@ -838,15 +845,16 @@ function* setForceSave(ctx, docId, forceSave, cmd, success, url) { if (commonDefines.c_oAscForceSaveTypes.Command !== forceSaveType) { let data = {type: forceSaveType, time: forceSave.getTime(), success: success}; if(commonDefines.c_oAscForceSaveTypes.Form === forceSaveType || commonDefines.c_oAscForceSaveTypes.Internal === forceSaveType) { - data = {code: commonDefines.c_oAscServerCommandErrors.NoError, time: null, inProgress: false}; + let code = success ? commonDefines.c_oAscServerCommandErrors.NoError : commonDefines.c_oAscServerCommandErrors.UnknownError; + data = {code: code, time: null, inProgress: false}; if (commonDefines.c_oAscForceSaveTypes.Internal === forceSaveType) { data.url = url; } let userId = cmd.getUserConnectionId(); docId = cmd.getUserConnectionDocId() || docId; - yield* publish(ctx, {type: commonDefines.c_oPublishType.rpc, ctx, docId, userId, data, responseKey: cmd.getResponseKey()}); + yield publish(ctx, {type: commonDefines.c_oPublishType.rpc, ctx, docId, userId, data, responseKey: cmd.getResponseKey()}); } else { - yield* publish(ctx, {type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, data: data}, cmd.getUserConnectionId()); + yield publish(ctx, {type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, data: data}, cmd.getUserConnectionId()); } } } @@ -910,7 +918,9 @@ async function applyForceSaveCache(ctx, docId, forceSave, type, opt_userConnecti } return res; } -async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, opt_queue, opt_pubsub, opt_conn, opt_initShardKey) { +async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_userId, opt_userConnectionId, + opt_userConnectionDocId, opt_userIndex, opt_responseKey, opt_baseUrl, + opt_queue, opt_pubsub, opt_conn, opt_initShardKey, opt_jsonParams) { ctx.logger.debug('startForceSave start'); let res = {code: commonDefines.c_oAscServerCommandErrors.NoError, time: null, inProgress: false}; let startedForceSave; @@ -928,7 +938,7 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ newChangesLastDate.setMilliseconds(0);//remove milliseconds avoid issues with MySQL datetime rounding let newChangesLastTime = newChangesLastDate.getTime(); let baseUrl = utils.getBaseUrlByConnection(ctx, opt_conn); - let changeInfo = getExternalChangeInfo(opt_conn.user, newChangesLastTime); + let changeInfo = getExternalChangeInfo(opt_conn.user, newChangesLastTime, opt_conn.lang); await editorData.setForceSave(ctx, docId, newChangesLastTime, 0, baseUrl, changeInfo, null); forceSave = await editorData.getForceSave(ctx, docId); } @@ -957,13 +967,6 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ forceSave.setAuthorUserId(opt_userId); forceSave.setAuthorUserIndex(opt_userIndex); - if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { - await co(publish(ctx, { - type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, - data: {type: type, time: forceSave.getTime(), start: true} - }, undefined, undefined, opt_pubsub)); - } - let priority; let expiration; if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { @@ -975,9 +978,15 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ //start new convert let status = await converterService.convertFromChanges(ctx, docId, baseUrl, forceSave, startedForceSave.changeInfo, opt_userdata, opt_formdata, opt_userConnectionId, opt_userConnectionDocId, opt_responseKey, priority, expiration, - opt_queue, undefined, opt_initShardKey); + opt_queue, undefined, opt_initShardKey, opt_jsonParams); if (constants.NO_ERROR === status.err) { res.time = forceSave.getTime(); + if (commonDefines.c_oAscForceSaveTypes.Timeout === type) { + await publish(ctx, { + type: commonDefines.c_oPublishType.forceSave, ctx: ctx, docId: docId, + data: {type: type, time: forceSave.getTime(), start: true} + }, undefined, undefined, opt_pubsub); + } } else { res.code = commonDefines.c_oAscServerCommandErrors.UnknownError; } @@ -986,8 +995,8 @@ async function startForceSave(ctx, docId, type, opt_userdata, opt_formdata, opt_ ctx.logger.debug('startForceSave end'); return res; } -function getExternalChangeInfo(user, date) { - return {user_id: user.id, user_id_original: user.idOriginal, user_name: user.username, change_date: date}; +function getExternalChangeInfo(user, date, lang) { + return {user_id: user.id, user_id_original: user.idOriginal, user_name: user.username, lang, change_date: date}; } let resetForceSaveAfterChanges = co.wrap(function*(ctx, docId, newChangesLastTime, puckerIndex, baseUrl, changeInfo) { const tenForceSaveEnable = ctx.getCfg('services.CoAuthoring.autoAssembly.enable', cfgForceSaveEnable); @@ -1031,9 +1040,11 @@ function* startRPC(ctx, conn, responseKey, data) { case 'sendForm': { let forceSaveRes; if (conn.user) { + //isPrint - to remove forms + let jsonParams = {'documentLayout': {'isPrint': true}}; forceSaveRes = yield startForceSave(ctx, docId, commonDefines.c_oAscForceSaveTypes.Form, undefined, data.formdata, conn.user.idOriginal, conn.user.id, undefined, conn.user.indexUser, - responseKey, undefined, undefined, undefined, conn); + responseKey, undefined, undefined, undefined, conn, undefined, jsonParams); } if (!forceSaveRes || commonDefines.c_oAscServerCommandErrors.NoError !== forceSaveRes.code || forceSaveRes.inProgress) { sendDataRpc(ctx, conn, responseKey, forceSaveRes); @@ -1199,7 +1210,7 @@ let onReplySendStatusDocument = co.wrap(function*(ctx, docId, replyData) { var oData = parseReplyData(ctx, replyData); if (!(oData && commonDefines.c_oAscServerCommandErrors.NoError == oData.error)) { // Error subscribing to callback, send warning - yield* publish(ctx, {type: commonDefines.c_oPublishType.warning, ctx: ctx, docId: docId, description: 'Error on save server subscription!'}); + yield publish(ctx, {type: commonDefines.c_oPublishType.warning, ctx: ctx, docId: docId, description: 'Error on save server subscription!'}); } }); function* publishCloseUsersConnection(ctx, docId, users, isOriginalId, code, description) { @@ -1208,7 +1219,7 @@ function* publishCloseUsersConnection(ctx, docId, users, isOriginalId, code, des map[val] = 1; return map; }, {}); - yield* publish(ctx, { + yield publish(ctx, { type: commonDefines.c_oPublishType.closeConnection, ctx: ctx, docId: docId, usersMap: usersMap, isOriginalId: isOriginalId, code: code, description: description }); @@ -1229,7 +1240,7 @@ function closeUsersConnection(ctx, docId, usersMap, isOriginalId, code, descript } function* dropUsersFromDocument(ctx, docId, users) { if (Array.isArray(users)) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: users, description: ''}); + yield publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: users, description: ''}); } } @@ -1289,10 +1300,10 @@ let unlockWopiDoc = co.wrap(function*(ctx, docId, opt_userIndex) { //wopi unlock var getRes = yield getCallback(ctx, docId, opt_userIndex); if (getRes && getRes.wopiParams && getRes.wopiParams.userAuth && 'view' !== getRes.wopiParams.userAuth.mode) { - yield wopiClient.unlock(ctx, getRes.wopiParams); + let unlockRes = yield wopiClient.unlock(ctx, getRes.wopiParams); let unlockInfo = wopiClient.getWopiUnlockMarker(getRes.wopiParams); - if (unlockInfo) { - yield canvasService.commandOpenStartPromise(ctx, docId, undefined, true, unlockInfo); + if (unlockInfo && unlockRes) { + yield canvasService.commandOpenStartPromise(ctx, docId, undefined, unlockInfo); } } }); @@ -1319,7 +1330,7 @@ function* cleanDocumentOnExitNoChanges(ctx, docId, opt_userId, opt_userIndex, op yield* cleanDocumentOnExit(ctx, docId, false, opt_userIndex); } -function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_noDelay, opt_initShardKey) { +function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_noDelay, opt_initShardKey) { return co(function*(){ const tenAscSaveTimeOutDelay = ctx.getCfg('services.CoAuthoring.server.savetimeoutdelay', cfgAscSaveTimeOutDelay); @@ -1337,7 +1348,7 @@ function createSaveTimer(ctx, docId, opt_userId, opt_userIndex, opt_queue, opt_n } while (true) { if (!sqlBase.isLockCriticalSection(docId)) { - canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_queue, opt_initShardKey); + yield canvasService.saveFromChanges(ctx, docId, updateTask.statusInfo, null, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_initShardKey); break; } yield utils.sleep(c_oAscLockTimeOutDelay); @@ -1474,6 +1485,7 @@ function getOpenFormatByEditor(editorType) { exports.c_oAscServerStatus = c_oAscServerStatus; exports.editorData = editorData; +exports.editorStat = editorStat; exports.sendData = sendData; exports.modifyConnectionForPassword = modifyConnectionForPassword; exports.parseUrl = parseUrl; @@ -1566,167 +1578,172 @@ exports.install = function(server, callbackFunction) { }); }); - io.on('connection', function(conn) { - if (!conn) { - operationContext.global.logger.error("null == conn"); - return; - } + io.on('connection', async function(conn) { let ctx = new operationContext.Context(); - ctx.initFromConnection(conn); - //todo - //yield ctx.initTenantCache(); - if (getIsShutdown()) { - sendFileError(ctx, conn, 'Server shutdow'); - return; - } - conn.baseUrl = utils.getBaseUrlByConnection(ctx, conn); - conn.sessionIsSendWarning = false; - conn.sessionTimeConnect = conn.sessionTimeLastAction = new Date().getTime(); + try { + if (!conn) { + operationContext.global.logger.error("null == conn"); + return; + } + ctx.initFromConnection(conn); + await ctx.initTenantCache(); + if (getIsShutdown()) { + sendFileError(ctx, conn, 'Server shutdow'); + return; + } + conn.baseUrl = utils.getBaseUrlByConnection(ctx, conn); + conn.sessionIsSendWarning = false; + conn.sessionTimeConnect = conn.sessionTimeLastAction = new Date().getTime(); - conn.on('message', function(data) { - return co(function* () { - var docId = 'null'; - let ctx = new operationContext.Context(); - try { - ctx.initFromConnection(conn); - yield ctx.initTenantCache(); - const tenErrorFiles = ctx.getCfg('FileConverter.converter.errorfiles', cfgErrorFiles); + conn.on('message', function(data) { + return co(function* () { + var docId = 'null'; + let ctx = new operationContext.Context(); + try { + ctx.initFromConnection(conn); + yield ctx.initTenantCache(); + const tenErrorFiles = ctx.getCfg('FileConverter.converter.errorfiles', cfgErrorFiles); - var startDate = null; - if(clientStatsD) { - startDate = new Date(); - } + var startDate = null; + if(clientStatsD) { + startDate = new Date(); + } - docId = conn.docId; - ctx.logger.info('data.type = %s', data.type); - if(getIsShutdown()) - { - ctx.logger.debug('Server shutdown receive data'); - return; - } - if (conn.isCiriticalError && ('message' == data.type || 'getLock' == data.type || 'saveChanges' == data.type || - 'isSaveLock' == data.type)) { - ctx.logger.warn("conn.isCiriticalError send command: type = %s", data.type); - sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); - conn.disconnect(true); - return; - } - if ((conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && - ('getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { - ctx.logger.warn("conn.user.view||isCloseCoAuthoring access deny: type = %s", data.type); - sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); - conn.disconnect(true); - return; - } - yield encryptPasswordParams(ctx, data); - switch (data.type) { - case 'auth' : - try { - yield* auth(ctx, conn, data); - } catch(err){ - ctx.logger.error('auth error: %s', err.stack); + docId = conn.docId; + ctx.logger.info('data.type = %s', data.type); + if(getIsShutdown()) + { + ctx.logger.debug('Server shutdown receive data'); + return; + } + if (conn.isCiriticalError && ('message' == data.type || 'getLock' == data.type || 'saveChanges' == data.type || + 'isSaveLock' == data.type)) { + ctx.logger.warn("conn.isCiriticalError send command: type = %s", data.type); sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); conn.disconnect(true); return; } - break; - case 'message' : - yield* onMessage(ctx, conn, data); - break; - case 'cursor' : - yield* onCursor(ctx, conn, data); - break; - case 'getLock' : - yield* getLock(ctx, conn, data, false); - break; - case 'saveChanges' : - yield* saveChanges(ctx, conn, data); - break; - case 'isSaveLock' : - yield* isSaveLock(ctx, conn, data); - break; - case 'unSaveLock' : - yield* unSaveLock(ctx, conn, -1, -1, -1); - break; // The index is sent -1, because this is an emergency withdrawal without saving - case 'getMessages' : - yield* getMessages(ctx, conn, data); - break; - case 'unLockDocument' : - yield* checkEndAuthLock(ctx, data.unlock, data.isSave, docId, conn.user.id, data.releaseLocks, data.deleteIndex, conn); - break; - case 'close': - yield* closeDocument(ctx, conn); - break; - case 'versionHistory' : { - let cmd = new commonDefines.InputCommand(data.cmd); - yield* versionHistory(ctx, conn, cmd); - break; - } - case 'openDocument' : { - var cmd = new commonDefines.InputCommand(data.message); - cmd.fillFromConnection(conn); - yield canvasService.openDocument(ctx, conn, cmd); - break; - } - case 'clientLog': - let level = data.level?.toLowerCase(); - if("trace" === level || "debug" === level || "info" === level || "warn" === level || "error" === level || "fatal" === level) { - ctx.logger[level]("clientLog: %s", data.msg); + if ((conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && + ('getLock' == data.type || 'saveChanges' == data.type || 'isSaveLock' == data.type)) { + ctx.logger.warn("conn.user.view||isCloseCoAuthoring access deny: type = %s", data.type); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); + return; } - if ("error" === level && tenErrorFiles && docId) { - let destDir = 'browser/' + docId; - yield storage.copyPath(ctx, docId, destDir, undefined, tenErrorFiles); - yield* saveErrorChanges(ctx, docId, destDir); + yield encryptPasswordParams(ctx, data); + switch (data.type) { + case 'auth' : + try { + yield* auth(ctx, conn, data); + } catch(err){ + ctx.logger.error('auth error: %s', err.stack); + sendDataDisconnectReason(ctx, conn, constants.ACCESS_DENIED_CODE, constants.ACCESS_DENIED_REASON); + conn.disconnect(true); + return; + } + break; + case 'message' : + yield* onMessage(ctx, conn, data); + break; + case 'cursor' : + yield* onCursor(ctx, conn, data); + break; + case 'getLock' : + yield getLock(ctx, conn, data, false); + break; + case 'saveChanges' : + yield* saveChanges(ctx, conn, data); + break; + case 'isSaveLock' : + yield* isSaveLock(ctx, conn, data); + break; + case 'unSaveLock' : + yield* unSaveLock(ctx, conn, -1, -1, -1); + break; // The index is sent -1, because this is an emergency withdrawal without saving + case 'getMessages' : + yield* getMessages(ctx, conn, data); + break; + case 'unLockDocument' : + yield* checkEndAuthLock(ctx, data.unlock, data.isSave, docId, conn.user.id, data.releaseLocks, data.deleteIndex, conn); + break; + case 'close': + yield* closeDocument(ctx, conn); + break; + case 'versionHistory' : { + let cmd = new commonDefines.InputCommand(data.cmd); + yield* versionHistory(ctx, conn, cmd); + break; + } + case 'openDocument' : { + var cmd = new commonDefines.InputCommand(data.message); + cmd.fillFromConnection(conn); + yield canvasService.openDocument(ctx, conn, cmd); + break; + } + case 'clientLog': + let level = data.level?.toLowerCase(); + if("trace" === level || "debug" === level || "info" === level || "warn" === level || "error" === level || "fatal" === level) { + ctx.logger[level]("clientLog: %s", data.msg); + } + if ("error" === level && tenErrorFiles && docId) { + let destDir = 'browser/' + docId; + yield storage.copyPath(ctx, docId, destDir, undefined, tenErrorFiles); + yield* saveErrorChanges(ctx, docId, destDir); + } + break; + case 'extendSession' : + ctx.logger.debug("extendSession idletime: %d", data.idletime); + conn.sessionIsSendWarning = false; + conn.sessionTimeLastAction = new Date().getTime() - data.idletime; + break; + case 'forceSaveStart' : + var forceSaveRes; + if (conn.user) { + forceSaveRes = yield startForceSave(ctx, docId, commonDefines.c_oAscForceSaveTypes.Button, undefined, undefined, conn.user.idOriginal, conn.user.id, undefined, conn.user.indexUser); + } else { + forceSaveRes = {code: commonDefines.c_oAscServerCommandErrors.UnknownError, time: null}; + } + sendData(ctx, conn, {type: "forceSaveStart", messages: forceSaveRes}); + break; + case 'rpc' : + yield* startRPC(ctx, conn, data.responseKey, data.data); + break; + case 'authChangesAck' : + delete conn.authChangesAck; + break; + default: + ctx.logger.debug("unknown command %j", data); + break; } - break; - case 'extendSession' : - ctx.logger.debug("extendSession idletime: %d", data.idletime); - conn.sessionIsSendWarning = false; - conn.sessionTimeLastAction = new Date().getTime() - data.idletime; - break; - case 'forceSaveStart' : - var forceSaveRes; - if (conn.user) { - forceSaveRes = yield startForceSave(ctx, docId, commonDefines.c_oAscForceSaveTypes.Button, undefined, undefined, conn.user.idOriginal, conn.user.id, undefined, conn.user.indexUser); - } else { - forceSaveRes = {code: commonDefines.c_oAscServerCommandErrors.UnknownError, time: null}; + if(clientStatsD) { + if('openDocument' != data.type) { + clientStatsD.timing('coauth.data.' + data.type, new Date() - startDate); + } } - sendData(ctx, conn, {type: "forceSaveStart", messages: forceSaveRes}); - break; - case 'rpc' : - yield* startRPC(ctx, conn, data.responseKey, data.data); - break; - case 'authChangesAck' : - delete conn.authChangesAck; - break; - default: - ctx.logger.debug("unknown command %j", data); - break; - } - if(clientStatsD) { - if('openDocument' != data.type) { - clientStatsD.timing('coauth.data.' + data.type, new Date() - startDate); + } catch (e) { + ctx.logger.error("error receiving response: type = %s %s", (data && data.type) ? data.type : 'null', e.stack); } - } - } catch (e) { - ctx.logger.error("error receiving response: type = %s %s", (data && data.type) ? data.type : 'null', e.stack); - } + }); }); - }); - conn.on("disconnect", function(reason) { - return co(function* () { - let ctx = new operationContext.Context(); - try { - ctx.initFromConnection(conn); - yield ctx.initTenantCache(); - yield* closeDocument(ctx, conn, reason); - } catch (err) { - ctx.logger.error('Error conn close: %s', err.stack); - } + conn.on("disconnect", function(reason) { + return co(function* () { + let ctx = new operationContext.Context(); + try { + ctx.initFromConnection(conn); + yield ctx.initTenantCache(); + yield* closeDocument(ctx, conn, reason); + } catch (err) { + ctx.logger.error('Error conn close: %s', err.stack); + } + }); }); - }); - _checkLicense(ctx, conn); + _checkLicense(ctx, conn); + } catch(err){ + ctx.logger.error('connection error: %s', err.stack); + sendDataDisconnectReason(ctx, conn, constants.DROP_CODE, constants.DROP_REASON); + conn.disconnect(true); + } }); io.engine.on("connection_error", (err) => { operationContext.global.logger.warn('io.connection_error code=%s, message=%s', err.code, err.message); @@ -1795,7 +1812,7 @@ exports.install = function(server, callbackFunction) { if (!participantsTimestamp) { participantsTimestamp = Date.now(); } - yield* publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participants}, docId, tmpUser.id); + yield publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participants}, docId, tmpUser.id); tmpUser.view = tmpView; // For this user, we remove the lock from saving @@ -1817,11 +1834,11 @@ exports.install = function(server, callbackFunction) { } } //Release locks - userLocks = yield* removeUserLocks(ctx, docId, conn.user.id); + userLocks = yield removeUserLocks(ctx, docId, conn.user.id); if (0 < userLocks.length) { //todo send nothing in case of close document //sendReleaseLock(conn, userLocks); - yield* publish(ctx, {type: commonDefines.c_oPublishType.releaseLock, ctx: ctx, docId: docId, userId: conn.user.id, locks: userLocks}, docId, conn.user.id); + yield publish(ctx, {type: commonDefines.c_oPublishType.releaseLock, ctx: ctx, docId: docId, userId: conn.user.id, locks: userLocks}, docId, conn.user.id); } // For this user, remove the Lock from the document @@ -1843,7 +1860,8 @@ exports.install = function(server, callbackFunction) { } if (needSaveChanges && !conn.encrypted) { // Send changes to save server - yield createSaveTimer(ctx, docId, tmpUser.idOriginal, userIndex); + let user_lcid = utilsDocService.localeToLCID(conn.lang); + yield createSaveTimer(ctx, docId, tmpUser.idOriginal, userIndex, user_lcid); } else if (needSendStatus) { yield* cleanDocumentOnExitNoChanges(ctx, docId, tmpUser.idOriginal, userIndex); } else { @@ -1909,31 +1927,19 @@ exports.install = function(server, callbackFunction) { return objChangesDocument; } - function* getAllLocks(ctx, docId) { - var docLockRes = []; - var docLock = yield editorData.getLocks(ctx, docId); - for (var i = 0; i < docLock.length; ++i) { - docLockRes.push(docLock[i]); - } - return docLockRes; - } - function* removeUserLocks(ctx, docId, userId) { - var userLocks = [], i; - var toCache = []; - var docLock = yield* getAllLocks(ctx, docId); - for (i = 0; i < docLock.length; ++i) { - var elem = docLock[i]; - if (elem.user === userId) { - userLocks.push(elem); - } else { - toCache.push(elem); + async function removeUserLocks(ctx, docId, userId) { + let locks = await editorData.getLocks(ctx, docId); + let res = []; + let toRemove = {}; + for (let lockId in locks) { + let lock = locks[lockId]; + if (lock.user === userId) { + toRemove[lockId] = lock; + res.push(lock); } } - //remove all - yield editorData.removeLocks(ctx, docId); - //set all - yield editorData.addLocks(ctx, docId, toCache); - return userLocks; + await editorData.removeLocks(ctx, docId, toRemove); + return res; } function* checkEndAuthLock(ctx, unlock, isSave, docId, userId, releaseLocks, deleteIndex, conn) { @@ -1955,7 +1961,7 @@ exports.install = function(server, callbackFunction) { var unlockRes = yield editorData.unlockAuth(ctx, docId, userId); if (commonDefines.c_oAscUnlockRes.Unlocked === unlockRes) { const participantsMap = yield getParticipantMap(ctx, docId); - yield* publish(ctx, { + yield publish(ctx, { type: commonDefines.c_oPublishType.auth, ctx: ctx, docId: docId, @@ -1969,10 +1975,10 @@ exports.install = function(server, callbackFunction) { //Release locks if (releaseLocks && conn) { - const userLocks = yield* removeUserLocks(ctx, docId, userId); + const userLocks = yield removeUserLocks(ctx, docId, userId); if (0 < userLocks.length) { sendReleaseLock(ctx, conn, userLocks); - yield* publish(ctx, { + yield publish(ctx, { type: commonDefines.c_oPublishType.releaseLock, ctx: ctx, docId: docId, @@ -2052,20 +2058,20 @@ exports.install = function(server, callbackFunction) { // Recalculation only for foreign Lock when saving on a client that added/deleted rows or columns function _recalcLockArray(userId, _locks, oRecalcIndexColumns, oRecalcIndexRows) { + let res = {}; if (null == _locks) { - return false; + return res; } - var count = _locks.length; var element = null, oRangeOrObjectId = null; - var i; var sheetId = -1; - var isModify = false; - for (i = 0; i < count; ++i) { + for (let lockId in _locks) { + let isModify = false; + let lock = _locks[lockId]; // we do not count for ourselves - if (userId === _locks[i].user) { + if (userId === lock.user) { continue; } - element = _locks[i].block; + element = lock.block; if (c_oAscLockTypeElem.Range !== element["type"] || c_oAscLockTypeElemSubType.InsertColumns === element["subType"] || c_oAscLockTypeElemSubType.InsertRows === element["subType"]) { @@ -2087,8 +2093,11 @@ exports.install = function(server, callbackFunction) { oRangeOrObjectId["r2"] = oRecalcIndexRows[sheetId].getLockMe2(oRangeOrObjectId["r2"]); isModify = true; } + if (isModify) { + res[lockId] = lock; + } } - return isModify; + return res; } function _addRecalcIndex(oRecalcIndex) { @@ -2223,7 +2232,10 @@ exports.install = function(server, callbackFunction) { return name; } function isEditMode(permissions, mode) { - //as in web-apps/apps/documenteditor/main/app/controller/Main.js + //like this.api.asc_setViewMode(!this.appOptions.isEdit && !this.appOptions.isRestrictedEdit); + //https://github.com/ONLYOFFICE/web-apps/blob/4a7879b4f88f315fe94d9f7d97c0ed8aa9f82221/apps/documenteditor/main/app/controller/Main.js#L1743 + //todo permissions in embed editor + //https://github.com/ONLYOFFICE/web-apps/blob/72b8350c71e7b314b63b8eec675e76156bb4a2e4/apps/documenteditor/forms/app/controller/ApplicationController.js#L627 return (!mode || mode !== 'view') && (!permissions || permissions.edit !== false || permissions.review === true || permissions.comment === true || permissions.fillForms === true); } @@ -2240,13 +2252,13 @@ exports.install = function(server, callbackFunction) { } if (decoded.queryParams) { let queryParams = decoded.queryParams; - data.lang = queryParams.lang || queryParams.ui || "en-US"; + data.lang = queryParams.lang || queryParams.ui || constants.TEMPLATES_DEFAULT_LOCALE; } - if (decoded.fileInfo) { + if (wopiClient.isWopiJwtToken(decoded)) { let fileInfo = decoded.fileInfo; + let queryParams = decoded.queryParams; if (openCmd) { - let fileType = fileInfo.BaseFileName ? fileInfo.BaseFileName.substr(fileInfo.BaseFileName.lastIndexOf('.') + 1) : ""; - openCmd.format = fileInfo.FileExtension ? fileInfo.FileExtension.substr(1) : fileType; + openCmd.format = wopiClient.getFileTypeByInfo(fileInfo); openCmd.title = fileInfo.BreadcrumbDocName || fileInfo.BaseFileName; } let name = fileInfo.IsAnonymousUser ? "" : fileInfo.UserFriendlyName; @@ -2260,11 +2272,15 @@ exports.install = function(server, callbackFunction) { openCmd.userid = fileInfo.UserId; } } + let permissionsEdit = !fileInfo.ReadOnly && fileInfo.UserCanWrite && queryParams.formsubmit !== "1"; + let permissionsFillForm = permissionsEdit || queryParams.formsubmit === "1"; let permissions = { - edit: !fileInfo.ReadOnly && fileInfo.UserCanWrite, + edit: permissionsEdit, review: (fileInfo.SupportsReviewing === false) ? false : (fileInfo.UserCanReview === false ? false : fileInfo.UserCanReview), copy: fileInfo.CopyPasteRestrictions !== "CurrentDocumentOnly" && fileInfo.CopyPasteRestrictions !== "BlockAll", - print: !fileInfo.DisablePrint && !fileInfo.HidePrintOption + print: !fileInfo.DisablePrint && !fileInfo.HidePrintOption, + chat: queryParams.dchat!=="1", + fillForms: permissionsFillForm }; //todo (review: undefiend) // res = deepEqual(data.permissions, permissions, {strict: true}); @@ -2342,7 +2358,8 @@ exports.install = function(server, callbackFunction) { if (null != edit.lang) { data.lang = edit.lang; } - if (null != edit.mode) { + //allow to restrict rights so don't use token mode in case of 'view' + if (null != edit.mode && 'view' !== data.mode) { data.mode = edit.mode; } if (edit.coEditing?.mode) { @@ -2398,7 +2415,7 @@ exports.install = function(server, callbackFunction) { } //todo make required fields - if (decoded.url || decoded.payload|| (decoded.key && !decoded.fileInfo)) { + if (decoded.url || decoded.payload|| (decoded.key && !wopiClient.isWopiJwtToken(decoded))) { ctx.logger.warn('fillDataFromJwt token has invalid format'); res = false; } @@ -2439,7 +2456,7 @@ exports.install = function(server, callbackFunction) { isDecoded = true; let decoded = checkJwtRes.decoded; let fillDataFromJwtRes = false; - if (decoded.fileInfo) { + if (wopiClient.isWopiJwtToken(decoded)) { //wopi fillDataFromJwtRes = fillDataFromWopiJwt(decoded, data); } else if (decoded.editorConfig && undefined !== decoded.editorConfig.ds_view) { @@ -2477,13 +2494,37 @@ exports.install = function(server, callbackFunction) { let docId = data.docid; const user = data.user; - let wopiParams = null; + let wopiParams = null, openedAtStr; if (data.documentCallbackUrl) { wopiParams = wopiClient.parseWopiCallback(ctx, data.documentCallbackUrl); if (wopiParams && wopiParams.userAuth) { conn.access_token_ttl = wopiParams.userAuth.access_token_ttl; } } + let cmd = null; + if (data.openCmd) { + cmd = new commonDefines.InputCommand(data.openCmd); + cmd.setDocId(docId); + if (isDecoded) { + cmd.setWithAuthorization(true); + } + } + //todo minimize select calls on opening + let result = yield taskResult.select(ctx, docId); + let resultRow = result.length > 0 ? result[0] : null; + if (wopiParams) { + let wopiParamsFull; + if (resultRow && resultRow.callback) { + wopiParamsFull = wopiClient.parseWopiCallback(ctx, data.documentCallbackUrl, resultRow.callback); + cmd?.setWopiParams(wopiParamsFull); + } + if (!wopiParamsFull || !wopiParamsFull.userAuth || !wopiParamsFull.commonInfo) { + ctx.logger.warn('invalid wopi callback (maybe postgres<9.5) %j', wopiParams); + sendDataDisconnectReason(ctx, conn, constants.DROP_CODE, constants.DROP_REASON); + conn.disconnect(true); + return; + } + } //get user index const bIsRestore = null != data.sessionId; let upsertRes = null; @@ -2503,10 +2544,10 @@ exports.install = function(server, callbackFunction) { } } let format = data.openCmd && data.openCmd.format; - upsertRes = yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByConnection(ctx, conn), true, data.documentCallbackUrl, format); - let isInserted = upsertRes.affectedRows == 1; - curIndexUser = isInserted ? 1 : upsertRes.insertId; - if ((isInserted || (wopiParams && 2 === curIndexUser)) && (undefined !== data.timezoneOffset || ctx.shardKey)) { + upsertRes = yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByConnection(ctx, conn), data.documentCallbackUrl, format); + curIndexUser = upsertRes.insertId; + //todo update additional in commandOpenStartPromise + if ((upsertRes.isInsert || (wopiParams && 2 === curIndexUser)) && (undefined !== data.timezoneOffset || ctx.shardKey || ctx.wopiSrc)) { //todo insert in commandOpenStartPromise. insert here for database compatibility if (false === canvasService.hasAdditionalCol) { let selectRes = yield taskResult.select(ctx, docId); @@ -2518,11 +2559,15 @@ exports.install = function(server, callbackFunction) { task.key = docId; if (undefined !== data.timezoneOffset) { //todo duplicate created_at because CURRENT_TIMESTAMP uses server timezone - task.additional = sqlBase.DocumentAdditional.prototype.setOpenedAt(Date.now(), data.timezoneOffset); + openedAtStr = sqlBase.DocumentAdditional.prototype.setOpenedAt(Date.now(), data.timezoneOffset); + task.additional = openedAtStr; } if (ctx.shardKey) { task.additional += sqlBase.DocumentAdditional.prototype.setShardKey(ctx.shardKey); } + if (ctx.wopiSrc) { + task.additional += sqlBase.DocumentAdditional.prototype.setWopiSrc(ctx.wopiSrc); + } yield taskResult.update(ctx, task); } else { ctx.logger.warn('auth unknown column "additional"'); @@ -2538,7 +2583,7 @@ exports.install = function(server, callbackFunction) { const curUserId = curUserIdOriginal + curIndexUser; conn.tenant = tenantManager.getTenantByConnection(ctx, conn); conn.docId = data.docid; - conn.permissions = data.permissions; + conn.permissions = data.permissions || {}; conn.user = { id: curUserId, idOriginal: curUserIdOriginal, @@ -2561,6 +2606,7 @@ exports.install = function(server, callbackFunction) { } conn.unsyncTime = null; conn.encrypted = data.encrypted; + conn.lang = data.lang; conn.supportAuthChangesAck = data.supportAuthChangesAck; const c_LR = constants.LICENSE_RESULT; @@ -2598,15 +2644,6 @@ exports.install = function(server, callbackFunction) { } } - let cmd = null; - if (data.openCmd) { - cmd = new commonDefines.InputCommand(data.openCmd); - cmd.fillFromConnection(conn); - if (isDecoded) { - cmd.setWithAuthorization(true); - } - } - // Situation when the user is already disabled from co-authoring if (bIsRestore && data.isCloseCoAuthoring) { conn.sessionId = data.sessionId;//restore old @@ -2627,20 +2664,6 @@ exports.install = function(server, callbackFunction) { } return; } - let result = yield taskResult.select(ctx, docId); - let resultRow = result.length > 0 ? result[0] : null; - if (cmd && resultRow && resultRow.callback) { - let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, resultRow.callback, curIndexUser); - let wopiParams = wopiClient.parseWopiCallback(ctx, userAuthStr, resultRow.callback); - cmd.setWopiParams(wopiParams); - if (wopiParams) { - documentCallback = null; - if (!wopiParams.userAuth || !wopiParams.commonInfo) { - yield* sendFileErrorAuth(ctx, conn, data.sessionId, `invalid wopi callback (maybe postgres<9.5) ${JSON.stringify(wopiParams)}`); - return; - } - } - } if (conn.user.idOriginal.length > constants.USER_ID_MAX_LENGTH) { //todo refactor DB and remove restrictions ctx.logger.warn('auth user id too long actual = %s; max = %s', curUserIdOriginal.length, constants.USER_ID_MAX_LENGTH); @@ -2719,7 +2742,7 @@ exports.install = function(server, callbackFunction) { if (bIsSuccessRestore) { // check locks var arrayBlocks = data['block']; - var getLockRes = yield* getLock(ctx, conn, data, true); + var getLockRes = yield getLock(ctx, conn, data, true); if (arrayBlocks && (0 === arrayBlocks.length || getLockRes)) { yield* authRestore(ctx, conn, data.sessionId); } else { @@ -2737,7 +2760,8 @@ exports.install = function(server, callbackFunction) { } } else { conn.sessionId = conn.id; - const endAuthRes = yield* endAuth(ctx, conn, false, documentCallback, canvasService.getOpenedAt(resultRow)); + let openedAt = openedAtStr ? sqlBase.DocumentAdditional.prototype.getOpenedAt(openedAtStr) : canvasService.getOpenedAt(resultRow); + const endAuthRes = yield* endAuth(ctx, conn, false, documentCallback, openedAt); if (endAuthRes && cmd) { yield canvasService.openDocument(ctx, conn, cmd, upsertRes, bIsRestore); } @@ -2838,7 +2862,7 @@ exports.install = function(server, callbackFunction) { //closing could happen during async action return false; } - yield* publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participantsMap, waitAuthUserId: waitAuthUserId}, docId, tmpUser.id); + yield publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: docId, userId: tmpUser.id, participantsTimestamp: participantsTimestamp, participants: participantsMap, waitAuthUserId: waitAuthUserId}, docId, tmpUser.id); return res; } @@ -2941,17 +2965,13 @@ exports.install = function(server, callbackFunction) { const tenTypesUpload = ctx.getCfg('services.CoAuthoring.utils.limits_image_types_upload', cfgTypesUpload); const docId = conn.docId; - let docLock; - if(EditorTypes.document == conn.editorType){ - docLock = {}; - let elem; - const allLocks = yield* getAllLocks(ctx, docId); - for(let i = 0 ; i < allLocks.length; ++i) { - elem = allLocks[i]; - docLock[elem.block] = elem; + let docLock = yield editorData.getLocks(ctx, docId); + if (EditorTypes.document !== conn.editorType){ + let docLockList = []; + for (let lockId in docLock) { + docLockList.push(docLock[lockId]); } - } else { - docLock = yield* getAllLocks(ctx, docId); + docLock = docLockList; } let allMessages = yield editorData.getMessages(ctx, docId); allMessages = allMessages.length > 0 ? allMessages : undefined;//todo client side @@ -2984,7 +3004,7 @@ exports.install = function(server, callbackFunction) { } function* onMessage(ctx, conn, data) { - if (false === conn.permissions.chat) { + if (false === conn.permissions?.chat) { ctx.logger.warn("insert message permissions.chat==false"); return; } @@ -2997,7 +3017,7 @@ exports.install = function(server, callbackFunction) { var messages = [msg]; sendDataMessage(ctx, conn, messages); - yield* publish(ctx, {type: commonDefines.c_oPublishType.message, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); + yield publish(ctx, {type: commonDefines.c_oPublishType.message, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); } function* onCursor(ctx, conn, data) { @@ -3008,100 +3028,55 @@ exports.install = function(server, callbackFunction) { ctx.logger.info("send cursor: %s", msg); var messages = [msg]; - yield* publish(ctx, {type: commonDefines.c_oPublishType.cursor, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); + yield publish(ctx, {type: commonDefines.c_oPublishType.cursor, ctx: ctx, docId: docId, userId: userId, messages: messages}, docId, userId); } - - function* getLock(ctx, conn, data, bIsRestore) { - ctx.logger.info("getLock"); - var fLock = null; + // For Word block is now string "guid" + // For Excel block is now object { sheetId, type, rangeOrObjectId, guid } + // For presentations, this is an object { type, val } or { type, slideId, objId } + async function getLock(ctx, conn, data, bIsRestore) { + ctx.logger.debug("getLock"); + var fCheckLock = null; switch (conn.editorType) { case EditorTypes.document: // Word - fLock = getLockWord; + fCheckLock = _checkLockWord; break; case EditorTypes.spreadsheet: // Excel - fLock = getLockExcel; + fCheckLock = _checkLockExcel; break; case EditorTypes.presentation: // PP - fLock = getLockPresentation; + fCheckLock = _checkLockPresentation; break; + default: + return false; } - return fLock ? yield* fLock(ctx, conn, data, bIsRestore) : false; - } - - function* getLockWord(ctx, conn, data, bIsRestore) { - var docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; - var i; - var checkRes = yield* _checkLock(ctx, docId, arrayBlocks); - var documentLocks = checkRes.documentLocks; - if (checkRes.res) { - //Ok. take lock - var toCache = []; - for (i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - var elem = {time: Date.now(), user: userId, block: block}; - documentLocks[block] = elem; - toCache.push(elem); - } - yield editorData.addLocks(ctx, docId, toCache); - } else if (bIsRestore) { - return false; - } - //to the one who made the request we return as quickly as possible - sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield* publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); - return true; - } - - // For Excel block is now object { sheetId, type, rangeOrObjectId, guid } - function* getLockExcel(ctx, conn, data, bIsRestore) { - var docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; - var i; - var checkRes = yield* _checkLockExcel(ctx, docId, arrayBlocks, userId); - var documentLocks = checkRes.documentLocks; - if (checkRes.res) { - //Ok. take lock - var toCache = []; - for (i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - var elem = {time: Date.now(), user: userId, block: block}; - documentLocks.push(elem); - toCache.push(elem); - } - yield editorData.addLocks(ctx, docId, toCache); - } else if (bIsRestore) { - return false; - } - //to the one who made the request we return as quickly as possible - sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield* publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); - return true; - } - - // For presentations, this is an object { type, val } or { type, slideId, objId } - function* getLockPresentation(ctx, conn, data, bIsRestore) { - var docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; - var i; - var checkRes = yield* _checkLockPresentation(ctx, docId, arrayBlocks, userId); - var documentLocks = checkRes.documentLocks; - if (checkRes.res) { - //Ok. take lock - var toCache = []; - for (i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - var elem = {time: Date.now(), user: userId, block: block}; - documentLocks.push(elem); - toCache.push(elem); - } - yield editorData.addLocks(ctx, docId, toCache); - } else if (bIsRestore) { - return false; + let docId = conn.docId, userId = conn.user.id, arrayBlocks = data.block; + let locks = arrayBlocks.reduce(function(map, block) { + //todo use one id + map[block.guid || block] = {time: Date.now(), user: userId, block: block}; + return map; + }, {}); + let addRes = await editorData.addLocksNX(ctx, docId, locks); + let documentLocks = addRes.allLocks; + let isAllAdded = Object.keys(addRes.lockConflict).length === 0; + if (!isAllAdded || !fCheckLock(ctx, docId, documentLocks, locks, arrayBlocks, userId)) { + //remove new locks + let toRemove = {}; + for (let lockId in locks) { + if (!addRes.lockConflict[lockId]) { + toRemove[lockId] = locks[lockId]; + delete documentLocks[lockId]; + } + } + await editorData.removeLocks(ctx, docId, toRemove); + if (bIsRestore) { + return false; + } } - //to the one who made the request we return as quickly as possible sendData(ctx, conn, {type: "getLock", locks: documentLocks}); - yield* publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); + await publish(ctx, {type: commonDefines.c_oPublishType.getLock, ctx: ctx, docId: docId, userId: userId, documentLocks: documentLocks}, docId, userId); return true; } @@ -3185,14 +3160,10 @@ exports.install = function(server, callbackFunction) { const oRecalcIndexRows = _addRecalcIndex(tmpAdditionalInfo["indexRows"]); // Now we need to recalculate indexes for lock elements if (null !== oRecalcIndexColumns || null !== oRecalcIndexRows) { - const docLock = yield* getAllLocks(ctx, docId); - if (_recalcLockArray(userId, docLock, oRecalcIndexColumns, oRecalcIndexRows)) { - let toCache = []; - for (let i = 0; i < docLock.length; ++i) { - toCache.push(docLock[i]); - } - yield editorData.removeLocks(ctx, docId); - yield editorData.addLocks(ctx, docId, toCache); + let docLock = yield editorData.getLocks(ctx, docId); + let docLockMod = _recalcLockArray(userId, docLock, oRecalcIndexColumns, oRecalcIndexRows); + if (Object.keys(docLockMod).length > 0) { + yield editorData.addLocks(ctx, docId, docLockMod); } } } @@ -3200,7 +3171,7 @@ exports.install = function(server, callbackFunction) { let userLocks = []; if (data.releaseLocks) { //Release locks - userLocks = yield* removeUserLocks(ctx, docId, userId); + userLocks = yield removeUserLocks(ctx, docId, userId); } // For this user, we remove Lock from the document if the unlock flag has arrived const checkEndAuthLockRes = yield* checkEndAuthLock(ctx, data.unlock, false, docId, userId); @@ -3221,14 +3192,14 @@ exports.install = function(server, callbackFunction) { value.time = value.time.getTime(); }) } - yield* publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, + yield publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, changes: changesToSend, startIndex: startIndex, changesIndex: puckerIndex, syncChangesIndex: puckerIndex, locks: arrLocks, excelAdditionalInfo: data.excelAdditionalInfo, endSaveChanges: data.endSaveChanges}, docId, userId); } // Automatically remove the lock ourselves and send the index to save yield* unSaveLock(ctx, conn, changesIndex, newChangesLastTime, puckerIndex); //last save - let changeInfo = getExternalChangeInfo(conn.user, newChangesLastTime); + let changeInfo = getExternalChangeInfo(conn.user, newChangesLastTime, conn.lang); yield resetForceSaveAfterChanges(ctx, docId, newChangesLastTime, puckerIndex, utils.getBaseUrlByConnection(ctx, conn), changeInfo); } else { let changesToSend = arrNewDocumentChanges; @@ -3239,13 +3210,13 @@ exports.install = function(server, callbackFunction) { value.time = value.time.getTime(); }) } - let isPublished = yield* publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, + let isPublished = yield publish(ctx, {type: commonDefines.c_oPublishType.changes, ctx: ctx, docId: docId, userId: userId, changes: changesToSend, startIndex: startIndex, changesIndex: puckerIndex, syncChangesIndex: puckerIndex, locks: [], excelAdditionalInfo: undefined, endSaveChanges: data.endSaveChanges}, docId, userId); sendData(ctx, conn, {type: 'savePartChanges', changesIndex: changesIndex, syncChangesIndex: puckerIndex}); if (!isPublished) { //stub for lockDocumentsTimerId - yield* publish(ctx, {type: commonDefines.c_oPublishType.changesNotify, ctx: ctx, docId: docId}); + yield publish(ctx, {type: commonDefines.c_oPublishType.changesNotify, ctx: ctx, docId: docId}); } } } @@ -3301,45 +3272,21 @@ exports.install = function(server, callbackFunction) { sendDataMessage(ctx, conn, allMessages); } - function* _checkLock(ctx, docId, arrayBlocks) { - // Data is array now - var isLock = false; - var allLocks = yield* getAllLocks(ctx, docId); - var documentLocks = {}; - for(var i = 0 ; i < allLocks.length; ++i) { - var elem = allLocks[i]; - documentLocks[elem.block] =elem; - } - if (arrayBlocks.length > 0) { - for (var i = 0; i < arrayBlocks.length; ++i) { - var block = arrayBlocks[i]; - ctx.logger.info("getLock id: %s", block); - if (documentLocks.hasOwnProperty(block) && documentLocks[block] !== null) { - isLock = true; - break; - } - } - } else { - isLock = true; - } - return {res: !isLock, documentLocks: documentLocks}; + function _checkLockWord(ctx, docId, documentLocks, newLocks, arrayBlocks, userId) { + return true; } - - function* _checkLockExcel(ctx, docId, arrayBlocks, userId) { + function _checkLockExcel(ctx, docId, documentLocks, newLocks, arrayBlocks, userId) { // Data is array now var documentLock; var isLock = false; var isExistInArray = false; var i, blockRange; - var documentLocks = yield* getAllLocks(ctx, docId); var lengthArray = (arrayBlocks) ? arrayBlocks.length : 0; for (i = 0; i < lengthArray && false === isLock; ++i) { blockRange = arrayBlocks[i]; - for (var keyLockInArray in documentLocks) { - if (true === isLock) { - break; - } - if (!documentLocks.hasOwnProperty(keyLockInArray)) { + for (let keyLockInArray in documentLocks) { + if (newLocks[keyLockInArray]) { + //skip just added continue; } documentLock = documentLocks[keyLockInArray]; @@ -3376,41 +3323,43 @@ exports.install = function(server, callbackFunction) { continue; } isLock = compareExcelBlock(blockRange, documentLock.block); + if (true === isLock) { + break; + } } } if (0 === lengthArray) { isLock = true; } - return {res: !isLock && !isExistInArray, documentLocks: documentLocks}; + return !isLock && !isExistInArray; } - function* _checkLockPresentation(ctx, docId, arrayBlocks, userId) { + function _checkLockPresentation(ctx, docId, documentLocks, newLocks, arrayBlocks, userId) { // Data is array now var isLock = false; - var i, documentLock, blockRange; - var documentLocks = yield* getAllLocks(ctx, docId); + var i, blockRange; var lengthArray = (arrayBlocks) ? arrayBlocks.length : 0; for (i = 0; i < lengthArray && false === isLock; ++i) { blockRange = arrayBlocks[i]; - for (var keyLockInArray in documentLocks) { - if (true === isLock) { - break; - } - if (!documentLocks.hasOwnProperty(keyLockInArray)) { + for (let keyLockInArray in documentLocks) { + if (newLocks[keyLockInArray]) { + //skip just added continue; } - documentLock = documentLocks[keyLockInArray]; - + let documentLock = documentLocks[keyLockInArray]; if (documentLock.user === userId || !(documentLock.block)) { continue; } isLock = comparePresentationBlock(blockRange, documentLock.block); + if (true === isLock) { + break; + } } } if (0 === lengthArray) { isLock = true; } - return {res: !isLock, documentLocks: documentLocks}; + return !isLock; } function _checkLicense(ctx, conn) { @@ -3477,13 +3426,13 @@ exports.install = function(server, callbackFunction) { if (licenseInfo.usersCount) { const nowUTC = getLicenseNowUtc(); if(isLiveViewer) { - const arrUsers = yield editorData.getPresenceUniqueViewUser(ctx, nowUTC); + const arrUsers = yield editorStat.getPresenceUniqueViewUser(ctx, nowUTC); if (arrUsers.length >= licenseInfo.usersViewCount && (-1 === arrUsers.findIndex((element) => {return element.userid === userId}))) { licenseType = c_LR.UsersViewCount; } licenseWarningLimitUsersView = licenseInfo.usersViewCount * tenWarningLimitPercents <= arrUsers.length; } else { - const arrUsers = yield editorData.getPresenceUniqueUser(ctx, nowUTC); + const arrUsers = yield editorStat.getPresenceUniqueUser(ctx, nowUTC); if (arrUsers.length >= licenseInfo.usersCount && (-1 === arrUsers.findIndex((element) => {return element.userid === userId}))) { licenseType = c_LR.UsersCount; } @@ -3491,14 +3440,14 @@ exports.install = function(server, callbackFunction) { } } else if(isLiveViewer) { const connectionsLiveCount = licenseInfo.connectionsView; - const liveViewerConnectionsCount = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + const liveViewerConnectionsCount = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); if (liveViewerConnectionsCount >= connectionsLiveCount) { licenseType = c_LR.ConnectionsLive; } licenseWarningLimitConnectionsLive = connectionsLiveCount * tenWarningLimitPercents <= liveViewerConnectionsCount; } else { const connectionsCount = licenseInfo.connections; - const editConnectionsCount = yield editorData.getEditorConnectionsCount(ctx, connections); + const editConnectionsCount = yield editorStat.getEditorConnectionsCount(ctx, connections); if (editConnectionsCount >= connectionsCount) { licenseType = c_LR.Connections; } @@ -3694,7 +3643,8 @@ exports.install = function(server, callbackFunction) { case commonDefines.c_oPublishType.shutdown: //flag prevent new socket connections and receive data from exist connections shutdownFlag = data.status; - ctx.logger.warn('start shutdown:%b', shutdownFlag); + wopiClient.setIsShutdown(shutdownFlag); + ctx.logger.warn('start shutdown:%s', shutdownFlag); if (shutdownFlag) { ctx.logger.warn('active connections: %d', connections.length); //do not stop the server, because sockets and all requests will be unavailable @@ -3742,7 +3692,7 @@ exports.install = function(server, callbackFunction) { if (hasChanges) { let participants = yield getParticipantMap(ctx, data.docId); let participantsTimestamp = Date.now(); - yield* publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: data.docId, userId: null, participantsTimestamp: participantsTimestamp, participants: participants}); + yield publish(ctx, {type: commonDefines.c_oPublishType.participantsState, ctx: ctx, docId: data.docId, userId: null, participantsTimestamp: participantsTimestamp, participants: participants}); } break; case commonDefines.c_oPublishType.rpc: @@ -3762,7 +3712,7 @@ exports.install = function(server, callbackFunction) { function* collectStats(ctx, countEdit, countLiveView, countView) { let now = Date.now(); - yield editorData.setEditorConnections(ctx, countEdit, countLiveView, countView, now, PRECISION); + yield editorStat.setEditorConnections(ctx, countEdit, countLiveView, countView, now, PRECISION); } function expireDoc() { return co(function* () { @@ -3839,16 +3789,16 @@ exports.install = function(server, callbackFunction) { ctx.setTenant(tenantId); let tenant = tenants[tenantId]; yield* collectStats(ctx, tenant.countEditByShard, tenant.countLiveViewByShard, tenant.countViewByShard); - yield editorData.setEditorConnectionsCountByShard(ctx, SHARD_ID, tenant.countEditByShard); - yield editorData.setLiveViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countLiveViewByShard); - yield editorData.setViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countViewByShard); + yield editorStat.setEditorConnectionsCountByShard(ctx, SHARD_ID, tenant.countEditByShard); + yield editorStat.setLiveViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countLiveViewByShard); + yield editorStat.setViewerConnectionsCountByShard(ctx, SHARD_ID, tenant.countViewByShard); if (clientStatsD) { //todo with multitenant - let countEdit = yield editorData.getEditorConnectionsCount(ctx, connections); + let countEdit = yield editorStat.getEditorConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.edit', countEdit); - let countLiveView = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + let countLiveView = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.liveview', countLiveView); - let countView = yield editorData.getViewerConnectionsCount(ctx, connections); + let countView = yield editorStat.getViewerConnectionsCount(ctx, connections); clientStatsD.gauge('expireDoc.connections.view', countView); } } @@ -3859,9 +3809,9 @@ exports.install = function(server, callbackFunction) { aggregationCtx.init(tenantManager.getDefautTenant(), ctx.docId, ctx.userId); //yield ctx.initTenantCache();//no need yield* collectStats(aggregationCtx, countEditByShard, countLiveViewByShard, countViewByShard); - yield editorData.setEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, countEditByShard); - yield editorData.setLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countLiveViewByShard); - yield editorData.setViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countViewByShard); + yield editorStat.setEditorConnectionsCountByShard(aggregationCtx, SHARD_ID, countEditByShard); + yield editorStat.setLiveViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countLiveViewByShard); + yield editorStat.setViewerConnectionsCountByShard(aggregationCtx, SHARD_ID, countViewByShard); } ctx.initDefault(); } catch (err) { @@ -3941,11 +3891,16 @@ exports.install = function(server, callbackFunction) { } }); if (-1 !== index || 0 === res.length) { - return editorData.connect().then(function() { - callbackFunction(); - }).catch(err => { - operationContext.global.logger.error('editorData error: %s', err.stack); - }); + return editorData.connect() + .then(function () { + return editorStat.connect(); + }) + .then(function () { + callbackFunction(); + }) + .catch(err => { + operationContext.global.logger.error('editorData error: %s', err.stack); + }); } else { operationContext.global.logger.error('DB table "%s" does not contain %s column, columns info: %j', tableName, tableRequiredColumn, res); } @@ -3970,13 +3925,18 @@ exports.healthCheck = function(req, res) { yield sqlBase.healthCheck(ctx); ctx.logger.debug('healthCheck database'); //check redis connection - if (editorData.isConnected()) { - yield editorData.ping(); + const healthData = yield editorData.healthCheck(); + if (healthData) { ctx.logger.debug('healthCheck editorData'); } else { - throw new Error('redis disconnected'); + throw new Error('editorData'); + } + const healthStat = yield editorStat.healthCheck(); + if (healthStat) { + ctx.logger.debug('healthCheck editorStat'); + } else { + throw new Error('editorStat'); } - const healthPubsub = yield pubsub.healthCheck(); if (healthPubsub) { ctx.logger.debug('healthCheck pubsub'); @@ -3991,18 +3951,12 @@ exports.healthCheck = function(req, res) { } //storage - const clusterId = cluster.isWorker ? cluster.worker.id : ''; - const tempName = 'hc_' + os.hostname() + '_' + clusterId + '_' + Math.round(Math.random() * HEALTH_CHECK_KEY_MAX); - const tempBuffer = Buffer.from([1, 2, 3, 4, 5]); - //It's proper to putObject one tempName - yield storage.putObject(ctx, tempName, tempBuffer, tempBuffer.length); - try { - //try to prevent case, when another process can remove same tempName - yield storage.deleteObject(ctx, tempName); - } catch (err) { - ctx.logger.warn('healthCheck error %s', err.stack); - } + yield storage.healthCheck(ctx); ctx.logger.debug('healthCheck storage'); + if (storage.isDiffrentPersistentStorage()) { + yield storage.healthCheck(ctx, cfgForgottenFiles); + ctx.logger.debug('healthCheck storage persistent'); + } output = true; ctx.logger.info('healthCheck end'); @@ -4064,7 +4018,7 @@ exports.licenseInfo = function(req, res) { view: {min: 0, avr: 0, max: 0} }; } - var redisRes = yield editorData.getEditorConnections(ctx); + var redisRes = yield editorStat.getEditorConnections(ctx); const now = Date.now(); if (redisRes.length > 0) { let expDocumentsStep95 = expDocumentsStep * 0.95; @@ -4125,8 +4079,8 @@ exports.licenseInfo = function(req, res) { } const nowUTC = getLicenseNowUtc(); let execRes; - execRes = yield editorData.getPresenceUniqueUser(ctx, nowUTC); - output.quota.edit.connectionsCount = yield editorData.getEditorConnectionsCount(ctx, connections); + execRes = yield editorStat.getPresenceUniqueUser(ctx, nowUTC); + output.quota.edit.connectionsCount = yield editorStat.getEditorConnectionsCount(ctx, connections); output.quota.edit.usersCount.unique = execRes.length; execRes.forEach(function(elem) { if (elem.anonym) { @@ -4134,8 +4088,8 @@ exports.licenseInfo = function(req, res) { } }); - execRes = yield editorData.getPresenceUniqueViewUser(ctx, nowUTC); - output.quota.view.connectionsCount = yield editorData.getLiveViewerConnectionsCount(ctx, connections); + execRes = yield editorStat.getPresenceUniqueViewUser(ctx, nowUTC); + output.quota.view.connectionsCount = yield editorStat.getLiveViewerConnectionsCount(ctx, connections); output.quota.view.usersCount.unique = execRes.length; execRes.forEach(function(elem) { if (elem.anonym) { @@ -4143,8 +4097,8 @@ exports.licenseInfo = function(req, res) { } }); - let byMonth = yield editorData.getPresenceUniqueUsersOfMonth(ctx); - let byMonthView = yield editorData.getPresenceUniqueViewUsersOfMonth(ctx); + let byMonth = yield editorStat.getPresenceUniqueUsersOfMonth(ctx); + let byMonthView = yield editorStat.getPresenceUniqueViewUsersOfMonth(ctx); let byMonthMerged = []; for (let i in byMonth) { if (byMonth.hasOwnProperty(i)) { @@ -4226,8 +4180,8 @@ function* findForgottenFile(ctx, docId) { function* commandLicense(ctx) { const nowUTC = getLicenseNowUtc(); - const users = yield editorData.getPresenceUniqueUser(ctx, nowUTC); - const users_view = yield editorData.getPresenceUniqueViewUser(ctx, nowUTC); + const users = yield editorStat.getPresenceUniqueUser(ctx, nowUTC); + const users_view = yield editorStat.getPresenceUniqueViewUser(ctx, nowUTC); const licenseInfo = yield tenantManager.getTenantLicense(ctx); return { @@ -4264,7 +4218,7 @@ function* commandHandle(ctx, params, req, output) { } case 'drop': { if (params.userid) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: [params.userid], description: params.description}); + yield publish(ctx, {type: commonDefines.c_oPublishType.drop, ctx: ctx, docId: docId, users: [params.userid], description: params.description}); } else if (params.users) { const users = (typeof params.users === 'string') ? JSON.parse(params.users) : params.users; yield* dropUsersFromDocument(ctx, docId, users); @@ -4291,7 +4245,7 @@ function* commandHandle(ctx, params, req, output) { } case 'meta': { if (params.meta) { - yield* publish(ctx, {type: commonDefines.c_oPublishType.meta, ctx: ctx, docId: docId, meta: params.meta}); + yield publish(ctx, {type: commonDefines.c_oPublishType.meta, ctx: ctx, docId: docId, meta: params.meta}); } else { output.error = commonDefines.c_oAscServerCommandErrors.UnknownCommand; } @@ -4321,7 +4275,7 @@ function* commandHandle(ctx, params, req, output) { break; } - yield storage.deleteObject(ctx, forgottenFile, tenForgottenFiles); + yield storage.deletePath(ctx, docId, tenForgottenFiles); break; } case 'getForgottenList': { @@ -4385,7 +4339,7 @@ exports.shutdown = function(req, res) { ctx.initFromRequest(req); yield ctx.initTenantCache(); ctx.logger.info('shutdown start'); - output = yield shutdown.shutdown(ctx, editorData, req.method === 'PUT'); + output = yield shutdown.shutdown(ctx, editorStat, req.method === 'PUT'); } catch (err) { ctx.logger.error('shutdown error %s', err.stack); } finally { diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 68e614e3a..2b9c44e35 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -38,7 +38,7 @@ var co = require('co'); const ms = require('ms'); const retry = require('retry'); const MultiRange = require('multi-integer-range').MultiRange; -var sqlBase = require('./baseConnector'); +var sqlBase = require('./databaseConnectors/baseConnector'); const utilsDocService = require('./utilsDocService'); var docsCoServer = require('./DocsCoServer'); var taskResult = require('./taskresult'); @@ -53,6 +53,7 @@ var statsDClient = require('./../../Common/sources/statsdclient'); var operationContext = require('./../../Common/sources/operationContext'); var tenantManager = require('./../../Common/sources/tenantManager'); var config = require('config'); +const path = require("path"); const cfgTypesUpload = config.get('services.CoAuthoring.utils.limits_image_types_upload'); const cfgImageSize = config.get('services.CoAuthoring.server.limits_image_size'); @@ -70,7 +71,7 @@ const cfgAssemblyFormatAsOrigin = config.get('services.CoAuthoring.server.assemb const cfgDownloadMaxBytes = config.get('FileConverter.converter.maxDownloadBytes'); const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout'); const cfgDownloadFileAllowExt = config.get('services.CoAuthoring.server.downloadFileAllowExt'); -const cfgAllowPrivateIPAddressForSignedRequests = config.get('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests'); +const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate'); var SAVE_TYPE_PART_START = 0; var SAVE_TYPE_PART = 1; @@ -161,7 +162,7 @@ function getOpenedAt(row) { function getOpenedAtJSONParams(row) { let openedAt = getOpenedAt(row); if (openedAt) { - return JSON.stringify({'documentLayout': {'openedAt': openedAt}}); + return {'documentLayout': {'openedAt': openedAt}}; } return undefined; } @@ -279,17 +280,13 @@ var getOutputData = co.wrap(function* (ctx, cmd, outputData, key, optConn, optAd outputData.setData(statusInfo); break; case commonDefines.FileStatus.Err: + outputData.setStatus('err'); + outputData.setData(statusInfo); + break; case commonDefines.FileStatus.ErrToReload: outputData.setStatus('err'); outputData.setData(statusInfo); - if (commonDefines.FileStatus.ErrToReload == status) { - let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); - let wopiParams = wopiClient.parseWopiCallback(ctx, userAuthStr); - if (!wopiParams) { - //todo rework ErrToReload to clean up on next open - yield cleanupCache(ctx, key); - } - } + yield cleanupErrToReload(ctx, key); break; case commonDefines.FileStatus.None: //this status has no handler @@ -378,38 +375,40 @@ function getSaveTask(ctx, cmd) { //} return queueData; } -function* getUpdateResponse(ctx, cmd) { +async function getUpdateResponse(ctx, cmd) { const tenOpenProtectedFile = ctx.getCfg('services.CoAuthoring.server.openProtectedFile', cfgOpenProtectedFile); var updateTask = new taskResult.TaskResultData(); updateTask.tenant = ctx.tenant; updateTask.key = cmd.getSaveKey() ? cmd.getSaveKey() : cmd.getDocId(); var statusInfo = cmd.getStatusInfo(); - if (constants.NO_ERROR == statusInfo) { + if (constants.NO_ERROR === statusInfo) { updateTask.status = commonDefines.FileStatus.Ok; let password = cmd.getPassword(); if (password) { if (false === hasPasswordCol) { - let selectRes = yield taskResult.select(ctx, updateTask.key); + let selectRes = await taskResult.select(ctx, updateTask.key); hasPasswordCol = selectRes.length > 0 && undefined !== selectRes[0].password; } if(hasPasswordCol) { updateTask.password = password; } } - } else if (constants.CONVERT_DOWNLOAD == statusInfo) { + } else if (constants.CONVERT_DOWNLOAD === statusInfo) { updateTask.status = commonDefines.FileStatus.ErrToReload; - } else if (constants.CONVERT_NEED_PARAMS == statusInfo) { + } else if (constants.CONVERT_LIMITS === statusInfo) { + updateTask.status = commonDefines.FileStatus.ErrToReload; + } else if (constants.CONVERT_NEED_PARAMS === statusInfo) { updateTask.status = commonDefines.FileStatus.NeedParams; - } else if (constants.CONVERT_DRM == statusInfo || constants.CONVERT_PASSWORD == statusInfo) { + } else if (constants.CONVERT_DRM === statusInfo || constants.CONVERT_PASSWORD === statusInfo) { if (tenOpenProtectedFile) { updateTask.status = commonDefines.FileStatus.NeedPassword; } else { updateTask.status = commonDefines.FileStatus.Err; } - } else if (constants.CONVERT_DRM_UNSUPPORTED == statusInfo) { + } else if (constants.CONVERT_DRM_UNSUPPORTED === statusInfo) { updateTask.status = commonDefines.FileStatus.Err; - } else if (constants.CONVERT_DEAD_LETTER == statusInfo) { + } else if (constants.CONVERT_DEAD_LETTER === statusInfo) { updateTask.status = commonDefines.FileStatus.ErrToReload; } else { updateTask.status = commonDefines.FileStatus.Err; @@ -420,32 +419,36 @@ function* getUpdateResponse(ctx, cmd) { var cleanupCache = co.wrap(function* (ctx, docId) { //todo redis ? var res = false; - let list = []; var removeRes = yield taskResult.remove(ctx, docId); if (removeRes.affectedRows > 0) { - list = yield storage.listObjects(ctx, docId); - yield storage.deleteObjects(ctx, list); + yield storage.deletePath(ctx, docId); res = true; } - ctx.logger.debug("cleanupCache docId=%s db.affectedRows=%d list.length=%d", docId, removeRes.affectedRows, list.length); + ctx.logger.debug("cleanupCache docId=%s db.affectedRows=%d", docId, removeRes.affectedRows); return res; }); var cleanupCacheIf = co.wrap(function* (ctx, mask) { //todo redis ? var res = false; - let list = []; var removeRes = yield taskResult.removeIf(ctx, mask); if (removeRes.affectedRows > 0) { sqlBase.deleteChanges(ctx, mask.key, null); - list = yield storage.listObjects(ctx, mask.key); - yield storage.deleteObjects(ctx, list); + yield storage.deletePath(ctx, mask.key); res = true; } - ctx.logger.debug("cleanupCacheIf db.affectedRows=%d list.length=%d", removeRes.affectedRows, list.length); + ctx.logger.debug("cleanupCacheIf db.affectedRows=%d", removeRes.affectedRows); return res; }); +async function cleanupErrToReload(ctx, key) { + let updateTask = new taskResult.TaskResultData(); + updateTask.tenant = ctx.tenant; + updateTask.key = key; + updateTask.status = commonDefines.FileStatus.None; + updateTask.statusInfo = constants.NO_ERROR; + await taskResult.update(ctx, updateTask); +} -function commandOpenStartPromise(ctx, docId, baseUrl, opt_updateUserIndex, opt_documentCallbackUrl, opt_format) { +function commandOpenStartPromise(ctx, docId, baseUrl, opt_documentCallbackUrl, opt_format) { var task = new taskResult.TaskResultData(); task.tenant = ctx.tenant; task.key = docId; @@ -459,7 +462,7 @@ function commandOpenStartPromise(ctx, docId, baseUrl, opt_updateUserIndex, opt_d if (opt_format) { task.changeId = formatChecker.getFormatFromString(opt_format); } - return taskResult.upsert(ctx, task, opt_updateUserIndex); + return taskResult.upsert(ctx, task); } function* commandOpen(ctx, conn, cmd, outputData, opt_upsertRes, opt_bIsRestore) { const tenForgottenFiles = ctx.getCfg('services.CoAuthoring.server.forgottenfiles', cfgForgottenFiles); @@ -470,9 +473,7 @@ function* commandOpen(ctx, conn, cmd, outputData, opt_upsertRes, opt_bIsRestore) } else { upsertRes = yield commandOpenStartPromise(ctx, cmd.getDocId(), utils.getBaseUrlByConnection(ctx, conn)); } - //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values - //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html - let bCreate = upsertRes.affectedRows == 1; + let bCreate = upsertRes.isInsert; let needAddTask = bCreate; if (!bCreate) { needAddTask = yield* commandOpenFillOutput(ctx, conn, cmd, outputData, opt_bIsRestore); @@ -623,10 +624,11 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, var selectRes = yield taskResult.select(ctx, cmd.getDocId()); var row = selectRes.length > 0 ? selectRes[0] : null; if (!row) { - return; + return false; } if (opt_initShardKey) { ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + ctx.setWopiSrc(sqlBase.DocumentAdditional.prototype.getWopiSrc(row.additional)); } yield* addRandomKeyTaskCmd(ctx, cmd); addPasswordToCmd(ctx, cmd, row.password); @@ -634,16 +636,13 @@ let commandSfctByCmd = co.wrap(function*(ctx, cmd, opt_priority, opt_expiration, let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback)); cmd.setOutputFormat(changeFormatByOrigin(ctx, row, cmd.getOutputFormat())); - cmd.setJsonParams(getOpenedAtJSONParams(row)); + cmd.appendJsonParams(getOpenedAtJSONParams(row)); var queueData = getSaveTask(ctx, cmd); queueData.setFromChanges(true); let priority = null != opt_priority ? opt_priority : constants.QUEUE_PRIORITY_LOW; yield* docsCoServer.addTask(queueData, priority, opt_queue, opt_expiration); + return true; }); -function* commandSfct(ctx, cmd, outputData) { - yield commandSfctByCmd(ctx, cmd); - outputData.setStatus('ok'); -} function isDisplayedImage(strName) { var res = 0; if (strName) { @@ -668,11 +667,11 @@ function* commandImgurls(ctx, conn, cmd, outputData) { const tenImageSize = ctx.getCfg('services.CoAuthoring.server.limits_image_size', cfgImageSize); const tenImageDownloadTimeout = ctx.getCfg('services.CoAuthoring.server.limits_image_download_timeout', cfgImageDownloadTimeout); const tenTokenEnableBrowser = ctx.getCfg('services.CoAuthoring.token.enable.browser', cfgTokenEnableBrowser); - const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests); var errorCode = constants.NO_ERROR; let urls = cmd.getData(); let authorizations = []; + let isInJwtToken = false; let token = cmd.getTokenDownload(); if (tenTokenEnableBrowser && token) { let checkJwtRes = yield docsCoServer.checkJwt(ctx, token, commonDefines.c_oAscSecretType.Browser); @@ -691,6 +690,7 @@ function* commandImgurls(ctx, conn, cmd, outputData) { authorizations[i] = [utils.fillJwtForRequest(ctx, {url: urls[i]}, secret, false)]; } } + isInJwtToken = true; } else { ctx.logger.warn('Error commandImgurls jwt: %s', checkJwtRes.description); errorCode = constants.VKEY_ENCRYPT; @@ -733,8 +733,7 @@ function* commandImgurls(ctx, conn, cmd, outputData) { } } //todo stream - const filterPrivate = !authorizations[i] || !tenAllowPrivateIPAddressForSignedRequests; - let getRes = yield utils.downloadUrlPromise(ctx, urlSource, tenImageDownloadTimeout, tenImageSize, authorizations[i], filterPrivate); + let getRes = yield utils.downloadUrlPromise(ctx, urlSource, tenImageDownloadTimeout, tenImageSize, authorizations[i], isInJwtToken); data = getRes.body; urlParsed = urlModule.parse(urlSource); } catch (e) { @@ -838,6 +837,7 @@ function* commandSaveFromOrigin(ctx, cmd, outputData, password) { if (docPassword.initial) { cmd.setPassword(docPassword.initial); } + //todo setLCID in browser var queueData = getSaveTask(ctx, cmd); queueData.setFromOrigin(true); queueData.setFromChanges(true); @@ -858,8 +858,11 @@ function* commandSetPassword(ctx, conn, cmd, outputData) { hasDocumentPassword = true; } } - ctx.logger.debug('commandSetPassword isEnterCorrectPassword=%s, hasDocumentPassword=%s, hasPasswordCol=%s', conn.isEnterCorrectPassword, hasDocumentPassword, hasPasswordCol); - if (tenOpenProtectedFile && (conn.isEnterCorrectPassword || !hasDocumentPassword) && hasPasswordCol) { + //https://github.com/ONLYOFFICE/web-apps/blob/4a7879b4f88f315fe94d9f7d97c0ed8aa9f82221/apps/documenteditor/main/app/controller/Main.js#L1652 + //this.appOptions.isPasswordSupport = this.appOptions.isEdit && this.api.asc_isProtectionSupport() && (this.permissions.protect!==false); + let isPasswordSupport = tenOpenProtectedFile && !conn.user?.view && false !== conn.permissions?.protect; + ctx.logger.debug('commandSetPassword isEnterCorrectPassword=%s, hasDocumentPassword=%s, hasPasswordCol=%s, isPasswordSupport=%s', conn.isEnterCorrectPassword, hasDocumentPassword, hasPasswordCol, isPasswordSupport); + if (isPasswordSupport && (conn.isEnterCorrectPassword || !hasDocumentPassword) && hasPasswordCol) { let updateMask = new taskResult.TaskResultData(); updateMask.tenant = ctx.tenant; updateMask.key = cmd.getDocId(); @@ -872,7 +875,7 @@ function* commandSetPassword(ctx, conn, cmd, outputData) { task.password = cmd.getPassword() || ""; let changeInfo = null; if (conn.user) { - changeInfo = task.innerPasswordChange = docsCoServer.getExternalChangeInfo(conn.user, newChangesLastDate.getTime()); + changeInfo = task.innerPasswordChange = docsCoServer.getExternalChangeInfo(conn.user, newChangesLastDate.getTime(), conn.lang); } var upsertRes = yield taskResult.updateIf(ctx, task, updateMask); @@ -1077,8 +1080,17 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { } else { try { if (wopiParams) { - let isAutoSave = forceSaveType !== commonDefines.c_oAscForceSaveTypes.Button && forceSaveType !== commonDefines.c_oAscForceSaveTypes.Form; - replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, true, isAutoSave, false); + if (outputSfc.getUrl()) { + if (forceSaveType === commonDefines.c_oAscForceSaveTypes.Form) { + yield processWopiSaveAs(ctx, cmd); + replyStr = JSON.stringify({error: 0}); + } else { + let isAutoSave = forceSaveType !== commonDefines.c_oAscForceSaveTypes.Button && forceSaveType !== commonDefines.c_oAscForceSaveTypes.Form; + replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, true, isAutoSave, false); + } + } else { + replyStr = JSON.stringify({error: 1, descr: "wopi: no file"}); + } } else { replyStr = yield docsCoServer.sendServerRequest(ctx, uri, outputSfc, checkAndFixAuthorizationLength); } @@ -1111,7 +1123,11 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { updateMask.statusInfo = updateIfTask.statusInfo; try { if (wopiParams) { - replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, !notModified, false, true); + if (outputSfc.getUrl()) { + replyStr = yield processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChangeId, !notModified, false, true); + } else { + replyStr = JSON.stringify({error: 1, descr: "wopi: no file"}); + } } else { replyStr = yield docsCoServer.sendServerRequest(ctx, uri, outputSfc, checkAndFixAuthorizationLength); } @@ -1178,6 +1194,8 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { } if (!isSfcm) { //todo simultaneous opening + //clean redis (redisKeyPresenceSet and redisKeyPresenceHash removed with last element) + yield docsCoServer.editorData.cleanDocumentOnExit(ctx, docId); //to unlock wopi file yield docsCoServer.unlockWopiDoc(ctx, docId, callbackUserIndex); //cleanupRes can be false in case of simultaneous opening. it is OK @@ -1205,7 +1223,7 @@ const commandSfcCallback = co.wrap(function*(ctx, cmd, isSfcm, isEncrypted) { if ((docsCoServer.getIsShutdown() && !isSfcm) || cmd.getRedisKey()) { let keyRedis = cmd.getRedisKey() ? cmd.getRedisKey() : redisKeyShutdown; - yield docsCoServer.editorData.removeShutdown(keyRedis, docId); + yield docsCoServer.editorStat.removeShutdown(keyRedis, docId); } ctx.logger.debug('End commandSfcCallback'); return replyStr; @@ -1216,19 +1234,13 @@ function* processWopiPutFile(ctx, docId, wopiParams, savePathDoc, userLastChange let streamObj = yield storage.createReadStream(ctx, savePathDoc); let postRes = yield wopiClient.putFile(ctx, wopiParams, null, streamObj.readStream, metadata.ContentLength, userLastChangeId, isModifiedByUser, isAutosave, isExitSave); if (postRes) { - if (postRes.body) { - try { - let body = JSON.parse(postRes.body); - //collabora nexcloud connector - if (body.LastModifiedTime) { - let lastModifiedTimeInfo = wopiClient.getWopiModifiedMarker(wopiParams, body.LastModifiedTime); - yield commandOpenStartPromise(ctx, docId, undefined, true, lastModifiedTimeInfo); - } - } catch (e) { - ctx.logger.debug('processWopiPutFile error: %s', e.stack); - } - } res = '{"error": 0}'; + let body = wopiClient.parsePutFileResponse(ctx, postRes); + //collabora nexcloud connector + if (body?.LastModifiedTime) { + let lastModifiedTimeInfo = wopiClient.getWopiModifiedMarker(wopiParams, body.LastModifiedTime); + yield commandOpenStartPromise(ctx, docId, undefined, lastModifiedTimeInfo); + } } return res; } @@ -1427,9 +1439,6 @@ exports.downloadAs = function(req, res) { case 'sendmm': yield* commandSendMailMerge(ctx, cmd, outputData); break; - case 'sfct': - yield* commandSfct(ctx, cmd, outputData); - break; default: outputData.setStatus('err'); outputData.setData(constants.UNKNOWN); @@ -1527,7 +1536,10 @@ function getPrintFileUrl(ctx, docId, baseUrl, filename) { let userFriendlyName = encodeURIComponent(filename.replace(/\//g, "%2f")); let res = `${baseUrl}/printfile/${encodeURIComponent(docId)}/${userFriendlyName}?token=${encodeURIComponent(token)}`; if (ctx.shardKey) { - res += `&${constants.SHARED_KEY_NAME}=${encodeURIComponent(ctx.shardKey)}`; + res += `&${constants.SHARD_KEY_API_NAME}=${encodeURIComponent(ctx.shardKey)}`; + } + if (ctx.wopiSrc) { + res += `&${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(ctx.wopiSrc)}`; } res += `&filename=${userFriendlyName}`; return res; @@ -1608,23 +1620,44 @@ exports.downloadFile = function(req, res) { const tenDownloadMaxBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgDownloadMaxBytes); const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout); const tenDownloadFileAllowExt = ctx.getCfg('services.CoAuthoring.server.downloadFileAllowExt', cfgDownloadFileAllowExt); - const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests); + const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); let authorization; + let isInJwtToken = false; let errorDescription; + let headers, fromTemplate; let authRes = yield docsCoServer.getRequestParams(ctx, req); if (authRes.code === constants.NO_ERROR) { let decoded = authRes.params; if (decoded.changesUrl) { url = decoded.changesUrl; + isInJwtToken = true; } else if (decoded.document && -1 !== tenDownloadFileAllowExt.indexOf(decoded.document.fileType)) { url = decoded.document.url; + isInJwtToken = true; } else if (decoded.url && -1 !== tenDownloadFileAllowExt.indexOf(decoded.fileType)) { url = decoded.url; + isInJwtToken = true; + } else if (wopiClient.isWopiJwtToken(decoded)) { + if (decoded.fileInfo.Size === 0) { + //editnew case + fromTemplate = pathModule.extname(decoded.fileInfo.BaseFileName).substring(1); + } else { + ({url, headers} = yield wopiClient.getWopiFileUrl(ctx, decoded.fileInfo, decoded.userAuth)); + let filterStatus = yield wopiClient.checkIpFilter(ctx, url); + if (0 === filterStatus) { + //todo false? (true because it passed checkIpFilter for wopi) + //todo use directIfIn + isInJwtToken = true; + } else { + errorDescription = 'access deny'; + } + } } else if (!tenTokenEnableBrowser) { //todo token required if (decoded.url) { url = decoded.url; + isInJwtToken = true; } } else { errorDescription = 'access deny'; @@ -1637,46 +1670,59 @@ exports.downloadFile = function(req, res) { res.sendStatus(403); return; } - if (utils.canIncludeOutboxAuthorization(ctx, url)) { - let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox); - authorization = utils.fillJwtForRequest(ctx, {url: url}, secret, false); - } - let urlParsed = urlModule.parse(url); - let filterStatus = yield* utils.checkHostFilter(ctx, urlParsed.hostname); - if (0 !== filterStatus) { - ctx.logger.warn('Error downloadFile checkIpFilter error: url = %s', url); - res.sendStatus(filterStatus); - return; - } - let headers; - if (req.get('Range')) { - headers = { - 'Range': req.get('Range') + if (fromTemplate) { + ctx.logger.debug('downloadFile from file template: %s', fromTemplate); + let locale = constants.TEMPLATES_DEFAULT_LOCALE; + let fileTemplatePath = pathModule.join(tenNewFileTemplate, locale, 'new.' + fromTemplate); + res.sendFile(pathModule.resolve(fileTemplatePath)); + } else { + if (utils.canIncludeOutboxAuthorization(ctx, url)) { + let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox); + authorization = utils.fillJwtForRequest(ctx, {url: url}, secret, false); + } + let urlParsed = urlModule.parse(url); + let filterStatus = yield* utils.checkHostFilter(ctx, urlParsed.hostname); + if (0 !== filterStatus) { + ctx.logger.warn('Error downloadFile checkIpFilter error: url = %s', url); + res.sendStatus(filterStatus); + return; + } + + if (req.get('Range')) { + if (!headers) { + headers = {}; + } + headers['Range'] = req.get('Range'); } - } - const filterPrivate = !authorization || !tenAllowPrivateIPAddressForSignedRequests; - yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, filterPrivate, headers, res); + yield utils.downloadUrlPromise(ctx, url, tenDownloadTimeout, tenDownloadMaxBytes, authorization, isInJwtToken, headers, res); + } if (clientStatsD) { clientStatsD.timing('coauth.downloadFile', new Date() - startDate); } } catch (err) { - ctx.logger.error('Error downloadFile: %s', err.stack); - //catch errors because status may be sent while piping to response - try { - if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { - res.sendStatus(408); - } else if (err.code === 'EMSGSIZE') { - res.sendStatus(413); - } else if (err.response) { - res.sendStatus(err.response.statusCode); - } else { - res.sendStatus(400); - } - } catch (err) { + if (err.code === "ERR_STREAM_PREMATURE_CLOSE") { + ctx.logger.debug('Error downloadFile: %s', err.stack); + } else { ctx.logger.error('Error downloadFile: %s', err.stack); + //catch errors because status may be sent while piping to response + if (!res.headersSent) { + try { + if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { + res.sendStatus(408); + } else if (err.code === 'EMSGSIZE') { + res.sendStatus(413); + } else if (err.response) { + res.sendStatus(err.response.statusCode); + } else { + res.sendStatus(400); + } + } catch (err) { + ctx.logger.error('Error downloadFile: %s', err.stack); + } + } } } finally { @@ -1684,7 +1730,7 @@ exports.downloadFile = function(req, res) { } }); }; -exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_queue, opt_initShardKey) { +exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId, opt_userIndex, opt_userLcid, opt_queue, opt_initShardKey) { return co(function* () { try { var startDate = null; @@ -1701,6 +1747,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId } if (opt_initShardKey) { ctx.setShardKey(sqlBase.DocumentAdditional.prototype.getShardKey(row.additional)); + ctx.setWopiSrc(sqlBase.DocumentAdditional.prototype.getWopiSrc(row.additional)); } var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfc'); @@ -1709,7 +1756,9 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId cmd.setStatusInfoIn(statusInfo); cmd.setUserActionId(opt_userId); cmd.setUserActionIndex(opt_userIndex); - cmd.setJsonParams(getOpenedAtJSONParams(row)); + cmd.appendJsonParams(getOpenedAtJSONParams(row)); + //todo lang and region are different + cmd.setLCID(opt_userLcid); let userAuthStr = sqlBase.UserCallback.prototype.getCallbackByUserIndex(ctx, row.callback); cmd.setWopiParams(wopiClient.parseWopiCallback(ctx, userAuthStr, row.callback)); addPasswordToCmd(ctx, cmd, row && row.password); @@ -1719,7 +1768,7 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId queueData.setFromChanges(true); yield* docsCoServer.addTask(queueData, constants.QUEUE_PRIORITY_NORMAL, opt_queue); if (docsCoServer.getIsShutdown()) { - yield docsCoServer.editorData.addShutdown(redisKeyShutdown, docId); + yield docsCoServer.editorStat.addShutdown(redisKeyShutdown, docId); } ctx.logger.debug('AddTask saveFromChanges'); } else { @@ -1736,6 +1785,18 @@ exports.saveFromChanges = function(ctx, docId, statusInfo, optFormat, opt_userId } }); }; + +async function processWopiSaveAs(ctx, cmd) { + const info = await docsCoServer.getCallback(ctx, cmd.getDocId(), cmd.getUserIndex()); + // info.wopiParams is null if it is not wopi + if (info?.wopiParams) { + const suggestedTargetType = `.${formatChecker.getStringFromFormat(cmd.getOutputFormat())}`; + const storageFilePath = `${cmd.getSaveKey()}/${cmd.getOutputPath()}`; + const stream = await storage.createReadStream(ctx, storageFilePath); + const { wopiSrc, access_token } = info.wopiParams.userAuth; + await wopiClient.putRelativeFile(ctx, wopiSrc, access_token, null, stream.readStream, stream.contentLength, suggestedTargetType, false); + } +} exports.receiveTask = function(data, ack) { return co(function* () { let ctx = new operationContext.Context(); @@ -1746,29 +1807,33 @@ exports.receiveTask = function(data, ack) { ctx.initFromTaskQueueData(task); yield ctx.initTenantCache(); ctx.logger.info('receiveTask start: %s', data); - var updateTask = yield* getUpdateResponse(ctx, cmd); + var updateTask = yield getUpdateResponse(ctx, cmd); var updateRes = yield taskResult.update(ctx, updateTask); if (updateRes.affectedRows > 0) { var outputData = new OutputData(cmd.getCommand()); var command = cmd.getCommand(); var additionalOutput = {needUrlKey: null, needUrlMethod: null, needUrlType: null, needUrlIsCorrectPassword: undefined, creationDate: undefined, openedAt: undefined}; - if ('open' == command || 'reopen' == command) { + if ('open' === command || 'reopen' === command) { yield getOutputData(ctx, cmd, outputData, cmd.getDocId(), null, additionalOutput); - } else if ('save' == command || 'savefromorigin' == command || 'sfct' == command) { - yield getOutputData(ctx, cmd, outputData, cmd.getSaveKey(), null, additionalOutput); - } else if ('sfcm' == command) { + } else if ('save' === command || 'savefromorigin' === command) { + let status = yield getOutputData(ctx, cmd, outputData, cmd.getSaveKey(), null, additionalOutput); + if (commonDefines.FileStatus.Ok === status && cmd.getIsSaveAs()) { + yield processWopiSaveAs(ctx, cmd); + //todo in case of wopi no need to send url. send it to avoid stubs in sdk + } + } else if ('sfcm' === command) { yield commandSfcCallback(ctx, cmd, true); - } else if ('sfc' == command) { + } else if ('sfc' === command) { yield commandSfcCallback(ctx, cmd, false); - } else if ('sendmm' == command) { + } else if ('sendmm' === command) { yield* commandSendMMCallback(ctx, cmd); - } else if ('conv' == command) { + } else if ('conv' === command) { //nothing } if (outputData.getStatus()) { ctx.logger.debug('receiveTask publish: %s', JSON.stringify(outputData)); var output = new OutputDataWrap('documentOpen', outputData); - yield* docsCoServer.publish(ctx, { + yield docsCoServer.publish(ctx, { type: commonDefines.c_oPublishType.receiveTask, ctx: ctx, cmd: cmd, output: output, needUrlKey: additionalOutput.needUrlKey, needUrlMethod: additionalOutput.needUrlMethod, @@ -1791,6 +1856,7 @@ exports.receiveTask = function(data, ack) { exports.cleanupCache = cleanupCache; exports.cleanupCacheIf = cleanupCacheIf; +exports.cleanupErrToReload = cleanupErrToReload; exports.getOpenedAt = getOpenedAt; exports.commandSfctByCmd = commandSfctByCmd; exports.commandOpenStartPromise = commandOpenStartPromise; diff --git a/DocService/sources/changes2forgotten.js b/DocService/sources/changes2forgotten.js index 483013546..75d48873f 100644 --- a/DocService/sources/changes2forgotten.js +++ b/DocService/sources/changes2forgotten.js @@ -41,10 +41,12 @@ var utils = require('./../../Common/sources/utils'); const storage = require('./../../Common/sources/storage-base'); const queueService = require('./../../Common/sources/taskqueueRabbitMQ'); const operationContext = require('./../../Common/sources/operationContext'); -const sqlBase = require('./baseConnector'); +const sqlBase = require('./databaseConnectors/baseConnector'); const docsCoServer = require('./DocsCoServer'); const taskResult = require('./taskresult'); -const editorDataStorage = require('./' + config.get('services.CoAuthoring.server.editorDataStorage')); +const cfgEditorDataStorage = config.get('services.CoAuthoring.server.editorDataStorage'); +const cfgEditorStatStorage = config.get('services.CoAuthoring.server.editorStatStorage'); +const editorStatStorage = require('./' + (cfgEditorStatStorage || cfgEditorDataStorage)); const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles'); const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); @@ -56,7 +58,7 @@ var WAIT_TIMEOUT = 30000; var LOOP_TIMEOUT = 1000; var EXEC_TIMEOUT = WAIT_TIMEOUT + utils.getConvertionTimeout(undefined); -let addSqlParam = sqlBase.baseConnector.addSqlParameter; +let addSqlParam = sqlBase.addSqlParameter; function updateDoc(ctx, docId, status, callback) { return new Promise(function(resolve, reject) { @@ -66,7 +68,7 @@ function updateDoc(ctx, docId, status, callback) { let p3 = addSqlParam(ctx.tenant, values); let p4 = addSqlParam(docId, values); let sqlCommand = `UPDATE ${cfgTableResult} SET status=${p1},callback=${p2} WHERE tenant=${p3} AND id=${p4};`; - sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { + sqlBase.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); } else { @@ -81,7 +83,7 @@ function shutdown() { var res = true; let ctx = new operationContext.Context(); try { - let editorData = new editorDataStorage(); + let editorStat = editorStatStorage.EditorStat ? new editorStatStorage.EditorStat() : new editorStatStorage(); ctx.logger.debug('shutdown start:' + EXEC_TIMEOUT); //redisKeyShutdown is not a simple counter, so it doesn't get decremented by a build that started before Shutdown started @@ -130,9 +132,9 @@ function shutdown() { yield ctx.initTenantCache(); yield updateDoc(ctx, docId, commonDefines.FileStatus.Ok, ""); - yield editorData.addShutdown(redisKeyShutdown, docId); + yield editorStat.addShutdown(redisKeyShutdown, docId); ctx.logger.debug('shutdown createSaveTimerPromise %s', docId); - yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true); + yield docsCoServer.createSaveTimer(ctx, docId, null, null, null, queue, true); } ctx.initDefault(); //sleep because of bugs in createSaveTimerPromise @@ -140,7 +142,7 @@ function shutdown() { let startTime = new Date().getTime(); while (true) { - let remainingFiles = yield editorData.getShutdownCount(redisKeyShutdown); + let remainingFiles = yield editorStat.getShutdownCount(redisKeyShutdown); ctx.logger.debug('shutdown remaining files:%d', remainingFiles); let curTime = new Date().getTime() - startTime; if (curTime >= EXEC_TIMEOUT || remainingFiles <= 0) { @@ -169,7 +171,7 @@ function shutdown() { //todo needs to check queues, because there may be long conversions running before Shutdown //clean up - yield editorData.cleanupShutdown(redisKeyShutdown); + yield editorStat.cleanupShutdown(redisKeyShutdown); yield pubsub.close(); yield queue.close(); diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index 9e2faf6da..2cf7781fc 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -35,7 +35,6 @@ const path = require('path'); var config = require('config'); var co = require('co'); -const locale = require('windows-locale'); const mime = require('mime'); var taskResult = require('./taskresult'); var utils = require('./../../Common/sources/utils'); @@ -49,7 +48,8 @@ var formatChecker = require('./../../Common/sources/formatchecker'); var statsDClient = require('./../../Common/sources/statsdclient'); var storageBase = require('./../../Common/sources/storage-base'); var operationContext = require('./../../Common/sources/operationContext'); -const sqlBase = require('./baseConnector'); +const sqlBase = require('./databaseConnectors/baseConnector'); +const utilsDocService = require("./utilsDocService"); const cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.browser'); @@ -83,12 +83,12 @@ function* getConvertStatus(ctx, docId, encryptedUserPassword, selectRes, opt_che } break; case commonDefines.FileStatus.Err: + status.err = row.status_info; + break; case commonDefines.FileStatus.ErrToReload: case commonDefines.FileStatus.NeedPassword: status.err = row.status_info; - if (commonDefines.FileStatus.ErrToReload == row.status || commonDefines.FileStatus.NeedPassword == row.status) { - yield canvasService.cleanupCache(ctx, docId); - } + yield canvasService.cleanupErrToReload(ctx, docId); break; case commonDefines.FileStatus.NeedParams: case commonDefines.FileStatus.SaveVersion: @@ -139,17 +139,16 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, task.status = commonDefines.FileStatus.WaitQueue; task.statusInfo = constants.NO_ERROR; - let upsertRes = yield taskResult.upsert(ctx, task); - //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values - //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html - bCreate = upsertRes.affectedRows == 1; + const upsertRes = yield taskResult.upsert(ctx, task); + bCreate = upsertRes.isInsert; } var selectRes; var status; if (!bCreate) { selectRes = yield taskResult.select(ctx, docId); status = yield* getConvertStatus(ctx, cmd.getDocId() ,cmd.getPassword(), selectRes, opt_checkPassword); - } else { + } + if (bCreate || (commonDefines.FileStatus.None === selectRes?.[0]?.status)) { var queueData = new commonDefines.TaskQueueData(); queueData.setCtx(ctx); queueData.setCmd(cmd); @@ -184,8 +183,9 @@ function* convertByCmd(ctx, cmd, async, opt_fileTo, opt_taskExist, opt_priority, return status; } -async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChangeInfo, opt_userdata, opt_formdata, opt_userConnectionId, - opt_userConnectionDocId, opt_responseKey, opt_priority, opt_expiration, opt_queue, opt_redisKey, opt_initShardKey) { +async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChangeInfo, opt_userdata, opt_formdata, + opt_userConnectionId, opt_userConnectionDocId, opt_responseKey, opt_priority, + opt_expiration, opt_queue, opt_redisKey, opt_initShardKey, opt_jsonParams) { var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfcm'); cmd.setDocId(docId); @@ -195,6 +195,10 @@ async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChange cmd.setDelimiter(commonDefines.c_oAscCsvDelimiter.Comma); cmd.setForceSave(forceSave); cmd.setExternalChangeInfo(externalChangeInfo); + if (externalChangeInfo.lang) { + //todo lang and region are different + cmd.setLCID(utilsDocService.localeToLCID(externalChangeInfo.lang)); + } if (opt_userdata) { cmd.setUserData(opt_userdata); } @@ -214,8 +218,14 @@ async function convertFromChanges(ctx, docId, baseUrl, forceSave, externalChange if (opt_redisKey) { cmd.setRedisKey(opt_redisKey); } + if (opt_jsonParams) { + cmd.appendJsonParams(opt_jsonParams); + } - await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey); + let commandSfctByCmdRes = await canvasService.commandSfctByCmd(ctx, cmd, opt_priority, opt_expiration, opt_queue, opt_initShardKey); + if (!commandSfctByCmdRes) { + return new commonDefines.ConvertStatus(constants.UNKNOWN); + } var fileTo = constants.OUTPUT_NAME; let outputExt = formatChecker.getStringFromFormat(cmd.getOutputFormat()); if (outputExt) { @@ -251,8 +261,7 @@ function convertRequest(req, res, isJson) { } let filetype = params.filetype || params.fileType || ''; let outputtype = params.outputtype || params.outputType || ''; - let docId = 'conv_' + params.key + '_' + outputtype; - ctx.setDocId(docId); + ctx.setDocId(params.key); if (params.key && !constants.DOC_ID_REGEX.test(params.key)) { ctx.logger.warn('convertRequest unexpected key = %s', params.key); @@ -270,6 +279,19 @@ function convertRequest(req, res, isJson) { utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.CONVERT_PARAMS), isJson); return; } + if (params.pdf) { + if (true === params.pdf.pdfa && constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === outputFormat) { + outputFormat = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA; + } else if (false === params.pdf.pdfa && constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === outputFormat) { + outputFormat = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF; + } + if (params.pdf.form && (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === outputFormat || + constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === outputFormat)) { + outputFormat = constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM_PDF; + } + } + //todo use hash of params as id + let docId = 'conv_' + params.key + '_' + outputFormat; var cmd = new commonDefines.InputCommand(); cmd.setCommand('conv'); cmd.setUrl(params.url); @@ -283,8 +305,8 @@ function convertRequest(req, res, isJson) { cmd.setDelimiter(parseIntParam(params.delimiter) || commonDefines.c_oAscCsvDelimiter.Comma); if(undefined != params.delimiterChar) cmd.setDelimiterChar(params.delimiterChar); - if (params.region && locale[params.region.toLowerCase()]) { - cmd.setLCID(locale[params.region.toLowerCase()].id); + if (params.region) { + cmd.setLCID(utilsDocService.localeToLCID(params.region)); } let jsonParams = {}; if (params.documentLayout) { @@ -297,7 +319,7 @@ function convertRequest(req, res, isJson) { jsonParams['watermark'] = params.watermark; } if (Object.keys(jsonParams).length > 0) { - cmd.setJsonParams(JSON.stringify(jsonParams)); + cmd.appendJsonParams(jsonParams); } if (params.password) { if (params.password.length > constants.PASSWORD_MAX_LENGTH) { @@ -517,22 +539,22 @@ function convertTo(req, res) { cmd.setOutputFormat(outputFormat); cmd.setCodepage(commonDefines.c_oAscCodePageUtf8); cmd.setDelimiter(commonDefines.c_oAscCsvDelimiter.Comma); - if (lang && locale[lang.toLowerCase()]) { - cmd.setLCID(locale[lang.toLowerCase()].id); + if (lang) { + cmd.setLCID(utilsDocService.localeToLCID(lang)); } if (fullSheetPreview) { - cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': { + cmd.appendJsonParams({'spreadsheetLayout': { "ignorePrintArea": true, "fitToWidth": 1, "fitToHeight": 1 - }})); + }}); } else { - cmd.setJsonParams(JSON.stringify({'spreadsheetLayout': { + cmd.appendJsonParams({'spreadsheetLayout': { "ignorePrintArea": true, "fitToWidth": 0, "fitToHeight": 0, "scale": 100 - }})); + }}); } fileTo = constants.OUTPUT_NAME; @@ -630,7 +652,7 @@ function getConverterHtmlHandler(req, res) { ctx.setDocId(docId); if (!(wopiSrc && access_token && access_token && targetext && docId) || constants.AVS_OFFICESTUDIO_FILE_UNKNOWN === formatChecker.getFormatFromString(targetext)) { - ctx.logger.debug('convert-and-edit-handler invalid params: wopiSrc=%s; access_token=%s; targetext=%s; docId=%s', wopiSrc, access_token, targetext, docId); + ctx.logger.debug('convert-and-edit-handler invalid params: WOPISrc=%s; access_token=%s; targetext=%s; docId=%s', wopiSrc, access_token, targetext, docId); utils.fillResponse(req, res, new commonDefines.ConvertStatus(constants.CONVERT_PARAMS), isJson); return; } diff --git a/DocService/sources/damengBaseConnector.js b/DocService/sources/damengBaseConnector.js deleted file mode 100644 index 815cb6201..000000000 --- a/DocService/sources/damengBaseConnector.js +++ /dev/null @@ -1,200 +0,0 @@ -/* - * (c) Copyright Ascensio System SIA 2010-2023 - * - * This program is a free software product. You can redistribute it and/or - * modify it under the terms of the GNU Affero General Public License (AGPL) - * version 3 as published by the Free Software Foundation. In accordance with - * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect - * that Ascensio System SIA expressly excludes the warranty of non-infringement - * of any third-party rights. - * - * This program is distributed WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For - * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html - * - * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish - * street, Riga, Latvia, EU, LV-1050. - * - * The interactive user interfaces in modified source and object code versions - * of the Program must display Appropriate Legal Notices, as required under - * Section 5 of the GNU AGPL version 3. - * - * Pursuant to Section 7(b) of the License you must retain the original Product - * logo when distributing the program. Pursuant to Section 7(e) we decline to - * grant you any rights under trademark law for use of our trademarks. - * - * All the Product's GUI elements, including illustrations and icon sets, as - * well as technical writing content are licensed under the terms of the - * Creative Commons Attribution-ShareAlike 4.0 International. See the License - * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode - * - */ - -'use strict'; - -const co = require('co'); -const connectorUtilities = require('./connectorUtilities'); -const db = require("dmdb"); -const config = require('config'); - -const cfgDbHost = config.get('services.CoAuthoring.sql.dbHost'); -const cfgDbPort = config.get('services.CoAuthoring.sql.dbPort'); -const cfgDbUser = config.get('services.CoAuthoring.sql.dbUser'); -const cfgDbPass = config.get('services.CoAuthoring.sql.dbPass'); -const cfgConnectionlimit = config.get('services.CoAuthoring.sql.connectionlimit'); -const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); -var cfgDamengExtraOptions = config.get('services.CoAuthoring.sql.damengExtraOptions'); - -let pool = null; -let connectString = `dm://${cfgDbUser}:${cfgDbPass}@${cfgDbHost}:${cfgDbPort}`; -let connectionConfig = { - connectString: connectString, - poolMax: cfgConnectionlimit, - poolMin: 0, - localTimezone: 0 -}; -config.util.extendDeep(connectionConfig, cfgDamengExtraOptions); - -function readLob(lob) { - return new Promise(function(resolve, reject) { - var blobData = Buffer.alloc(0); - var totalLength = 0; - lob.on('data', function(chunk) { - totalLength += chunk.length; - blobData = Buffer.concat([blobData, chunk], totalLength); - }); - lob.on('error', function(err) { - reject(err); - }); - lob.on('end', function() { - resolve(blobData); - }); - }); -} -function formatResult(result) { - return co(function *() { - let res = []; - if (result?.rows && result ?.metaData) { - for (let i = 0; i < result.rows.length; ++i) { - let row = result.rows[i]; - let out = {}; - for (let j = 0; j < result.metaData.length; ++j) { - let columnName = result.metaData[j].name.toLowerCase(); - if (row[j]?.on) { - let buf = yield readLob(row[j]); - out[columnName] = buf.toString('utf8'); - } else { - out[columnName] = row[j]; - } - } - res.push(out); - } - } - return res; - }); -} -exports.sqlQuery = function(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) { - return co(function *() { - var result = null; - var output = null; - var error = null; - try { - if (!pool) { - pool = yield db.createPool(connectionConfig); - } - let conn = yield pool.getConnection(); - result = yield conn.execute(sqlCommand, opt_values, {resultSet: false}); - if (conn) { - yield conn.close(); - } - output = result; - if (!opt_noModifyRes) { - if (result?.rows) { - output = yield formatResult(result); - } else if (result?.rowsAffected) { - output = {affectedRows: result.rowsAffected}; - } else { - output = {rows: [], affectedRows: 0}; - } - } - } catch (err) { - error = err; - if (!opt_noLog) { - ctx.logger.warn('sqlQuery error sqlCommand: %s: %s', sqlCommand.slice(0, 50), err.stack); - } - } finally { - if (callbackFunction) { - callbackFunction(error, output); - } - } - }); -}; -let addSqlParam = function (val, values) { - values.push({val: val}); - return ':' + values.length; -}; -exports.addSqlParameter = addSqlParam; -let concatParams = function (val1, val2) { - return `CONCAT(COALESCE(${val1}, ''), COALESCE(${val2}, ''))`; -}; -exports.concatParams = concatParams; - -exports.upsert = function(ctx, task, opt_updateUserIndex) { - return new Promise(function(resolve, reject) { - task.completeDefaults(); - let dateNow = new Date(); - let values = []; - let cbInsert = task.callback; - if (task.callback) { - let userCallback = new connectorUtilities.UserCallback(); - userCallback.fromValues(task.userIndex, task.callback); - cbInsert = userCallback.toSQLInsert(); - } - let p0 = addSqlParam(task.tenant, values); - let p1 = addSqlParam(task.key, values); - let p2 = addSqlParam(task.status, values); - let p3 = addSqlParam(task.statusInfo, values); - let p4 = addSqlParam(dateNow, values); - let p5 = addSqlParam(task.userIndex, values); - let p6 = addSqlParam(task.changeId, values); - let p7 = addSqlParam(cbInsert, values); - let p8 = addSqlParam(task.baseurl, values); - let p9 = addSqlParam(dateNow, values); - var sqlCommand = `MERGE INTO ${cfgTableResult} USING dual ON (tenant = ${p0} AND id = ${p1}) `; - sqlCommand += `WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) `; - sqlCommand += `VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}) `; - sqlCommand += `WHEN MATCHED THEN UPDATE SET last_open_date = ${p9}`; - if (task.callback) { - let p10 = addSqlParam(JSON.stringify(task.callback), values); - sqlCommand += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${p10}, '}')`; - } - if (task.baseurl) { - let p11 = addSqlParam(task.baseurl, values); - sqlCommand += `, baseurl = ${p11}`; - } - if (opt_updateUserIndex) { - sqlCommand += ', user_index = user_index + 1'; - } - sqlCommand += ';'; - sqlCommand += `SELECT user_index FROM ${cfgTableResult} WHERE tenant = ${p0} AND id = ${p1};`; - exports.sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - let out = {affectedRows: 0, insertId: 0}; - if (result?.length > 0) { - var first = result[0]; - out.affectedRows = task.userIndex !== first.user_index ? 2 : 1; - out.insertId = first.user_index; - } - resolve(out); - } - }, undefined, undefined, values); - }); -}; -exports.getTableColumns = function(ctx, tableName) { - //todo - return new Promise(function(resolve, reject) { - resolve([]); - }); -}; diff --git a/DocService/sources/baseConnector.js b/DocService/sources/databaseConnectors/baseConnector.js similarity index 56% rename from DocService/sources/baseConnector.js rename to DocService/sources/databaseConnectors/baseConnector.js index 92e17bd53..67626520c 100644 --- a/DocService/sources/baseConnector.js +++ b/DocService/sources/databaseConnectors/baseConnector.js @@ -1,371 +1,382 @@ -/* - * (c) Copyright Ascensio System SIA 2010-2023 - * - * This program is a free software product. You can redistribute it and/or - * modify it under the terms of the GNU Affero General Public License (AGPL) - * version 3 as published by the Free Software Foundation. In accordance with - * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect - * that Ascensio System SIA expressly excludes the warranty of non-infringement - * of any third-party rights. - * - * This program is distributed WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For - * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html - * - * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish - * street, Riga, Latvia, EU, LV-1050. - * - * The interactive user interfaces in modified source and object code versions - * of the Program must display Appropriate Legal Notices, as required under - * Section 5 of the GNU AGPL version 3. - * - * Pursuant to Section 7(b) of the License you must retain the original Product - * logo when distributing the program. Pursuant to Section 7(e) we decline to - * grant you any rights under trademark law for use of our trademarks. - * - * All the Product's GUI elements, including illustrations and icon sets, as - * well as technical writing content are licensed under the terms of the - * Creative Commons Attribution-ShareAlike 4.0 International. See the License - * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode - * - */ - -'use strict'; - -var sqlDataBaseType = { - mySql : 'mysql', - mariaDB : 'mariadb', - msSql : 'mssql', - postgreSql : 'postgres', - dameng : 'dameng', - oracle: 'oracle' -}; - -const connectorUtilities = require('./connectorUtilities'); -const utils = require('./../../Common/sources/utils'); -var bottleneck = require("bottleneck"); -var config = require('config'); -var configSql = config.get('services.CoAuthoring.sql'); -const dbType = configSql.get('type'); - -let baseConnector; -switch (dbType) { - case sqlDataBaseType.mySql: - case sqlDataBaseType.mariaDB: - baseConnector = require('./mySqlBaseConnector'); - break; - case sqlDataBaseType.msSql: - baseConnector = require('./msSqlServerConnector'); - break; - case sqlDataBaseType.dameng: - baseConnector = require('./damengBaseConnector'); - break; - case sqlDataBaseType.oracle: - baseConnector = require('./oracleBaseConnector'); - break; - default: - baseConnector = require('./postgreSqlBaseConnector'); - break; -} - -const cfgTableResult = configSql.get('tableResult'); -const cfgTableChanges = configSql.get('tableChanges'); - -var g_oCriticalSection = {}; -let isSupportFastInsert = !!baseConnector.insertChanges; -let addSqlParam = baseConnector.addSqlParameter; -var maxPacketSize = configSql.get('max_allowed_packet'); // The default size for a query to the database is 1Mb - 1 (because it does not write 1048575, but writes 1048574) -const cfgBottleneckGetChanges = config.get('bottleneck.getChanges'); - -let reservoirMaximum = cfgBottleneckGetChanges.reservoirIncreaseMaximum || cfgBottleneckGetChanges.reservoirRefreshAmount; -let group = new bottleneck.Group(cfgBottleneckGetChanges); - -function getChangesSize(changes) { - return changes.reduce((accumulator, currentValue) => accumulator + currentValue.change_data.length, 0); -} - -exports.baseConnector = baseConnector; -exports.insertChangesPromiseCompatibility = function (ctx, objChanges, docId, index, user) { - return new Promise(function(resolve, reject) { - _insertChangesCallback(ctx, 0, objChanges, docId, index, user, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); -}; -exports.insertChangesPromiseFast = function (ctx, objChanges, docId, index, user) { - return new Promise(function(resolve, reject) { - baseConnector.insertChanges(ctx, cfgTableChanges, 0, objChanges, docId, index, user, function(error, result, isSupported) { - isSupportFastInsert = isSupported; - if (error) { - if (!isSupportFastInsert) { - resolve(exports.insertChangesPromiseCompatibility(ctx, objChanges, docId, index, user)); - } else { - reject(error); - } - } else { - resolve(result); - } - }); - }); -}; -exports.insertChangesPromise = function (ctx, objChanges, docId, index, user) { - if (isSupportFastInsert) { - return exports.insertChangesPromiseFast(ctx, objChanges, docId, index, user); - } else { - return exports.insertChangesPromiseCompatibility(ctx, objChanges, docId, index, user); - } - -}; -function _getDateTime2(oDate) { - return oDate.toISOString().slice(0, 19).replace('T', ' '); -} - -exports.getDateTime = _getDateTime2; - -function _insertChangesCallback (ctx, startIndex, objChanges, docId, index, user, callback) { - var sqlCommand = `INSERT INTO ${cfgTableChanges} VALUES`; - var i = startIndex, l = objChanges.length, lengthUtf8Current = sqlCommand.length, lengthUtf8Row = 0, values = []; - if (i === l) - return; - - const indexBytes = 4; - const timeBytes = 8; - for (; i < l; ++i, ++index) { - //49 - length of "($1001,... $1008)," - //4 is max utf8 bytes per symbol - lengthUtf8Row = 49 + 4 * (ctx.tenant.length + docId.length + user.id.length + user.idOriginal.length + user.username.length + objChanges[i].change.length) + indexBytes + timeBytes; - if (lengthUtf8Row + lengthUtf8Current >= maxPacketSize && i > startIndex) { - sqlCommand += ';'; - (function(tmpStart, tmpIndex) { - baseConnector.sqlQuery(ctx, sqlCommand, function() { - // do not remove lock, but we continue to add - _insertChangesCallback(ctx, tmpStart, objChanges, docId, tmpIndex, user, callback); - }, undefined, undefined, values); - })(i, index); - return; - } - let p0 = addSqlParam(ctx.tenant, values); - let p1 = addSqlParam(docId, values); - let p2 = addSqlParam(index, values); - let p3 = addSqlParam(user.id, values); - let p4 = addSqlParam(user.idOriginal, values); - let p5 = addSqlParam(user.username, values); - let p6 = addSqlParam(objChanges[i].change, values); - let p7 = addSqlParam(objChanges[i].time, values); - if (i > startIndex) { - sqlCommand += ','; - } - sqlCommand += `(${p0},${p1},${p2},${p3},${p4},${p5},${p6},${p7})`; - lengthUtf8Current += lengthUtf8Row; - } - - sqlCommand += ';'; - baseConnector.sqlQuery(ctx, sqlCommand, callback, undefined, undefined, values); -} -exports.deleteChangesCallback = function(ctx, docId, deleteIndex, callback) { - let sqlCommand, values = []; - let p1 = addSqlParam(ctx.tenant, values); - let p2 = addSqlParam(docId, values); - if (null !== deleteIndex) { - let sqlParam2 = addSqlParam(deleteIndex, values); - sqlCommand = `DELETE FROM ${cfgTableChanges} WHERE tenant=${p1} AND id=${p2} AND change_id >= ${sqlParam2};`; - } else { - sqlCommand = `DELETE FROM ${cfgTableChanges} WHERE tenant=${p1} AND id=${p2};`; - } - baseConnector.sqlQuery(ctx, sqlCommand, callback, undefined, undefined, values); -}; -exports.deleteChangesPromise = function (ctx, docId, deleteIndex) { - return new Promise(function(resolve, reject) { - exports.deleteChangesCallback(ctx, docId, deleteIndex, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); -}; -exports.deleteChanges = function (ctx, docId, deleteIndex) { - lockCriticalSection(docId, function () {_deleteChanges(ctx, docId, deleteIndex);}); -}; -function _deleteChanges (ctx, docId, deleteIndex) { - exports.deleteChangesCallback(ctx, docId, deleteIndex, function () {unLockCriticalSection(docId);}); -} -exports.getChangesIndex = function(ctx, docId, callback) { - let values = []; - let p1 = addSqlParam(ctx.tenant, values); - let p2 = addSqlParam(docId, values); - var sqlCommand = `SELECT MAX(change_id) as change_id FROM ${cfgTableChanges} WHERE tenant=${p1} AND id=${p2};`; - baseConnector.sqlQuery(ctx, sqlCommand, callback, undefined, undefined, values); -}; -exports.getChangesIndexPromise = function(ctx, docId) { - return new Promise(function(resolve, reject) { - exports.getChangesIndex(ctx, docId, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); -}; -exports.getChangesPromise = function (ctx, docId, optStartIndex, optEndIndex, opt_time) { - let limiter = group.key(`${ctx.tenant}\t${docId}\tchanges`); - return limiter.schedule(() => { - return new Promise(function(resolve, reject) { - let values = []; - let sqlParam = addSqlParam(ctx.tenant, values); - let sqlWhere = `tenant=${sqlParam}`; - sqlParam = addSqlParam(docId, values); - sqlWhere += ` AND id=${sqlParam}`; - if (null != optStartIndex) { - sqlParam = addSqlParam(optStartIndex, values); - sqlWhere += ` AND change_id>=${sqlParam}`; - } - if (null != optEndIndex) { - sqlParam = addSqlParam(optEndIndex, values); - sqlWhere += ` AND change_id<${sqlParam}`; - } - if (null != opt_time) { - if (!(opt_time instanceof Date)) { - opt_time = new Date(opt_time); - } - sqlParam = addSqlParam(opt_time, values); - sqlWhere += ` AND change_date<=${sqlParam}`; - } - sqlWhere += ' ORDER BY change_id ASC'; - var sqlCommand = `SELECT * FROM ${cfgTableChanges} WHERE ${sqlWhere};`; - - baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - if (reservoirMaximum > 0) { - let size = Math.min(getChangesSize(result), reservoirMaximum); - let cur = limiter.incrementReservoir(-size).then((cur) => { - ctx.logger.debug("getChangesPromise bottleneck reservoir cur=%s", cur); - resolve(result); - }); - } else { - resolve(result); - } - } - }, undefined, undefined, values); - }); - }); -}; -exports.getDocumentsWithChanges = baseConnector.getDocumentsWithChanges ?? function (ctx) { - return new Promise(function(resolve, reject) { - const sqlCommand = `SELECT * FROM ${cfgTableResult} WHERE EXISTS(SELECT id FROM ${cfgTableChanges} WHERE tenant=${cfgTableResult}.tenant AND id = ${cfgTableResult}.id LIMIT 1);`; - baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }, false, false); - }); -} -exports.getExpired = baseConnector.getExpired ?? function(ctx, maxCount, expireSeconds) { - return new Promise(function(resolve, reject) { - const values = []; - const expireDate = new Date(); - utils.addSeconds(expireDate, -expireSeconds); - const date = addSqlParam(expireDate, values); - const count = addSqlParam(maxCount, values); - const sqlCommand = `SELECT tenant, id FROM ${cfgTableResult} WHERE last_open_date <= ${date}` + - ` AND NOT EXISTS(SELECT tenant, id FROM ${cfgTableChanges} WHERE ${cfgTableChanges}.tenant = ${cfgTableResult}.tenant AND ${cfgTableChanges}.id = ${cfgTableResult}.id LIMIT 1) LIMIT ${count};`; - baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }, false, false, values); - }); -} - -exports.isLockCriticalSection = function (id) { - return !!(g_oCriticalSection[id]); -}; - -// critical section -function lockCriticalSection (id, callback) { - if (g_oCriticalSection[id]) { - // wait - g_oCriticalSection[id].push(callback); - return; - } - // lock - g_oCriticalSection[id] = []; - g_oCriticalSection[id].push(callback); - callback(); -} -function unLockCriticalSection (id) { - var arrCallbacks = g_oCriticalSection[id]; - arrCallbacks.shift(); - if (0 < arrCallbacks.length) - arrCallbacks[0](); - else - delete g_oCriticalSection[id]; -} -exports.healthCheck = function (ctx) { - return new Promise(function(resolve, reject) { - //SELECT 1; usefull for H2, MySQL, Microsoft SQL Server, PostgreSQL, SQLite - //http://stackoverflow.com/questions/3668506/efficient-sql-test-query-or-validation-query-that-will-work-across-all-or-most - let sql; - switch (dbType) { - case sqlDataBaseType.oracle: { - sql = 'SELECT 1 FROM DUAL'; - break; - } - default: { - sql = 'SELECT 1;'; - } - } - - baseConnector.sqlQuery(ctx, sql, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); -}; - -exports.getEmptyCallbacks = baseConnector.getEmptyCallbacks ?? function(ctx) { - return new Promise(function(resolve, reject) { - const sqlCommand = `SELECT DISTINCT t1.tenant, t1.id FROM ${cfgTableChanges} t1 LEFT JOIN ${cfgTableResult} t2 ON t2.tenant = t1.tenant AND t2.id = t1.id WHERE t2.callback = '';`; - baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); -}; -exports.getTableColumns = function(ctx, tableName) { - if (baseConnector.getTableColumns) { - return baseConnector.getTableColumns(ctx, tableName); - } else { - return new Promise(function(resolve, reject) { - const sqlCommand = `SELECT column_name FROM information_schema.COLUMNS WHERE TABLE_NAME = '${tableName}';`; - baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); - } -}; -exports.UserCallback = connectorUtilities.UserCallback; -exports.DocumentPassword = connectorUtilities.DocumentPassword; -exports.DocumentAdditional = connectorUtilities.DocumentAdditional; \ No newline at end of file +/* + * (c) Copyright Ascensio System SIA 2010-2023 + * + * This program is a free software product. You can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License (AGPL) + * version 3 as published by the Free Software Foundation. In accordance with + * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect + * that Ascensio System SIA expressly excludes the warranty of non-infringement + * of any third-party rights. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For + * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html + * + * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish + * street, Riga, Latvia, EU, LV-1050. + * + * The interactive user interfaces in modified source and object code versions + * of the Program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU AGPL version 3. + * + * Pursuant to Section 7(b) of the License you must retain the original Product + * logo when distributing the program. Pursuant to Section 7(e) we decline to + * grant you any rights under trademark law for use of our trademarks. + * + * All the Product's GUI elements, including illustrations and icon sets, as + * well as technical writing content are licensed under the terms of the + * Creative Commons Attribution-ShareAlike 4.0 International. See the License + * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + * + */ + +'use strict'; + +const sqlDataBaseType = { + mySql : 'mysql', + mariaDB : 'mariadb', + msSql : 'mssql', + postgreSql : 'postgres', + dameng : 'dameng', + oracle : 'oracle' +}; + +const connectorUtilities = require('./connectorUtilities'); +const utils = require('../../../Common/sources/utils'); +const bottleneck = require('bottleneck'); +const config = require('config'); + +const configSql = config.get('services.CoAuthoring.sql'); +const cfgTableResult = configSql.get('tableResult'); +const cfgTableChanges = configSql.get('tableChanges'); +const maxPacketSize = configSql.get('max_allowed_packet'); // The default size for a query to the database is 1Mb - 1 (because it does not write 1048575, but writes 1048574) +const cfgBottleneckGetChanges = config.get('bottleneck.getChanges'); +const dbType = configSql.get('type'); + +const reservoirMaximum = cfgBottleneckGetChanges.reservoirIncreaseMaximum || cfgBottleneckGetChanges.reservoirRefreshAmount; +const group = new bottleneck.Group(cfgBottleneckGetChanges); +const g_oCriticalSection = {}; + +let dbInstance; +switch (dbType) { + case sqlDataBaseType.mySql: + case sqlDataBaseType.mariaDB: + dbInstance = require('./mysqlConnector'); + break; + case sqlDataBaseType.msSql: + dbInstance = require('./mssqlConnector'); + break; + case sqlDataBaseType.dameng: + dbInstance = require('./damengConnector'); + break; + case sqlDataBaseType.oracle: + dbInstance = require('./oracleConnector'); + break; + default: + dbInstance = require('./postgreConnector'); + break; +} + +let isSupportFastInsert = !!dbInstance.insertChanges; +const addSqlParameter = dbInstance.addSqlParameter; + +function getChangesSize(changes) { + return changes.reduce((accumulator, currentValue) => accumulator + currentValue.change_data.length, 0); +} + +function insertChangesPromiseCompatibility(ctx, objChanges, docId, index, user) { + return new Promise(function(resolve, reject) { + _insertChangesCallback(ctx, 0, objChanges, docId, index, user, function(error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); +} + +function insertChangesPromiseFast(ctx, objChanges, docId, index, user) { + return new Promise(function(resolve, reject) { + dbInstance.insertChanges(ctx, cfgTableChanges, 0, objChanges, docId, index, user, function(error, result, isSupported) { + isSupportFastInsert = isSupported; + if (error) { + if (!isSupportFastInsert) { + resolve(insertChangesPromiseCompatibility(ctx, objChanges, docId, index, user)); + } else { + reject(error); + } + } else { + resolve(result); + } + }); + }); +} + +function insertChangesPromise(ctx, objChanges, docId, index, user) { + if (isSupportFastInsert) { + return insertChangesPromiseFast(ctx, objChanges, docId, index, user); + } else { + return insertChangesPromiseCompatibility(ctx, objChanges, docId, index, user); + } +} + +function _getDateTime2(oDate) { + return oDate.toISOString().slice(0, 19).replace('T', ' '); +} + +function _insertChangesCallback(ctx, startIndex, objChanges, docId, index, user, callback) { + var sqlCommand = `INSERT INTO ${cfgTableChanges} VALUES`; + var i = startIndex, l = objChanges.length, lengthUtf8Current = sqlCommand.length, lengthUtf8Row = 0, values = []; + if (i === l) + return; + + const indexBytes = 4; + const timeBytes = 8; + for (; i < l; ++i, ++index) { + //49 - length of "($1001,... $1008)," + //4 is max utf8 bytes per symbol + lengthUtf8Row = 49 + 4 * (ctx.tenant.length + docId.length + user.id.length + user.idOriginal.length + user.username.length + objChanges[i].change.length) + indexBytes + timeBytes; + if (lengthUtf8Row + lengthUtf8Current >= maxPacketSize && i > startIndex) { + sqlCommand += ';'; + (function(tmpStart, tmpIndex) { + dbInstance.sqlQuery(ctx, sqlCommand, function() { + // do not remove lock, but we continue to add + _insertChangesCallback(ctx, tmpStart, objChanges, docId, tmpIndex, user, callback); + }, undefined, undefined, values); + })(i, index); + return; + } + let p0 = addSqlParameter(ctx.tenant, values); + let p1 = addSqlParameter(docId, values); + let p2 = addSqlParameter(index, values); + let p3 = addSqlParameter(user.id, values); + let p4 = addSqlParameter(user.idOriginal, values); + let p5 = addSqlParameter(user.username, values); + let p6 = addSqlParameter(objChanges[i].change, values); + let p7 = addSqlParameter(objChanges[i].time, values); + if (i > startIndex) { + sqlCommand += ','; + } + sqlCommand += `(${p0},${p1},${p2},${p3},${p4},${p5},${p6},${p7})`; + lengthUtf8Current += lengthUtf8Row; + } + + sqlCommand += ';'; + dbInstance.sqlQuery(ctx, sqlCommand, callback, undefined, undefined, values); +} + +function deleteChangesCallback(ctx, docId, deleteIndex, callback) { + let sqlCommand, values = []; + let p1 = addSqlParameter(ctx.tenant, values); + let p2 = addSqlParameter(docId, values); + if (null !== deleteIndex) { + let sqlParam2 = addSqlParameter(deleteIndex, values); + sqlCommand = `DELETE FROM ${cfgTableChanges} WHERE tenant=${p1} AND id=${p2} AND change_id >= ${sqlParam2};`; + } else { + sqlCommand = `DELETE FROM ${cfgTableChanges} WHERE tenant=${p1} AND id=${p2};`; + } + dbInstance.sqlQuery(ctx, sqlCommand, callback, undefined, undefined, values); +} + +function deleteChangesPromise(ctx, docId, deleteIndex) { + return new Promise(function(resolve, reject) { + deleteChangesCallback(ctx, docId, deleteIndex, function(error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); +} + +function deleteChanges(ctx, docId, deleteIndex) { + lockCriticalSection(docId, function () {_deleteChanges(ctx, docId, deleteIndex);}); +} + +function _deleteChanges (ctx, docId, deleteIndex) { + deleteChangesCallback(ctx, docId, deleteIndex, function () {unLockCriticalSection(docId);}); +} + +function getChangesIndex(ctx, docId, callback) { + let values = []; + let p1 = addSqlParameter(ctx.tenant, values); + let p2 = addSqlParameter(docId, values); + var sqlCommand = `SELECT MAX(change_id) as change_id FROM ${cfgTableChanges} WHERE tenant=${p1} AND id=${p2};`; + dbInstance.sqlQuery(ctx, sqlCommand, callback, undefined, undefined, values); +} + +function getChangesIndexPromise(ctx, docId) { + return new Promise(function(resolve, reject) { + getChangesIndex(ctx, docId, function(error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); +} + +function getChangesPromise(ctx, docId, optStartIndex, optEndIndex, opt_time) { + let limiter = group.key(`${ctx.tenant}\t${docId}\tchanges`); + return limiter.schedule(() => { + return new Promise(function(resolve, reject) { + let values = []; + let sqlParam = addSqlParameter(ctx.tenant, values); + let sqlWhere = `tenant=${sqlParam}`; + sqlParam = addSqlParameter(docId, values); + sqlWhere += ` AND id=${sqlParam}`; + if (null != optStartIndex) { + sqlParam = addSqlParameter(optStartIndex, values); + sqlWhere += ` AND change_id>=${sqlParam}`; + } + if (null != optEndIndex) { + sqlParam = addSqlParameter(optEndIndex, values); + sqlWhere += ` AND change_id<${sqlParam}`; + } + if (null != opt_time) { + if (!(opt_time instanceof Date)) { + opt_time = new Date(opt_time); + } + sqlParam = addSqlParameter(opt_time, values); + sqlWhere += ` AND change_date<=${sqlParam}`; + } + sqlWhere += ' ORDER BY change_id ASC'; + var sqlCommand = `SELECT * FROM ${cfgTableChanges} WHERE ${sqlWhere};`; + + dbInstance.sqlQuery(ctx, sqlCommand, function(error, result) { + if (error) { + reject(error); + } else { + if (reservoirMaximum > 0) { + let size = Math.min(getChangesSize(result), reservoirMaximum); + let cur = limiter.incrementReservoir(-size).then((cur) => { + ctx.logger.debug("getChangesPromise bottleneck reservoir cur=%s", cur); + resolve(result); + }); + } else { + resolve(result); + } + } + }, undefined, undefined, values); + }); + }); +} + +function getDocumentsWithChanges(ctx) { + return new Promise(function(resolve, reject) { + const sqlCommand = `SELECT * FROM ${cfgTableResult} WHERE EXISTS(SELECT id FROM ${cfgTableChanges} WHERE tenant=${cfgTableResult}.tenant AND id = ${cfgTableResult}.id LIMIT 1);`; + dbInstance.sqlQuery(ctx, sqlCommand, function(error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }, false, false); + }); +} + + +function getExpired(ctx, maxCount, expireSeconds) { + return new Promise(function(resolve, reject) { + const values = []; + const expireDate = new Date(); + utils.addSeconds(expireDate, -expireSeconds); + const date = addSqlParameter(expireDate, values); + const count = addSqlParameter(maxCount, values); + const sqlCommand = `SELECT * FROM ${cfgTableResult} WHERE last_open_date <= ${date}` + + ` AND NOT EXISTS(SELECT tenant, id FROM ${cfgTableChanges} WHERE ${cfgTableChanges}.tenant = ${cfgTableResult}.tenant AND ${cfgTableChanges}.id = ${cfgTableResult}.id LIMIT 1) LIMIT ${count};`; + dbInstance.sqlQuery(ctx, sqlCommand, function(error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }, false, false, values); + }); +} + +function isLockCriticalSection(id) { + return !!(g_oCriticalSection[id]); +} + +// critical section +function lockCriticalSection(id, callback) { + if (g_oCriticalSection[id]) { + // wait + g_oCriticalSection[id].push(callback); + return; + } + // lock + g_oCriticalSection[id] = []; + g_oCriticalSection[id].push(callback); + callback(); +} + +function unLockCriticalSection(id) { + var arrCallbacks = g_oCriticalSection[id]; + arrCallbacks.shift(); + if (0 < arrCallbacks.length) + arrCallbacks[0](); + else + delete g_oCriticalSection[id]; +} + +function healthCheck(ctx) { + return new Promise(function(resolve, reject) { + //SELECT 1; usefull for H2, MySQL, Microsoft SQL Server, PostgreSQL, SQLite + //http://stackoverflow.com/questions/3668506/efficient-sql-test-query-or-validation-query-that-will-work-across-all-or-most + dbInstance.sqlQuery(ctx, 'SELECT 1;', function(error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); +} + +function getEmptyCallbacks(ctx) { + return new Promise(function(resolve, reject) { + const sqlCommand = `SELECT DISTINCT t1.tenant, t1.id FROM ${cfgTableChanges} t1 LEFT JOIN ${cfgTableResult} t2 ON t2.tenant = t1.tenant AND t2.id = t1.id WHERE t2.callback = '';`; + dbInstance.sqlQuery(ctx, sqlCommand, function(error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); +} + +function getTableColumns(ctx, tableName) { + return new Promise(function(resolve, reject) { + const sqlCommand = `SELECT column_name as "column_name" FROM information_schema.COLUMNS WHERE TABLE_NAME = '${tableName}';`; + dbInstance.sqlQuery(ctx, sqlCommand, function(error, result) { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); +} + +module.exports = { + insertChangesPromise, + deleteChangesPromise, + deleteChanges, + getChangesIndexPromise, + getChangesPromise, + isLockCriticalSection, + getDocumentsWithChanges, + getExpired, + healthCheck, + getEmptyCallbacks, + getTableColumns, + getDateTime: _getDateTime2, + ...connectorUtilities, + ...dbInstance +}; \ No newline at end of file diff --git a/DocService/sources/connectorUtilities.js b/DocService/sources/databaseConnectors/connectorUtilities.js similarity index 90% rename from DocService/sources/connectorUtilities.js rename to DocService/sources/databaseConnectors/connectorUtilities.js index eb658604f..a1b5c1a7c 100644 --- a/DocService/sources/connectorUtilities.js +++ b/DocService/sources/databaseConnectors/connectorUtilities.js @@ -1,4 +1,4 @@ -const constants = require('./../../Common/sources/constants'); +const constants = require('../../../Common/sources/constants'); function UserCallback() { this.userIndex = undefined; @@ -59,7 +59,6 @@ UserCallback.prototype.getCallbacks = function(ctx, callbacksStr) { } return res; }; -exports.UserCallback = UserCallback; function DocumentPassword() { this.password = undefined; @@ -111,7 +110,6 @@ DocumentPassword.prototype.hasPasswordChanges = function(ctx, docPasswordStr) { let docPassword = this.getDocPassword(ctx, docPasswordStr); return docPassword.initial !== docPassword.current; }; -exports.DocumentPassword = DocumentPassword; function DocumentAdditional() { this.data = []; @@ -152,6 +150,7 @@ DocumentAdditional.prototype.getOpenedAt = function(str) { }); return res; }; + DocumentAdditional.prototype.setShardKey = function(shardKey) { let additional = new DocumentAdditional(); additional.data.push({shardKey}); @@ -169,4 +168,25 @@ DocumentAdditional.prototype.getShardKey = function(str) { return res; }; -exports.DocumentAdditional = DocumentAdditional; +DocumentAdditional.prototype.setWopiSrc = function(wopiSrc) { + let additional = new DocumentAdditional(); + additional.data.push({wopiSrc}); + return additional.toSQLInsert(); +}; +DocumentAdditional.prototype.getWopiSrc = function(str) { + let res; + let val = new DocumentAdditional(); + val.fromString(str); + val.data.forEach((elem) => { + if (elem.wopiSrc) { + res = elem.wopiSrc; + } + }); + return res; +}; + +module.exports = { + UserCallback, + DocumentPassword, + DocumentAdditional +} \ No newline at end of file diff --git a/DocService/sources/databaseConnectors/damengConnector.js b/DocService/sources/databaseConnectors/damengConnector.js new file mode 100644 index 000000000..16c77b4fa --- /dev/null +++ b/DocService/sources/databaseConnectors/damengConnector.js @@ -0,0 +1,223 @@ +/* + * (c) Copyright Ascensio System SIA 2010-2023 + * + * This program is a free software product. You can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License (AGPL) + * version 3 as published by the Free Software Foundation. In accordance with + * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect + * that Ascensio System SIA expressly excludes the warranty of non-infringement + * of any third-party rights. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For + * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html + * + * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish + * street, Riga, Latvia, EU, LV-1050. + * + * The interactive user interfaces in modified source and object code versions + * of the Program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU AGPL version 3. + * + * Pursuant to Section 7(b) of the License you must retain the original Product + * logo when distributing the program. Pursuant to Section 7(e) we decline to + * grant you any rights under trademark law for use of our trademarks. + * + * All the Product's GUI elements, including illustrations and icon sets, as + * well as technical writing content are licensed under the terms of the + * Creative Commons Attribution-ShareAlike 4.0 International. See the License + * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + * + */ + +'use strict'; + +const connectorUtilities = require('./connectorUtilities'); +const db = require('dmdb'); +const config = require('config'); + +const configSql = config.get('services.CoAuthoring.sql'); +const cfgDbHost = configSql.get('dbHost'); +const cfgDbPort = configSql.get('dbPort'); +const cfgDbUser = configSql.get('dbUser'); +const cfgDbPass = configSql.get('dbPass'); +const cfgConnectionLimit = configSql.get('connectionlimit'); +const cfgTableResult = configSql.get('tableResult'); +const cfgDamengExtraOptions = configSql.get('damengExtraOptions'); +const forceClosingCountdownMs = 2000; + +// dmdb driver separates PoolAttributes and ConnectionAttributes. +// For some reason if you use pool you must define connection attributes in connectString, they are not included in config object, and pool.getConnection() can't configure it. +const poolHostInfo = `dm://${cfgDbUser}:${cfgDbPass}@${cfgDbHost}:${cfgDbPort}`; +const connectionOptions = Object.entries(cfgDamengExtraOptions).map(option => option.join('=')).join('&'); + +let pool = null; +const poolConfig = { + // String format dm://username:password@host:port[?prop1=val1[&prop2=val2]] + connectString: `${poolHostInfo}${connectionOptions.length > 0 ? '?' : ''}${connectionOptions}`, + poolMax: cfgConnectionLimit, + poolMin: 0 +}; + +function readLob(lob) { + return new Promise(function(resolve, reject) { + let blobData = Buffer.alloc(0); + let totalLength = 0; + + lob.on('data', function(chunk) { + totalLength += chunk.length; + blobData = Buffer.concat([blobData, chunk], totalLength); + }); + + lob.on('error', function(err) { + reject(err); + }); + + lob.on('end', function() { + resolve(blobData); + }); + }); +} + +async function formatResult(result) { + const res = []; + if (result?.rows && result?.metaData) { + for (let i = 0; i < result.rows.length; ++i) { + const row = result.rows[i]; + const out = {}; + for (let j = 0; j < result.metaData.length; ++j) { + let columnName = result.metaData[j].name; + if (row[j]?.on) { + const buf = await readLob(row[j]); + out[columnName] = buf.toString('utf8'); + } else { + out[columnName] = row[j]; + } + } + + res.push(out); + } + } + + return res; +} + +function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes = false, opt_noLog = false, opt_values = []) { + return executeQuery(ctx, sqlCommand, opt_values, opt_noModifyRes, opt_noLog).then( + result => callbackFunction?.(null, result), + error => callbackFunction?.(error) + ); +} + +async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, noLog = false) { + let connection = null; + try { + if (!pool) { + pool = await db.createPool(poolConfig); + } + + connection = await pool.getConnection(); + const result = await connection.execute(sqlCommand, values, { resultSet: false }); + + let output = result; + if (!noModifyRes) { + if (result?.rows) { + output = await formatResult(result); + } else if (result?.rowsAffected) { + output = { affectedRows: result.rowsAffected }; + } else { + output = { rows: [], affectedRows: 0 }; + } + } + + return output; + } catch (error) { + if (!noLog) { + ctx.logger.warn('sqlQuery error sqlCommand: %s: %s', sqlCommand.slice(0, 50), error.stack); + } + + throw error; + } finally { + connection?.close(); + } +} + +function closePool() { + return pool.close(forceClosingCountdownMs); +} + +function addSqlParameter(val, values) { + values.push({ val: val }); + return `:${values.length}`; +} + +function concatParams(val1, val2) { + return `CONCAT(COALESCE(${val1}, ''), COALESCE(${val2}, ''))`; +} + +async function getTableColumns(ctx, tableName) { + const result = await executeQuery(ctx, `SELECT column_name FROM DBA_TAB_COLUMNS WHERE table_name = '${tableName.toUpperCase()}';`); + return result.map(row => { return { column_name: row.column_name.toLowerCase() }}); +} + +async function upsert(ctx, task) { + task.completeDefaults(); + let dateNow = new Date(); + let values = []; + + let cbInsert = task.callback; + if (task.callback) { + let userCallback = new connectorUtilities.UserCallback(); + userCallback.fromValues(task.userIndex, task.callback); + cbInsert = userCallback.toSQLInsert(); + } + + const p0 = addSqlParameter(task.tenant, values); + const p1 = addSqlParameter(task.key, values); + const p2 = addSqlParameter(task.status, values); + const p3 = addSqlParameter(task.statusInfo, values); + const p4 = addSqlParameter(dateNow, values); + const p5 = addSqlParameter(task.userIndex, values); + const p6 = addSqlParameter(task.changeId, values); + const p7 = addSqlParameter(cbInsert, values); + const p8 = addSqlParameter(task.baseurl, values); + const p9 = addSqlParameter(dateNow, values); + + let sqlCommand = `MERGE INTO ${cfgTableResult} USING dual ON (tenant = ${p0} AND id = ${p1}) `; + sqlCommand += `WHEN NOT MATCHED THEN INSERT (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) `; + sqlCommand += `VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}) `; + sqlCommand += `WHEN MATCHED THEN UPDATE SET last_open_date = ${p9}`; + + if (task.callback) { + let p10 = addSqlParameter(JSON.stringify(task.callback), values); + sqlCommand += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${p10}, '}')`; + } + + if (task.baseurl) { + let p11 = addSqlParameter(task.baseurl, values); + sqlCommand += `, baseurl = ${p11}`; + } + + sqlCommand += ', user_index = user_index + 1'; + sqlCommand += ';'; + sqlCommand += `SELECT user_index FROM ${cfgTableResult} WHERE tenant = ${p0} AND id = ${p1};`; + + const out = {}; + const result = await executeQuery(ctx, sqlCommand, values); + if (result?.length > 0) { + const first = result[0]; + out.isInsert = task.userIndex === first.user_index; + out.insertId = first.user_index; + } + + return out; +} + +module.exports = { + sqlQuery, + closePool, + addSqlParameter, + concatParams, + getTableColumns, + upsert +}; diff --git a/DocService/sources/msSqlServerConnector.js b/DocService/sources/databaseConnectors/mssqlConnector.js similarity index 91% rename from DocService/sources/msSqlServerConnector.js rename to DocService/sources/databaseConnectors/mssqlConnector.js index c90c2ea19..af42a9f4d 100644 --- a/DocService/sources/msSqlServerConnector.js +++ b/DocService/sources/databaseConnectors/mssqlConnector.js @@ -32,10 +32,10 @@ 'use strict'; -const sql = require("mssql"); +const sql = require('mssql'); const config = require('config'); const connectorUtilities = require('./connectorUtilities'); -const utils = require('./../../Common/sources/utils'); +const utils = require('../../../Common/sources/utils'); const configSql = config.get('services.CoAuthoring.sql'); const cfgTableResult = configSql.get('tableResult'); @@ -50,12 +50,11 @@ const connectionConfiguration = { database: configSql.get('dbName'), pool: { max: configSql.get('connectionlimit'), - min: 0, - idleTimeoutMillis: 30000 + min: 0 } }; const additionalOptions = configSql.get('msSqlExtraOptions'); -const configuration = Object.assign({}, connectionConfiguration, additionalOptions); +const configuration = utils.deepMergeObjects({}, connectionConfiguration, additionalOptions); const placeholderPrefix = 'ph_'; @@ -72,17 +71,17 @@ function errorHandle(message, error, ctx) { function dataType(value) { let type = sql.TYPES.NChar(1); switch (typeof value) { - case "number": { + case 'number': { type = sql.TYPES.Decimal(18, 0); break; } - case "string": { + case 'string': { type = sql.TYPES.NVarChar(sql.MAX); break; } - case "object": { + case 'object': { if (value instanceof Date) { - type = sql.TYPES.DateTime() + type = sql.TYPES.DateTime(); } break; @@ -120,19 +119,19 @@ function registerPlaceholderValues(values, statement) { } function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes = false, opt_noLog = false, opt_values = {}) { - return executeSql(ctx, sqlCommand, opt_values, opt_noModifyRes, opt_noLog).then( + return executeQuery(ctx, sqlCommand, opt_values, opt_noModifyRes, opt_noLog).then( result => callbackFunction?.(null, result), error => callbackFunction?.(error) ); } -async function executeSql(ctx, sqlCommand, values = {}, noModifyRes = false, noLog = false) { +async function executeQuery(ctx, sqlCommand, values = {}, noModifyRes = false, noLog = false) { try { await sql.connect(configuration); const statement = new sql.PreparedStatement(); const placeholders = convertPlaceholdersValues(values); - registerPlaceholderValues(placeholders, statement) + registerPlaceholderValues(placeholders, statement); await statement.prepare(sqlCommand); const result = await statement.execute(placeholders); @@ -145,7 +144,7 @@ async function executeSql(ctx, sqlCommand, values = {}, noModifyRes = false, noL let output = result; if (!noModifyRes) { if (result.recordset) { - output = result.recordset + output = result.recordset; } else { output = { affectedRows: result.rowsAffected.pop() }; } @@ -174,6 +173,10 @@ async function executeBulk(ctx, table) { } } +function closePool() { + return sql.close(); +} + function addSqlParameterObjectBased(parameter, name, type, accumulatedObject) { if (accumulatedObject._typesMetadata === undefined) { accumulatedObject._typesMetadata = {}; @@ -197,14 +200,14 @@ function concatParams(...parameters) { function getTableColumns(ctx, tableName) { const sqlCommand = `SELECT column_name FROM information_schema.COLUMNS WHERE TABLE_NAME = '${tableName}' AND TABLE_SCHEMA = 'dbo';`; - return executeSql(ctx, sqlCommand); + return executeQuery(ctx, sqlCommand); } function getDocumentsWithChanges(ctx) { const existingId = `SELECT TOP(1) id FROM ${cfgTableChanges} WHERE tenant=${cfgTableResult}.tenant AND id = ${cfgTableResult}.id`; const sqlCommand = `SELECT * FROM ${cfgTableResult} WHERE EXISTS(${existingId});`; - return executeSql(ctx, sqlCommand); + return executeQuery(ctx, sqlCommand); } function getExpired(ctx, maxCount, expireSeconds) { @@ -217,10 +220,10 @@ function getExpired(ctx, maxCount, expireSeconds) { const notExistingTenantAndId = `SELECT TOP(1) tenant, id FROM ${cfgTableChanges} WHERE ${cfgTableChanges}.tenant = ${cfgTableResult}.tenant AND ${cfgTableChanges}.id = ${cfgTableResult}.id` const sqlCommand = `SELECT TOP(${count}) * FROM ${cfgTableResult} WHERE last_open_date <= ${date} AND NOT EXISTS(${notExistingTenantAndId});`; - return executeSql(ctx, sqlCommand, values); + return executeQuery(ctx, sqlCommand, values); } -async function upsert(ctx, task, opt_updateUserIndex) { +async function upsert(ctx, task) { task.completeDefaults(); let cbInsert = task.callback; @@ -250,7 +253,7 @@ async function upsert(ctx, task, opt_updateUserIndex) { const lastOpenDate = insertValuesPlaceholder[4]; const baseUrl = insertValuesPlaceholder[8]; const insertValues = insertValuesPlaceholder.join(', '); - const columns = ['tenant', 'id', 'status', 'status_info', 'last_open_date', 'user_index', 'change_id', 'callback', 'baseurl'] + const columns = ['tenant', 'id', 'status', 'status_info', 'last_open_date', 'user_index', 'change_id', 'callback', 'baseurl']; const sourceColumns = columns.join(', '); const sourceValues = columns.map(column => `source.${column}`).join(', '); @@ -270,9 +273,7 @@ async function upsert(ctx, task, opt_updateUserIndex) { updateColumns += `, target.baseurl = ${baseUrl}`; } - if (opt_updateUserIndex) { - updateColumns += ', target.user_index = target.user_index + 1'; - } + updateColumns += ', target.user_index = target.user_index + 1'; let sqlMerge = `MERGE INTO ${cfgTableResult} AS target ` + `USING(VALUES(${insertValues})) AS source(${sourceColumns}) ` @@ -281,11 +282,11 @@ async function upsert(ctx, task, opt_updateUserIndex) { + `WHEN NOT MATCHED THEN INSERT(${sourceColumns}) VALUES(${sourceValues}) ` + `OUTPUT $ACTION as action, INSERTED.user_index as insertId;`; - const result = await executeSql(ctx, sqlMerge, values, true); + const result = await executeQuery(ctx, sqlMerge, values, true); const insertId = result.recordset[0].insertId; - const affectedRows = result.recordset[0].action === 'UPDATE' ? 2 : 1; + const isInsert = result.recordset[0].action === 'INSERT'; - return { affectedRows, insertId }; + return { isInsert, insertId }; } function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { @@ -327,11 +328,12 @@ async function insertChangesAsync(ctx, tableChanges, startIndex, objChanges, doc result.affectedRows += recursiveValue.affectedRows; } - return result + return result; } module.exports = { sqlQuery, + closePool, addSqlParameter, concatParams, getTableColumns, diff --git a/DocService/sources/databaseConnectors/mysqlConnector.js b/DocService/sources/databaseConnectors/mysqlConnector.js new file mode 100644 index 000000000..f295588ce --- /dev/null +++ b/DocService/sources/databaseConnectors/mysqlConnector.js @@ -0,0 +1,168 @@ +/* + * (c) Copyright Ascensio System SIA 2010-2023 + * + * This program is a free software product. You can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License (AGPL) + * version 3 as published by the Free Software Foundation. In accordance with + * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect + * that Ascensio System SIA expressly excludes the warranty of non-infringement + * of any third-party rights. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For + * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html + * + * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish + * street, Riga, Latvia, EU, LV-1050. + * + * The interactive user interfaces in modified source and object code versions + * of the Program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU AGPL version 3. + * + * Pursuant to Section 7(b) of the License you must retain the original Product + * logo when distributing the program. Pursuant to Section 7(e) we decline to + * grant you any rights under trademark law for use of our trademarks. + * + * All the Product's GUI elements, including illustrations and icon sets, as + * well as technical writing content are licensed under the terms of the + * Creative Commons Attribution-ShareAlike 4.0 International. See the License + * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + * + */ + +'use strict'; + +const mysql = require('mysql2/promise'); +const connectorUtilities = require('./connectorUtilities'); +const config = require('config'); + +const configSql = config.get('services.CoAuthoring.sql'); +const cfgTableResult = configSql.get('tableResult'); + +const connectionConfiguration = { + host : configSql.get('dbHost'), + port : parseInt(configSql.get('dbPort')), + user : configSql.get('dbUser'), + password : configSql.get('dbPass'), + database : configSql.get('dbName'), + charset : configSql.get('charset'), + connectionLimit : configSql.get('connectionlimit'), + timezone : 'Z', + flags : '-FOUND_ROWS' +}; + +const additionalOptions = configSql.get('mysqlExtraOptions'); +const configuration = Object.assign({}, connectionConfiguration, additionalOptions); + +let pool = mysql.createPool(configuration); + +function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes = false, opt_noLog = false, opt_values = []) { + return executeQuery(ctx, sqlCommand, opt_values, opt_noModifyRes, opt_noLog).then( + result => callbackFunction?.(null, result), + error => callbackFunction?.(error) + ); +} + +async function executeQuery(ctx, sqlCommand, values = [], noModifyRes = false, noLog = false) { + let connection = null; + try { + connection = await pool.getConnection(); + + const result = await connection.query(sqlCommand, values); + + let output; + if (!noModifyRes) { + output = result[0]?.affectedRows ? { affectedRows: result[0].affectedRows } : result[0]; + } else { + output = result[0]; + } + + return output ?? { rows: [], affectedRows: 0 }; + } catch (error) { + if (!noLog) { + ctx.logger.error(`sqlQuery() error while executing query: ${sqlCommand}\n${error.stack}`); + } + + throw error; + } finally { + if (connection) { + try { + // Put the connection back in the pool + connection.release(); + } catch (error) { + if (!noLog) { + ctx.logger.error(`connection.release() error while executing query: ${sqlCommand}\n${error.stack}`); + } + } + } + } +} + +async function closePool() { + return await pool.end(); +} + +function addSqlParameter(parameter, accumulatedArray) { + accumulatedArray.push(parameter); + return '?'; +} + +function concatParams(firstParameter, secondParameter) { + return `CONCAT(COALESCE(${firstParameter}, ''), COALESCE(${secondParameter}, ''))`; +} + +async function upsert(ctx, task) { + task.completeDefaults(); + const dateNow = new Date(); + + let cbInsert = task.callback; + if (task.callback) { + const userCallback = new connectorUtilities.UserCallback(); + userCallback.fromValues(task.userIndex, task.callback); + cbInsert = userCallback.toSQLInsert(); + } + + const values = []; + const valuesPlaceholder = [ + addSqlParameter(task.tenant, values), + addSqlParameter(task.key, values), + addSqlParameter(task.status, values), + addSqlParameter(task.statusInfo, values), + addSqlParameter(dateNow, values), + addSqlParameter(task.userIndex, values), + addSqlParameter(task.changeId, values), + addSqlParameter(cbInsert, values), + addSqlParameter(task.baseurl, values) + ]; + + let updateStatement = `last_open_date = ${addSqlParameter(dateNow, values)}`; + if (task.callback) { + let callbackPlaceholder = addSqlParameter(JSON.stringify(task.callback), values); + updateStatement += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${callbackPlaceholder}, '}')`; + } + + if (task.baseurl) { + let baseUrlPlaceholder = addSqlParameter(task.baseurl, values); + updateStatement += `, baseurl = ${baseUrlPlaceholder}`; + } + + updateStatement += ', user_index = LAST_INSERT_ID(user_index + 1);'; + + const sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl) `+ + `VALUES (${valuesPlaceholder.join(', ')}) ` + + `ON DUPLICATE KEY UPDATE ${updateStatement}`; + + const result = await executeQuery(ctx, sqlCommand, values, true); + const insertId = result.affectedRows === 1 ? task.userIndex : result.insertId; + //if CLIENT_FOUND_ROWS don't specify 1 row is inserted , 2 row is updated, and 0 row is set to its current values + //http://dev.mysql.com/doc/refman/5.7/en/insert-on-duplicate.html + const isInsert = result.affectedRows === 1; + + return { isInsert, insertId }; +} + +module.exports.sqlQuery = sqlQuery; +module.exports.closePool = closePool; +module.exports.addSqlParameter = addSqlParameter; +module.exports.concatParams = concatParams; +module.exports.upsert = upsert; diff --git a/DocService/sources/oracleBaseConnector.js b/DocService/sources/databaseConnectors/oracleConnector.js similarity index 94% rename from DocService/sources/oracleBaseConnector.js rename to DocService/sources/databaseConnectors/oracleConnector.js index e67000c7c..eb9e8035e 100644 --- a/DocService/sources/oracleBaseConnector.js +++ b/DocService/sources/databaseConnectors/oracleConnector.js @@ -35,8 +35,7 @@ const oracledb = require('oracledb'); const config = require('config'); const connectorUtilities = require('./connectorUtilities'); -const utils = require('./../../Common/sources/utils'); -const {result} = require("underscore"); +const utils = require('../../../Common/sources/utils'); const configSql = config.get('services.CoAuthoring.sql'); const cfgTableResult = configSql.get('tableResult'); @@ -52,6 +51,7 @@ const connectionConfiguration = { }; const additionalOptions = configSql.get('oracleExtraOptions'); const configuration = Object.assign({}, connectionConfiguration, additionalOptions); +const forceClosingCountdownMs = 2000; let pool = null; oracledb.fetchAsString = [ oracledb.NCLOB, oracledb.CLOB ]; @@ -153,6 +153,14 @@ async function executeBunch(ctx, sqlCommand, values = [], noLog = false) { } } +function closePool() { + return pool?.close(forceClosingCountdownMs); +} + +function healthCheck(ctx) { + return executeQuery(ctx, 'SELECT 1 FROM DUAL'); +} + function addSqlParameter(parameter, accumulatedArray) { const currentIndex = accumulatedArray.push(parameter) - 1; return `:${currentIndex}`; @@ -192,7 +200,7 @@ function getExpired(ctx, maxCount, expireSeconds) { return executeQuery(ctx, sqlCommand, values); } -function makeUpdateSql(dateNow, task, values, opt_updateUserIndex) { +function makeUpdateSql(dateNow, task, values) { const lastOpenDate = addSqlParameter(dateNow, values); let callback = ''; @@ -207,15 +215,12 @@ function makeUpdateSql(dateNow, task, values, opt_updateUserIndex) { baseUrl = `, baseurl = ${parameter}`; } - let userIndex = ''; - if (opt_updateUserIndex) { - userIndex = ', user_index = user_index + 1'; - } + const userIndex = ', user_index = user_index + 1'; - const updateQuery = `last_open_date = ${lastOpenDate}${callback}${baseUrl}${userIndex}` + const updateQuery = `last_open_date = ${lastOpenDate}${callback}${baseUrl}${userIndex}`; const tenant = addSqlParameter(task.tenant, values); const id = addSqlParameter(task.key, values); - const condition = `tenant = ${tenant} AND id = ${id}` + const condition = `tenant = ${tenant} AND id = ${id}`; const returning = addSqlParameter({ type: oracledb.NUMBER, dir: oracledb.BIND_OUT }, values); @@ -226,7 +231,7 @@ function getReturnedValue(returned) { return returned?.outBinds?.pop()?.pop(); } -async function upsert(ctx, task, opt_updateUserIndex) { +async function upsert(ctx, task) { task.completeDefaults(); let cbInsert = task.callback; @@ -259,17 +264,17 @@ async function upsert(ctx, task, opt_updateUserIndex) { const insertResult = await executeQuery(ctx, sqlInsertTry, insertValues, true, true); const insertId = getReturnedValue(insertResult); - return { affectedRows: 1, insertId }; + return { isInsert: true, insertId }; } catch (insertError) { if (insertError.code !== 'ORA-00001') { throw insertError; } const values = []; - const updateResult = await executeQuery(ctx, makeUpdateSql(dateNow, task, values, opt_updateUserIndex), values, true); + const updateResult = await executeQuery(ctx, makeUpdateSql(dateNow, task, values), values, true); const insertId = getReturnedValue(updateResult); - return { affectedRows: 2, insertId }; + return { isInsert: false, insertId }; } } @@ -328,7 +333,7 @@ async function insertChangesAsync(ctx, tableChanges, startIndex, objChanges, doc placeholder.push(`:${i}`); } - const sqlInsert = `INSERT /*+ APPEND_VALUES*/INTO ${tableChanges} VALUES(${placeholder.join(',')})` + const sqlInsert = `INSERT /*+ APPEND_VALUES*/INTO ${tableChanges} VALUES(${placeholder.join(',')})`; const result = await executeBunch(ctx, sqlInsert, values); if (packetCapacityReached) { @@ -341,6 +346,8 @@ async function insertChangesAsync(ctx, tableChanges, startIndex, objChanges, doc module.exports = { sqlQuery, + closePool, + healthCheck, addSqlParameter, concatParams, getTableColumns, diff --git a/DocService/sources/postgreSqlBaseConnector.js b/DocService/sources/databaseConnectors/postgreConnector.js similarity index 79% rename from DocService/sources/postgreSqlBaseConnector.js rename to DocService/sources/databaseConnectors/postgreConnector.js index 18123edda..2f8448293 100644 --- a/DocService/sources/postgreSqlBaseConnector.js +++ b/DocService/sources/databaseConnectors/postgreConnector.js @@ -1,225 +1,246 @@ -/* - * (c) Copyright Ascensio System SIA 2010-2023 - * - * This program is a free software product. You can redistribute it and/or - * modify it under the terms of the GNU Affero General Public License (AGPL) - * version 3 as published by the Free Software Foundation. In accordance with - * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect - * that Ascensio System SIA expressly excludes the warranty of non-infringement - * of any third-party rights. - * - * This program is distributed WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For - * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html - * - * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish - * street, Riga, Latvia, EU, LV-1050. - * - * The interactive user interfaces in modified source and object code versions - * of the Program must display Appropriate Legal Notices, as required under - * Section 5 of the GNU AGPL version 3. - * - * Pursuant to Section 7(b) of the License you must retain the original Product - * logo when distributing the program. Pursuant to Section 7(e) we decline to - * grant you any rights under trademark law for use of our trademarks. - * - * All the Product's GUI elements, including illustrations and icon sets, as - * well as technical writing content are licensed under the terms of the - * Creative Commons Attribution-ShareAlike 4.0 International. See the License - * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode - * - */ - -'use strict'; - -var pg = require('pg'); -var co = require('co'); -var types = require('pg').types; -const connectorUtilities = require('./connectorUtilities'); -const config = require('config'); -var configSql = config.get('services.CoAuthoring.sql'); -const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); -var pgPoolExtraOptions = configSql.get('pgPoolExtraOptions'); -const cfgEditor = config.get('services.CoAuthoring.editor'); - -let connectionConfig = { - host: configSql.get('dbHost'), - port: parseInt(configSql.get('dbPort')), - user: configSql.get('dbUser'), - password: configSql.get('dbPass'), - database: configSql.get('dbName'), - max: configSql.get('connectionlimit'), - min: 0, - ssl: false, - idleTimeoutMillis: 30000 -}; -//clone pgPoolExtraOptions to resolve 'TypeError: Cannot redefine property: key' in pg-pool -config.util.extendDeep(connectionConfig, pgPoolExtraOptions); -var pool = new pg.Pool(connectionConfig); -//todo datetime timezone -pg.defaults.parseInputDatesAsUTC = true; -types.setTypeParser(1114, function(stringValue) { - return new Date(stringValue + '+0000'); -}); -types.setTypeParser(1184, function(stringValue) { - return new Date(stringValue + '+0000'); -}); - -var maxPacketSize = configSql.get('max_allowed_packet'); - -exports.sqlQuery = function(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) { - co(function *() { - var result = null; - var error = null; - try { - result = yield pool.query(sqlCommand, opt_values); - } catch (err) { - error = err; - if (!opt_noLog) { - ctx.logger.warn('sqlQuery error sqlCommand: %s: %s', sqlCommand.slice(0, 50), err.stack); - } - } finally { - if (callbackFunction) { - var output = result; - if (result && !opt_noModifyRes) { - if ('SELECT' === result.command) { - output = result.rows; - } else { - output = {affectedRows: result.rowCount}; - } - } - callbackFunction(error, output); - } - } - }); -}; -let addSqlParam = function (val, values) { - values.push(val); - return '$' + values.length; -}; -exports.addSqlParameter = addSqlParam; -let concatParams = function (val1, val2) { - return `COALESCE(${val1}, '') || COALESCE(${val2}, '')`; -}; -exports.concatParams = concatParams; -var isSupportOnConflict = true; - -function getUpsertString(task, values) { - task.completeDefaults(); - let dateNow = new Date(); - let cbInsert = task.callback; - if (isSupportOnConflict && task.callback) { - let userCallback = new connectorUtilities.UserCallback(); - userCallback.fromValues(task.userIndex, task.callback); - cbInsert = userCallback.toSQLInsert(); - } - let p0 = addSqlParam(task.tenant, values); - let p1 = addSqlParam(task.key, values); - let p2 = addSqlParam(task.status, values); - let p3 = addSqlParam(task.statusInfo, values); - let p4 = addSqlParam(dateNow, values); - let p5 = addSqlParam(task.userIndex, values); - let p6 = addSqlParam(task.changeId, values); - let p7 = addSqlParam(cbInsert, values); - let p8 = addSqlParam(task.baseurl, values); - if (isSupportOnConflict) { - let p9 = addSqlParam(dateNow, values); - //http://stackoverflow.com/questions/34762732/how-to-find-out-if-an-upsert-was-an-update-with-postgresql-9-5-upsert - let sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)`; - sqlCommand += ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8})`; - sqlCommand += ` ON CONFLICT (tenant, id) DO UPDATE SET last_open_date = ${p9}`; - if (task.callback) { - let p10 = addSqlParam(JSON.stringify(task.callback), values); - sqlCommand += `, callback = ${cfgTableResult}.callback || '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' `; - sqlCommand += ` || (${cfgTableResult}.user_index + 1)::text || ',"callback":' || ${p10}::text || '}'`; - } - if (task.baseurl) { - let p11 = addSqlParam(task.baseurl, values); - sqlCommand += `, baseurl = ${p11}`; - } - sqlCommand += `, user_index = ${cfgTableResult}.user_index + 1 RETURNING user_index as userindex;`; - return sqlCommand; - } else { - return `SELECT * FROM merge_db(${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8});`; - } -} -exports.upsert = function(ctx, task) { - return new Promise(function(resolve, reject) { - let values = []; - var sqlCommand = getUpsertString(task, values); - exports.sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - if (isSupportOnConflict && '42601' === error.code) { - //SYNTAX ERROR - isSupportOnConflict = false; - ctx.logger.warn('checkIsSupportOnConflict false'); - resolve(exports.upsert(ctx, task)); - } else { - reject(error); - } - } else { - if (result && result.rows.length > 0) { - var first = result.rows[0]; - result = {affectedRows: 0, insertId: 0}; - result.affectedRows = task.userIndex !== first.userindex ? 2 : 1; - result.insertId = first.userindex; - } - resolve(result); - } - }, true, undefined, values); - }); -}; -exports.insertChanges = function(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { - let i = startIndex; - if (i >= objChanges.length) { - return; - } - let isSupported = true; - let tenant = []; - let id = []; - let changeId = []; - let userId = []; - let userIdOriginal = []; - let username = []; - let change = []; - let time = []; - //Postgres 9.4 multi-argument unnest - let sqlCommand = `INSERT INTO ${tableChanges} (tenant, id, change_id, user_id, user_id_original, user_name, change_data, change_date) `; - let changesType = cfgEditor['binaryChanges'] ? 'bytea' : 'text'; - sqlCommand += `SELECT * FROM UNNEST ($1::text[], $2::text[], $3::int[], $4::text[], $5::text[], $6::text[], $7::${changesType}[], $8::timestamp[]);`; - let values = [tenant, id, changeId, userId, userIdOriginal, username, change, time]; - let curLength = sqlCommand.length; - for (; i < objChanges.length; ++i) { - //4 is max utf8 bytes per symbol - curLength += 4 * (docId.length + user.id.length + user.idOriginal.length + user.username.length + objChanges[i].change.length) + 4 + 8; - if (curLength >= maxPacketSize && i > startIndex) { - exports.sqlQuery(ctx, sqlCommand, function(error, output) { - if (error && '42883' == error.code) { - isSupported = false; - ctx.logger.warn('postgresql does not support UNNEST'); - } - if (error) { - callback(error, output, isSupported); - } else { - exports.insertChanges(ctx, tableChanges, i, objChanges, docId, index, user, callback); - } - }, undefined, undefined, values); - return; - } - tenant.push(ctx.tenant); - id.push(docId); - changeId.push(index++); - userId.push(user.id); - userIdOriginal.push(user.idOriginal); - username.push(user.username); - change.push(objChanges[i].change); - time.push(objChanges[i].time); - } - exports.sqlQuery(ctx, sqlCommand, function(error, output) { - if (error && '42883' == error.code) { - isSupported = false; - ctx.logger.warn('postgresql does not support UNNEST'); - } - callback(error, output, isSupported); - }, undefined, undefined, values); -}; +/* + * (c) Copyright Ascensio System SIA 2010-2023 + * + * This program is a free software product. You can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License (AGPL) + * version 3 as published by the Free Software Foundation. In accordance with + * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect + * that Ascensio System SIA expressly excludes the warranty of non-infringement + * of any third-party rights. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For + * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html + * + * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish + * street, Riga, Latvia, EU, LV-1050. + * + * The interactive user interfaces in modified source and object code versions + * of the Program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU AGPL version 3. + * + * Pursuant to Section 7(b) of the License you must retain the original Product + * logo when distributing the program. Pursuant to Section 7(e) we decline to + * grant you any rights under trademark law for use of our trademarks. + * + * All the Product's GUI elements, including illustrations and icon sets, as + * well as technical writing content are licensed under the terms of the + * Creative Commons Attribution-ShareAlike 4.0 International. See the License + * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + * + */ + +'use strict'; + +var pg = require('pg'); +var co = require('co'); +var types = require('pg').types; +const connectorUtilities = require('./connectorUtilities'); +const operationContext = require('../../../Common/sources/operationContext'); +const config = require('config'); +var configSql = config.get('services.CoAuthoring.sql'); +const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); +var pgPoolExtraOptions = configSql.get('pgPoolExtraOptions'); +const cfgEditor = config.get('services.CoAuthoring.editor'); + +let connectionConfig = { + host: configSql.get('dbHost'), + port: parseInt(configSql.get('dbPort')), + user: configSql.get('dbUser'), + password: configSql.get('dbPass'), + database: configSql.get('dbName'), + max: configSql.get('connectionlimit'), + min: 0, + ssl: false +}; +//clone pgPoolExtraOptions to resolve 'TypeError: Cannot redefine property: key' in pg-pool +//timeouts from https://github.com/brianc/node-postgres/issues/3018#issuecomment-1619729794 +config.util.extendDeep(connectionConfig, pgPoolExtraOptions); +var pool = new pg.Pool(connectionConfig); +//listen "error" event otherwise - unhandled exception(https://github.com/brianc/node-postgres/issues/2764#issuecomment-1163475426) +pool.on('error', (err, client) => { + operationContext.global.logger.error(`postgresql pool error %s`, err.stack) +}) +//todo datetime timezone +pg.defaults.parseInputDatesAsUTC = true; +types.setTypeParser(1114, function(stringValue) { + return new Date(stringValue + '+0000'); +}); +types.setTypeParser(1184, function(stringValue) { + return new Date(stringValue + '+0000'); +}); + +var maxPacketSize = configSql.get('max_allowed_packet'); + +function sqlQuery(ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) { + co(function *() { + var result = null; + var error = null; + try { + result = yield pool.query(sqlCommand, opt_values); + } catch (err) { + error = err; + if (!opt_noLog) { + ctx.logger.warn('sqlQuery error sqlCommand: %s: %s', sqlCommand.slice(0, 50), err.stack); + } + } finally { + if (callbackFunction) { + var output = result; + if (result && !opt_noModifyRes) { + if ('SELECT' === result.command) { + output = result.rows; + } else { + output = {affectedRows: result.rowCount}; + } + } + callbackFunction(error, output); + } + } + }); +} + +function closePool() { + pool.end(); +} + +function addSqlParameter(val, values) { + values.push(val); + return '$' + values.length; +} + +function concatParams(val1, val2) { + return `COALESCE(${val1}, '') || COALESCE(${val2}, '')`; +} + +var isSupportOnConflict = true; + +function getUpsertString(task, values) { + task.completeDefaults(); + let dateNow = new Date(); + let cbInsert = task.callback; + if (isSupportOnConflict && task.callback) { + let userCallback = new connectorUtilities.UserCallback(); + userCallback.fromValues(task.userIndex, task.callback); + cbInsert = userCallback.toSQLInsert(); + } + let p0 = addSqlParameter(task.tenant, values); + let p1 = addSqlParameter(task.key, values); + let p2 = addSqlParameter(task.status, values); + let p3 = addSqlParameter(task.statusInfo, values); + let p4 = addSqlParameter(dateNow, values); + let p5 = addSqlParameter(task.userIndex, values); + let p6 = addSqlParameter(task.changeId, values); + let p7 = addSqlParameter(cbInsert, values); + let p8 = addSqlParameter(task.baseurl, values); + if (isSupportOnConflict) { + let p9 = addSqlParameter(dateNow, values); + //http://stackoverflow.com/questions/34762732/how-to-find-out-if-an-upsert-was-an-update-with-postgresql-9-5-upsert + let sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)`; + sqlCommand += ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8})`; + sqlCommand += ` ON CONFLICT (tenant, id) DO UPDATE SET last_open_date = ${p9}`; + if (task.callback) { + let p10 = addSqlParameter(JSON.stringify(task.callback), values); + sqlCommand += `, callback = ${cfgTableResult}.callback || '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' `; + sqlCommand += ` || (${cfgTableResult}.user_index + 1)::text || ',"callback":' || ${p10}::text || '}'`; + } + if (task.baseurl) { + let p11 = addSqlParameter(task.baseurl, values); + sqlCommand += `, baseurl = ${p11}`; + } + sqlCommand += `, user_index = ${cfgTableResult}.user_index + 1 RETURNING user_index as userindex;`; + return sqlCommand; + } else { + return `SELECT * FROM merge_db(${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8});`; + } +} + +function upsert(ctx, task) { + return new Promise(function(resolve, reject) { + let values = []; + var sqlCommand = getUpsertString(task, values); + sqlQuery(ctx, sqlCommand, function(error, result) { + if (error) { + if (isSupportOnConflict && '42601' === error.code) { + //SYNTAX ERROR + isSupportOnConflict = false; + ctx.logger.warn('checkIsSupportOnConflict false'); + resolve(upsert(ctx, task)); + } else { + reject(error); + } + } else { + if (result && result.rows.length > 0) { + var first = result.rows[0]; + result = {}; + result.isInsert = task.userIndex === first.userindex; + result.insertId = first.userindex; + } + resolve(result); + } + }, true, undefined, values); + }); +} + +function insertChanges(ctx, tableChanges, startIndex, objChanges, docId, index, user, callback) { + let i = startIndex; + if (i >= objChanges.length) { + return; + } + let isSupported = true; + let tenant = []; + let id = []; + let changeId = []; + let userId = []; + let userIdOriginal = []; + let username = []; + let change = []; + let time = []; + //Postgres 9.4 multi-argument unnest + let sqlCommand = `INSERT INTO ${tableChanges} (tenant, id, change_id, user_id, user_id_original, user_name, change_data, change_date) `; + let changesType = cfgEditor['binaryChanges'] ? 'bytea' : 'text'; + sqlCommand += `SELECT * FROM UNNEST ($1::text[], $2::text[], $3::int[], $4::text[], $5::text[], $6::text[], $7::${changesType}[], $8::timestamp[]);`; + let values = [tenant, id, changeId, userId, userIdOriginal, username, change, time]; + let curLength = sqlCommand.length; + for (; i < objChanges.length; ++i) { + //4 is max utf8 bytes per symbol + curLength += 4 * (docId.length + user.id.length + user.idOriginal.length + user.username.length + objChanges[i].change.length) + 4 + 8; + if (curLength >= maxPacketSize && i > startIndex) { + sqlQuery(ctx, sqlCommand, function(error, output) { + if (error && '42883' == error.code) { + isSupported = false; + ctx.logger.warn('postgresql does not support UNNEST'); + } + if (error) { + callback(error, output, isSupported); + } else { + insertChanges(ctx, tableChanges, i, objChanges, docId, index, user, callback); + } + }, undefined, undefined, values); + return; + } + tenant.push(ctx.tenant); + id.push(docId); + changeId.push(index++); + userId.push(user.id); + userIdOriginal.push(user.idOriginal); + username.push(user.username); + change.push(objChanges[i].change); + time.push(objChanges[i].time); + } + sqlQuery(ctx, sqlCommand, function(error, output) { + if (error && '42883' == error.code) { + isSupported = false; + ctx.logger.warn('postgresql does not support UNNEST'); + } + callback(error, output, isSupported); + }, undefined, undefined, values); +} + +module.exports = { + sqlQuery, + closePool, + addSqlParameter, + concatParams, + upsert, + insertChanges +}; \ No newline at end of file diff --git a/DocService/sources/editorDataMemory.js b/DocService/sources/editorDataMemory.js index 0951c40b4..cfed80559 100644 --- a/DocService/sources/editorDataMemory.js +++ b/DocService/sources/editorDataMemory.js @@ -39,19 +39,29 @@ const tenantManager = require('./../../Common/sources/tenantManager'); const cfgExpMonthUniqueUsers = ms(config.get('services.CoAuthoring.expire.monthUniqueUsers')); +function EditorCommon() { +} +EditorCommon.prototype.connect = async function () {}; +EditorCommon.prototype.isConnected = function() { + return true; +}; +EditorCommon.prototype.ping = async function() {return "PONG"}; +EditorCommon.prototype.close = async function() {}; +EditorCommon.prototype.healthCheck = async function() { + if (this.isConnected()) { + await this.ping(); + return true; + } + return false; +}; + function EditorData() { + EditorCommon.call(this); this.data = {}; this.forceSaveTimer = {}; - this.uniqueUser = {}; - this.uniqueUsersOfMonth = {}; - this.uniqueViewUser = {}; - this.uniqueViewUsersOfMonth = {}; - this.shutdown = {}; - this.stat = {}; } -EditorData.prototype.connect = function() { - return Promise.resolve(); -}; +EditorData.prototype = Object.create(EditorCommon.prototype); +EditorData.prototype.constructor = EditorData; EditorData.prototype._getDocumentData = function(ctx, docId) { let tenantData = this.data[ctx.tenant]; if (!tenantData) { @@ -73,7 +83,7 @@ EditorData.prototype._checkAndLock = function(ctx, name, docId, fencingToken, tt const expireAt = now + ttl * 1000; data[name] = {fencingToken: fencingToken, expireAt: expireAt}; } - return Promise.resolve(res); + return res; }; EditorData.prototype._checkAndUnlock = function(ctx, name, docId, fencingToken) { let data = this._getDocumentData(ctx, docId); @@ -90,106 +100,117 @@ EditorData.prototype._checkAndUnlock = function(ctx, name, docId, fencingToken) res = commonDefines.c_oAscUnlockRes.Empty; delete data[name]; } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresence = function(ctx, docId, userId, userInfo) { - return Promise.resolve(); -}; -EditorData.prototype.updatePresence = function(ctx, docId, userId) { - return Promise.resolve(); -}; -EditorData.prototype.removePresence = function(ctx, docId, userId) { - return Promise.resolve(); -}; -EditorData.prototype.getPresence = function(ctx, docId, connections) { +EditorData.prototype.addPresence = async function(ctx, docId, userId, userInfo) {}; +EditorData.prototype.updatePresence = async function(ctx, docId, userId) {}; +EditorData.prototype.removePresence = async function(ctx, docId, userId) {}; +EditorData.prototype.getPresence = async function(ctx, docId, connections) { let hvals = []; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (conn.docId === docId && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - hvals.push(utils.getConnectionInfoStr(conn)); + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (conn.docId === docId && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + hvals.push(utils.getConnectionInfoStr(conn)); + } } } - return Promise.resolve(hvals); + return hvals; }; -EditorData.prototype.lockSave = function(ctx, docId, userId, ttl) { +EditorData.prototype.lockSave = async function(ctx, docId, userId, ttl) { return this._checkAndLock(ctx, 'lockSave', docId, userId, ttl); }; -EditorData.prototype.unlockSave = function(ctx, docId, userId) { +EditorData.prototype.unlockSave = async function(ctx, docId, userId) { return this._checkAndUnlock(ctx, 'lockSave', docId, userId); }; -EditorData.prototype.lockAuth = function(ctx, docId, userId, ttl) { +EditorData.prototype.lockAuth = async function(ctx, docId, userId, ttl) { return this._checkAndLock(ctx, 'lockAuth', docId, userId, ttl); }; -EditorData.prototype.unlockAuth = function(ctx, docId, userId) { +EditorData.prototype.unlockAuth = async function(ctx, docId, userId) { return this._checkAndUnlock(ctx, 'lockAuth', docId, userId); }; -EditorData.prototype.getDocumentPresenceExpired = function(now) { - return Promise.resolve([]); -}; -EditorData.prototype.removePresenceDocument = function(ctx, docId) { - return Promise.resolve(); +EditorData.prototype.getDocumentPresenceExpired = async function(now) { + return []; }; +EditorData.prototype.removePresenceDocument = async function(ctx, docId) {}; -EditorData.prototype.addLocks = function(ctx, docId, locks) { +EditorData.prototype.addLocks = async function(ctx, docId, locks) { + let data = this._getDocumentData(ctx, docId); + if (!data.locks) { + data.locks = {}; + } + Object.assign(data.locks, locks); +}; +EditorData.prototype.addLocksNX = async function(ctx, docId, locks) { let data = this._getDocumentData(ctx, docId); if (!data.locks) { - data.locks = []; + data.locks = {}; + } + let lockConflict = {}; + for (let lockId in locks) { + if (undefined === data.locks[lockId]) { + data.locks[lockId] = locks[lockId]; + } else { + lockConflict[lockId] = locks[lockId]; + } + } + return {lockConflict, allLocks: data.locks}; +}; +EditorData.prototype.removeLocks = async function(ctx, docId, locks) { + let data = this._getDocumentData(ctx, docId); + if (data.locks) { + for (let lockId in locks) { + delete data.locks[lockId]; + } } - data.locks = data.locks.concat(locks); - return Promise.resolve(); }; -EditorData.prototype.removeLocks = function(ctx, docId) { +EditorData.prototype.removeAllLocks = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.locks = undefined; - return Promise.resolve(); }; -EditorData.prototype.getLocks = function(ctx, docId) { +EditorData.prototype.getLocks = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return Promise.resolve(data.locks || []); + return data.locks || {}; }; -EditorData.prototype.addMessage = function(ctx, docId, msg) { +EditorData.prototype.addMessage = async function(ctx, docId, msg) { let data = this._getDocumentData(ctx, docId); if (!data.messages) { data.messages = []; } data.messages.push(msg); - return Promise.resolve(); }; -EditorData.prototype.removeMessages = function(ctx, docId) { +EditorData.prototype.removeMessages = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.messages = undefined; - return Promise.resolve(); }; -EditorData.prototype.getMessages = function(ctx, docId) { +EditorData.prototype.getMessages = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return Promise.resolve(data.messages || []); + return data.messages || []; }; -EditorData.prototype.setSaved = function(ctx, docId, status) { +EditorData.prototype.setSaved = async function(ctx, docId, status) { let data = this._getDocumentData(ctx, docId); data.saved = status; - return Promise.resolve(); }; -EditorData.prototype.getdelSaved = function(ctx, docId) { +EditorData.prototype.getdelSaved = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); let res = data.saved; - data.saved = undefined; - return Promise.resolve(res); + data.saved = null; + return res; }; -EditorData.prototype.setForceSave = function(ctx, docId, time, index, baseUrl, changeInfo, convertInfo) { +EditorData.prototype.setForceSave = async function(ctx, docId, time, index, baseUrl, changeInfo, convertInfo) { let data = this._getDocumentData(ctx, docId); data.forceSave = {time, index, baseUrl, changeInfo, started: false, ended: false, convertInfo}; - return Promise.resolve(); }; -EditorData.prototype.getForceSave = function(ctx, docId) { +EditorData.prototype.getForceSave = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); - return Promise.resolve(data.forceSave || null); + return data.forceSave || null; }; -EditorData.prototype.checkAndStartForceSave = function(ctx, docId) { +EditorData.prototype.checkAndStartForceSave = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); let res; if (data.forceSave && !data.forceSave.started) { @@ -197,9 +218,9 @@ EditorData.prototype.checkAndStartForceSave = function(ctx, docId) { data.forceSave.ended = false; res = data.forceSave; } - return Promise.resolve(res); + return res; }; -EditorData.prototype.checkAndSetForceSave = function(ctx, docId, time, index, started, ended, convertInfo) { +EditorData.prototype.checkAndSetForceSave = async function(ctx, docId, time, index, started, ended, convertInfo) { let data = this._getDocumentData(ctx, docId); let res; if (data.forceSave && time === data.forceSave.time && index === data.forceSave.index) { @@ -208,15 +229,14 @@ EditorData.prototype.checkAndSetForceSave = function(ctx, docId, time, index, st data.forceSave.convertInfo = convertInfo; res = data.forceSave; } - return Promise.resolve(res); + return res; }; -EditorData.prototype.removeForceSave = function(ctx, docId) { +EditorData.prototype.removeForceSave = async function(ctx, docId) { let data = this._getDocumentData(ctx, docId); data.forceSave = undefined; - return Promise.resolve(); }; -EditorData.prototype.cleanDocumentOnExit = function(ctx, docId) { +EditorData.prototype.cleanDocumentOnExit = async function(ctx, docId) { let tenantData = this.data[ctx.tenant]; if (tenantData) { delete tenantData[docId]; @@ -225,10 +245,9 @@ EditorData.prototype.cleanDocumentOnExit = function(ctx, docId) { if (tenantTimer) { delete tenantTimer[docId]; } - return Promise.resolve(); }; -EditorData.prototype.addForceSaveTimerNX = function(ctx, docId, expireAt) { +EditorData.prototype.addForceSaveTimerNX = async function(ctx, docId, expireAt) { let tenantTimer = this.forceSaveTimer[ctx.tenant]; if (!tenantTimer) { this.forceSaveTimer[ctx.tenant] = tenantTimer = {}; @@ -236,9 +255,8 @@ EditorData.prototype.addForceSaveTimerNX = function(ctx, docId, expireAt) { if (!tenantTimer[docId]) { tenantTimer[docId] = expireAt; } - return Promise.resolve(); }; -EditorData.prototype.getForceSaveTimer = function(now) { +EditorData.prototype.getForceSaveTimer = async function(now) { let res = []; for (let tenant in this.forceSaveTimer) { if (this.forceSaveTimer.hasOwnProperty(tenant)) { @@ -253,18 +271,29 @@ EditorData.prototype.getForceSaveTimer = function(now) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueUser = function(ctx, userId, expireAt, userInfo) { +function EditorStat() { + EditorCommon.call(this); + this.uniqueUser = {}; + this.uniqueUsersOfMonth = {}; + this.uniqueViewUser = {}; + this.uniqueViewUsersOfMonth = {}; + this.stat = {}; + this.shutdown = {}; + this.license = {}; +} +EditorStat.prototype = Object.create(EditorCommon.prototype); +EditorStat.prototype.constructor = EditorStat; +EditorStat.prototype.addPresenceUniqueUser = async function(ctx, userId, expireAt, userInfo) { let tenantUser = this.uniqueUser[ctx.tenant]; if (!tenantUser) { this.uniqueUser[ctx.tenant] = tenantUser = {}; } tenantUser[userId] = {expireAt: expireAt, userInfo: userInfo}; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueUser = function(ctx, nowUTC) { +EditorStat.prototype.getPresenceUniqueUser = async function(ctx, nowUTC) { let res = []; let tenantUser = this.uniqueUser[ctx.tenant]; if (!tenantUser) { @@ -282,9 +311,9 @@ EditorData.prototype.getPresenceUniqueUser = function(ctx, nowUTC) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueUsersOfMonth = function(ctx, userId, period, userInfo) { +EditorStat.prototype.addPresenceUniqueUsersOfMonth = async function(ctx, userId, period, userInfo) { let tenantUser = this.uniqueUsersOfMonth[ctx.tenant]; if (!tenantUser) { this.uniqueUsersOfMonth[ctx.tenant] = tenantUser = {}; @@ -294,9 +323,8 @@ EditorData.prototype.addPresenceUniqueUsersOfMonth = function(ctx, userId, perio tenantUser[period] = {expireAt: expireAt, data: {}}; } tenantUser[period].data[userId] = userInfo; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueUsersOfMonth = function(ctx) { +EditorStat.prototype.getPresenceUniqueUsersOfMonth = async function(ctx) { let res = {}; let nowUTC = Date.now(); let tenantUser = this.uniqueUsersOfMonth[ctx.tenant]; @@ -313,18 +341,17 @@ EditorData.prototype.getPresenceUniqueUsersOfMonth = function(ctx) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueViewUser = function(ctx, userId, expireAt, userInfo) { +EditorStat.prototype.addPresenceUniqueViewUser = async function(ctx, userId, expireAt, userInfo) { let tenantUser = this.uniqueViewUser[ctx.tenant]; if (!tenantUser) { this.uniqueViewUser[ctx.tenant] = tenantUser = {}; } tenantUser[userId] = {expireAt: expireAt, userInfo: userInfo}; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueViewUser = function(ctx, nowUTC) { +EditorStat.prototype.getPresenceUniqueViewUser = async function(ctx, nowUTC) { let res = []; let tenantUser = this.uniqueViewUser[ctx.tenant]; if (!tenantUser) { @@ -342,9 +369,9 @@ EditorData.prototype.getPresenceUniqueViewUser = function(ctx, nowUTC) { } } } - return Promise.resolve(res); + return res; }; -EditorData.prototype.addPresenceUniqueViewUsersOfMonth = function(ctx, userId, period, userInfo) { +EditorStat.prototype.addPresenceUniqueViewUsersOfMonth = async function(ctx, userId, period, userInfo) { let tenantUser = this.uniqueViewUsersOfMonth[ctx.tenant]; if (!tenantUser) { this.uniqueViewUsersOfMonth[ctx.tenant] = tenantUser = {}; @@ -354,9 +381,8 @@ EditorData.prototype.addPresenceUniqueViewUsersOfMonth = function(ctx, userId, p tenantUser[period] = {expireAt: expireAt, data: {}}; } tenantUser[period].data[userId] = userInfo; - return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueViewUsersOfMonth = function(ctx) { +EditorStat.prototype.getPresenceUniqueViewUsersOfMonth = async function(ctx) { let res = {}; let nowUTC = Date.now(); let tenantUser = this.uniqueViewUsersOfMonth[ctx.tenant]; @@ -373,10 +399,9 @@ EditorData.prototype.getPresenceUniqueViewUsersOfMonth = function(ctx) { } } } - return Promise.resolve(res); + return res; }; - -EditorData.prototype.setEditorConnections = function(ctx, countEdit, countLiveView, countView, now, precision) { +EditorStat.prototype.setEditorConnections = async function(ctx, countEdit, countLiveView, countView, now, precision) { let tenantStat = this.stat[ctx.tenant]; if (!tenantStat) { this.stat[ctx.tenant] = tenantStat = []; @@ -387,79 +412,69 @@ EditorData.prototype.setEditorConnections = function(ctx, countEdit, countLiveVi i++; } tenantStat.splice(0, i); - return Promise.resolve(); }; -EditorData.prototype.getEditorConnections = function(ctx) { +EditorStat.prototype.getEditorConnections = async function(ctx) { let tenantStat = this.stat[ctx.tenant]; if (!tenantStat) { this.stat[ctx.tenant] = tenantStat = []; } - return Promise.resolve(tenantStat); -}; -EditorData.prototype.setEditorConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); + return tenantStat; }; -EditorData.prototype.incrEditorConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.getEditorConnectionsCount = function(ctx, connections) { +EditorStat.prototype.setEditorConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.incrEditorConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.getEditorConnectionsCount = async function(ctx, connections) { let count = 0; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (!(conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - count++; + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (!(conn.isCloseCoAuthoring || (conn.user && conn.user.view)) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + count++; + } } } - return Promise.resolve(count); -}; -EditorData.prototype.setViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.incrViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); + return count; }; -EditorData.prototype.getViewerConnectionsCount = function(ctx, connections) { +EditorStat.prototype.setViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.incrViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.getViewerConnectionsCount = async function(ctx, connections) { let count = 0; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (conn.isCloseCoAuthoring || (conn.user && conn.user.view) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - count++; + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (conn.isCloseCoAuthoring || (conn.user && conn.user.view) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + count++; + } } } - return Promise.resolve(count); -}; -EditorData.prototype.setLiveViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); -}; -EditorData.prototype.incrLiveViewerConnectionsCountByShard = function(ctx, shardId, count) { - return Promise.resolve(); + return count; }; -EditorData.prototype.getLiveViewerConnectionsCount = function(ctx, connections) { +EditorStat.prototype.setLiveViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.incrLiveViewerConnectionsCountByShard = async function(ctx, shardId, count) {}; +EditorStat.prototype.getLiveViewerConnectionsCount = async function(ctx, connections) { let count = 0; - for (let i = 0; i < connections.length; ++i) { - let conn = connections[i]; - if (utils.isLiveViewer(conn) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { - count++; + if (connections) { + for (let i = 0; i < connections.length; ++i) { + let conn = connections[i]; + if (utils.isLiveViewer(conn) && ctx.tenant === tenantManager.getTenantByConnection(ctx, conn)) { + count++; + } } } - return Promise.resolve(count); + return count; }; - -EditorData.prototype.addShutdown = function(key, docId) { +EditorStat.prototype.addShutdown = async function(key, docId) { if (!this.shutdown[key]) { this.shutdown[key] = {}; } this.shutdown[key][docId] = 1; - return Promise.resolve(); }; -EditorData.prototype.removeShutdown = function(key, docId) { +EditorStat.prototype.removeShutdown = async function(key, docId) { if (!this.shutdown[key]) { this.shutdown[key] = {}; } delete this.shutdown[key][docId]; - return Promise.resolve(); }; -EditorData.prototype.getShutdownCount = function(key) { +EditorStat.prototype.getShutdownCount = async function(key) { let count = 0; if (this.shutdown[key]) { for (let docId in this.shutdown[key]) { @@ -468,31 +483,22 @@ EditorData.prototype.getShutdownCount = function(key) { } } } - return Promise.resolve(count); + return count; }; -EditorData.prototype.cleanupShutdown = function(key) { +EditorStat.prototype.cleanupShutdown = async function(key) { delete this.shutdown[key]; - return Promise.resolve(); }; - -EditorData.prototype.setLicense = function(key, val) { - return Promise.resolve(); +EditorStat.prototype.setLicense = async function(key, val) { + this.license[key] = val; }; -EditorData.prototype.getLicense = function(key) { - return Promise.resolve(null); +EditorStat.prototype.getLicense = async function(key) { + return this.license[key] || null; }; -EditorData.prototype.removeLicense = function(key) { - return Promise.resolve(); +EditorStat.prototype.removeLicense = async function(key) { + delete this.license[key]; }; -EditorData.prototype.isConnected = function() { - return true; -}; -EditorData.prototype.ping = function() { - return Promise.resolve(); -}; -EditorData.prototype.close = function() { - return Promise.resolve(); -}; - -module.exports = EditorData; +module.exports = { + EditorData, + EditorStat +} diff --git a/DocService/sources/gc.js b/DocService/sources/gc.js index d2ae650d5..6c734a2f1 100644 --- a/DocService/sources/gc.js +++ b/DocService/sources/gc.js @@ -47,7 +47,7 @@ var commondefines = require('./../../Common/sources/commondefines'); var queueService = require('./../../Common/sources/taskqueueRabbitMQ'); var operationContext = require('./../../Common/sources/operationContext'); var pubsubService = require('./pubsubRabbitMQ'); -const sqlBase = require("./baseConnector"); +const sqlBase = require("./databaseConnectors/baseConnector"); var cfgExpFilesCron = config.get('services.CoAuthoring.expire.filesCron'); var cfgExpDocumentsCron = config.get('services.CoAuthoring.expire.documentsCron'); @@ -78,7 +78,8 @@ var checkFileExpire = function(expireSeconds) { let tenant = expired[i].tenant; let docId = expired[i].id; let shardKey = sqlBase.DocumentAdditional.prototype.getShardKey(expired[i].additional); - ctx.init(tenant, docId, ctx.userId, shardKey); + let wopiSrc = sqlBase.DocumentAdditional.prototype.getWopiSrc(expired[i].additional); + ctx.init(tenant, docId, ctx.userId, shardKey, wopiSrc); yield ctx.initTenantCache(); //todo tenant //check that no one is in the document @@ -125,7 +126,7 @@ var checkDocumentExpire = function() { var hasChanges = yield docsCoServer.hasChanges(ctx, docId); if (hasChanges) { //todo opt_initShardKey from getDocumentPresenceExpired data or from db - yield docsCoServer.createSaveTimer(ctx, docId, null, null, queue, true, true); + yield docsCoServer.createSaveTimer(ctx, docId, null, null, null, queue, true, true); startSaveCount++; } else { yield docsCoServer.cleanDocumentOnExitNoChangesPromise(ctx, docId); diff --git a/DocService/sources/mySqlBaseConnector.js b/DocService/sources/mySqlBaseConnector.js deleted file mode 100644 index c32efa20c..000000000 --- a/DocService/sources/mySqlBaseConnector.js +++ /dev/null @@ -1,133 +0,0 @@ -/* - * (c) Copyright Ascensio System SIA 2010-2023 - * - * This program is a free software product. You can redistribute it and/or - * modify it under the terms of the GNU Affero General Public License (AGPL) - * version 3 as published by the Free Software Foundation. In accordance with - * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect - * that Ascensio System SIA expressly excludes the warranty of non-infringement - * of any third-party rights. - * - * This program is distributed WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For - * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html - * - * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish - * street, Riga, Latvia, EU, LV-1050. - * - * The interactive user interfaces in modified source and object code versions - * of the Program must display Appropriate Legal Notices, as required under - * Section 5 of the GNU AGPL version 3. - * - * Pursuant to Section 7(b) of the License you must retain the original Product - * logo when distributing the program. Pursuant to Section 7(e) we decline to - * grant you any rights under trademark law for use of our trademarks. - * - * All the Product's GUI elements, including illustrations and icon sets, as - * well as technical writing content are licensed under the terms of the - * Creative Commons Attribution-ShareAlike 4.0 International. See the License - * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode - * - */ - -'use strict'; - -var mysql = require('mysql2'); -var connectorUtilities = require('./connectorUtilities'); -const config = require('config'); - -const configSql = config.get('services.CoAuthoring.sql'); -const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); - -var pool = mysql.createPool({ - host : configSql.get('dbHost'), - port : parseInt(configSql.get('dbPort')), - user : configSql.get('dbUser'), - password : configSql.get('dbPass'), - database : configSql.get('dbName'), - charset : configSql.get('charset'), - connectionLimit : configSql.get('connectionlimit'), - timezone : 'Z', - flags : '-FOUND_ROWS' -}); - -exports.sqlQuery = function (ctx, sqlCommand, callbackFunction, opt_noModifyRes, opt_noLog, opt_values) { - pool.getConnection(function(err, connection) { - if (err) { - ctx.logger.error('pool.getConnection error: %s', err); - if (callbackFunction) callbackFunction(err, null); - return; - } - let queryCallback = function (error, result) { - connection.release(); - if (error) { - ctx.logger.error('________________________error_____________________'); - ctx.logger.error('sqlQuery: %s sqlCommand: %s', error.code, sqlCommand); - ctx.logger.error(error); - ctx.logger.error('_____________________end_error_____________________'); - } - if (callbackFunction) callbackFunction(error, result); - }; - if(opt_values){ - connection.query(sqlCommand, opt_values, queryCallback); - } else { - connection.query(sqlCommand, queryCallback); - } - }); -}; -let addSqlParam = function (val, values) { - values.push(val); - return '?'; -}; -exports.addSqlParameter = addSqlParam; -let concatParams = function (val1, val2) { - return `CONCAT(COALESCE(${val1}, ''), COALESCE(${val2}, ''))`; -}; -exports.concatParams = concatParams; - -exports.upsert = function(ctx, task, opt_updateUserIndex) { - return new Promise(function(resolve, reject) { - task.completeDefaults(); - let dateNow = new Date(); - let values = []; - let cbInsert = task.callback; - if (task.callback) { - let userCallback = new connectorUtilities.UserCallback(); - userCallback.fromValues(task.userIndex, task.callback); - cbInsert = userCallback.toSQLInsert(); - } - let p0 = addSqlParam(task.tenant, values); - let p1 = addSqlParam(task.key, values); - let p2 = addSqlParam(task.status, values); - let p3 = addSqlParam(task.statusInfo, values); - let p4 = addSqlParam(dateNow, values); - let p5 = addSqlParam(task.userIndex, values); - let p6 = addSqlParam(task.changeId, values); - let p7 = addSqlParam(cbInsert, values); - let p8 = addSqlParam(task.baseurl, values); - let p9 = addSqlParam(dateNow, values); - var sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)`+ - ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8}) ON DUPLICATE KEY UPDATE` + - ` last_open_date = ${p9}`; - if (task.callback) { - let p10 = addSqlParam(JSON.stringify(task.callback), values); - sqlCommand += `, callback = CONCAT(callback , '${connectorUtilities.UserCallback.prototype.delimiter}{"userIndex":' , (user_index + 1) , ',"callback":', ${p10}, '}')`; - } - if (task.baseurl) { - let p11 = addSqlParam(task.baseurl, values); - sqlCommand += `, baseurl = ${p11}`; - } - if (opt_updateUserIndex) { - sqlCommand += ', user_index = LAST_INSERT_ID(user_index + 1)'; - } - sqlCommand += ';'; - - exports.sqlQuery(ctx, sqlCommand, function(error, result) { - if (error) { - reject(error); - } else { - resolve(result); - } - }, undefined, undefined, values); - }); -}; diff --git a/DocService/sources/pubsubRabbitMQ.js b/DocService/sources/pubsubRabbitMQ.js index 57707285f..fb820a8fc 100644 --- a/DocService/sources/pubsubRabbitMQ.js +++ b/DocService/sources/pubsubRabbitMQ.js @@ -151,7 +151,7 @@ function repeat(pubsub) { } function publishRabbit(pubsub, data) { return new Promise(function (resolve, reject) { - //Channels act like stream.Writable when you call publish or sendToQueue: they return either true, meaning �keep sending�, or false, meaning �please wait for a �drain� event�. + //Channels act like stream.Writable when you call publish or sendToQueue: they return either true, meaning “keep sending”, or false, meaning “please wait for a ‘drain’ event”. let keepSending = pubsub.channelPublish.publish(pubsub.exchangePublish, '', data); if (!keepSending) { //todo (node:4308) MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 drain listeners added to [Sender]. Use emitter.setMaxListeners() to increase limit diff --git a/DocService/sources/routes/static.js b/DocService/sources/routes/static.js new file mode 100644 index 000000000..6051ae2c0 --- /dev/null +++ b/DocService/sources/routes/static.js @@ -0,0 +1,104 @@ +/* + * (c) Copyright Ascensio System SIA 2010-2023 + * + * This program is a free software product. You can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License (AGPL) + * version 3 as published by the Free Software Foundation. In accordance with + * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect + * that Ascensio System SIA expressly excludes the warranty of non-infringement + * of any third-party rights. + * + * This program is distributed WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For + * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html + * + * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish + * street, Riga, Latvia, EU, LV-1050. + * + * The interactive user interfaces in modified source and object code versions + * of the Program must display Appropriate Legal Notices, as required under + * Section 5 of the GNU AGPL version 3. + * + * Pursuant to Section 7(b) of the License you must retain the original Product + * logo when distributing the program. Pursuant to Section 7(e) we decline to + * grant you any rights under trademark law for use of our trademarks. + * + * All the Product's GUI elements, including illustrations and icon sets, as + * well as technical writing content are licensed under the terms of the + * Creative Commons Attribution-ShareAlike 4.0 International. See the License + * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode + * + */ + +'use strict'; +const express = require('express'); +const config = require("config"); +const operationContext = require('./../../../Common/sources/operationContext'); +const utils = require('./../../../Common/sources/utils'); +const storage = require('./../../../Common/sources/storage-base'); +const urlModule = require("url"); +const path = require("path"); +const mime = require("mime"); + +const cfgStaticContent = config.has('services.CoAuthoring.server.static_content') ? config.get('services.CoAuthoring.server.static_content') : {}; +const cfgCacheStorage = config.get('storage'); +const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); +const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles'); +const cfgErrorFiles = config.get('FileConverter.converter.errorfiles'); + +const router = express.Router(); + +function initCacheRouter(cfgStorage, routs) { + const bucketName = cfgStorage.bucketName; + const storageFolderName = cfgStorage.storageFolderName; + const folderPath = cfgStorage.fs.folderPath; + routs.forEach((rout) => { + //special dirs are empty by default + if (!rout) { + return; + } + let rootPath = path.join(folderPath, rout); + router.use(`/${bucketName}/${storageFolderName}/${rout}`, (req, res, next) => { + const index = req.url.lastIndexOf('/'); + if ('GET' === req.method && index > 0) { + let sendFileOptions = { + root: rootPath, dotfiles: 'deny', headers: { + 'Content-Disposition': 'attachment' + } + }; + const urlParsed = urlModule.parse(req.url); + if (urlParsed && urlParsed.pathname) { + const filename = decodeURIComponent(path.basename(urlParsed.pathname)); + sendFileOptions.headers['Content-Type'] = mime.getType(filename); + } + const realUrl = decodeURI(req.url.substring(0, index)); + res.sendFile(realUrl, sendFileOptions, (err) => { + if (err) { + operationContext.global.logger.error(err); + res.status(400).end(); + } + }); + } else { + res.sendStatus(404); + } + }); + }); +} + +for (let i in cfgStaticContent) { + if (cfgStaticContent.hasOwnProperty(i)) { + router.use(i, express.static(cfgStaticContent[i]['path'], cfgStaticContent[i]['options'])); + } +} +if (storage.needServeStatic()) { + initCacheRouter(cfgCacheStorage, [cfgCacheStorage.cacheFolderName]); +} +if (storage.needServeStatic(cfgForgottenFiles)) { + let persistentRouts = [cfgForgottenFiles, cfgErrorFiles]; + persistentRouts.filter((rout) => {return rout && rout.length > 0;}); + if (persistentRouts.length > 0) { + initCacheRouter(cfgPersistentStorage, [cfgForgottenFiles, cfgErrorFiles]); + } +} + +module.exports = router; diff --git a/DocService/sources/server.js b/DocService/sources/server.js index 880983149..09c9e3bf3 100644 --- a/DocService/sources/server.js +++ b/DocService/sources/server.js @@ -57,7 +57,7 @@ const utils = require('./../../Common/sources/utils'); const commonDefines = require('./../../Common/sources/commondefines'); const operationContext = require('./../../Common/sources/operationContext'); const tenantManager = require('./../../Common/sources/tenantManager'); -const configStorage = config.get('storage'); +const staticRouter = require('./routes/static'); const cfgWopiEnable = config.get('wopi.enable'); const cfgWopiDummyEnable = config.get('wopi.dummy.enable'); @@ -132,44 +132,6 @@ updateLicense(); fs.watchFile(cfgLicenseFile, updateLicense); setInterval(updateLicense, 86400000); -if (config.has('services.CoAuthoring.server.static_content')) { - const staticContent = config.get('services.CoAuthoring.server.static_content'); - for (let i in staticContent) { - if (staticContent.hasOwnProperty(i)) { - app.use(i, express.static(staticContent[i]['path'], staticContent[i]['options'])); - } - } -} - -if (configStorage.has('fs.folderPath')) { - const cfgBucketName = configStorage.get('bucketName'); - const cfgStorageFolderName = configStorage.get('storageFolderName'); - app.use('/' + cfgBucketName + '/' + cfgStorageFolderName, (req, res, next) => { - const index = req.url.lastIndexOf('/'); - if ('GET' === req.method && index > 0) { - let sendFileOptions = { - root: configStorage.get('fs.folderPath'), dotfiles: 'deny', headers: { - 'Content-Disposition': 'attachment' - } - }; - const urlParsed = urlModule.parse(req.url); - if (urlParsed && urlParsed.pathname) { - const filename = decodeURIComponent(path.basename(urlParsed.pathname)); - sendFileOptions.headers['Content-Type'] = mime.getType(filename); - } - const realUrl = decodeURI(req.url.substring(0, index)); - res.sendFile(realUrl, sendFileOptions, (err) => { - if (err) { - operationContext.global.logger.error(err); - res.status(400).end(); - } - }); - } else { - res.sendStatus(404); - } - }); -} - try { fs.watch(config.get('services.CoAuthoring.plugins.path'), updatePlugins); } catch (e) { @@ -211,6 +173,9 @@ docsCoServer.install(server, () => { } }); }); + + app.use('/', staticRouter); + const rawFileParser = bodyParser.raw( {inflate: true, limit: config.get('services.CoAuthoring.server.limits_tempfile_upload'), type: function() {return true;}}); const urleEcodedParser = bodyParser.urlencoded({ extended: false }); diff --git a/DocService/sources/shutdown.js b/DocService/sources/shutdown.js index edd92320a..d85e9c1db 100644 --- a/DocService/sources/shutdown.js +++ b/DocService/sources/shutdown.js @@ -47,7 +47,7 @@ var WAIT_TIMEOUT = 30000; var LOOP_TIMEOUT = 1000; var EXEC_TIMEOUT = WAIT_TIMEOUT + utils.getConvertionTimeout(undefined); -exports.shutdown = function(ctx, editorData, status) { +exports.shutdown = function(ctx, editorStat, status) { return co(function*() { var res = true; try { @@ -55,7 +55,7 @@ exports.shutdown = function(ctx, editorData, status) { //redisKeyShutdown is not a simple counter, so it doesn't get decremented by a build that started before Shutdown started //reset redisKeyShutdown just in case the previous run didn't finish - yield editorData.cleanupShutdown(redisKeyShutdown); + yield editorStat.cleanupShutdown(redisKeyShutdown); var pubsub = new pubsubService(); yield pubsub.initPromise(); @@ -76,7 +76,7 @@ exports.shutdown = function(ctx, editorData, status) { ctx.logger.debug('shutdown timeout'); break; } - var remainingFiles = yield editorData.getShutdownCount(redisKeyShutdown); + var remainingFiles = yield editorStat.getShutdownCount(redisKeyShutdown); ctx.logger.debug('shutdown remaining files:%d', remainingFiles); if (!isStartWait && remainingFiles <= 0) { break; @@ -85,7 +85,7 @@ exports.shutdown = function(ctx, editorData, status) { } //todo need to check the queues, because there may be long conversions running before Shutdown //clean up - yield editorData.cleanupShutdown(redisKeyShutdown); + yield editorStat.cleanupShutdown(redisKeyShutdown); yield pubsub.close(); ctx.logger.debug('shutdown end'); diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js index 9a651c562..d22451976 100644 --- a/DocService/sources/taskresult.js +++ b/DocService/sources/taskresult.js @@ -33,7 +33,7 @@ 'use strict'; const crypto = require('crypto'); -var sqlBase = require('./baseConnector'); +var sqlBase = require('./databaseConnectors/baseConnector'); var constants = require('./../../Common/sources/constants'); var commonDefines = require('./../../Common/sources/commondefines'); var tenantManager = require('./../../Common/sources/tenantManager'); @@ -41,8 +41,8 @@ var config = require('config'); const cfgTableResult = config.get('services.CoAuthoring.sql.tableResult'); -let addSqlParam = sqlBase.baseConnector.addSqlParameter; -let concatParams = sqlBase.baseConnector.concatParams; +let addSqlParam = sqlBase.addSqlParameter; +let concatParams = sqlBase.concatParams; var RANDOM_KEY_MAX = 10000; @@ -95,8 +95,8 @@ TaskResultData.prototype.completeDefaults = function() { } }; -function upsert(ctx, task, opt_updateUserIndex) { - return sqlBase.baseConnector.upsert(ctx, task, opt_updateUserIndex); +function upsert(ctx, task) { + return sqlBase.upsert(ctx, task); } function select(ctx, docId) { @@ -105,7 +105,7 @@ function select(ctx, docId) { let p1 = addSqlParam(ctx.tenant, values); let p2 = addSqlParam(docId, values); let sqlCommand = `SELECT * FROM ${cfgTableResult} WHERE tenant=${p1} AND id=${p2};`; - sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { + sqlBase.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); } else { @@ -174,7 +174,7 @@ function update(ctx, task, setPassword) { let p1 = addSqlParam(task.tenant, values); let p2 = addSqlParam(task.key, values); let sqlCommand = `UPDATE ${cfgTableResult} SET ${sqlSet} WHERE tenant=${p1} AND id=${p2};`; - sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { + sqlBase.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); } else { @@ -194,7 +194,7 @@ function updateIf(ctx, task, mask) { let sqlSet = commandArg.join(', '); let sqlWhere = commandArgMask.join(' AND '); let sqlCommand = `UPDATE ${cfgTableResult} SET ${sqlSet} WHERE ${sqlWhere};`; - sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { + sqlBase.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); } else { @@ -245,7 +245,7 @@ function addRandomKey(ctx, task, opt_prefix, opt_size) { let p8 = addSqlParam(task.baseurl, values); let sqlCommand = `INSERT INTO ${cfgTableResult} (tenant, id, status, status_info, last_open_date, user_index, change_id, callback, baseurl)` + ` VALUES (${p0}, ${p1}, ${p2}, ${p3}, ${p4}, ${p5}, ${p6}, ${p7}, ${p8});`; - sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { + sqlBase.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); } else { @@ -286,7 +286,7 @@ function remove(ctx, docId) { let p1 = addSqlParam(ctx.tenant, values); let p2 = addSqlParam(docId, values); const sqlCommand = `DELETE FROM ${cfgTableResult} WHERE tenant=${p1} AND id=${p2};`; - sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { + sqlBase.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); } else { @@ -303,7 +303,7 @@ function removeIf(ctx, mask) { commandArgMask.push('id=' + addSqlParam(mask.key, values)); let sqlWhere = commandArgMask.join(' AND '); const sqlCommand = `DELETE FROM ${cfgTableResult} WHERE ${sqlWhere};`; - sqlBase.baseConnector.sqlQuery(ctx, sqlCommand, function(error, result) { + sqlBase.sqlQuery(ctx, sqlCommand, function(error, result) { if (error) { reject(error); } else { diff --git a/DocService/sources/utilsDocService.js b/DocService/sources/utilsDocService.js index a6575ac8a..378b357d8 100644 --- a/DocService/sources/utilsDocService.js +++ b/DocService/sources/utilsDocService.js @@ -34,6 +34,7 @@ const exifParser = require("exif-parser"); const Jimp = require("jimp"); +const locale = require('windows-locale'); async function fixImageExifRotation(ctx, buffer) { if (!buffer) { @@ -59,7 +60,17 @@ async function fixImageExifRotation(ctx, buffer) { } return buffer; } +/** + * + * @param {string} lang + * @returns {number | undefined} + */ +function localeToLCID(lang) { + let elem = locale[lang && lang.toLowerCase()]; + return elem && elem.id; +} module.exports = { - fixImageExifRotation + fixImageExifRotation, + localeToLCID }; \ No newline at end of file diff --git a/DocService/sources/wopiClient.js b/DocService/sources/wopiClient.js index 49bb15371..f024e7646 100644 --- a/DocService/sources/wopiClient.js +++ b/DocService/sources/wopiClient.js @@ -33,14 +33,15 @@ 'use strict'; const path = require('path'); -const fs = require('fs'); -const {stat} = require('node:fs/promises'); const { pipeline } = require('node:stream/promises'); const crypto = require('crypto'); +let util = require('util'); const {URL} = require('url'); const co = require('co'); const jwt = require('jsonwebtoken'); const config = require('config'); +const { createReadStream } = require('fs'); +const { stat, lstat, readdir } = require('fs/promises'); const utf7 = require('utf7'); const mimeDB = require('mime-db'); const xmlbuilder2 = require('xmlbuilder2'); @@ -48,9 +49,10 @@ const logger = require('./../../Common/sources/logger'); const utils = require('./../../Common/sources/utils'); const constants = require('./../../Common/sources/constants'); const commonDefines = require('./../../Common/sources/commondefines'); +const formatChecker = require('./../../Common/sources/formatchecker'); const operationContext = require('./../../Common/sources/operationContext'); const tenantManager = require('./../../Common/sources/tenantManager'); -const sqlBase = require('./baseConnector'); +const sqlBase = require('./databaseConnectors/baseConnector'); const taskResult = require('./taskresult'); const canvasService = require('./canvasservice'); const converterService = require('./converterservice'); @@ -60,20 +62,24 @@ const cfgTokenOutboxAlgorithm = config.get('services.CoAuthoring.token.outbox.al const cfgTokenOutboxExpires = config.get('services.CoAuthoring.token.outbox.expires'); const cfgTokenEnableBrowser = config.get('services.CoAuthoring.token.enable.browser'); const cfgCallbackRequestTimeout = config.get('services.CoAuthoring.server.callbackRequestTimeout'); -const cfgAllowPrivateIPAddressForSignedRequests = config.get('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests'); +const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate'); const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout'); +const cfgMaxDownloadBytes = config.get('FileConverter.converter.maxDownloadBytes'); const cfgWopiFileInfoBlockList = config.get('wopi.fileInfoBlockList'); const cfgWopiWopiZone = config.get('wopi.wopiZone'); const cfgWopiPdfView = config.get('wopi.pdfView'); +const cfgWopiPdfEdit = config.get('wopi.pdfEdit'); const cfgWopiWordView = config.get('wopi.wordView'); const cfgWopiWordEdit = config.get('wopi.wordEdit'); const cfgWopiCellView = config.get('wopi.cellView'); const cfgWopiCellEdit = config.get('wopi.cellEdit'); const cfgWopiSlideView = config.get('wopi.slideView'); const cfgWopiSlideEdit = config.get('wopi.slideEdit'); +const cfgWopiForms = config.get('wopi.forms'); const cfgWopiFavIconUrlWord = config.get('wopi.favIconUrlWord'); const cfgWopiFavIconUrlCell = config.get('wopi.favIconUrlCell'); const cfgWopiFavIconUrlSlide = config.get('wopi.favIconUrlSlide'); +const cfgWopiFavIconUrlPdf = config.get('wopi.favIconUrlPdf'); const cfgWopiPublicKey = config.get('wopi.publicKey'); const cfgWopiModulus = config.get('wopi.modulus'); const cfgWopiExponent = config.get('wopi.exponent'); @@ -85,6 +91,13 @@ const cfgWopiPrivateKeyOld = config.get('wopi.privateKeyOld'); const cfgWopiHost = config.get('wopi.host'); const cfgWopiDummySampleFilePath = config.get('wopi.dummy.sampleFilePath'); +let cryptoSign = util.promisify(crypto.sign); + +let templatesFolderLocalesCache = null; +let templatesFolderExtsCache = null; +const templateFilesSizeCache = {}; +let shutdownFlag = false; + let mimeTypesByExt = (function() { let mimeTypesByExt = {}; for (let mimeType in mimeDB) { @@ -103,9 +116,24 @@ let mimeTypesByExt = (function() { return mimeTypesByExt; })(); +async function getTemplatesFolderExts(ctx){ + //find available template files + if (templatesFolderExtsCache === null) { + const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); + const dirContent = await readdir(`${tenNewFileTemplate}/${constants.TEMPLATES_DEFAULT_LOCALE}/`, { withFileTypes: true }); + templatesFolderExtsCache = dirContent + .filter(dirObject => dirObject.isFile()) + .reduce((result, item, index, array) => { + let ext = path.extname(item.name).substring(1); + result[ext] = ext; + return result; + }, {}); + } + return templatesFolderExtsCache; +} + function discovery(req, res) { return co(function*() { - let output = ''; const xml = xmlbuilder2.create({version: '1.0', encoding: 'utf-8'}); let ctx = new operationContext.Context(); try { @@ -114,15 +142,18 @@ function discovery(req, res) { ctx.logger.info('wopiDiscovery start'); const tenWopiWopiZone = ctx.getCfg('wopi.wopiZone', cfgWopiWopiZone); const tenWopiPdfView = ctx.getCfg('wopi.pdfView', cfgWopiPdfView); + const tenWopiPdfEdit = ctx.getCfg('wopi.pdfEdit', cfgWopiPdfEdit); const tenWopiWordView = ctx.getCfg('wopi.wordView', cfgWopiWordView); const tenWopiWordEdit = ctx.getCfg('wopi.wordEdit', cfgWopiWordEdit); const tenWopiCellView = ctx.getCfg('wopi.cellView', cfgWopiCellView); const tenWopiCellEdit = ctx.getCfg('wopi.cellEdit', cfgWopiCellEdit); const tenWopiSlideView = ctx.getCfg('wopi.slideView', cfgWopiSlideView); const tenWopiSlideEdit = ctx.getCfg('wopi.slideEdit', cfgWopiSlideEdit); + const tenWopiForms = ctx.getCfg('wopi.forms', cfgWopiForms); const tenWopiFavIconUrlWord = ctx.getCfg('wopi.favIconUrlWord', cfgWopiFavIconUrlWord); const tenWopiFavIconUrlCell = ctx.getCfg('wopi.favIconUrlCell', cfgWopiFavIconUrlCell); const tenWopiFavIconUrlSlide = ctx.getCfg('wopi.favIconUrlSlide', cfgWopiFavIconUrlSlide); + const tenWopiFavIconUrlPdf = ctx.getCfg('wopi.favIconUrlSlide', cfgWopiFavIconUrlPdf); const tenWopiPublicKey = ctx.getCfg('wopi.publicKey', cfgWopiPublicKey); const tenWopiModulus = ctx.getCfg('wopi.modulus', cfgWopiModulus); const tenWopiExponent = ctx.getCfg('wopi.exponent', cfgWopiExponent); @@ -132,20 +163,27 @@ function discovery(req, res) { const tenWopiHost = ctx.getCfg('wopi.host', cfgWopiHost); let baseUrl = tenWopiHost || utils.getBaseUrlByRequest(ctx, req); - let names = ['Word','Excel','PowerPoint']; - let favIconUrls = [tenWopiFavIconUrlWord, tenWopiFavIconUrlCell, tenWopiFavIconUrlSlide]; + let names = ['Word','Excel','PowerPoint','Pdf']; + let favIconUrls = [tenWopiFavIconUrlWord, tenWopiFavIconUrlCell, tenWopiFavIconUrlSlide, tenWopiFavIconUrlPdf]; let exts = [ - {targetext: 'docx', view: tenWopiPdfView.concat(tenWopiWordView), edit: tenWopiWordEdit}, + {targetext: 'docx', view: tenWopiWordView, edit: tenWopiWordEdit}, {targetext: 'xlsx', view: tenWopiCellView, edit: tenWopiCellEdit}, - {targetext: 'pptx', view: tenWopiSlideView, edit: tenWopiSlideEdit} + {targetext: 'pptx', view: tenWopiSlideView, edit: tenWopiSlideEdit}, + {targetext: null, view: tenWopiPdfView, edit: tenWopiPdfEdit} ]; + let documentTypes = [`word`, `cell`, `slide`, `pdf`]; + + let templatesFolderExtsCache = yield getTemplatesFolderExts(ctx); + let formsExts = tenWopiForms.reduce((result, item, index, array) => { + result[item] = item; + return result; + }, {}); let templateStart = `${baseUrl}/hosting/wopi`; let templateEnd = `<rs=DC_LLCC&><dchat=DISABLE_CHAT&><embed=EMBEDDED&>`; templateEnd += `<fs=FULLSCREEN&><hid=HOST_SESSION_ID&><rec=RECORDING&>`; templateEnd += `<sc=SESSION_CONTEXT&><thm=THEME_ID&><ui=UI_LLCC&>`; templateEnd += `<wopisrc=WOPI_SOURCE&>&`; - let documentTypes = [`word`, `cell`, `slide`]; let xmlZone = xml.ele('wopi-discovery').ele('net-zone', { name: tenWopiWopiZone }); //start section for MS WOPI connectors for(let i = 0; i < names.length; ++i) { @@ -160,12 +198,13 @@ function discovery(req, res) { let urlTemplateMobileView = `${templateStart}/${documentTypes[i]}/view?mobile=1&${templateEnd}`; let urlTemplateEdit = `${templateStart}/${documentTypes[i]}/edit?${templateEnd}`; let urlTemplateMobileEdit = `${templateStart}/${documentTypes[i]}/edit?mobile=1&${templateEnd}`; + let urlTemplateFormSubmit = `${templateStart}/${documentTypes[i]}/edit?formsubmit=1&${templateEnd}`; let xmlApp = xmlZone.ele('app', {name: name, favIconUrl: favIconUrl}); for (let j = 0; j < ext.view.length; ++j) { - xmlApp.ele('action', {name: 'view', ext: ext.view[j], urlsrc: urlTemplateView}).up(); + xmlApp.ele('action', {name: 'view', ext: ext.view[j], default: 'true', urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: ext.view[j], urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: ext.view[j], urlsrc: urlTemplateMobileView}).up(); - if (-1 === tenWopiPdfView.indexOf(ext.view[j])) { + if (ext.targetext) { let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; xmlApp.ele('action', {name: 'convert', ext: ext.view[j], targetext: ext.targetext, requires: 'update', urlsrc: urlConvert}).up(); } @@ -174,12 +213,16 @@ function discovery(req, res) { xmlApp.ele('action', {name: 'view', ext: ext.edit[j], urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: ext.edit[j], urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: ext.edit[j], urlsrc: urlTemplateMobileView}).up(); - if ("oform" !== ext.edit[j]) { - //todo config - xmlApp.ele('action', {name: 'editnew', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + if (formsExts[ext.edit[j]]) { + xmlApp.ele('action', {name: 'formsubmit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } else { + xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); } - xmlApp.ele('action', {name: 'edit', ext: ext.edit[j], default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); xmlApp.ele('action', {name: 'mobileEdit', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); + if (templatesFolderExtsCache[ext.edit[j]]) { + xmlApp.ele('action', {name: 'editnew', ext: ext.edit[j], requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } } xmlApp.up(); } @@ -192,6 +235,7 @@ function discovery(req, res) { let urlTemplateMobileView = `${templateStart}/${documentTypes[i]}/view?mobile=1&${templateEnd}`; let urlTemplateEdit = `${templateStart}/${documentTypes[i]}/edit?${templateEnd}`; let urlTemplateMobileEdit = `${templateStart}/${documentTypes[i]}/edit?mobile=1&${templateEnd}`; + let urlTemplateFormSubmit = `${templateStart}/${documentTypes[i]}/edit?formsubmit=1&${templateEnd}`; for (let j = 0; j < ext.view.length; ++j) { let mimeTypes = mimeTypesByExt[ext.view[j]]; if (mimeTypes) { @@ -200,7 +244,7 @@ function discovery(req, res) { xmlApp.ele('action', {name: 'view', ext: '', default: 'true', urlsrc: urlTemplateView}).up(); xmlApp.ele('action', {name: 'embedview', ext: '', urlsrc: urlTemplateEmbedView}).up(); xmlApp.ele('action', {name: 'mobileView', ext: '', urlsrc: urlTemplateMobileView}).up(); - if (-1 === tenWopiPdfView.indexOf(ext.view[j])) { + if (ext.targetext) { let urlConvert = `${templateStart}/convert-and-edit/${ext.view[j]}/${ext.targetext}?${templateEnd}`; xmlApp.ele('action', {name: 'convert', ext: '', targetext: ext.targetext, requires: 'update', urlsrc: urlConvert}).up(); } @@ -213,8 +257,16 @@ function discovery(req, res) { if (mimeTypes) { mimeTypes.forEach((value) => { let xmlApp = xmlZone.ele('app', {name: value}); - xmlApp.ele('action', {name: 'edit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + if (formsExts[ext.edit[j]]) { + xmlApp.ele('action', {name: 'formsubmit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateFormSubmit}).up(); + xmlApp.ele('action', {name: 'edit', ext: '', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } else { + xmlApp.ele('action', {name: 'edit', ext: '', default: 'true', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } xmlApp.ele('action', {name: 'mobileEdit', ext: '', requires: 'locks,update', urlsrc: urlTemplateMobileEdit}).up(); + if (templatesFolderExtsCache[ext.edit[j]]) { + xmlApp.ele('action', {name: 'editnew', ext: '', requires: 'locks,update', urlsrc: urlTemplateEdit}).up(); + } xmlApp.up(); }); } @@ -226,9 +278,11 @@ function discovery(req, res) { //end section for collabora nexcloud connectors let xmlDiscovery = xmlZone.up(); if (tenWopiPublicKeyOld && tenWopiPublicKey) { + let exponent = numberToBase64(tenWopiExponent); + let exponentOld = numberToBase64(tenWopiExponentOld); xmlDiscovery.ele('proof-key', { - oldvalue: tenWopiPublicKeyOld, oldmodulus: tenWopiModulusOld, oldexponent: tenWopiExponentOld, - value: tenWopiPublicKey, modulus: tenWopiModulus, exponent: tenWopiExponent + oldvalue: tenWopiPublicKeyOld, oldmodulus: tenWopiModulusOld, oldexponent: exponentOld, + value: tenWopiPublicKey, modulus: tenWopiModulus, exponent: exponent }).up(); } xmlDiscovery.up(); @@ -281,6 +335,33 @@ function getWopiUnlockMarker(wopiParams) { function getWopiModifiedMarker(wopiParams, lastModifiedTime) { return JSON.stringify(Object.assign({fileInfo: {LastModifiedTime: lastModifiedTime}}, wopiParams.userAuth)); } +function getFileTypeByInfo(fileInfo) { + let fileType = fileInfo.BaseFileName ? fileInfo.BaseFileName.substr(fileInfo.BaseFileName.lastIndexOf('.') + 1) : ""; + fileType = fileInfo.FileExtension ? fileInfo.FileExtension.substr(1) : fileType; + return fileType.toLowerCase(); +} +async function getWopiFileUrl(ctx, fileInfo, userAuth) { + const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); + let url; + let headers = {'X-WOPI-MaxExpectedSize': tenMaxDownloadBytes}; + if (fileInfo?.FileUrl) { + //Requests to the FileUrl can not be signed using proof keys. The FileUrl is used exactly as provided by the host, so it does not necessarily include the access token, which is required to construct the expected proof. + url = fileInfo.FileUrl; + } else if (fileInfo?.TemplateSource) { + url = fileInfo.TemplateSource; + } else if (userAuth) { + url = `${userAuth.wopiSrc}/contents?access_token=${userAuth.access_token}`; + await fillStandardHeaders(ctx, headers, url, userAuth.access_token); + } + ctx.logger.debug('getWopiFileUrl url=%s; headers=%j', url, headers); + return {url, headers}; +} +function isWopiJwtToken(decoded) { + return !!decoded.fileInfo; +} +function setIsShutdown(val) { + shutdownFlag = val; +} function getLastModifiedTimeFromCallbacks(callbacks) { for (let i = callbacks.length; i >= 0; --i) { let callback = callbacks[i]; @@ -339,7 +420,7 @@ function checkAndInvalidateCache(ctx, docId, fileInfo) { ctx.logger.debug('wopiEditor unlockMarkStr=%s', unlockMarkStr); let hasUnlockMarker = isWopiUnlockMarker(unlockMarkStr); ctx.logger.debug('wopiEditor hasUnlockMarker=%s', hasUnlockMarker); - if (hasUnlockMarker) { + if (hasUnlockMarker || !commonInfo.fileInfo.SupportsLocks) { let fileInfoVersion = fileInfo.Version; let cacheVersion = commonInfo.fileInfo.Version; let fileInfoModified = fileInfo.LastModifiedTime; @@ -366,9 +447,62 @@ function checkAndInvalidateCache(ctx, docId, fileInfo) { return res; }); } +function parsePutFileResponse(ctx, postRes) { + let body = null + if (postRes.body) { + try { + //collabora nexcloud connector + body = JSON.parse(postRes.body); + } catch (e) { + ctx.logger.debug('wopi PutFile body parse error: %s', e.stack); + } + } + return body; +} +async function checkAndReplaceEmptyFile(ctx, fileInfo, wopiSrc, access_token, access_token_ttl, lang, ui, fileType) { + // TODO: throw error if format not supported? + if (fileInfo.Size === 0 && fileType.length !== 0) { + const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); + + //Create new files using Office for the web + const wopiParams = getWopiParams(undefined, fileInfo, wopiSrc, access_token, access_token_ttl); + + if (templatesFolderLocalesCache === null) { + const dirContent = await readdir(`${tenNewFileTemplate}/`, {withFileTypes: true}); + templatesFolderLocalesCache = dirContent.filter(dirObject => dirObject.isDirectory()) + .map(dirObject => dirObject.name); + } + + const localePrefix = lang || ui || 'en'; + let locale = constants.TEMPLATES_FOLDER_LOCALE_COLLISON_MAP[localePrefix] ?? + templatesFolderLocalesCache.find(locale => locale.startsWith(localePrefix)); + if (locale === undefined) { + locale = constants.TEMPLATES_DEFAULT_LOCALE; + } + + const filePath = `${tenNewFileTemplate}/${locale}/new.${fileType}`; + if (!templateFilesSizeCache[filePath]) { + templateFilesSizeCache[filePath] = await lstat(filePath); + } + + const templateFileInfo = templateFilesSizeCache[filePath]; + const templateFileStream = createReadStream(filePath); + let postRes = await putFile(ctx, wopiParams, undefined, templateFileStream, templateFileInfo.size, fileInfo.UserId, false, false, false); + if (postRes) { + //update Size + fileInfo.Size = templateFileInfo.size; + let body = parsePutFileResponse(ctx, postRes); + //collabora nexcloud connector + if (body?.LastModifiedTime) { + //update LastModifiedTime + fileInfo.LastModifiedTime = body.LastModifiedTime; + } + } + } +} function getEditorHtml(req, res) { return co(function*() { - let params = {key: undefined, fileInfo: {}, userAuth: {}, queryParams: req.query, token: undefined, documentType: undefined}; + let params = {key: undefined, fileInfo: {}, userAuth: {}, queryParams: req.query, token: undefined, documentType: undefined, docs_api_config: {}}; let ctx = new operationContext.Context(); try { ctx.initFromRequest(req); @@ -390,8 +524,14 @@ function getEditorHtml(req, res) { let mode = req.params.mode; let sc = req.query['sc']; let hostSessionId = req.query['hid']; + let lang = req.query['lang']; + let ui = req.query['ui']; let access_token = req.body['access_token'] || ""; let access_token_ttl = parseInt(req.body['access_token_ttl']) || 0; + let docs_api_config = req.body['docs_api_config']; + if (docs_api_config) { + params.docs_api_config = JSON.parse(docs_api_config); + } let fileInfo = params.fileInfo = yield checkFileInfo(ctx, wopiSrc, access_token, sc); @@ -399,6 +539,10 @@ function getEditorHtml(req, res) { params.fileInfo = {}; return; } + const fileType = getFileTypeByInfo(fileInfo); + if (!shutdownFlag) { + yield checkAndReplaceEmptyFile(ctx, fileInfo, wopiSrc, access_token, access_token_ttl, lang, ui, fileType); + } if (!fileInfo.UserCanWrite) { mode = 'view'; @@ -432,21 +576,22 @@ function getEditorHtml(req, res) { params.fileInfo = {}; return; } - //save common info - if (undefined === lockId) { - let fileType = fileInfo.BaseFileName ? fileInfo.BaseFileName.substr(fileInfo.BaseFileName.lastIndexOf('.') + 1) : ""; - fileType = fileInfo.FileExtension ? fileInfo.FileExtension.substr(1) : fileType; - lockId = crypto.randomBytes(16).toString('base64'); - let commonInfo = JSON.stringify({lockId: lockId, fileInfo: fileInfo}); - yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByRequest(ctx, req), 1, commonInfo, fileType); - } + if (!shutdownFlag) { + //save common info + if (undefined === lockId) { + //Use deterministic(not random) lockId to fix issues with forgotten openings due to integrator failures + lockId = docId; + let commonInfo = JSON.stringify({lockId: lockId, fileInfo: fileInfo}); + yield canvasService.commandOpenStartPromise(ctx, docId, utils.getBaseUrlByRequest(ctx, req), commonInfo, fileType); + } - //Lock - if ('view' !== mode) { - let lockRes = yield lock(ctx, 'LOCK', lockId, fileInfo, userAuth); - if (!lockRes) { - params.fileInfo = {}; - return; + //Lock + if ('view' !== mode) { + let lockRes = yield lock(ctx, 'LOCK', lockId, fileInfo, userAuth); + if (!lockRes) { + params.fileInfo = {}; + return; + } } } @@ -497,7 +642,7 @@ function getConverterHtml(req, res) { let targetext = req.params.targetext; if (!(wopiSrc && access_token && access_token_ttl && ext && targetext)) { - ctx.logger.debug('convert-and-edit invalid params: wopiSrc=%s; access_token=%s; access_token_ttl=%s; ext=%s; targetext=%s', wopiSrc, access_token, access_token_ttl, ext, targetext); + ctx.logger.debug('convert-and-edit invalid params: WOPISrc=%s; access_token=%s; access_token_ttl=%s; ext=%s; targetext=%s', wopiSrc, access_token, access_token_ttl, ext, targetext); return; } @@ -507,21 +652,21 @@ function getConverterHtml(req, res) { return; } - let wopiParams = getWopiParams(null, fileInfo, wopiSrc, access_token, access_token_ttl); + let wopiParams = getWopiParams(undefined, fileInfo, wopiSrc, access_token, access_token_ttl); let docId = yield converterService.convertAndEdit(ctx, wopiParams, ext, targetext); if (docId) { let baseUrl = tenWopiHost || utils.getBaseUrlByRequest(ctx, req); params.statusHandler = `${baseUrl}/hosting/wopi/convert-and-edit-handler`; - params.statusHandler += `?wopiSrc=${encodeURI(wopiSrc)}&access_token=${encodeURI(access_token)}`; - params.statusHandler += `&targetext=${encodeURI(targetext)}&docId=${encodeURI(docId)}`; + params.statusHandler += `?${constants.SHARD_KEY_WOPI_NAME}=${encodeURIComponent(wopiSrc)}&access_token=${encodeURIComponent(access_token)}`; + params.statusHandler += `&targetext=${encodeURIComponent(targetext)}&docId=${encodeURIComponent(docId)}`; if (tenTokenEnableBrowser) { let tokenData = {docId: docId}; let options = {algorithm: tenTokenOutboxAlgorithm, expiresIn: tenTokenOutboxExpires}; let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Browser); let token = jwt.sign(tokenData, secret, options); - params.statusHandler += `&token=${encodeURI(token)}`; + params.statusHandler += `&token=${encodeURIComponent(token)}`; } } } catch (err) { @@ -561,7 +706,7 @@ function putFile(ctx, wopiParams, data, dataStream, dataSize, userLastChangeId, let commonInfo = wopiParams.commonInfo; //todo add all the users who contributed changes to the document in this PutFile request to X-WOPI-Editors let headers = {'X-WOPI-Override': 'PUT', 'X-WOPI-Lock': commonInfo.lockId, 'X-WOPI-Editors': userLastChangeId}; - fillStandardHeaders(ctx, headers, uri, userAuth.access_token); + yield fillStandardHeaders(ctx, headers, uri, userAuth.access_token); headers['X-LOOL-WOPI-IsModifiedByUser'] = isModifiedByUser; headers['X-LOOL-WOPI-IsAutosave'] = isAutosave; headers['X-LOOL-WOPI-IsExitSave'] = isExitSave; @@ -569,6 +714,7 @@ function putFile(ctx, wopiParams, data, dataStream, dataSize, userLastChangeId, //collabora nexcloud connector headers['X-LOOL-WOPI-Timestamp'] = wopiParams.LastModifiedTime; } + headers['Content-Type'] = mime.getType(getFileTypeByInfo(fileInfo)); ctx.logger.debug('wopi PutFile request uri=%s headers=%j', uri, headers); postRes = yield utils.postRequestPromise(ctx, uri, data, dataStream, dataSize, tenCallbackRequestTimeout, undefined, headers); @@ -598,9 +744,12 @@ function putRelativeFile(ctx, wopiSrc, access_token, data, dataStream, dataSize, return postRes; } - let headers = {'X-WOPI-Override': 'PUT_RELATIVE', 'X-WOPI-SuggestedTarget': utf7.encode(suggestedTarget), - 'X-WOPI-FileConversion': isFileConversion}; - fillStandardHeaders(ctx, headers, uri, access_token); + let headers = {'X-WOPI-Override': 'PUT_RELATIVE', 'X-WOPI-SuggestedTarget': utf7.encode(suggestedTarget)}; + if (isFileConversion) { + headers['X-WOPI-FileConversion'] = isFileConversion; + } + yield fillStandardHeaders(ctx, headers, uri, access_token); + headers['Content-Type'] = mime.getType(suggestedTarget); ctx.logger.debug('wopi putRelativeFile request uri=%s headers=%j', uri, headers); postRes = yield utils.postRequestPromise(ctx, uri, data, dataStream, dataSize, tenCallbackRequestTimeout, undefined, headers); @@ -638,7 +787,7 @@ function renameFile(ctx, wopiParams, name) { let commonInfo = wopiParams.commonInfo; let headers = {'X-WOPI-Override': 'RENAME_FILE', 'X-WOPI-Lock': commonInfo.lockId, 'X-WOPI-RequestedName': utf7.encode(name)}; - fillStandardHeaders(ctx, headers, uri, userAuth.access_token); + yield fillStandardHeaders(ctx, headers, uri, userAuth.access_token); ctx.logger.debug('wopi RenameFile request uri=%s headers=%j', uri, headers); let postRes = yield utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); @@ -665,7 +814,6 @@ function checkFileInfo(ctx, wopiSrc, access_token, opt_sc) { let fileInfo = undefined; try { ctx.logger.info('wopi checkFileInfo start'); - const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests); const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout); let uri = `${encodeURI(wopiSrc)}?access_token=${encodeURIComponent(access_token)}`; @@ -677,10 +825,12 @@ function checkFileInfo(ctx, wopiSrc, access_token, opt_sc) { if (opt_sc) { headers['X-WOPI-SessionContext'] = opt_sc; } - fillStandardHeaders(ctx, headers, uri, access_token); + yield fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi checkFileInfo request uri=%s headers=%j', uri, headers); - const filterPrivate = !tenAllowPrivateIPAddressForSignedRequests; - let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, undefined, undefined, filterPrivate, headers); + //todo false? (true because it passed checkIpFilter for wopi) + //todo use directIfIn + let isInJwtToken = true; + let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, undefined, undefined, isInJwtToken, headers); ctx.logger.debug(`wopi checkFileInfo headers=%j body=%s`, getRes.response.headers, getRes.body); fileInfo = JSON.parse(getRes.body); } catch (err) { @@ -711,7 +861,7 @@ function lock(ctx, command, lockId, fileInfo, userAuth) { } let headers = {"X-WOPI-Override": command, "X-WOPI-Lock": lockId}; - fillStandardHeaders(ctx, headers, uri, access_token); + yield fillStandardHeaders(ctx, headers, uri, access_token); ctx.logger.debug('wopi %s request uri=%s headers=%j', command, uri, headers); let postRes = yield utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); ctx.logger.debug('wopi %s response headers=%j', command, postRes.response.headers); @@ -727,40 +877,41 @@ function lock(ctx, command, lockId, fileInfo, userAuth) { return res; }); } -function unlock(ctx, wopiParams) { - return co(function* () { - try { - ctx.logger.info('wopi Unlock start'); - const tenCallbackRequestTimeout = ctx.getCfg('services.CoAuthoring.server.callbackRequestTimeout', cfgCallbackRequestTimeout); +async function unlock(ctx, wopiParams) { + let res = false; + try { + ctx.logger.info('wopi Unlock start'); + const tenCallbackRequestTimeout = ctx.getCfg('services.CoAuthoring.server.callbackRequestTimeout', cfgCallbackRequestTimeout); - if (!wopiParams.userAuth || !wopiParams.commonInfo) { + if (!wopiParams.userAuth || !wopiParams.commonInfo) { + return; + } + let fileInfo = wopiParams.commonInfo.fileInfo; + if (fileInfo && fileInfo.SupportsLocks) { + let wopiSrc = wopiParams.userAuth.wopiSrc; + let lockId = wopiParams.commonInfo.lockId; + let access_token = wopiParams.userAuth.access_token; + let uri = `${wopiSrc}?access_token=${access_token}`; + let filterStatus = await checkIpFilter(ctx, uri); + if (0 !== filterStatus) { return; } - let fileInfo = wopiParams.commonInfo.fileInfo; - if (fileInfo && fileInfo.SupportsLocks) { - let wopiSrc = wopiParams.userAuth.wopiSrc; - let lockId = wopiParams.commonInfo.lockId; - let access_token = wopiParams.userAuth.access_token; - let uri = `${wopiSrc}?access_token=${access_token}`; - let filterStatus = yield checkIpFilter(ctx, uri); - if (0 !== filterStatus) { - return; - } - let headers = {"X-WOPI-Override": "UNLOCK", "X-WOPI-Lock": lockId}; - fillStandardHeaders(ctx, headers, uri, access_token); - ctx.logger.debug('wopi Unlock request uri=%s headers=%j', uri, headers); - let postRes = yield utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); - ctx.logger.debug('wopi Unlock response headers=%j', postRes.response.headers); - } else { - ctx.logger.info('wopi SupportsLocks = false'); - } - } catch (err) { - ctx.logger.error('wopi error Unlock:%s', err.stack); - } finally { - ctx.logger.info('wopi Unlock end'); + let headers = {"X-WOPI-Override": "UNLOCK", "X-WOPI-Lock": lockId}; + await fillStandardHeaders(ctx, headers, uri, access_token); + ctx.logger.debug('wopi Unlock request uri=%s headers=%j', uri, headers); + let postRes = await utils.postRequestPromise(ctx, uri, undefined, undefined, undefined, tenCallbackRequestTimeout, undefined, headers); + ctx.logger.debug('wopi Unlock response headers=%j', postRes.response.headers); + res = true; + } else { + ctx.logger.info('wopi SupportsLocks = false'); } - }); + } catch (err) { + ctx.logger.error('wopi error Unlock:%s', err.stack); + } finally { + ctx.logger.info('wopi Unlock end'); + } + return res; } function generateProofBuffer(url, accessToken, timeStamp) { const accessTokenBytes = Buffer.from(accessToken, 'utf8'); @@ -781,28 +932,32 @@ function generateProofBuffer(url, accessToken, timeStamp) { buffer.writeBigUInt64BE(timeStamp, offset); return buffer; } -function generateProofSign(url, accessToken, timeStamp, privateKey) { - let signer = crypto.createSign('RSA-SHA256'); - signer.update(generateProofBuffer(url, accessToken, timeStamp)); - return signer.sign({key:privateKey}, "base64"); -} -function generateProof(ctx, url, accessToken, timeStamp) { - const tenWopiPrivateKey = ctx.getCfg('wopi.privateKey', cfgWopiPrivateKey); - let privateKey = `-----BEGIN RSA PRIVATE KEY-----\n${tenWopiPrivateKey}\n-----END RSA PRIVATE KEY-----`; - return generateProofSign(url, accessToken, timeStamp, privateKey); + +async function generateProofSign(url, accessToken, timeStamp, privateKey) { + let data = generateProofBuffer(url, accessToken, timeStamp); + let sign = await cryptoSign('RSA-SHA256', data, privateKey); + return sign.toString('base64'); } -function generateProofOld(ctx, url, accessToken, timeStamp) { - const tenWopiPrivateKeyOld = ctx.getCfg('wopi.privateKeyOld', cfgWopiPrivateKeyOld); - let privateKey = `-----BEGIN RSA PRIVATE KEY-----\n${tenWopiPrivateKeyOld}\n-----END RSA PRIVATE KEY-----`; - return generateProofSign(url, accessToken, timeStamp, privateKey); + +function numberToBase64(val) { + // Convert to hexadecimal + let hexString = val.toString(16); + //Ensure the hexadecimal string has an even length + if (hexString.length % 2 !== 0) { + hexString = '0' + hexString; + } + //Convert the hexadecimal string to a buffer + const buffer = Buffer.from(hexString, 'hex'); + return buffer.toString('base64'); } -function fillStandardHeaders(ctx, headers, url, access_token) { + +async function fillStandardHeaders(ctx, headers, url, access_token) { let timeStamp = utils.getDateTimeTicks(new Date()); const tenWopiPrivateKey = ctx.getCfg('wopi.privateKey', cfgWopiPrivateKey); const tenWopiPrivateKeyOld = ctx.getCfg('wopi.privateKeyOld', cfgWopiPrivateKeyOld); if (tenWopiPrivateKey && tenWopiPrivateKeyOld) { - headers['X-WOPI-Proof'] = generateProof(ctx, url, access_token, timeStamp); - headers['X-WOPI-ProofOld'] = generateProofOld(ctx, url, access_token, timeStamp); + headers['X-WOPI-Proof'] = await generateProofSign(url, access_token, timeStamp, tenWopiPrivateKey); + headers['X-WOPI-ProofOld'] = await generateProofSign(url, access_token, timeStamp, tenWopiPrivateKeyOld); headers['X-WOPI-TimeStamp'] = timeStamp; headers['X-WOPI-ClientVersion'] = commonDefines.buildVersion + '.' + commonDefines.buildNumber; // todo @@ -884,11 +1039,16 @@ async function dummyGetFile(req, res) { res.setHeader('Content-Type', mime.getType(tenWopiDummySampleFilePath)); await pipeline( - fs.createReadStream(tenWopiDummySampleFilePath), + createReadStream(tenWopiDummySampleFilePath), res, ); } catch (err) { - ctx.logger.error('dummyGetFile error:%s', err.stack); + if (err.code === "ERR_STREAM_PREMATURE_CLOSE") { + //xhr.abort case + ctx.logger.debug('dummyGetFile error: %s', err.stack); + } else { + ctx.logger.error('dummyGetFile error:%s', err.stack); + } } finally { if (!res.headersSent) { res.sendStatus(400); @@ -899,21 +1059,25 @@ function dummyOk(req, res) { res.sendStatus(200); } +exports.checkIpFilter = checkIpFilter; exports.discovery = discovery; exports.collaboraCapabilities = collaboraCapabilities; exports.parseWopiCallback = parseWopiCallback; exports.getEditorHtml = getEditorHtml; exports.getConverterHtml = getConverterHtml; exports.putFile = putFile; +exports.parsePutFileResponse = parsePutFileResponse; exports.putRelativeFile = putRelativeFile; exports.renameFile = renameFile; exports.lock = lock; exports.unlock = unlock; -exports.generateProof = generateProof; -exports.generateProofOld = generateProofOld; exports.fillStandardHeaders = fillStandardHeaders; exports.getWopiUnlockMarker = getWopiUnlockMarker; exports.getWopiModifiedMarker = getWopiModifiedMarker; +exports.getFileTypeByInfo = getFileTypeByInfo; +exports.getWopiFileUrl = getWopiFileUrl; +exports.isWopiJwtToken = isWopiJwtToken; +exports.setIsShutdown = setIsShutdown; exports.dummyCheckFileInfo = dummyCheckFileInfo; exports.dummyGetFile = dummyGetFile; exports.dummyOk = dummyOk; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 41341913f..436f87f58 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -46,7 +46,7 @@ var commonDefines = require('./../../Common/sources/commondefines'); var storage = require('./../../Common/sources/storage-base'); var utils = require('./../../Common/sources/utils'); var constants = require('./../../Common/sources/constants'); -var baseConnector = require('./../../DocService/sources/baseConnector'); +var baseConnector = require('../../DocService/sources/databaseConnectors/baseConnector'); const wopiClient = require('./../../DocService/sources/wopiClient'); const taskResult = require('./../../DocService/sources/taskresult'); var statsDClient = require('./../../Common/sources/statsdclient'); @@ -74,8 +74,9 @@ const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles const cfgForgottenFilesName = config.get('services.CoAuthoring.server.forgottenfilesname'); const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate'); const cfgEditor = config.get('services.CoAuthoring.editor'); -const cfgAllowPrivateIPAddressForSignedRequests = config.get('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests'); const cfgRequesFilteringAgent = config.get('services.CoAuthoring.request-filtering-agent'); +const cfgExternalRequestDirectIfIn = config.get('externalRequest.directIfIn'); +const cfgExternalRequestAction = config.get('externalRequest.action'); //windows limit 512(2048) https://msdn.microsoft.com/en-us/library/6e3b887c.aspx //Ubuntu 14.04 limit 4096 http://underyx.me/2015/05/18/raising-the-maximum-number-of-file-descriptors.html @@ -90,6 +91,7 @@ var exitCodesReturn = [constants.CONVERT_PARAMS, constants.CONVERT_NEED_PARAMS, var exitCodesMinorError = [constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM, constants.CONVERT_DRM_UNSUPPORTED, constants.CONVERT_PASSWORD]; var exitCodesUpload = [constants.NO_ERROR, constants.CONVERT_CORRUPTED, constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM, constants.CONVERT_DRM_UNSUPPORTED]; +var exitCodesCopyOrigin = [constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM]; let inputLimitsXmlCache; function TaskQueueDataConvert(ctx, task) { @@ -122,7 +124,7 @@ function TaskQueueDataConvert(ctx, task) { this.mailMergeSend = cmd.mailmergesend; this.thumbnail = cmd.thumbnail; this.textParams = cmd.getTextParams(); - this.jsonParams = cmd.getJsonParams(); + this.jsonParams = JSON.stringify(cmd.getJsonParams()); this.lcid = cmd.getLCID(); this.password = cmd.getPassword(); this.savePassword = cmd.getSavePassword(); @@ -164,7 +166,7 @@ TaskQueueDataConvert.prototype = { xml += this.serializeXmlProp('m_bIsNoBase64', this.noBase64); xml += this.serializeXmlProp('m_sConvertToOrigin', this.convertToOrigin); xml += this.serializeLimit(ctx); - xml += this.serializeOptions(ctx); + xml += this.serializeOptions(ctx, false); xml += ''; fs.writeFileSync(fsPath, xml, {encoding: 'utf8'}); }, @@ -187,12 +189,45 @@ TaskQueueDataConvert.prototype = { return xml; }); }, - serializeOptions: function (ctx) { + serializeOptions: function (ctx, isInJwtToken) { const tenRequesFilteringAgent = ctx.getCfg('services.CoAuthoring.request-filtering-agent', cfgRequesFilteringAgent); + const tenExternalRequestDirectIfIn = ctx.getCfg('externalRequest.directIfIn', cfgExternalRequestDirectIfIn); + const tenExternalRequestAction = ctx.getCfg('externalRequest.action', cfgExternalRequestAction); + let allowList = tenExternalRequestDirectIfIn.allowList; + let allowNetworkRequest = tenExternalRequestAction.allow; + let allowPrivateIP = !tenExternalRequestAction.blockPrivateIP && tenRequesFilteringAgent.allowPrivateIPAddress; + let proxyUrl = tenExternalRequestAction.proxyUrl; + let proxyUser = tenExternalRequestAction.proxyUser; + let proxyHeaders = tenExternalRequestAction.proxyHeaders; + if (allowList.length === 0 && tenExternalRequestDirectIfIn.jwtToken && isInJwtToken) { + allowNetworkRequest = true; + allowPrivateIP = true; + proxyUrl = ""; + proxyUser = null; + proxyHeaders = {}; + } let xml = ""; xml += ''; - xml += this.serializeXmlProp('allowNetworkRequest', true); - xml += this.serializeXmlProp('allowPrivateIP', tenRequesFilteringAgent.allowPrivateIPAddress); + if (allowList.length > 0) { + xml += this.serializeXmlProp('allowList', allowList.join(';')); + } + xml += this.serializeXmlProp('allowNetworkRequest', allowNetworkRequest); + xml += this.serializeXmlProp('allowPrivateIP', allowPrivateIP); + if (proxyUrl) { + xml += this.serializeXmlProp('proxy', proxyUrl); + } + if (proxyUser) { + let user = proxyUser.username; + let pass = proxyUser.password; + xml += this.serializeXmlProp('proxyUser', `${user}:${pass}`); + } + let proxyHeadersStr= []; + for (let name in proxyHeaders) { + proxyHeadersStr.push(`${name}:${proxyHeaders[name]}`); + } + if (proxyHeadersStr.length > 0) { + xml += this.serializeXmlProp('proxyHeader', proxyHeadersStr.join(';')); + } xml += ''; return xml; }, @@ -257,6 +292,7 @@ TaskQueueDataConvert.prototype = { }, serializeXmlProp: function(name, value) { var xml = ''; + //todo check empty and undefined (password?) if (null != value) { xml += '<' + name + '>'; xml += utils.encodeXml(value.toString()); @@ -306,13 +342,15 @@ function* isUselessConvertion(ctx, task, cmd) { return constants.NO_ERROR; } async function changeFormatToExtendedPdf(ctx, dataConvert, cmd) { + let forceSave = cmd.getForceSave(); + let isSendForm = forceSave && forceSave.getType() === commonDefines.c_oAscForceSaveTypes.Form; let originFormat = cmd.getOriginFormat(); let isOriginFormatWithForms = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === originFormat || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM === originFormat || constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCXF === originFormat; let isFormatToPdf = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === dataConvert.formatTo || constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === dataConvert.formatTo; - if (isFormatToPdf && isOriginFormatWithForms) { + if (isFormatToPdf && isOriginFormatWithForms && !isSendForm) { let format = await formatChecker.getDocumentFormatByFile(dataConvert.fileFrom); if (constants.AVS_OFFICESTUDIO_FILE_CANVAS_WORD === format) { ctx.logger.debug('change format to extended pdf'); @@ -323,7 +361,7 @@ async function changeFormatToExtendedPdf(ctx, dataConvert, cmd) { function* replaceEmptyFile(ctx, fileFrom, ext, _lcid) { const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate); if (!fs.existsSync(fileFrom) || 0 === fs.lstatSync(fileFrom).size) { - let locale = 'en-US'; + let locale = constants.TEMPLATES_DEFAULT_LOCALE; if (_lcid) { let localeNew = lcid.from(_lcid); if (localeNew) { @@ -335,18 +373,28 @@ function* replaceEmptyFile(ctx, fileFrom, ext, _lcid) { } } } - ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); - let format = formatChecker.getFormatFromString(ext); - if (formatChecker.isDocumentFormat(format)) { - fs.copyFileSync(path.join(tenNewFileTemplate, locale, 'new.docx'), fileFrom); - } else if (formatChecker.isSpreadsheetFormat(format)) { - fs.copyFileSync(path.join(tenNewFileTemplate, locale, 'new.xlsx'), fileFrom); - } else if (formatChecker.isPresentationFormat(format)) { - fs.copyFileSync(path.join(tenNewFileTemplate, locale, 'new.pptx'), fileFrom); + let fileTemplatePath = path.join(tenNewFileTemplate, locale, 'new.'); + if (fs.existsSync(fileTemplatePath + ext)) { + ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); + fs.copyFileSync(fileTemplatePath + ext, fileFrom); + } else { + let format = formatChecker.getFormatFromString(ext); + let editorFormat; + if (formatChecker.isDocumentFormat(format)) { + editorFormat = 'docx'; + } else if (formatChecker.isSpreadsheetFormat(format)) { + editorFormat = 'xlsx'; + } else if (formatChecker.isPresentationFormat(format)) { + editorFormat = 'pptx'; + } + if (fs.existsSync(fileTemplatePath + editorFormat)) { + ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale); + fs.copyFileSync(fileTemplatePath + editorFormat, fileFrom); + } } } } -function* downloadFile(ctx, uri, fileFrom, withAuthorization, filterPrivate, opt_headers) { +function* downloadFile(ctx, uri, fileFrom, withAuthorization, isInJwtToken, opt_headers) { const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout); const tenDownloadAttemptMaxCount = ctx.getCfg('FileConverter.converter.downloadAttemptMaxCount', cfgDownloadAttemptMaxCount); @@ -365,7 +413,7 @@ function* downloadFile(ctx, uri, fileFrom, withAuthorization, filterPrivate, opt let secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox); authorization = utils.fillJwtForRequest(ctx, {url: uri}, secret, false); } - let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, tenMaxDownloadBytes, authorization, filterPrivate, opt_headers); + let getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, tenMaxDownloadBytes, authorization, isInJwtToken, opt_headers); data = getRes.body; sha256 = getRes.sha256; res = constants.NO_ERROR; @@ -394,7 +442,6 @@ function* downloadFile(ctx, uri, fileFrom, withAuthorization, filterPrivate, opt return res; } function* downloadFileFromStorage(ctx, strPath, dir, opt_specialDir) { - const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); var list = yield storage.listObjects(ctx, strPath, opt_specialDir); ctx.logger.debug('downloadFileFromStorage list %s', list.toString()); //create dirs @@ -860,6 +907,13 @@ function* postProcess(ctx, cmd, dataConvert, tempDirs, childRes, error, isTimeou ctx.logger.debug('ExitCode (code=%d;signal=%s;error:%d)', exitCode, exitSignal, error); } if (-1 !== exitCodesUpload.indexOf(error)) { + if (-1 !== exitCodesCopyOrigin.indexOf(error)) { + let originPath = path.join(path.dirname(dataConvert.fileTo), "origin" + path.extname(dataConvert.fileFrom)); + if (!fs.existsSync(dataConvert.fileTo)) { + fs.copyFileSync(dataConvert.fileFrom, originPath); + ctx.logger.debug('copyOrigin complete'); + } + } //todo clarify calcChecksum conditions let calcChecksum = (0 === (constants.AVS_OFFICESTUDIO_FILE_CANVAS & cmd.getOutputFormat())); yield* processUploadToStorage(ctx, tempDirs.result, dataConvert.key, calcChecksum); @@ -897,7 +951,7 @@ function* postProcess(ctx, cmd, dataConvert, tempDirs, childRes, error, isTimeou return queueData; } -function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task) { +function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken) { const tenX2tPath = ctx.getCfg('FileConverter.converter.x2tPath', cfgX2tPath); const tenDocbuilderPath = ctx.getCfg('FileConverter.converter.docbuilderPath', cfgDocbuilderPath); const tenArgs = ctx.getCfg('FileConverter.converter.args', cfgArgs); @@ -926,7 +980,7 @@ function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, g if (builderParams.argument) { childArgs.push(`--argument=${JSON.stringify(builderParams.argument)}`); } - childArgs.push('--options=' + dataConvert.serializeOptions(ctx)); + childArgs.push('--options=' + dataConvert.serializeOptions(ctx, isInJwtToken)); childArgs.push(dataConvert.fileFrom); } let timeoutId; @@ -967,10 +1021,8 @@ function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, g } function* ExecuteTask(ctx, task) { - const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes); const tenForgottenFiles = ctx.getCfg('services.CoAuthoring.server.forgottenfiles', cfgForgottenFiles); const tenForgottenFilesName = ctx.getCfg('services.CoAuthoring.server.forgottenfilesname', cfgForgottenFilesName); - const tenAllowPrivateIPAddressForSignedRequests = ctx.getCfg('services.CoAuthoring.server.allowPrivateIPAddressForSignedRequests', cfgAllowPrivateIPAddressForSignedRequests); var startDate = null; var curDate = null; if(clientStatsD) { @@ -988,6 +1040,7 @@ function* ExecuteTask(ctx, task) { dataConvert.fileTo = fileTo ? path.join(tempDirs.result, fileTo) : ''; let builderParams = cmd.getBuilderParams(); let authorProps = {lastModifiedBy: null, modified: null}; + let isInJwtToken = cmd.getWithAuthorization(); error = yield* isUselessConvertion(ctx, task, cmd); if (constants.NO_ERROR !== error) { ; @@ -997,30 +1050,18 @@ function* ExecuteTask(ctx, task) { if (utils.checkPathTraversal(ctx, dataConvert.key, tempDirs.source, dataConvert.fileFrom)) { let url = cmd.getUrl(); let withAuthorization = cmd.getWithAuthorization(); - let filterPrivate = !withAuthorization || !tenAllowPrivateIPAddressForSignedRequests; let headers; let fileSize; let wopiParams = cmd.getWopiParams(); if (wopiParams) { withAuthorization = false; - filterPrivate = !tenAllowPrivateIPAddressForSignedRequests; + isInJwtToken = true; let fileInfo = wopiParams.commonInfo?.fileInfo; - let userAuth = wopiParams.userAuth; fileSize = fileInfo?.Size; - if (fileInfo?.FileUrl) { - //Requests to the FileUrl can not be signed using proof keys. The FileUrl is used exactly as provided by the host, so it does not necessarily include the access token, which is required to construct the expected proof. - url = fileInfo.FileUrl; - } else if (fileInfo?.TemplateSource) { - url = fileInfo.TemplateSource; - } else if (userAuth) { - url = `${userAuth.wopiSrc}/contents?access_token=${userAuth.access_token}`; - headers = {'X-WOPI-MaxExpectedSize': tenMaxDownloadBytes}; - wopiClient.fillStandardHeaders(ctx, headers, url, userAuth.access_token); - } - ctx.logger.debug('wopi url=%s; headers=%j', url, headers); + ({url, headers} = yield wopiClient.getWopiFileUrl(ctx, fileInfo, wopiParams.userAuth)); } if (undefined === fileSize || fileSize > 0) { - error = yield* downloadFile(ctx, url, dataConvert.fileFrom, withAuthorization, filterPrivate, headers); + error = yield* downloadFile(ctx, url, dataConvert.fileFrom, withAuthorization, isInJwtToken, headers); } if (constants.NO_ERROR === error) { yield* replaceEmptyFile(ctx, dataConvert.fileFrom, format, cmd.getLCID()); @@ -1066,7 +1107,7 @@ function* ExecuteTask(ctx, task) { let childRes = null; let isTimeout = false; if (constants.NO_ERROR === error) { - ({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task)); + ({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken)); if (childRes && 0 !== childRes.status && !isTimeout && task.getFromChanges() && constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML !== dataConvert.formatTo && !formatChecker.isOOXFormat(dataConvert.formatTo) && !cmd.getWopiParams()) { @@ -1075,7 +1116,7 @@ function* ExecuteTask(ctx, task) { let extNew = '.' + formatChecker.getStringFromFormat(constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML); dataConvert.formatTo = constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML; dataConvert.fileTo = dataConvert.fileTo.slice(0, -extOld.length) + extNew; - ({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task)); + ({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken)); } if(clientStatsD) { clientStatsD.timing('conv.spawnSync', new Date() - curDate); diff --git a/SpellChecker/npm-shrinkwrap.json b/SpellChecker/npm-shrinkwrap.json index ecc51d9d3..7cd0edf11 100644 --- a/SpellChecker/npm-shrinkwrap.json +++ b/SpellChecker/npm-shrinkwrap.json @@ -5,40 +5,54 @@ "requires": true, "dependencies": { "accepts": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", - "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "requires": { - "mime-types": "~2.1.18", - "negotiator": "0.6.1" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" } }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, "body-parser": { - "version": "1.18.3", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", - "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "requires": { - "bytes": "3.0.0", - "content-type": "~1.0.4", + "bytes": "3.1.2", + "content-type": "~1.0.5", "debug": "2.6.9", - "depd": "~1.1.2", - "http-errors": "~1.6.3", - "iconv-lite": "0.4.23", - "on-finished": "~2.3.0", - "qs": "6.5.2", - "raw-body": "2.3.3", - "type-is": "~1.6.16" + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" } }, "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "call-bind": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "requires": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + } }, "co": { "version": "4.6.0", @@ -54,24 +68,34 @@ } }, "content-disposition": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", - "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } + } }, "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" }, "cookie": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", - "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" }, "cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" }, "debug": { "version": "2.6.9", @@ -81,71 +105,102 @@ "ms": "2.0.0" } }, + "define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "requires": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + } + }, "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" }, "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" + }, + "es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "requires": { + "get-intrinsic": "^1.2.4" + } + }, + "es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, "express": { - "version": "4.16.4", - "resolved": "https://registry.npmjs.org/express/-/express-4.16.4.tgz", - "integrity": "sha512-j12Uuyb4FMrd/qQAm6uCHAkPtO8FDTRJZBDd5D2KOL2eLaz1yUNdUB/NOIyq0iU4q4cFarsUCrnFDPBcnksuOg==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "requires": { - "accepts": "~1.3.5", + "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.18.3", - "content-disposition": "0.5.2", + "body-parser": "1.20.2", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.3.1", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", - "depd": "~1.1.2", + "depd": "2.0.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.1.1", + "finalhandler": "1.2.0", "fresh": "0.5.2", + "http-errors": "2.0.0", "merge-descriptors": "1.0.1", "methods": "~1.1.2", - "on-finished": "~2.3.0", - "parseurl": "~1.3.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.4", - "qs": "6.5.2", - "range-parser": "~1.2.0", - "safe-buffer": "5.1.2", - "send": "0.16.2", - "serve-static": "1.13.2", - "setprototypeof": "1.1.0", - "statuses": "~1.4.0", - "type-is": "~1.6.16", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } } }, "faye-websocket": { @@ -157,38 +212,90 @@ } }, "finalhandler": { - "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", - "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", + "version": "1.2.0", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "requires": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "on-finished": "~2.3.0", - "parseurl": "~1.3.2", - "statuses": "~1.4.0", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", "unpipe": "~1.0.0" } }, "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + }, + "function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" + }, + "get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "requires": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + } + }, + "gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "requires": { + "get-intrinsic": "^1.1.3" + } + }, + "has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "requires": { + "es-define-property": "^1.0.0" + } + }, + "has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==" + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "requires": { + "function-bind": "^1.1.2" + } }, "http-errors": { - "version": "1.6.3", - "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", - "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "version": "2.0.0", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", "requires": { - "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.0", - "statuses": ">= 1.4.0 < 2" + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" } }, "http-parser-js": { @@ -197,22 +304,22 @@ "integrity": "sha512-t7hjvef/5HEK7RWTdUzVUhl8zkEu+LlaE0IYzdMuvbSDipxBRpOn4Uhw8ZyECEa808iVT8XCjzo6xmYt4CiLZg==" }, "iconv-lite": { - "version": "0.4.23", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", - "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "requires": { "safer-buffer": ">= 2.1.2 < 3" } }, "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ipaddr.js": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", - "integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4=" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, "json5": { "version": "1.0.1", @@ -225,34 +332,34 @@ "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" }, "mime": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", - "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" }, "mime-db": { - "version": "1.37.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", - "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==" + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, "mime-types": { - "version": "2.1.21", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", - "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "requires": { - "mime-db": "~1.37.0" + "mime-db": "1.52.0" } }, "minimist": { @@ -263,12 +370,12 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, "negotiator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", - "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, "node-addon-api": { "version": "3.0.2", @@ -282,51 +389,59 @@ "node-addon-api": "*" } }, + "object-inspect": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", + "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==" + }, "on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "requires": { "ee-first": "1.1.1" } }, "parseurl": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", - "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, "proxy-addr": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", - "integrity": "sha512-5erio2h9jp5CHGwcybmxmVqHmnCBZeewlfJ0pex+UW7Qny7OOZXTtH56TGNyBizkgiOwhJtMKrVzDTeKcySZwA==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "requires": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.8.0" + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" } }, "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } }, "range-parser": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", - "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, "raw-body": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", - "integrity": "sha512-9esiElv1BrZoI3rCDuOuKCBRbuApGGaDPQfjSflGxdy4oyzqghxu6klEkkVIvBje+FF0BX9coEv8KqW6X/7njw==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.3", - "iconv-lite": "0.4.23", + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", "unpipe": "1.0.0" } }, @@ -341,40 +456,71 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "send": { - "version": "0.16.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", - "integrity": "sha512-E64YFPUssFHEFBvpbbjr44NCLtI1AohxQ8ZSiJjQLskAdKuriYEP6VyGEsRDH8ScozGpkaX1BGvhanqCwkcEZw==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "requires": { "debug": "2.6.9", - "depd": "~1.1.2", - "destroy": "~1.0.4", + "depd": "2.0.0", + "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.6.2", - "mime": "1.4.1", - "ms": "2.0.0", - "on-finished": "~2.3.0", - "range-parser": "~1.2.0", - "statuses": "~1.4.0" + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } } }, "serve-static": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", - "integrity": "sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", - "parseurl": "~1.3.2", - "send": "0.16.2" + "parseurl": "~1.3.3", + "send": "0.18.0" + } + }, + "set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "requires": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" } }, "setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "side-channel": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "requires": { + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + } }, "sockjs": { "version": "0.3.21", @@ -387,28 +533,33 @@ } }, "statuses": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", - "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, "type-is": { - "version": "1.6.16", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", - "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==", + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "requires": { "media-typer": "0.3.0", - "mime-types": "~2.1.18" + "mime-types": "~2.1.24" } }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, "uuid": { "version": "3.4.0", @@ -418,7 +569,7 @@ "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, "websocket-driver": { "version": "0.7.4", diff --git a/SpellChecker/package.json b/SpellChecker/package.json index ebb74c971..62d2d250b 100644 --- a/SpellChecker/package.json +++ b/SpellChecker/package.json @@ -7,7 +7,7 @@ "dependencies": { "co": "4.6.0", "config": "2.0.1", - "express": "4.16.4", + "express": "4.19.2", "nodehun": "git+https://git@github.com/ONLYOFFICE/nodehun.git#2411a56828c7d58214c61781b4a5c63d18adba99", "sockjs": "0.3.21" }, diff --git a/package.json b/package.json index 359730ecc..33b596698 100644 --- a/package.json +++ b/package.json @@ -71,8 +71,9 @@ "scripts": { "perf-expired": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/checkFileExpire.js", "perf-exif": "cd ./DocService&& cross-env NODE_ENV=development-windows NODE_CONFIG_DIR=../Common/config node ../tests/perf/fixImageExifRotation.js", - "unit tests": "cd ./DocService && jest unit --config=../tests/jest.config.js", - "integration tests": "cd ./DocService && jest integration --config=../tests/jest.config.js", + "unit tests": "cd ./DocService && jest unit --inject-globals=false --config=../tests/jest.config.js", + "integration tests with server instance": "cd ./DocService && jest integration/withServerInstance --inject-globals=false --config=../tests/jest.config.js", + "integration database tests": "cd ./DocService && jest integration/databaseTests --inject-globals=false --config=../tests/jest.config.js", "tests": "cd ./DocService && jest --inject-globals=false --config=../tests/jest.config.js", "install:Common": "npm ci --prefix ./Common", "install:DocService": "npm ci --prefix ./DocService", diff --git a/schema/dameng/createdb.sql b/schema/dameng/createdb.sql index b71586c0e..329e98e5a 100644 --- a/schema/dameng/createdb.sql +++ b/schema/dameng/createdb.sql @@ -7,7 +7,7 @@ -- ---------------------------- -- Table structure for doc_changes -- ---------------------------- -CREATE TABLE onlyoffice.doc_changes +CREATE TABLE doc_changes ( tenant varchar(255) NOT NULL, id varchar(255) NOT NULL, @@ -23,7 +23,7 @@ PRIMARY KEY (tenant, id, change_id) -- ---------------------------- -- Table structure for task_result -- ---------------------------- -CREATE TABLE onlyoffice.task_result +CREATE TABLE task_result ( tenant varchar(255) NOT NULL, id varchar(255) NOT NULL, diff --git a/schema/mssql/createdb.sql b/schema/mssql/createdb.sql index 920b651ca..634357d7f 100644 --- a/schema/mssql/createdb.sql +++ b/schema/mssql/createdb.sql @@ -1,3 +1,6 @@ +CREATE DATABASE onlyoffice; +GO + USE onlyoffice; CREATE TABLE doc_changes( @@ -27,4 +30,6 @@ CREATE TABLE task_result ( additional NVARCHAR(MAX) NULL, UNIQUE (tenant, id), CONSTRAINT unsigned_task_result CHECK(change_id BETWEEN 0 AND 4294967295 AND user_index BETWEEN 0 AND 4294967295) -); \ No newline at end of file +); + +GO \ No newline at end of file diff --git a/tests/integration/databaseTests/baseConnector.tests.js b/tests/integration/databaseTests/baseConnector.tests.js new file mode 100644 index 000000000..08ba5c06e --- /dev/null +++ b/tests/integration/databaseTests/baseConnector.tests.js @@ -0,0 +1,440 @@ +const { describe, test, expect, afterAll } = require('@jest/globals'); +const config = require('../../../Common/node_modules/config'); + +const baseConnector = require('../../../DocService/sources/databaseConnectors/baseConnector'); +const operationContext = require('../../../Common/sources/operationContext'); +const taskResult = require('../../../DocService/sources/taskresult'); +const commonDefines = require('../../../Common/sources/commondefines'); +const constants = require('../../../Common/sources/constants'); +const configSql = config.get('services.CoAuthoring.sql'); + +const ctx = new operationContext.Context(); +const cfgDbType = configSql.get('type'); +const cfgTableResult = configSql.get('tableResult'); +const cfgTableChanges = configSql.get('tableChanges'); +const dbTypes = { + oracle: { + number: 'NUMBER', + string: 'NVARCHAR(50)' + }, + mssql: { + number: 'INT', + string: 'NVARCHAR(50)' + }, + mysql: { + number: 'INT', + string: 'VARCHAR(50)' + }, + dameng: { + number: 'INT', + string: 'VARCHAR(50)' + }, + postgres: { + number: 'INT', + string: 'VARCHAR(50)' + }, + number: function () { + return this[cfgDbType].number; + }, + string: function () { + return this[cfgDbType].string; + } +} + +const insertCases = { + 5: 'baseConnector-insert()-tester-5-rows', + 500: 'baseConnector-insert()-tester-500-rows', + 1000: 'baseConnector-insert()-tester-1000-rows', + 5000: 'baseConnector-insert()-tester-5000-rows', + 10000: 'baseConnector-insert()-tester-10000-rows' +}; +const changesCases = { + range: 'baseConnector-getChangesPromise()-tester', + index: 'baseConnector-getChangesIndexPromise()-tester', + delete: 'baseConnector-deleteChangesPromise()-tester' +}; +const emptyCallbacksCase = [ + 'baseConnector-getEmptyCallbacks()-tester-0', + 'baseConnector-getEmptyCallbacks()-tester-1', + 'baseConnector-getEmptyCallbacks()-tester-2', + 'baseConnector-getEmptyCallbacks()-tester-3', + 'baseConnector-getEmptyCallbacks()-tester-4', +]; +const documentsWithChangesCase = [ + 'baseConnector-getDocumentsWithChanges()-tester-0', + 'baseConnector-getDocumentsWithChanges()-tester-1' +]; +const getExpiredCase = [ + 'baseConnector-getExpired()-tester-0', + 'baseConnector-getExpired()-tester-1', + 'baseConnector-getExpired()-tester-2', +]; +const upsertCases = { + insert: 'baseConnector-upsert()-tester-row-inserted', + update: 'baseConnector-upsert()-tester-row-updated' +}; + +function createChanges(changesLength, date) { + const objChanges = [ + { + docid: '__ffff_127.0.0.1new.docx41692082262909', + change: '"64;AgAAADEA//8BAG+X6xGnEAMAjgAAAAIAAAAEAAAABAAAAAUAAACCAAAAggAAAA4AAAAwAC4AMAAuADAALgAwAA=="', + time: date, + user: 'uid-18', + useridoriginal: 'uid-1' + } + ]; + + const length = changesLength - 1; + for (let i = 1; i <= length; i++) { + objChanges.push( + { + docid: '__ffff_127.0.0.1new.docx41692082262909', + change: '"39;CgAAADcAXwA2ADQAMAACABwAAQAAAAAAAAABAAAALgAAAAAAAAAA"', + time: date, + user: 'uid-18', + useridoriginal: 'uid-1' + } + ); + } + + return objChanges; +} + +async function getRowsCountById(table, id) { + const result = await executeSql(`SELECT COUNT(id) AS count FROM ${table} WHERE id = '${id}';`); + // Return type of COUNT() in postgres is bigint which treats as string by connector. Dameng DB returns js bigint type. + return Number(result[0].count); +} + +async function noRowsExistenceCheck(table, id) { + const noRows = await getRowsCountById(table, id); + expect(noRows).toEqual(0); +} + +function deleteRowsByIds(table, ids) { + const idToDelete = ids.map(id => `id = '${id}'`).join(' OR '); + return executeSql(`DELETE FROM ${table} WHERE ${idToDelete};`); +} + +function executeSql(sql, values = []) { + return new Promise((resolve, reject) => { + baseConnector.sqlQuery(ctx, sql, function (error, result) { + if (error) { + reject(error) + } else { + resolve(result) + } + }, false, false, values); + }); +} + +function createTask(id, callback = '', baseurl = '') { + const task = new taskResult.TaskResultData(); + task.tenant = ctx.tenant; + task.key = id; + task.status = commonDefines.FileStatus.None; + task.statusInfo = constants.NO_ERROR; + task.callback = callback; + task.baseurl = baseurl; + task.completeDefaults(); + + return task; +} + +function insertIntoResultTable(dateNow, task) { + let cbInsert = task.callback; + if (task.callback) { + const userCallback = new baseConnector.UserCallback(); + userCallback.fromValues(task.userIndex, task.callback); + cbInsert = userCallback.toSQLInsert(); + } + + const columns = ['tenant', 'id', 'status', 'status_info', 'last_open_date', 'user_index', 'change_id', 'callback', 'baseurl']; + const values = []; + const placeholder = [ + baseConnector.addSqlParameter(task.tenant, values), + baseConnector.addSqlParameter(task.key, values), + baseConnector.addSqlParameter(task.status, values), + baseConnector.addSqlParameter(task.statusInfo, values), + baseConnector.addSqlParameter(dateNow, values), + baseConnector.addSqlParameter(task.userIndex, values), + baseConnector.addSqlParameter(task.changeId, values), + baseConnector.addSqlParameter(cbInsert, values), + baseConnector.addSqlParameter(task.baseurl, values) + ]; + + return executeSql(`INSERT INTO ${cfgTableResult}(${columns.join(', ')}) VALUES(${placeholder.join(', ')});`, values); +} + +afterAll(async function () { + const insertIds = Object.values(insertCases); + const changesIds = Object.values(changesCases); + const upsertIds = Object.values(upsertCases); + + const tableChangesIds = [...emptyCallbacksCase, ...documentsWithChangesCase, ...changesIds, ...insertIds]; + const tableResultIds = [...emptyCallbacksCase, ...documentsWithChangesCase, ...getExpiredCase, ...upsertIds]; + + const deletionPool = [ + deleteRowsByIds(cfgTableChanges, tableChangesIds), + deleteRowsByIds(cfgTableResult, tableResultIds), + executeSql('DROP TABLE test_table;') + ]; + + await Promise.allSettled(deletionPool); + baseConnector.closePool?.(); +}); + +// Assumed that at least default DB was installed and configured. +describe('Base database connector', function () { + test('Availability of configured DB', async function () { + const result = await baseConnector.healthCheck(ctx); + + expect(result.length).toEqual(1); + }); + + test('Correct return format of requested rows', async function() { + const result = await baseConnector.healthCheck(ctx); + + // The [[constructor]] field is referring to a parent class instance, so for Object-like values it is equal to itself. + expect(result.constructor).toEqual(Array); + // SQL in healthCheck() request column with value 1, so we expect only one value. The default format, that used here is [{ columnName: columnValue }, { columnName: columnValue }]. + expect(result.length).toEqual(1); + expect(result[0].constructor).toEqual(Object); + expect(Object.values(result[0]).length).toEqual(1); + // Value itself. + expect(Object.values(result[0])[0]).toEqual(1); + }); + + test('Correct return format of changing in DB', async function () { + const createTableSql = `CREATE TABLE test_table(num ${dbTypes.number()});` + const alterTableSql = `INSERT INTO test_table VALUES(1);`; + + await executeSql(createTableSql); + const result = await executeSql(alterTableSql); + + expect(result).toEqual({ affectedRows: 1 }); + }); + + describe('DB tables existence', function () { + const tables = { + [cfgTableResult]: [ + { column_name: 'tenant' }, + { column_name: 'id' }, + { column_name: 'status' }, + { column_name: 'status_info' }, + { column_name: 'created_at' }, + { column_name: 'last_open_date' }, + { column_name: 'user_index' }, + { column_name: 'change_id' }, + { column_name: 'callback' }, + { column_name: 'baseurl' }, + { column_name: 'password' }, + { column_name: 'additional' } + ], + [cfgTableChanges]: [ + { column_name: 'tenant' }, + { column_name: 'id' }, + { column_name: 'change_id' }, + { column_name: 'user_id' }, + { column_name: 'user_id_original' }, + { column_name: 'user_name' }, + { column_name: 'change_data' }, + { column_name: 'change_date' } + ] + }; + + for (const table in tables) { + test(`${table} table existence`, async function () { + const result = await baseConnector.getTableColumns(ctx, table); + for (const row of tables[table]) { + expect(result).toContainEqual(row); + } + }); + } + }); + + describe('Changes manipulations', function () { + const date = new Date(); + const index = 0; + const user = { + id: 'uid-18', + idOriginal: 'uid-1', + username: 'John Smith', + indexUser: 8, + view: false + }; + + describe('Add changes', function () { + for (const testCase in insertCases) { + test(`${testCase} rows inserted`, async function () { + const docId = insertCases[testCase]; + const objChanges = createChanges(+testCase, date); + + await noRowsExistenceCheck(cfgTableChanges, docId); + + await baseConnector.insertChangesPromise(ctx, objChanges, docId, index, user); + const result = await getRowsCountById(cfgTableChanges, docId); + + expect(result).toEqual(objChanges.length); + }); + } + }); + + describe('Get and delete changes', function () { + const changesCount = 10; + const objChanges = createChanges(changesCount, date); + + test('Get changes in range', async function () { + const docId = changesCases.range; + const additionalChangesCount = 5; + const dayBefore = new Date(); + dayBefore.setDate(dayBefore.getDate() - 1); + const limitedByDateChanges = createChanges(additionalChangesCount, dayBefore); + const fullChanges = [...objChanges, ...limitedByDateChanges]; + + await noRowsExistenceCheck(cfgTableChanges, docId); + + await baseConnector.insertChangesPromise(ctx, fullChanges, docId, index, user); + + const result = await baseConnector.getChangesPromise(ctx, docId, index, changesCount); + expect(result.length).toEqual(changesCount); + + dayBefore.setSeconds(dayBefore.getSeconds() + 1); + const resultByDate = await baseConnector.getChangesPromise(ctx, docId, index, changesCount + additionalChangesCount, dayBefore); + expect(resultByDate.length).toEqual(additionalChangesCount); + }); + + test('Get changes index', async function () { + const docId = changesCases.index; + + await noRowsExistenceCheck(cfgTableChanges, docId); + + await baseConnector.insertChangesPromise(ctx, objChanges, docId, index, user); + + const result = await baseConnector.getChangesIndexPromise(ctx, docId); + + // We created 10 changes rows, change_id: 0..9, changes index is MAX(change_id). + const expected = [{ change_id: 9 }]; + expect(result).toEqual(expected); + }); + + test('Delete changes', async function () { + const docId = changesCases.delete; + + await baseConnector.insertChangesPromise(ctx, objChanges, docId, index, user); + + // Deleting 6 rows. + await baseConnector.deleteChangesPromise(ctx, docId, 4); + + const result = await getRowsCountById(cfgTableChanges, docId); + + // Rest rows. + expect(result).toEqual(4); + }); + }); + + test('Get empty callbacks' , async function () { + const idCount = 5; + const notNullCallbacks = idCount - 2; + + const resultBefore = await baseConnector.getEmptyCallbacks(ctx); + + // Adding non-empty callbacks. + for (let i = 0; i < notNullCallbacks; i++) { + const task = createTask(emptyCallbacksCase[i], 'some_callback'); + await insertIntoResultTable(date, task); + } + + // Adding empty callbacks. + for (let i = notNullCallbacks; i < idCount; i++) { + const task = createTask(emptyCallbacksCase[i], ''); + await insertIntoResultTable(date, task); + } + + // Adding same amount of changes with same tenant and id. + const objChanges = createChanges(1, date); + for (let i = 0; i < idCount; i++) { + await baseConnector.insertChangesPromise(ctx, objChanges, emptyCallbacksCase[i], index, user); + } + + const resultAfter = await baseConnector.getEmptyCallbacks(ctx); + + expect(resultAfter.length).toEqual(resultBefore.length + idCount - notNullCallbacks); + }); + + test('Get documents with changes', async function () { + const objChanges = createChanges(1, date); + + const resultBeforeNewRows = await baseConnector.getDocumentsWithChanges(ctx); + + for (const id of documentsWithChangesCase) { + const task = createTask(id); + await Promise.all([ + baseConnector.insertChangesPromise(ctx, objChanges, id, index, user), + insertIntoResultTable(date, task) + ]); + } + + const resultAfterNewRows = await baseConnector.getDocumentsWithChanges(ctx); + expect(resultAfterNewRows.length).toEqual(resultBeforeNewRows.length + documentsWithChangesCase.length); + }); + + test('Get expired', async function () { + const maxCount = 100; + const dayBefore = new Date(); + dayBefore.setDate(dayBefore.getDate() - 1); + + const resultBeforeNewRows = await baseConnector.getExpired(ctx, maxCount, 0); + + for (const id of getExpiredCase) { + const task = createTask(id); + await insertIntoResultTable(dayBefore, task); + } + + // 3 rows were added. + const resultAfterNewRows = await baseConnector.getExpired(ctx, maxCount + 3, 0); + + expect(resultAfterNewRows.length).toEqual(resultBeforeNewRows.length + getExpiredCase.length); + }); + }); + + describe('upsert() method', function () { + test('New row inserted', async function () { + const task = createTask(upsertCases.insert); + + await noRowsExistenceCheck(cfgTableResult, task.key); + + const result = await baseConnector.upsert(ctx, task); + + // isInsert should be true because of insert operation, insertId should be 1 by default. + const expected = { isInsert: true, insertId: 1 }; + expect(result).toEqual(expected); + + const insertedResult = await getRowsCountById(cfgTableResult, task.key); + + expect(insertedResult).toEqual(1); + }); + + test('Row updated', async function () { + const task = createTask(upsertCases.update, '', 'some-url'); + + await noRowsExistenceCheck(cfgTableResult, task.key); + + await baseConnector.upsert(ctx, task); + + // Changing baseurl to verify upsert() changing the row. + task.baseurl = 'some-updated-url'; + const result = await baseConnector.upsert(ctx, task); + + // isInsert should be false because of update operation, insertId should be 2 by updating clause. + const expected = { isInsert: false, insertId: 2 }; + expect(result).toEqual(expected); + + const updatedRow = await executeSql(`SELECT id, baseurl FROM ${cfgTableResult} WHERE id = '${task.key}';`); + + const expectedUrlChanges = [{ id: task.key, baseurl: 'some-updated-url' }]; + expect(updatedRow).toEqual(expectedUrlChanges); + }); + }); +}); \ No newline at end of file diff --git a/tests/integration/storage.tests.js b/tests/integration/storage.tests.js deleted file mode 100644 index 836bede06..000000000 --- a/tests/integration/storage.tests.js +++ /dev/null @@ -1,154 +0,0 @@ -const {jest, describe, test, expect} = require('@jest/globals'); -const http = require('http'); -const https = require('https'); -const fs = require('fs'); - -const operationContext = require('../../Common/sources/operationContext'); -const storage = require('../../Common/sources/storage-base'); -const utils = require('../../Common/sources/utils'); -const commonDefines = require("../../Common/sources/commondefines"); -const config = require('../../Common/node_modules/config'); - -const cfgStorageName = config.get('storage.name'); - -const ctx = operationContext.global; -const rand = Math.floor(Math.random() * 1000000); -const testDir = "DocService-DocsCoServer-storage-" + rand; -const baseUrl = "http://localhost:8000"; -const urlType = commonDefines.c_oAscUrlTypes.Session; -let testFile1 = testDir + "/test1.txt"; -let testFile2 = testDir + "/test2.txt"; -let testFile3 = testDir + "/test3.txt"; -let testFileData1 = "test1"; -let testFileData2 = "test2"; -let testFileData3 = testFileData2; - -console.debug(`testDir: ${testDir}`) - -function request(url) { - return new Promise(resolve => { - let module = url.startsWith('https') ? https : http; - module.get(url, response => { - let data = ''; - response.on('data', _data => (data += _data)); - response.on('end', () => resolve(data)); - }); - }); -} -function runTestForDir(specialDir) { - test("start listObjects", async () => { - let list = await storage.listObjects(ctx, testDir, specialDir); - expect(list).toEqual([]); - }); - test("putObject", async () => { - let buffer = Buffer.from(testFileData1); - await storage.putObject(ctx, testFile1, buffer, buffer.length, specialDir); - let list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile1].sort()); - }); - if ("storage-fs" === cfgStorageName) { - test("todo UploadObject in fs", async () => { - let buffer = Buffer.from(testFileData2); - await storage.putObject(ctx, testFile2, buffer, buffer.length, specialDir); - let list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile1, testFile2].sort()); - }); - } else { - test("uploadObject", async () => { - const spy = jest.spyOn(fs, 'createReadStream').mockReturnValue(testFileData2); - await storage.uploadObject(ctx, testFile2, "createReadStream.txt", specialDir); - let list = await storage.listObjects(ctx, testDir, specialDir); - expect(spy).toHaveBeenCalled(); - expect(list.sort()).toEqual([testFile1, testFile2].sort()); - }); - } - test("copyObject", async () => { - await storage.copyObject(ctx, testFile2, testFile3, specialDir, specialDir); - // let buffer = Buffer.from(testFileData3); - // await storage.putObject(ctx, testFile3, buffer, buffer.length, specialDir); - let list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); - }); - test("headObject", async () => { - let output; - output = await storage.headObject(ctx, testFile1, specialDir); - expect(output).toHaveProperty("ContentLength", testFileData1.length); - - output = await storage.headObject(ctx, testFile2, specialDir); - expect(output).toHaveProperty("ContentLength", testFileData2.length); - - output = await storage.headObject(ctx, testFile3, specialDir); - expect(output).toHaveProperty("ContentLength", testFileData3.length); - }); - test("getObject", async () => { - let output; - output = await storage.getObject(ctx, testFile1, specialDir); - expect(output.toString("utf8")).toEqual(testFileData1); - - output = await storage.getObject(ctx, testFile2, specialDir); - expect(output.toString("utf8")).toEqual(testFileData2); - - output = await storage.getObject(ctx, testFile3, specialDir); - expect(output.toString("utf8")).toEqual(testFileData3); - }); - test("createReadStream", async () => { - let output, outputText; - - output = await storage.createReadStream(ctx, testFile1, specialDir); - await utils.sleep(100); - outputText = await utils.stream2Buffer(output.readStream); - await utils.sleep(100); - expect(outputText.toString("utf8")).toEqual(testFileData1); - - output = await storage.createReadStream(ctx, testFile2, specialDir); - outputText = await utils.stream2Buffer(output.readStream); - expect(outputText.toString("utf8")).toEqual(testFileData2); - - output = await storage.createReadStream(ctx, testFile3, specialDir); - outputText = await utils.stream2Buffer(output.readStream); - expect(outputText.toString("utf8")).toEqual(testFileData3); - }); - test("getSignedUrl", async () => { - let url, data; - url = await storage.getSignedUrl(ctx, baseUrl, testFile1, urlType, undefined, undefined, specialDir); - data = await request(url); - expect(data).toEqual(testFileData1); - - url = await storage.getSignedUrl(ctx, baseUrl, testFile2, urlType, undefined, undefined, specialDir); - data = await request(url); - expect(data).toEqual(testFileData2); - - url = await storage.getSignedUrl(ctx, baseUrl, testFile3, urlType, undefined, undefined, specialDir); - data = await request(url); - expect(data).toEqual(testFileData3); - }); - test("deleteObject", async () => { - let list; - list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); - - await storage.deleteObject(ctx, testFile1, specialDir); - - list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile2, testFile3].sort()); - }); - test("deleteObjects", async () => { - let list; - list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([testFile2, testFile3].sort()); - - await storage.deleteObjects(ctx, list, specialDir); - - list = await storage.listObjects(ctx, testDir, specialDir); - expect(list.sort()).toEqual([].sort()); - }); -} - -// Assumed, that server is already up. -describe('storage common dir', function () { - runTestForDir(""); -}); - -describe('storage forgotten dir', function () { - runTestForDir("forgotten"); -}); \ No newline at end of file diff --git a/tests/integration/forgottenFilesCommnads.tests.js b/tests/integration/withServerInstance/forgottenFilesCommnads.tests.js similarity index 91% rename from tests/integration/forgottenFilesCommnads.tests.js rename to tests/integration/withServerInstance/forgottenFilesCommnads.tests.js index 42bd97648..38554d569 100644 --- a/tests/integration/forgottenFilesCommnads.tests.js +++ b/tests/integration/withServerInstance/forgottenFilesCommnads.tests.js @@ -1,12 +1,13 @@ const { describe, test, expect, afterAll, beforeAll } = require('@jest/globals'); const http = require('http'); -const { signToken } = require('../../DocService/sources/DocsCoServer'); -const storage = require('../../Common/sources/storage-base'); -const constants = require('../../Common/sources/commondefines'); -const operationContext = require('../../Common/sources/operationContext'); -const utils = require("../../Common/sources/utils"); -const config = require('../../Common/node_modules/config'); +const { signToken } = require('../../../DocService/sources/DocsCoServer'); +const storage = require('../../../Common/sources/storage-base'); +const constants = require('../../../Common/sources/commondefines'); +const operationContext = require('../../../Common/sources/operationContext'); +const utils = require("../../../Common/sources/utils"); + +const config = require('../../../Common/node_modules/config'); const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles'); const cfgForgottenFilesName = config.get('services.CoAuthoring.server.forgottenfilesname'); @@ -18,7 +19,7 @@ const cfgStorageName = config.get('storage.name'); const cfgEndpoint = config.get('storage.endpoint'); const cfgBucketName = config.get('storage.bucketName'); const ctx = new operationContext.Context(); -//yield ctx.initTenantCache();//no need + const testFilesNames = { get: 'DocService-DocsCoServer-forgottenFilesCommands-getForgotten-integration-test', delete1: 'DocService-DocsCoServer-forgottenFilesCommands-deleteForgotten-integration-test', @@ -85,9 +86,11 @@ beforeAll(async function () { afterAll(async function () { const keys = await storage.listObjects(ctx, '', cfgForgottenFiles); - const deletePromises = keys.filter(key => key.includes('DocService-DocsCoServer-forgottenFilesCommands')) - .map(filteredKey => storage.deleteObject(ctx, filteredKey, cfgForgottenFiles)); - + const keysDirectories = getKeysDirectories(keys); + const deletePromises = keysDirectories.filter(key => key.includes('DocService-DocsCoServer-forgottenFilesCommands')) + .map(filteredKey => storage.deletePath(ctx, filteredKey, cfgForgottenFiles)); + console.log(`keys:`+JSON.stringify(keys)); + console.log(`keysDirectories:`+JSON.stringify(keysDirectories)); return Promise.allSettled(deletePromises); }); diff --git a/tests/integration/withServerInstance/storage.tests.js b/tests/integration/withServerInstance/storage.tests.js new file mode 100644 index 000000000..b86b3beab --- /dev/null +++ b/tests/integration/withServerInstance/storage.tests.js @@ -0,0 +1,291 @@ +const {jest, describe, test, expect} = require('@jest/globals'); +const http = require('http'); +const https = require('https'); +const fs = require('fs'); +const { Readable } = require('stream'); + +let testFileData1 = "test1"; +let testFileData2 = "test22"; +let testFileData3 = "test333"; +let testFileData4 = testFileData3; + +jest.mock("fs/promises", () => ({ + ...jest.requireActual('fs/promises'), + cp: jest.fn().mockImplementation((from, to) => fs.writeFileSync(to, testFileData3)) +})); +const { cp } = require('fs/promises'); + +const operationContext = require('../../../Common/sources/operationContext'); +const tenantManager = require('../../../Common/sources/tenantManager'); +const storage = require('../../../Common/sources/storage-base'); +const utils = require('../../../Common/sources/utils'); +const commonDefines = require("../../../Common/sources/commondefines"); +const config = require('../../../Common/node_modules/config'); + +const cfgCacheStorage = config.get('storage'); +const cfgPersistentStorage = utils.deepMergeObjects({}, cfgCacheStorage, config.get('persistentStorage')); + +const ctx = operationContext.global; +const rand = Math.floor(Math.random() * 1000000); +const testDir = "DocService-DocsCoServer-storage-" + rand; +const baseUrl = "http://localhost:8000"; +const urlType = commonDefines.c_oAscUrlTypes.Session; +let testFile1 = testDir + "/test1.txt"; +let testFile2 = testDir + "/test2.txt"; +let testFile3 = testDir + "/test3.txt"; +let testFile4 = testDir + "/test4.txt"; +let specialDirCache = ""; +let specialDirForgotten = "forgotten"; + +console.debug(`testDir: ${testDir}`) + +function getStorageCfg(specialDir) { + return specialDir ? cfgPersistentStorage : cfgCacheStorage; +} + +function request(url) { + return new Promise(resolve => { + let module = url.startsWith('https') ? https : http; + module.get(url, response => { + let data = ''; + response.on('data', _data => (data += _data)); + response.on('end', () => resolve(data)); + }); + }); +} +function runTestForDir(ctx, isMultitenantMode, specialDir) { + let oldMultitenantMode = tenantManager.isMultitenantMode(); + test("start listObjects", async () => { + //todo set in all tests do not rely on test order + tenantManager.setMultitenantMode(isMultitenantMode); + + let list = await storage.listObjects(ctx, testDir, specialDir); + expect(list).toEqual([]); + }); + test("putObject", async () => { + let buffer = Buffer.from(testFileData1); + let res = await storage.putObject(ctx, testFile1, buffer, buffer.length, specialDir); + expect(res).toEqual(undefined); + let list = await storage.listObjects(ctx, testDir, specialDir); + expect(list.sort()).toEqual([testFile1].sort()); + }); + test("putObject-stream", async () => { + let buffer = Buffer.from(testFileData2); + const stream = Readable.from(buffer); + let res = await storage.putObject(ctx, testFile2, stream, buffer.length, specialDir); + expect(res).toEqual(undefined); + let list = await storage.listObjects(ctx, testDir, specialDir); + expect(list.sort()).toEqual([testFile1, testFile2].sort()); + }); + if ("storage-fs" === getStorageCfg(specialDir).name) { + test("UploadObject", async () => { + let res = await storage.uploadObject(ctx, testFile3, "createReadStream.txt", specialDir); + expect(res).toEqual(undefined); + expect(cp).toHaveBeenCalled(); + let list = await storage.listObjects(ctx, testDir, specialDir); + expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); + }); + } else { + test("uploadObject", async () => { + const spy = jest.spyOn(fs, 'createReadStream').mockReturnValue(testFileData3); + let res = await storage.uploadObject(ctx, testFile3, "createReadStream.txt", specialDir); + expect(res).toEqual(undefined); + let list = await storage.listObjects(ctx, testDir, specialDir); + expect(spy).toHaveBeenCalled(); + expect(list.sort()).toEqual([testFile1, testFile2, testFile3].sort()); + spy.mockRestore(); + }); + } + test("copyObject", async () => { + let res = await storage.copyObject(ctx, testFile3, testFile4, specialDir, specialDir); + expect(res).toEqual(undefined); + // let buffer = Buffer.from(testFileData3); + // await storage.putObject(ctx, testFile3, buffer, buffer.length, specialDir); + let list = await storage.listObjects(ctx, testDir, specialDir); + expect(list.sort()).toEqual([testFile1, testFile2, testFile3, testFile4].sort()); + }); + test("headObject", async () => { + let output; + output = await storage.headObject(ctx, testFile1, specialDir); + expect(output).toMatchObject({ContentLength: testFileData1.length}); + + output = await storage.headObject(ctx, testFile2, specialDir); + expect(output).toMatchObject({ContentLength: testFileData2.length}); + + output = await storage.headObject(ctx, testFile3, specialDir); + expect(output).toMatchObject({ContentLength: testFileData3.length}); + + output = await storage.headObject(ctx, testFile4, specialDir); + expect(output).toMatchObject({ContentLength: testFileData4.length}); + }); + test("getObject", async () => { + let output; + output = await storage.getObject(ctx, testFile1, specialDir); + expect(output.toString("utf8")).toEqual(testFileData1); + + output = await storage.getObject(ctx, testFile2, specialDir); + expect(output.toString("utf8")).toEqual(testFileData2); + + output = await storage.getObject(ctx, testFile3, specialDir); + expect(output.toString("utf8")).toEqual(testFileData3); + + output = await storage.getObject(ctx, testFile4, specialDir); + expect(output.toString("utf8")).toEqual(testFileData4); + }); + test("createReadStream", async () => { + let output, outputText; + + output = await storage.createReadStream(ctx, testFile1, specialDir); + await utils.sleep(100); + expect(output.contentLength).toEqual(testFileData1.length); + outputText = await utils.stream2Buffer(output.readStream); + await utils.sleep(100); + expect(outputText.toString("utf8")).toEqual(testFileData1); + + output = await storage.createReadStream(ctx, testFile2, specialDir); + expect(output.contentLength).toEqual(testFileData2.length); + outputText = await utils.stream2Buffer(output.readStream); + expect(outputText.toString("utf8")).toEqual(testFileData2); + + output = await storage.createReadStream(ctx, testFile3, specialDir); + expect(output.contentLength).toEqual(testFileData3.length); + outputText = await utils.stream2Buffer(output.readStream); + expect(outputText.toString("utf8")).toEqual(testFileData3); + }); + test("getSignedUrl", async () => { + let url, urls, data; + url = await storage.getSignedUrl(ctx, baseUrl, testFile1, urlType, undefined, undefined, specialDir); + data = await request(url); + expect(data).toEqual(testFileData1); + + url = await storage.getSignedUrl(ctx, baseUrl, testFile2, urlType, undefined, undefined, specialDir); + data = await request(url); + expect(data).toEqual(testFileData2); + + url = await storage.getSignedUrl(ctx, baseUrl, testFile3, urlType, undefined, undefined, specialDir); + data = await request(url); + expect(data).toEqual(testFileData3); + + url = await storage.getSignedUrl(ctx, baseUrl, testFile4, urlType, undefined, undefined, specialDir); + data = await request(url); + expect(data).toEqual(testFileData4); + }); + test("getSignedUrls", async () => { + let urls, data; + urls = await storage.getSignedUrls(ctx, baseUrl, testDir, urlType, undefined, specialDir); + data = []; + for(let i in urls) { + data.push(await request(urls[i])); + } + expect(data.sort()).toEqual([testFileData1, testFileData2, testFileData3, testFileData4].sort()); + }); + test("getSignedUrlsArrayByArray", async () => { + let urls, data; + urls = await storage.getSignedUrlsArrayByArray(ctx, baseUrl, [testFile1, testFile2], urlType, specialDir); + data = []; + for(let i = 0; i < urls.length; ++i) { + data.push(await request(urls[i])); + } + expect(data.sort()).toEqual([testFileData1, testFileData2].sort()); + }); + test("getSignedUrlsByArray", async () => { + let urls, data; + urls = await storage.getSignedUrlsByArray(ctx, baseUrl, [testFile3, testFile4], undefined, urlType, specialDir); + data = []; + for(let i in urls) { + data.push(await request(urls[i])); + } + expect(data.sort()).toEqual([testFileData3, testFileData4].sort()); + }); + test("deleteObject", async () => { + let list; + list = await storage.listObjects(ctx, testDir, specialDir); + expect(list.sort()).toEqual([testFile1, testFile2, testFile3, testFile4].sort()); + + let res = await storage.deleteObject(ctx, testFile1, specialDir); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDir); + expect(list.sort()).toEqual([testFile2, testFile3, testFile4].sort()); + }); + test("deletePath", async () => { + let list; + list = await storage.listObjects(ctx, testDir, specialDir); + expect(list.sort()).toEqual([testFile2, testFile3, testFile4].sort()); + + let res = await storage.deletePath(ctx, testDir, specialDir); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDir); + expect(list.sort()).toEqual([].sort()); + + tenantManager.setMultitenantMode(oldMultitenantMode); + }); +} + +// Assumed, that server is already up. +describe('storage common dir', function () { + runTestForDir(ctx, false, specialDirCache); +}); + +describe('storage forgotten dir', function () { + runTestForDir(ctx, false, specialDirForgotten); +}); + +describe('storage common dir with tenants', function () { + runTestForDir(ctx, true, specialDirCache); +}); + +describe('storage forgotten dir with tenants', function () { + runTestForDir(ctx, true, specialDirForgotten); +}); + +describe('storage mix common and forgotten dir', function () { + test("putObject", async () => { + tenantManager.setMultitenantMode(false); + + let buffer = Buffer.from(testFileData1); + let res = await storage.putObject(ctx, testFile1, buffer, buffer.length, specialDirCache); + expect(res).toEqual(undefined); + let list = await storage.listObjects(ctx, testDir, specialDirCache); + expect(list.sort()).toEqual([testFile1].sort()); + + buffer = Buffer.from(testFileData2); + res = await storage.putObject(ctx, testFile2, buffer, buffer.length, specialDirForgotten); + expect(res).toEqual(undefined); + list = await storage.listObjects(ctx, testDir, specialDirForgotten); + expect(list.sort()).toEqual([testFile2].sort()); + }); + + test("copyPath", async () => { + let list, res; + res = await storage.copyPath(ctx, testDir, testDir, specialDirCache, specialDirForgotten); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirForgotten); + expect(list.sort()).toEqual([testFile1, testFile2].sort()); + }); + test("copyObject", async () => { + let list, res; + res = await storage.copyObject(ctx, testFile2, testFile2, specialDirForgotten, specialDirCache); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirCache); + expect(list.sort()).toEqual([testFile1, testFile2].sort()); + }); + + test("deletePath", async () => { + let list, res; + res = await storage.deletePath(ctx, testDir, specialDirCache); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirCache); + expect(list.sort()).toEqual([].sort()); + + res = await storage.deletePath(ctx, testDir, specialDirForgotten); + expect(res).toEqual(undefined); + + list = await storage.listObjects(ctx, testDir, specialDirForgotten); + expect(list.sort()).toEqual([].sort()); + }); +}); diff --git a/tests/perf/checkFileExpire.js b/tests/perf/checkFileExpire.js index 6a75356a4..406253d1b 100644 --- a/tests/perf/checkFileExpire.js +++ b/tests/perf/checkFileExpire.js @@ -68,7 +68,7 @@ async function beforeStart() { taskResult.remove = timerify(taskResult.remove, "remove"); storage.putObject = timerify(storage.putObject, "putObject"); storage.listObjects = timerify(storage.listObjects, "listObjects"); - storage.deleteObjects = timerify(storage.deleteObjects, "deleteObjects"); + storageFs.deletePath = timerify(storageFs.deletePath, "deletePath"); storageFs.deleteObject = timerify(storageFs.deleteObject, "deleteObject"); docsCoServer.getEditorsCountPromise = timerify(docsCoServer.getEditorsCountPromise, "getEditorsCountPromise"); diff --git a/tests/unit/sample.tests.js b/tests/unit/sample.tests.js new file mode 100644 index 000000000..c0b759e01 --- /dev/null +++ b/tests/unit/sample.tests.js @@ -0,0 +1,11 @@ +const { describe, test, expect } = require('@jest/globals'); + +describe('Successful and failure tests', function () { + test('Successful test', function () { + expect(true).toBeTruthy(); + }); + + test.skip('Failure test', function () { + expect(true).toBeFalsy(); + }); +}); \ No newline at end of file