Merge branch 'develop' into feature/backup

# Conflicts:
#	common/ASC.Data.Backup.Core/Storage/ConsumerBackupStorage.cs
#	config/nginx/onlyoffice.conf
#	packages/asc-web-common/constants/index.js
#	products/ASC.Files/Client/src/components/panels/StyledPanels.js
#	products/ASC.Files/Client/webpack.config.js
#	web/ASC.Web.Api/Models/SettingsWrapper.cs
#	web/ASC.Web.Client/src/components/NavMenu/sub-components/header.js

ASC.Web.slnf

@@ -22,6 +22,7 @@
       "common\\services\\ASC.ElasticSearch\\ASC.ElasticSearch.csproj",
       "common\\services\\ASC.Feed.Aggregator\\ASC.Feed.Aggregator.csproj",
       "common\\services\\ASC.Notify\\ASC.Notify.csproj",
+      "common\\services\\ASC.Socket.IO.Svc\\ASC.Socket.IO.Svc.csproj",
       "common\\services\\ASC.Studio.Notify\\ASC.Studio.Notify.csproj",
       "common\\services\\ASC.TelegramService\\ASC.TelegramService.csproj",
       "common\\services\\ASC.UrlShortener.Svc\\ASC.UrlShortener.Svc.csproj",

README.md

@@ -1,27 +1,17 @@
-# AppServer
-Platform for building your own scalable online office with ONLYOFFICE modules
+# ONLYOFFICE App Server
 
 ## Overview
-App Server is a platform for building your own online office by connecting ONLYOFFICE modules packaged as separate apps.
 
-## How to install
-[coming soon]
+ONLYOFFICE App Server is a platform based on .NET Core and React engines which comprises document management features and makes it possible to implement advanced folder management. 
 
-## Connectable apps
+## Functionality
 
-* **People** - ONLYOFFICE module for creating your team’s profiles and setting their roles (user, guest, admin).
-
-## Plans
-
-These ONLYOFFICE modules will be available for connecting to App Server soon.
-* **Documents** for managing and sharing your documents.
-* **Chat** to exchange instant messages with your team.
-* **Mail** to manage correspondence sufficiently. 
-* **Calendar** for creating timetables and shared calendars.
-* **Projects** to organize your team’s tasks and track progress.
-* **CRM** for sales management and productivity improvement.
-
-You will also be able to connect third-party apps or create new ones using ONLYOFFICE component library.
+* Cross-platform: web and mobile environment.
+* Integration with online editors (text documents, spreadsheets, presentations, forms).
+* External sharing for editing, reviewing, commenting, reading.
+* Version history.
+* File and folder search.
+* Connecting Dropbox, Google Drive, Box, OneDrive, Nextcloud, ownCloud, kDrive, Yandex.Disk.
 
 ## Licensing 
 
@@ -31,8 +21,8 @@ App Server is released under AGPLv3 license. See the LICENSE file for more infor
 
 Official website: [https://www.onlyoffice.com](https://www.onlyoffice.com/?utm_source=github&utm_medium=cpc&utm_campaign=AppServer "https://www.onlyoffice.com/?utm_source=github&utm_medium=cpc&utm_campaign=AppServer")
 
-Components source code - [https://github.com/ONLYOFFICE/AppComponents](https://github.com/ONLYOFFICE/AppComponents)
+Code repository: [https://github.com/ONLYOFFICE/AppServer](https://github.com/ONLYOFFICE/AppServer)
 
 ## User feedback and support
 
-If you have any problems or questions about App Server, use the issues section here, in this repository.
+If you have any problems or questions about App Server, use the Issues section in this repository or visit our [official forum](https://forum.onlyoffice.com/).

build/build.backend.bat

@@ -11,8 +11,6 @@ call start\stop.bat nopause
 dotnet build ..\asc.web.slnf  /fl1 /flp1:logfile=asc.web.log;verbosity=normal
 echo.
 
-call start\start.bat nopause
-
 echo install nodejs projects dependencies...
 echo.
 
@@ -25,6 +23,10 @@ for /R "scripts\" %%f in (*.bat) do (
 
 echo.
 
+call start\start.bat nopause
+
+echo.
+
 if "%1"=="nopause" goto end
 pause
 

build/build.static.bat

@@ -36,14 +36,10 @@ powershell -Command "(gc build\deploy\nginx\onlyoffice.conf) -replace '#', '' |
 xcopy config\nginx\sites-enabled\* build\deploy\nginx\sites-enabled\ /E /R /Y
 
 REM fix paths
-powershell -Command "(gc build\deploy\nginx\sites-enabled\onlyoffice-calendar.conf) -replace 'ROOTPATH', '%~dp0deploy\products\ASC.Calendar\client' -replace '\\', '/' | Out-File -encoding ASCII build\deploy\nginx\sites-enabled\onlyoffice-calendar.conf"
-powershell -Command "(gc build\deploy\nginx\sites-enabled\onlyoffice-crm.conf) -replace 'ROOTPATH', '%~dp0deploy\products\ASC.CRM\client' -replace '\\', '/' | Out-File -encoding ASCII build\deploy\nginx\sites-enabled\onlyoffice-crm.conf"
 powershell -Command "(gc build\deploy\nginx\sites-enabled\onlyoffice-editor.conf) -replace 'ROOTPATH', '%~dp0deploy\products\ASC.Files\editor' -replace '\\', '/' | Out-File -encoding ASCII build\deploy\nginx\sites-enabled\onlyoffice-editor.conf"
 powershell -Command "(gc build\deploy\nginx\sites-enabled\onlyoffice-files.conf) -replace 'ROOTPATH', '%~dp0deploy\products\ASC.Files\client' -replace '\\', '/' | Out-File -encoding ASCII build\deploy\nginx\sites-enabled\onlyoffice-files.conf"
 powershell -Command "(gc build\deploy\nginx\sites-enabled\onlyoffice-login.conf) -replace 'ROOTPATH', '%~dp0deploy\studio\login' -replace '\\', '/' | Out-File -encoding ASCII build\deploy\nginx\sites-enabled\onlyoffice-login.conf"
-powershell -Command "(gc build\deploy\nginx\sites-enabled\onlyoffice-mail.conf) -replace 'ROOTPATH', '%~dp0deploy\products\ASC.Mail\client' -replace '\\', '/' | Out-File -encoding ASCII build\deploy\nginx\sites-enabled\onlyoffice-mail.conf"
 powershell -Command "(gc build\deploy\nginx\sites-enabled\onlyoffice-people.conf) -replace 'ROOTPATH', '%~dp0deploy\products\ASC.People\client' -replace '\\', '/' | Out-File -encoding ASCII build\deploy\nginx\sites-enabled\onlyoffice-people.conf"
-powershell -Command "(gc build\deploy\nginx\sites-enabled\onlyoffice-projects.conf) -replace 'ROOTPATH', '%~dp0deploy\products\ASC.Projects\client' -replace '\\', '/' | Out-File -encoding ASCII build\deploy\nginx\sites-enabled\onlyoffice-projects.conf"
 powershell -Command "(gc build\deploy\nginx\sites-enabled\onlyoffice-studio.conf) -replace 'ROOTPATH', '%~dp0deploy\studio\client' -replace '\\', '/' | Out-File -encoding ASCII build\deploy\nginx\sites-enabled\onlyoffice-studio.conf"
 
 REM restart nginx
@@ -57,7 +53,7 @@ echo service nginx start
 call sc start nginx > nul
 
 if NOT %errorlevel% == 0 (
-	echo Couldn't restarte Onlyoffice%%~nf service			
+	echo Couldn't restart Onlyoffice%%~nf service			
 )
 
 )

build/install/OneClickInstall/install-Debian/install-preq.sh

@@ -55,14 +55,16 @@ rm packages-microsoft-prod.deb
 
 #install kafka
 PRODUCT_DIR="/var/www/${product}"
-if [ "$(ls -A "$PRODUCT_DIR/services/kafka" 2> /dev/null)" == "" ]; then
+if [ "$(ls "$PRODUCT_DIR/services/kafka" 2> /dev/null)" == "" ]; then
 	mkdir -p ${PRODUCT_DIR}/services/
 	if ! cat /etc/passwd | grep -q "kafka"; then
 		adduser --quiet --home ${PRODUCT_DIR}/services/kafka --system kafka
 	fi
 	cd ${PRODUCT_DIR}/services/kafka
-	curl https://downloads.apache.org/kafka/2.7.2/kafka_2.13-2.7.2.tgz -O
-	tar xzf kafka_*.tgz --strip 1 && rm -rf kafka_*.tgz
+	KAFKA_VERSION=$(curl https://downloads.apache.org/kafka/ | grep -Eo '3.1.[0-9]' | tail -1)
+	KAFKA_ARCHIVE=$(curl https://downloads.apache.org/kafka/$KAFKA_VERSION/ | grep -Eo "kafka_2.[0-9][0-9]-$KAFKA_VERSION.tgz" | tail -1)
+	curl https://downloads.apache.org/kafka/$KAFKA_VERSION/$KAFKA_ARCHIVE -O
+	tar xzf $KAFKA_ARCHIVE --strip 1 && rm -rf $KAFKA_ARCHIVE
 	chown -R kafka ${PRODUCT_DIR}/services/kafka
 	cd -
 fi
@@ -107,12 +109,16 @@ if ! dpkg -l | grep -q "mysql-server"; then
 	MYSQL_SERVER_PASS=${MYSQL_SERVER_PASS:-"$(cat /dev/urandom | tr -dc A-Za-z0-9 | head -c 12)"}
 
 	# setup mysql 8.0 package
-	curl -OL http://dev.mysql.com/get/mysql-apt-config_0.8.15-1_all.deb
+	MYSQL_PACKAGE_NAME="mysql-apt-config_0.8.22-1_all.deb"
+	if [ "$DIST" = "debian" ] && [ "$DISTRIB_CODENAME" = "stretch" ]; then
+		MYSQL_PACKAGE_NAME="mysql-apt-config_0.8.16-1_all.deb"
+	fi
+	curl -OL http://dev.mysql.com/get/${MYSQL_PACKAGE_NAME}
 	echo "mysql-apt-config mysql-apt-config/repo-codename  select  $DISTRIB_CODENAME" | debconf-set-selections
 	echo "mysql-apt-config mysql-apt-config/repo-distro  select  $DIST" | debconf-set-selections
 	echo "mysql-apt-config mysql-apt-config/select-server  select  mysql-8.0" | debconf-set-selections
-	DEBIAN_FRONTEND=noninteractive dpkg -i mysql-apt-config_0.8.15-1_all.deb
-	rm -f mysql-apt-config_0.8.15-1_all.deb
+	DEBIAN_FRONTEND=noninteractive dpkg -i ${MYSQL_PACKAGE_NAME}
+	rm -f ${MYSQL_PACKAGE_NAME}
 
 	echo mysql-community-server mysql-community-server/root-pass password ${MYSQL_SERVER_PASS} | debconf-set-selections
 	echo mysql-community-server mysql-community-server/re-root-pass password ${MYSQL_SERVER_PASS} | debconf-set-selections

build/install/OneClickInstall/install-Docker.sh

@@ -70,6 +70,7 @@ KAFKA_HOST=""
 
 ELK_HOST=""
 
+DOCUMENT_SERVER_IMAGE_NAME=onlyoffice/4testing-documentserver-ee:latest
 DOCUMENT_SERVER_JWT_SECRET=""
 DOCUMENT_SERVER_HOST=""
 
@@ -282,6 +283,13 @@ while [ "$1" != "" ]; do
 				shift
 			fi
 		;;
+
+		-di | --documentserverimage )
+			if [ "$2" != "" ]; then
+				DOCUMENT_SERVER_IMAGE_NAME=$2
+				shift
+			fi
+		;;
 
 		-? | -h | --help )
 			echo "  Usage: bash $HELP_TARGET [PARAMETER] [[PARAMETER], ...]"
@@ -291,8 +299,9 @@ while [ "$1" != "" ]; do
 			echo "      -un, --username                   dockerhub username"
 			echo "      -p, --password                    dockerhub password"
 			echo "      -ias, --installappserver          install or update appserver (true|false)"
-			echo "      -vas, --versionappserver          select the version to install appserver (latest|develop|version number)"
+			echo "      -tag, --dockertag                 select the version to install appserver (latest|develop|version number)"
 			echo "      -ids, --installdocumentserver     install or update document server (true|false)"
+			echo "      -di, --documentserverimage        document server image name"
 			echo "      -imysql, --installmysql           install or update mysql (true|false)"			
 			echo "      -ikafka, --installkafka           install or update kafka (true|false)"
 			echo "      -mysqlrp, --mysqlrootpassword     mysql server root password"
@@ -497,8 +506,8 @@ check_hardware () {
 }
 
 install_service () {
-	COMMAND_NAME=$1
-	PACKAGE_NAME=$2
+	local COMMAND_NAME=$1
+	local PACKAGE_NAME=$2
 
 	PACKAGE_NAME=${PACKAGE_NAME:-"$COMMAND_NAME"}
 
@@ -698,8 +707,8 @@ create_network () {
 }
 
 get_container_env_parameter () {
-	CONTAINER_NAME=$1;
-	PARAMETER_NAME=$2;
+	local CONTAINER_NAME=$1;
+	local PARAMETER_NAME=$2;
 	VALUE="";
 
 	if [[ -z ${CONTAINER_NAME} ]]; then
@@ -770,45 +779,28 @@ set_core_machinekey () {
 
 	if [[ -z ${CORE_MACHINEKEY} ]] && [[ "$UPDATE" != "true" ]]; then
 		APP_CORE_MACHINEKEY=$(get_random_str 12);
+		mkdir -p ${BASE_DIR}/.private/
 		echo $APP_CORE_MACHINEKEY > ${BASE_DIR}/.private/machinekey
 	fi
 }
 
 download_files () {
-	mkdir -p ${BASE_DIR}
-	mkdir -p ${BASE_DIR}/.private/
-	mkdir -p ${BASE_DIR}/config/mysql/conf.d/
-
-	if ! command_exists wget; then
-		install_service wget
-	fi
-
-	DOWNLOAD_URL_PREFIX="https://raw.githubusercontent.com/ONLYOFFICE/${PRODUCT}/${GIT_BRANCH}/build/install/docker"
-	wget -q -O $BASE_DIR/.env "${DOWNLOAD_URL_PREFIX}/.env"
-	wget -q -O $BASE_DIR/db.yml "${DOWNLOAD_URL_PREFIX}/db.yml"
-	wget -q -O $BASE_DIR/ds.yml "${DOWNLOAD_URL_PREFIX}/ds.yml"
-	wget -q -O $BASE_DIR/kafka.yml "${DOWNLOAD_URL_PREFIX}/kafka.yml"
-	wget -q -O $BASE_DIR/appserver.yml "${DOWNLOAD_URL_PREFIX}/appserver.yml"
-	wget -q -O $BASE_DIR/config/createdb.sql "${DOWNLOAD_URL_PREFIX}/config/createdb.sql"
-	wget -q -O $BASE_DIR/config/onlyoffice.sql "${DOWNLOAD_URL_PREFIX}/config/onlyoffice.sql"
-	wget -q -O $BASE_DIR/config/onlyoffice.data.sql "${DOWNLOAD_URL_PREFIX}/config/onlyoffice.data.sql"
-	wget -q -O $BASE_DIR/config/mysql/conf.d/mysql.cnf "${DOWNLOAD_URL_PREFIX}/config/mysql/conf.d/mysql.cnf"
-	wget -q -O $BASE_DIR/config/onlyoffice.resources.sql "${DOWNLOAD_URL_PREFIX}/config/onlyoffice.resources.sql"
-	wget -q -O $BASE_DIR/config/onlyoffice.upgradev110.sql "${DOWNLOAD_URL_PREFIX}/config/onlyoffice.upgradev110.sql"
-	wget -q -O $BASE_DIR/config/onlyoffice.upgradev111.sql "${DOWNLOAD_URL_PREFIX}/config/onlyoffice.upgradev111.sql"
-	wget -q -O $BASE_DIR/config/onlyoffice.upgradev115.sql "${DOWNLOAD_URL_PREFIX}/config/onlyoffice.upgradev115.sql"
-
-	if [[ -n ${STATUS} ]]; then
-		sed -i "s/STATUS=.*/STATUS=\"${STATUS}\"/g" $BASE_DIR/.env
+	if ! command_exists svn; then
+		install_service svn subversion
 	fi
+
+	svn export --force https://github.com/ONLYOFFICE/${PRODUCT}/branches/${GIT_BRANCH}/build/install/docker/ ${BASE_DIR}
+	svn export --force https://github.com/ONLYOFFICE/CommunityServer/branches/master/build/sql/ ${BASE_DIR}/config/ #Download SQL scripts
+
+	reconfigure STATUS ${STATUS}
 }
 
 reconfigure () {
-	VARIABLE_NAME=$1
-	VARIABLE_VALUE=$(echo $2 | sed -e 's/;/%/g' -e 's/=/%/g' -e 's/!/%/g')
+	local VARIABLE_NAME=$1
+	local VARIABLE_VALUE=$2
 
 	if [[ -n ${VARIABLE_VALUE} ]]; then
-		sed -i "s/${VARIABLE_NAME}=.*/${VARIABLE_NAME}=${VARIABLE_VALUE}/g" $BASE_DIR/.env
+		sed -i "s~${VARIABLE_NAME}=.*~${VARIABLE_NAME}=${VARIABLE_VALUE}~g" $BASE_DIR/.env
 	fi
 }
 
@@ -818,8 +810,8 @@ install_mysql_server () {
 	fi
 
 	if [[ -z ${MYSQL_PASSWORD} ]] && [[ -z ${MYSQL_ROOT_PASSWORD} ]]; then
-		MYSQL_PASSWORD=$(get_random_str 20);
-		MYSQL_ROOT_PASSWORD=$(get_random_str 20);
+		MYSQL_PASSWORD=$(get_random_str 20 | sed -e 's/;/%/g' -e 's/=/%/g' -e 's/!/%/g');
+		MYSQL_ROOT_PASSWORD=$(get_random_str 20 | sed -e 's/;/%/g' -e 's/=/%/g' -e 's/!/%/g');
 	elif [[ -z ${MYSQL_PASSWORD} ]] || [[ -z ${MYSQL_ROOT_PASSWORD} ]]; then
 		MYSQL_PASSWORD=${MYSQL_PASSWORD:-"$MYSQL_ROOT_PASSWORD"}
 		MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-"$MYSQL_PASSWORD"}
@@ -839,6 +831,7 @@ install_document_server () {
 		install_docker_compose
 	fi
 
+	reconfigure DOCUMENT_SERVER_IMAGE_NAME ${DOCUMENT_SERVER_IMAGE_NAME}
 	reconfigure DOCUMENT_SERVER_JWT_SECRET ${DOCUMENT_SERVER_JWT_SECRET}
 	reconfigure DOCUMENT_SERVER_HOST ${DOCUMENT_SERVER_HOST}
 
@@ -868,6 +861,89 @@ install_appserver () {
 	fi
 
 	docker-compose -f $BASE_DIR/appserver.yml up -d
+	docker-compose -f $BASE_DIR/notify.yml up -d
+}
+
+get_local_image_RepoDigests() {
+   local CONTAINER_IMAGE=$1;
+   LOCAL_IMAGE_RepoDigest=$(docker inspect --format='{{index .RepoDigests 0}}' $CONTAINER_IMAGE)
+   if [ -z ${LOCAL_IMAGE_RepoDigest} ]; then
+        echo "Local docker image not found, check the name of docker image $CONTAINER_IMAGE"
+        exit 1 
+   fi
+   echo $LOCAL_IMAGE_RepoDigest
+}
+
+check_pull_image() {
+    local CONTAINER_IMAGE=$1;
+    CHECK_STATUS_IMAGE="$(docker pull  $CONTAINER_IMAGE | grep Status | awk '{print $2" "$3" "$4" "$5" "$6}')"
+    if [ "${CHECK_STATUS_IMAGE}" == "Image is up to date" ]; then
+        echo "No updates required"
+    fi
+}
+
+check_image_RepoDigest() {
+    local OLD_LOCAL_IMAGE_RepoDigest=$1
+    local NEW_LOCAL_IMAGE_RepoDigest=$2
+    if [ "${OLD_LOCAL_IMAGE_RepoDigest}" == "${NEW_LOCAL_IMAGE_RepoDigest}" ]; then
+       CHECK_RepoDigest="false";
+    else
+       CHECK_RepoDigest="true";
+    fi
+}
+
+docker_image_update() {
+    docker-compose -f $BASE_DIR/notify.yml -f $BASE_DIR/appserver.yml down --volumes
+    docker-compose -f $BASE_DIR/build.yml pull
+}
+
+update_appserver () {
+	if ! command_exists docker-compose; then
+		install_docker_compose
+	fi
+
+	IMAGE_NAME="onlyoffice-api"
+	CONTAINER_IMAGE=$(docker inspect --format='{{.Config.Image}}' $IMAGE_NAME)
+
+	OLD_LOCAL_IMAGE_RepoDigest=$(get_local_image_RepoDigests "${CONTAINER_IMAGE}")
+	check_pull_image "${CONTAINER_IMAGE}"
+	NEW_LOCAL_IMAGE_RepoDigest=$(get_local_image_RepoDigests "${CONTAINER_IMAGE}")
+	check_image_RepoDigest ${OLD_LOCAL_IMAGE_RepoDigest} ${NEW_LOCAL_IMAGE_RepoDigest}
+
+	if [ ${CHECK_RepoDigest} == "true" ]; then
+		docker_image_update
+	fi
+}
+
+save_parameter() {
+	local VARIABLE_NAME=$1
+	local VARIABLE_VALUE=$2
+
+	if [[ -z ${VARIABLE_VALUE} ]]; then
+		sed -n "/.*${VARIABLE_NAME}=/s///p" $BASE_DIR/.env
+	else
+		echo $VARIABLE_VALUE
+	fi
+}
+
+save_parameters_from_configs() {
+	MYSQL_DATABASE=$(save_parameter MYSQL_DATABASE $MYSQL_DATABASE)
+	MYSQL_USER=$(save_parameter MYSQL_USER $MYSQL_USER)
+	MYSQL_PASSWORD=$(save_parameter MYSQL_PASSWORD $MYSQL_PASSWORD)
+	MYSQL_ROOT_PASSWORD=$(save_parameter MYSQL_ROOT_PASSWORD $MYSQL_ROOT_PASSWORD)
+	MYSQL_HOST=$(save_parameter MYSQL_HOST $MYSQL_HOST)
+	DOCUMENT_SERVER_JWT_SECRET=$(save_parameter DOCUMENT_SERVER_JWT_SECRET $DOCUMENT_SERVER_JWT_SECRET)
+	DOCUMENT_SERVER_HOST=$(save_parameter DOCUMENT_SERVER_HOST $DOCUMENT_SERVER_HOST)
+	ZOO_PORT=$(save_parameter ZOO_PORT $ZOO_PORT)
+	ZOO_HOST=$(save_parameter ZOO_HOST $ZOO_HOST)
+	KAFKA_HOST=$(save_parameter KAFKA_HOST $KAFKA_HOST)
+	ELK_HOST=$(save_parameter ELK_HOST $ELK_HOST)
+	SERVICE_PORT=$(save_parameter SERVICE_PORT $SERVICE_PORT)
+	APP_CORE_MACHINEKEY=$(save_parameter APP_CORE_MACHINEKEY $APP_CORE_MACHINEKEY)
+	APP_CORE_BASE_DOMAIN=$(save_parameter APP_CORE_BASE_DOMAIN $APP_CORE_BASE_DOMAIN)
+	if [ ${EXTERNAL_PORT} = "8092" ]; then 
+		EXTERNAL_PORT=$(grep -oP '(?<=- ).*?(?=:8092)' /app/onlyoffice/appserver.yml)
+	fi
 }
 
 start_installation () {
@@ -894,6 +970,10 @@ start_installation () {
 
 	docker_login
 
+	if [ "$UPDATE" = "true" ]; then
+		save_parameters_from_configs
+	fi
+
 	download_files
 
 	set_jwt_secret
@@ -902,6 +982,10 @@ start_installation () {
 
 	create_network
 
+	if [ "$UPDATE" = "true" ]; then
+		update_appserver
+	fi
+
 	if [ "$INSTALL_MYSQL_SERVER" == "true" ]; then
 		install_mysql_server
 	fi