diff --git a/.drone.star b/.drone.star index c6c3ca779c9..5fb6023e4cf 100644 --- a/.drone.star +++ b/.drone.star @@ -1,8 +1,9 @@ config = { 'apiTests': { - 'coreBranch': 'master', - 'coreCommit': 'e9850b40657ff78f32cb5585ec00342fe07a5ff2', - 'numberOfParts': 4 + 'coreBranch': 'uid-gid-user-create', + 'coreCommit': 'c731fc3ba0858811aef6d3b56002f67b6f9f42f8', + 'numberOfParts': 4, + 'numberOfPartsEos': 10, }, 'uiTests': { 'phoenixBranch': 'master', @@ -95,6 +96,11 @@ def testPipelines(ctx): pipelines.append(coreApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfParts'], 'owncloud')) pipelines.append(coreApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfParts'], 'ocis')) + for runPart in range(1, config['apiTests']['numberOfPartsEos'] + 1): + pipelines.append(coreApiTestsEosStorage(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfPartsEos'])) + + pipelines.append(eosCleanup(ctx, config['apiTests']['numberOfPartsEos'])) + pipelines += uiTests(ctx, config['uiTests']['phoenixBranch'], config['uiTests']['phoenixCommit']) return pipelines @@ -316,6 +322,155 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n }, } +def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, number_of_parts = 1): + return { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'coreApiTests-Eos-Storage-%s' % (part_number), + 'platform': { + 'os': 'linux', + 'arch': 'amd64', + }, + 'steps': + cloneCoreRepos(coreBranch, coreCommit) + [ + { + 'name': 'oC10ApiTests-%s' % (part_number), + 'image': 'owncloudci/php:7.2', + 'pull': 'always', + 'environment' : { + 'SKELETON_DIR': '/srv/app/tmp/testing/data/apiSkeleton', + 'TEST_OCIS':'true', + 'BEHAT_FILTER_TAGS': '~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~@local_storage', + 'DIVIDE_INTO_NUM_PARTS': number_of_parts, + 'RUN_PART': part_number, + 'EXPECTED_FAILURES_FILE': '/drone/src/tests/acceptance/expected-failures-on-EOS-storage.txt', + 'DELETE_USER_DATA_CMD': 'ssh -tt root@$IPADDR docker exec -it mgm-master eos -r 0 0 rm -r /eos/dockertest/reva/users/%s', + 'DRONE_COMMIT_ID': ctx.build.commit, + 'HCLOUD_TOKEN': { + 'from_secret': 'hcloud_token', + }, + 'DRONE_HCLOUD_USER': 'dipak', + }, + 'commands': [ + # Install Go + 'wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz', + 'mkdir -p /usr/local/bin', + 'tar xf go1.14.2.linux-amd64.tar.gz -C /usr/local', + 'ln -s /usr/local/go/bin/* /usr/local/bin', + + # Install Hcloud Cli + 'go get -u github.com/hetznercloud/cli/cmd/hcloud', + 'ln -s /root/go/bin/* /usr/local/bin', + + 'ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""', + 'hcloud ssh-key create --name droneci-eos-test-$DRONE_COMMIT_ID-$RUN_PART --public-key-from-file /root/.ssh/id_rsa.pub', + + 'export SERVER_NAME=droneci-eos-test-$DRONE_COMMIT_ID-$RUN_PART', + + # Create a new machine on hcloud for eos + 'hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $SERVER_NAME --name $SERVER_NAME --label owner=$DRONE_HCLOUD_USER --label for=test --label from=eos-compose', + + # time for the server to start up + 'sleep 15', + + 'export IPADDR=$(hcloud server ip $SERVER_NAME)', + 'export TEST_SERVER_URL=https://$IPADDR:9200', + + 'ssh -o StrictHostKeyChecking=no root@$IPADDR ls', + + # Create an eos machine + 'cd /drone/src', + '/drone/src/tests/spawn_eos.sh', + 'cd /srv/app/testrunner', + + # Run tests + 'make test-acceptance-api', + + # Delete the eos machine + 'hcloud server delete droneci-eos-test-%s-%s' % (ctx.build.commit, part_number), + 'hcloud ssh-key delete droneci-eos-test-%s-%s' % (ctx.build.commit, part_number), + ], + 'volumes': [{ + 'name': 'gopath', + 'path': '/srv/app', + }] + }, + ], + 'volumes': [ + { + 'name': 'gopath', + 'temp': {}, + }, + ], + 'trigger': { + 'ref': [ + 'refs/heads/master', + 'refs/tags/**', + 'refs/pull/**', + ], + }, + } + +def eosCleanup(ctx, parts): + depends_on = [] + steps = [] + for parts_number in range(1, parts + 1): + depends_on.append('coreApiTests-Eos-Storage-%s' % (parts_number)) + steps.append('hcloud server delete droneci-eos-test-%s-%s' % (ctx.build.commit, parts_number)) + steps.append('hcloud ssh-key delete droneci-eos-test-%s-%s' % (ctx.build.commit, parts_number)) + + return { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'eos-cleanup', + 'platform': { + 'os': 'linux', + 'arch': 'amd64', + }, + 'steps': [ + { + 'name': 'eos-cleanup', + 'image': 'owncloudci/php:7.2', + 'pull': 'always', + 'environment' : { + 'HCLOUD_TOKEN': { + 'from_secret': 'hcloud_token', + }, + 'DRONE_HCLOUD_USER': 'dipak', + }, + 'commands': [ + # Install Go + 'wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz', + 'mkdir -p /usr/local/bin', + 'tar xf go1.14.2.linux-amd64.tar.gz -C /usr/local', + 'ln -s /usr/local/go/bin/* /usr/local/bin', + + # Install Hcloud Cli + 'go get -u github.com/hetznercloud/cli/cmd/hcloud', + 'ln -s /root/go/bin/* /usr/local/bin', + ] + steps, + 'volumes': [{ + 'name': 'gopath', + 'path': '/srv/app', + }] + }, + ], + 'volumes': [ + { + 'name': 'gopath', + 'temp': {}, + }, + ], + 'depends_on': depends_on, + 'trigger': { + 'ref': [ + 'refs/heads/master', + 'refs/tags/**', + 'refs/pull/**', + ], + }, + } + def uiTests(ctx, phoenixBranch, phoenixCommit): suiteNames = getUITestSuiteNames() return [uiTestPipeline(suiteName, phoenixBranch, phoenixCommit) for suiteName in suiteNames] diff --git a/docker-compose.yml b/docker-compose.yml index 0d8023b790a..b49269bdbf0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ services: - .:/ocis - ../ocis-reva:/ocis-reva - ../reva:/reva + - ./config:/ocis-config environment: # ocis log level will be used for all services OCIS_LOG_LEVEL: debug @@ -46,6 +47,9 @@ services: REVA_STORAGE_EOS_NAMESPACE: "/eos/dockertest/reva/users" # the eos end xrdcopy binaries use this env var to find the eos mgm EOS_MGM_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094} + # uses identifier registration from config directory + KONNECTD_IDENTIFIER_REGISTRATION_CONF: /ocis-config/identifier-registration.yaml + mgm-master: container_name: mgm-master diff --git a/go.sum b/go.sum index adca2f5e08f..2b645f219f0 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,7 @@ github.com/aws/aws-sdk-go v1.33.19 h1:SMna0QLInNqm+nNL9tb7OVWTqSfNYSxrCa2adnyVth github.com/aws/aws-sdk-go v1.33.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.2 h1:9vCknCdTAmmV4ht7lPuda7aJXzllXwEQyCMZKJHjBrM= github.com/aws/aws-sdk-go v1.34.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.12 h1:7UbBEYDUa4uW0YmRnOd806MS1yoJMcaodBWDzvBShAI= github.com/aws/aws-sdk-go v1.34.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-xray-sdk-go v0.9.4/go.mod h1:XtMKdBQfpVut+tJEwI7+dJFRxxRdxHDyVNp2tHXRq04= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= @@ -324,6 +325,7 @@ github.com/cs3org/reva v1.1.1-0.20200819100654-dcbf0c8ea187 h1:ZyDXH+zy5KPOH5AM5 github.com/cs3org/reva v1.1.1-0.20200819100654-dcbf0c8ea187/go.mod h1:gkGtoun9QABW8k7jzAWWxB9jO64DJ1lreSzTucci/ok= github.com/cs3org/reva v1.2.1-0.20200826162318-c0f54e1f37ea h1:xVyJvR+GoaBrRJV6GnDflgm9bzkmjchCBBg0nMiHu6M= github.com/cs3org/reva v1.2.1-0.20200826162318-c0f54e1f37ea/go.mod h1:FvYB+UPpuPCw0hH42qHVR1R2atZyy69METZI/zEaST8= +github.com/cs3org/reva v1.2.1-0.20200911111727-51649e37df2d h1:vohUfrOZVrbbzt7JUICBDX8rjcedLbuEsmuOJtx8mWY= github.com/cs3org/reva v1.2.1-0.20200911111727-51649e37df2d/go.mod h1:Q1/JB40ZKtlhZUxz+ZqB/O/VlrnQX//NYdPPRPsP0UE= github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= diff --git a/start-eos.sh b/start-eos.sh new file mode 100644 index 00000000000..a70d5af546a --- /dev/null +++ b/start-eos.sh @@ -0,0 +1,16 @@ +#!/bin/bash +docker-compose exec -d ocis /start-ldap + +# time for ldap service to starup within ocis container +sleep 5 + +# Configure ocis +docker-compose exec ocis id einstein +docker-compose exec ocis /ocis/bin/ocis kill reva-users +docker-compose exec ocis /ocis/bin/ocis run reva-users +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home +docker-compose exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home-data +docker-compose exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data + + diff --git a/tests/config/drone/create-config.json.sh b/tests/config/drone/create-config.json.sh new file mode 100755 index 00000000000..9c4a0d152ee --- /dev/null +++ b/tests/config/drone/create-config.json.sh @@ -0,0 +1,20 @@ +#!/bin/bash +echo "{ + \"server\":\"https://${OCIS_DOMAIN}:9200\", + \"theme\":\"owncloud\", + \"version\":\"0.1.0\", + \"openIdConnect\":{ + \"metadata_url\":\"https://${OCIS_DOMAIN}:9200/.well-known/openid-configuration\", + \"authority\":\"https://${OCIS_DOMAIN}:9200\", + \"client_id\":\"phoenix\", + \"response_type\":\"code\", + \"scope\":\"openid profile email\" + }, + \"apps\":[\"files\",\"draw-io\",\"pdf-viewer\",\"markdown-editor\",\"media-viewer\"], + \"external_apps\":[ + {\"id\":\"accounts\",\"path\":\"https://${OCIS_DOMAIN}:9200/accounts.js\"}, + {\"id\":\"settings\",\"path\":\"https://${OCIS_DOMAIN}:9200/settings.js\"} + ], + \"options\":{\"hideSearchBar\":true} +}" > $PWD/config.json + diff --git a/tests/config/drone/create-identifier-registration.sh b/tests/config/drone/create-identifier-registration.sh new file mode 100755 index 00000000000..4d27d9249d4 --- /dev/null +++ b/tests/config/drone/create-identifier-registration.sh @@ -0,0 +1,15 @@ +#!/bin/bash +echo "--- +# OpenID Connect client registry. +clients: + - id: phoenix + name: OCIS + application_type: web + insecure: yes + trusted: yes + redirect_uris: + - https://${OCIS_DOMAIN}:9200/oidc-callback.html + - https://${OCIS_DOMAIN}:9200/ + origins: + - https://${OCIS_DOMAIN}:9200 +authorities:" > $PWD/identifier-registration.yaml diff --git a/tests/config/drone/setup-eos.sh b/tests/config/drone/setup-eos.sh new file mode 100755 index 00000000000..fccff43ac7d --- /dev/null +++ b/tests/config/drone/setup-eos.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# wait for ocis to start +until $(curl -k --output /dev/null --silent --head --fail https://localhost:9200); +do + echo '.' + sleep 5 +done + +docker-compose down + +echo $OCIS_DOMAIN + +OCIS_DOMAIN=${OCIS_DOMAIN} docker-compose up -d + +# wait for ocis to start +until $(curl -k --output /dev/null --silent --head --fail https://localhost:9200); +do + echo '.' + sleep 5 +done + +sleep 5 + +docker-compose exec -d ocis /start-ldap + +# time for ldap service to starup within ocis container +sleep 5 + +# Configure ocis +docker-compose exec ocis id einstein +docker-compose exec ocis /ocis/bin/ocis kill reva-users +docker-compose exec ocis /ocis/bin/ocis run reva-users +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home +docker-compose exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home-data +docker-compose exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data + diff --git a/tests/spawn_eos.sh b/tests/spawn_eos.sh new file mode 100755 index 00000000000..8cf719c61a0 --- /dev/null +++ b/tests/spawn_eos.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# ME=$DRONE_HCLOUD_USER +# SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID}-${RUN_PART} + +# # setup ssh keys for hcloud +# ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N "" +# hcloud ssh-key create --name drone-${DRONE_COMMIT_ID}-${RUN_PART} --public-key-from-file /root/.ssh/id_rsa.pub + +# # Create a new machine on hcloud for eos +# hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $SERVER_NAME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose +# # time for the server to start up +# sleep 15 + +# IPADDR=$(hcloud server ip $SERVER_NAME) +# export IPADDR=$IPADDR +# export TEST_SERVER_URL=https://${IPADDR}:9200 + +# ssh -o StrictHostKeyChecking=no root@$IPADDR + + +# Setup system and clone ocis +ssh -tt root@$IPADDR apt-get update -y +ssh -tt root@$IPADDR apt-get install -y git screen docker.io docker-compose ldap-utils +ssh -tt root@$IPADDR git clone https://github.com/owncloud/ocis.git /ocis +ssh -tt root@$IPADDR "cd /ocis && git checkout $DRONE_COMMIT_ID" + +# Create necessary files +ssh -tt root@$IPADDR "mkdir -p /ocis/tests/eos-config" +ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" +ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" + +# run ocis with eos +ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose up -d" + +# Some necessary configuration for eos +ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/setup-eos.sh"