From 6f7185c44123c632274d5efc2a296ee258071301 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Thu, 3 Sep 2020 10:32:40 +0545 Subject: [PATCH 01/13] wip - setting up tests with eos on drone-ci --- docker-compose-eos-ci.yml | 179 ++++++++++++++++++ tests/config/drone/create-config.json.sh | 20 ++ .../drone/create-identifier-registration.sh | 15 ++ tests/config/drone/setup-eos.sh | 19 ++ tests/spwan_eos.sh | 28 +++ 5 files changed, 261 insertions(+) create mode 100644 docker-compose-eos-ci.yml create mode 100644 tests/config/drone/create-config.json.sh create mode 100644 tests/config/drone/create-identifier-registration.sh create mode 100644 tests/config/drone/setup-eos.sh create mode 100644 tests/spwan_eos.sh diff --git a/docker-compose-eos-ci.yml b/docker-compose-eos-ci.yml new file mode 100644 index 00000000000..ab5efdec465 --- /dev/null +++ b/docker-compose-eos-ci.yml @@ -0,0 +1,179 @@ +--- +version: '3.5' + +networks: + testnet: + name: testnet + +services: + ocis: + container_name: ocis + image: owncloud/eos-ocis-dev:latest + tty: true + privileged: true + stdin_open: true + ports: + - 9200:9200 + env_file: + - ./config/eos-docker.env + hostname: ocis + networks: + - testnet + volumes: + - .:/ocis + - ../ocis-reva:/ocis-reva + - ../reva:/reva + - ./tests/config/drone:/eos-config + environment: + # ocis log level will be used for all services + OCIS_LOG_LEVEL: debug + # domain setup + # TODO currently the below lines hardcode the port to 9200, use an OCIS_URL that includes protocol and port + OCIS_DOMAIN: ${OCIS_DOMAIN:-localhost} + PROXY_OIDC_ISSUER: https://${OCIS_DOMAIN:-localhost}:9200 + KONNECTD_ISS: https://${OCIS_DOMAIN:-localhost}:9200 + PHOENIX_OIDC_AUTHORITY: https://${OCIS_DOMAIN:-localhost}:9200 + PHOENIX_OIDC_METADATA_URL: https://${OCIS_DOMAIN:-localhost}:9200/.well-known/openid-configuration + PHOENIX_WEB_CONFIG_SERVER: https://${OCIS_DOMAIN:-localhost}:9200 + REVA_OIDC_ISSUER: https://${OCIS_DOMAIN:-localhost}:9200 + REVA_LDAP_IDP: https://${OCIS_DOMAIN:-localhost}:9200 + # TODO make id the default in ocis-reva + REVA_STORAGE_EOS_LAYOUT: "{{substr 0 1 .Id.OpaqueId}}/{{.Id.OpaqueId}}" + REVA_FRONTEND_URL: https://${OCIS_DOMAIN:-localhost}:9200 + REVA_DATAGATEWAY_URL: https://${OCIS_DOMAIN:-localhost}:9200/data + # common eos settings used for both drivers: eos and eoshome + REVA_STORAGE_EOS_MASTER_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094} + REVA_STORAGE_EOS_SLAVE_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094} + REVA_STORAGE_EOS_NAMESPACE: "/eos/dockertest/reva/users" + # the eos end xrdcopy binaries use this env var to find the eos mgm + EOS_MGM_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094} + PHOENIX_WEB_CONFIG: /eos-config/config.json + KONNECTD_IDENTIFIER_REGISTRATION_CONF: /eos-config/identifier-registration.yml + + mgm-master: + container_name: mgm-master + image: owncloud/eos-mgm:4.6.5 + tty: true + privileged: true + stdin_open: true + env_file: + - ./config/eos-docker.env + hostname: mgm-master.testnet + networks: + - testnet + volumes: + - eos-mgm-master-log:/var/log/eos + - eos-mgm-master-config:/var/eos/config + - eos-mgm-master-ns-queue:/var/eos/ns-queue + # this volume kills mgm-master during startup + # - ./e/master/var/eos/md:/var/eos/md + environment: + EOS_SET_MASTER: 1 + + mq-master: + container_name: mq-master + image: owncloud/eos-mq:4.6.5 + tty: true + privileged: true + stdin_open: true + env_file: + - ./config/eos-docker.env + hostname: mq-master.testnet + networks: + - testnet + volumes: + - eos-mq-master-log:/var/log/eos + - eos-mq-master-config:/var/eos/config + - eos-mq-master-ns-queue:/var/eos/ns-queue + environment: + EOS_SET_MASTER: 1 + + fst: + container_name: fst + image: owncloud/eos-fst:4.6.5 + tty: true + privileged: true + stdin_open: true + env_file: + - ./config/eos-docker.env + hostname: fst.testnet + networks: + - testnet + volumes: + - eos-fst-log:/var/log/eos + - eos-fst-disks:/disks + environment: + EOS_MGM_URL: "root://mgm-master.testnet" + + quark-1: + container_name: quark-1 + image: owncloud/eos-qdb:4.6.5 + tty: true + privileged: true + stdin_open: true + env_file: + - ./config/eos-docker.env + hostname: quark-1.testnet + networks: + - testnet + volumes: + - eos-quarkdb1:/var/lib/quarkdb + environment: + EOS_QDB_DIR: "/var/lib/quarkdb/eosns" + EOS_QDB_PORT: "7777" + EOS_QDB_MODE: "raft" + EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b" + EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777" + + quark-2: + container_name: quark-2 + image: owncloud/eos-qdb:4.6.5 + tty: true + privileged: true + stdin_open: true + env_file: + - ./config/eos-docker.env + hostname: quark-2.testnet + networks: + - testnet + volumes: + - eos-quarkdb2:/var/lib/quarkdb + environment: + EOS_QDB_DIR: "/var/lib/quarkdb/eosns" + EOS_QDB_PORT: "7777" + EOS_QDB_MODE: "raft" + EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b" + EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777" + + quark-3: + container_name: quark-3 + image: owncloud/eos-qdb:4.6.5 + tty: true + privileged: true + stdin_open: true + env_file: + - ./config/eos-docker.env + hostname: quark-3.testnet + networks: + - testnet + volumes: + - eos-quarkdb3:/var/lib/quarkdb + environment: + EOS_QDB_DIR: "/var/lib/quarkdb/eosns" + EOS_QDB_PORT: "7777" + EOS_QDB_MODE: "raft" + EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b" + EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777" + +volumes: + eos-mgm-master-log: + eos-mgm-master-config: + eos-mgm-master-ns-queue: + eos-mq-master-log: + eos-mq-master-config: + eos-mq-master-ns-queue: + eos-fst-log: + eos-fst-disks: + eos-quarkdb1: + eos-quarkdb2: + eos-quarkdb3: diff --git a/tests/config/drone/create-config.json.sh b/tests/config/drone/create-config.json.sh new file mode 100644 index 00000000000..9c4a0d152ee --- /dev/null +++ b/tests/config/drone/create-config.json.sh @@ -0,0 +1,20 @@ +#!/bin/bash +echo "{ + \"server\":\"https://${OCIS_DOMAIN}:9200\", + \"theme\":\"owncloud\", + \"version\":\"0.1.0\", + \"openIdConnect\":{ + \"metadata_url\":\"https://${OCIS_DOMAIN}:9200/.well-known/openid-configuration\", + \"authority\":\"https://${OCIS_DOMAIN}:9200\", + \"client_id\":\"phoenix\", + \"response_type\":\"code\", + \"scope\":\"openid profile email\" + }, + \"apps\":[\"files\",\"draw-io\",\"pdf-viewer\",\"markdown-editor\",\"media-viewer\"], + \"external_apps\":[ + {\"id\":\"accounts\",\"path\":\"https://${OCIS_DOMAIN}:9200/accounts.js\"}, + {\"id\":\"settings\",\"path\":\"https://${OCIS_DOMAIN}:9200/settings.js\"} + ], + \"options\":{\"hideSearchBar\":true} +}" > $PWD/config.json + diff --git a/tests/config/drone/create-identifier-registration.sh b/tests/config/drone/create-identifier-registration.sh new file mode 100644 index 00000000000..5fb5870aeb7 --- /dev/null +++ b/tests/config/drone/create-identifier-registration.sh @@ -0,0 +1,15 @@ +#!/bin/bash +echo "--- +# OpenID Connect client registry. +clients: + - id: phoenix + name: OCIS + application_type: web + insecure: yes + trusted: yes + redirect_uris: + - https://${OCIS_DOMAIN}:9200/oidc-callback.html + - https://${OCIS_DOMAIN}:9200/ + origins: + - https://${OCIS_DOMAIN}:9200 +authorities:" > $PWD/identifier-registration.yml diff --git a/tests/config/drone/setup-eos.sh b/tests/config/drone/setup-eos.sh new file mode 100644 index 00000000000..8c655b231bf --- /dev/null +++ b/tests/config/drone/setup-eos.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# wait for ocis to start +until $(curl -k --output /dev/null --silent --head --fail https://localhost:9200); +do + printf '.' + sleep 2 +done + +docker-compose -f ./docker-compose-eos-ci.yml exec -d ocis /start-ldap +sleep 5 +docker-compose -f ./docker-compose-eos-ci.yml exec ocis id einstein +docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-users +docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis run reva-users +docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-storage-home +docker-compose -f ./docker-compose-eos-ci.yml exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home +docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-storage-home-data +docker-compose -f ./docker-compose-eos-ci.yml exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data + diff --git a/tests/spwan_eos.sh b/tests/spwan_eos.sh new file mode 100644 index 00000000000..3cab62af06f --- /dev/null +++ b/tests/spwan_eos.sh @@ -0,0 +1,28 @@ +#!/bin/bash +ME=$DRONE_HCLOUD_USER +SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID} + +# Create a new machine on hcloud for eos +hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $ME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose + +IPADDR=$(hcloud server ip $SERVER_NAME) +OCIS_DOMAIN=$(hcloud server ip $SERVER_NAME) + +# timeout 180 while [[ \"$(curl -k -v -s -o /dev/null -w ''%{http_code}'' https://:9200)\" != \"200\" ]]; do sleep 2; done +# sleep 15 + +# Setup system and clone ocis +ssh -t root@$IPADDR apt-get update -y +ssh -t root@$IPADDR apt-get install -y git screen docker.io docker-compose ldap-utils +ssh -t root@$IPADDR git clone https://github.com/owncloud/ocis.git /ocis +ssh -t root@$IPADDR "cd /ocis && git checkout $DRONE_COMMIT_ID" + +# Create necessary files +ssh -t root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" +ssh -t root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" + +# run ocis with eos +ssh -t root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose -f ./docker-compose-eos-ci.yml up -d" + +# Some necessary configuration for eos +ssh -t root@$IPADDR "cd /ocis && bash /ocis/tests/config/drone/setup-eos.sh" From 14da797e081c976a7ed2c2069aea8f5e00a0af44 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Fri, 4 Sep 2020 09:36:39 +0545 Subject: [PATCH 02/13] Add new pipeline for eos testing --- .drone.star | 62 +++++++++++++++++++++++++++++++++ go.sum | 2 ++ tests/config/drone/setup-eos.sh | 4 +++ tests/spwan_eos.sh | 10 +++--- 4 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.drone.star b/.drone.star index c6c3ca779c9..be697544f92 100644 --- a/.drone.star +++ b/.drone.star @@ -95,6 +95,9 @@ def testPipelines(ctx): pipelines.append(coreApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfParts'], 'owncloud')) pipelines.append(coreApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfParts'], 'ocis')) + for runPart in range(1, config['apiTests']['numberOfParts'] + 1): + pipelines.append(coreApiTestsEosStorage(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfParts'])) + pipelines += uiTests(ctx, config['uiTests']['phoenixBranch'], config['uiTests']['phoenixCommit']) return pipelines @@ -316,6 +319,65 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n }, } +def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, number_of_parts = 1): + return { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'coreApiTests-Eos-Storage-%s' % (part_number), + 'platform': { + 'os': 'linux', + 'arch': 'amd64', + }, + 'steps': + cloneCoreRepos(coreBranch, coreCommit) + [ + { + 'name': 'oC10ApiTests-%s' % (part_number), + 'image': 'owncloudci/php:7.2', + 'pull': 'always', + 'environment' : { + 'SKELETON_DIR': '/srv/app/tmp/testing/data/apiSkeleton', + 'TEST_OCIS':'true', + 'BEHAT_FILTER_TAGS': '~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~@local_storage', + 'DIVIDE_INTO_NUM_PARTS': number_of_parts, + 'RUN_PART': part_number, + 'EXPECTED_FAILURES_FILE': '/drone/src/tests/acceptance/expected-failures-on-EOS-storage.txt', + 'DELETE_USER_DATA_CMD': 'ssh -t root@${IPADDR} docker exec -it mgm-master eos -r 0 0 rm -r /eos/dockertest/reva/users/%s', + 'DRONE_COMMIT_ID': ctx.build.commit + }, + 'commands': [ + 'wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz', + 'mkdir -p /usr/local/bin', + 'tar xf go1.14.2.linux-amd64.tar.gz -C /usr/local', + 'ln -s /usr/local/go/bin/* /usr/local/bin', + + 'go get -u github.com/hetznercloud/cli/cmd/hcloud', + + 'cd /drone/src', + '/drone/src/tests/spwan_eos.sh', + 'cd /srv/app/testrunner', + 'make test-acceptance-api', + ], + 'volumes': [{ + 'name': 'gopath', + 'path': '/srv/app', + }] + }, + ], + 'volumes': [ + { + 'name': 'gopath', + 'temp': {}, + }, + ], + 'trigger': { + 'ref': [ + 'refs/heads/master', + 'refs/tags/**', + 'refs/pull/**', + ], + }, + } + def uiTests(ctx, phoenixBranch, phoenixCommit): suiteNames = getUITestSuiteNames() return [uiTestPipeline(suiteName, phoenixBranch, phoenixCommit) for suiteName in suiteNames] diff --git a/go.sum b/go.sum index adca2f5e08f..2b645f219f0 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,7 @@ github.com/aws/aws-sdk-go v1.33.19 h1:SMna0QLInNqm+nNL9tb7OVWTqSfNYSxrCa2adnyVth github.com/aws/aws-sdk-go v1.33.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.2 h1:9vCknCdTAmmV4ht7lPuda7aJXzllXwEQyCMZKJHjBrM= github.com/aws/aws-sdk-go v1.34.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.12 h1:7UbBEYDUa4uW0YmRnOd806MS1yoJMcaodBWDzvBShAI= github.com/aws/aws-sdk-go v1.34.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-xray-sdk-go v0.9.4/go.mod h1:XtMKdBQfpVut+tJEwI7+dJFRxxRdxHDyVNp2tHXRq04= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= @@ -324,6 +325,7 @@ github.com/cs3org/reva v1.1.1-0.20200819100654-dcbf0c8ea187 h1:ZyDXH+zy5KPOH5AM5 github.com/cs3org/reva v1.1.1-0.20200819100654-dcbf0c8ea187/go.mod h1:gkGtoun9QABW8k7jzAWWxB9jO64DJ1lreSzTucci/ok= github.com/cs3org/reva v1.2.1-0.20200826162318-c0f54e1f37ea h1:xVyJvR+GoaBrRJV6GnDflgm9bzkmjchCBBg0nMiHu6M= github.com/cs3org/reva v1.2.1-0.20200826162318-c0f54e1f37ea/go.mod h1:FvYB+UPpuPCw0hH42qHVR1R2atZyy69METZI/zEaST8= +github.com/cs3org/reva v1.2.1-0.20200911111727-51649e37df2d h1:vohUfrOZVrbbzt7JUICBDX8rjcedLbuEsmuOJtx8mWY= github.com/cs3org/reva v1.2.1-0.20200911111727-51649e37df2d/go.mod h1:Q1/JB40ZKtlhZUxz+ZqB/O/VlrnQX//NYdPPRPsP0UE= github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= diff --git a/tests/config/drone/setup-eos.sh b/tests/config/drone/setup-eos.sh index 8c655b231bf..2fca2178d12 100644 --- a/tests/config/drone/setup-eos.sh +++ b/tests/config/drone/setup-eos.sh @@ -8,7 +8,11 @@ do done docker-compose -f ./docker-compose-eos-ci.yml exec -d ocis /start-ldap + +# time for ldap service to starup within ocis container sleep 5 + +# Configure ocis docker-compose -f ./docker-compose-eos-ci.yml exec ocis id einstein docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-users docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis run reva-users diff --git a/tests/spwan_eos.sh b/tests/spwan_eos.sh index 3cab62af06f..cefd1b795fc 100644 --- a/tests/spwan_eos.sh +++ b/tests/spwan_eos.sh @@ -1,15 +1,15 @@ #!/bin/bash ME=$DRONE_HCLOUD_USER -SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID} +SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID}-${PART_NUMBER} # Create a new machine on hcloud for eos hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $ME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose +# time for the server to start up +sleep 15 IPADDR=$(hcloud server ip $SERVER_NAME) -OCIS_DOMAIN=$(hcloud server ip $SERVER_NAME) - -# timeout 180 while [[ \"$(curl -k -v -s -o /dev/null -w ''%{http_code}'' https://:9200)\" != \"200\" ]]; do sleep 2; done -# sleep 15 +export IPADDR=$IPADDR +export TEST_SERVER_URL=https://${IPADDR}:9200 # Setup system and clone ocis ssh -t root@$IPADDR apt-get update -y From 00be87544514552dc8cec591635a95b7b8153900 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Fri, 4 Sep 2020 09:55:48 +0545 Subject: [PATCH 03/13] Add executable permission for eos scripts --- .drone.star | 11 ++++++++++- tests/config/drone/create-config.json.sh | 0 tests/config/drone/create-identifier-registration.sh | 0 tests/config/drone/setup-eos.sh | 0 tests/{spwan_eos.sh => spawn_eos.sh} | 2 +- 5 files changed, 11 insertions(+), 2 deletions(-) mode change 100644 => 100755 tests/config/drone/create-config.json.sh mode change 100644 => 100755 tests/config/drone/create-identifier-registration.sh mode change 100644 => 100755 tests/config/drone/setup-eos.sh rename tests/{spwan_eos.sh => spawn_eos.sh} (95%) mode change 100644 => 100755 diff --git a/.drone.star b/.drone.star index be697544f92..ef60bce831e 100644 --- a/.drone.star +++ b/.drone.star @@ -345,17 +345,26 @@ def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_num 'DRONE_COMMIT_ID': ctx.build.commit }, 'commands': [ + # Install Go 'wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz', 'mkdir -p /usr/local/bin', 'tar xf go1.14.2.linux-amd64.tar.gz -C /usr/local', 'ln -s /usr/local/go/bin/* /usr/local/bin', + # Install Hcloud Cli 'go get -u github.com/hetznercloud/cli/cmd/hcloud', + 'ln -s /root/go/bin/* /usr/local/bin', + # Create an eos machine 'cd /drone/src', - '/drone/src/tests/spwan_eos.sh', + '/drone/src/tests/spawn_eos.sh', 'cd /srv/app/testrunner', + + # Run tests 'make test-acceptance-api', + + # Delete the eos machine + 'hcloud server delete droneci-eos-test-%s-%s' % (ctx.build.commit, part_number) ], 'volumes': [{ 'name': 'gopath', diff --git a/tests/config/drone/create-config.json.sh b/tests/config/drone/create-config.json.sh old mode 100644 new mode 100755 diff --git a/tests/config/drone/create-identifier-registration.sh b/tests/config/drone/create-identifier-registration.sh old mode 100644 new mode 100755 diff --git a/tests/config/drone/setup-eos.sh b/tests/config/drone/setup-eos.sh old mode 100644 new mode 100755 diff --git a/tests/spwan_eos.sh b/tests/spawn_eos.sh old mode 100644 new mode 100755 similarity index 95% rename from tests/spwan_eos.sh rename to tests/spawn_eos.sh index cefd1b795fc..3a094fb0563 --- a/tests/spwan_eos.sh +++ b/tests/spawn_eos.sh @@ -1,6 +1,6 @@ #!/bin/bash ME=$DRONE_HCLOUD_USER -SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID}-${PART_NUMBER} +SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID}-${RUN_PART} # Create a new machine on hcloud for eos hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $ME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose From 54aa62cb0c564029c9d34d64990630fa6bebbdb4 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Sun, 6 Sep 2020 11:59:41 +0545 Subject: [PATCH 04/13] set ssh keys --- .drone.star | 6 +++++- tests/spawn_eos.sh | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.drone.star b/.drone.star index ef60bce831e..dedfafad9ab 100644 --- a/.drone.star +++ b/.drone.star @@ -342,7 +342,11 @@ def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_num 'RUN_PART': part_number, 'EXPECTED_FAILURES_FILE': '/drone/src/tests/acceptance/expected-failures-on-EOS-storage.txt', 'DELETE_USER_DATA_CMD': 'ssh -t root@${IPADDR} docker exec -it mgm-master eos -r 0 0 rm -r /eos/dockertest/reva/users/%s', - 'DRONE_COMMIT_ID': ctx.build.commit + 'DRONE_COMMIT_ID': ctx.build.commit, + 'HCLOUD_TOKEN': { + 'from_secret': 'hcloud_token', + }, + 'DRONE_HCLOUD_USER': 'dipak', }, 'commands': [ # Install Go diff --git a/tests/spawn_eos.sh b/tests/spawn_eos.sh index 3a094fb0563..ee932911d15 100755 --- a/tests/spawn_eos.sh +++ b/tests/spawn_eos.sh @@ -2,8 +2,12 @@ ME=$DRONE_HCLOUD_USER SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID}-${RUN_PART} +# setup ssh keys for hcloud +ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N "" +hcloud ssh-key --name drone-%s-%s --public-key-from-file /root/.ssh/id_rsa.pub + # Create a new machine on hcloud for eos -hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $ME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose +hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $SERVER_NAME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose # time for the server to start up sleep 15 @@ -11,6 +15,9 @@ IPADDR=$(hcloud server ip $SERVER_NAME) export IPADDR=$IPADDR export TEST_SERVER_URL=https://${IPADDR}:9200 +ssh -o StrictHostKeyChecking=no root@$IPADDR + + # Setup system and clone ocis ssh -t root@$IPADDR apt-get update -y ssh -t root@$IPADDR apt-get install -y git screen docker.io docker-compose ldap-utils From c2e8cd3d9f9924fffdb78a65e137d3e6e13ce1d9 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Mon, 7 Sep 2020 09:40:17 +0545 Subject: [PATCH 05/13] refactor scripts --- .drone.star | 14 ++++++++++++++ tests/spawn_eos.sh | 22 +++++++++++----------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.drone.star b/.drone.star index dedfafad9ab..caca45c5edb 100644 --- a/.drone.star +++ b/.drone.star @@ -359,6 +359,20 @@ def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_num 'go get -u github.com/hetznercloud/cli/cmd/hcloud', 'ln -s /root/go/bin/* /usr/local/bin', + 'ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""', + 'hcloud ssh-key create --name drone-$DRONE_COMMIT_ID-$RUN_PART --public-key-from-file /root/.ssh/id_rsa.pub', + + # Create a new machine on hcloud for eos + 'hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $SERVER_NAME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose', + # time for the server to start up + 'sleep 15', + + 'export IPADDR=$(hcloud server ip ${SERVER_NAME})', + 'export IPADDR=${IPADDR}', + 'export TEST_SERVER_URL=https://${IPADDR}:9200', + + 'ssh -o StrictHostKeyChecking=no root@$IPADDR', + # Create an eos machine 'cd /drone/src', '/drone/src/tests/spawn_eos.sh', diff --git a/tests/spawn_eos.sh b/tests/spawn_eos.sh index ee932911d15..483d1be9107 100755 --- a/tests/spawn_eos.sh +++ b/tests/spawn_eos.sh @@ -2,20 +2,20 @@ ME=$DRONE_HCLOUD_USER SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID}-${RUN_PART} -# setup ssh keys for hcloud -ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N "" -hcloud ssh-key --name drone-%s-%s --public-key-from-file /root/.ssh/id_rsa.pub +# # setup ssh keys for hcloud +# ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N "" +# hcloud ssh-key create --name drone-${DRONE_COMMIT_ID}-${RUN_PART} --public-key-from-file /root/.ssh/id_rsa.pub -# Create a new machine on hcloud for eos -hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $SERVER_NAME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose -# time for the server to start up -sleep 15 +# # Create a new machine on hcloud for eos +# hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $SERVER_NAME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose +# # time for the server to start up +# sleep 15 -IPADDR=$(hcloud server ip $SERVER_NAME) -export IPADDR=$IPADDR -export TEST_SERVER_URL=https://${IPADDR}:9200 +# IPADDR=$(hcloud server ip $SERVER_NAME) +# export IPADDR=$IPADDR +# export TEST_SERVER_URL=https://${IPADDR}:9200 -ssh -o StrictHostKeyChecking=no root@$IPADDR +# ssh -o StrictHostKeyChecking=no root@$IPADDR # Setup system and clone ocis From 0ddadadc44ce55f71f699f8a0973b79c5ce10909 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Mon, 7 Sep 2020 12:05:36 +0545 Subject: [PATCH 06/13] fix drone file --- .drone.star | 17 ++++++++++------- tests/config/drone/setup-eos.sh | 4 ++-- tests/spawn_eos.sh | 20 ++++++++++---------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.drone.star b/.drone.star index caca45c5edb..35412d3086b 100644 --- a/.drone.star +++ b/.drone.star @@ -341,7 +341,7 @@ def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_num 'DIVIDE_INTO_NUM_PARTS': number_of_parts, 'RUN_PART': part_number, 'EXPECTED_FAILURES_FILE': '/drone/src/tests/acceptance/expected-failures-on-EOS-storage.txt', - 'DELETE_USER_DATA_CMD': 'ssh -t root@${IPADDR} docker exec -it mgm-master eos -r 0 0 rm -r /eos/dockertest/reva/users/%s', + 'DELETE_USER_DATA_CMD': 'ssh -t root@$IPADDR docker exec -it mgm-master eos -r 0 0 rm -r /eos/dockertest/reva/users/%s', 'DRONE_COMMIT_ID': ctx.build.commit, 'HCLOUD_TOKEN': { 'from_secret': 'hcloud_token', @@ -360,16 +360,18 @@ def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_num 'ln -s /root/go/bin/* /usr/local/bin', 'ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""', - 'hcloud ssh-key create --name drone-$DRONE_COMMIT_ID-$RUN_PART --public-key-from-file /root/.ssh/id_rsa.pub', + 'hcloud ssh-key create --name droneci-eos-test-$DRONE_COMMIT_ID-$RUN_PART --public-key-from-file /root/.ssh/id_rsa.pub', + + 'export SERVER_NAME=droneci-eos-test-$DRONE_COMMIT_ID-$RUN_PART', # Create a new machine on hcloud for eos - 'hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $SERVER_NAME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose', + 'hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $SERVER_NAME --name $SERVER_NAME --label owner=$DRONE_HCLOUD_USER --label for=test --label from=eos-compose', + # time for the server to start up 'sleep 15', - 'export IPADDR=$(hcloud server ip ${SERVER_NAME})', - 'export IPADDR=${IPADDR}', - 'export TEST_SERVER_URL=https://${IPADDR}:9200', + 'export IPADDR=$(hcloud server ip $SERVER_NAME)', + 'export TEST_SERVER_URL=https://$IPADDR:9200', 'ssh -o StrictHostKeyChecking=no root@$IPADDR', @@ -382,7 +384,8 @@ def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_num 'make test-acceptance-api', # Delete the eos machine - 'hcloud server delete droneci-eos-test-%s-%s' % (ctx.build.commit, part_number) + 'hcloud server delete droneci-eos-test-%s-%s' % (ctx.build.commit, part_number), + 'hcloud ssh-key delete droneci-eos-test-%s-%s' % (ctx.build.commit, part_number), ], 'volumes': [{ 'name': 'gopath', diff --git a/tests/config/drone/setup-eos.sh b/tests/config/drone/setup-eos.sh index 2fca2178d12..927bbef7efd 100755 --- a/tests/config/drone/setup-eos.sh +++ b/tests/config/drone/setup-eos.sh @@ -3,8 +3,8 @@ # wait for ocis to start until $(curl -k --output /dev/null --silent --head --fail https://localhost:9200); do - printf '.' - sleep 2 + echo '.' + sleep 5 done docker-compose -f ./docker-compose-eos-ci.yml exec -d ocis /start-ldap diff --git a/tests/spawn_eos.sh b/tests/spawn_eos.sh index 483d1be9107..146011923fb 100755 --- a/tests/spawn_eos.sh +++ b/tests/spawn_eos.sh @@ -1,6 +1,6 @@ #!/bin/bash -ME=$DRONE_HCLOUD_USER -SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID}-${RUN_PART} +# ME=$DRONE_HCLOUD_USER +# SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID}-${RUN_PART} # # setup ssh keys for hcloud # ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N "" @@ -19,17 +19,17 @@ SERVER_NAME=droneci-eos-test-${DRONE_COMMIT_ID}-${RUN_PART} # Setup system and clone ocis -ssh -t root@$IPADDR apt-get update -y -ssh -t root@$IPADDR apt-get install -y git screen docker.io docker-compose ldap-utils -ssh -t root@$IPADDR git clone https://github.com/owncloud/ocis.git /ocis -ssh -t root@$IPADDR "cd /ocis && git checkout $DRONE_COMMIT_ID" +ssh -tt root@$IPADDR apt-get update -y +ssh -tt root@$IPADDR apt-get install -y git screen docker.io docker-compose ldap-utils +ssh -tt root@$IPADDR git clone https://github.com/owncloud/ocis.git /ocis +ssh -tt root@$IPADDR "cd /ocis && git checkout $DRONE_COMMIT_ID" # Create necessary files -ssh -t root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" -ssh -t root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" +ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" +ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" # run ocis with eos -ssh -t root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose -f ./docker-compose-eos-ci.yml up -d" +ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose -f ./docker-compose-eos-ci.yml up -d" # Some necessary configuration for eos -ssh -t root@$IPADDR "cd /ocis && bash /ocis/tests/config/drone/setup-eos.sh" +ssh -tt root@$IPADDR "cd /ocis && bash /ocis/tests/config/drone/setup-eos.sh" From 20d813599e27d8810f9b2e3acc1829c68e2d03c7 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Mon, 7 Sep 2020 19:36:46 +0545 Subject: [PATCH 07/13] fix docker-compose file --- .drone.star | 4 ++-- docker-compose-eos-ci.yml | 2 +- .../drone/create-identifier-registration.sh | 2 +- tests/config/drone/setup-eos.sh | 16 ++++++++-------- tests/spawn_eos.sh | 9 ++++++--- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/.drone.star b/.drone.star index 35412d3086b..d68ff626897 100644 --- a/.drone.star +++ b/.drone.star @@ -341,7 +341,7 @@ def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_num 'DIVIDE_INTO_NUM_PARTS': number_of_parts, 'RUN_PART': part_number, 'EXPECTED_FAILURES_FILE': '/drone/src/tests/acceptance/expected-failures-on-EOS-storage.txt', - 'DELETE_USER_DATA_CMD': 'ssh -t root@$IPADDR docker exec -it mgm-master eos -r 0 0 rm -r /eos/dockertest/reva/users/%s', + 'DELETE_USER_DATA_CMD': 'ssh -tt root@$IPADDR docker exec -it mgm-master eos -r 0 0 rm -r /eos/dockertest/reva/users/%s', 'DRONE_COMMIT_ID': ctx.build.commit, 'HCLOUD_TOKEN': { 'from_secret': 'hcloud_token', @@ -373,7 +373,7 @@ def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_num 'export IPADDR=$(hcloud server ip $SERVER_NAME)', 'export TEST_SERVER_URL=https://$IPADDR:9200', - 'ssh -o StrictHostKeyChecking=no root@$IPADDR', + 'ssh -o StrictHostKeyChecking=no root@$IPADDR ls', # Create an eos machine 'cd /drone/src', diff --git a/docker-compose-eos-ci.yml b/docker-compose-eos-ci.yml index ab5efdec465..ec16643acd1 100644 --- a/docker-compose-eos-ci.yml +++ b/docker-compose-eos-ci.yml @@ -48,7 +48,7 @@ services: # the eos end xrdcopy binaries use this env var to find the eos mgm EOS_MGM_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094} PHOENIX_WEB_CONFIG: /eos-config/config.json - KONNECTD_IDENTIFIER_REGISTRATION_CONF: /eos-config/identifier-registration.yml + KONNECTD_IDENTIFIER_REGISTRATION_CONF: /eos-config/identifier-registration.yaml mgm-master: container_name: mgm-master diff --git a/tests/config/drone/create-identifier-registration.sh b/tests/config/drone/create-identifier-registration.sh index 5fb5870aeb7..4d27d9249d4 100755 --- a/tests/config/drone/create-identifier-registration.sh +++ b/tests/config/drone/create-identifier-registration.sh @@ -12,4 +12,4 @@ clients: - https://${OCIS_DOMAIN}:9200/ origins: - https://${OCIS_DOMAIN}:9200 -authorities:" > $PWD/identifier-registration.yml +authorities:" > $PWD/identifier-registration.yaml diff --git a/tests/config/drone/setup-eos.sh b/tests/config/drone/setup-eos.sh index 927bbef7efd..37159cea07e 100755 --- a/tests/config/drone/setup-eos.sh +++ b/tests/config/drone/setup-eos.sh @@ -7,17 +7,17 @@ do sleep 5 done -docker-compose -f ./docker-compose-eos-ci.yml exec -d ocis /start-ldap +docker-compose exec -d ocis /start-ldap # time for ldap service to starup within ocis container sleep 5 # Configure ocis -docker-compose -f ./docker-compose-eos-ci.yml exec ocis id einstein -docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-users -docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis run reva-users -docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-storage-home -docker-compose -f ./docker-compose-eos-ci.yml exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home -docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-storage-home-data -docker-compose -f ./docker-compose-eos-ci.yml exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data +docker-compose exec ocis id einstein +docker-compose exec ocis /ocis/bin/ocis kill reva-users +docker-compose exec ocis /ocis/bin/ocis run reva-users +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home +docker-compose exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home-data +docker-compose exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data diff --git a/tests/spawn_eos.sh b/tests/spawn_eos.sh index 146011923fb..7ffc3e043cd 100755 --- a/tests/spawn_eos.sh +++ b/tests/spawn_eos.sh @@ -25,11 +25,14 @@ ssh -tt root@$IPADDR git clone https://github.com/owncloud/ocis.git /ocis ssh -tt root@$IPADDR "cd /ocis && git checkout $DRONE_COMMIT_ID" # Create necessary files -ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" -ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" +# ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" +# ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" + +ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" +ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" # run ocis with eos -ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose -f ./docker-compose-eos-ci.yml up -d" +ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose up -d" # Some necessary configuration for eos ssh -tt root@$IPADDR "cd /ocis && bash /ocis/tests/config/drone/setup-eos.sh" From 954be2836a2281a3008ca2283981360db9dd1666 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Tue, 8 Sep 2020 10:32:35 +0545 Subject: [PATCH 08/13] remove duplicate --- tests/spawn_eos.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/spawn_eos.sh b/tests/spawn_eos.sh index 7ffc3e043cd..d69b8668b41 100755 --- a/tests/spawn_eos.sh +++ b/tests/spawn_eos.sh @@ -28,7 +28,6 @@ ssh -tt root@$IPADDR "cd /ocis && git checkout $DRONE_COMMIT_ID" # ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" # ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" -ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" # run ocis with eos From 7e01ddf83052cf234dcaa0b98ba1fe9d5b92a372 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Tue, 8 Sep 2020 11:50:04 +0545 Subject: [PATCH 09/13] Fix config files --- tests/config/drone/setup-eos.sh | 16 ++++++++-------- tests/spawn_eos.sh | 8 +++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/config/drone/setup-eos.sh b/tests/config/drone/setup-eos.sh index 37159cea07e..4c46dda613a 100755 --- a/tests/config/drone/setup-eos.sh +++ b/tests/config/drone/setup-eos.sh @@ -7,17 +7,17 @@ do sleep 5 done -docker-compose exec -d ocis /start-ldap +docker-compose -f ./docker-compose-eos-ci.yml exec -d ocis /start-ldap # time for ldap service to starup within ocis container sleep 5 # Configure ocis -docker-compose exec ocis id einstein -docker-compose exec ocis /ocis/bin/ocis kill reva-users -docker-compose exec ocis /ocis/bin/ocis run reva-users -docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home -docker-compose exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home -docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home-data -docker-compose exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data +docker-compose -f ./docker-compose-eos-ci.yml exec ocis id einstein +docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-users +docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis run reva-users +docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-storage-home +docker-compose -f ./docker-compose-eos-ci.yml exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home +docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-storage-home-data +docker-compose -f ./docker-compose-eos-ci.yml exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data diff --git a/tests/spawn_eos.sh b/tests/spawn_eos.sh index d69b8668b41..146011923fb 100755 --- a/tests/spawn_eos.sh +++ b/tests/spawn_eos.sh @@ -25,13 +25,11 @@ ssh -tt root@$IPADDR git clone https://github.com/owncloud/ocis.git /ocis ssh -tt root@$IPADDR "cd /ocis && git checkout $DRONE_COMMIT_ID" # Create necessary files -# ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" -# ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" - -ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" +ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" +ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" # run ocis with eos -ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose up -d" +ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose -f ./docker-compose-eos-ci.yml up -d" # Some necessary configuration for eos ssh -tt root@$IPADDR "cd /ocis && bash /ocis/tests/config/drone/setup-eos.sh" From 8a8748699f5c679482111d7ef4bbbeca6dd0912d Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Thu, 10 Sep 2020 15:48:24 +0545 Subject: [PATCH 10/13] remove unused compose setup --- docker-compose-eos-ci.yml | 179 -------------------------------- docker-compose.yml | 4 + tests/config/drone/setup-eos.sh | 16 +-- tests/spawn_eos.sh | 7 +- 4 files changed, 16 insertions(+), 190 deletions(-) delete mode 100644 docker-compose-eos-ci.yml diff --git a/docker-compose-eos-ci.yml b/docker-compose-eos-ci.yml deleted file mode 100644 index ec16643acd1..00000000000 --- a/docker-compose-eos-ci.yml +++ /dev/null @@ -1,179 +0,0 @@ ---- -version: '3.5' - -networks: - testnet: - name: testnet - -services: - ocis: - container_name: ocis - image: owncloud/eos-ocis-dev:latest - tty: true - privileged: true - stdin_open: true - ports: - - 9200:9200 - env_file: - - ./config/eos-docker.env - hostname: ocis - networks: - - testnet - volumes: - - .:/ocis - - ../ocis-reva:/ocis-reva - - ../reva:/reva - - ./tests/config/drone:/eos-config - environment: - # ocis log level will be used for all services - OCIS_LOG_LEVEL: debug - # domain setup - # TODO currently the below lines hardcode the port to 9200, use an OCIS_URL that includes protocol and port - OCIS_DOMAIN: ${OCIS_DOMAIN:-localhost} - PROXY_OIDC_ISSUER: https://${OCIS_DOMAIN:-localhost}:9200 - KONNECTD_ISS: https://${OCIS_DOMAIN:-localhost}:9200 - PHOENIX_OIDC_AUTHORITY: https://${OCIS_DOMAIN:-localhost}:9200 - PHOENIX_OIDC_METADATA_URL: https://${OCIS_DOMAIN:-localhost}:9200/.well-known/openid-configuration - PHOENIX_WEB_CONFIG_SERVER: https://${OCIS_DOMAIN:-localhost}:9200 - REVA_OIDC_ISSUER: https://${OCIS_DOMAIN:-localhost}:9200 - REVA_LDAP_IDP: https://${OCIS_DOMAIN:-localhost}:9200 - # TODO make id the default in ocis-reva - REVA_STORAGE_EOS_LAYOUT: "{{substr 0 1 .Id.OpaqueId}}/{{.Id.OpaqueId}}" - REVA_FRONTEND_URL: https://${OCIS_DOMAIN:-localhost}:9200 - REVA_DATAGATEWAY_URL: https://${OCIS_DOMAIN:-localhost}:9200/data - # common eos settings used for both drivers: eos and eoshome - REVA_STORAGE_EOS_MASTER_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094} - REVA_STORAGE_EOS_SLAVE_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094} - REVA_STORAGE_EOS_NAMESPACE: "/eos/dockertest/reva/users" - # the eos end xrdcopy binaries use this env var to find the eos mgm - EOS_MGM_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094} - PHOENIX_WEB_CONFIG: /eos-config/config.json - KONNECTD_IDENTIFIER_REGISTRATION_CONF: /eos-config/identifier-registration.yaml - - mgm-master: - container_name: mgm-master - image: owncloud/eos-mgm:4.6.5 - tty: true - privileged: true - stdin_open: true - env_file: - - ./config/eos-docker.env - hostname: mgm-master.testnet - networks: - - testnet - volumes: - - eos-mgm-master-log:/var/log/eos - - eos-mgm-master-config:/var/eos/config - - eos-mgm-master-ns-queue:/var/eos/ns-queue - # this volume kills mgm-master during startup - # - ./e/master/var/eos/md:/var/eos/md - environment: - EOS_SET_MASTER: 1 - - mq-master: - container_name: mq-master - image: owncloud/eos-mq:4.6.5 - tty: true - privileged: true - stdin_open: true - env_file: - - ./config/eos-docker.env - hostname: mq-master.testnet - networks: - - testnet - volumes: - - eos-mq-master-log:/var/log/eos - - eos-mq-master-config:/var/eos/config - - eos-mq-master-ns-queue:/var/eos/ns-queue - environment: - EOS_SET_MASTER: 1 - - fst: - container_name: fst - image: owncloud/eos-fst:4.6.5 - tty: true - privileged: true - stdin_open: true - env_file: - - ./config/eos-docker.env - hostname: fst.testnet - networks: - - testnet - volumes: - - eos-fst-log:/var/log/eos - - eos-fst-disks:/disks - environment: - EOS_MGM_URL: "root://mgm-master.testnet" - - quark-1: - container_name: quark-1 - image: owncloud/eos-qdb:4.6.5 - tty: true - privileged: true - stdin_open: true - env_file: - - ./config/eos-docker.env - hostname: quark-1.testnet - networks: - - testnet - volumes: - - eos-quarkdb1:/var/lib/quarkdb - environment: - EOS_QDB_DIR: "/var/lib/quarkdb/eosns" - EOS_QDB_PORT: "7777" - EOS_QDB_MODE: "raft" - EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b" - EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777" - - quark-2: - container_name: quark-2 - image: owncloud/eos-qdb:4.6.5 - tty: true - privileged: true - stdin_open: true - env_file: - - ./config/eos-docker.env - hostname: quark-2.testnet - networks: - - testnet - volumes: - - eos-quarkdb2:/var/lib/quarkdb - environment: - EOS_QDB_DIR: "/var/lib/quarkdb/eosns" - EOS_QDB_PORT: "7777" - EOS_QDB_MODE: "raft" - EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b" - EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777" - - quark-3: - container_name: quark-3 - image: owncloud/eos-qdb:4.6.5 - tty: true - privileged: true - stdin_open: true - env_file: - - ./config/eos-docker.env - hostname: quark-3.testnet - networks: - - testnet - volumes: - - eos-quarkdb3:/var/lib/quarkdb - environment: - EOS_QDB_DIR: "/var/lib/quarkdb/eosns" - EOS_QDB_PORT: "7777" - EOS_QDB_MODE: "raft" - EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b" - EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777" - -volumes: - eos-mgm-master-log: - eos-mgm-master-config: - eos-mgm-master-ns-queue: - eos-mq-master-log: - eos-mq-master-config: - eos-mq-master-ns-queue: - eos-fst-log: - eos-fst-disks: - eos-quarkdb1: - eos-quarkdb2: - eos-quarkdb3: diff --git a/docker-compose.yml b/docker-compose.yml index 0d8023b790a..b49269bdbf0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ services: - .:/ocis - ../ocis-reva:/ocis-reva - ../reva:/reva + - ./config:/ocis-config environment: # ocis log level will be used for all services OCIS_LOG_LEVEL: debug @@ -46,6 +47,9 @@ services: REVA_STORAGE_EOS_NAMESPACE: "/eos/dockertest/reva/users" # the eos end xrdcopy binaries use this env var to find the eos mgm EOS_MGM_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094} + # uses identifier registration from config directory + KONNECTD_IDENTIFIER_REGISTRATION_CONF: /ocis-config/identifier-registration.yaml + mgm-master: container_name: mgm-master diff --git a/tests/config/drone/setup-eos.sh b/tests/config/drone/setup-eos.sh index 4c46dda613a..37159cea07e 100755 --- a/tests/config/drone/setup-eos.sh +++ b/tests/config/drone/setup-eos.sh @@ -7,17 +7,17 @@ do sleep 5 done -docker-compose -f ./docker-compose-eos-ci.yml exec -d ocis /start-ldap +docker-compose exec -d ocis /start-ldap # time for ldap service to starup within ocis container sleep 5 # Configure ocis -docker-compose -f ./docker-compose-eos-ci.yml exec ocis id einstein -docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-users -docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis run reva-users -docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-storage-home -docker-compose -f ./docker-compose-eos-ci.yml exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home -docker-compose -f ./docker-compose-eos-ci.yml exec ocis /ocis/bin/ocis kill reva-storage-home-data -docker-compose -f ./docker-compose-eos-ci.yml exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data +docker-compose exec ocis id einstein +docker-compose exec ocis /ocis/bin/ocis kill reva-users +docker-compose exec ocis /ocis/bin/ocis run reva-users +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home +docker-compose exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home-data +docker-compose exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data diff --git a/tests/spawn_eos.sh b/tests/spawn_eos.sh index 146011923fb..1cdb689abd4 100755 --- a/tests/spawn_eos.sh +++ b/tests/spawn_eos.sh @@ -25,11 +25,12 @@ ssh -tt root@$IPADDR git clone https://github.com/owncloud/ocis.git /ocis ssh -tt root@$IPADDR "cd /ocis && git checkout $DRONE_COMMIT_ID" # Create necessary files -ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" -ssh -tt root@$IPADDR "cd /ocis/tests/config/drone && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" +ssh -tt root@$IPADDR "mkdir -p /ocis/tests/eos-config" +ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-config.json.sh" +ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/create-identifier-registration.sh" # run ocis with eos -ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose -f ./docker-compose-eos-ci.yml up -d" +ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose up -d" # Some necessary configuration for eos ssh -tt root@$IPADDR "cd /ocis && bash /ocis/tests/config/drone/setup-eos.sh" From 73941e0dea6e6cafbb5f4c8596f9007ae594f65e Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Mon, 14 Sep 2020 09:02:02 +0545 Subject: [PATCH 11/13] update core commit id --- .drone.star | 6 +++--- start-eos.sh | 16 ++++++++++++++++ tests/config/drone/setup-eos.sh | 15 +++++++++++++++ tests/spawn_eos.sh | 2 +- 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 start-eos.sh diff --git a/.drone.star b/.drone.star index d68ff626897..9e1f4ff5a60 100644 --- a/.drone.star +++ b/.drone.star @@ -1,8 +1,8 @@ config = { 'apiTests': { - 'coreBranch': 'master', - 'coreCommit': 'e9850b40657ff78f32cb5585ec00342fe07a5ff2', - 'numberOfParts': 4 + 'coreBranch': 'uid-gid-user-create', + 'coreCommit': 'c731fc3ba0858811aef6d3b56002f67b6f9f42f8', + 'numberOfParts': 1 }, 'uiTests': { 'phoenixBranch': 'master', diff --git a/start-eos.sh b/start-eos.sh new file mode 100644 index 00000000000..a70d5af546a --- /dev/null +++ b/start-eos.sh @@ -0,0 +1,16 @@ +#!/bin/bash +docker-compose exec -d ocis /start-ldap + +# time for ldap service to starup within ocis container +sleep 5 + +# Configure ocis +docker-compose exec ocis id einstein +docker-compose exec ocis /ocis/bin/ocis kill reva-users +docker-compose exec ocis /ocis/bin/ocis run reva-users +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home +docker-compose exec -e REVA_STORAGE_HOME_DRIVER=eoshome -e REVA_STORAGE_HOME_MOUNT_ID=1284d238-aa92-42ce-bdc4-0b0000009158 ocis ./bin/ocis run reva-storage-home +docker-compose exec ocis /ocis/bin/ocis kill reva-storage-home-data +docker-compose exec -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome ocis ./bin/ocis run reva-storage-home-data + + diff --git a/tests/config/drone/setup-eos.sh b/tests/config/drone/setup-eos.sh index 37159cea07e..fccff43ac7d 100755 --- a/tests/config/drone/setup-eos.sh +++ b/tests/config/drone/setup-eos.sh @@ -7,6 +7,21 @@ do sleep 5 done +docker-compose down + +echo $OCIS_DOMAIN + +OCIS_DOMAIN=${OCIS_DOMAIN} docker-compose up -d + +# wait for ocis to start +until $(curl -k --output /dev/null --silent --head --fail https://localhost:9200); +do + echo '.' + sleep 5 +done + +sleep 5 + docker-compose exec -d ocis /start-ldap # time for ldap service to starup within ocis container diff --git a/tests/spawn_eos.sh b/tests/spawn_eos.sh index 1cdb689abd4..8cf719c61a0 100755 --- a/tests/spawn_eos.sh +++ b/tests/spawn_eos.sh @@ -33,4 +33,4 @@ ssh -tt root@$IPADDR "cd /ocis/config && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/ ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} docker-compose up -d" # Some necessary configuration for eos -ssh -tt root@$IPADDR "cd /ocis && bash /ocis/tests/config/drone/setup-eos.sh" +ssh -tt root@$IPADDR "cd /ocis && OCIS_DOMAIN=${IPADDR} bash /ocis/tests/config/drone/setup-eos.sh" From 4f0c5f2ca720765e5478baa1115eac890459a3a2 Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Mon, 14 Sep 2020 11:00:16 +0545 Subject: [PATCH 12/13] 10 parts --- .drone.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 9e1f4ff5a60..49c293532a2 100644 --- a/.drone.star +++ b/.drone.star @@ -2,7 +2,7 @@ config = { 'apiTests': { 'coreBranch': 'uid-gid-user-create', 'coreCommit': 'c731fc3ba0858811aef6d3b56002f67b6f9f42f8', - 'numberOfParts': 1 + 'numberOfParts': 10 }, 'uiTests': { 'phoenixBranch': 'master', From c9349dbef5b08371ec07f1f65e6fe048f6379a7f Mon Sep 17 00:00:00 2001 From: Dipak Acharya Date: Mon, 14 Sep 2020 11:12:48 +0545 Subject: [PATCH 13/13] refactor cleanup in drone.star --- .drone.star | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/.drone.star b/.drone.star index 49c293532a2..5fb6023e4cf 100644 --- a/.drone.star +++ b/.drone.star @@ -2,7 +2,8 @@ config = { 'apiTests': { 'coreBranch': 'uid-gid-user-create', 'coreCommit': 'c731fc3ba0858811aef6d3b56002f67b6f9f42f8', - 'numberOfParts': 10 + 'numberOfParts': 4, + 'numberOfPartsEos': 10, }, 'uiTests': { 'phoenixBranch': 'master', @@ -95,8 +96,10 @@ def testPipelines(ctx): pipelines.append(coreApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfParts'], 'owncloud')) pipelines.append(coreApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfParts'], 'ocis')) - for runPart in range(1, config['apiTests']['numberOfParts'] + 1): - pipelines.append(coreApiTestsEosStorage(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfParts'])) + for runPart in range(1, config['apiTests']['numberOfPartsEos'] + 1): + pipelines.append(coreApiTestsEosStorage(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], runPart, config['apiTests']['numberOfPartsEos'])) + + pipelines.append(eosCleanup(ctx, config['apiTests']['numberOfPartsEos'])) pipelines += uiTests(ctx, config['uiTests']['phoenixBranch'], config['uiTests']['phoenixCommit']) return pipelines @@ -408,6 +411,66 @@ def coreApiTestsEosStorage(ctx, coreBranch = 'master', coreCommit = '', part_num }, } +def eosCleanup(ctx, parts): + depends_on = [] + steps = [] + for parts_number in range(1, parts + 1): + depends_on.append('coreApiTests-Eos-Storage-%s' % (parts_number)) + steps.append('hcloud server delete droneci-eos-test-%s-%s' % (ctx.build.commit, parts_number)) + steps.append('hcloud ssh-key delete droneci-eos-test-%s-%s' % (ctx.build.commit, parts_number)) + + return { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'eos-cleanup', + 'platform': { + 'os': 'linux', + 'arch': 'amd64', + }, + 'steps': [ + { + 'name': 'eos-cleanup', + 'image': 'owncloudci/php:7.2', + 'pull': 'always', + 'environment' : { + 'HCLOUD_TOKEN': { + 'from_secret': 'hcloud_token', + }, + 'DRONE_HCLOUD_USER': 'dipak', + }, + 'commands': [ + # Install Go + 'wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz', + 'mkdir -p /usr/local/bin', + 'tar xf go1.14.2.linux-amd64.tar.gz -C /usr/local', + 'ln -s /usr/local/go/bin/* /usr/local/bin', + + # Install Hcloud Cli + 'go get -u github.com/hetznercloud/cli/cmd/hcloud', + 'ln -s /root/go/bin/* /usr/local/bin', + ] + steps, + 'volumes': [{ + 'name': 'gopath', + 'path': '/srv/app', + }] + }, + ], + 'volumes': [ + { + 'name': 'gopath', + 'temp': {}, + }, + ], + 'depends_on': depends_on, + 'trigger': { + 'ref': [ + 'refs/heads/master', + 'refs/tags/**', + 'refs/pull/**', + ], + }, + } + def uiTests(ctx, phoenixBranch, phoenixCommit): suiteNames = getUITestSuiteNames() return [uiTestPipeline(suiteName, phoenixBranch, phoenixCommit) for suiteName in suiteNames]