[WIP] acceptance tests with EOS

examples/eos-compose-acceptance-tests/.gitignore

@@ -0,0 +1,5 @@
+/eos-mgm.iml
+/.idea
+e/
+*.swp
+.env

examples/eos-compose-acceptance-tests/README.md

@@ -0,0 +1,75 @@
+# EOS for oCIS
+
+# Setup
+
+Setup on hcloud cx21 and install some dependencies (imho needs cx21 to have enough ram for building ocis)
+
+First check for correct hcloud project: `hcloud context list'
+
+```
+# set your name here for labels and ssh-key setup
+ME=$(whoami) 
+SERVER_NAME=eos-ocis-test
+
+# create server
+hcloud server create --type cx21 --image ubuntu-20.04 --ssh-key $ME --name $SERVER_NAME --label owner=$ME --label for=test --label from=eos-compose
+
+IPADDR=$(hcloud server ip $SERVER_NAME)
+
+ssh -t root@$IPADDR apt-get update -y
+ssh -t root@$IPADDR apt-get install -y git screen docker.io docker-compose ldap-utils
+ssh -t root@$IPADDR git clone https://github.com/owncloud-docker/compose-playground.git
+ssh -t root@$IPADDR "cd compose-playground/examples/eos-compose-acceptance-tests && ./build"
+```
+
+Anytime:
+- Check IP `hcloud server ip $SERVER_NAME`
+- Access `ssh root@IPADDR`
+
+# Run
+
+1. Access the hcloud machine or run localy ...
+2. Set your domain or IP in .env 
+3. Start via docker compose 
+
+```
+echo "OCIS_DOMAIN=localhost" > .env
+docker-compose up -d
+```
+
+# Run tests
+```
+make test-acceptance-api \
+TEST_SERVER_URL=https://localhost:9200 \
+TEST_EXTERNAL_USER_BACKENDS=true \
+TEST_OCIS=true \
+BEHAT_FILTER_TAGS='~@skipOnOcis&&~@skipOnLDAP&&@TestAlsoOnExternalUserBackend&&~@local_storage' \
+SKELETON_DIR=apps/testing/data/apiSkeleton \
+BEHAT_FEATURE='<feature>'
+```
+
+# More commands
+
+add users for manual testing
+```
+ldapadd -x -D "cn=admin,dc=owncloud,dc=com" -w admin -H ldap://localhost -f ./config/example-ldap-users-groups.ldif
+```
+
+In the mgm-master or ocis container you can access eos cli
+
+```
+eos vid ls
+eos whoami
+eos -r 0 0 whoami
+eos -r 2 2 whoami
+eos -r 20000 30000 whoami
+
+eos -r 0 0 ls -la /eos/dockertest/reva/users
+```
+
+Also see file `check` for more system checks.
+
+# Limitations
+
+- in LDAP the `cn` field has to match the `uid` field
+- the display name for the user is taken from the `sn` field in LDAP

examples/eos-compose-acceptance-tests/build

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+docker build -t eos/base ./eos-base
+docker build -t eos/qdb ./eos-qdb
+docker build -t eos/mgm ./eos-mgm
+docker build -t eos/mq ./eos-mq
+docker build -t eos/fst ./eos-fst
+docker build -t eos/ocis:master -t eos/ocis:latest ./eos-ocis
+
+# docker build -t eos/ocis:1.0.0-beta4 --build-arg BRANCH=v1.0.0-beta4 ./eos-ocis
+# docker build -t eos/eosd ./eos-eosd
+# docker build -t eos/eosxd ./eos-eosxd

examples/eos-compose-acceptance-tests/check

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+echo "===== [mgm] Expect to see running quarkdb: ====="
+docker-compose exec mgm-master sh -c 'redis-cli -h quark-1.testnet -p 7777 -a "$(cat /etc/eos.client.keytab)" raft-info'
+docker-compose exec mgm-master sh -c 'redis-cli -h quark-2.testnet -p 7777 -a "$(cat /etc/eos.client.keytab)" raft-info'
+docker-compose exec mgm-master sh -c 'redis-cli -h quark-3.testnet -p 7777 -a "$(cat /etc/eos.client.keytab)" raft-info'
+docker-compose exec mgm-master sh -c 'redis-cli -h quark-3.testnet -p 7777 -a "$(cat /etc/eos.client.keytab)" quarkdb-health'
+
+echo "===== [mgm] Expect to see *sudo and effective uid 0 here: ====="
+docker-compose exec mgm-master eos whoami
+docker-compose exec mgm-master eos ns
+docker-compose exec mgm-master eos node ls
+
+echo; echo "===== [mgm] Expect to see vid: ====="
+docker-compose exec mgm-master eos -r 0 0 -b vid ls
+
+echo; echo "===== [ocis] Expect to see *sudo and effective uid 0 here: ====="
+docker-compose exec ocis eos whoami
+docker-compose exec ocis eos -r 0 0 whoami
+
+echo; echo "===== [qdb] Print QDB infos ====="
+for q in "quark-1" "quark-2" "quark-3"; do
+  echo; echo "===== ${q} ====="; echo;
+  docker-compose exec ${q} redis-cli -p 7777 raft-info
+done
+
+echo; echo "===== [mgm] Check auth and id ====="
+docker-compose exec mgm-master authconfig --test | grep ena 	# expect: nss_ldap is enabled
+docker-compose exec mgm-master id marie			# expect: uid=20001(marie) gid=30000(users) ...
+
+echo; echo "===== [ocis] Check auth and id ====="
+docker-compose exec ocis authconfig --test | grep ena 	# expect: nss_ldap is enabled
+docker-compose exec ocis id marie			# expect: uid=20001(marie) gid=30000(users) ...
+
+echo; echo "===== RUNNING SOME MORE CHECKS ====="
+docker-compose exec mgm-master eos group set default.0 on
+docker-compose exec mgm-master eos space set default on
+docker-compose exec mgm-master eos ls -l eos
+docker-compose exec mgm-master eos fs ls
+docker-compose exec mgm-master eos space ls
+docker-compose exec mgm-master eos group ls
+
+# echo; echo "===== TESTING EOSD & EOSXD MOUNTS =====";
+# sudo docker exec mgm-master eos -r 0 0 -b attr set sys.acl="u:2:rwxc" /eos/test
+
+# for dir in "eos" "eosxd"; do
+#   testfile="e/${dir}/test/hello.txt"
+
+#   echo; echo "${dir}: Writing a test file.."
+#   sudo echo "  hello, world!" > ${testfile}
+#   echo; echo "Reading the test file.."
+#   sudo cat ${testfile}
+# done
+
+
+
+# fst           | ls: cannot access /hostdev/mapper/: No such file or directory
+# fst           | ls: cannot access /dev/mapper/luks*: No such file or directory
+# fst           | ls: cannot access /dev/sd*: No such file or directory
+# fst           | ls: cannot access /dev/mapper/*: No such file or directory
+# fst           | ls: cannot access /dev/sd*: No such file or directory
+# fst           | ls: cannot access /dev/mapper/*: No such file or directory
+# # mgm-master    | chown: cannot access '/tmp/eos.mgm': No such file or directory
+# ocis          | 2020-05-22T17:41:14Z WRN policy-selector not configured. Will always use first policy: 'reva' service=proxy
+
+# quark-2       | Plugin No such file or directory loading protocol /usr/lib64/libXrdQuarkDB-4.so
+# quark-2       | Config Falling back to using /usr/lib64/libXrdQuarkDB.so
+# quark-2       | Plugin Unable to find  plugin XrdgetProtocolPort in protocol /usr/lib64/libXrdQuarkDB.so
+# quark-2       | Config Unable to load protocol plugin /usr/lib64/libXrdQuarkDB.so
+# quark-2       | Config warning: only '/tmp' will be exported.
+# quark-2       | Config warning: 'xrootd.seclib' not specified; strong authentication disabled!
+# quark-2       | Config warning: 'xrootd.prepare logdir' not specified; prepare tracking disabled.
+# quark-2       | ------ xrootd protocol initialization completed.
+# quark-2       | Plugin No such file or directory loading protocol /usr/lib64/libXrdQuarkDB-4.so
+# quark-2       | Config Falling back to using /usr/lib64/libXrdQuarkDB.so
+# # mgm-master    | Couldn't send data to graphite host: [Errno 111] Connection refused
+# #   eos_graphite.py has hardcoded ip:port 
+
+# quark-2       | 200522 17:41:07 001 XrdConfig: Unable to create home directory //quarkdb; permission denied
+# quark-2       | ------ xrootd [email protected]:7777 initialization completed.
+# ocis          | 2020-05-22T17:41:15Z WRN No tls certificate provided, using a generated one service=proxy
+# ocis          | 2020-05-22T17:41:15Z ERR error starting the grpc server error="unable to register services: rgrpc: grpc service usershareprovider could not be started,: error loading the file containing the shares: error opening/creating the file: /var/tmp/reva/shares.json: open /var/tmp/reva/shares.json: no such file or directory" service=reva
+# ocis          | 2020-05-22T17:41:16Z WRN missing --encryption-secret parameter, using random encyption secret with 32 bytes service=konnectd
+# ocis          | 2020-05-22T17:41:16Z WRN missing --signing-private-key parameter, using random 2048 bit signing key alg=PS256 service=konnectd
+
+

examples/eos-compose-acceptance-tests/config/eos-docker.env

@@ -0,0 +1,12 @@
+EOS_MQ_URL=mq-master.testnet
+EOS_MGM_ALIAS=mgm-master.testnet
+EOS_QDB_NODES=quark-1.testnet:7777 quark-2.testnet:7777 quark-3.testnet:7777
+EOS_LDAP_HOST=ldap.testnet:389
+EOS_GEOTAG=test
+EOS_INSTANCE_NAME=eostest
+EOS_MAIL_CC=eos@localhost
+EOS_USE_QDB=1
+EOS_USE_QDB_MASTER=1
+EOS_NS_ACCOUNTING=1
+EOS_SYNCTIME_ACCOUNTING=1
+EOS_UTF8=1

examples/eos-compose-acceptance-tests/config/example-ldap-users-groups.ldif

@@ -0,0 +1,129 @@
+# This LDIF file contains examples of groups and users that could be created
+# in LDAP when testing. It is not used by the automated tests. The automated
+# tests create LDAP groups and users on-the-fly.
+#
+# Examples here might be useful if you are manually setting up some LDAP entries
+# for local development and testing.
+dn: ou=TestGroups,dc=owncloud,dc=com
+objectclass: top
+objectclass: organizationalUnit
+ou: TestGroups
+
+dn: cn=generalgroup,ou=TestGroups,dc=owncloud,dc=com
+cn: generalgroup
+gidnumber: 5000
+objectclass: top
+objectclass: posixGroup
+
+dn: cn=grp1,ou=TestGroups,dc=owncloud,dc=com
+cn: grp1
+gidnumber: 5001
+memberuid: user1
+memberuid: user2
+objectclass: top
+objectclass: posixGroup
+
+dn: cn=grp2,ou=TestGroups,dc=owncloud,dc=com
+cn: grp2
+gidnumber: 5002
+memberuid: user3
+objectclass: top
+objectclass: posixGroup
+
+dn: cn=grp3,ou=TestGroups,dc=owncloud,dc=com
+cn: grp3
+gidnumber: 5003
+objectclass: top
+objectclass: posixGroup
+
+dn: cn=grp4,ou=TestGroups,dc=owncloud,dc=com
+cn: grp4
+memberuid: user1
+gidnumber: 5004
+objectclass: top
+objectclass: posixGroup
+
+dn: ou=TestUsers,dc=owncloud,dc=com
+objectclass: top
+objectclass: organizationalUnit
+ou: TestUsers
+
+dn: uid=user0,ou=TestUsers,dc=owncloud,dc=com
+cn: user0
+sn: Zero
+displayname: User Zero
+gecos: User0
+gidnumber: 5000
+givenname: User0
+homedirectory: /home/openldap/user0
+loginshell: /bin/bash
+mail: [email protected]
+objectclass: posixAccount
+objectclass: inetOrgPerson
+uid: user0
+uidnumber: 30000
+userpassword: 123456
+
+dn: uid=user1,ou=TestUsers,dc=owncloud,dc=com
+cn: user1
+sn: One
+displayname: User One
+gecos: User1
+gidnumber: 5000
+givenname: User1
+homedirectory: /home/openldap/user1
+loginshell: /bin/bash
+mail: [email protected]
+objectclass: posixAccount
+objectclass: inetOrgPerson
+uid: user1
+uidnumber: 30001
+userpassword: 1234
+
+dn: uid=user2,ou=TestUsers,dc=owncloud,dc=com
+cn: user2
+sn: Two
+displayname: User Two
+gecos: User2
+gidnumber: 5000
+givenname: User2
+homedirectory: /home/openldap/user2
+loginshell: /bin/bash
+mail: [email protected]
+objectclass: posixAccount
+objectclass: inetOrgPerson
+uid: user2
+uidnumber: 30002
+userpassword: AaBb2Cc3Dd4
+
+dn: uid=user3,ou=TestUsers,dc=owncloud,dc=com
+cn: user3
+sn: Three
+displayname: User Three
+gecos: User3
+gidnumber: 5000
+givenname: User3
+homedirectory: /home/openldap/user3
+loginshell: /bin/bash
+mail: [email protected]
+objectclass: posixAccount
+objectclass: inetOrgPerson
+uid: user3
+uidnumber: 30003
+userpassword: aVeryLongPassword42TheMeaningOfLife
+
+dn: uid=user4,ou=TestUsers,dc=owncloud,dc=com
+cn: user4
+sn: Four
+displayname: User Four
+gecos: User4
+gidnumber: 5000
+givenname: User4
+homedirectory: /home/openldap/user4
+loginshell: /bin/bash
+mail: [email protected]
+objectclass: posixAccount
+objectclass: inetOrgPerson
+uid: user4
+uidnumber: 30004
+userpassword: ThisIsThe4thAlternatePwd