From f6ae9ca062e023d093be3d795744827fb723a1e2 Mon Sep 17 00:00:00 2001 From: saw-jan Date: Fri, 19 Nov 2021 12:59:30 +0545 Subject: [PATCH] [POC] Tests for parallel deployment add owncloud-selector cookie working tests remove unused files build ocis from source branch check users in ldap, oc10 and ocis update proxy config use ocis selector initially build ocis from source use ocis latest image buil ocis from source refactor fix test run script user 33 and alpine image for ocis server delete local compose files run tests with ocis latest refactor drone config [volumes] Run each suite in a pipeline refactor namings use run.sh to run tests move TestHelpers folder reuse behat context params refactor to use contexts and helpers from core repo remove contexts and helpers added bootstrap file to bootstrap core's test files add api tag remove unnecessary code implement expected-failures file remove unnecessary confgs fix starlark style rename context file add nightly trigger refactor drone config generate trigger as per the event remove keycloak services --- .drone.env | 2 +- .drone.star | 477 +++++++++++++++++- Makefile | 11 +- tests/parallelDeployAcceptance/.gitignore | 2 + .../parallelDeployAcceptance/config/behat.yml | 40 ++ .../config/ldap_users_groups.ldif | 7 + .../drone/ldap/ldif/10_owncloud_schema.ldif | 10 + .../drone/oc10/10-custom-config.sh | 33 ++ .../drone/oc10/ldap-config.tmpl.json | 53 ++ .../drone/oc10/oidc.config.php | 22 + .../drone/ocis/proxy.json | 109 ++++ .../expected-failures-API.md | 5 + .../apiShareManagement/acceptShares.feature | 30 ++ .../apiWebdavOperations/downloadFile.feature | 146 ++++++ .../features/bootstrap/ParallelContext.php | 71 +++ .../features/bootstrap/bootstrap.php | 37 ++ 16 files changed, 1051 insertions(+), 4 deletions(-) create mode 100644 tests/parallelDeployAcceptance/.gitignore create mode 100644 tests/parallelDeployAcceptance/config/behat.yml create mode 100644 tests/parallelDeployAcceptance/config/ldap_users_groups.ldif create mode 100644 tests/parallelDeployAcceptance/drone/ldap/ldif/10_owncloud_schema.ldif create mode 100755 tests/parallelDeployAcceptance/drone/oc10/10-custom-config.sh create mode 100755 tests/parallelDeployAcceptance/drone/oc10/ldap-config.tmpl.json create mode 100644 tests/parallelDeployAcceptance/drone/oc10/oidc.config.php create mode 100644 tests/parallelDeployAcceptance/drone/ocis/proxy.json create mode 100644 tests/parallelDeployAcceptance/expected-failures-API.md create mode 100644 tests/parallelDeployAcceptance/features/apiShareManagement/acceptShares.feature create mode 100644 tests/parallelDeployAcceptance/features/apiWebdavOperations/downloadFile.feature create mode 100644 tests/parallelDeployAcceptance/features/bootstrap/ParallelContext.php create mode 100644 tests/parallelDeployAcceptance/features/bootstrap/bootstrap.php diff --git a/.drone.env b/.drone.env index 484f572405c..4b99674b298 100644 --- a/.drone.env +++ b/.drone.env @@ -1,5 +1,5 @@ # The test runner source for API tests -CORE_COMMITID=38ee68e6358443e980ba5e7036cf1fb554443814 +CORE_COMMITID=40923975aac8eb17663f8a222e7a2b9c3a1a6948 CORE_BRANCH=acceptance-test-changes-waiting-2021-11 # The test runner source for UI tests diff --git a/.drone.star b/.drone.star index 7d03e5571ad..ba09622e435 100644 --- a/.drone.star +++ b/.drone.star @@ -8,6 +8,7 @@ OC_CI_NODEJS = "owncloudci/nodejs:14" OC_CI_PHP = "owncloudci/php:7.4" OC_CI_WAIT_FOR = "owncloudci/wait-for:latest" MINIO_MC = "minio/mc:RELEASE.2021-10-07T04-19-58Z" +REDIS = "redis:6-alpine" # configuration config = { @@ -53,6 +54,24 @@ config = { "skip": False, "earlyFail": True, }, + "parallelApiTests": { + "apiSharing": { + "suites": [ + "apiShareManagement", + ], + "skip": False, + "earlyFail": True, + "cron": "nightly", + }, + "apiWebdav": { + "suites": [ + "apiWebdavOperations", + ], + "skip": False, + "earlyFail": True, + "cron": "nightly", + }, + }, "rocketchat": { "channel": "ocis-internal", "from_secret": "private_rocketchat", @@ -232,6 +251,9 @@ def testPipelines(ctx): if "skip" not in config["settingsUITests"] or not config["settingsUITests"]["skip"]: pipelines.append(settingsUITests(ctx)) + if "parallelApiTests" in config: + pipelines += parallelDeployAcceptancePipeline(ctx) + return pipelines def testOcisModule(ctx, module): @@ -1473,7 +1495,7 @@ def redis(): return [ { "name": "redis", - "image": "redis:6-alpine", + "image": REDIS, }, ] @@ -1837,3 +1859,456 @@ def pipelineSanityChecks(ctx, pipelines): for image in images.keys(): print(" %sx\t%s" % (images[image], image)) + +"""Parallel Deployment configs +""" + +#images +OC_OCIS = "owncloud/ocis:latest" +OC_OC10 = "owncloud/server:10" +OC_UBUNTU = "owncloud/ubuntu:18.04" +MARIADB = "mariadb:10.6" +OSIXIA_OPENLDAP = "osixia/openldap:latest" +QUAY_IO_KEYCLOAK = "quay.io/keycloak/keycloak:latest" +POSTGRES = "postgres:alpine" + +# configs +OCIS_URL = "https://ocis:9200" +OCIS_DOMAIN = "ocis:9200" +OC10_URL = "http://oc10:8080" +PARALLEL_DEPLOY_CONFIG_PATH = "/drone/src/tests/parallelDeployAcceptance/drone" + +# step volumes +stepVolumeOC10Templates = \ + { + "name": "oc10-templates", + "path": "/etc/templates", + } +stepVolumeOC10PreServer = \ + { + "name": "preserver-config", + "path": "/etc/pre_server.d", + } +stepVolumeOC10Apps = \ + { + "name": "core-apps", + "path": "/var/www/owncloud/apps", + } +stepVolumeOC10OCISData = \ + { + "name": "data", + "path": "/mnt/data", + } +stepVolumeOCISConfig = \ + { + "name": "proxy-config", + "path": "/etc/ocis", + } + +# pipeline volumes +pipeOC10TemplatesVol = \ + { + "name": "oc10-templates", + "temp": {}, + } +pipeOC10PreServerVol = \ + { + "name": "preserver-config", + "temp": {}, + } +pipeOC10AppsVol = \ + { + "name": "core-apps", + "temp": {}, + } +pipeOC10OCISSharedVol = \ + { + "name": "data", + "temp": {}, + } +pipeOCISConfigVol = \ + { + "name": "proxy-config", + "temp": {}, + } + +def parallelDeployAcceptancePipeline(ctx): + pipelines = [] + + default = { + "filterTags": "~@skip", + } + + for category, params in config["parallelApiTests"].items(): + if "skip" in params and params["skip"]: + return pipelines + + early_fail = params["earlyFail"] if "earlyFail" in params else False + + if type(params["suites"]) == "list": + suites = {} + for suite in params["suites"]: + suites[suite] = suite + else: + suites = params["suites"] + + for suite, suiteName in suites.items(): + params = {} + for item in default: + params[item] = params[item] if item in params else default[item] + + environment = {} + environment["BEHAT_FILTER_TAGS"] = params["filterTags"] + environment["BEHAT_SUITE"] = suite + + pipeline = { + "kind": "pipeline", + "type": "docker", + "name": "parallel-%s" % (suiteName), + "platform": { + "os": "linux", + "arch": "amd64", + }, + "steps": cloneCoreRepos() + + copyConfigs() + + waitForServices() + + oC10Server() + + owncloudLog() + + fixSharedDataPermissions() + + latestOcisServer() + + parallelAcceptance(environment) + + failEarly(ctx, early_fail), + "services": oc10DbService() + + ldapService() + + redis(), + "volumes": [ + pipeOC10TemplatesVol, + pipeOC10PreServerVol, + pipeOC10AppsVol, + pipeOC10OCISSharedVol, + pipeOCISConfigVol, + pipelineVolumeOC10Tests, + ], + "trigger": {}, + } + + if (ctx.build.event == "cron"): + pipeline["trigger"]["cron"] = params["cron"] if "cron" in params and params["cron"] != "" else "nightly" + else: + pipeline["trigger"]["ref"] = [ + "refs/heads/master", + "refs/tags/v*", + "refs/pull/**", + ] + + pipelines.append(pipeline) + + return pipelines + +def parallelAcceptance(env): + environment = { + "TEST_SERVER_URL": OCIS_URL, + "TEST_OC10_URL": OC10_URL, + "TEST_PARALLEL_DEPLOYMENT": "true", + "TEST_OCIS": "true", + "TEST_WITH_LDAP": "true", + "REVA_LDAP_PORT": 636, + "REVA_LDAP_BASE_DN": "dc=owncloud,dc=com", + "REVA_LDAP_HOSTNAME": "openldap", + "REVA_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com", + "SKELETON_DIR": "/var/www/owncloud/apps/testing/data/apiSkeleton", + "PATH_TO_CORE": "/srv/app/testrunner", + "OCIS_REVA_DATA_ROOT": "/mnt/data/", + "EXPECTED_FAILURES_FILE": "/drone/src/tests/parallelDeployAcceptance/expected-failures-API.md", + "OCIS_SKELETON_STRATEGY": "copy", + "SEND_SCENARIO_LINE_REFERENCES": "true", + "UPLOAD_DELETE_WAIT_TIME": "1", + } + environment.update(env) + + return [{ + "name": "acceptance-tests", + "image": OC_CI_PHP, + "environment": environment, + "commands": [ + "make test-paralleldeployment-api", + ], + "depends_on": ["clone-core-repos", "wait-for-oc10", "wait-for-ocis"], + "volumes": [ + stepVolumeOC10Apps, + stepVolumeOC10Tests, + stepVolumeOC10OCISData, + ], + }] + +def latestOcisServer(): + environment = { + # Keycloak IDP specific configuration + "PROXY_OIDC_ISSUER": "https://keycloak/auth/realmsowncloud", + "WEB_OIDC_AUTHORITY": "https://keycloak/auth/realms/owncloud", + "WEB_OIDC_CLIENT_ID": "ocis-web", + "WEB_OIDC_METADATA_URL": "https://keycloak/auth/realms/owncloud/.well-known/openid-configuration", + "STORAGE_OIDC_ISSUER": "https://keycloak", + "STORAGE_LDAP_IDP": "https://keycloak/auth/realms/owncloud", + "WEB_OIDC_SCOPE": "openid profile email owncloud", + # LDAP bind + "STORAGE_LDAP_HOSTNAME": "openldap", + "STORAGE_LDAP_PORT": 636, + "STORAGE_LDAP_INSECURE": "true", + "STORAGE_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com", + "STORAGE_LDAP_BIND_PASSWORD": "admin", + # LDAP user settings + "PROXY_AUTOPROVISION_ACCOUNTS": "true", # automatically create users when they login + "PROXY_ACCOUNT_BACKEND_TYPE": "cs3", # proxy should get users from CS3APIS (which gets it from LDAP) + "PROXY_USER_OIDC_CLAIM": "ocis.user.uuid", # claim was added in Keycloak + "PROXY_USER_CS3_CLAIM": "userid", # equals STORAGE_LDAP_USER_SCHEMA_UID + "STORAGE_LDAP_BASE_DN": "dc=owncloud,dc=com", + "STORAGE_LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn", + "STORAGE_LDAP_GROUP_SCHEMA_GID_NUMBER": "gidnumber", + "STORAGE_LDAP_GROUP_SCHEMA_GID": "cn", + "STORAGE_LDAP_GROUP_SCHEMA_MAIL": "mail", + "STORAGE_LDAP_GROUPATTRIBUTEFILTER": "(&(objectclass=posixGroup)(objectclass=owncloud)({{attr}}={{value}}))", + "STORAGE_LDAP_GROUPFILTER": "(&(objectclass=groupOfUniqueNames)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))", + "STORAGE_LDAP_GROUPMEMBERFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))", + "STORAGE_LDAP_USERGROUPFILTER": "(&(objectclass=posixGroup)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))", + "STORAGE_LDAP_USER_SCHEMA_CN": "cn", + "STORAGE_LDAP_USER_SCHEMA_DISPLAYNAME": "displayname", + "STORAGE_LDAP_USER_SCHEMA_GID_NUMBER": "gidnumber", + "STORAGE_LDAP_USER_SCHEMA_MAIL": "mail", + "STORAGE_LDAP_USER_SCHEMA_UID_NUMBER": "uidnumber", + "STORAGE_LDAP_USER_SCHEMA_UID": "ownclouduuid", + "STORAGE_LDAP_LOGINFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(uid={{login}})(mail={{login}})))", + "STORAGE_LDAP_USERATTRIBUTEFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)({{attr}}={{value}}))", + "STORAGE_LDAP_USERFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(ownclouduuid={{.OpaqueId}})(uid={{.OpaqueId}})))", + "STORAGE_LDAP_USERFINDFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)))", + # ownCloud storage driver + "STORAGE_HOME_DRIVER": "owncloudsql", + "STORAGE_USERS_DRIVER": "owncloudsql", + "STORAGE_METADATA_DRIVER": "ocis", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR": "/mnt/data/files", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR": "/tmp", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER": "/Shares", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT": "{{.Username}}", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME": "owncloud", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD": "owncloud", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBHOST": "oc10-db", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPORT": 3306, + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME": "owncloud", + # TODO: redis is not yet supported + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR": "redis:6379", + # ownCloud storage readonly + # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303 + "OCIS_STORAGE_READ_ONLY": "false", + # General oCIS config + "OCIS_LOG_LEVEL": "error", + "OCIS_URL": OCIS_URL, + "PROXY_TLS": "true", + # change default secrets + "OCIS_JWT_SECRET": "Pive-Fumkiu4", + "STORAGE_TRANSFER_SECRET": "replace-me-with-a-transfer-secret", + "OCIS_MACHINE_AUTH_API_KEY": "change-me-please", + "OCIS_INSECURE": "true", + "PROXY_ENABLE_BASIC_AUTH": "true", + } + + return [ + { + "name": "ocis", + "image": OC_OCIS, + "environment": environment, + "detach": True, + "commands": [ + "ocis server", + ], + "volumes": [ + stepVolumeOC10OCISData, + stepVolumeOCISConfig, + ], + "user": "33:33", + "depends_on": ["fix-permissions"], + }, + { + "name": "wait-for-ocis", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it ocis:9200 -t 300", + ], + "depends_on": ["wait-for-oc10"], + }, + ] + +def oC10Server(): + return [ + { + "name": "oc10", + "image": OC_OC10, + "pull": "always", + "detach": True, + "environment": { + # can be switched to "web" + "OWNCLOUD_DEFAULT_APP": "files", + "OWNCLOUD_WEB_REWRITE_LINKS": "false", + # script / config variables + "IDP_OIDC_ISSUER": "https://keycloak/auth/realms/owncloud", + "IDP_OIDC_CLIENT_SECRET": "oc10-oidc-secret", + "CLOUD_DOMAIN": OCIS_DOMAIN, + # LDAP bind configuration + "LDAP_HOST": "openldap", + "LDAP_PORT": 389, + "STORAGE_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com", + "STORAGE_LDAP_BIND_PASSWORD": "admin", + # LDAP user configuration + "LDAP_BASE_DN": "dc=owncloud,dc=com", + "LDAP_USER_SCHEMA_DISPLAYNAME": "displayname", + "LDAP_LOGINFILTER": "(&(objectclass=owncloud)(|(uid=%uid)(mail=%uid)))", + "LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn", + "LDAP_USER_SCHEMA_NAME_ATTR": "uid", + "LDAP_GROUPFILTER": "(&(objectclass=groupOfUniqueNames)(objectclass=owncloud))", + "LDAP_USER_SCHEMA_UID": "ownclouduuid", + "LDAP_USERATTRIBUTEFILTERS": "uid", # ownCloudUUID;cn;uid;mail + "LDAP_USER_SCHEMA_MAIL": "mail", + "LDAP_USERFILTER": "(&(objectclass=owncloud))", + "LDAP_GROUP_MEMBER_ASSOC_ATTR": "uniqueMember", + # database + "OWNCLOUD_DB_TYPE": "mysql", + "OWNCLOUD_DB_NAME": "owncloud", + "OWNCLOUD_DB_USERNAME": "owncloud", + "OWNCLOUD_DB_PASSWORD": "owncloud", + "OWNCLOUD_DB_HOST": "oc10-db", + "OWNCLOUD_ADMIN_USERNAME": "admin", + "OWNCLOUD_ADMIN_PASSWORD": "admin", + "OWNCLOUD_MYSQL_UTF8MB4": "true", + # redis + "OWNCLOUD_REDIS_ENABLED": "true", + "OWNCLOUD_REDIS_HOST": "redis", + # ownCloud config + "OWNCLOUD_TRUSTED_PROXIES": OCIS_DOMAIN, + "OWNCLOUD_OVERWRITE_PROTOCOL": "https", + "OWNCLOUD_OVERWRITE_HOST": OCIS_DOMAIN, + "OWNCLOUD_APPS_ENABLE": "openidconnect,oauth2,user_ldap,graphapi", + "OWNCLOUD_LOG_LEVEL": 2, + "OWNCLOUD_LOG_FILE": "/mnt/data/owncloud.log", + }, + "volumes": [ + stepVolumeOC10OCISData, + stepVolumeOC10Apps, + stepVolumeOC10Templates, + stepVolumeOC10PreServer, + ], + "depends_on": ["wait-for-services", "copy-configs"], + }, + { + "name": "wait-for-oc10", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it oc10:8080 -t 300", + ], + "depends_on": ["wait-for-services"], + }, + ] + +def ldapService(): + return [{ + "name": "openldap", + "image": OSIXIA_OPENLDAP, + "pull": "always", + "environment": { + "LDAP_TLS_VERIFY_CLIENT": "never", + "LDAP_DOMAIN": "owncloud.com", + "LDAP_ORGANISATION": "owncloud", + "LDAP_ADMIN_PASSWORD": "admin", + "LDAP_RFC2307BIS_SCHEMA": "true", + "LDAP_REMOVE_CONFIG_AFTER_SETUP": "false", + "LDAP_SEED_INTERNAL_LDIF_PATH": "%s/ldap/ldif" % (PARALLEL_DEPLOY_CONFIG_PATH), + }, + "command": [ + "--copy-service", + "--loglevel", + "debug", + ], + }] + +def oc10DbService(): + return [ + { + "name": "oc10-db", + "image": MARIADB, + "pull": "always", + "environment": { + "MYSQL_ROOT_PASSWORD": "owncloud", + "MYSQL_USER": "owncloud", + "MYSQL_PASSWORD": "owncloud", + "MYSQL_DATABASE": "owncloud", + }, + "command": [ + "--max-allowed-packet=128M", + "--innodb-log-file-size=64M", + "--innodb-read-only-compressed=OFF", + ], + }, + ] + +def copyConfigs(): + return [{ + "name": "copy-configs", + "image": OC_OC10, + "pull": "always", + "commands": [ + # ocis proxy config + "mkdir -p /etc/ocis", + "cp %s/ocis/proxy.json /etc/ocis/proxy.json" % (PARALLEL_DEPLOY_CONFIG_PATH), + # oc10 configs + "mkdir -p /etc/templates", + "mkdir -p /etc/pre_server.d", + "cp %s/oc10/oidc.config.php /etc/templates/oidc.config.php" % (PARALLEL_DEPLOY_CONFIG_PATH), + "cp %s/oc10/ldap-config.tmpl.json /etc/templates/ldap-config.tmpl.json" % (PARALLEL_DEPLOY_CONFIG_PATH), + "cp %s/oc10/10-custom-config.sh /etc/pre_server.d/10-custom-config.sh" % (PARALLEL_DEPLOY_CONFIG_PATH), + ], + "volumes": [ + stepVolumeOCISConfig, + stepVolumeOC10Templates, + stepVolumeOC10PreServer, + ], + }] + +def owncloudLog(): + return [{ + "name": "owncloud-log", + "image": OC_UBUNTU, + "pull": "always", + "detach": True, + "commands": [ + "tail -f /mnt/data/owncloud.log", + ], + "volumes": [ + stepVolumeOC10OCISData, + ], + "depends_on": ["wait-for-oc10"], + }] + +def fixSharedDataPermissions(): + return [{ + "name": "fix-permissions", + "image": OC_CI_PHP, + "pull": "always", + "commands": [ + "chown -R www-data:www-data /var/www/owncloud/apps", + "chmod -R 777 /var/www/owncloud/apps", + "chmod -R 777 /mnt/data/", + ], + "volumes": [ + stepVolumeOC10Apps, + stepVolumeOC10OCISData, + ], + "depends_on": ["wait-for-oc10"], + }] + +def waitForServices(): + return [{ + "name": "wait-for-services", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it oc10-db:3306 -t 300", + "wait-for -it openldap:636 -t 300", + ], + }] diff --git a/Makefile b/Makefile index f7d22aa8bd4..de9c6924ba6 100644 --- a/Makefile +++ b/Makefile @@ -51,8 +51,9 @@ help: @echo @echo -e "${GREEN}Testing with test suite natively installed:${RESET}\n" @echo -e "${PURPLE}\tdocs: https://owncloud.dev/ocis/development/testing/#testing-with-test-suite-natively-installed${RESET}\n" - @echo -e "\tmake test-acceptance-api\t${BLUE}run API acceptance tests${RESET}" - @echo -e "\tmake clean-tests\t\t${BLUE}delete API tests framework dependencies${RESET}" + @echo -e "\tmake test-acceptance-api\t\t${BLUE}run API acceptance tests${RESET}" + @echo -e "\tmake test-paralleldeployment-api\t${BLUE}run API acceptance tests for parallel deployment${RESET}" + @echo -e "\tmake clean-tests\t\t\t${BLUE}delete API tests framework dependencies${RESET}" @echo @echo -e "${BLACK}---------------------------------------------------------${RESET}" @echo @@ -83,11 +84,17 @@ clean-tests: @rm -Rf vendor-bin/**/vendor vendor-bin/**/composer.lock tests/acceptance/output BEHAT_BIN=vendor-bin/behat/vendor/bin/behat +# behat config file for parallel deployment tests +PARALLEL_BEHAT_YML=tests/parallelDeployAcceptance/config/behat.yml .PHONY: test-acceptance-api test-acceptance-api: vendor-bin/behat/vendor BEHAT_BIN=$(BEHAT_BIN) $(PATH_TO_CORE)/tests/acceptance/run.sh --remote --type api +.PHONY: test-paralleldeployment-api +test-paralleldeployment-api: vendor-bin/behat/vendor + BEHAT_BIN=$(BEHAT_BIN) BEHAT_YML=$(PARALLEL_BEHAT_YML) $(PATH_TO_CORE)/tests/acceptance/run.sh --type api + vendor/bamarni/composer-bin-plugin: composer.lock composer install diff --git a/tests/parallelDeployAcceptance/.gitignore b/tests/parallelDeployAcceptance/.gitignore new file mode 100644 index 00000000000..b3423b476ef --- /dev/null +++ b/tests/parallelDeployAcceptance/.gitignore @@ -0,0 +1,2 @@ +!config +output \ No newline at end of file diff --git a/tests/parallelDeployAcceptance/config/behat.yml b/tests/parallelDeployAcceptance/config/behat.yml new file mode 100644 index 00000000000..eecebc81098 --- /dev/null +++ b/tests/parallelDeployAcceptance/config/behat.yml @@ -0,0 +1,40 @@ +default: + autoload: + "": "%paths.base%/../features/bootstrap" + + suites: + apiShareManagement: + paths: + - "%paths.base%/../features/apiShareManagement" + context: &common_ldap_suite_context + parameters: + ldapAdminPassword: admin + ldapUsersOU: TestUsers + ldapGroupsOU: TestGroups + ldapInitialUserFilePath: /ldap_users_groups.ldif + contexts: + - ParallelContext: + - FeatureContext: &common_feature_context_params + baseUrl: https://ocis:9200 + adminUsername: admin + adminPassword: admin + regularUserPassword: 1234 + ocPath: apps/testing/api/v1/occ + - OccContext: + + apiWebdavOperations: + paths: + - "%paths.base%/../features/apiWebdavOperations" + context: *common_ldap_suite_context + contexts: + - ParallelContext: + - FeatureContext: *common_feature_context_params + + extensions: + jarnaiz\JUnitFormatter\JUnitFormatterExtension: + filename: report.xml + outputDir: "%paths.base%/../output/" + + rdx\behatvars\BehatVariablesExtension: ~ + + Cjm\Behat\StepThroughExtension: ~ diff --git a/tests/parallelDeployAcceptance/config/ldap_users_groups.ldif b/tests/parallelDeployAcceptance/config/ldap_users_groups.ldif new file mode 100644 index 00000000000..9f76f70b1cb --- /dev/null +++ b/tests/parallelDeployAcceptance/config/ldap_users_groups.ldif @@ -0,0 +1,7 @@ +dn: ou=TestUsers,dc=owncloud,dc=com +objectClass: organizationalUnit +ou: TestUsers + +dn: ou=TestGroups,dc=owncloud,dc=com +objectClass: organizationalUnit +ou: TestGroups \ No newline at end of file diff --git a/tests/parallelDeployAcceptance/drone/ldap/ldif/10_owncloud_schema.ldif b/tests/parallelDeployAcceptance/drone/ldap/ldif/10_owncloud_schema.ldif new file mode 100644 index 00000000000..bff48c367e5 --- /dev/null +++ b/tests/parallelDeployAcceptance/drone/ldap/ldif/10_owncloud_schema.ldif @@ -0,0 +1,10 @@ +# This LDIF files describes the ownCloud schema and can be used to +# add two optional attributes: ownCloudQuota and ownCloudUUID +# The ownCloudUUID is used to store a unique, non-reassignable, persistent identifier for users and groups +dn: cn=owncloud,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: owncloud +olcAttributeTypes: ( 1.3.6.1.4.1.39430.1.1.1 NAME 'ownCloudQuota' DESC 'User Quota (e.g. 2 GB)' EQUALITY caseExactMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) +olcAttributeTypes: ( 1.3.6.1.4.1.39430.1.1.2 NAME 'ownCloudUUID' DESC 'A non-reassignable and persistent account ID)' EQUALITY uuidMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.1.16.1 SINGLE-VALUE ) +olcAttributeTypes: ( 1.3.6.1.4.1.39430.1.1.3 NAME 'ownCloudSelector' DESC 'A selector attribute for a route in the ownCloud Infinte Scale proxy)' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) +olcObjectClasses: ( 1.3.6.1.4.1.39430.1.2.1 NAME 'ownCloud' DESC 'ownCloud LDAP Schema' AUXILIARY MAY ( ownCloudQuota $ ownCloudUUID $ ownCloudSelector ) ) diff --git a/tests/parallelDeployAcceptance/drone/oc10/10-custom-config.sh b/tests/parallelDeployAcceptance/drone/oc10/10-custom-config.sh new file mode 100755 index 00000000000..47bcf2668e1 --- /dev/null +++ b/tests/parallelDeployAcceptance/drone/oc10/10-custom-config.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +echo "Writing custom config files..." + +# openidconnect +gomplate \ + -f /etc/templates/oidc.config.php \ + -o ${OWNCLOUD_VOLUME_CONFIG}/oidc.config.php + +# we need at least version 2.1.0 of the oenidconnect app +occ market:upgrade --major openidconnect +occ app:enable openidconnect + +# user LDAP +gomplate \ + -f /etc/templates/ldap-config.tmpl.json \ + -o ${OWNCLOUD_VOLUME_CONFIG}/ldap-config.json + +CONFIG=$(cat ${OWNCLOUD_VOLUME_CONFIG}/ldap-config.json) +occ config:import <<< $CONFIG + +occ ldap:test-config "s01" +occ app:enable user_ldap +/bin/bash -c 'occ user:sync "OCA\User_LDAP\User_Proxy" -r -m remove' + +occ market:upgrade --major web +occ app:enable web + +# enable testing app +echo "Cloning and enabling testing app..." +git clone --depth 1 https://github.com/owncloud/testing.git /var/www/owncloud/apps/testing +occ app:enable testing + +true diff --git a/tests/parallelDeployAcceptance/drone/oc10/ldap-config.tmpl.json b/tests/parallelDeployAcceptance/drone/oc10/ldap-config.tmpl.json new file mode 100755 index 00000000000..93c65670ff0 --- /dev/null +++ b/tests/parallelDeployAcceptance/drone/oc10/ldap-config.tmpl.json @@ -0,0 +1,53 @@ +{ + "apps": { + "user_ldap": { + "s01has_memberof_filter_support": "0", + "s01home_folder_naming_rule": "", + "s01last_jpegPhoto_lookup": "0", + "s01ldap_agent_password": "{{ .Env.STORAGE_LDAP_BIND_PASSWORD | base64.Encode }}", + "s01ldap_attributes_for_group_search": "", + "s01ldap_attributes_for_user_search": "{{ .Env.LDAP_USERATTRIBUTEFILTERS }}", + "s01ldap_backup_host": "", + "s01ldap_backup_port": "", + "s01ldap_base_groups": "{{ .Env.LDAP_BASE_DN }}", + "s01ldap_base_users": "{{ .Env.LDAP_BASE_DN }}", + "s01ldap_base": "{{ .Env.LDAP_BASE_DN }}", + "s01ldap_cache_ttl": "60", + "s01ldap_configuration_active": "1", + "s01ldap_display_name": "{{ .Env.LDAP_USER_SCHEMA_DISPLAYNAME }}", + "s01ldap_dn": "{{ .Env.STORAGE_LDAP_BIND_DN }}", + "s01ldap_dynamic_group_member_url": "", + "s01ldap_email_attr": "{{ .Env.LDAP_USER_SCHEMA_MAIL }}", + "s01ldap_experienced_admin": "1", + "s01ldap_expert_username_attr": "{{ .Env.LDAP_USER_SCHEMA_NAME_ATTR }}", + "s01ldap_expert_uuid_group_attr": "", + "s01ldap_expert_uuid_user_attr": "{{ .Env.LDAP_USER_SCHEMA_UID }}", + "s01ldap_group_display_name": "{{ .Env.LDAP_GROUP_SCHEMA_DISPLAYNAME }}", + "s01ldap_group_filter_mode": "0", + "s01ldap_group_filter": "{{ .Env.LDAP_GROUPFILTER }}", + "s01ldap_group_member_assoc_attribute": "{{ .Env.LDAP_GROUP_MEMBER_ASSOC_ATTR }}", + "s01ldap_groupfilter_groups": "", + "s01ldap_groupfilter_objectclass": "", + "s01ldap_host": "{{ .Env.LDAP_HOST }}", + "s01ldap_login_filter_mode": "0", + "s01ldap_login_filter": "{{ .Env.LDAP_LOGINFILTER }}", + "s01ldap_loginfilter_attributes": "", + "s01ldap_loginfilter_email": "1", + "s01ldap_loginfilter_username": "1", + "s01ldap_nested_groups": "0", + "s01ldap_override_main_server": "", + "s01ldap_paging_size": "100", + "s01ldap_port": "{{ .Env.LDAP_PORT }}", + "s01ldap_quota_attr": "", + "s01ldap_quota_def": "", + "s01ldap_tls": "0", + "s01ldap_turn_off_cert_check": "0", + "s01ldap_user_display_name_2": "", + "s01ldap_user_filter_mode": "0", + "s01ldap_userfilter_groups": "", + "s01ldap_userfilter_objectclass": "", + "s01ldap_userlist_filter": "{{ .Env.LDAP_USERFILTER }}", + "s01use_memberof_to_detect_membership": "1" + } + } +} diff --git a/tests/parallelDeployAcceptance/drone/oc10/oidc.config.php b/tests/parallelDeployAcceptance/drone/oc10/oidc.config.php new file mode 100644 index 00000000000..5de9f9d3e61 --- /dev/null +++ b/tests/parallelDeployAcceptance/drone/oc10/oidc.config.php @@ -0,0 +1,22 @@ + [ + 'provider-url' => getenv('IDP_OIDC_ISSUER'), + 'client-id' => 'oc10', + 'client-secret' => getenv('IDP_OIDC_CLIENT_SECRET'), + 'loginButtonName' => 'OpenId Connect', + 'search-attribute' => 'preferred_username', + 'mode' => 'userid', + 'autoRedirectOnLoginPage' => true, + 'insecure' => true, + 'post_logout_redirect_uri' => 'https://' . getenv('CLOUD_DOMAIN'), + ], + ]; + return $config; +} + +$CONFIG = getOIDCConfigFromEnv(); diff --git a/tests/parallelDeployAcceptance/drone/ocis/proxy.json b/tests/parallelDeployAcceptance/drone/ocis/proxy.json new file mode 100644 index 00000000000..6333545dac0 --- /dev/null +++ b/tests/parallelDeployAcceptance/drone/ocis/proxy.json @@ -0,0 +1,109 @@ +{ + "log": { + "level": "PROXY_LOG_LEVEL" + }, + "policy_selector": { + "claims": { + "default_policy": "oc10", + "unauthenticated_policy": "oc10" + } + }, + "policies": [ + { + "name": "ocis", + "routes": [ + { + "endpoint": "/", + "backend": "http://localhost:9100" + }, + { + "endpoint": "/.well-known/", + "backend": "http://localhost:9130" + }, + { + "type": "regex", + "endpoint": "/ocs/v[12].php/cloud/user/signing-key", + "backend": "http://localhost:9110" + }, + { + "endpoint": "/ocs/", + "backend": "http://localhost:9140" + }, + { + "type": "query", + "endpoint": "/remote.php/?preview=1", + "backend": "http://localhost:9115" + }, + { + "endpoint": "/remote.php/", + "backend": "http://localhost:9140" + }, + { + "endpoint": "/dav/", + "backend": "http://localhost:9140" + }, + { + "endpoint": "/webdav/", + "backend": "http://localhost:9140" + }, + { + "endpoint": "/status.php", + "backend": "http://localhost:9140" + }, + { + "endpoint": "/index.php/", + "backend": "http://localhost:9140" + }, + { + "endpoint": "/index.php/login", + "backend": "http://localhost:9100" + }, + { + "endpoint": "/login", + "backend": "http://localhost:9100" + }, + { + "endpoint": "/data", + "backend": "http://localhost:9140" + }, + { + "endpoint": "/graph/", + "backend": "http://localhost:9120" + }, + { + "endpoint": "/app/", + "backend": "http://localhost:9140" + }, + { + "endpoint": "/archiver", + "backend": "http://localhost:9140" + }, + { + "endpoint": "/graph-explorer/", + "backend": "http://localhost:9135" + }, + { + "endpoint": "/api/v0/settings", + "backend": "http://localhost:9190" + }, + { + "endpoint": "/settings.js", + "backend": "http://localhost:9190" + } + ] + }, + { + "name": "oc10", + "routes": [ + { + "endpoint": "/", + "backend": "http://oc10:8080" + }, + { + "endpoint": "/data", + "backend": "http://localhost:9140" + } + ] + } + ] +} diff --git a/tests/parallelDeployAcceptance/expected-failures-API.md b/tests/parallelDeployAcceptance/expected-failures-API.md new file mode 100644 index 00000000000..2e698adcbe8 --- /dev/null +++ b/tests/parallelDeployAcceptance/expected-failures-API.md @@ -0,0 +1,5 @@ +## Scenarios that are expected to fail in parallel deployment + +#### [[WIP] Add a SharesStorageProvider and an oc10 sql share manager](https://github.com/owncloud/ocis/pull/2232) + +- [apiShareManagement/acceptShares.feature:22](https://github.com/owncloud/ocis/blob/master/tests/parallelDeployAcceptance/features/apiShareManagement/acceptShares.feature#L22) diff --git a/tests/parallelDeployAcceptance/features/apiShareManagement/acceptShares.feature b/tests/parallelDeployAcceptance/features/apiShareManagement/acceptShares.feature new file mode 100644 index 00000000000..0092e309edc --- /dev/null +++ b/tests/parallelDeployAcceptance/features/apiShareManagement/acceptShares.feature @@ -0,0 +1,30 @@ +# Sharing tests currently doesn't work +# Accessing oc10 shares from ocis still WIP in PR #2232 +# https://github.com/owncloud/ocis/pull/2232 +@api +Feature: sharing files and folders + As a user + I want to share files/folders with other users + So that I can give access to my files/folders to others + + + Background: + Given using "oc10" as owncloud selector + And the administrator has set the default folder for received shares to "Shares" + And auto-accept shares has been disabled + And using OCS API version "1" + And using new DAV path + And user "Alice" has been created with default attributes and without skeleton files + And user "Brian" has been created with default attributes and without skeleton files + And user "Alice" has uploaded file with content "ownCloud test text file" to "textfile.txt" + + + Scenario: accept a pending share + Given user "Alice" has shared folder "/textfile.txt" with user "Brian" + And using "ocis" as owncloud selector + When user "Brian" accepts share "/textfile.txt" offered by user "Alice" using the sharing API + Then the OCS status code should be "100" + And the HTTP status code should be "200" + And the sharing API should report to user "Brian" that these shares are in the accepted state + | path | + | /Shares/textfile.txt | \ No newline at end of file diff --git a/tests/parallelDeployAcceptance/features/apiWebdavOperations/downloadFile.feature b/tests/parallelDeployAcceptance/features/apiWebdavOperations/downloadFile.feature new file mode 100644 index 00000000000..1db4f76b9c8 --- /dev/null +++ b/tests/parallelDeployAcceptance/features/apiWebdavOperations/downloadFile.feature @@ -0,0 +1,146 @@ +@api +Feature: download file + As a user + I want to be able to download files + So that I can work wih local copies of files on my client system + + Background: + Given using "oc10" as owncloud selector + And user "Alice" has been created with default attributes and without skeleton files + And user "Alice" has uploaded file with content "ownCloud test text file" to "textfile.txt" + + + Scenario Outline: download a file + Given using "ocis" as owncloud selector + And using DAV path + When user "Alice" downloads file "textfile.txt" using the WebDAV API + Then the downloaded content should be "ownCloud test text file" + Examples: + | dav_version | + | old | + | new | + + + Scenario Outline: download a file with range + Given using "ocis" as owncloud selector + And using DAV path + When user "Alice" downloads file "textfile.txt" with range "bytes=0-7" using the WebDAV API + Then the downloaded content should be "ownCloud" + Examples: + | dav_version | + | old | + | new | + + + Scenario: Get the size of a file + Given using "ocis" as owncloud selector + When user "Alice" gets the size of file "textfile.txt" using the WebDAV API + Then the HTTP status code should be "207" + And the size of the file should be "23" + + + Scenario Outline: Download a file with comma in the filename + Given using DAV path + And user "Alice" has uploaded file with content "file with comma in filename" to + And using "ocis" as owncloud selector + When user "Alice" downloads file using the WebDAV API + Then the downloaded content should be "file with comma in filename" + Examples: + | dav_version | filename | + | old | "sample,1.txt" | + | old | ",,,.txt" | + | old | ",,,.," | + | new | "sample,1.txt" | + | new | ",,,.txt" | + | new | ",,,.," | + + + Scenario Outline: download a file with single part ranges + Given using "ocis" as owncloud selector + And using DAV path + When user "Alice" downloads file "textfile.txt" with range "bytes=0-7" using the WebDAV API + Then the HTTP status code should be "206" + And the following headers should be set + | header | value | + | Content-Length | 8 | + | Content-Range | bytes 0-7/23 | + And the downloaded content should be "ownCloud" + Examples: + | dav_version | + | old | + | new | + + + Scenario Outline: download a file with last byte range out of bounds + Given using "ocis" as owncloud selector + And using DAV path + When user "Alice" downloads file "textfile.txt" with range "bytes=0-24" using the WebDAV API + Then the HTTP status code should be "206" + And the downloaded content should be "ownCloud test text file" + Examples: + | dav_version | + | old | + | new | + + + Scenario Outline: download a range at the end of a file + Given using "ocis" as owncloud selector + And using DAV path + When user "Alice" downloads file "textfile.txt" with range "bytes=-4" using the WebDAV API + Then the HTTP status code should be "206" + And the downloaded content should be "file" + Examples: + | dav_version | + | old | + | new | + + + Scenario Outline: download a file with range out of bounds + Given using "ocis" as owncloud selector + And using DAV path + When user "Alice" downloads file "textfile.txt" with range "bytes=24-30" using the WebDAV API + Then the HTTP status code should be "416" + Examples: + | dav_version | + | old | + | new | + + + Scenario Outline: download a hidden file + Given using DAV path + And user "Alice" has created folder "FOLDER" + And user "Alice" has uploaded the following files with content "hidden file" + | path | + | .hidden_file | + | FOLDER/.hidden_file | + And using "ocis" as owncloud selector + When user "Alice" downloads file ".hidden_file" using the WebDAV API + Then the HTTP status code should be "200" + And the downloaded content should be "hidden file" + When user "Alice" downloads file "FOLDER/.hidden_file" using the WebDAV API + Then the HTTP status code should be "200" + And the downloaded content should be "hidden file" + Examples: + | dav_version | + | old | + | new | + + + Scenario Outline: Downloading a file should serve security headers + Given using "ocis" as owncloud selector + And using DAV path + When user "Alice" downloads file "textfile.txt" using the WebDAV API + Then the following headers should be set + | header | value | + | Content-Disposition | attachment; filename*=UTF-8''textfile.txt; filename="textfile.txt" | + | Content-Security-Policy | default-src 'none'; | + | X-Content-Type-Options | nosniff | + | X-Download-Options | noopen | + | X-Frame-Options | SAMEORIGIN | + | X-Permitted-Cross-Domain-Policies | none | + | X-Robots-Tag | none | + | X-XSS-Protection | 1; mode=block | + Examples: + | dav_version | + | old | + | new | diff --git a/tests/parallelDeployAcceptance/features/bootstrap/ParallelContext.php b/tests/parallelDeployAcceptance/features/bootstrap/ParallelContext.php new file mode 100644 index 00000000000..23e6202ce87 --- /dev/null +++ b/tests/parallelDeployAcceptance/features/bootstrap/ParallelContext.php @@ -0,0 +1,71 @@ + + * @copyright Copyright (c) 2021 Sajan Gurung sajan@jankaritech.com + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, + * as published by the Free Software Foundation; + * either version 3 of the License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see + * + */ + +use Behat\Behat\Context\Context; +use Behat\Behat\Hook\Scope\BeforeScenarioScope; +use TestHelpers\SetupHelper; +use TestHelpers\HttpRequestHelper; + +require_once 'bootstrap.php'; + +/** + * Steps related to parallel deploy setup + */ +class ParallelContext implements Context { + + /** + * @var FeatureContext + */ + private FeatureContext $featureContext; + + /** + * @BeforeScenario + * + * @param BeforeScenarioScope $scope + * + * @return void + * + * @throws Exception + */ + public function setUpScenario(BeforeScenarioScope $scope): void { + $environment = $scope->getEnvironment(); + $this->featureContext = $environment->getContext('FeatureContext'); + SetupHelper::init( + $this->featureContext->getAdminUsername(), + $this->featureContext->getAdminPassword(), + $this->featureContext->getBaseUrl(), + $this->featureContext->getOcPath() + ); + } + + /** + * @Given using :selector as owncloud selector + * + * @param string $selector 'ocis' or 'oc10' + * + * @return void + */ + public function usingOwncloudSelector(string $selector): void { + $this->featureContext->setOCSelector($selector); + HttpRequestHelper::setOCSelectorCookie("owncloud-selector=$selector;path=/;"); + } +} diff --git a/tests/parallelDeployAcceptance/features/bootstrap/bootstrap.php b/tests/parallelDeployAcceptance/features/bootstrap/bootstrap.php new file mode 100644 index 00000000000..ec7e80734a1 --- /dev/null +++ b/tests/parallelDeployAcceptance/features/bootstrap/bootstrap.php @@ -0,0 +1,37 @@ + + * @copyright Copyright (c) 2020 Phil Davis phil@jankaritech.com + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, + * as published by the Free Software Foundation; + * either version 3 of the License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see + * + */ + +$pathToCore = \getenv('PATH_TO_CORE'); +if ($pathToCore === false) { + $pathToCore = "../core"; +} + +require_once $pathToCore . '/tests/acceptance/features/bootstrap/bootstrap.php'; + +$classLoader = new \Composer\Autoload\ClassLoader(); +$classLoader->addPsr4( + "", + $pathToCore . "/tests/acceptance/features/bootstrap", + true +); + +$classLoader->register();