[Tests-Only] Implement expected failures file (#1036)

.drone.yml

@@ -338,13 +338,33 @@ steps:
       - /drone/src/cmd/revad/revad -c storage-publiclink.toml &
       - /drone/src/cmd/revad/revad -c ldap-users.toml
 
-  - name: oC10APIAcceptanceTests
+  - name: clone-oC10-test-repos
     image: owncloudci/php:7.2
     commands:
       - git clone -b master --depth=1 https://github.com/owncloud/testing.git /drone/src/tmp/testing
       - git clone -b master --single-branch --no-tags https://github.com/owncloud/core.git /drone/src/tmp/testrunner
       - cd /drone/src/tmp/testrunner
-      - git checkout 03c68e17ebab61da2ebb0a9f541a6cffac0908ac
+      - git checkout 5d8d76600f451b985662024f730cee1a40444c7f
+
+  - name: localAPIAcceptanceTestsOcStorage
+    image: owncloudci/php:7.2
+    commands:
+      - make test-acceptance-api
+    environment:
+      TEST_SERVER_URL: 'http://revad-services:20080'
+      OCIS_REVA_DATA_ROOT: '/drone/src/tmp/reva/'
+      SKELETON_DIR: '/drone/src/tmp/testing/data/apiSkeleton'
+      TEST_EXTERNAL_USER_BACKENDS: 'true'
+      REVA_LDAP_HOSTNAME: 'ldap'
+      TEST_OCIS: 'true'
+      TEST_REVA: 'true'
+      BEHAT_FILTER_TAGS: '~@skipOnOcis-OC-Storage'
+      PATH_TO_CORE: '/drone/src/tmp/testrunner'
+
+  - name: oC10APIAcceptanceTests
+    image: owncloudci/php:7.2
+    commands:
+      - cd /drone/src/tmp/testrunner
       - make test-acceptance-api
     environment:
       TEST_SERVER_URL: 'http://revad-services:20080'
@@ -354,7 +374,8 @@ steps:
       REVA_LDAP_HOSTNAME: 'ldap'
       TEST_OCIS: 'true'
       TEST_REVA: 'true'
-      BEHAT_FILTER_TAGS: '~@skipOnOcis&&~@skipOnOcis-OC-Storage&&~@preview-extension-required'
+      BEHAT_FILTER_TAGS: '~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~@preview-extension-required'
+      EXPECTED_FAILURES_FILE: '/drone/src/tests/acceptance/expected-failures.txt'
 
 services:
   - 'name': 'ldap'

.gitignore

@@ -39,3 +39,10 @@ docs/themes/
 
 dist/
 bin/
+
+# API acceptance tests
+composer.lock
+/vendor
+vendor-bin/**/vendor
+vendor-bin/**/composer.lock
+tests/acceptance/output

Makefile

@@ -88,3 +88,21 @@ dist: default
 	go run tools/create-artifacts/main.go -version ${VERSION} -commit ${GIT_COMMIT} -goversion ${GO_VERSION}
 
 all: deps default
+
+BEHAT_BIN=vendor-bin/behat/vendor/bin/behat
+
+.PHONY: test-acceptance-api
+test-acceptance-api: vendor-bin/behat/vendor
+	BEHAT_BIN=$(BEHAT_BIN) $(PATH_TO_CORE)/tests/acceptance/run.sh --remote --type api
+
+vendor/bamarni/composer-bin-plugin: composer.lock
+	composer install
+
+vendor-bin/behat/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/behat/composer.lock
+	composer bin behat install --no-progress
+
+vendor-bin/behat/composer.lock: vendor-bin/behat/composer.json
+	@echo behat composer.lock is not up to date.
+
+composer.lock: composer.json
+	@echo composer.lock is not up to date.

composer.json

@@ -0,0 +1,18 @@
+{
+  "name": "cs3org/reva",
+  "config" : {
+    "platform": {
+      "php": "7.2"
+    }
+  },
+  "require": {
+  },
+  "require-dev": {
+    "bamarni/composer-bin-plugin": "^1.4"
+  },
+  "extra": {
+    "bamarni-bin": {
+      "bin-links": false
+    }
+  }
+}

tests/acceptance/config/behat.yml

@@ -0,0 +1,34 @@
+default:
+  autoload:
+    '': '%paths.base%/../features/bootstrap'
+
+  suites:
+    apiOcisSpecific:
+      paths:
+        - '%paths.base%/../features/apiOcisSpecific'
+      context: &common_ldap_suite_context
+        parameters:
+          ldapAdminPassword: admin
+          ldapUsersOU: TestUsers
+          ldapGroupsOU: TestGroups
+          ldapInitialUserFilePath: /../../config/ldap-users.ldif
+      contexts:
+        - RevaContext:
+        - OccContext:
+        - FeatureContext: &common_feature_context_params
+            baseUrl:  http://localhost:8080
+            adminUsername: admin
+            adminPassword: admin
+            regularUserPassword: 123456
+            ocPath: apps/testing/api/v1/occ
+        - CapabilitiesContext:
+        - ChecksumContext:
+        - FavoritesContext:
+        - FilesVersionsContext:
+        - PublicWebDavContext:
+        - WebDavPropertiesContext:
+
+  extensions:
+    jarnaiz\JUnitFormatter\JUnitFormatterExtension:
+      filename: report.xml
+      outputDir: '%paths.base%/../output/'