Merge branch 'master' into poc-actions-rpc

mattermost · Jul 5, 2024 · a1fd7bc · a1fd7bc
2 parents a691490 + b596430
commit a1fd7bc
Show file tree

Hide file tree

Showing 396 changed files with 14,970 additions and 12,794 deletions.
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
@@ -1,25 +1,34 @@
-Per Mattermost guidelines, GitHub issues are for bug reports: <https://handbook.mattermost.com/contributors/contributors/ways-to-contribute/>.
+Before filing a bug report, please check the following:
 
-For troubleshooting see: https://forum.mattermost.com/.
-For feature proposals see: https://mattermost.com/suggestions/
-
-If you've found a bug--something appears unintentional--please follow these steps:
-
-1. Confirm you’re filing a new issue. [Search existing tickets in Jira](https://mattermost.atlassian.net/jira/software/c/projects/MM/issues/) to ensure that the ticket does not already exist.
-2. Confirm your issue does not involve security. Otherwise, please see our [Responsible Disclosure Policy](https://mattermost.com/security-vulnerability-report/).
-3. [File a new issue](https://github.com/mattermost/mattermost/issues/new) using the format below. Mattermost will confirm steps to reproduce and file in Jira, or ask for more details if there is trouble reproducing it. If there's already an existing bug in Jira, it will be linked back to the GitHub issue so you can track when it gets fixed. 
+1. Confirm you’re filing a new issue. [Search existing tickets in Jira](https://mattermost.atlassian.net/jira/software/c/projects/MM/issues/) and [existing issues in GitHub](https://github.com/mattermost/mattermost/issues) to ensure that the ticket does not already exist.
+2. Confirm your issue does not involve security. Please submit security issues via our [Responsible Disclosure Policy](https://mattermost.com/security-vulnerability-report/).
+3. Confirm your issue is not a feature request. Please submit feature requests on [UserVoice](https://mattermost.uservoice.com/forums/306457-general).
+4. Confirm your issue is not a troubleshooting question. Please submit troubleshooting questions on the [forum](https://forum.mattermost.com/).
+5. File a bug report using the format below. Mattermost will confirm steps to reproduce and file in Jira, or ask for more details if there is trouble reproducing it. If there's already an existing bug in Jira, it will be linked back to the GitHub issue so you can track when it gets fixed.
 
 #### Summary
-Bug report in one concise sentence
+Bug report in one concise sentence.
 
 #### Steps to reproduce
-How can we reproduce the issue (what version are you using?)
+What are the steps to reproduce the issue?
+
+#### Operating System
+What Operating System do you use? (examples: Windows 10 x64, macOS Ventura 13.2 Apple Silicon, Ubuntu Linux 22.04 LTS x64)
+
+#### Mattermost Server Version
+Which version of the Mattermost Server did this occur on? (You can find your Mattermost Server version via **Mattermost Menu** > **About Mattermost**)
 
 #### Expected behavior
-Describe your issue in detail
+What is the expected behaviour?
+
+#### Observed behavior
+What issue did you see happen?
+
+#### Log Output
+Output from the log files if applicable.
 
-#### Observed behavior (that appears unintentional) 
-What did you see happen? Please include relevant error messages and/or screenshots.
+#### Additional Information
+Additional helpful information, such as error messages and/or screenshots.
 
 #### Possible fixes
-If you can, link to the line of code that might be responsible for the problem
+If you can, link to the line of code that might be responsible for the problem.
diff --git a/.github/workflows/e2e-fulltests-ci.yml b/.github/workflows/e2e-fulltests-ci.yml
@@ -30,7 +30,7 @@ on:
         default: NONE
 
 concurrency:
-  group: "${{ github.workflow }}-${{ inputs.PR_NUMBER || inputs.ref }}-${{ inputs.MM_ENV }}"
+  group: "${{ github.workflow }}-${{ inputs.REPORT_TYPE }}-${{ inputs.PR_NUMBER || inputs.ref }}-${{ inputs.MM_ENV }}"
   cancel-in-progress: true
 
 jobs:
@@ -58,6 +58,7 @@ jobs:
             curl -fsSL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -H "Authorization: Bearer ${{ github.token }}" "${{ github.api_url }}/repos/${{ github.repository }}/pulls/${PR_NUMBER}" -o pr.json
             echo "commit_sha=$(jq -r .head.sha <pr.json)" >> $GITHUB_OUTPUT
             echo "BRANCH=server-pr-${PR_NUMBER}" >> $GITHUB_OUTPUT
+            # For reference, the real branch name may be retrievable as follows:
             #echo "BRANCH=$(jq -r .head.ref <pr.json)" >> $GITHUB_OUTPUT
           else
             echo "commit_sha=$(git rev-parse --verify HEAD)" >> $GITHUB_OUTPUT
@@ -77,18 +78,32 @@ jobs:
     outputs:
       status_check_context: "${{ steps.generate.outputs.status_check_context }}"
       workers_number: "${{ steps.generate.outputs.workers_number }}"
+      server_uppercase: "${{ steps.generate.outputs.server_uppercase }}" # Required for license selection
+      SERVER: "${{ steps.generate.outputs.SERVER }}"
       ENABLED_DOCKER_SERVICES: "${{ steps.generate.outputs.ENABLED_DOCKER_SERVICES }}"
       TEST_FILTER: "${{ steps.generate.outputs.TEST_FILTER }}"
       BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}"
+      TM4J_ENABLE: "${{ steps.generate.outputs.TM4J_ENABLE }}"
     env:
       GH_TOKEN: "${{ github.token }}"
       PR_NUMBER: "${{ inputs.PR_NUMBER || '' }}"
+      REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
       # We could exclude the @smoke group for PRs, but then we wouldn't have it in the report
       TEST_FILTER_PR: >-
         --stage="@prod"
         --excludeGroup="@te_only,@cloud_only,@high_availability"
         --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap"
         --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
+      TEST_FILTER_DAILY_ONPREM: >-
+        --stage="@prod"
+        --excludeGroup="@te_only,@cloud_only,@high_availability"
+        --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
+        --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
+      TEST_FILTER_DAILY_CLOUD: >-
+        --stage="@prod"
+        --excludeGroup="@not_cloud,@cloud_trial,@e20_only,@te_only,@high_availability,@license_removal"
+        --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
+        --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa"
       MM_ENV: "${{ inputs.MM_ENV || '' }}"
     steps:
       - name: ci/generate-test-variables
@@ -99,19 +114,35 @@ jobs:
           # Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
           BUILD_ID_PREFIX="${{ github.run_id }}_${{ github.run_attempt }}-${COMMIT_SHA::7}"
           MM_ENV_HASH=$(md5sum -z <<<"$MM_ENV" | cut -c-8)
-          case "${{ inputs.REPORT_TYPE }}" in
+          SERVER=onprem
+          case "$REPORT_TYPE" in
             NONE | PR)
               echo "status_check_context=E2E Tests/test${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
               echo "workers_number=20" >> $GITHUB_OUTPUT
-              echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT
               echo "TEST_FILTER=$TEST_FILTER_PR" >> $GITHUB_OUTPUT
-              echo "BUILD_ID=${BUILD_ID_PREFIX}-pr-onprem-ent" >> $GITHUB_OUTPUT
+              echo "BUILD_ID=${BUILD_ID_PREFIX}-${REPORT_TYPE@L}-${SERVER}-ent" >> $GITHUB_OUTPUT
+              ;;
+            MASTER | MASTER_UNSTABLE | CLOUD | CLOUD_UNSTABLE)
+              IS_TEST_UNSTABLE=$(sed -n -E 's/^.*(UNSTABLE).*$/\1/p' <<< "$REPORT_TYPE") # Variable is UNSTABLE if report type is for unstable tests
+              if grep -q CLOUD <<<$REPORT_TYPE; then
+                SERVER=cloud
+              fi
+              TEST_FILTER_VARIABLE="TEST_FILTER_DAILY_${SERVER@U}"
+              BUILD_ID_SUFFIX="${IS_TEST_UNSTABLE:+unstable-}daily-${SERVER}-ent"
+              echo "status_check_context=E2E Tests/test-${BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
+              echo "workers_number=10" >> $GITHUB_OUTPUT # Daily tests are not time critical, and it's more efficient to run on fewer workers
+              echo "TEST_FILTER=${!TEST_FILTER_VARIABLE} ${IS_TEST_UNSTABLE:+--invert}" >> $GITHUB_OUTPUT
+              echo "BUILD_ID=${BUILD_ID_PREFIX}-${BUILD_ID_SUFFIX}" >> $GITHUB_OUTPUT
+              echo "TM4J_ENABLE=true" >> $GITHUB_OUTPUT
               ;;
             *)
-              # TODO implement other test types, in the future
+              # TODO implement release testing, in the future
               echo "Fatal: unimplemented test type. Aborting."
               exit 1
           esac
+          echo "SERVER=${SERVER}" >> $GITHUB_OUTPUT
+          echo "server_uppercase=${SERVER@U}" >> $GITHUB_OUTPUT
+          echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT
       - name: ci/notify-user
         run: |
           if [ -n "$PR_NUMBER" ]; then
@@ -142,18 +173,20 @@ jobs:
       testcase_failure_fatal: false
       run_preflight_checks: false
       enable_reporting: true
+      SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
       ENABLED_DOCKER_SERVICES: "${{ needs.generate-test-variables.outputs.ENABLED_DOCKER_SERVICES }}"
       TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER }}"
       MM_ENV: "${{ inputs.MM_ENV || '' }}"
       BRANCH: "${{ needs.resolve-ref.outputs.BRANCH }}"
       BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
       REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
     secrets:
-      MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
+      MM_LICENSE: "${{ secrets[format('MM_E2E_TEST_LICENSE_{0}_ENT', needs.generate-test-variables.outputs.server_uppercase)] }}"
       AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
       AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_TOKEN }}"
       PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
       REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
-      ### These are disabled until release tests are implemented
-      #REPORT_TM4J_API_KEY: "${{ secrets.MM_E2E_TM4J_API_KEY }}"
-      #REPORT_TEST_CYCLE_LINK_PREFIX: "${{ secrets.MM_E2E_TEST_CYCLE_LINK_PREFIX }}"
+      REPORT_TM4J_API_KEY: "${{ needs.generate-test-variables.outputs.TM4J_ENABLE == 'true' && secrets.MM_E2E_TM4J_API_KEY || '' }}"
+      REPORT_TM4J_TEST_CYCLE_LINK_PREFIX: "${{ secrets.MM_E2E_TEST_CYCLE_LINK_PREFIX }}"
+      CWS_URL: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_URL || '' }}"
+      CWS_EXTRA_HTTP_HEADERS: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS || '' }}"
diff --git a/.github/workflows/e2e-tests-ci-template.yml b/.github/workflows/e2e-tests-ci-template.yml
@@ -30,6 +30,10 @@ on:
         type: boolean
         required: false
         default: false
+      SERVER:
+        type: string # Valid values are: onprem, cloud
+        required: false
+        default: onprem
       ENABLED_DOCKER_SERVICES:
         type: string
         required: false
@@ -63,6 +67,10 @@ on:
         required: false
       REPORT_TM4J_TEST_CYCLE_LINK_PREFIX:
         required: false
+      CWS_URL:
+        required: false
+      CWS_EXTRA_HTTP_HEADERS:
+        required: false
 
 jobs:
   update-initial-status:
@@ -150,21 +158,31 @@ jobs:
     runs-on: ubuntu-22.04
     needs:
       - update-initial-status
+    defaults:
+      run:
+        shell: bash
     outputs:
-      workers: "${{ steps.workers.outputs.workers }}"
+      workers: "${{ steps.generate.outputs.workers }}"
+      SERVER_IMAGE: "${{ steps.generate.outputs.SERVER_IMAGE }}"
     steps:
       - name: ci/checkout-repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           ref: ${{ inputs.commit_sha }}
           fetch-depth: 0
-      - name: ci/generate-workers
-        id: workers
+      - name: ci/generate-build-variables
+        id: generate
         env:
           WORKERS: ${{ inputs.workers_number }}
+          REPORT_TYPE: ${{ inputs.REPORT_TYPE }}
         run: |
           [ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0
           echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT
+          if grep -qE '^((MASTER|CLOUD)(_UNSTABLE)?)$' <<<"$REPORT_TYPE"; then
+            # For MASTER, MASTER_UNSTABLE, CLOUD and CLOUD_UNSTABLE runs, always use the latest master image
+            # The commit_sha is still used for reporting the test result, and for the testing scripts and test cases
+            echo "SERVER_IMAGE=mattermostdevelopment/mattermost-enterprise-edition:master" >> $GITHUB_OUTPUT
+          fi
 
   generate-test-cycle:
     runs-on: ubuntu-22.04
@@ -237,6 +255,8 @@ jobs:
     env:
       AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
       AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
+      SERVER: "${{ inputs.SERVER }}"
+      SERVER_IMAGE: "${{ needs.generate-build-variables.outputs.SERVER_IMAGE }}"
       MM_LICENSE: "${{ secrets.MM_LICENSE }}"
       ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}"
       TEST_FILTER: "${{ inputs.TEST_FILTER }}"
@@ -245,6 +265,8 @@ jobs:
       BUILD_ID: "${{ inputs.BUILD_ID }}"
       CI_BASE_URL: "http://localhost:8065/?worker_index=${{ matrix.worker_index }}"
       CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
+      CWS_URL: "${{ secrets.CWS_URL }}"
+      CWS_EXTRA_HTTP_HEADERS: "${{ secrets.CWS_EXTRA_HTTP_HEADERS }}"
     steps:
       - name: ci/checkout-repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -282,6 +304,7 @@ jobs:
     runs-on: ubuntu-22.04
     needs:
       - test
+      - generate-build-variables
     defaults:
       run:
         shell: bash
@@ -322,6 +345,8 @@ jobs:
         if: "${{ inputs.enable_reporting }}"
         env:
           TYPE: "${{ inputs.REPORT_TYPE }}"
+          SERVER: "${{ inputs.SERVER }}"
+          SERVER_IMAGE: "${{ needs.generate-build-variables.outputs.SERVER_IMAGE }}"
           WEBHOOK_URL: "${{ secrets.REPORT_WEBHOOK_URL }}"
           BRANCH: "${{ inputs.BRANCH }}"
           BUILD_ID: "${{ inputs.BUILD_ID }}"

diff --git a/api/Makefile b/api/Makefile
@@ -40,6 +40,7 @@ build-v4: node_modules playbooks
 	@cat $(V4_SRC)/roles.yaml >> $(V4_YAML)
 	@cat $(V4_SRC)/schemes.yaml >> $(V4_YAML)
 	@cat $(V4_SRC)/service_terms.yaml >> $(V4_YAML)
+	@cat $(V4_SRC)/remoteclusters.yaml >> $(V4_YAML)
 	@cat $(V4_SRC)/sharedchannels.yaml >> $(V4_YAML)
 	@cat $(V4_SRC)/reactions.yaml >> $(V4_YAML)
 	@cat $(V4_SRC)/actions.yaml >> $(V4_YAML)
@@ -53,14 +54,15 @@ build-v4: node_modules playbooks
 	@cat $(V4_SRC)/bookmarks.yaml >> $(V4_YAML)
 	@cat $(V4_SRC)/reports.yaml >> $(V4_YAML)
 	@cat $(V4_SRC)/limits.yaml >> $(V4_YAML)
+	@cat $(V4_SRC)/logs.yaml >> $(V4_YAML)
 	@cat $(V4_SRC)/outgoing_oauth_connections.yaml >> $(V4_YAML)
 	@if [ -r $(PLAYBOOKS_SRC)/paths.yaml ]; then cat $(PLAYBOOKS_SRC)/paths.yaml >> $(V4_YAML); fi
 	@if [ -r $(PLAYBOOKS_SRC)/merged-definitions.yaml ]; then cat $(PLAYBOOKS_SRC)/merged-definitions.yaml >> $(V4_YAML); else cat $(V4_SRC)/definitions.yaml >> $(V4_YAML); fi
 	@echo Extracting code samples
 	cd server && go run . $(V4_YAML)
 
 	@node_modules/.bin/swagger-cli validate $(V4_YAML)
-	@node_modules/.bin/redoc-cli -t ./v4/html/ssr_template.hbs build ./v4/html/static/mattermost-openapi-v4.yaml -o ./v4/html/index.html --options.noAutoAuth  --options.suppressWarnings
+	@node_modules/.bin/redocly -t ./v4/html/ssr_template.hbs build-docs ./v4/html/static/mattermost-openapi-v4.yaml -o ./v4/html/index.html
 	@echo Complete
 
 node_modules: package.json $(wildcard package-lock.json)
@@ -72,7 +74,7 @@ node_modules: package.json $(wildcard package-lock.json)
 run:
 	@echo Starting redoc server
 
-	@node_modules/.bin/redoc-cli -t ./v4/html/ssr_template.hbs serve ./v4/html/static/mattermost-openapi-v4.yaml --options.noAutoAuth  --options.suppressWarnings  --ssr
+	@node_modules/.bin/redocly preview-docs ./v4/html/static/mattermost-openapi-v4.yaml
 
 clean:
 	@echo Cleaning