Merge master into 4.0 branch

.github/dependabot.yml

@@ -3,6 +3,7 @@
 
 version: 2
 updates:
+# For openrefine java deps
 - package-ecosystem: maven
   directory: "/"
   schedule:
@@ -13,11 +14,18 @@ updates:
     versions:
     - "> 1.4.12"
     - "< 2"
+# For documentation website
 - package-ecosystem: "npm" # For Yarn
-  directory: "/"
+  directory: "docs/"
   schedule:
     interval: "daily"
+# For github actions
 - package-ecosystem: "github-actions"
   directory: "/"
   schedule:
     interval: "daily"
+# For cypress test_suite
+- package-ecosystem: "npm"
+  directory: "main/tests/cypress"
+  schedule:
+    interval: "daily"

.github/workflows/pull_request.yml

@@ -1,9 +1,14 @@
-name: Java CI
+name: Continuous Integration
 
-on: [pull_request_target]
+on:
+  pull_request_target:
+    paths-ignore:
+      - 'docs/**'
+
+permissions: read-all
 
 jobs:
-  build:
+  server_tests:
     strategy:
       matrix:
         java: [ 8, 14 ]
@@ -38,18 +43,22 @@ jobs:
 
     steps:
     - uses: actions/[email protected]
+      with:
+        ref: ${{ github.event.pull_request.head.ref }}
+        repository: ${{ github.event.pull_request.head.repo.full_name }}
 
     - name: Restore dependency cache
-      uses: actions/[email protected].3
+      uses: actions/[email protected].6
       with:
         path: ~/.m2/repository
         key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
             ${{ runner.os }}-maven-
 
     - name: Set up Java ${{ matrix.java }}
-      uses: actions/setup-java@v1
+      uses: actions/setup-java@v2
       with:
+        distribution: 'adopt'
         java-version: ${{ matrix.java }}
 
     - name: Configure connections to databases
@@ -72,48 +81,93 @@ jobs:
       run: |
         mvn prepare-package -DskipTests=true
         mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }}
-  cypress_tests:
-    strategy:
-      matrix:
-          browser: ['chrome']
 
+  prepare_ui_test_matrix:
     runs-on: ubuntu-latest
-
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-    - uses: actions/[email protected]
-
-    - name: Restore dependency cache
-      uses: actions/[email protected]
-      with:
+      - uses: actions/[email protected]
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+      - id: set-matrix
+        run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
+        env:
+          browsers: chrome
+  ui_test:
+    needs: prepare_ui_test_matrix
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
+    steps:
+      - uses: actions/[email protected]
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+      - name: Restore dependency cache
+        uses: actions/[email protected]
+        with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-              ${{ runner.os }}-maven-
+            ${{ runner.os }}-maven-
 
-    - name: Set up Java 8
-      uses: actions/setup-java@v1
-      with:
+      - name: Set up Java 8
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
           java-version: 8
 
-    - name: Build OpenRefine
-      run: ./refine build
-
-    - name: Restore Tests dependency cache
-      uses: actions/[email protected]
-      with:
-          path: '**/node_modules'
+      - name: Build OpenRefine
+        run: ./refine build
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+
+      - name: Install Edge
+        if: matrix.browser == 'edge'
+        run: |
+          sudo curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
+          sudo install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/
+          sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge-dev.list'
+          sudo rm microsoft.gpg
+          sudo apt-get update
+          sudo apt-get install microsoft-edge-dev
+
+      - name: Restore Tests dependency cache
+        uses: actions/[email protected]
+        with:
+          path: |
+            ~/cache
+            ~/.cache
+            **/node_modules
+            !~/cache/exclude
           key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
           restore-keys: |
-              ${{ runner.os }}-yarn
+            ${{ runner.os }}-yarn
 
-    - name: Install test dependencies
-      run: |
+      - name: Install test dependencies
+        run: |
           cd ./main/tests/cypress
+          npm i -g yarn
           yarn install
 
-    - name: Test with Cypress on ${{ matrix.browser }}
-      run: | 
-        echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
-        echo REFINE_MEMORY=4096M >> ./refine.ini
-        ./refine ui_test ${{ matrix.browser }} s5du3k "${{ secrets.CYPRESS_RECORD_KEY }}"
-
+      - name: Test with Cypress on ${{ matrix.browser }}
+        run: | 
+          echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
+          echo REFINE_MEMORY=4096M >> ./refine.ini
+          ./refine ui_tests
+        env:
+          CYPRESS_BROWSER: ${{ matrix.browser }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+          CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
+          CYPRESS_SPECS: ${{ matrix.specs }}

.github/workflows/release_manager.sh

@@ -2,7 +2,7 @@
 
 
 # Build Release Message
-API_JSON=$(printf '{"tag_name": "%s","target_commitish": "master","name": "OpenRefine snapshot %s","body": "This is a snapshot of the development version of OpenRefine, made on %s.\\n\\nThis contains the latest new features and bug fixes, but might not have been tested as thoroughly as official releases.\\n\\nMake sure you [back up your workspace](https://github.com/OpenRefine/OpenRefine/wiki/Back-Up-OpenRefine-Data) to avoid data loss and report any issues found with this version on [the mailing list](https://groups.google.com/forum/#!forum/openrefine).","draft": false,"prerelease": true}' ${OR_VERSION} ${OR_VERSION} "$(date -u +"%c")" )
+API_JSON=$(printf '{"tag_name": "%s","target_commitish": "master","name": "OpenRefine snapshot %s","body": "This is a snapshot of the development version of OpenRefine, made on %s.\\n\\nThis contains the latest new features and bug fixes, but might not have been tested as thoroughly as official releases.\\n\\nMake sure you [back up your workspace](https://docs.openrefine.org/manual/installing#back-up-your-data) to avoid data loss and report any issues found with this version on [the mailing list](https://groups.google.com/forum/#!forum/openrefine).","draft": false,"prerelease": true}' ${OR_VERSION} ${OR_VERSION} "$(date -u +"%c")" )
 
 # Create Release
-echo $( curl --silent --data "${API_JSON}" "https://api.github.com/repos/${RELEASE_REPO_OWNER}/OpenRefine-snapshot-releases/releases?access_token=${RELEASE_REPO_TOKEN}" | jq -r '.upload_url' )
+echo $( curl --silent -H "Authorization: token ${RELEASE_REPO_TOKEN}" --data "${API_JSON}" "https://api.github.com/repos/${RELEASE_REPO_OWNER}/OpenRefine-snapshot-releases/releases" | jq -r '.upload_url' )

.github/workflows/snapshot_release.yml

@@ -3,63 +3,101 @@ name: Snapshot release
 on:
   push:
     branches:
-      - master
+      - 'master'
+    paths-ignore:
+      - 'docs/**'
 
 jobs:
-  cypress_tests:
+  prepare_ui_test_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/[email protected]
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+      - id: set-matrix
+        run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
+        env:
+          browsers: chrome,edge
+  ui_test:
+    needs: prepare_ui_test_matrix
     runs-on: ubuntu-latest
-
     strategy:
-      matrix:
-          browser: ['edge', 'chrome']
-
+      matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
     steps:
-    - uses: actions/[email protected]
-
-    - name: Restore dependency cache
-      uses: actions/[email protected]
-      with:
+      - uses: actions/[email protected]
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+      - name: Restore dependency cache
+        uses: actions/[email protected]
+        with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-              ${{ runner.os }}-maven-
+            ${{ runner.os }}-maven-
 
-    - name: Set up Java 8
-      uses: actions/setup-java@v1
-      with:
+      - name: Set up Java 8
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
           java-version: 8
 
-    - name: Install Edge
-      if: matrix.browser == 'edge'
-      run: | 
-        sudo curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
-        sudo install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/
-        sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge-dev.list'
-        sudo rm microsoft.gpg
-        sudo apt-get update
-        sudo apt-get install microsoft-edge-dev
-
-    - name: Build OpenRefine
-      run: ./refine build
-
-    - name: Restore Tests dependency cache
-      uses: actions/[email protected]
-      with:
-          path: '**/node_modules'
+      - name: Build OpenRefine
+        run: ./refine build
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+
+      - name: Install Edge
+        if: matrix.browser == 'edge'
+        run: |
+          sudo curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
+          sudo install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/
+          sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge-dev.list'
+          sudo rm microsoft.gpg
+          sudo apt-get update
+          sudo apt-get install microsoft-edge-dev
+
+      - name: Restore Tests dependency cache
+        uses: actions/[email protected]
+        with:
+          path: |
+            ~/cache
+            ~/.cache
+            **/node_modules
+            !~/cache/exclude
           key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
           restore-keys: |
-              ${{ runner.os }}-yarn
+            ${{ runner.os }}-yarn
 
-    - name: Install test dependencies
-      run: |
+      - name: Install test dependencies
+        run: |
           cd ./main/tests/cypress
+          npm i -g yarn
           yarn install
 
-    - name: Test with Cypress on ${{ matrix.browser }}
-      run: | 
-        echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
-        echo REFINE_MEMORY=4096M >> ./refine.ini
-        ./refine ui_test ${{ matrix.browser }} s5du3k "${{ secrets.CYPRESS_RECORD_KEY }}"
+      - name: Test with Cypress on ${{ matrix.browser }}
+        run: | 
+          echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
+          echo REFINE_MEMORY=4096M >> ./refine.ini
+          ./refine ui_tests
+        env:
+          CYPRESS_BROWSER: ${{ matrix.browser }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+          CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
+          CYPRESS_SPECS: ${{ matrix.specs }}
+
 
   build:
 
@@ -97,17 +135,18 @@ jobs:
         fetch-depth: 0 # This is wasteful, but needed for git describe
 
     - name: Restore dependency cache
-      uses: actions/[email protected].3
+      uses: actions/[email protected].6
       with:
         path: ~/.m2/repository
         key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
             ${{ runner.os }}-maven-
 
-    - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
+    - name: Set up Java 8
+      uses: actions/setup-java@v2
       with:
-        java-version: 1.8
+        distribution: 'adopt'
+        java-version: 8
 
     - name: Install genisoimage and jq
       run: sudo apt-get install genisoimage jq
@@ -190,7 +229,7 @@ jobs:
     - name: Delete older releases
       id: delete-older-releases
       if: github.repository == 'OpenRefine/OpenRefine'
-      uses: dev-drprasad/delete-older-releases@v0.1.0
+      uses: dev-drprasad/delete-older-releases@v0.2.0
       with:
         repo: OpenRefine/OpenRefine-snapshot-releases
         # Specifies number of latest releases (sorted by created_at) to keep. Pass 0 if you want to delete all releases

CONTRIBUTING.md

@@ -31,9 +31,11 @@ All developers including new distributions and plugin developers are invited to
 
 ### How to submit PR's (pull requests), patches, and bug fixes
 
+- Read [Your first pull request](https://github.com/OpenRefine/OpenRefine/wiki/Your-first-pull-request)
 - Avoid merging master in your branch because it makes code review a lot harder.
 - If you want to keep your branch up to date with our master, it would be nicer if you could just rebase your branch instead. That would keep the history a lot cleaner.
 - Please avoid adding unrelated changes in the PR. Do a separate PR and rebase once they get merged can work really well.
+- It is important that pull requests are used systematically, even by those who have the rights to merge them.
 
 If you make trivial changes, you can send them directly via a pull request. **Please make your changes in a new git branch and send your patch**, including appropriate test cases.