WIP: python3.10/11, opensearch, invenio upgrades

.github/workflows/e2e-workflow-server.yml

@@ -2,7 +2,7 @@ name: E2E - Server changes with python package installations
 
 on:
   pull_request:
-    branches: [master, ant, faser]
+    branches: [master, next]
     paths:
       - "setup.py"
       - "requirements.txt"

.github/workflows/e2e-workflow-ui.yml

@@ -3,7 +3,7 @@ on:
   push:
     branches: uitest
   pull_request:
-    branches: [master, ant, faser]
+    branches: [master, next]
     paths:
       - "ui/**"
       - ".github/workflows/ui-test.yml"

.github/workflows/e2e-workflow.yml

@@ -5,11 +5,12 @@ on:
       - master
       - qa
       - production
+      - next
   pull_request:
     branches:
       - qa
       - production
-      - faser
+      - next
 jobs:
   Cypress:
     name: Cypress
@@ -25,7 +26,7 @@ jobs:
         run: docker-compose -f docker-compose.e2e.yml run web-api sh scripts/clean-and-init.sh
 
       - name: Update save folder
-        run: docker-compose -f docker-compose.e2e.yml run web-api  cap files location local var/data --default
+        run: docker-compose -f docker-compose.e2e.yml run web-api cap files location local var/data --default
 
       - name: Sleep for 180 seconds
         run: sleep 180s

.github/workflows/harbour-build-ui.yml

@@ -2,14 +2,13 @@ name: Build and export CAP UI image
 
 on:
   push:
-    branches: ['master', 'dev', 'qa', 'test']
-    paths:
-      - "ui/**"
-      - ".github/workflows/ui-test.yml"
+    branches: ["master", "dev", "qa", "test", "with-formule"]
+    # paths:
+    #   - "ui/**"
+    #   - ".github/workflows/ui-test.yml"
 
 jobs:
   build-image:
-    needs: UITest
     name: Build and export ui image
     runs-on: ubuntu-20.04
     outputs:
@@ -23,6 +22,10 @@ jobs:
         run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
         id: get_branch
 
+      - name: Extract short commit SHA
+        id: get_short_sha
+        run: echo "SENTRY_RELEASE=$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_OUTPUT
+
       - id: build-ui
         name: Build and export
         uses: cern-sis/gh-workflows/.github/actions/[email protected]
@@ -36,46 +39,8 @@ jobs:
           username: ${{ secrets.HARBOR_USERNAME }}
           password: ${{ secrets.HARBOR_PASSWORD }}
           build-args: |
-            BRANCH_NAME=${{ steps.get_branch.outputs.BRANCH_NAME }}
-            PIWIK_ENV=${{ steps.get_branch.outputs.BRANCH_NAME }}
-            CAP_PIWIK_URL=${{ secrets.CAP_PIWIK_URL }}
-            CAP_PIWIK_SITEID_DEV=${{ secrets.CAP_PIWIK_SITEID_DEV }}
-            CAP_PIWIK_SITEID_PROD=${{ secrets.CAP_PIWIK_SITEID_PROD }}
-            CAP_PIWIK_SITEID_QA=${{ secrets.CAP_PIWIK_SITEID_QA }}
-            CAP_PIWIK_SITEID_TEST=${{ secrets.CAP_PIWIK_SITEID_TEST }}
-
-      - id: build-ui-e2e
-        name: Build and export for end to end image
-        uses: cern-sis/gh-workflows/.github/actions/[email protected]
-        if: ${{ steps.get_branch.outputs.BRANCH_NAME }} == 'qa'
-        with:
-          dockerfile: ./docker/harbour/ui/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          image: cern-sis/cap-ui
-          registry: registry.cern.ch
-          cache: false
-          tags: qa-e2e
-          username: ${{ secrets.HARBOR_USERNAME }}
-          password: ${{ secrets.HARBOR_PASSWORD }}
-          build-args: |
-            BRANCH_NAME=${{ steps.get_branch.outputs.BRANCH_NAME }}
-            ENABLE_E2E=True
-            PIWIK_ENV=${{ steps.get_branch.outputs.BRANCH_NAME }}
-            CAP_PIWIK_URL=${{ secrets.CAP_PIWIK_URL }}
-            CAP_PIWIK_SITEID_QA=${{ secrets.CAP_PIWIK_SITEID_QA }}
-
-  deploy:
-    needs: build-image
-    name: update-image
-    runs-on: ubuntu-20.04
-
-    steps:
-      - name: send event
-        uses: cern-sis/gh-workflows/.github/actions/[email protected]
-        with:
-          event-type: update
-          token: ${{ secrets.PAT_FIRE_EVENTS_ON_CERN_SIS_KUBERNETES_CAP }}
-          repo: cern-sis/kubernetes-cap
-          images: |
-            cern-sis/cap-ui@{{ needs.build-image.outputs.image-id }}
-
+            SENTRY_URL=${{ secrets.CAP_SENTRY_URL }}
+            SENTRY_ORG=${{ secrets.CAP_SENTRY_ORG }}
+            SENTRY_PROJECT=${{ secrets.CAP_SENTRY_PROJECT }}
+            SENTRY_AUTH_TOKEN=${{ secrets.CAP_SENTRY_AUTH_TOKEN }}
+            SENTRY_RELEASE=${{ steps.get_short_sha.outputs.SENTRY_RELEASE }}

.github/workflows/lint-commit.yml

@@ -2,7 +2,7 @@ name: Lint Commit
 
 on:
   pull_request:
-    branches: [ master ]
+    branches: [ master, next ]
 
 jobs:
   lint-shellcheck:
@@ -28,7 +28,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Checking commit quality (messages, signatures, spelling checks, etc)
         env:
@@ -44,7 +44,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Check Python manifest completeness
         run: |
@@ -59,7 +59,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Check compliance with Python docstring conventions
         run: |

.github/workflows/lint-push.yml

@@ -2,7 +2,7 @@ name: Lint Source
 
 on:
   push:
-    branches: [ master ]
+    branches: [master, next]
 
 jobs:
   lint-shellcheck:
@@ -23,7 +23,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: Check compliance with Python docstring conventions
         run: |
@@ -38,7 +38,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: Check compliance with pep8, pyflakes and circular complexity
         run: |
@@ -55,7 +55,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: Run black linter for python
         run: |
@@ -70,7 +70,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: isort your imports, so you don't have to.
         run: |
@@ -85,7 +85,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2-beta
         with:
-          node-version: "10"
+          node-version: "20"
 
       - name: Run prettier
         run: |
@@ -100,7 +100,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: Check Python manifest completeness
         run: |
@@ -120,7 +120,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: Checking commit quality (messages, signatures, spelling checks, etc)
         env:

.github/workflows/ui-test.yml

@@ -1,12 +1,12 @@
 name: UI test
 on:
   push:
-    branches: [master, ant]
+    branches: [master, next]
     paths:
       - "ui/**"
       - ".github/workflows/ui-test.yml"
   pull_request:
-    branches: [master, ant, faser]
+    branches: [master, next]
     paths:
       - "ui/**"
       - ".github/workflows/ui-test.yml"
@@ -20,7 +20,7 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2-beta
         with:
-          node-version: "14"
+          node-version: "20"
 
       - name: Build
         working-directory: ./ui

.github/workflows/web-api.yml

@@ -2,13 +2,13 @@ name: Backend tests
 
 on:
   push:
-    branches: [ master, ant ]
+    branches: [master, next]
     paths-ignore:
-    - 'ui/**'
+      - "ui/**"
   pull_request:
-    branches: [ master, ant, faser ]
+    branches: [master, next]
     paths-ignore:
-    - 'ui/**'
+      - "ui/**"
 
 jobs:
   lint-python-changes:
@@ -19,10 +19,11 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: Run darker
         run: |
+          pip install flake8
           pip install darker[isort]
           darker . --check -i -L flake8
 
@@ -40,40 +41,44 @@ jobs:
           POSTGRES_PASSWORD: cap
           POSTGRES_DB: cap
         ports:
-        - 5432:5432
+          - 5432:5432
       rabbitmq:
         image: rabbitmq:3-management
         ports:
           - 5672:5672
-      elasticsearchd:
-        image: docker.elastic.co/elasticsearch/elasticsearch:5.6.4
+      search:
+        image: opensearchproject/opensearch:2.2.0
         env:
+          node.name: search
           bootstrap.memory_lock: true
-          ES_JAVA_OPTS: -Xms1024m -Xmx1024m
+          OPENSEARCH_JAVA_OPTS: -Xms1024m -Xmx1024m
           discovery.type: single-node
-          xpack.security.enabled: false
+          discovery.seed_hosts: search
+          DISABLE_SECURITY_PLUGIN: true
         ports:
           - 9200:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
 
     steps:
       - uses: actions/checkout@v2
 
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: Install system dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev libkrb5-dev
+          sudo apt-get install libsasl2-dev python3-dev libldap2-dev libssl-dev libkrb5-dev
 
       - name: Install dependencies
         run: |
-          pip install --upgrade pip==20.2.4
-          pip install -r requirements.txt
-          pip install -e .[all,docs,tests,ldap]
-          pip install -r requirements-local-forks.txt
+          pip install .
 
       - name: Run integration and unit tests
         run: ./run-tests.sh --check-pytest

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
         args: ["--config=.flake8"]
 
   - repo: https://github.com/pycqa/isort
-    rev: "5.10.1"
+    rev: "5.12.0"
     hooks:
       - id: isort