Add tests + arm build (#1)

* added .gitignore * tests * arm + test flow * added test flow * fixed runners * poetry * added curl * poetry run RLTest * python 3.8 in poetry * fixed yum matrix * fixed matrix runner in yum tests * python 3.10 * fail-fast is flase * build essential * added `--fix-missing --fix-broken` flags to Install deps * another attempt (explicit libc6 install) * reordering * install python from package manager * -y * curl * remove tls * yum * remove python-* from yum * poetry 1.5.1 for old python3 * yum updates * python3 + old poetry * un-poetry * typo * fixed requirements.txt * rltest remove version * pip3 install * call pip from python3 on yum * unable to find req file in yum * --user * --user in the right place * need to checkout first * order * python3 devel * tidy up * another attempt * attempt to include docker * cleanup * remove leftover poetry * implemented alternative layout * version bump-up * added tests alternative * update image * more cleanup * enabled back deployment * fast print system info * attempt to improve name * cleanup * return of the "if" * added ARM64 * removed ARM64 (for now) * shrink arm machine * increase a bit --------- Co-authored-by: GuyAv46 <[email protected]>
RedisLabsModules · Sep 21, 2023 · b5b6d5f · b5b6d5f
1 parent e19f782
commit b5b6d5f
Show file tree

Hide file tree

Showing 14 changed files with 667 additions and 96 deletions.
diff --git a/.github/workflows/common-container.yml b/.github/workflows/common-container.yml
@@ -0,0 +1,81 @@
+name: Common flow (container)
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        default: 'ubuntu-latest'
+        type: string
+      container:
+        required: true
+        type: string
+      os-nickname:
+        type: string
+      REDIS_REF:
+        type: string
+        default: '7.2'
+      deploy:
+        type: boolean
+        default: false
+      dependencies_command:
+        type: string
+      python-setup:
+        type: string
+
+
+jobs:
+  common-build:
+    runs-on: ${{ inputs.runner }}
+    container: ${{ inputs.container }}
+    steps:
+    - name: Install Deps
+      run: |
+        export DEBIAN_FRONTEND=noninteractive
+        ${{ inputs.dependencies_command }}
+    - uses: actions/checkout@v3
+    - uses: kenchan0130/actions-system-info@master
+      id: system-info
+    - name: Clone Redis
+      uses: actions/checkout@v3
+      with:
+        repository: 'redis/redis'
+        ref: ${{ inputs.REDIS_REF }}
+        path: redis
+    - name: Install Python
+      if: ${{ inputs.python-setup }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-setup }}
+    - name: Install Python Dependencies
+      run: pip3 install -r requirments.txt
+    - name: Install Redis Server
+      working-directory: redis
+      run: make -j `nproc` install
+    - name: Make
+      run: make
+    - name: test
+      run: RLTest --module bin/rediscompat.so
+    - name: Rename
+      if: ${{ inputs.deploy }}
+      run: mv ./bin/rediscompat.so
+        ${{ format('{0}.{1}-{2}.{3}.so',
+          './bin/rediscompat',
+          inputs.os-nickname || format('{0}-{1}', steps.system-info.outputs.name, steps.system-info.outputs.release),
+          runner.arch,
+          '1.0.0') }}
+    - name: Upload artifact
+      if: ${{ inputs.deploy }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: RedisStackStub
+        path: ./bin
+        retention-days: 1
+    - name: Upload S3
+      if: ${{ inputs.deploy }}
+      uses: shallwefootball/s3-upload-action@master
+      with:
+        aws_key_id: ${{ secrets.REDISLABSMODULES_ACCESS_KEY }}
+        aws_secret_access_key: ${{ secrets.REDISLABSMODULES_SECRET_ACCESS_KEY }}
+        aws_bucket: 'redismodules'
+        source_dir: ./bin
+        destination_dir: rediscompat
diff --git a/.github/workflows/common-flow.yml b/.github/workflows/common-flow.yml
@@ -0,0 +1,87 @@
+name: Common CI Workflow
+
+on:
+  workflow_call:
+    inputs:
+      deploy:
+        type: boolean
+        default: false
+      fast-fail:
+        type: boolean
+        default: true
+
+
+jobs:
+  ubuntu-build:
+    strategy:
+      fail-fast: ${{ inputs.fast-fail }}
+      matrix:
+        os: ["ubuntu:18.04", "ubuntu:20.04", "ubuntu:22.04", "debian:bullseye"]
+    secrets: inherit
+    uses: ./.github/workflows/common-container.yml
+    with:
+      container: ${{ matrix.os }}
+      deploy: ${{ inputs.deploy }}
+      dependencies_command: 'apt update && apt install make gcc build-essential python3 python3-pip curl -y'
+
+  ubuntu-build-arm:
+    strategy:
+      fail-fast: ${{ inputs.fast-fail }}
+      matrix:
+        os: ["ubuntu:18.04", "ubuntu:20.04", "ubuntu:22.04", "debian:bullseye"]
+    secrets: inherit
+    uses: ./.github/workflows/self-hosted-arm.yml
+    with:
+      container: ${{ matrix.os }}
+      deploy: ${{ inputs.deploy }}
+      dependencies_command: 'apt update && apt install make gcc build-essential python3 python3-pip curl -y'
+
+  yum-flavors:
+    strategy:
+      fail-fast: ${{ inputs.fast-fail }}
+      matrix:
+        include:
+          - os: "centos:centos7"
+            osnick: "centos7"
+          - os: "amazonlinux:2"
+            osnick: "amazonlinux"
+          - os: "rockylinux:8"
+            osnick: "rockylinux"
+    secrets: inherit
+    uses: ./.github/workflows/common-container.yml
+    with:
+      container: ${{ matrix.os }}
+      os-nickname: ${{ matrix.osnick }}
+      deploy: ${{ inputs.deploy }}
+      dependencies_command: 'yum install make gcc tar gzip curl python3 python3-devel -y'
+
+  yum-flavors-arm:
+    strategy:
+      fail-fast: ${{ inputs.fast-fail }}
+      matrix:
+        include:
+          - os: "centos:centos7"
+            osnick: "centos7"
+          - os: "amazonlinux:2"
+            osnick: "amazonlinux"
+          - os: "rockylinux:8"
+            osnick: "rockylinux"
+    secrets: inherit
+    uses: ./.github/workflows/self-hosted-arm.yml
+    with:
+      container: ${{ matrix.os }}
+      os-nickname: ${{ matrix.osnick }}
+      deploy: ${{ inputs.deploy }}
+      dependencies_command: 'yum install make gcc tar gzip curl python3 python3-devel -y'
+
+  mac-build:
+    strategy:
+      fail-fast: ${{ inputs.fast-fail }}
+      matrix:
+        runners: [macos-12]
+    secrets: inherit
+    uses: ./.github/workflows/common-runner.yml
+    with:
+      runner: ${{ matrix.runners }}
+      python-setup: '3.10'
+      deploy: ${{ inputs.deploy }}
diff --git a/.github/workflows/common-runner.yml b/.github/workflows/common-runner.yml
@@ -0,0 +1,77 @@
+name: Common flow (runnser)
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        required: true
+        type: string
+      os-nickname:
+        type: string
+      REDIS_REF:
+        type: string
+        default: '7.2'
+      deploy:
+        type: boolean
+        default: false
+      dependencies_command:
+        type: string
+      python-setup:
+        type: string
+
+
+jobs:
+  common-build:
+    runs-on: ${{ inputs.runner }}
+    steps:
+    - name: Install Deps
+      run: |
+        export DEBIAN_FRONTEND=noninteractive
+        ${{ inputs.dependencies_command }}
+    - uses: actions/checkout@v3
+    - uses: kenchan0130/actions-system-info@master
+      id: system-info
+    - name: Clone Redis
+      uses: actions/checkout@v3
+      with:
+        repository: 'redis/redis'
+        ref: ${{ inputs.REDIS_REF }}
+        path: redis
+    - name: Install Python
+      if: ${{ inputs.python-setup }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-setup }}
+    - name: Install Python Dependencies
+      run: pip3 install -r requirments.txt
+    - name: Install Redis Server
+      working-directory: redis
+      run: make -j `nproc` install
+    - name: Make
+      run: make
+    - name: test
+      run: RLTest --module bin/rediscompat.so
+    - name: Rename
+      if: ${{ inputs.deploy }}
+      run: mv ./bin/rediscompat.so
+        ${{ format('{0}.{1}-{2}.{3}.so',
+            './bin/rediscompat',
+            inputs.os-nickname || format('{0}-{1}', steps.system-info.outputs.name, steps.system-info.outputs.release),
+            runner.arch,
+            '1.0.0') }}
+    - name: Upload artifact
+      if: ${{ inputs.deploy }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: RedisStackStub
+        path: ./bin
+        retention-days: 1
+    - name: Upload S3
+      if: ${{ inputs.deploy }}
+      uses: shallwefootball/s3-upload-action@master
+      with:
+        aws_key_id: ${{ secrets.REDISLABSMODULES_ACCESS_KEY }}
+        aws_secret_access_key: ${{ secrets.REDISLABSMODULES_SECRET_ACCESS_KEY }}
+        aws_bucket: 'redismodules'
+        source_dir: ./bin
+        destination_dir: rediscompat
diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml
@@ -6,98 +6,9 @@ on:
   workflow_dispatch:
 
 jobs:
-  ubuntu-build:
-    strategy:
-      matrix:
-        os: ["ubuntu:18.04", "ubuntu:20.04", "ubuntu:22.04", "debian:bullseye"]
-    runs-on: ubuntu-latest
-    container: ${{ matrix.os }}
-    steps:
-    - name: apt update
-      run: apt-get update
-    - name: Install deps
-      run: apt-get install make gcc -y
-    - uses: actions/checkout@v3
-    - uses: kenchan0130/actions-system-info@master
-      id: system-info
-    - name: Make
-      run: make
-    - name: Rename
-      run: mv ./bin/rediscompat.so ${{format('{0}.{1}-{2}.{3}', './bin/rediscompat', steps.system-info.outputs.name, runner.arch, '1.0.0.so') }}
-    - name: Upload artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: RedisStackStub
-        path: ./bin
-        retention-days: 1
-    - name: Upload S3
-      uses: shallwefootball/s3-upload-action@master
-      with:
-        aws_key_id: ${{ secrets.REDISLABSMODULES_ACCESS_KEY }}
-        aws_secret_access_key: ${{ secrets.REDISLABSMODULES_SECRET_ACCESS_KEY }}
-        aws_bucket: 'redismodules'
-        source_dir: ./bin
-        destination_dir: rediscompat
-
-  yum-flavors:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          - os: "centos:centos7"
-            osnick: "centos7"
-          - os: "amazonlinux:2"
-            osnick: "amazonlinux"
-          - os: "rockylinux:8"
-            osnick: "rockylinux"
-    container: ${{ matrix.os }}
-    steps:
-    - name: Install deps
-      run: yum install make gcc tar gzip -y
-    - uses: actions/checkout@v3
-    - name: Make
-      run: make
-    - name: Rename
-      run: mv ./bin/rediscompat.so ${{format('{0}.{1}-{2}.{3}', './bin/rediscompat', matrix.osnick, runner.arch, '1.0.0.so') }}
-    - name: upload artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: Upload artifact
-        path: ./bin
-        retention-days: 1
-    - name: Upload S3
-      uses: shallwefootball/s3-upload-action@master
-      with:
-        aws_key_id: ${{ secrets.REDISLABSMODULES_ACCESS_KEY }}
-        aws_secret_access_key: ${{ secrets.REDISLABSMODULES_SECRET_ACCESS_KEY }}
-        aws_bucket: 'redismodules'
-        source_dir: ./bin
-        destination_dir: rediscompat
-
-  mac-build:
-    strategy:
-      matrix:
-        runners: [macos-12, ARM64]
-    runs-on: ${{ matrix.runners }}
-    steps:
-    - uses: actions/checkout@v3
-    - uses: kenchan0130/actions-system-info@master
-      id: system-info
-    - name: Make
-      run: make
-    - name: Rename
-      run: mv ./bin/rediscompat.so ${{format('{0}.{1}-{2}.{3}', './bin/rediscompat', steps.system-info.outputs.name, runner.arch, '1.0.0.so') }}
-    - name: upload artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: Upload artifact
-        path: ./bin
-        retention-days: 1
-    - name: Upload S3
-      uses: shallwefootball/s3-upload-action@master
-      with:
-        aws_key_id: ${{ secrets.REDISLABSMODULES_ACCESS_KEY }}
-        aws_secret_access_key: ${{ secrets.REDISLABSMODULES_SECRET_ACCESS_KEY }}
-        aws_bucket: 'redismodules'
-        source_dir: ./bin
-        destination_dir: rediscompat
+  run-deploy:
+    uses: ./.github/workflows/common-flow.yml
+    secrets: inherit
+    with:
+      deploy: true
+      fast-fail: true