Release/0.11.0 🎉

.github/workflows/compliance-test.yaml

@@ -0,0 +1,135 @@
+# Workflow for running the TES compliance suite against Funnel
+
+# This includes the following steps:
+#  1. Build Funnel and store the resulting binary artifact
+#  2. Install tes-compliance-suite and run against every version of TES simultaneously
+#  3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish
+#     the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/
+
+# Optionally debug via SSH
+# Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability
+#
+# To use this step uncomment and place anywhere in the build steps. The build will pause on this step and
+# output a ssh address associated with the Github action worker. Helpful for debugging build steps and
+# and intermediary files/artifacts.
+#
+# - name: "Debug: Package dependancies for tmate (CentOS)"
+#   run: |
+#     yum install -y xz
+#     ln -s /bin/true /bin/apt-get
+#
+# - name: Setup tmate session
+#   uses: mxschmitt/action-tmate@v3
+
+name: Compliance Test
+
+on:
+  push:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: quay.io/ohsu-comp-bio/slurm
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.21
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Build
+      run: make build
+
+    - name: Store funnel
+      uses: actions/upload-artifact@v2
+      with:
+        name: funnelBin
+        path: funnel
+
+  compliance:
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [1.0.0, 1.1.0]
+        db: ["boltdb", "mongodb"]
+        compute: ["local", "slurm"]
+    needs: build
+    runs-on: ubuntu-latest
+    container:
+      image: quay.io/ohsu-comp-bio/slurm
+      options: --hostname slurmctl --cap-add sys_admin
+    steps:
+      # Required to access the 'tests/mongo.config.yml' file
+      # Perhaps uploading it as an artifact would be more efficient?
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: funnelBin
+
+      - name: Start Funnel server
+        run: |
+          touch config.yml
+          if [ ${{ matrix.db }} = "mongodb" ]; then
+            make start-mongodb
+            cat `pwd`/tests/mongo.config.yml >> config.yml
+            # Required for Funnel to connect MongoDB
+            echo "172.17.0.1 localhost" >> /etc/hosts
+          elif [ ${{ matrix.compute }} = "slurm" ]; then
+            cat `pwd`/tests/slurm.config.yml >> config.yml
+            cp config.yml /opt/funnel_config.yml
+            # Start Slurm
+            /usr/local/bin/docker-entrypoint.sh
+          fi
+          chmod +x funnel
+          FLAGS="--config `pwd`/config.yml"
+          ./funnel server run $FLAGS &> funnel.logs &
+
+      - name: Run OpenAPI Test Runner
+        run: |
+          # Clone the 'upstream' OpenAPI Test Runner when PR #65 is merged
+          # https://github.com/elixir-cloud-aai/openapi-test-runner/pull/65
+          # git clone https://github.com/elixir-cloud-aai/openapi-test-runner
+          git clone https://github.com/ohsu-comp-bio/openapi-test-runner -b fix/create-and-filter-task
+
+          cd openapi-test-runner
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+          python setup.py install  
+          openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/"
+
+      - name: Install TES compliance suite
+        run: |
+          git clone https://github.com/lbeckman314/tes-compliance-suite -b feature/tesv1.1
+          cd tes-compliance-suite
+          /root/.pyenv/shims/python3 -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+          python setup.py install
+          mkdir reports
+
+      - name: Test compliance (Report Publishing)
+        run: |
+          cd tes-compliance-suite
+          source venv/bin/activate
+          tes-compliance-suite report --version "${{ matrix.version }}" --server "http://localhost:8000/"
+
+  start-report-deployment:
+    needs: compliance
+    runs-on: ubuntu-latest
+    steps:
+      # https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#create-a-repository-dispatch-event
+      - name: Start report generation
+        uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_AUTHOR: "Funnel"
+          PAYLOAD_REVISION: "3"
+        with:
+          repository: ohsu-comp-bio/funnel-compliance
+          event: start-report
+          github_pat: ${{ secrets.ACTIONS_TOKEN }}
+

.github/workflows/nextflow.yaml

@@ -0,0 +1,67 @@
+name: Nextflow Test
+
+on:
+  push:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.21
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Build Funnel (if cache does not exist)
+      run: make build
+
+    - name: Store Funnel
+      uses: actions/upload-artifact@v2
+      with:
+        name: funnelBin
+        path: funnel
+
+  nextflow:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+    - name: Download Funnel
+      uses: actions/download-artifact@v2
+      with:
+        name: funnelBin
+        path: funnel
+
+    - name: Start Funnel
+      working-directory: 
+      run: |
+        cd funnel/
+        chmod +x ./funnel
+        ./funnel server --LocalStorage.AllowedDirs $HOME run &
+
+    - name: Install Nextflow
+      run: |
+        cd ..
+        git clone https://github.com/nextflow-io/nextflow/ -b tes-update-1.1
+        cd nextflow
+        make compile
+
+    - name: Install nf-canary and GA4GH-TES plugin
+      run: |
+        cd ..
+        git clone https://github.com/seqeralabs/nf-canary
+        cd nf-canary
+        cat <<EOF >> nextflow.config
+          plugins {
+            id 'nf-ga4gh'
+          }
+          process.executor = 'tes'
+          tes.endpoint = 'http://localhost:8000'
+        EOF
+
+    - name: Run nf-canary tests
+      run: |
+        cd ../nf-canary
+        ../nextflow/launch.sh run main.nf

.github/workflows/s3-test.yaml

@@ -0,0 +1,48 @@
+# Credit: rhnvrm
+# Adapted from: https://rohanverma.net/blog/2021/02/09/minio-github-actions/
+
+name: S3 Integration Test
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  s3Test-integration:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup minio
+        run: |
+          docker run -d -p 9000:9000 --name minio \
+                     -e "MINIO_ROOT_USER=minioadmin" \
+                     -e "MINIO_ROOT_PASSWORD=minioadmin" \
+                     -v /tmp/data:/data \
+                     -v /tmp/config:/root/.minio \
+                     minio/minio server /data
+
+          cat <<EOF > funnel.config.yml
+          LocalStorage:
+            Disabled: true
+          AmazonS3:
+            Disabled: true
+          GoogleStorage:
+            Disabled: true
+          HTTPStorage:
+            Disabled: true
+          FTPStorage:
+            Disabled: true
+          GenericS3:
+            - Disabled: false
+              Endpoint: "localhost:9000"
+              Key: "minioadmin"
+              Secret: "minioadmin"
+          EOF
+
+          wget https://github.com/ohsu-comp-bio/funnel/releases/download/untagged-217841a99d14ccfe289c/funnel-darwin-arm64-0.11.0.tar.gz
+          tar -zxvf funnel-darwin-arm64-0.11.0.tar.gz
+          chmod +x funnel
+          ./funnel server run --config funnel.config.yml &
+          ./funnel task run examples/s3-test.yml

.github/workflows/tests.yaml

@@ -1,36 +1,36 @@
-name: Go
+name: Go Tests
 
-on: [ pull_request ]
+on:
+  push:
 
 jobs:
-
   lint:
     name: lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.21
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.52.2
-          args: --timeout 3m --verbose
+          version: latest
+          args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple -D ineffassign
 
   build:
     runs-on: ubuntu-latest
     steps:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.18
+        go-version: 1.21
 
     - name: Check out code
       uses: actions/checkout@v2
 
     - name: Build
-      run: go build -v ./
+      run: make build
 
     - name: Store funnel
       uses: actions/upload-artifact@v2
@@ -44,23 +44,30 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.18
+        go-version: 1.21
     - name: Check out code
       uses: actions/checkout@v2
+
     - name: Unit Tests
-      run: make test
+      run: make test-verbose
 
   mongoTest:
     runs-on: ubuntu-latest
     needs: build
     steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.21
+
     - name: Check out code
       uses: actions/checkout@v2
 
     - name: Download funnel bin
       uses: actions/download-artifact@v2
       with:
         name: funnelBin
+
     - name: MongoTest
       run: |
         chmod +x funnel
@@ -72,6 +79,10 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.21
     - name: Check out code
       uses: actions/checkout@v2
 
@@ -83,3 +94,48 @@ jobs:
       run: |
         chmod +x funnel
         make test-badger
+
+  slurmTest:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.21
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Download funnel bin
+      uses: actions/download-artifact@v2
+      with:
+        name: funnelBin
+
+    - name: Slurm Test
+      run: |
+        chmod +x funnel
+        make test-slurm
+
+  s3Test:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.21
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Download funnel bin
+      uses: actions/download-artifact@v2
+      with:
+        name: funnelBin
+
+    - name: S3 Test
+      run: |
+        chmod +x funnel
+        make start-generic-s3
+        sleep 10
+        make test-generic-s3
+

.gitignore

@@ -21,3 +21,5 @@ docker/funnel
 deployments/kubernetes/**/funnel
 deployments/kubernetes/**/*.json
 funnel
+dist/
+.vscode