[FEA] more docs, image cleanup, module renames

.dockerignore

@@ -4,11 +4,9 @@
 **/*.zip
 **/*.*.o
 **/*.arrow
-**/*.ipynb
 **/*.geojson
 **/*.parquet
 **/*.tar.gz
-**/yarn.lock
 **/lerna-debug.log
 **/docker-compose*.yml
 **/compile_commands.json

.env.sample

@@ -7,13 +7,13 @@ UID=1000
 PARALLEL_LEVEL=4
 
 # `nvidia/cudagl` base image CUDA version
-CUDA_VERSION=11.4.2
+# CUDA_VERSION=11.4.2
 
 # `nvidia/cudagl` base image Ubuntu version
-LINUX_VERSION=ubuntu20.04
+# LINUX_VERSION=ubuntu20.04
 
 # RAPIDS version to use
-RAPIDS_VERSION=21.12.00
+# RAPIDS_VERSION=21.12.00
 
 # How long sccache should wait until it considers a compile job timed out.
 # This number should be large, because C++ and CUDA can take a long time to compile.

.github/actions/build-and-publish-image/action.yml

@@ -0,0 +1,73 @@
+name: build-and-publish-image
+
+description: "Build and publish a Docker image"
+
+inputs:
+  push:
+    default: false
+    required: false
+    description: "Push the image to the container registry"
+  file:
+    required: true
+    description: "Dockerfile to build"
+  tags:
+    required: true
+    description: "Image tags to publish"
+  secret-files:
+    default: ""
+    required: false
+    description: "Optional file of secrets to mount"
+  platforms:
+    default: ""
+    required: false
+    description: "Optional platforms to build"
+  build-args:
+    default: ""
+    required: false
+    description: "Optional build arguments to use"
+
+outputs:
+  digest:
+    description: "Image content-addressable identifier"
+    value: ${{ steps.docker-build.outputs.digest }}
+  metadata:
+    description: "Build result metadata"
+    value: ${{ steps.docker-build.outputs.metadata }}
+
+runs:
+  using: composite
+  steps:
+    - name: Free up disk space
+      shell: bash
+      run: |
+        df -h
+        docker images
+        sudo swapoff -a
+        sudo rm -f /swapfile
+        sudo apt clean
+        sudo rm -rf /opt/ghc "$CONDA" \
+          /usr/share/swift \
+          /usr/share/dotnet \
+          /usr/local/lib/android \
+          /home/linuxbrew/.linuxbrew \
+          /opt/hostedtoolcache/CodeQL
+        docker rmi $(docker image ls -aq) || true
+        df -h
+        docker images
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        buildkitd-flags: --debug
+    - name: Build image
+      id: docker-build
+      uses: docker/build-push-action@v2
+      with:
+        context: .
+        push: ${{ inputs.push }}
+        file: ${{ inputs.file }}
+        tags: ${{ inputs.tags }}
+        platforms: ${{ inputs.platforms }}
+        build-args: ${{ inputs.build-args }}
+        secret-files: ${{ inputs.secret-files }}

.github/workflows/build-devel-main-image.yml

@@ -0,0 +1,51 @@
+name: Build devel main image
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-publish-devel-main-image:
+    name: Build devel main image
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: true
+      matrix:
+        CUDA: [11.0.3, 11.2.2, 11.4.2]
+        LINUX: [ubuntu18.04, ubuntu20.04]
+    env:
+      ARCH: amd64
+      NODE: 16.10.0
+      RAPIDS: 21.12.00
+      REPOSITORY: ghcr.io/rapidsai/node
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Login to container registry
+        uses: docker/login-action@v1
+        if: github.event_name == 'push'
+        with:
+          registry: ghcr.io
+          username: ${{ github.token }}
+          password: ${{ github.repository_owner }}
+      - name: Initialize sccache_credentials
+        run: |
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> /tmp/sccache_credentials
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> /tmp/sccache_credentials
+      - name: Build and push image
+        uses: ./.github/actions/build-and-publish-image
+        with:
+          push: ${{ github.event_name == 'push' }}
+          secret-files: |
+            "sccache_credentials=/tmp/sccache_credentials"
+          file: dockerfiles/devel/main.Dockerfile
+          tags: |
+            ${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cudagl${{ matrix.CUDA }}-${{ matrix.LINUX }}-main-${{ env.ARCH }}
+          build-args: |
+            "NODE_VERSION=${{ env.NODE }}"
+            "SCCACHE_IDLE_TIMEOUT=32768"
+            "SCCACHE_REGION=us-west-2"
+            "SCCACHE_BUCKET=node-rapids-sccache"
+            "FROM_IMAGE=nvidia/cudagl:${{ matrix.CUDA }}-devel-${{ matrix.LINUX }}"

.github/workflows/build-devel-package-image.yml

@@ -0,0 +1,54 @@
+name: Build devel packages image
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-publish-devel-packages-image:
+    needs:
+      - build-and-publish-devel-main-image
+    name: Build devel packages image
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: true
+      matrix:
+        CUDA: [11.0.3, 11.2.2, 11.4.2]
+        LINUX: [ubuntu18.04, ubuntu20.04]
+    env:
+      ARCH: amd64
+      NODE: 16.10.0
+      RAPIDS: 21.12.00
+      REPOSITORY: ghcr.io/rapidsai/node
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Login to container registry
+        uses: docker/login-action@v1
+        if: github.event_name == 'push'
+        with:
+          registry: ghcr.io
+          username: ${{ github.token }}
+          password: ${{ github.repository_owner }}
+      - name: Initialize sccache_credentials
+        run: |
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> /tmp/sccache_credentials
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> /tmp/sccache_credentials
+      - name: Build and push image
+        uses: ./.github/actions/build-and-publish-image
+        with:
+          push: ${{ github.event_name == 'push' }}
+          secret-files: |
+            "sccache_credentials=/tmp/sccache_credentials"
+          file: dockerfiles/devel/package.Dockerfile
+          tags: |
+            ${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cudagl${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages-${{ env.ARCH }}
+          build-args: |
+            "PARALLEL_LEVEL=1"
+            "SCCACHE_IDLE_TIMEOUT=32768"
+            "SCCACHE_REGION=us-west-2"
+            "SCCACHE_BUCKET=node-rapids-sccache"
+            "RAPIDS_VERSION=${{ env.RAPIDS }}"
+            "FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cudagl${{ matrix.CUDA }}-${{ matrix.LINUX }}-main-${{ env.ARCH }}"

.github/workflows/build-runtime-cuda-base-image.yml

@@ -0,0 +1,51 @@
+name: Build runtime cuda-base image
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-publish-runtime-cuda-base-image:
+    needs:
+      - build-and-publish-devel-main-image
+    name: Build runtime cuda-base image
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: true
+      matrix:
+        CUDA: [11.0.3, 11.2.2, 11.4.2]
+        LINUX: [ubuntu18.04, ubuntu20.04]
+    env:
+      ARCH: amd64
+      NODE: 16.10.0
+      RAPIDS: 21.12.00
+      REPOSITORY: ghcr.io/rapidsai/node
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Login to container registry
+        uses: docker/login-action@v1
+        if: github.event_name == 'push'
+        with:
+          registry: ghcr.io
+          username: ${{ github.token }}
+          password: ${{ github.repository_owner }}
+      - name: Initialize sccache_credentials
+        run: |
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> /tmp/sccache_credentials
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> /tmp/sccache_credentials
+      - name: Build and push image
+        uses: ./.github/actions/build-and-publish-image
+        with:
+          push: ${{ github.event_name == 'push' }}
+          secret-files: |
+            "sccache_credentials=/tmp/sccache_credentials"
+          file: dockerfiles/runtime/base.Dockerfile
+          tags: |
+            ${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base-${{ env.ARCH }}
+          build-args: |
+            "UID=1000"
+            "FROM_IMAGE=nvidia/cuda:${{ matrix.CUDA }}-runtime-${{ matrix.LINUX }}"
+            "DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cudagl${{ matrix.CUDA }}-${{ matrix.LINUX }}-main-${{ env.ARCH }}"

.github/workflows/build-runtime-cudagl-base-image.yml

@@ -0,0 +1,51 @@
+name: Build runtime cudagl-base image
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-publish-runtime-cudagl-base-image:
+    needs:
+      - build-and-publish-devel-main-image
+    name: Build runtime cudagl-base image
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: true
+      matrix:
+        CUDA: [11.0.3, 11.2.2, 11.4.2]
+        LINUX: [ubuntu18.04, ubuntu20.04]
+    env:
+      ARCH: amd64
+      NODE: 16.10.0
+      RAPIDS: 21.12.00
+      REPOSITORY: ghcr.io/rapidsai/node
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Login to container registry
+        uses: docker/login-action@v1
+        if: github.event_name == 'push'
+        with:
+          registry: ghcr.io
+          username: ${{ github.token }}
+          password: ${{ github.repository_owner }}
+      - name: Initialize sccache_credentials
+        run: |
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> /tmp/sccache_credentials
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> /tmp/sccache_credentials
+      - name: Build and push image
+        uses: ./.github/actions/build-and-publish-image
+        with:
+          push: ${{ github.event_name == 'push' }}
+          secret-files: |
+            "sccache_credentials=/tmp/sccache_credentials"
+          file: dockerfiles/runtime/base.Dockerfile
+          tags: |
+            ${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cudagl${{ matrix.CUDA }}-${{ matrix.LINUX }}-base-${{ env.ARCH }}
+          build-args: |
+            "UID=1000"
+            "FROM_IMAGE=nvidia/cudagl:${{ matrix.CUDA }}-runtime-${{ matrix.LINUX }}"
+            "DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cudagl${{ matrix.CUDA }}-${{ matrix.LINUX }}-main-${{ env.ARCH }}"

.github/workflows/build-runtime-cudf-image.yml

@@ -0,0 +1,51 @@
+name: Build runtime cudf image
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-publish-runtime-cudf-image:
+    needs:
+      - build-and-publish-devel-packages-image
+      - build-and-publish-runtime-cuda-base-image
+    name: Build runtime cudf image
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: true
+      matrix:
+        CUDA: [11.0.3, 11.2.2, 11.4.2]
+        LINUX: [ubuntu18.04, ubuntu20.04]
+    env:
+      ARCH: amd64
+      NODE: 16.10.0
+      RAPIDS: 21.12.00
+      REPOSITORY: ghcr.io/rapidsai/node
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Login to container registry
+        uses: docker/login-action@v1
+        if: github.event_name == 'push'
+        with:
+          registry: ghcr.io
+          username: ${{ github.token }}
+          password: ${{ github.repository_owner }}
+      - name: Initialize sccache_credentials
+        run: |
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> /tmp/sccache_credentials
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> /tmp/sccache_credentials
+      - name: Build and push image
+        uses: ./.github/actions/build-and-publish-image
+        with:
+          push: ${{ github.event_name == 'push' }}
+          secret-files: |
+            "sccache_credentials=/tmp/sccache_credentials"
+          file: dockerfiles/runtime/cudf.Dockerfile
+          tags: |
+            ${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-cudf-${{ env.ARCH }}
+          build-args: |
+            "FROM_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-runtime-node${{ env.NODE }}-cuda${{ matrix.CUDA }}-${{ matrix.LINUX }}-base-${{ env.ARCH }}"
+            "DEVEL_IMAGE=${{ env.REPOSITORY }}:${{ env.RAPIDS }}-devel-node${{ env.NODE }}-cudagl${{ matrix.CUDA }}-${{ matrix.LINUX }}-packages-${{ env.ARCH }}"