[tokenizers] Return score for QA inference (#3555) #280
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Native S3 Huggingface tokenizers | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - 'master' | |
| paths: | |
| - extensions/tokenizers/rust/** | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| build-tokenizers-jni-linux: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amazonlinux:2 | |
| env: | |
| JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto | |
| steps: | |
| - name: Install Environment | |
| run: | | |
| yum -y groupinstall "Development Tools" | |
| yum -y install patch perl-IPC-Cmd cmake3 curl jq | |
| yum -y install java-17-amazon-corretto-devel | |
| ln -s /usr/bin/cmake3 /usr/bin/cmake | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| pip3 install awscli wheel setuptools --upgrade | |
| - uses: taiki-e/checkout-action@v1 | |
| - name: Release JNI prep | |
| run: | | |
| source "$HOME/.cargo/env" | |
| ./gradlew :extensions:tokenizers:compileJNI | |
| PYTORCH_PRECXX11=true ./gradlew -Pjni :extensions:tokenizers:test | |
| - name: Build djl-converter wheel | |
| working-directory: extensions/tokenizers/src/main/python/ | |
| run: ./setup.py bdist_wheel | |
| - name: Configure AWS Credentials | |
| run: | | |
| oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ | |
| "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value") | |
| echo "::add-mask::$oidc_token" | |
| read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \ | |
| --region "us-east-2" \ | |
| --role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \ | |
| --role-session-name "build-tokenizers-jni-linux" \ | |
| --web-identity-token "$oidc_token" \ | |
| --query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \ | |
| --output text)" | |
| echo "::add-mask::$AWS_ACCESS_KEY_ID" | |
| echo "::add-mask::$AWS_SECRET_ACCESS_KEY" | |
| echo "::add-mask::$AWS_SESSION_TOKEN" | |
| echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV" | |
| echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV" | |
| echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV" | |
| - name: Copy files to S3 with the AWS CLI | |
| run: | | |
| DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml) | |
| TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)" | |
| aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/linux-x86_64/cpu s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/linux-x86_64/cpu/ | |
| aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*" | |
| aws s3 cp extensions/tokenizers/src/main/python/dist/djl_converter-*.whl s3://djl-ai/publish/djl_converter/ | |
| aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/djl_converter/*" | |
| build-tokenizer-jni-windows: | |
| runs-on: windows-latest | |
| steps: | |
| - name: Install Environment | |
| run: | | |
| choco install -y mingw | |
| - uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 17 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Release CPU JNI | |
| shell: cmd | |
| run: | | |
| call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 | |
| gradlew :extensions:tokenizers:compileJNI | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role | |
| aws-region: us-east-2 | |
| - name: Copy files to S3 with the AWS CLI | |
| shell: bash | |
| run: | | |
| DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml) | |
| TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)" | |
| aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/win-x86_64 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/win-x86_64/ | |
| aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*" | |
| build-tokenizers-jni-arm64-osx: | |
| if: ${{ github.repository == 'deepjavalibrary/djl' && always() }} | |
| runs-on: macos-latest-xlarge | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: 17 | |
| distribution: corretto | |
| architecture: aarch64 | |
| - uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Release JNI prep | |
| run: | | |
| ./gradlew :extensions:tokenizers:compileJNI | |
| ./gradlew -Pjni :extensions:tokenizers:test | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role | |
| aws-region: us-east-2 | |
| - name: Copy files to S3 with the AWS CLI | |
| shell: bash | |
| run: | | |
| DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml) | |
| TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)" | |
| aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/osx-aarch64 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/osx-aarch64/ | |
| aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*" | |
| create-runners: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: [ self-hosted, scheduler ] | |
| steps: | |
| - name: Create new Graviton instance | |
| id: create_aarch64 | |
| run: | | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \ | |
| https://api.github.com/repos/deepjavalibrary/djl/actions/runners/registration-token \ | |
| --fail \ | |
| | jq '.token' | tr -d '"' ) | |
| ./start_instance.sh action_graviton $token djl | |
| - name: Create new G5 instance | |
| id: create_gpu | |
| run: | | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| token=$( curl -X POST -H "Authorization: token ${{ secrets.ACTION_RUNNER_PERSONAL_TOKEN }}" \ | |
| https://api.github.com/repos/deepjavalibrary/djl/actions/runners/registration-token \ | |
| --fail \ | |
| | jq '.token' | tr -d '"' ) | |
| ./start_instance.sh action_g5 $token djl | |
| outputs: | |
| aarch64_instance_id: ${{ steps.create_aarch64.outputs.action_graviton_instance_id }} | |
| gpu_instance_id: ${{ steps.create_gpu.outputs.action_g5_instance_id }} | |
| build-tokenizer-jni-aarch64: | |
| if: ${{ github.repository == 'deepjavalibrary/djl' && always() }} | |
| runs-on: [ self-hosted, aarch64 ] | |
| timeout-minutes: 30 | |
| needs: create-runners | |
| container: | |
| image: amazonlinux:2 | |
| env: | |
| JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto.aarch64 | |
| steps: | |
| - name: Install Environment | |
| run: | | |
| yum -y groupinstall "Development Tools" | |
| yum -y install patch perl-IPC-Cmd cmake3 curl jq | |
| yum -y install java-17-amazon-corretto-devel | |
| ln -s /usr/bin/cmake3 /usr/bin/cmake | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| pip3 install awscli --upgrade | |
| - uses: taiki-e/checkout-action@v1 | |
| - name: Release JNI prep | |
| run: | | |
| source "$HOME/.cargo/env" | |
| ./gradlew :extensions:tokenizers:compileJNI | |
| PYTORCH_PRECXX11=true ./gradlew -Pjni :extensions:tokenizers:test | |
| - name: Configure AWS Credentials | |
| run: | | |
| oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ | |
| "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value") | |
| echo "::add-mask::$oidc_token" | |
| read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \ | |
| --region "us-east-2" \ | |
| --role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \ | |
| --role-session-name "build-tokenizer-jni-aarch64" \ | |
| --web-identity-token "$oidc_token" \ | |
| --query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \ | |
| --output text)" | |
| echo "::add-mask::$AWS_ACCESS_KEY_ID" | |
| echo "::add-mask::$AWS_SECRET_ACCESS_KEY" | |
| echo "::add-mask::$AWS_SESSION_TOKEN" | |
| echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV" | |
| echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV" | |
| echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV" | |
| - name: Copy files to S3 with the AWS CLI | |
| run: | | |
| DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml) | |
| TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)" | |
| aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/linux-aarch64 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/linux-aarch64/ | |
| aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*" | |
| build-tokenizers-jni-cu122: | |
| if: github.repository == 'deepjavalibrary/djl' | |
| runs-on: [ self-hosted, g5 ] | |
| timeout-minutes: 60 | |
| needs: create-runners | |
| container: | |
| image: nvidia/cuda:12.2.2-devel-ubuntu20.04 | |
| options: --gpus all --runtime=nvidia | |
| env: | |
| CUDA_VERSION: cu122 | |
| steps: | |
| - name: Install Environment | |
| run: | | |
| apt-get -y update | |
| apt-get -y install curl git python3-pip | |
| pip install awscli wheel setuptools --upgrade | |
| - uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'corretto' | |
| java-version: 17 | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Release JNI prep | |
| run: | | |
| . "$HOME/.cargo/env" | |
| ./gradlew :extensions:tokenizers:compileJNI -Pcuda=${{ env.CUDA_VERSION }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role | |
| aws-region: us-east-2 | |
| - name: Copy files to S3 with the AWS CLI | |
| run: | | |
| DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml) | |
| TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)" | |
| aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/linux-x86_64/ s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/${DJL_VERSION}/linux-x86_64/ | |
| aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*" | |
| stop-runners: | |
| if: ${{ github.repository == 'deepjavalibrary/djl' && always() }} | |
| runs-on: [ self-hosted, scheduler ] | |
| needs: [ create-runners, build-tokenizer-jni-aarch64, build-tokenizers-jni-cu122 ] | |
| steps: | |
| - name: Stop all instances | |
| run: | | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| instance_id=${{ needs.create-runners.outputs.aarch64_instance_id }} | |
| ./stop_instance.sh $instance_id | |
| cd /home/ubuntu/djl_benchmark_script/scripts | |
| instance_id=${{ needs.create-runners.outputs.gpu_instance_id }} | |
| ./stop_instance.sh $instance_id |