Skip to content

Commit

Permalink
Qualcomm Adreno CI hacks.
Browse files Browse the repository at this point in the history
srkreddy1238 committed Nov 12, 2024
1 parent d5b9f5c commit 51260a6
Showing 3 changed files with 373 additions and 1 deletion.
365 changes: 365 additions & 0 deletions .github/workflows/adreno_linaro_release_gen.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,365 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# GH actions.

name: Adreno Linaro Build Test and Release

on:
workflow_dispatch:

concurrency:
group: CI-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true

jobs:
Lint:
if: ${{ github.repository == 'gpgpu/apache-tvm' }}
runs-on: tvm-linux-ci
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'
ref: 'linaro'
- name: Run Lint
shell: bash -l {0}
run: |
SCCACHE_SERVER_PORT=8766 ./tests/scripts/ci.py lint -d tvm.ci_lint
Adreno-Tests-V3:
needs: [Lint]
if: ${{ github.repository == 'gpgpu/apache-tvm' }}
runs-on: adreno-runner
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'
ref: 'linaro'
- name: Build and run tests
shell: bash -l {0}
run: |
set -e
if ! git remote -v | grep ^gpgpu ; then git remote add gpgpu [email protected]:gpgpu/apache-tvm.git ; fi
git fetch gpgpu
git checkout gpgpu/adreno_ci -- pytest_deselect_ids.txt
git restore --staged pytest_deselect_ids.txt
# Test V4 device with V3 CLML SDK
ADRENO_OPENCL=${ADRENO_OPENCL_V3} \
SCCACHE_SERVER_PORT=8769 \
ANDROID_SERIAL=`cat /etc/tvm-ci-device-v4` \
python ./tests/scripts/ci.py adreno -d tvm.ci_adreno --test
# Test V3 device with V3 CLML SDK
ADRENO_OPENCL=${ADRENO_OPENCL_V3} \
SCCACHE_SERVER_PORT=8769 \
ANDROID_SERIAL=`cat /etc/tvm-ci-device` \
python ./tests/scripts/ci.py adreno -d tvm.ci_adreno --test
Adreno-Tests-V4:
needs: [Adreno-Tests-V3]
if: ${{ github.repository == 'gpgpu/apache-tvm' }}
runs-on: adreno-runner
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'
ref: 'linaro'
- name: Build and run tests
shell: bash -l {0}
run: |
set -e
if ! git remote -v | grep ^gpgpu ; then git remote add gpgpu [email protected]:gpgpu/apache-tvm.git ; fi
git fetch gpgpu
git checkout gpgpu/adreno_ci -- pytest_deselect_ids.txt
git restore --staged pytest_deselect_ids.txt
# Test V4 device with V4 CLML SDK
ADRENO_OPENCL=${ADRENO_OPENCL_V4} \
SCCACHE_SERVER_PORT=8767 \
ANDROID_SERIAL=`cat /etc/tvm-ci-device-v4` \
python ./tests/scripts/ci.py adreno -d tvm.ci_adreno --test
# Test V3 device with V4 CLML SDK
ADRENO_OPENCL=${ADRENO_OPENCL_V4} \
SCCACHE_SERVER_PORT=8767 \
ANDROID_SERIAL=`cat /etc/tvm-ci-device` \
python ./tests/scripts/ci.py adreno -d tvm.ci_adreno --test
GPU-Tests:
needs: [Adreno-Tests-V4]
if: ${{ github.repository == 'gpgpu/apache-tvm' }}
runs-on: gpu-runner
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'
ref: 'linaro'
- name: Build and run topi tests
shell: bash -l {0}
run: |
set -e
if ! git remote -v | grep ^gpgpu ; then git remote add gpgpu [email protected]:gpgpu/apache-tvm.git ; fi
git fetch gpgpu
git checkout gpgpu/adreno_ci -- pytest_deselect_ids.txt
git restore --staged pytest_deselect_ids.txt
SCCACHE_SERVER_PORT=8768 python ./tests/scripts/ci.py gpu -d adreno_ci_gpu --topi
- name: Build cpptest
shell: bash -l {0}
run: |
./docker/bash.sh adreno_ci_gpu --env SCCACHE_SERVER_PORT=8768 python3 ./tests/scripts/task_build.py --cmake-target cpptest --build-dir build-gpu
- name: Run cpp unit tests
shell: bash -l {0}
run: |
SCCACHE_SERVER_PORT=8768 ./tests/scripts/ci.py gpu -d adreno_ci_gpu -c
- name: Run unit tests
shell: bash -l {0}
run: |
SCCACHE_SERVER_PORT=8768 ./tests/scripts/ci.py gpu -d adreno_ci_gpu -u
- name: Run front end tests
shell: bash -l {0}
run: |
SCCACHE_SERVER_PORT=8768 ./tests/scripts/ci.py gpu -d adreno_ci_gpu -f
Windows-Build:
needs: [GPU-Tests]
if: ${{ github.repository == 'gpgpu/apache-tvm' }}
runs-on: tvm-windows
steps:
- uses: actions/checkout@v3
with:
submodules: 'recursive'
ref: 'linaro'
- name: Compile x86
shell: powershell
run: |
Push-Location
New-Item -ItemType Directory -Force -Path "./build"
cd build
cp ../cmake/config.cmake .
Add-Content config.cmake "set(USE_OPENCL $ENV:OPENCL_SDK_ADRENO_X86)"
Add-Content config.cmake "set(USE_LLVM $ENV:LLVM_CONFIG)"
#Add-Content config.cmake "set(USE_CLML $ENV:OPENCL_SDK_ADRENO_X86)"
#Add-Content config.cmake "set(USE_CLML_GRAPH_EXECUTOR $ENV:OPENCL_SDK_ADRENO_X86)"
Add-Content config.cmake "set(USE_CPP_RTVM ON)"
Add-Content config.cmake "set(USE_RPC ON)"
Add-Content config.cmake "set(USE_CPP_RPC ON)"
Add-Content config.cmake "set(USE_GRAPH_EXECUTOR ON)"
Add-Content config.cmake "set(USE_KALLOC_ALIGNMENT 32)"
Add-Content config.cmake "set(USE_OPENCL_EXTN_QCOM ON)"
cmake .. -G "Visual Studio 17 2022" -A x64
cmake --build . --config Release --parallel $env:NUMBER_OF_PROCESSORS
Pop-Location
- name: Check x86 Build Status
shell: powershell
run: |
$env:PYTHONPATH = "$pwd\python"
python -c "import tvm; print(tvm.__path__)"
- name: Compile Arm64
shell: powershell
run: |
Push-Location
New-Item -ItemType Directory -Force -Path "./build-arm64"
cd build-arm64
cp ../cmake/config.cmake .
Add-Content config.cmake "set(USE_OPENCL $ENV:OPENCL_SDK_ADRENO_ARM64)"
#Add-Content config.cmake "set(USE_CLML $ENV:OPENCL_SDK_ADRENO_ARM64)"
#Add-Content config.cmake "set(USE_CLML_GRAPH_EXECUTOR $ENV:OPENCL_SDK_ADRENO_ARM64)"
Add-Content config.cmake "set(USE_CPP_RTVM ON)"
Add-Content config.cmake "set(USE_RPC ON)"
Add-Content config.cmake "set(USE_CPP_RPC ON)"
Add-Content config.cmake "set(USE_GRAPH_EXECUTOR ON)"
Add-Content config.cmake "set(USE_KALLOC_ALIGNMENT 32)"
Add-Content config.cmake "set(USE_OPENCL_EXTN_QCOM ON)"
cmake .. -G "Visual Studio 17 2022" -A ARM64
cmake --build . --config Release --parallel $env:NUMBER_OF_PROCESSORS --target rtvm tvm_rpc
Pop-Location
BuildPackageLinux:
needs: [Windows-Build]
if: github.repository == 'gpgpu/apache-tvm'
runs-on: tvm-linux-ci
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'
ref: 'linaro'
fetch-depth: 0
- name: Build CPU Host Wheel
shell: bash -l {0}
run: |
set -e
docker run --rm --pid=host -w /workspace \
-v ${PWD}:/workspace \
-v ${ADRENO_OPENCL}:/adreno_opencl \
-e ADRENO_OPENCL=/adreno_opencl \
-e ENV_USER_ID=`id -u` \
-e ENV_GROUP_ID=`id -g` \
mlcaidev/package-cu121 \
conda run -n py38 ./ci/adreno/build_lib.sh
docker run --rm --pid=host -w /workspace \
-v ${PWD}:/workspace \
-v ${ADRENO_OPENCL}:/adreno_opencl \
-e ADRENO_OPENCL=/adreno_opencl \
-e ENV_USER_ID=`id -u` \
-e ENV_GROUP_ID=`id -g` \
-e WHEEL_EXT_NAME="_adreno_cpu" \
mlcaidev/package-cu121 \
conda run -n py38 ./ci/adreno/build_wheel.sh
mkdir -p linaro
cp wheels/*.whl ./linaro/ -rfa
mv wheels/*.whl ${CI_PACKAGE_REPO_LINARO}/
docker run --rm --pid=host -w /workspace \
-v ${PWD}:/workspace \
-v ${ADRENO_OPENCL}:/adreno_opencl \
-e ADRENO_OPENCL=/adreno_opencl \
-e ENV_USER_ID=`id -u` \
-e ENV_GROUP_ID=`id -g` \
mlcaidev/package-cu121 \
conda run -n py38 ./ci/adreno/build_clean.sh
- name: Build CUDA Host Wheel
shell: bash -l {0}
run: |
set -e
docker run --rm --pid=host -w /workspace \
-v ${PWD}:/workspace \
-v ${ADRENO_OPENCL}:/adreno_opencl \
-e ADRENO_OPENCL=/adreno_opencl \
-e ENV_USER_ID=`id -u` \
-e ENV_GROUP_ID=`id -g` \
-e GPU=cuda \
mlcaidev/package-cu121 \
conda run -n py38 ./ci/adreno/build_lib.sh
docker run --rm --pid=host -w /workspace \
-v ${PWD}:/workspace \
-v ${ADRENO_OPENCL}:/adreno_opencl \
-e ADRENO_OPENCL=/adreno_opencl \
-e ENV_USER_ID=`id -u` \
-e ENV_GROUP_ID=`id -g` \
-e WHEEL_EXT_NAME="_adreno_cuda" \
-e GPU=cuda \
mlcaidev/package-cu121 \
conda run -n py38 ./ci/adreno/build_wheel.sh
cp wheels/*.whl ./linaro/ -rfa
mv wheels/*.whl ${CI_PACKAGE_REPO_LINARO}/
docker run --rm --pid=host -w /workspace \
-v ${PWD}:/workspace \
-v ${ADRENO_OPENCL}:/adreno_opencl \
-e ADRENO_OPENCL=/adreno_opencl \
-e ENV_USER_ID=`id -u` \
-e ENV_GROUP_ID=`id -g` \
-e GPU=cuda \
mlcaidev/package-cu121 \
conda run -n py38 ./ci/adreno/build_clean.sh
- name: Build Target
shell: bash -l {0}
run: |
set -e
mkdir -p build-adreno-target
cd build-adreno-target
cp ../cmake/config.cmake . -f
echo set\(USE_CLML "${ADRENO_OPENCL}"\) >> config.cmake
echo set\(USE_CLML_GRAPH_EXECUTOR "${ADRENO_OPENCL}"\) >> config.cmake
echo set\(USE_RPC ON\) >> config.cmake
echo set\(USE_CPP_RPC ON\) >> config.cmake
echo set\(USE_CPP_RTVM ON\) >> config.cmake
echo set\(USE_GRAPH_EXECUTOR ON\) >> config.cmake
echo set\(USE_LIBBACKTRACE AUTO\) >> config.cmake
echo set\(USE_KALLOC_ALIGNMENT 32\) >> config.cmake
echo set\(ANDROID_ABI arm64-v8a\) >> config.cmake
echo set\(ANDROID_PLATFORM android-28\) >> config.cmake
echo set\(MACHINE_NAME aarch64-linux-gnu\) >> config.cmake
cmake -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \
-DANDROID_ABI=arm64-v8a \
-DANDROID_PLATFORM=android-28 \
-DCMAKE_SYSTEM_VERSION=1 \
-DCMAKE_FIND_ROOT_PATH="${ADRENO_OPENCL}" \
-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
-DCMAKE_CXX_COMPILER="${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android28-clang++" \
-DCMAKE_C_COMPILER="${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android28-clang" \
-DMACHINE_NAME="aarch64-linux-gnu" ..
cmake --build . --parallel $(nproc)
cd -
- name: Build Adreno Utils
shell: bash -l {0}
run: |
set -e
cp build-adreno-target/tvm_rpc adreno-utils/ -rfa
cp build-adreno-target/rtvm adreno-utils/ -rfa
cp build-adreno-target/libtvm_runtime.so adreno-utils/ -rfa
tar jcvfp adreno-utils-linux-arm64.tar.bz2 adreno-utils
cp adreno-utils-linux-arm64.tar.bz2 ./linaro/ -rfa
gh release delete tvm-linaro --cleanup-tag --yes --repo "$LINARO_RELEASE_REPO_TVM" || true
gh release create tvm-linaro --latest --title tvm-linaro --repo "$LINARO_RELEASE_REPO_TVM" --notes "TVM Nightly builds"
gh release upload tvm-linaro ./linaro/* --clobber --repo "$LINARO_RELEASE_REPO_TVM"
rm -rf linaro
mv adreno-utils-linux-arm64.tar.bz2 ${CI_PACKAGE_REPO_LINARO}/
BuildPackageWindows:
needs: [BuildPackageLinux]
if: ${{ github.repository == 'gpgpu/apache-tvm' }}
runs-on: tvm-windows
steps:
- uses: actions/checkout@v3
with:
submodules: 'recursive'
ref: 'linaro'
- name: Compile X86
shell: powershell
run: |
Push-Location
New-Item -ItemType Directory -Force -Path "./build"
cd build
cp ../cmake/config.cmake .
Add-Content config.cmake "set(USE_OPENCL $ENV:OPENCL_SDK_ADRENO_X86)"
Add-Content config.cmake "set(USE_LLVM $ENV:LLVM_CONFIG)"
Add-Content config.cmake "set(USE_CPP_RTVM ON)"
Add-Content config.cmake "set(USE_RPC ON)"
Add-Content config.cmake "set(USE_CPP_RPC ON)"
Add-Content config.cmake "set(USE_GRAPH_EXECUTOR ON)"
Add-Content config.cmake "set(USE_KALLOC_ALIGNMENT 32)"
Add-Content config.cmake "set(USE_OPENCL_EXTN_QCOM ON)"
cmake .. -G "Visual Studio 17 2022" -A x64
cmake --build . --config Release --parallel $env:NUMBER_OF_PROCESSORS
Pop-Location
- name: Check Build X86 Status
shell: powershell
run: |
$env:PYTHONPATH = "$pwd\python"
python -c "import tvm; print(tvm.__path__)"
- name: Compile Arm64
shell: powershell
run: |
Push-Location
New-Item -ItemType Directory -Force -Path "./build-arm64"
cd build-arm64
cp ../cmake/config.cmake .
Add-Content config.cmake "set(USE_OPENCL $ENV:OPENCL_SDK_ADRENO_ARM64)"
Add-Content config.cmake "set(USE_CPP_RTVM ON)"
Add-Content config.cmake "set(USE_RPC ON)"
Add-Content config.cmake "set(USE_CPP_RPC ON)"
Add-Content config.cmake "set(USE_GRAPH_EXECUTOR ON)"
Add-Content config.cmake "set(USE_KALLOC_ALIGNMENT 32)"
Add-Content config.cmake "set(USE_OPENCL_EXTN_QCOM ON)"
cmake .. -G "Visual Studio 17 2022" -A ARM64
cmake --build . --config Release --parallel $env:NUMBER_OF_PROCESSORS --target rtvm tvm_rpc
Pop-Location
- name: Build Package and Publish
shell: powershell
run: |
Push-Location
cd python
$Env:WHEEL_EXT_NAME="_adreno_cpu_x86"
python.exe .\setup.py bdist_wheel --plat-name win_amd64
Pop-Location
Compress-Archive -Path ".\\build-arm64\Release\*" -CompressionLevel Optimal -DestinationPath adreno-utils-win-arm64.zip -Force
foreach ($i in Get-ChildItem -Path python\dist) {gh release upload tvm-linaro .\python\dist\$i --clobber --repo $Env:LINARO_RELEASE_REPO_TVM}
gh release upload tvm-linaro adreno-utils-win-arm64.zip --clobber --repo $Env:LINARO_RELEASE_REPO_TVM
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3 changes: 3 additions & 0 deletions python/tvm/contrib/download.py
Original file line number Diff line number Diff line change
@@ -23,6 +23,9 @@
import shutil
import tempfile
import time
import ssl

ssl._create_default_https_context = ssl._create_unverified_context

LOG = logging.getLogger("download")

6 changes: 5 additions & 1 deletion tests/scripts/setup-pytest-env.sh
Original file line number Diff line number Diff line change
@@ -83,13 +83,17 @@ function run_pytest() {
extra_args+=("-n=$DEFAULT_PARALLELISM")
fi

if [ -f pytest_deselect_ids.txt ] ; then
DESELECT_LIST=`cat pytest_deselect_ids.txt`
fi

exit_code=0
set +e
TVM_FFI=${ffi_type} python3 -m pytest \
-o "junit_suite_name=${suite_name}" \
"--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \
"--junit-prefix=${ffi_type}" \
"${extra_args[@]}" || exit_code=$?
"${extra_args[@]}" ${DESELECT_LIST} || exit_code=$?
# Pytest will return error code -5 if no test is collected.
if [ "$exit_code" -ne "0" ] && [ "$exit_code" -ne "5" ]; then
pytest_errors+=("${suite_name}: $@")

0 comments on commit 51260a6

Please sign in to comment.