Skip to content

Commit

Permalink
Merge branch 'develop' into roc-6.3.x
Browse files Browse the repository at this point in the history
fix merge conflicts

fix merge conflict
  • Loading branch information
peterjunpark committed Dec 20, 2024
2 parents 8c03653 + bf9727d commit 93b17c5
Show file tree
Hide file tree
Showing 55 changed files with 1,890 additions and 2,207 deletions.
1 change: 1 addition & 0 deletions .azuredevops/components/MIOpen.yml
Original file line number Diff line number Diff line change
Expand Up @@ -197,3 +197,4 @@ jobs:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
parameters:
componentName: MIOpen
testParameters: '-VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml --exclude-regex test_rnn_seq_api'
1 change: 1 addition & 0 deletions .azuredevops/components/llvm-project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ jobs:
componentName: comgr
extraBuildFlags: >-
-DCMAKE_PREFIX_PATH="$(Build.SourcesDirectory)/llvm/build;$(Build.SourcesDirectory)/amd/device-libs/build"
-DCOMGR_DISABLE_SPIRV=1
-DCMAKE_BUILD_TYPE=Release
cmakeBuildDir: 'amd/comgr/build'
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
Expand Down
166 changes: 166 additions & 0 deletions .azuredevops/components/omniperf.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
parameters:
- name: checkoutRepo
type: string
default: 'self'
- name: checkoutRef
type: string
default: ''
- name: aptPackages
type: object
default:
- cmake
- python3-pip
- name: pipModules
type: object
default:
- astunparse==1.6.2
- colorlover
- dash>=1.12.0
- matplotlib
- numpy>=1.17.5
- pandas>=1.4.3
- pymongo
- pyyaml
- tabulate
- tqdm
- dash-svg
- dash-bootstrap-components
- kaleido
- setuptools
- plotille
- mock
- pytest
- pytest-cov
- pytest-xdist
- name: rocmDependencies
type: object
default:
- clr
- llvm-project
- rocm-cmake
- rocm-core
- rocminfo
- ROCR-Runtime
- rocprofiler
- rocprofiler-register
- roctracer

jobs:
- job: omniperf
variables:
- group: common
- template: /.azuredevops/variables-global.yml
pool:
vmImage: ${{ variables.BASE_BUILD_POOL }}
workspace:
clean: all
strategy:
matrix:
gfx942:
JOB_GPU_TARGET: gfx942
steps:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
parameters:
aptPackages: ${{ parameters.aptPackages }}
pipModules: ${{ parameters.pipModules }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
parameters:
checkoutRepo: ${{ parameters.checkoutRepo }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
parameters:
${{ if eq(parameters.checkoutRef, '') }}:
dependencySource: staging
${{ elseif ne(parameters.checkoutRef, '') }}:
dependencySource: tag-builds
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
parameters:
dependencyList: ${{ parameters.rocmDependencies }}
gpuTarget: $(JOB_GPU_TARGET)
# CI case: download latest default branch build
${{ if eq(parameters.checkoutRef, '') }}:
dependencySource: staging
# manual build case: triggered by ROCm/ROCm repo
${{ elseif ne(parameters.checkoutRef, '') }}:
dependencySource: tag-builds
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
parameters:
gpuTarget: $(JOB_GPU_TARGET)

- job: omniperf_testing
dependsOn: omniperf
condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
variables:
- group: common
- template: /.azuredevops/variables-global.yml
- name: PYTHON_VERSION
value: 3.10
pool: $(JOB_TEST_POOL)
workspace:
clean: all
strategy:
matrix:
gfx942:
JOB_GPU_TARGET: gfx942
JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
steps:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
parameters:
aptPackages: ${{ parameters.aptPackages }}
pipModules: ${{ parameters.pipModules }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
parameters:
checkoutRepo: ${{ parameters.checkoutRepo }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
parameters:
${{ if eq(parameters.checkoutRef, '') }}:
dependencySource: staging
${{ elseif ne(parameters.checkoutRef, '') }}:
dependencySource: tag-builds
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
parameters:
dependencyList: ${{ parameters.rocmDependencies }}
gpuTarget: $(JOB_GPU_TARGET)
${{ if eq(parameters.checkoutRef, '') }}:
dependencySource: staging
${{ elseif ne(parameters.checkoutRef, '') }}:
dependencySource: tag-builds
- task: Bash@3
displayName: Add ROCm binaries to PATH
inputs:
targetType: inline
script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
- task: Bash@3
displayName: Add ROCm compilers to PATH
inputs:
targetType: inline
script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
parameters:
extraBuildFlags: >-
-DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
-DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-DCMAKE_BUILD_TYPE=Release
-DENABLE_TESTS=ON
-DINSTALL_TESTS=ON
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
parameters:
componentName: omniperf
testDir: $(Build.BinariesDirectory)/libexec/omniperf
testExecutable: export OMNIPERF_ARCH_OVERRIDE="MI300X"; ctest
- task: Bash@3
displayName: Remove ROCm binaries from PATH
inputs:
targetType: inline
script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
- task: Bash@3
displayName: Remove ROCm compilers from PATH
inputs:
targetType: inline
script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
1 change: 1 addition & 0 deletions .azuredevops/components/rocPyDecode.yml
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ jobs:
parameters:
dependencyList: ${{ parameters.rocmDependencies }}
gpuTarget: $(JOB_GPU_TARGET)
setupHIPLibrarySymlinks: true
${{ if eq(parameters.checkoutRef, '') }}:
dependencySource: staging
${{ elseif ne(parameters.checkoutRef, '') }}:
Expand Down
1 change: 1 addition & 0 deletions .azuredevops/components/rocprofiler-sdk.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ parameters:
- ROCR-Runtime
- rocprofiler-register
- roctracer
- aomp

jobs:
- job: rocprofilersdk
Expand Down
8 changes: 8 additions & 0 deletions .azuredevops/components/rocprofiler-systems.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ parameters:
- rocprofiler
- rocprofiler-register
- roctracer
- rocprofiler-sdk

jobs:
- job: rocprofiler_systems
Expand All @@ -73,6 +74,12 @@ jobs:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
parameters:
checkoutRepo: ${{ parameters.checkoutRepo }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
parameters:
${{ if eq(parameters.checkoutRef, '') }}:
dependencySource: staging
${{ elseif ne(parameters.checkoutRef, '') }}:
dependencySource: tag-builds
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
parameters:
dependencyList: ${{ parameters.rocmDependencies }}
Expand Down Expand Up @@ -109,6 +116,7 @@ jobs:
-DROCPROFSYS_BUILD_TESTING=ON
-DROCPROFSYS_BUILD_DYNINST=ON
-DROCPROFSYS_BUILD_LIBUNWIND=ON
-DROCPROFSYS_DISABLE_EXAMPLES="openmp-target"
-DDYNINST_BUILD_TBB=ON
-DDYNINST_BUILD_ELFUTILS=ON
-DDYNINST_BUILD_LIBIBERTY=ON
Expand Down
133 changes: 73 additions & 60 deletions .azuredevops/nightly/pytorch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,10 @@ parameters:
- binary_ufuncs
- autograd
# - inductor/torchinductor takes too long
# set to false to disable torchvision build and test
- name: includeVision
type: boolean
default: false

trigger: none
pr: none
Expand Down Expand Up @@ -237,6 +241,12 @@ jobs:
git clone https://github.com/pytorch/builder.git --depth=1 --recurse-submodules
sudo ln -s $(Build.SourcesDirectory)/builder /builder
workingDirectory: $(Build.SourcesDirectory)
- task: Bash@3
displayName: Temporarily Patch CK Submodule
inputs:
targetType: inline
script: git pull origin develop
workingDirectory: $(Build.SourcesDirectory)/pytorch/third_party/composable_kernel
- task: Bash@3
displayName: Install patchelf
inputs:
Expand Down Expand Up @@ -296,59 +306,60 @@ jobs:
sourceDir: /remote/wheelhouserocm$(ROCM_VERSION)
contentsString: '*.whl'
# common helper source for pytorch vision and audio
- task: Bash@3
displayName: git clone pytorch test-infra
inputs:
targetType: inline
script: git clone https://github.com/pytorch/test-infra.git --depth=1 --recurse-submodules
workingDirectory: $(Build.SourcesDirectory)
- task: Bash@3
displayName: install package helper
inputs:
targetType: inline
script: python3 -m pip install test-infra/tools/pkg-helpers
workingDirectory: $(Build.SourcesDirectory)
- task: Bash@3
displayName: pytorch pkg helpers
inputs:
targetType: inline
script: CU_VERSION=${CU_VERSION} CHANNEL=${CHANNEL} python -m pytorch_pkg_helpers
# get torch vision source and build
- task: Bash@3
displayName: git clone pytorch vision
inputs:
targetType: inline
script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
workingDirectory: $(Build.SourcesDirectory)
- task: Bash@3
displayName: Build vision
inputs:
targetType: inline
script: >-
TORCH_PACKAGE_NAME=torch.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
TORCHVISION_PACKAGE_NAME=torchvision.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
PYTORCH_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
BUILD_VERSION=$(cat $(Build.SourcesDirectory)/vision/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
python3 setup.py bdist_wheel
workingDirectory: $(Build.SourcesDirectory)/vision
- task: Bash@3
displayName: Relocate vision
inputs:
targetType: inline
script: python3 packaging/wheel/relocate.py
workingDirectory: $(Build.SourcesDirectory)/vision
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
parameters:
sourceDir: $(Build.SourcesDirectory)/vision/dist
contentsString: '*.whl'
clean: false
- ${{ if eq(parameters.includeVision, true) }}:
- task: Bash@3
displayName: git clone pytorch test-infra
inputs:
targetType: inline
script: git clone https://github.com/pytorch/test-infra.git --depth=1 --recurse-submodules
workingDirectory: $(Build.SourcesDirectory)
- task: Bash@3
displayName: install package helper
inputs:
targetType: inline
script: python3 -m pip install test-infra/tools/pkg-helpers
workingDirectory: $(Build.SourcesDirectory)
- task: Bash@3
displayName: pytorch pkg helpers
inputs:
targetType: inline
script: CU_VERSION=${CU_VERSION} CHANNEL=${CHANNEL} python -m pytorch_pkg_helpers
# get torch vision source and build
- task: Bash@3
displayName: git clone pytorch vision
inputs:
targetType: inline
script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
workingDirectory: $(Build.SourcesDirectory)
- task: Bash@3
displayName: Build vision
inputs:
targetType: inline
script: >-
TORCH_PACKAGE_NAME=torch.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
TORCHVISION_PACKAGE_NAME=torchvision.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
PYTORCH_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
BUILD_VERSION=$(cat $(Build.SourcesDirectory)/vision/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
python3 setup.py bdist_wheel
workingDirectory: $(Build.SourcesDirectory)/vision
- task: Bash@3
displayName: Relocate vision
inputs:
targetType: inline
script: python3 packaging/wheel/relocate.py
workingDirectory: $(Build.SourcesDirectory)/vision
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
parameters:
sourceDir: $(Build.SourcesDirectory)/vision/dist
contentsString: '*.whl'
clean: false
- task: PublishPipelineArtifact@1
displayName: 'wheel file Publish'
retryCountOnTaskFailure: 3
inputs:
targetPath: $(Build.BinariesDirectory)

- job: torchvision_testing
- job: pytorch_testing
dependsOn: pytorch
condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
variables:
Expand Down Expand Up @@ -401,12 +412,13 @@ jobs:
targetType: inline
script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
workingDirectory: $(Build.SourcesDirectory)
- task: Bash@3
displayName: git clone pytorch vision
inputs:
targetType: inline
script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
workingDirectory: $(Build.SourcesDirectory)
- ${{ if eq(parameters.includeVision, true) }}:
- task: Bash@3
displayName: git clone pytorch vision
inputs:
targetType: inline
script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
workingDirectory: $(Build.SourcesDirectory)
- task: Bash@3
displayName: Install Wheel Files
inputs:
Expand Down Expand Up @@ -510,13 +522,14 @@ jobs:
script: pytest test/test_${{ torchTest }}.py
# Reference on what tests to run for torchvision found in private repo:
# https://github.com/ROCm/rocAutomation/blob/jenkins-pipelines/pytorch/pytorch_ci/test_torchvision.sh#L51
- task: Bash@3
displayName: Test vision/transforms
continueOnError: true
inputs:
targetType: inline
script: pytest test/test_transforms.py
workingDirectory: $(Build.SourcesDirectory)/vision
- ${{ if eq(parameters.includeVision, true) }}:
- task: Bash@3
displayName: Test vision/transforms
continueOnError: true
inputs:
targetType: inline
script: pytest test/test_transforms.py
workingDirectory: $(Build.SourcesDirectory)/vision
- task: Bash@3
displayName: Uninstall Wheel Files
inputs:
Expand Down
Loading

0 comments on commit 93b17c5

Please sign in to comment.