Skip to content

Fix bf16 support issues #270

Fix bf16 support issues

Fix bf16 support issues #270

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
# This workflow is used for FBGEMM_GPU-CPU CI as well as nightly builds of
# FBGEMM_GPU-CPU against PyTorch-CPU Nightly.
name: FBGEMM_GPU-CPU CI
on:
# PR Trigger (enabled for regression checks and debugging)
#
pull_request:
branches:
- main
# Push Trigger (enable to catch errors coming out of multiple merges)
#
push:
branches:
- main
# Cron Trigger (UTC)
#
# Based on the Conda page for PyTorch-nightly, the CPU nightly releases appear
# around 00:30 PST every day
#
schedule:
- cron: '45 12 * * *'
# Manual Trigger
#
workflow_dispatch:
inputs:
publish_to_pypi:
description: Publish Artifact to PyPI
type: boolean
required: false
default: false
concurrency:
# Cancel previous runs in the PR if a new commit is pushed
# https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
# Build on CPU hosts, run tests, and upload to GHA
build_artifact:
runs-on: ${{ matrix.host-machine.instance }}
container:
image: amazonlinux:2023
options: --user root
defaults:
run:
shell: bash
env:
PRELUDE: .github/scripts/setup_env.bash
BUILD_ENV: build_binary
continue-on-error: true
strategy:
# Don't fast-fail all the other builds if one of the them fails
fail-fast: false
matrix:
host-machine: [
{ arch: x86, instance: "linux.4xlarge" },
{ arch: arm, instance: "linux.arm64.2xlarge" },
]
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
steps:
- name: Setup Build Container
run: yum update -y; yum install -y binutils findutils git pciutils sudo wget which
- name: Checkout the Repository
uses: actions/checkout@v4
with:
submodules: true
- name: Display System Info
run: . $PRELUDE; print_system_info
- name: Display GPU Info
run: . $PRELUDE; print_gpu_info
- name: Setup Miniconda
run: . $PRELUDE; setup_miniconda $HOME/miniconda
- name: Create Conda Environment
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
- name: Install C/C++ Compilers
run: . $PRELUDE; install_cxx_compiler $BUILD_ENV
- name: Install Build Tools
run: . $PRELUDE; install_build_tools $BUILD_ENV
- name: Install PyTorch-CPU Nightly
run: . $PRELUDE; install_pytorch_pip $BUILD_ENV nightly cpu
- name: Collect PyTorch Environment Info
if: ${{ success() || failure() }}
run: . $PRELUDE; collect_pytorch_env_info $BUILD_ENV
- name: Prepare FBGEMM_GPU Build
run: . $PRELUDE; cd fbgemm_gpu; prepare_fbgemm_gpu_build $BUILD_ENV
- name: Build FBGEMM_GPU Wheel
run: . $PRELUDE; cd fbgemm_gpu; build_fbgemm_gpu_package $BUILD_ENV nightly cpu
- name: Upload Built Wheel as GHA Artifact
uses: actions/upload-artifact@v3
with:
name: fbgemm_gpu_nightly_cpu_${{ matrix.host-machine.arch }}_${{ matrix.python-version }}.whl
path: fbgemm_gpu/dist/fbgemm_gpu_nightly_cpu-*.whl
# Download the built artifact from GHA, test on GPU, and push to PyPI
test_and_publish_artifact:
runs-on: ${{ matrix.host-machine.instance }}
container:
image: amazonlinux:2023
options: --user root
defaults:
run:
shell: bash
env:
PRELUDE: .github/scripts/setup_env.bash
BUILD_ENV: build_binary
strategy:
fail-fast: false
matrix:
host-machine: [
{ arch: x86, instance: "linux.4xlarge" },
{ arch: arm, instance: "linux.arm64.2xlarge" },
]
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
needs: build_artifact
steps:
- name: Setup Build Container
run: yum update -y; yum install -y binutils findutils git pciutils sudo wget which
- name: Checkout the Repository
uses: actions/checkout@v4
with:
submodules: true
- name: Download Wheel Artifact from GHA
uses: actions/download-artifact@v3
with:
name: fbgemm_gpu_nightly_cpu_${{ matrix.host-machine.arch }}_${{ matrix.python-version }}.whl
- name: Display System Info
run: . $PRELUDE; print_system_info; print_ec2_info
- name: Display GPU Info
run: . $PRELUDE; print_gpu_info
- name: Setup Miniconda
run: . $PRELUDE; setup_miniconda $HOME/miniconda
- name: Create Conda Environment
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
- name: Install C/C++ Compilers
# CXX compiler is needed for inductor used by torchrec.
run: . $PRELUDE; install_cxx_compiler $BUILD_ENV
- name: Install PyTorch-CPU Nightly
run: . $PRELUDE; install_pytorch_pip $BUILD_ENV nightly cpu
- name: Collect PyTorch Environment Info
if: ${{ success() || failure() }}
run: . $PRELUDE; collect_pytorch_env_info $BUILD_ENV
- name: Prepare FBGEMM_GPU Build
run: . $PRELUDE; cd fbgemm_gpu; prepare_fbgemm_gpu_build $BUILD_ENV
- name: Install FBGEMM_GPU Wheel
run: |
. $PRELUDE
pwd; ls -la .
install_fbgemm_gpu_wheel $BUILD_ENV *.whl
- name: Test with PyTest
timeout-minutes: 15
run: . $PRELUDE; cd fbgemm_gpu/test; run_fbgemm_gpu_tests $BUILD_ENV cpu
- name: Push Wheel to PyPI
if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_to_pypi == 'true') }}
env:
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
run: . $PRELUDE; publish_to_pypi $BUILD_ENV fbgemm_gpu_nightly_cpu-*.whl "$PYPI_TOKEN"
build_and_test_ubuntu:
runs-on: ${{ matrix.host-machine.instance }}
container:
image: ${{ matrix.container-image }}
options: --user root
defaults:
run:
shell: bash
env:
PRELUDE: .github/scripts/setup_env.bash
BUILD_ENV: build_binary
strategy:
fail-fast: false
matrix:
host-machine: [
{ arch: x86, instance: "linux.4xlarge" },
{ arch: arm, instance: "linux.arm64.2xlarge" },
]
container-image: [ "ubuntu:20.04", "ubuntu:22.04" ]
python-version: [ "3.11" ]
steps:
- name: Setup Build Container
run: |
apt update -y
apt install -y binutils build-essential git pciutils sudo wget
git config --global --add safe.directory '*'
- name: Checkout the Repository
uses: actions/checkout@v4
with:
submodules: true
- name: Display System Info
run: . $PRELUDE; print_system_info
- name: Display GPU Info
run: . $PRELUDE; print_gpu_info
- name: Setup Miniconda
run: . $PRELUDE; setup_miniconda $HOME/miniconda
- name: Create Conda Environment
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
- name: Install Build Tools
run: . $PRELUDE; install_build_tools $BUILD_ENV
- name: Install PyTorch
run: . $PRELUDE; install_pytorch_pip $BUILD_ENV nightly cpu
- name: Collect PyTorch Environment Info
if: ${{ success() || failure() }}
run: . $PRELUDE; collect_pytorch_env_info $BUILD_ENV
- name: Prepare FBGEMM_GPU Build
run: . $PRELUDE; cd fbgemm_gpu; prepare_fbgemm_gpu_build $BUILD_ENV
- name: Build + Install FBGEMM_GPU (CPU version)
run: . $PRELUDE; cd fbgemm_gpu; build_fbgemm_gpu_install $BUILD_ENV cpu
- name: Test FBGEMM_GPU-CPU Nightly Installation
timeout-minutes: 15
run: . $PRELUDE; cd fbgemm_gpu/test; run_fbgemm_gpu_tests $BUILD_ENV cpu