From 7918935c696748a0aa184a91f878c021bd936043 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 <35255081+nguyenhoangthuan99@users.noreply.github.com> Date: Tue, 7 Jan 2025 08:43:26 +0700 Subject: [PATCH] Feat/python package ci (#1792) * feat: add ci for python package * feat: add ci for python package * feat: add ci for python package * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test package unix * test package mac arm * test package using miniconda * test package using miniconda * test package using miniconda * test package using miniconda * test package using miniconda * test package using miniconda * test package using miniconda * test package using miniconda * test package using miniconda * test package using miniconda * test package using miniconda * Add upload artifact linux * feat: add codesign for macos * Test CI window * Test CI window * Test CI window * Test CI window * test CI windows * test CI windows * test CI windows include hidden file * test CI macos include hidden file * test CI macos include hidden file * test CI macos include hidden file * chore: add package pipeline for 4 os * chore: add package pipeline for 4 os change compression level * chore: add package pipeline for 4 os optimize linux size * chore: add package pipeline for 4 os optimize linux size and fix windows * chore: add package pipeline fix windows * Feat python package codesign (#1780) * feat: add codesign for macos * feat: add codesign for macos * fix: notary python zipped folder --------- Co-authored-by: Hien To * Update python-package.yml * Update python-package.yml * test: package fish speech * test: package fish speech * test: rerun windows * feat: package env for ichigo-wrapper * feat: package env for ichigo-wrapper * feat: package env for ichigo-wrapper * feat: package env for whispervq * feat: package env for fish-speech * feat: package env for fish-speech mac * Fix: increase timeout for macos notarize * Update python-package.yml * Fix: upload venv to hf instead of github releas * Update: test run new CI for package python * Update: test run new CI for package python * Update: test run new CI for package python * Update: test run new CI for package python windows * Update: test run new CI for package python windows * Finished: venv package * feat: init CI for upload python script to huggingface * Finished CI for upload python scripts --------- Co-authored-by: Hien To Co-authored-by: hiento09 <136591877+hiento09@users.noreply.github.com> --- .github/workflows/python-script-package.yml | 72 +++++ .github/workflows/python-venv-package.yml | 275 ++++++++++++++++++++ 2 files changed, 347 insertions(+) create mode 100644 .github/workflows/python-script-package.yml create mode 100644 .github/workflows/python-venv-package.yml diff --git a/.github/workflows/python-script-package.yml b/.github/workflows/python-script-package.yml new file mode 100644 index 000000000..5ea65be9c --- /dev/null +++ b/.github/workflows/python-script-package.yml @@ -0,0 +1,72 @@ +name: Build and Package Python Code + +on: + workflow_dispatch: + inputs: + model_dir: + description: "Path to model directory in github repo" + required: true + repo_name: + description: "name of repo to be checked out" + required: true + branch_name: + description: "name of branch to be checked out" + required: true + default: main + hf_repo: + description: "name of huggingface repo to be pushed" + required: true + hf_prefix_branch: + description: "prefix of hf branch" + required: false + +env: + MODEL_DIR: ${{ inputs.model_dir }} + REPO_NAME: ${{ inputs.repo_name}} + BRANCH_NAME: ${{ inputs.branch_name }} + HF_REPO: ${{ inputs.hf_repo }} + HF_PREFIX_BRANCH: ${{ inputs.hf_prefix_branch }} + +jobs: + build-and-test: + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 3600 + strategy: + fail-fast: false + matrix: + include: + - os: "linux" + name: "amd64" + runs-on: "ubuntu-20-04-cuda-12-0" + - os: "mac" + name: "amd64" + runs-on: "macos-selfhosted-12" + - os: "mac" + name: "arm64" + runs-on: "macos-selfhosted-12-arm64" + - os: "windows" + name: "amd64" + runs-on: "windows-cuda-12-0" + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + with: + submodules: recursive + repository: ${{env.REPO_NAME}} + ref: ${{env.BRANCH_NAME}} + - name: use python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install hf-transfer huggingface_hub + + - name: Upload Artifact + run: | + huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential + cd ${{env.MODEL_DIR}} && huggingface-cli upload ${{env.HF_REPO}} . . --revision ${{env.HF_PREFIX_BRANCH}}-${{ matrix.os }}-${{ matrix.name }} + huggingface-cli logout \ No newline at end of file diff --git a/.github/workflows/python-venv-package.yml b/.github/workflows/python-venv-package.yml new file mode 100644 index 000000000..8bed4eb97 --- /dev/null +++ b/.github/workflows/python-venv-package.yml @@ -0,0 +1,275 @@ +name: Build and Package Python Virtual Environment + +on: + workflow_dispatch: + inputs: + model_dir: + description: "Path to model directory in github repo" + required: true + model_name: + description: "name of model to be release" + required: true + repo_name: + description: "name of repo to be checked out" + required: true + branch_name: + description: "name of branch to be checked out" + required: true + default: main + hf_repo: + description: "name of huggingface repo to be pushed" + required: true + hf_prefix_branch: + description: "prefix of hf branch" + required: false + + + +env: + MODEL_DIR: ${{ inputs.model_dir }} + MODEL_NAME: ${{ inputs.model_name }} + REPO_NAME: ${{ inputs.repo_name }} + BRANCH_NAME: ${{ inputs.branch_name }} + HF_REPO: ${{ inputs.hf_repo }} + HF_PREFIX_BRANCH: ${{ inputs.hf_prefix_branch }} + +jobs: + build-and-test: + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 3600 + strategy: + fail-fast: false + matrix: + include: + - os: "linux" + name: "amd64" + runs-on: "ubuntu-20-04-cuda-12-0" + - os: "mac" + name: "amd64" + runs-on: "macos-selfhosted-12" + - os: "mac" + name: "arm64" + runs-on: "macos-selfhosted-12-arm64" + - os: "windows" + name: "amd64" + runs-on: "windows-cuda-12-0" + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + with: + submodules: recursive + repository: ${{env.REPO_NAME}} + ref: ${{env.BRANCH_NAME}} + - uses: conda-incubator/setup-miniconda@v3 + if: runner.os != 'windows' + with: + auto-update-conda: true + python-version: 3.11 + - name: use python + if : runner.os == 'windows' + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Get Cer for code signing + if: runner.os == 'macOS' + run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12 + shell: bash + env: + CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }} + + - uses: apple-actions/import-codesign-certs@v2 + continue-on-error: true + if: runner.os == 'macOS' + with: + p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }} + p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }} + + - name: Get Cer for code signing + if: runner.os == 'macOS' + run: base64 -d <<< "$NOTARIZE_P8_BASE64" > /tmp/notary-key.p8 + shell: bash + env: + NOTARIZE_P8_BASE64: ${{ secrets.NOTARIZE_P8_BASE64 }} + + - name: Install dependencies Windows + if: runner.os == 'windows' + shell: pwsh + run: | + python3 -m pip install fastapi + python3 -m pip freeze | % { python3 -m pip uninstall -y $_ } + python3 -m pip install --upgrade pip + python3 -m pip install -I -r ${{env.MODEL_DIR}}/requirements.cuda.txt + python3 -m pip install python-dotenv + - name: Install dependencies Linux + if: runner.os == 'linux' + run: | + conda create -y -n ${{env.MODEL_NAME}} python=3.11 + source $HOME/miniconda3/bin/activate base + conda init + conda activate ${{env.MODEL_NAME}} + python -m pip install fastapi + python -m pip freeze | xargs python -m pip uninstall -y + python -m pip install --upgrade pip + python -m pip install -r ${{env.MODEL_DIR}}/requirements.cuda.txt + python -m pip install python-dotenv + - name: Install dependencies Mac + if: runner.os == 'macOS' + run: | + conda create -y -n ${{env.MODEL_NAME}} python=3.11 + source $HOME/miniconda3/bin/activate base + conda init + conda activate ${{env.MODEL_NAME}} + python -m pip install fastapi + python -m pip freeze | xargs python -m pip uninstall -y + python -m pip install --upgrade pip + python -m pip install -r ${{env.MODEL_DIR}}/requirements.txt + python -m pip install python-dotenv + + - name: prepare python package windows + if : runner.os == 'windows' + shell: pwsh + run: | + $pythonPath = where.exe python + echo "Python path (where.exe): $pythonPath" + $pythonFolder = Split-Path -Path "$pythonPath" -Parent + echo "PYTHON_FOLDER=$pythonFolder" >> $env:GITHUB_ENV + copy "$pythonFolder\python*.*" "$pythonFolder\Scripts\" + + - name: prepare python package macos + if : runner.os == 'macOs' + run: | + source $HOME/miniconda3/bin/activate base + conda init + conda activate ${{env.MODEL_NAME}} + PYTHON_PATH=$(which python) + echo $PYTHON_PATH + PYTHON_FOLDER=$(dirname $(dirname "$PYTHON_PATH")) + echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV + echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}" + - name: prepare python package linux + if : runner.os == 'linux' + run: | + source $HOME/miniconda3/bin/activate base + conda init + conda activate ${{env.MODEL_NAME}} + PYTHON_PATH=$(which python) + echo $PYTHON_PATH + PYTHON_FOLDER=$(dirname $(dirname "$PYTHON_PATH")) + rm -rf $PYTHON_FOLDER/lib/python3.1 + echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV + echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}" + + - name: create plist file + if: runner.os == 'macOS' + run: | + cat << EOF > /tmp/entitlements.plist + + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + + + com.apple.security.app-sandbox + + com.apple.security.network.client + + com.apple.security.network.server + + com.apple.security.device.audio-input + + com.apple.security.device.microphone + + com.apple.security.device.camera + + com.apple.security.files.user-selected.read-write + + com.apple.security.cs.disable-library-validation + + com.apple.security.cs.allow-dyld-environment-variables + + com.apple.security.cs.allow-executable-memory + + + + EOF + + - name: Notary macOS Binary + if: runner.os == 'macOS' + run: | + codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python + codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python3 + # Code sign all .so files and .dylib files + + find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime {} \; + + curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sudo sh -s -- -b /usr/local/bin + # Notarize the binary + quill notarize ${{env.PYTHON_FOLDER}}/bin/python + quill notarize ${{env.PYTHON_FOLDER}}/bin/python3 + find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec quill notarize {} \; + env: + QUILL_NOTARY_KEY_ID: ${{ secrets.NOTARY_KEY_ID }} + QUILL_NOTARY_ISSUER: ${{ secrets.NOTARY_ISSUER }} + QUILL_NOTARY_KEY: "/tmp/notary-key.p8" + + + - name: Upload Artifact MacOS + if : runner.os == 'macOS' + run: | + brew install zip + cd ${{env.PYTHON_FOLDER}} && zip -r venv.zip * + conda create -y -n hf-upload python=3.11 + source $HOME/miniconda3/bin/activate base + conda init + conda activate hf-upload + python -m pip install hf-transfer huggingface_hub + huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential + huggingface-cli upload ${{env.HF_REPO}} venv.zip --revision ${{env.HF_PREFIX_BRANCH}}-${{ matrix.os }}-${{ matrix.name }} + rm -rf venv.zip + huggingface-cli logout + + - name: Upload Artifact Linux + if : runner.os == 'linux' + run: | + sudo apt-get install -y zip + cd ${{env.PYTHON_FOLDER}} && zip -r venv.zip * + conda create -y -n hf-upload python=3.11 + source $HOME/miniconda3/bin/activate base + conda init + conda activate hf-upload + python -m pip install hf-transfer huggingface_hub + huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential + huggingface-cli upload ${{env.HF_REPO}} venv.zip --revision ${{env.HF_PREFIX_BRANCH}}-${{ matrix.os }}-${{ matrix.name }} + rm -rf venv.zip + huggingface-cli logout + + + - name: Upload Artifact Windows + if : runner.os == 'windows' + shell: pwsh + run: | + Compress-Archive -Path ${{env.PYTHON_FOLDER}}/* -DestinationPath venv.zip + python -m pip install hf-transfer huggingface_hub + huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential + huggingface-cli upload ${{env.HF_REPO}} venv.zip --revision ${{env.HF_PREFIX_BRANCH}}-${{ matrix.os }}-${{ matrix.name }} + rm venv.zip + huggingface-cli logout + + + - name: Post Upload windows + if : runner.os == 'windows' + run: | + rm ${{env.PYTHON_FOLDER}}/Scripts/python*.* + + - name: Remove Keychain + continue-on-error: true + if: always() && runner.os == 'macOS' + run: | + security delete-keychain signing_temp.keychain