ci: cache PyTorch home (#1574)

* ci: cache PyTorch home
* pip cache on GPU

.azure/gpu-pipeline.yml

@@ -55,6 +55,7 @@ jobs:
     - bash: |
         echo "##vso[task.setvariable variable=CONTAINER_ID]$(head -1 /proc/self/cgroup|cut -d/ -f3)"
         echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
+        echo "##vso[task.setvariable variable=PIP_CACHE_DIR]/var/tmp/pip"
       displayName: 'Set environment variables'
 
     - bash: |
@@ -66,6 +67,7 @@ jobs:
         echo $CONTAINER_ID
         python --version
         pip --version
+        pip cache dir
         pip list
         python -c "import torch ; print(torch.cuda.get_arch_list())"
       displayName: 'Image info & NVIDIA'
@@ -87,9 +89,8 @@ jobs:
         set -ex
         sudo apt-get update -qq --fix-missing
         sudo apt-get install -y build-essential gcc g++ cmake ffmpeg git libsndfile1 unzip --no-install-recommends
-        pip install pip -U
+        # pip install pip -U
         pip install -e . -r ./requirements/devel.txt
-        # pip install pandas --force-reinstall  # fixing some strange numpy error for oldest config.
         pip install mkl-service==2.4.0  # needed for the gpu multiprocessing
         pip list
       displayName: 'Install environment'

.github/actions/pull-caches/action.yml

@@ -36,6 +36,8 @@ runs:
   - name: Define caches
     id: cache_dirs
     run: |
+      torch_cache=$(python -c "import os ; print(os.path.join(os.getcwd(), '_ci-cache_pytorch'))")
+      echo "TORCH_HOME=$torch_cache" >> $GITHUB_ENV
       hf_cache=$(python -c "import os ; print(os.path.join(os.getcwd(), '_ci-cache_huggingface'))")
       echo "TRANSFORMERS_CACHE=$hf_cache" >> $GITHUB_ENV
     shell: bash
@@ -54,6 +56,22 @@ runs:
       ls -lh ${{ inputs.pypi-dir }}
     shell: bash
 
+  - name: Cache Torch
+    continue-on-error: true
+    uses: actions/cache/restore@v3
+    with:
+      path: ${{ env.TORCH_HOME }}
+      key: cache-pytorch
+
+  - name: Restored PT
+    if: ${{ runner.os == 'Linux' }}
+    run: |
+      mkdir -p $TORCH_HOME
+      printf "list $TORCH_HOME:\n"
+      sudo apt install -q -y tree
+      tree -h $TORCH_HOME
+    shell: bash
+
   - name: Cache HF
     continue-on-error: true
     uses: actions/cache/restore@v3

.github/actions/push-caches/action.yml

@@ -78,6 +78,20 @@ runs:
       path: ${{ inputs.pypi-dir }}
       key: ${{ inputs.pypi-key }}
 
+  - name: Post PT
+    if: ${{ runner.os == 'Linux' }}
+    run: |
+      printf "list $TORCH_HOME:\n"
+      tree -h $TORCH_HOME
+    shell: bash
+
+  - name: Cache Torch
+    continue-on-error: true
+    uses: actions/cache/save@v3
+    with:
+      path: ${{ env.TORCH_HOME }}
+      key: cache-pytorch
+
   - name: Post HF
     run: |
       printf "list $TRANSFORMERS_CACHE:\n"