From 79a74968b508b1a54ae73e4a56e6fb42d1ca46ed Mon Sep 17 00:00:00 2001 From: vmoens Date: Mon, 9 Oct 2023 11:48:22 +0100 Subject: [PATCH] init --- .circleci/config.yml | 753 ------------------ .../unittest/linux/scripts/environment.yml | 0 .../unittest/linux/scripts/install.sh | 0 .../unittest/linux/scripts/post_process.sh | 0 .../linux/scripts/run-clang-format.py | 0 .../unittest/linux/scripts/run_test.sh | 0 .../unittest/linux/scripts/setup_env.sh | 0 .../linux_stable/scripts/environment.yml | 0 .../unittest/linux_stable/scripts/install.sh | 0 .../linux_stable/scripts/post_process.sh | 0 .../linux_stable/scripts/run-clang-format.py | 0 .../unittest/linux_stable/scripts/run_test.sh | 0 .../linux_stable/scripts/setup_env.sh | 0 .../linux_torchrec/scripts/environment.yml | 0 .../linux_torchrec/scripts/install.sh | 0 .../linux_torchrec/scripts/post_process.sh | 0 .../scripts/run-clang-format.py | 0 .../linux_torchrec/scripts/run_test.sh | 0 .../linux_torchrec/scripts/setup_env.sh | 0 .../rl_linux_optdeps/scripts/environment.yml | 0 .../rl_linux_optdeps/scripts/install.sh | 0 .../rl_linux_optdeps/scripts/post_process.sh | 0 .../scripts/run-clang-format.py | 0 .../rl_linux_optdeps/scripts/run_test.sh | 0 .../rl_linux_optdeps/scripts/setup_env.sh | 0 .github/workflows/lint.yml | 2 +- .github/workflows/test-linux.yml | 32 +- .github/workflows/test-macos.yml | 16 +- .github/workflows/test-rl-gpu.yml | 8 +- setup.cfg | 2 +- 30 files changed, 30 insertions(+), 783 deletions(-) delete mode 100644 .circleci/config.yml rename {.circleci => .github}/unittest/linux/scripts/environment.yml (100%) rename {.circleci => .github}/unittest/linux/scripts/install.sh (100%) rename {.circleci => .github}/unittest/linux/scripts/post_process.sh (100%) rename {.circleci => .github}/unittest/linux/scripts/run-clang-format.py (100%) rename {.circleci => .github}/unittest/linux/scripts/run_test.sh (100%) rename {.circleci => .github}/unittest/linux/scripts/setup_env.sh (100%) rename {.circleci => .github}/unittest/linux_stable/scripts/environment.yml (100%) rename {.circleci => .github}/unittest/linux_stable/scripts/install.sh (100%) rename {.circleci => .github}/unittest/linux_stable/scripts/post_process.sh (100%) rename {.circleci => .github}/unittest/linux_stable/scripts/run-clang-format.py (100%) rename {.circleci => .github}/unittest/linux_stable/scripts/run_test.sh (100%) rename {.circleci => .github}/unittest/linux_stable/scripts/setup_env.sh (100%) rename {.circleci => .github}/unittest/linux_torchrec/scripts/environment.yml (100%) rename {.circleci => .github}/unittest/linux_torchrec/scripts/install.sh (100%) rename {.circleci => .github}/unittest/linux_torchrec/scripts/post_process.sh (100%) rename {.circleci => .github}/unittest/linux_torchrec/scripts/run-clang-format.py (100%) rename {.circleci => .github}/unittest/linux_torchrec/scripts/run_test.sh (100%) rename {.circleci => .github}/unittest/linux_torchrec/scripts/setup_env.sh (100%) rename {.circleci => .github}/unittest/rl_linux_optdeps/scripts/environment.yml (100%) rename {.circleci => .github}/unittest/rl_linux_optdeps/scripts/install.sh (100%) rename {.circleci => .github}/unittest/rl_linux_optdeps/scripts/post_process.sh (100%) rename {.circleci => .github}/unittest/rl_linux_optdeps/scripts/run-clang-format.py (100%) rename {.circleci => .github}/unittest/rl_linux_optdeps/scripts/run_test.sh (100%) rename {.circleci => .github}/unittest/rl_linux_optdeps/scripts/setup_env.sh (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 783d935b3..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,753 +0,0 @@ -version: 2.1 - -executors: - windows-cpu: - machine: - resource_class: windows.xlarge - image: windows-server-2019-vs2019:stable - shell: bash.exe - - windows-gpu: - machine: - resource_class: windows.gpu.nvidia.medium - image: windows-server-2019-nvidia:stable - shell: bash.exe - - -commands: - checkout_merge: - description: "checkout merge branch" - steps: - - checkout - designate_upload_channel: - description: "inserts the correct upload channel into ${BASH_ENV}" - steps: - - run: - name: adding UPLOAD_CHANNEL to BASH_ENV - command: | - our_upload_channel=nightly - # On tags upload to test instead - if [[ -n "${CIRCLE_TAG}" ]] || [[ ${CIRCLE_BRANCH} =~ release/* ]]; then - our_upload_channel=test - fi - echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - apt_install: - parameters: - args: - type: string - descr: - type: string - default: "" - update: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> apt install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: | - <<# parameters.update >> sudo apt update -qy <> - sudo apt install << parameters.args >> - pip_install: - parameters: - args: - type: string - descr: - type: string - default: "" - user: - type: boolean - default: true - steps: - - run: - name: > - <<^ parameters.descr >> pip install << parameters.args >> <> - <<# parameters.descr >> << parameters.descr >> <> - command: > - pip install - <<# parameters.user >> --user <> - --progress-bar=off - << parameters.args >> - - install_tensordict: - parameters: - editable: - type: boolean - default: true - steps: - - pip_install: - args: --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - descr: Install PyTorch from nightly releases - - pip_install: - args: --no-build-isolation <<# parameters.editable >> --editable <> . - descr: Install tensordict <<# parameters.editable >> in editable mode <> - - -binary_common: &binary_common - parameters: - # Edit these defaults to do a release - build_version: - description: "version number of release binary; by default, build a nightly" - type: string - default: "" - pytorch_version: - description: "PyTorch version to build against; by default, use a nightly" - type: string - default: "" - # Don't edit these - python_version: - description: "Python version to build against (e.g., 3.7)" - type: string - cu_version: - description: "CUDA version to build against, in CU format (e.g., cpu or cu100)" - type: string - default: "cpu" - unicode_abi: - description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)" - type: string - default: "" - wheel_docker_image: - description: "Wheel only: what docker image to use" - type: string - default: "pytorch/manylinux-cuda113" - conda_docker_image: - description: "Conda only: what docker image to use" - type: string - default: "pytorch/conda-builder:cpu" - environment: - PYTHON_VERSION: << parameters.python_version >> - PYTORCH_VERSION: << parameters.pytorch_version >> - UNICODE_ABI: << parameters.unicode_abi >> - CU_VERSION: << parameters.cu_version >> - -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: tensordict/smoke_test:latest - -jobs: -# circleci_consistency: -# docker: -# - image: circleci/python:3.7 -# steps: -# - checkout -# - pip_install: -# args: jinja2 pyyaml -# - run: -# name: Check CircleCI config consistency -# command: | -# python .circleci/regenerate.py -# git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) - - lint_python_and_config: - docker: - - image: circleci/python:3.8 - steps: - - checkout - - pip_install: - args: pre-commit - descr: Install lint utilities - - run: - name: Install pre-commit hooks - command: pre-commit install-hooks - - run: - name: Lint Python code and config files - command: pre-commit run --all-files - - run: - name: Required lint modifications - when: on_fail - command: git --no-pager diff - - # lint_c: - # docker: - # - image: circleci/python:3.7 - # steps: - # - apt_install: - # args: libtinfo5 - # descr: Install additional system libraries - # - checkout - # - run: - # name: Install lint utilities - # command: | - # curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format - # chmod +x clang-format - # sudo mv clang-format /opt/clang-format - # - run: - # name: Lint C code - # command: ./.circleci/unittest/linux/scripts/run-clang-format.py -r tensordict/csrc --clang-format-executable /opt/clang-format - # - run: - # name: Required lint modifications - # when: on_fail - # command: git --no-pager diff - - type_check_python: - docker: - - image: circleci/python:3.8 - steps: - - checkout - - pip_install: - args: cmake ninja - descr: Install CMake and Ninja - - install_tensordict: - editable: true - - pip_install: - args: mypy - descr: Install Python type check utilities - - run: - name: Check Python types statically - command: mypy --install-types --non-interactive --config-file mypy.ini - - binary_linux_wheel: - <<: *binary_common - docker: - - image: << parameters.wheel_docker_image >> - resource_class: 2xlarge+ - steps: - - checkout_merge - - designate_upload_channel - - run: packaging/build_wheels.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_macos_wheel: - <<: *binary_common - macos: - xcode: "14.0" - steps: - - checkout_merge - - designate_upload_channel - - run: - # Cannot easily deduplicate this as source'ing activate - # will set environment variables which we need to propagate - # to build_wheel.sh - command: | - curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh - sh conda.sh -b - source $HOME/miniconda3/bin/activate - packaging/build_wheels.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - unittest_linux_cpu: - <<: *binary_common - - docker: - - image: "pytorch/manylinux-cuda113" - resource_class: 2xlarge+ - - environment: - TAR_OPTIONS: --no-same-owner - PYTHON_VERSION: << parameters.python_version >> - CU_VERSION: << parameters.cu_version >> - - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - keys: - - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - - save_cache: - - key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install tensordict - command: .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Codecov upload - command: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -t ${CODECOV_TOKEN} -s ./ -Z -F linux-cpu - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.medium - environment: - image_name: "pytorch/manylinux-cuda113" - TAR_OPTIONS: --no-same-owner - PYTHON_VERSION: << parameters.python_version >> - CU_VERSION: << parameters.cu_version >> - - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env -# - run: -# # Here we create an envlist file that contains some env variables that we want the docker container to be aware of. -# # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables. -# # They're available in all the other workflows (OSX and Windows). -# # But here, we're running the unittest_linux_gpu workflows in a docker container, where those variables aren't accessible. -# # So instead we dump the variables we need in env.list and we pass that file when invoking "docker run". -# name: export CIRCLECI env var -# command: echo "CIRCLECI=true" >> ./env.list - - run: - name: Install tensordict -# command: bash .circleci/unittest/linux/scripts/install.sh - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: bash .circleci/unittest/linux/scripts/run_test.sh -# command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Codecov upload - command: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -t ${CODECOV_TOKEN} -s ./ -Z -F linux-gpu - - run: - name: Post Process - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_linux_torchrec_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.medium - environment: - image_name: "pytorch/manylinux-cuda113" - TAR_OPTIONS: --no-same-owner - PYTHON_VERSION: << parameters.python_version >> - CU_VERSION: << parameters.cu_version >> - - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_torchrec/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux_torchrec/scripts/setup_env.sh - - save_cache: - - key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_torchrec/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env -# - run: -# # Here we create an envlist file that contains some env variables that we want the docker container to be aware of. -# # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables. -# # They're available in all the other workflows (OSX and Windows). -# # But here, we're running the unittest_linux_gpu workflows in a docker container, where those variables aren't accessible. -# # So instead we dump the variables we need in env.list and we pass that file when invoking "docker run". -# name: export CIRCLECI env var -# command: echo "CIRCLECI=true" >> ./env.list - - run: - name: Install tensordict -# command: bash .circleci/unittest/linux_torchrec/scripts/install.sh - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux_torchrec/scripts/install.sh - - run: - name: Run tests - command: bash .circleci/unittest/linux_torchrec/scripts/run_test.sh -# command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux_torchrec/scripts/run_test.sh - - run: - name: Codecov upload - command: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -t ${CODECOV_TOKEN} -s ./ -Z -F linux-torchrec-gpu - - run: - name: Post Process - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux_torchrec/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_linux_stable_cpu: - <<: *binary_common - - docker: - - image: "pytorch/manylinux-cuda113" - resource_class: 2xlarge+ - - environment: - TAR_OPTIONS: --no-same-owner - PYTHON_VERSION: << parameters.python_version >> - CU_VERSION: << parameters.cu_version >> - - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/linux_stable/scripts/setup_env.sh - - - save_cache: - - key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install tensordict - command: .circleci/unittest/linux_stable/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/linux_stable/scripts/run_test.sh - - run: - name: Codecov upload - command: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -t ${CODECOV_TOKEN} -s ./ -Z -F linux-stable-cpu - - run: - name: Post process - command: .circleci/unittest/linux_stable/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_linux_stable_gpu: - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.medium - environment: - image_name: "pytorch/manylinux-cuda113" - TAR_OPTIONS: --no-same-owner - PYTHON_VERSION: << parameters.python_version >> - CU_VERSION: << parameters.cu_version >> - - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux_stable/scripts/setup_env.sh - - save_cache: - - key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install tensordict -# command: bash .circleci/unittest/linux_stable/scripts/install.sh - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: bash .circleci/unittest/linux_stable/scripts/run_test.sh -# command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Codecov upload - command: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -t ${CODECOV_TOKEN} -s ./ -Z -F linux-stable-gpu - - run: - name: Post Process - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux_stable/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_macos_cpu: - <<: *binary_common - macos: - xcode: "13.0" - - resource_class: large - steps: - - checkout - - designate_upload_channel - - run: - name: Install wget - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget - # Disable brew auto update which is very slow - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: - - key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env - - run: - name: Install tensordict - command: .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Codecov upload - command: | - curl -Os https://uploader.codecov.io/latest/macos/codecov - chmod +x codecov - ./codecov -t ${CODECOV_TOKEN} -s ./ -Z -F linux-macos-cpu - - run: - name: Post process - command: .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results - - unittest_rl_linux_optdeps_gpu: - # A hopefully temporary pipeline - <<: *binary_common - machine: - image: ubuntu-2004-cuda-11.4:202110-01 - resource_class: gpu.nvidia.medium - environment: - image_name: "nvidia/cudagl:11.4.0-base" - TAR_OPTIONS: --no-same-owner - PYTHON_VERSION: << parameters.python_version >> - CU_VERSION: << parameters.cu_version >> - - steps: - - checkout - - designate_upload_channel - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: - - keys: - - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/rl_linux_optdeps/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - - run: - name: Setup - command: .circleci/unittest/rl_linux_optdeps/scripts/setup_env.sh - - save_cache: - - key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/rl_linux_optdeps/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - paths: - - conda - - env -# - run: -# # Here we create an envlist file that contains some env variables that we want the docker container to be aware of. -# # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables. -# # They're available in all the other workflows (OSX and Windows). -# # But here, we're running the unittest_linux_gpu workflows in a docker container, where those variables aren't accessible. -# # So instead we dump the variables we need in env.list and we pass that file when invoking "docker run". -# name: export CIRCLECI env var -# command: echo "CIRCLECI=true" >> ./env.list - - run: - name: Install torchrl -# command: bash .circleci/unittest/rl_linux_optdeps/scripts/install.sh - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/rl_linux_optdeps/scripts/install.sh - - run: - name: Run tests - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/rl_linux_optdeps/scripts/run_test.sh -# command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Codecov upload - command: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov -t ${CODECOV_TOKEN} -s ./ -Z -F linux-outdeps-gpu - - run: - name: Post Process - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/rl_linux_optdeps/scripts/post_process.sh - - store_test_results: - path: test-results - -workflows: - lint: - jobs: -# - circleci_consistency - - lint_python_and_config -# - lint_c -# - type_check_python - - build: - jobs: - - binary_linux_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_linux_wheel_py3.8_cpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cuda102 - - - binary_linux_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_linux_wheel_py3.9_cpu - python_version: '3.9' - wheel_docker_image: pytorch/manylinux-cuda102 - - - binary_linux_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_linux_wheel_py3.10_cpu - python_version: '3.10' - wheel_docker_image: pytorch/manylinux-cuda102 - - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_macos_wheel_py3.8_cpu - python_version: '3.8' - wheel_docker_image: pytorch/manylinux-cuda102 - - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_macos_wheel_py3.9_cpu - python_version: '3.9' - wheel_docker_image: pytorch/manylinux-cuda102 - - - binary_macos_wheel: - conda_docker_image: pytorch/conda-builder:cpu - cu_version: cpu - name: binary_macos_wheel_py3.10_cpu - python_version: '3.10' - wheel_docker_image: pytorch/manylinux-cuda102 - - unittest: - jobs: - - unittest_macos_cpu: - cu_version: cpu - name: unittest_macos_cpu_py3.8 - python_version: '3.8' - - unittest_linux_cpu: - cu_version: cpu - name: unittest_linux_cpu_py3.8 - python_version: '3.8' - - unittest_linux_gpu: - cu_version: cu113 - name: unittest_linux_gpu_py3.8 - python_version: '3.8' - - unittest_linux_stable_cpu: - cu_version: cpu - name: unittest_linux_stable_cpu_py3.8 - python_version: '3.8' - - unittest_linux_stable_gpu: - cu_version: cu113 - name: unittest_linux_stable_gpu_py3.8 - python_version: '3.8' - - - unittest_macos_cpu: - cu_version: cpu - name: unittest_macos_cpu_py3.9 - python_version: '3.9' - - unittest_linux_cpu: - cu_version: cpu - name: unittest_linux_cpu_py3.9 - python_version: '3.9' - - unittest_linux_gpu: - cu_version: cu113 - name: unittest_linux_gpu_py3.9 - python_version: '3.9' - - unittest_linux_stable_cpu: - cu_version: cpu - name: unittest_linux_stable_cpu_py3.9 - python_version: '3.9' - - unittest_linux_stable_gpu: - cu_version: cu113 - name: unittest_linux_stable_gpu_py3.9 - python_version: '3.9' - - unittest_macos_cpu: - cu_version: cpu - name: unittest_macos_cpu_py3.10 - python_version: '3.10' - - unittest_linux_cpu: - cu_version: cpu - name: unittest_linux_cpu_py3.10 - python_version: '3.10' - - unittest_linux_gpu: - cu_version: cu113 - name: unittest_linux_gpu_py3.10 - python_version: '3.10' - - unittest_linux_stable_cpu: - cu_version: cpu - name: unittest_linux_stable_cpu_py3.10 - python_version: '3.10' - - unittest_linux_stable_gpu: - cu_version: cu113 - name: unittest_linux_stable_gpu_py3.10 - python_version: '3.10' - - - unittest_rl_linux_optdeps_gpu: - cu_version: cu117 - name: unittest_rl_linux_optdeps_gpu_py3.9 - python_version: '3.9' - filters: - branches: - ignore: main diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.github/unittest/linux/scripts/environment.yml similarity index 100% rename from .circleci/unittest/linux/scripts/environment.yml rename to .github/unittest/linux/scripts/environment.yml diff --git a/.circleci/unittest/linux/scripts/install.sh b/.github/unittest/linux/scripts/install.sh similarity index 100% rename from .circleci/unittest/linux/scripts/install.sh rename to .github/unittest/linux/scripts/install.sh diff --git a/.circleci/unittest/linux/scripts/post_process.sh b/.github/unittest/linux/scripts/post_process.sh similarity index 100% rename from .circleci/unittest/linux/scripts/post_process.sh rename to .github/unittest/linux/scripts/post_process.sh diff --git a/.circleci/unittest/linux/scripts/run-clang-format.py b/.github/unittest/linux/scripts/run-clang-format.py similarity index 100% rename from .circleci/unittest/linux/scripts/run-clang-format.py rename to .github/unittest/linux/scripts/run-clang-format.py diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.github/unittest/linux/scripts/run_test.sh similarity index 100% rename from .circleci/unittest/linux/scripts/run_test.sh rename to .github/unittest/linux/scripts/run_test.sh diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.github/unittest/linux/scripts/setup_env.sh similarity index 100% rename from .circleci/unittest/linux/scripts/setup_env.sh rename to .github/unittest/linux/scripts/setup_env.sh diff --git a/.circleci/unittest/linux_stable/scripts/environment.yml b/.github/unittest/linux_stable/scripts/environment.yml similarity index 100% rename from .circleci/unittest/linux_stable/scripts/environment.yml rename to .github/unittest/linux_stable/scripts/environment.yml diff --git a/.circleci/unittest/linux_stable/scripts/install.sh b/.github/unittest/linux_stable/scripts/install.sh similarity index 100% rename from .circleci/unittest/linux_stable/scripts/install.sh rename to .github/unittest/linux_stable/scripts/install.sh diff --git a/.circleci/unittest/linux_stable/scripts/post_process.sh b/.github/unittest/linux_stable/scripts/post_process.sh similarity index 100% rename from .circleci/unittest/linux_stable/scripts/post_process.sh rename to .github/unittest/linux_stable/scripts/post_process.sh diff --git a/.circleci/unittest/linux_stable/scripts/run-clang-format.py b/.github/unittest/linux_stable/scripts/run-clang-format.py similarity index 100% rename from .circleci/unittest/linux_stable/scripts/run-clang-format.py rename to .github/unittest/linux_stable/scripts/run-clang-format.py diff --git a/.circleci/unittest/linux_stable/scripts/run_test.sh b/.github/unittest/linux_stable/scripts/run_test.sh similarity index 100% rename from .circleci/unittest/linux_stable/scripts/run_test.sh rename to .github/unittest/linux_stable/scripts/run_test.sh diff --git a/.circleci/unittest/linux_stable/scripts/setup_env.sh b/.github/unittest/linux_stable/scripts/setup_env.sh similarity index 100% rename from .circleci/unittest/linux_stable/scripts/setup_env.sh rename to .github/unittest/linux_stable/scripts/setup_env.sh diff --git a/.circleci/unittest/linux_torchrec/scripts/environment.yml b/.github/unittest/linux_torchrec/scripts/environment.yml similarity index 100% rename from .circleci/unittest/linux_torchrec/scripts/environment.yml rename to .github/unittest/linux_torchrec/scripts/environment.yml diff --git a/.circleci/unittest/linux_torchrec/scripts/install.sh b/.github/unittest/linux_torchrec/scripts/install.sh similarity index 100% rename from .circleci/unittest/linux_torchrec/scripts/install.sh rename to .github/unittest/linux_torchrec/scripts/install.sh diff --git a/.circleci/unittest/linux_torchrec/scripts/post_process.sh b/.github/unittest/linux_torchrec/scripts/post_process.sh similarity index 100% rename from .circleci/unittest/linux_torchrec/scripts/post_process.sh rename to .github/unittest/linux_torchrec/scripts/post_process.sh diff --git a/.circleci/unittest/linux_torchrec/scripts/run-clang-format.py b/.github/unittest/linux_torchrec/scripts/run-clang-format.py similarity index 100% rename from .circleci/unittest/linux_torchrec/scripts/run-clang-format.py rename to .github/unittest/linux_torchrec/scripts/run-clang-format.py diff --git a/.circleci/unittest/linux_torchrec/scripts/run_test.sh b/.github/unittest/linux_torchrec/scripts/run_test.sh similarity index 100% rename from .circleci/unittest/linux_torchrec/scripts/run_test.sh rename to .github/unittest/linux_torchrec/scripts/run_test.sh diff --git a/.circleci/unittest/linux_torchrec/scripts/setup_env.sh b/.github/unittest/linux_torchrec/scripts/setup_env.sh similarity index 100% rename from .circleci/unittest/linux_torchrec/scripts/setup_env.sh rename to .github/unittest/linux_torchrec/scripts/setup_env.sh diff --git a/.circleci/unittest/rl_linux_optdeps/scripts/environment.yml b/.github/unittest/rl_linux_optdeps/scripts/environment.yml similarity index 100% rename from .circleci/unittest/rl_linux_optdeps/scripts/environment.yml rename to .github/unittest/rl_linux_optdeps/scripts/environment.yml diff --git a/.circleci/unittest/rl_linux_optdeps/scripts/install.sh b/.github/unittest/rl_linux_optdeps/scripts/install.sh similarity index 100% rename from .circleci/unittest/rl_linux_optdeps/scripts/install.sh rename to .github/unittest/rl_linux_optdeps/scripts/install.sh diff --git a/.circleci/unittest/rl_linux_optdeps/scripts/post_process.sh b/.github/unittest/rl_linux_optdeps/scripts/post_process.sh similarity index 100% rename from .circleci/unittest/rl_linux_optdeps/scripts/post_process.sh rename to .github/unittest/rl_linux_optdeps/scripts/post_process.sh diff --git a/.circleci/unittest/rl_linux_optdeps/scripts/run-clang-format.py b/.github/unittest/rl_linux_optdeps/scripts/run-clang-format.py similarity index 100% rename from .circleci/unittest/rl_linux_optdeps/scripts/run-clang-format.py rename to .github/unittest/rl_linux_optdeps/scripts/run-clang-format.py diff --git a/.circleci/unittest/rl_linux_optdeps/scripts/run_test.sh b/.github/unittest/rl_linux_optdeps/scripts/run_test.sh similarity index 100% rename from .circleci/unittest/rl_linux_optdeps/scripts/run_test.sh rename to .github/unittest/rl_linux_optdeps/scripts/run_test.sh diff --git a/.circleci/unittest/rl_linux_optdeps/scripts/setup_env.sh b/.github/unittest/rl_linux_optdeps/scripts/setup_env.sh similarity index 100% rename from .circleci/unittest/rl_linux_optdeps/scripts/setup_env.sh rename to .github/unittest/rl_linux_optdeps/scripts/setup_env.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 97f0d3250..0818bd228 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -66,7 +66,7 @@ jobs: echo '::group::Lint C source' set +e - ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchrl/csrc --clang-format-executable ./clang-format + ./.github/unittest/linux/scripts/run-clang-format.py -r torchrl/csrc --clang-format-executable ./clang-format if [ $? -ne 0 ]; then git --no-pager diff diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index afaf2d7af..c15842f9e 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -46,10 +46,10 @@ jobs: echo "CU_VERSION: $CU_VERSION" ## setup_env.sh - bash .circleci/unittest/linux/scripts/setup_env.sh - bash .circleci/unittest/linux/scripts/install.sh - bash .circleci/unittest/linux/scripts/run_test.sh - bash .circleci/unittest/linux/scripts/post_process.sh + bash .github/unittest/linux/scripts/setup_env.sh + bash .github/unittest/linux/scripts/install.sh + bash .github/unittest/linux/scripts/run_test.sh + bash .github/unittest/linux/scripts/post_process.sh test-cpu: strategy: @@ -72,10 +72,10 @@ jobs: echo "CU_VERSION: $CU_VERSION" ## setup_env.sh - bash .circleci/unittest/linux/scripts/setup_env.sh - bash .circleci/unittest/linux/scripts/install.sh - bash .circleci/unittest/linux/scripts/run_test.sh - bash .circleci/unittest/linux/scripts/post_process.sh + bash .github/unittest/linux/scripts/setup_env.sh + bash .github/unittest/linux/scripts/install.sh + bash .github/unittest/linux/scripts/run_test.sh + bash .github/unittest/linux/scripts/post_process.sh test-stable-gpu: strategy: @@ -105,10 +105,10 @@ jobs: echo "CU_VERSION: $CU_VERSION" ## setup_env.sh - bash .circleci/unittest/linux/scripts/setup_env.sh - bash .circleci/unittest/linux/scripts/install.sh - bash .circleci/unittest/linux/scripts/run_test.sh - bash .circleci/unittest/linux/scripts/post_process.sh + bash .github/unittest/linux/scripts/setup_env.sh + bash .github/unittest/linux/scripts/install.sh + bash .github/unittest/linux/scripts/run_test.sh + bash .github/unittest/linux/scripts/post_process.sh test-stable-cpu: strategy: @@ -131,7 +131,7 @@ jobs: echo "CU_VERSION: $CU_VERSION" ## setup_env.sh - bash .circleci/unittest/linux/scripts/setup_env.sh - bash .circleci/unittest/linux/scripts/install.sh - bash .circleci/unittest/linux/scripts/run_test.sh - bash .circleci/unittest/linux/scripts/post_process.sh + bash .github/unittest/linux/scripts/setup_env.sh + bash .github/unittest/linux/scripts/install.sh + bash .github/unittest/linux/scripts/run_test.sh + bash .github/unittest/linux/scripts/post_process.sh diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 4347957c6..fc8218c9d 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -42,10 +42,10 @@ jobs: echo "CU_VERSION: $CU_VERSION" ## setup_env.sh - bash .circleci/unittest/linux/scripts/setup_env.sh - bash .circleci/unittest/linux/scripts/install.sh - bash .circleci/unittest/linux/scripts/run_test.sh - bash .circleci/unittest/linux/scripts/post_process.sh + bash .github/unittest/linux/scripts/setup_env.sh + bash .github/unittest/linux/scripts/install.sh + bash .github/unittest/linux/scripts/run_test.sh + bash .github/unittest/linux/scripts/post_process.sh tests-silicon: strategy: @@ -71,7 +71,7 @@ jobs: echo "CU_VERSION: $CU_VERSION" ## setup_env.sh - bash .circleci/unittest/linux/scripts/setup_env.sh - bash .circleci/unittest/linux/scripts/install.sh - bash .circleci/unittest/linux/scripts/run_test.sh - bash .circleci/unittest/linux/scripts/post_process.sh + bash .github/unittest/linux/scripts/setup_env.sh + bash .github/unittest/linux/scripts/install.sh + bash .github/unittest/linux/scripts/run_test.sh + bash .github/unittest/linux/scripts/post_process.sh diff --git a/.github/workflows/test-rl-gpu.yml b/.github/workflows/test-rl-gpu.yml index d1f97de6e..ab284d754 100644 --- a/.github/workflows/test-rl-gpu.yml +++ b/.github/workflows/test-rl-gpu.yml @@ -45,7 +45,7 @@ jobs: echo "CU_VERSION: $CU_VERSION" ## setup_env.sh - bash .circleci/unittest/rl_linux_optdeps/scripts/setup_env.sh - bash .circleci/unittest/rl_linux_optdeps/scripts/install.sh - bash .circleci/unittest/rl_linux_optdeps/scripts/run_test.sh - bash .circleci/unittest/rl_linux_optdeps/scripts/post_process.sh + bash .github/unittest/rl_linux_optdeps/scripts/setup_env.sh + bash .github/unittest/rl_linux_optdeps/scripts/install.sh + bash .github/unittest/rl_linux_optdeps/scripts/run_test.sh + bash .github/unittest/rl_linux_optdeps/scripts/post_process.sh diff --git a/setup.cfg b/setup.cfg index ef057bb52..6e2c14d79 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,4 +27,4 @@ add-ignore = D100, D104, D105, D107, D102 ignore-decorators = test_* ; test/*.py -; .circleci/* +; .github/*