From 0293cea1bc8afe4ebccce05bcf21f5509880f82a Mon Sep 17 00:00:00 2001 From: Kiuk Chung Date: Mon, 26 Jul 2021 17:19:57 -0700 Subject: [PATCH] Remove torchelastic circleci configs (since torchelastic has been upstreamed to pytorch and tsm to torchx) Summary: All the code in https://github.com/pytorch/elastic have either been upstreamed to https://github.com/pytorch/pytorch or https://github.com/pytorch/torchx. Removing unnecessary CI actions since they test nothing and are useless but get kicked off and fail 100% of the time. Reviewed By: d4l3k Differential Revision: D29919766 fbshipit-source-id: 556fff83a8bd97a25359854c5fc4364f53d689bf --- .circleci/config.yml | 206 -------------------------------- torchelastic/tsm/test/suites.py | 50 -------- 2 files changed, 256 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 torchelastic/tsm/test/suites.py diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 036ec2ba..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,206 +0,0 @@ -# Python CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-python/ for more details -# -version: 2 - -# ------------------------------------------------------------------------------ -# Test envs -# ------------------------------------------------------------------------------ -cpu: &cpu - machine: - image: ubuntu-1604:201903-01 - resource_class: medium - -gpu: &gpu - environment: - CUDA_VERSION: "10.1" - machine: - image: ubuntu-1604-cuda-10.1:201909-23 - resource_class: gpu.nvidia.medium # turing (https://circleci.com/docs/2.0/configuration-reference/#gpu-executor-linux) - -# ------------------------------------------------------------------------------ -# test setup commands -# ------------------------------------------------------------------------------ -install_pyenv: &install_pyenv - - run: - name: Install pyenv - working_directory: ~/ - command: | - rm -rf $(pyenv root) - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - pyenv update - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - -install_python: &install_python - - run: - name: Install Python - command: | - pyenv install -s 3.8.4 - pyenv global 3.8.4 - -setup_venv: &setup_venv - - run: - name: Setup Python Virtual Environment - command: | - pyenv virtualenv -f 3.8.4 venv38 - pyenv local venv38 - pip install --upgrade pip - -install_dep: &install_dep - - run: - name: Install Dependencies - command: | - echo "In venv: $(pyenv local) - $(python --version), $(pip --version)" - python --version - pip --version - # TODO remove when torch-1.8.0 releases - pip uninstall -y torch - if [[ -z ${CUDA_VERSION} ]]; then - pip install --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - else - pip install --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html - fi - pip install --progress-bar off -r requirements.txt - pip install --progress-bar off black isort - pip list - -check_cuda: &check_cuda - - run: - name: Check CUDA - working_directory: ~/ - command: | - # CUDA installed with machine image - nvidia-smi - -install_etcd: &install_etcd - - run: - name: Install etcd - command: | - # pulls etcd server binary for spinning up a standalone etcd server for unittests - # see test/p2p/etcd_server_fixture.py - # needs to run AFTER checkout since it uses the install_etcd script in the repo - echo "current directory is $(pwd)" - echo "etcd install directory is: $ETCD_INSTALL_DIR" - examples/bin/install_etcd -d $ETCD_INSTALL_DIR - $ETCD_INSTALL_DIR/etcd --version - $ETCD_INSTALL_DIR/etcdctl version - -run_tests: &run_tests - - run: - name: Run Tests - command: | - python setup.py test -s torchelastic.test.suites.unittests -v - -# ------------------------------------------------------------------------------ -# tests -# ------------------------------------------------------------------------------ -jobs: - linter: - <<: *cpu - working_directory: ~/torchelastic - steps: - - checkout - - - <<: *install_pyenv - - # Download and cache python virtual environments (python + deps) - # Share cache with cpu_tests (only restore from) - - restore_cache: - keys: - - v1-cpu-venvs-{{ arch }}-{{ checksum "requirements.txt" }}- - - - <<: *install_python - - - <<: *setup_venv - - - <<: *install_dep - - - run: - name: run isort and black - command: | - bash ./scripts/formatter_python.sh - - cpu_tests: - <<: *cpu - - environment: - ETCD_INSTALL_DIR: /tmp/etcd - TORCHELASTIC_ETCD_BINARY_PATH: /tmp/etcd/etcd - - working_directory: ~/torchelastic - - steps: - - checkout - - - <<: *install_etcd - - - <<: *install_pyenv - - # Download and cache python virtual environments (python + deps) - - restore_cache: - keys: - - v1-cpu-venvs-{{ arch }}-{{ checksum "requirements.txt" }}- - - - <<: *install_python - - - <<: *setup_venv - - - <<: *install_dep - - - save_cache: - key: v1-cpu-venvs-{{ arch }}-{{ checksum "requirements.txt" }}-{{ epoch }} - paths: - - /opt/circleci/.pyenv/versions - - - - <<: *run_tests - - - gpu_tests: - <<: *gpu - - working_directory: ~/torchelastic - - steps: - - checkout - - - <<: *check_cuda - - - <<: *install_pyenv - - # Download and cache python virtual environments (python + deps) - - restore_cache: - keys: - - v1-gpu-venvs-{{ arch }}-{{ checksum "requirements.txt" }}- - - - <<: *install_python - - - <<: *setup_venv - - - <<: *install_dep - - - run: - name: Check CUDA Available - command: python -c "import torch; assert torch.cuda.is_available(), 'CUDA not available'" - - - save_cache: - paths: - - /opt/circleci/.pyenv/versions - key: v1-gpu-venvs-{{ arch }}-{{ checksum "requirements.txt" }}-{{ epoch }} - - - <<: *run_tests - -# ------------------------------------------------------------------------------------- -# Workflows -# ------------------------------------------------------------------------------------- -workflows: - version: 2 - build_and_test: - jobs: - - linter - - cpu_tests - # no gpu tests for now; uncomment to enable - # - gpu_tests diff --git a/torchelastic/tsm/test/suites.py b/torchelastic/tsm/test/suites.py deleted file mode 100644 index f86aaebb..00000000 --- a/torchelastic/tsm/test/suites.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import os -import random -import unittest -from itertools import chain - - -def _circleci_parallelism(suite): - """Allow for parallelism in CircleCI for speedier tests..""" - if int(os.environ.get("CIRCLE_NODE_TOTAL", 0)) <= 1: - # either not running on circleci, or we're not using parallelism. - return suite - # tests are automatically sorted by discover, so we will get the same ordering - # on all hosts. - total = int(os.environ["CIRCLE_NODE_TOTAL"]) - index = int(os.environ["CIRCLE_NODE_INDEX"]) - - # right now each test is corresponds to a /file/. Certain files are slower than - # others, so we want to flatten it - tests = [testfile._tests for testfile in suite._tests] - tests = list(chain.from_iterable(tests)) - random.Random(42).shuffle(tests) - tests = [t for i, t in enumerate(tests) if i % total == index] - return unittest.TestSuite(tests) - - -def unittests(): - """ - Short tests. - - Runs on CircleCI on every commit. Returns everything in the tests root directory. - """ - test_loader = unittest.TestLoader() - test_suite = test_loader.discover( - "torchelastic/tsm", pattern="*_test.py", top_level_dir="." - ) - test_suite = _circleci_parallelism(test_suite) - return test_suite - - -if __name__ == "__main__": - runner = unittest.TextTestRunner() - runner.run(unittests())