diff --git a/.circleci/helper-scripts/wrap-with-windows-conda.ps1 b/.circleci/helper-scripts/wrap-with-windows-conda.ps1 index 5f99fa6f6..4a33c6b2e 100644 --- a/.circleci/helper-scripts/wrap-with-windows-conda.ps1 +++ b/.circleci/helper-scripts/wrap-with-windows-conda.ps1 @@ -33,8 +33,8 @@ $Env:CONDA_DLL_SEARCH_MODIFICATION_ENABLE += 1 #echo "Current python version before activation:" #python --version -echo "About to run: bash packaging/build_conda.sh $a" -bash packaging/build_conda.sh $a +echo "About to run: bash $a" +bash $a # Required to propagate bash error out of Powershell: -exit $LastExitCode \ No newline at end of file +exit $LastExitCode diff --git a/README.md b/README.md index 3fbe8fd30..01e18dcbe 100644 --- a/README.md +++ b/README.md @@ -10,5 +10,3 @@ Folders: - **windows** : scripts to build Windows wheels - **cron** : scripts to drive all of the above scripts across multiple configurations together - **analytics** : scripts to pull wheel download count from our AWS s3 logs -- **test_community_repos** : scripts that test built binaries with various downstream repos that are high-priority and use pytorch -- **test_imports_docker** : some smoke tests to import torch in combination with other package imports, to check binary stability / compatibility diff --git a/build_binaries.sh b/build_binaries.sh deleted file mode 100755 index d71afb996..000000000 --- a/build_binaries.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -e - -pushd wheel -./build_all.sh -popd - -pushd conda -./build_all.sh -popd diff --git a/cron/README.md b/cron/README.md deleted file mode 100644 index c95629c78..000000000 --- a/cron/README.md +++ /dev/null @@ -1,92 +0,0 @@ -# Pytorch Binary Nightly Cron Jobs - -This folder contains all the scripts needed for nightly cron jobs which build -and upload the Pytorch binaries every night. - -# Entrypoints - -To manually build a single package: - - cron/build_mac.sh (conda|wheel|libtorch) py_version - Builds either a conda package or wheel for Mac (it must be run on a Mac). - - cron/build_docker.sh (conda|manywheel|libtorch) py_version cuda_version - Builds either a conda package, manywheel, or libtorch package in a docker. - -To manually build multiple packages: - cron/build_multiple.sh (conda|manywheel|wheel|libtorch) pyver[,py_ver...] cuda_ver[,cuda_ver...] - See the comment in cron/build_multiple.sh for example invocations. All packages will be built in serial. Essentially just calls into build_docker.sh or build_mac.sh - -How cron builds packages: -Cron uses ./cron/start_cron.sh, which clones the latest builder repo and then calls cron/build_cron.sh in that. build_cron.sh contains the schedule (which jobs run on which machines) and starts a few queues of 4 jobs each in parallel on each machine. build_cron.sh will also upload the successful packages and clean old nightlies folders. - - -# How the scripts work - -- 3 Linux machines and 1 Mac -- Started by cron every morning -- Each day a new nightlies folder is created like 2018_09_12 that holds all the - builds for the day - -# Example Program Flow - -Let's say the date is September 12, 2018 - -* cron will run based on hellemn's crontab -* cron will call hellemn/builder/cron/cron_start.sh - * This will clone the builder master into /scratch/hellemn/nightlies/2018_09_12/builder - * N.B. how the crontab points to my personal builder checkout, which then clones master's builder and calls into that. This is so that we can always use the latest master of the builder repo without having to manually update the github checkouts on each machine. Caveat, if cron_start.sh is ever changed then the repos will have to be manually updated on every machine. -* nightlies/2018_09_12/builder/cron/build_cron.sh will be called with the machine number [0-2]. This machine number corresponds to the tasks in builder/cron/build_cron.sh, which delegates 13 builds to each machine in 4 parallel tasks - * Each parallel task of 3-4 builds each is logged to 2018_09_12/logs/master/ - * While a build is ongoing its output is logged to 2018_09_12/logs/packagetype_pythonversion_cpucudaversion.log. Whenever a build fails, its log is moved to 2018_09_12/logs/failed. When a build succeeds, its log is moved to 2018_09_12/logs/succeeded. - * N.B., if the machine runs out of memory then sometimes these logs can't be written. -* When all builds finish, cron/upload.sh is called to upload the successful ones. This uses hellemn's credentials, which are sourced from a file that is not in the git repository. -* Since it's the 12th, all builds from the 5th earlier are deleted. In this case, rm -rf nightlies/2018_09_07 will be called. - -# Folder structure - -Each day's build creates a folder with the following structure - -``` -/scratch/hellemn/nightlies/2018_09_12/ - - # The latest builder repo, pytorch/builder's master and pytorch/pytorch's master - builder/ - pytorch/ - - logs/ - master/ - # cron_start.log - # build_cron.log - # upload.log - # Logs for build_multiple.sh calls, - # e.g. manywheel_2.7m2.7mu3.5m_cu92.log - - # When builds finish their logs are moved to one of these folders - failed/ - succeeded/ - - # Currently running builds will have their logs here - # e.g. manywheel_3.6_cpu.log - - # These subdirs hold final built packages. Not all subdirs will exist, - # depending on the jobs running on this machine. Packages are copied here - # after dependencies are tweaked with but before any testing is done, so - # they may be broken. - wheelhousecpu/ - wheelhouse80/ - wheelhouse90/ - wheelhouse92/ - conda_pkgs/ - mac_wheels/ - mac_conda_pkgs/ - - # On mac builds, each build also gets their own folder with their own copy - # pytorch and conda to ensure they don't interfere with each other - wheel_build_dirs/ - wheel_3.6_cpu/ - conda/ - pytorch/ - dist/ # Unfinished packages -``` - diff --git a/cron/backfill_binary_sizes.sh b/cron/backfill_binary_sizes.sh deleted file mode 100755 index beeb49d5b..000000000 --- a/cron/backfill_binary_sizes.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -set -ex - -# This script is used to backfill binary sizes (backfill the .json files stored -# at s3://pytorch/nightly_logs/binary_sizes/). All it does it call -# upload_binary_sizes.sh for a range of dates. This script should not be called -# often, so the ranges of the dates are hardcoded in. - -# The upload_binary_sizes.sh script expects dates with underscores in -# 2019_01_01 format. -# You should make sure that end date is actually after start date, and that -# both are in underscore format -START_DATE="2019_08_03" -END_DATE="$(date +%Y_%m_%d)" - -# The upload_binary_sizes script needs to construct the entire Pytorch binary -# versions in order to query conda for the sizes. There is a built-in -# assumption in many places that the version is like 1.1.0.dev20190101. -# N.B. this means you cannot use this script across a version boundary, as the -# script will only work for part of the range -CURRENT_PYTORCH_VERSION_PREAMBLE="1.2.0.dev" - -# TODO actually compare times instead of string comparisons. It's easy to get -# an infinite loop this way. -current_date="$START_DATE" -while [ "$current_date" != "$END_DATE" ]; do - ./upload_binary_sizes.sh "$current_date" "$CURRENT_PYTORCH_VERSION_PREAMBLE" - - # The date command understands 20190101 and 2019-01-01 but not 2019_01_01 - # without work, so we just remove the '_' in the calculation - current_date="$(date +%Y_%m_%d -d "$(echo $current_date | tr -d '_') + 1 day")" -done diff --git a/cron/build_cron.sh b/cron/build_cron.sh deleted file mode 100755 index 82b81caba..000000000 --- a/cron/build_cron.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/bin/bash - -set -ex -echo "build_cron.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$" -SOURCE_DIR=$(cd $(dirname $0) && pwd) -source "${SOURCE_DIR}/nightly_defaults.sh" - -# Script hardcoded to the number of worker machines we have. -# Divides work amongst the workers and runs the jobs in parallel on each worker -# -# Command line arguments -# DESIRED_PYTHONS -# All Python versions to build for, separated by commas, in format '2.7mu' -# for manywheels or in format '2.7' for conda/mac-wheels e.g. -# '2.7m,2.7mu,3.5m,3.6m' or '2.7,3.7' . This can also just be the word -# 'all', which will expand to all supported python versions. -# -# DESIRED_CUDAS -# All CUDA versions to build for including 'cpu', separated by commas, in -# format 'cpu' or 'cu92' or 'cu100' etc. e.g. 'cpu,cu92' or 'cu92,cu100' -# . This can also just be the word 'all', which will expand to all -# supported cpu/CUDA versions. - -# On mac there is only one machine, so not specifying which machine is fine -if [[ "$(uname)" == 'Darwin' ]]; then - which_worker='mac' -else - if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters. Require which worker I am [0-2] or 'mac'" - echo "e.g. ./build_cron.sh 0" - exit 1 - fi - - which_worker=$1 - - # This file is hardcoded to exactly 3 linux workers and 1 mac worker - if [[ "$which_worker" != 0 && "$which_worker" != 1 && "$which_worker" != 2 ]]; then - echo "Illegal parameter. This script is made for exactly 3 workers." - echo "You must give me a worker number out of [0, 1, 2] or 'mac'" - exit 1 - fi -fi - -# Clear out old logs. When we re-run jobs (after patching a change) then old -# logs should be removed to avoid confusion. -rm -rf "$FAILED_LOG_DIR" || true -rm -rf "$SUCCEEDED_LOG_DIR" || true -mkdir -p "$FAILED_LOG_DIR" -mkdir -p "$SUCCEEDED_LOG_DIR" -log_root="$NIGHTLIES_FOLDER/logs/master" -mkdir -p "$log_root" - -# Divy up the tasks -# -# There are currently 41 jobs and 3 machines -# Each machine should run its 12/13 jobs in 5 parallel batches, about -# conda jobs and gpu jobs take longer -# -# The jobs is the combination of all: -# manywheel X [2.7m 2.7mu 3.5m 3.6m 3.7m] X [cpu cu92 cu100] -# conda X [2.7 3.5 3.6 3.7 ] X [cpu cu92 cu100] -# wheel X [2.7 3.5 3.6 3.7 ] X [cpu ] -# libtorch X [2.7m ] X [cpu cu92 cu100] (linux) -# libtorch X [2.7 ] X [cpu ] (mac) -# -# cpu builds ~ 15 minutes. gpu builds > 1 hr -# Try to divide the cpu jobs evenly among the tasks -if [[ "$which_worker" == 0 ]]; then - # manywheel 2.7m,2.7mu,3.5m all - tasks=( - 'manywheel 2.7m cpu,cu92' - 'manywheel 2.7mu cpu,cu92' - 'manywheel 3.5m cpu,cu92' - 'manywheel 2.7m,2.7mu,3.5m cu100' - 'libtorch 2.7m cpu' - ) -elif [[ "$which_worker" == 1 ]]; then - # manywheel 3.6m,3.7, all - # conda 2.7 all - tasks=( - 'manywheel 3.6m cpu,cu92' - 'manywheel 3.7m cpu,cu92' - 'conda 2.7 cpu,cu92' - 'manywheel 3.6m,3.7m cu100 -- conda 2.7 cu100' - 'libtorch 2.7m cu92' - ) -elif [[ "$which_worker" == 2 ]]; then - # conda 3.5,3.6,3.7 all - tasks=( - 'conda 3.5 cpu,cu92' - 'conda 3.6 cpu,cu92' - 'conda 3.7 cpu,cu92' - 'conda 3.5,3.6,3.7 cu100' - 'libtorch 2.7m cu100' - ) -elif [[ "$which_worker" == 'mac' ]]; then - # wheel all - # conda all cpu - tasks=( - 'wheel 2.7,3.5,3.6 cpu' - 'wheel 3.7 cpu -- conda 2.7 cpu' - 'conda 3.5,3.6,3.7 cpu' - 'libtorch 2.7 cpu' - ) -fi - -# Run the tasks -child_pids=() -for task in "${tasks[@]}"; do - log_file="$log_root/$(echo $task | tr ' ' '_' | tr -d ',-').log" - "${NIGHTLIES_BUILDER_ROOT}/cron/build_multiple.sh" $task > "$log_file" 2>&1 & - child_pid="$!" - echo "Starting [build_multiple.sh $task] at $(date) with pid $child_pid" - child_pids+=("$child_pid") -done - -# We would like to always upload and delete old build folders -set +e -first_ret=0 - -# Wait for all the jobs to finish -echo "Waiting for all jobs to finish at $(date)" -for child_pid in "${child_pids[@]}"; do - wait "$child_pid" -done -echo "All jobs finished! at $(date)" - -# Count the total number of failures -failed_jobs=($(ls $FAILED_LOG_DIR)) -echo "Detected ${#failed_jobs[@]} failed builds" - -# Email everyone if the jobs failed -if [[ "${#failed_jobs[@]}" != 0 ]]; then - echo "Emailing all of $NIGHTLIES_EMAIL_LIST" - if [[ "(uname)" == 'Darwin' ]]; then - # `mail` on mac doesn't expect the -t - dash_t='' - else - dash_t='-t' - fi - mail -s "$NIGHTLIES_DATE nightlies failed" $dash_t "$NIGHTLIES_EMAIL_LIST" <<< \ -"On $(uname -a) -On $(date) -Nightly jobs failed. Failed jobs are: ${failed_jobs[@]}" - ret="$?" - if [[ "$first_ret" == 0 ]]; then - first_ret="$ret" - fi -fi - -# Upload the working binaries and all of the logs -# Only upload automatically on the current day, not on manual re-runs of past -# days -if [[ "$NIGHTLIES_DATE" == "$(date +%Y_%m_%d)" ]]; then - - # Upload successful binaries and all of the logs - succeeded_jobs=($(ls $SUCCEEDED_LOG_DIR)) - echo "Uploading all of these succesful jobs\n: ${succeeded_jobs[@]}" - "${NIGHTLIES_BUILDER_ROOT}/cron/upload.sh" ${succeeded_jobs[@]} > "${log_root}/upload.log" 2>&1 - ret="$?" - if [[ "$ret" != 0 && "$first_ret" == 0 ]]; then - echo "FAILED upload.sh" - first_ret="$ret" - fi -fi - -# Regardless of failures, clean up the old build folders so that we don't run -# out of memory -# Only run the clean on the current day, not on manual re-runs of past days -if [[ "$(basename $NIGHTLIES_FOLDER)" == "$NIGHTLIES_DATE" ]]; then - "${NIGHTLIES_BUILDER_ROOT}/cron/clean.sh" > "${log_root}/clean.sh" 2>&1 - ret="$?" - if [[ "$ret" != 0 && "$first_ret" == 0 ]]; then - echo "FAILED clean.sh" - first_ret="$ret" - fi -fi - -exit "$first_ret" diff --git a/cron/build_docker.sh b/cron/build_docker.sh deleted file mode 100755 index 91cdf7cbc..000000000 --- a/cron/build_docker.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/bash - -set -ex - -echo "build_docker.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$" -SOURCE_DIR=$(cd $(dirname $0) && pwd) -source "${SOURCE_DIR}/nightly_defaults.sh" - -# Handles building for manywheels and linux conda packages. - -# Parameters -############################################################################## -if [[ "$#" != 3 ]]; then - if [[ -z "$DESIRED_PYTHON" || -z "$DESIRED_CUDA" || -z "$PACKAGE_TYPE" ]]; then - echo "The env variabled PACKAGE_TYPE must be set to 'conda' or 'manywheel' or 'libtorch'" - echo "The env variabled DESIRED_PYTHON must be set like '2.7mu' or '3.6m' etc" - echo "The env variabled DESIRED_CUDA must be set like 'cpu' or 'cu92' etc" - exit 1 - fi - package_type="$PACKAGE_TYPE" - desired_python="$DESIRED_PYTHON" - desired_cuda="$DESIRED_CUDA" -else - package_type="$1" - desired_python="$2" - desired_cuda="$3" -fi -if [[ "$package_type" != 'conda' && "$package_type" != 'manywheel' && "$package_type" != 'libtorch' ]]; then - echo "The package type must be 'conda' or 'manywheel' or 'libtorch'" - exit 1 -fi - -echo "Building a $package_type package for python$desired_python and $desired_cuda" -echo "Starting to run the build at $(date)" - -# Organize folders by os, package_type, and cuda version -host_package_dir="$(nightlies_package_folder $package_type $desired_cuda)" -mkdir -p "$host_package_dir" || true - -# This is the name of the package dir dockerside -docker_package_dir="/host_machine_pkgs" - -# Map cuda/python/storage dirs for conda or manywheel -python_nodot="${desired_python:0:1}${desired_python:2:1}" -if [[ "$desired_cuda" == 'cpu' ]]; then - build_for_cpu=1 -else - cuda_nodot="${desired_cuda:2}" -fi -if [[ "$package_type" == 'conda' ]]; then - export TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '_' '-')" - desired_python="${desired_python:0:3}" - if [[ -n "$build_for_cpu" ]]; then - desired_cuda='cpu' - else - desired_cuda="$cuda_nodot" - fi - build_script='/remote/conda/build_pytorch.sh' - docker_image="soumith/conda-cuda" -else - export TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '-' '_')" - if [[ "$package_type" == 'libtorch' ]]; then - building_pythonless=1 - fi - if [[ "$desired_python" == '2.7mu' ]]; then - desired_python='cp27-cp27mu' - else - desired_python="cp${python_nodot}-cp${python_nodot}m" - fi - # desired_cuda is correct - if [[ -n "$build_for_cpu" ]]; then - build_script='/remote/manywheel/build_cpu.sh' - else - build_script='/remote/manywheel/build.sh' - fi - if [[ -n "$build_for_cpu" ]]; then - docker_image="soumith/manylinux-cuda100" - else - docker_image="soumith/manylinux-cuda$cuda_nodot" - fi -fi -if [[ -n "$ON_SUCCESS_WRITE_ME" ]]; then - success_folder="$(dirname $ON_SUCCESS_WRITE_ME)" - success_basename="$(basename $ON_SUCCESS_WRITE_ME)" -fi - -# Pull a fresh copy of the docker image -docker pull "$docker_image" - -# Build up Docker Arguments -############################################################################## -docker_args="" - -# Needs pseudo-TTY for /bin/cat to hang around -docker_args+="-t" - -# Detach so we can use docker exec to run stuff -docker_args+=" -d" - -# Increase shared memory size so that we can run bigger models in Docker container -# See: https://github.com/pytorch/pytorch/issues/2244 -#docker_args+=" --shm-size 8G" - -# Mount the folder that will collect the finished packages -docker_args+=" -v ${host_package_dir}:${docker_package_dir}" - -# Mount the folder that stores the file in which to write SUCCESS at the end -if [[ -n "$success_folder" ]]; then - docker_args+=" -v $success_folder:/statuses" -fi - -# Run Docker as the user of this script -# Without this line, all writes to the host filesystem (all folders in -v -# flags) will be as root, and won't actually be deletable on the host machine -# after the docker exits. But with this flag, yum and other CUDA functionality -# have troubles accessing files they need. So instead of using this line we -# just start up another docker to delete this files at the end - #docker_args+=" --user $(id -u):$(id -g)" - -# Image -docker_args+=" ${docker_image}" -############################################################################## - -# We start a container and detach it such that we can run -# a series of commands without nuking the container -echo "Starting container for image ${docker_image}" -id=$(nvidia-docker run ${docker_args} /bin/cat) - -trap "echo 'Stopping container...' && -docker rm -f $id > /dev/null" EXIT - -# Copy pytorch/builder and pytorch/pytorch into the container -nvidia-docker cp "$NIGHTLIES_BUILDER_ROOT" "$id:/remote" -nvidia-docker cp "$NIGHTLIES_PYTORCH_ROOT" "$id:/pytorch" - -# I found the only way to make the command below return the proper -# exit code is by splitting run and exec. Executing run directly -# doesn't propagate a non-zero exit code properly. -( - echo "export DESIRED_PYTHON=${desired_python}" - echo "export DESIRED_CUDA=${desired_cuda}" - # the following line is true from the docker's perspective - echo "export PYTORCH_FINAL_PACKAGE_DIR=${docker_package_dir}" - echo "export CMAKE_ARGS=${CMAKE_ARGS[@]}" - echo "export EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]}" - echo "export RUN_TEST_PARAMS=${RUN_TEST_PARAMS}" - echo "export TORCH_PACKAGE_NAME=${TORCH_PACKAGE_NAME}" - echo "export PYTORCH_BUILD_VERSION=${PYTORCH_BUILD_VERSION}" - echo "export PYTORCH_BUILD_NUMBER=${PYTORCH_BUILD_NUMBER}" - echo "export OVERRIDE_PACKAGE_VERSION=${OVERRIDE_PACKAGE_VERSION}" - echo "export TORCH_CONDA_BUILD_FOLDER=${TORCH_CONDA_BUILD_FOLDER}" - echo "export DEBUG=${DEBUG}" - echo "export ON_SUCCESS_WRITE_ME=/statuses/$success_basename" - - echo "export BUILD_PYTHONLESS=${building_pythonless}" - - # TODO improve avx512 detection instead of disabling all of fbgemm - echo "export USE_FBGEMM=0" - - echo "cd /" - - # Run the build script - echo "$build_script" - - # Mark this build as a success. build_multiple expects this file to be - # written if the build succeeds - # Note the ' instead of " so the variables are all evaluated within the - # docker - echo 'ret=$?' - echo 'if [[ $ret == 0 && -n $ON_SUCCESS_WRITE_ME ]]; then' - echo ' echo 'SUCCESS' > $ON_SUCCESS_WRITE_ME' - echo 'fi' - - echo 'exit $ret' -) | nvidia-docker exec -i "$id" bash -echo "docker run exited with $?" - -exit 0 diff --git a/cron/build_mac.sh b/cron/build_mac.sh deleted file mode 100755 index 98536df89..000000000 --- a/cron/build_mac.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -set -ex -echo "build_mac.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$ with pid $$" -SOURCE_DIR=$(cd $(dirname $0) && pwd) -source "${SOURCE_DIR}/nightly_defaults.sh" - -# Handles building for mac wheels and mac conda packages and mac libtorch packages. -# Env variables that should be set: - -# Parameters -############################################################################## -if [[ "$#" != 2 ]]; then - if [[ -z "$DESIRED_PYTHON" || -z "$PACKAGE_TYPE" ]]; then - echo "The env variabled PACKAGE_TYPE must be set to 'conda' or 'manywheel' or 'libtorch'" - echo "The env variabled DESIRED_PYTHON must be set like '2.7mu' or '3.6m' etc" - exit 1 - fi - package_type="$PACKAGE_TYPE" - desired_python="$DESIRED_PYTHON" -else - package_type="$1" - desired_python="$2" -fi -if [[ "$package_type" != 'conda' && "$package_type" != 'wheel' && "$package_type" != 'libtorch' ]]; then - echo "The package type must be 'conda' or 'wheel' or 'libtorch'" - exit 1 -fi -echo "$(date) Building a $package_type MacOS package for python$desired_python" -export PYTORCH_FINAL_PACKAGE_DIR="$(nightlies_package_folder $package_type cpu)" -mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" - -# Setup a workdir -############################################################################## -workdir="${NIGHTLIES_FOLDER}/wheel_build_dirs/${package_type}_${desired_python}" -export MAC_PACKAGE_WORK_DIR="$workdir" -rm -rf "$workdir" -mkdir -p "$workdir" - -# Copy the pytorch directory into the workdir -cp -R "$NIGHTLIES_PYTORCH_ROOT" "$workdir" - -# Copy the builder directory into the workdir -# This is needed b/c the conda scripts can alter the meta.yaml -cp -R "$NIGHTLIES_BUILDER_ROOT" "$workdir" - -# Build the package -############################################################################## -if [[ "$package_type" == 'conda' ]]; then - export TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '_' '-')" - "${workdir}/builder/conda/build_pytorch.sh" cpu "$PYTORCH_BUILD_VERSION" "$PYTORCH_BUILD_NUMBER" - ret="$?" -else - if [[ "$package_type" == 'libtorch' ]]; then - export BUILD_PYTHONLESS=1 - fi - export TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '-' '_')" - "${workdir}/builder/wheel/build_wheel.sh" "$desired_python" "$PYTORCH_BUILD_VERSION" "$PYTORCH_BUILD_NUMBER" - ret="$?" -fi - -# Mark this build as a success. build_multiple expects this file to be -# written if the build succeeds - if [[ "$ret" == 0 && -n "$ON_SUCCESS_WRITE_ME" ]]; then - echo 'SUCCESS' > "$ON_SUCCESS_WRITE_ME" - fi - - exit "$ret" diff --git a/cron/build_multiple.sh b/cron/build_multiple.sh deleted file mode 100755 index 6eb12a6a0..000000000 --- a/cron/build_multiple.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash - -set -ex -echo "build_multiple.sh at $(pwd) starting at $(date) on $(uname -a)" -SOURCE_DIR=$(cd $(dirname $0) && pwd) -source "${SOURCE_DIR}/nightly_defaults.sh" - -# Builds a set of packages sequentially with nice output and logging. -# De-duping is not done. If you specify something twice, it will get built twice. -# -# Command line arguments -# DESIRED_PYTHONS -# All Python versions to build for, separated by commas, in format '2.7mu' -# for manywheels or in format '2.7' for conda/mac-wheels e.g. -# '2.7m,2.7mu,3.5m,3.6m' or '2.7,3.7' . This can also just be the word -# 'all', which will expand to all supported python versions. -# -# DESIRED_CUDAS -# All CUDA versions to build for including 'cpu', separated by commas, in -# format 'cpu' or 'cu92' or 'cu100' etc. e.g. 'cpu,cu92' or 'cu92,cu100' -# . This can also just be the word 'all', which will expand to all -# supported cpu/CUDA versions. - -if [ "$#" -lt 3 ]; then - echo 'Illegal number of parameters' - echo ' build_multiple.sh [conda|manywheel|wheel] DESIRED_PYTHON,s DESIRED_CUDA,s' - echo 'e.g. build_multiple.sh manywheel 2.7mu,3.5m,3.6m cpu,cu92' - echo 'e.g. build_multiple.sh conda,manywheel 2.7 all' - echo ' DESIRED_PYTHONs must match: \d.\d(mu?)?' - echo ' DESIRED_CUDAs must match : (cpu|cu\d\d)' -fi - -nice_time () { - echo "$(($1 / 60)) minutes and $(($1 % 60)) seconds" -} - -# Save all configurations into a big list and loop through them later -# Read through sets of -all_configs=() -while [[ $# -gt 0 ]]; do - - # Read the next configuration - IFS=, all_packages=($1) - shift - IFS=, all_pythons=($1) - shift - IFS=, all_cuda=($1) - - # Expand 'all's and add all combos to the list of configurations - for package_type in "${all_packages[@]}"; do - if [[ "${all_pythons[0]}" == 'all' ]]; then - if [[ "$package_type" == 'conda' || "$package_type" == 'wheel' ]]; then - all_pythons=('2.7' '3.5' '3.6' '3.7') - else - all_pythons=('2.7m' '2.7mu' '3.5m' '3.6m' '3.7m') - fi - fi - if [[ "${all_cuda[0]}" == 'all' ]]; then - all_cuda=('cpu' 'cu92' 'cu100' 'cu101') - fi - for py_ver in "${all_pythons[@]}"; do - for cuda_ver in "${all_cuda[@]}"; do - all_configs+=("$package_type,$py_ver,$cuda_ver") - done - done - done - shift - - # Allow -- as harmless dividers for readability - if [[ "$1" == '--' ]]; then - shift - fi -done - - -set +x -echo -echo "$(date) :: Starting PyTorch binary build for $NIGHTLIES_DATE" -echo "Building all of [${all_configs[@]}]" -set -x - -# Build over all combinations -failed_builds=() -good_builds=() - -for config in "${all_configs[@]}"; do - IFS=, confs=($config) - package_type="${confs[0]}" - py_ver="${confs[1]}" - cuda_ver="${confs[2]}" - build_tag="${package_type}_${py_ver}_${cuda_ver}" - - # When the run build is running it will write logs to the logs/ folder. If - # the build succeeds (as detected inside the docker image since exit codes - # aren't being propogated), then the build will write 'SUCCESS' to its log in - # logs/succeeded/ . When the build is over, we check if that file has been - # written and if so move the log to logs/succeeded/ ; otherwise the build has - # failed and logs are moved to logs/failed/ - log_name="${RUNNING_LOG_DIR}/${build_tag}.log" - failed_log_loc="${FAILED_LOG_DIR}/${build_tag}.log" - succeeded_log_loc="${SUCCEEDED_LOG_DIR}/${build_tag}.log" - mkdir -p "$FAILED_LOG_DIR" || true - mkdir -p "$SUCCEEDED_LOG_DIR" || true - rm -f "$failed_log_loc" - rm -f "$succeeded_log_loc" - - # Swap build script out on Macs - if [[ "$(uname)" == 'Darwin' ]]; then - build_script="${NIGHTLIES_BUILDER_ROOT}/cron/build_mac.sh" - else - build_script="${NIGHTLIES_BUILDER_ROOT}/cron/build_docker.sh" - fi - - # Swap timeout out for libtorch - if [[ "$package_type" == libtorch ]]; then - _timeout="$PYTORCH_NIGHTLIES_LIBTORCH_TIMEOUT" - else - _timeout="$PYTORCH_NIGHTLIES_TIMEOUT" - fi - - set +x - echo - echo "##############################" - echo "$(date) :: Starting $package_type for py$py_ver and $cuda_ver" - echo "Writing to log: $log_name" - - set +e - set -x - SECONDS=0 - PACKAGE_TYPE="$package_type" \ - DESIRED_PYTHON="$py_ver" \ - DESIRED_CUDA="$cuda_ver" \ - ON_SUCCESS_WRITE_ME="$succeeded_log_loc" \ - $PORTABLE_TIMEOUT "$_timeout" \ - "$build_script" > "$log_name" 2>&1 - ret="$?" - duration="$SECONDS" - set -e - - # Keep track of the failed builds - if [[ -f "$succeeded_log_loc" ]]; then - set +x - echo "$(date) :: Finished $build_tag in $(nice_time $duration)" - echo "$(date) :: Status: SUCCESS!" - rm -f "$succeeded_log_loc" - mv "$log_name" "$succeeded_log_loc" - good_builds+=("$build_tag") - else - set +x - echo "$(date) :: Finished $build_tag in $(nice_time $duration)" - echo "$(date) :: Status: FAILURE" - >&2 echo "$(date) :: Status: FAILed building $build_tag" - mv "$log_name" "$failed_log_loc" - failed_builds+=("$build_tag") - fi - - echo "################################################################################" - set -x -done - -set +x -echo "$(date) :: All builds finished." -echo "Final status:" -echo " Failed : ${#failed_builds[@]}" -for build in "${failed_builds[@]}"; do - IFS=_ params=("$build") - echo " ${params[0]} ${params[1]} ${params[2]}" -done -echo " Success : ${#good_builds[@]}" -for build in "${good_builds[@]}"; do - IFS=_ params=("$build") - echo " ${params[0]} ${params[1]} ${params[2]}" -done diff --git a/cron/clean.sh b/cron/clean.sh deleted file mode 100755 index 730b6fdf1..000000000 --- a/cron/clean.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -# cron/clean.sh -# Usage: -# ./cron/clean.sh -# This is always called by the nightly jobs with no parameters. When called -# this way, this will delete all nightly folders (date folders) that are at -# least DAYS_TO_KEEP days old. e.g. if DAYS_TO_KEEP is 5 and today is -# 2018_09_25, then 2018_09_20 and all earlier dates will be deleted. -# Technically, all folders in the nightlies root folder that are lexicogocially -# <= 2018_09_20 will be deleted. This will only work if the NIGHTLIES_DATE -# matches the current date, to avoid accidentally wiping date folders when -# manually running previous days. -# Usage: -# ./cron/clean.sh [experiment] -# When called with parameters, it will delete those explicit folders. This -# assumes that the parameters are either full paths or directories relative to -# the current nightlies root folder (for today's date). - -set -ex -echo "clean.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$" -SOURCE_DIR=$(cd $(dirname $0) && pwd) -source "${SOURCE_DIR}/nightly_defaults.sh" - -# Define a remove function that will work on both Mac and Linux -if [[ "$(uname)" == Darwin ]]; then - remove_dir () { - # Don't accidentally delete the nightlies folder - if [[ "$1" == nightlies ]]; then - echo "If you really really want to delete the entire nightlies folder" - echo "then you'll have to delete this message." - exit 1 - fi - rm -rf "$1" - } -else - # So, the dockers don't have a proper user setup, so they all run as root. - # When they write their finished packages to the host system, they can't be - # deleted without sudo access. Actually fixing the permissions in the - # docker involves both setting up a correct user and adding sudo in the - # right places to yum, pip, conda, and CUDA functionality. Instead of all - # that, we run the rm command in a different docker image, since the - # dockers all run as root. - remove_dir () { - # Don't accidentally delete the nightlies folder - if [[ "$1" == nightlies ]]; then - echo "If you really really want to delete the entire nightlies folder" - echo "then you'll have to delete this message." - exit 1 - fi - docker run -v "$(dirname $1)":/remote soumith/conda-cuda rm -rf "/remote/$(basename $1)" - } -fi - -# If given a folder to delete, delete it without question, and then don't purge -# all the dockers -if [[ "$#" -gt 0 ]]; then - while [[ "$#" -gt 0 ]]; do - cur_dir="$1" - if [[ "${cur_dir:0:1}" == '/' ]]; then - remove_dir "$cur_dir" - else - # Assume that all dirs are in the NIGHTLIES_ROOT_FOLDER - remove_dir "${NIGHTLIES_ROOT_FOLDER}/${cur_dir}" - fi - shift - done - exit 0 -fi - - -# Delete everything older than a specified number of days (default is 5) -if [[ "$(uname)" == 'Darwin' ]]; then - cutoff_date=$(date -v "-${DAYS_TO_KEEP}d" +%Y_%m_%d) -else - cutoff_date=$(date --date="$DAYS_TO_KEEP days ago" +%Y_%m_%d) -fi - -# Loop through the nightlies folder deleting all date like objects -any_removal_failed=0 -for build_dir in "$NIGHTLIES_ROOT_FOLDER"/*; do - cur_date="$(basename $build_dir)" - if [[ "$cur_date" < "$cutoff_date" || "$cur_date" == "$cutoff_date" ]]; then - echo "DELETING BUILD_FOLDER $build_dir !!" - - # Remove the folder - # Technically, this should condition on whether a mac or docker - # produces the packages, but the linux jobs only run on linux machines - # so this is fine. - remove_dir "$build_dir" - - # Make sure the rm worked, in this case we want this next command to - # fail - set +e - ls "$build_dir" >/dev/null 2>&1 - ret="$?" - set -e - if [[ "$ret" == 0 ]]; then - any_removal_failed=1 - echo "ERROR | " - echo "ERROR | Could not remove $build_dir" - echo "ERROR | Please try to delete $build_dir manually" - echo "ERROR | Then fix builder/cron/clean.sh" - fi - fi -done - -# Purge all dockers -if [[ "$(uname)" != Darwin ]]; then - docker ps -aq | xargs -I {} docker rm {} --force - yes | docker system prune -fi - -exit "$any_removal_failed" diff --git a/cron/cron_start.sh b/cron/cron_start.sh deleted file mode 100755 index 433aafcf3..000000000 --- a/cron/cron_start.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -set -ex - -# Minimal bootstrap to clone the latest pytorch/builder repo and then call -# /that/ repo's build_cron.sh - -echo "cron_start.sh at $(pwd) starting at $(date) on $(uname -a)" - -# BUILDER_REPO -# The Github org/user whose fork of builder to check out (git clone -# https://github.com//builder.git). This will always be cloned -# fresh to build with. Default is 'pytorch' -if [[ -z "$BUILDER_REPO" ]]; then - export BUILDER_REPO='pytorch' -fi - -# BUILD_BRANCH -# The branch of builder to checkout for building (git checkout ). -# This can either be the name of the branch (e.g. git checkout -# my_branch_name) or can be a git commit (git checkout 4b2674n...). Default -# is 'master' -if [[ -z "$BUILDER_BRANCH" ]]; then - export BUILDER_BRANCH='master' -fi - -# N.B. NIGHTLIES_ROOT_FOLDER and NIGHTLIES_DATE are also set in nightly_defaults.sh -# and should be kept the same in both places -if [[ -z "$NIGHTLIES_ROOT_FOLDER" ]]; then - if [[ "$(uname)" == 'Darwin' ]]; then - export NIGHTLIES_ROOT_FOLDER='/Users/administrator/nightlies' - else - export NIGHTLIES_ROOT_FOLDER='/scratch/hellemn/nightlies' - fi -fi -if [[ -z "$NIGHTLIES_DATE" ]]; then - export NIGHTLIES_DATE="$(date +%Y_%m_%d)" -fi -if [[ -z "$NIGHTLIES_FOLDER" ]]; then - export NIGHTLIES_FOLDER="$NIGHTLIES_ROOT_FOLDER/$NIGHTLIES_DATE" -fi -mkdir -p "$NIGHTLIES_FOLDER" || true - - -# Clone the requested builder checkout -# This script already exists in a builder repo somewhere, but we don't want to -# manually update the Github repos on every worker machine every time a change -# is made to the builder repo, so we re-clone the latest builder repo, and then -# call /that/ repo's build_cron.sh. We keep this script instead of cloning this -# in the crontab itself for ease of debugging. -if [[ ! -d "$NIGHTLIES_FOLDER/builder" ]]; then - pushd "$NIGHTLIES_FOLDER" - rm -rf builder - git clone "https://github.com/${BUILDER_REPO}/builder.git" - pushd builder - git checkout "$BUILDER_BRANCH" - popd - popd -fi - -# Now call the build_cron.sh of the new pytorch/builder, which is more recent -# than the repo that this script exists in -"${NIGHTLIES_FOLDER}/builder/cron/build_cron.sh" "$@" diff --git a/cron/crontab_0 b/cron/crontab_0 deleted file mode 100644 index 5d42a9855..000000000 --- a/cron/crontab_0 +++ /dev/null @@ -1,6 +0,0 @@ -# m h dom mon dow command -# The only difference between these crontabs is the argument passed to -# cron_start.sh -MAILTO='hellemn@fb.com' -4 0 * * * mkdir -p /scratch/hellemn/nightlies/$(date +\%Y_\%m_\%d)/logs/master -5 0 * * * /scratch/hellemn/git/pjh5/builder/cron/cron_start.sh 0 > /scratch/hellemn/nightlies/$(date +\%Y_\%m_\%d)/logs/master/start_cron.log 2>&1 diff --git a/cron/crontab_1 b/cron/crontab_1 deleted file mode 100644 index 99561a804..000000000 --- a/cron/crontab_1 +++ /dev/null @@ -1,7 +0,0 @@ -# m h dom mon dow command -# The only difference between these crontabs is the argument passed to -# cron_start.sh -# N.B. it appears that cron runs an hour ahead on this machine -MAILTO='hellemn@fb.com' -4 1 * * * mkdir -p /scratch/hellemn/nightlies/$(date +\%Y_\%m_\%d)/logs/master -5 1 * * * /scratch/hellemn/git/pjh5/builder/cron/cron_start.sh 1 > /scratch/hellemn/nightlies/$(date +\%Y_\%m_\%d)/logs/master/start_cron.log 2>&1 diff --git a/cron/crontab_2 b/cron/crontab_2 deleted file mode 100644 index 18acdff29..000000000 --- a/cron/crontab_2 +++ /dev/null @@ -1,7 +0,0 @@ -# m h dom mon dow command -# The only difference between these crontabs is the argument passed to -# cron_start.sh -# N.B. it appears that cron runs an hour ahead on this machine -MAILTO='hellemn@fb.com' -4 1 * * * mkdir -p /scratch/hellemn/nightlies/$(date +\%Y_\%m_\%d)/logs/master -5 1 * * * /scratch/hellemn/git/pjh5/builder/cron/cron_start.sh 2 > /scratch/hellemn/nightlies/$(date +\%Y_\%m_\%d)/logs/master/start_cron.log 2>&1 diff --git a/cron/crontab_mac b/cron/crontab_mac deleted file mode 100644 index ed02f1f0d..000000000 --- a/cron/crontab_mac +++ /dev/null @@ -1,6 +0,0 @@ -# m h dom mon dow command -# The only difference between these crontabs is the argument passed to -# cron_start.sh -MAILTO='hellemn@fb.com' -4 0 * * * mkdir -p /Users/administrator/nightlies/$(date +\%Y_\%m_\%d)/logs/master -5 0 * * * /Users/administrator/git/pjh5/builder/cron/cron_start.sh > /Users/administrator/nightlies/$(date +\%Y_\%m_\%d)/logs/master/cron_start.log 2>&1 diff --git a/cron/parse_conda_json.py b/cron/parse_conda_json.py deleted file mode 100644 index c04c5cd67..000000000 --- a/cron/parse_conda_json.py +++ /dev/null @@ -1,63 +0,0 @@ -import json -import sys - -# Usage: -# parse_conda_json.py input_file output_file -# Reads the result of a `conda search --json` into lines of ' -# ' - -inputfile = sys.argv[1] -outputfile = sys.argv[2] - -data = [] - -with open(inputfile, 'rb') as jsonfile: - rawdata = json.load(jsonfile) - - # conda search returns format {'pytorch-nightly': [{key:val}...]} - pkg_name = list(rawdata.keys())[0] - print('parse_conda_json.py:: Parsing package {}'.format(pkg_name)) - - # Loop through versions found, keeping only 'build', and size - # size is in bytes - for result in rawdata[pkg_name]: - # N.B. platform is queried as 'linux-64' but is stores as linux, and as - # 'osx-64' but stored as 'darwin' - plat = 'linux' if 'linux' in result['platform'] else 'macos' - size = result['size'] - - # 'build' is of the form - # linux CUDA builds: 'py2.7_cuda8.0.61_cudnn7.1.2_0' - # linux CPU builds: 'py2.7_cpu_0' - # MacOS builds: 'py2.7_0' - build = result['build'].split('_') - - print('parse_conda_json.py:: Size of {} conda {} is {}'.format(plat, build, size)) - if plat == 'macos': - # We expect a format like 'py2.7_0' - assert len(build) == 2, "Unexpected MacOS build string {}".format(build) - else: - # We expect a format like: - # CUDA builds: 'py2.7_cuda8.0.61_cudnn7.1.2_0' - # CPU builds: 'py2.7_cpu_0' - assert len(build) in (3,4), "Unexpected Linux build string {}".format(build) - - # Python versions are of form 'py#.#' , we discard the 'py' - py_ver = build[0][2:] - - # CUDA versions are of the form 'cuda10.0.61', we replace 'cuda' with - # 'cu' and keep only the major and minor values - if build[1].startswith('cuda'): - cu_ver = build[1][4:].split('.') - assert len(cu_ver) == 3, "Unexpected cuda format {}".format(cu_ver) - cu_ver = 'cu' + ''.join((cu_ver[0], cu_ver[1])) - else: - cu_ver = 'cpu' - - data.append((plat, py_ver, cu_ver, size)) - -# Write the sizes out in log_name format of conda_2.7_cu80 -print("parse_conda_json.py:: Writing log_name format to {}".format(outputfile)) -with open(outputfile, 'a') as outfile: - for plat, py_ver, cu_ver, size in data: - outfile.write("{} conda {} {} {}\n".format(plat, py_ver, cu_ver, size)) diff --git a/cron/upload.sh b/cron/upload.sh deleted file mode 100755 index 9f8d63811..000000000 --- a/cron/upload.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/bin/bash - -set -ex -echo "upload.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$" -SOURCE_DIR=$(cd $(dirname $0) && pwd) -source "${SOURCE_DIR}/nightly_defaults.sh" - -# Upload all nightly packages. -# This has two use cases: -# In the regular nightlies use-case this script is passed all the successful -# logs -# ./cron/upload.sh conda_2.7_cpu.log manywheel_2.7mu_cu80.log ... -# and only those corresponding packages are uploaded. -# Otherwise, if given no parameters this will upload all packages. -# -# In both use cases, this will upload all of the logs. There is no flag to -# control this. If you are manually calling this function then you are probably -# overwriting some binaries in the cloud, so the corresponding logs should be -# updated to reflect the new visible binaries. - -upload_it () { - pkg_type="$1" - cuda_ver="$2" - pkg="$3" - - if [[ "$pkg_type" == 'conda' ]]; then - echo "Uploading $pkg to anaconda" - anaconda upload "$pkg" -u pytorch --label main --force --no-progress - elif [[ "$pkg_type" == 'libtorch' ]]; then - s3_dir="s3://pytorch/libtorch/${PIP_UPLOAD_FOLDER}${cuda_ver}/" - echo "Uploading $pkg to $s3_dir" - aws s3 cp "$pkg" "$s3_dir" --acl public-read --quiet - else - uploaded_a_wheel=1 - s3_dir="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${cuda_ver}/" - echo "Uploading $pkg to $s3_dir" - aws s3 cp "$pkg" "$s3_dir" --acl public-read --quiet - fi -} - -# Location of the temporary miniconda that is downloaded to install conda-build -# and aws to upload finished packages -conda_uploader_root="${NIGHTLIES_FOLDER}/miniconda" - -# Set-up tools we need to upload -############################################################################## -# This needs both 'aws' and 'anaconda-client' with proper credentials for each. -# The function check_if_uploaders_installed below will echo text if one of -# these is not installed (but doesn't check credentials). If we need to install -# one, then we first try to add $conda_uploader_root to the path and -# then check again. If the tools still don't work then we remove the old -# CONDA_UPLOADER_ISNTALLATION and install a new one. Any further failures will -# exit later in the script. -# aws and anaconda-client will always be installed into the 'upload_env' conda -# environment - -# Source the credentials if given -if [[ -x "$PYTORCH_CREDENTIALS_FILE" ]]; then - source "$PYTORCH_CREDENTIALS_FILE" -fi - -# This function is used to determine if both 'aws' and 'anaconda-client' are -# installed. N.B. this does not check if credentials are valid. -function check_if_uploaders_installed() { - conda --version >/dev/null 2>&1 - if [[ "$?" != 0 ]]; then - echo "conda is not installed" - fi - aws --version >/dev/null 2>&1 - if [[ "$?" != 0 ]]; then - echo "aws is not installed" - fi - anaconda upload -h >/dev/null 2>&1 - if [[ "$?" != 0 ]]; then - echo "anaconda-client is not installed" - fi -} - -# First try to source conda_uploader_root. This should trigger in the -# case of manual re-runs. -if [[ -d "$conda_uploader_root" && -n "$(check_if_uploaders_installed)" ]]; then - export PATH="$conda_uploader_root/bin:$PATH" - source activate upload_env || true -fi - -# Download miniconda so that we can install aws and anaconda-client on it -if [[ -n "$(check_if_uploaders_installed)" ]]; then - rm -rf "$conda_uploader_root" - miniconda_sh="${NIGHTLIES_FOLDER}/miniconda.sh" - if [[ "$(uname)" == 'Darwin' ]]; then - curl https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "$miniconda_sh" - else - curl https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o "$miniconda_sh" - fi - chmod +x "$miniconda_sh" && \ - "$miniconda_sh" -b -p "$conda_uploader_root" && \ - rm "$miniconda_sh" - export PATH="$conda_uploader_root/bin:$PATH" - - # Create an env to ensure that a Python exists - conda create -qyn upload_env python=3.6 - source activate upload_env - - # Install aws and anaconda client - pip install awscli - conda install -y anaconda-client - yes | anaconda login --username "$PYTORCH_ANACONDA_USERNAME" --password "$PYTORCH_ANACONDA_PASSWORD" -fi - - -# Upload all of the logs -############################################################################## -"${NIGHTLIES_BUILDER_ROOT}/cron/upload_logs.sh" - - -# Upload all [passed in] packages -############################################################################## -packages_to_upload=() -if [[ "$#" -eq 0 ]]; then - # If not given any specific packages to upload, then upload everything that - # we can find - # Packages are organized by type and CUDA/cpu version so we have to loop - # over these to find all the packages - _ALL_PKG_TYPES=("manywheel" "wheel" "conda" "libtorch") - _ALL_CUDA_VERSIONS=("cpu" "cu92" "cu100" "cu101") - for pkg_type in "${_ALL_PKG_TYPES[@]}"; do - for cuda_ver in "${_ALL_CUDA_VERSIONS[@]}"; do - pkg_dir="$(nightlies_package_folder $pkg_type $cuda_ver)" - if [[ ! -d "$pkg_dir" || -z "$(ls $pkg_dir)" ]]; then - continue - fi - for pkg in $pkg_dir/*; do - upload_it "$pkg_type" "$cuda_ver" "$pkg" - done - done - done -else - # Else we're given a bunch of log names, turn these into exact packages - # This is really fragile - all_configs=() - while [[ $# -gt 0 ]]; do - IFS=, confs=($(basename $1 .log | tr '_' ',')) - pkg_type="${confs[0]}" - py_ver="${confs[1]}" - cuda_ver="${confs[2]}" - pkg_dir="$(nightlies_package_folder $pkg_type $cuda_ver)" - - # Map e.g. 2.7mu -> cp27mu - if [[ "${#py_ver}" -gt 3 ]]; then - if [[ "$py_ver" == '2.7mu' ]]; then - py_ver="cp27mu" - else - py_ver="cp${py_ver:0:1}${py_ver:2:1}m" - fi - fi - - # On Darwin, map 2.7 -> cp27 without the m - if [[ "$(uname)" == 'Darwin' && "$pkg_type" != 'conda' ]]; then - py_ver="cp${py_ver:0:1}${py_ver:2:1}" - fi - - if [[ "$pkg_type" == 'libtorch' ]]; then - # Libtorch builds create a lot of different variants for each cuda - # version and ignores the python version. The package dir in this - # case will contain shared/static with/without deps. We just upload - # all the packages we find - for pkg in $pkg_dir/*; do - upload_it 'libtorch' "$cuda_ver" "$pkg" - done - else - # Conda/wheel/manywheel - Find the exact package to upload - # This package dir contains packages of different python versions, - # some of which may have failed tests. We need to find the exact - # python version that succeeded to upload. - # We need to match - or _ after the python version to avoid - # matching cp27mu when we're trying to mach cp27m. - # N.B. this will only work when the name in both conda and wheel - # packages does follow the python version with a - or _ - set +e - unset pkg - pkg="$(ls $pkg_dir | grep $py_ver[-_])" - set -e - if [[ -n "$pkg" ]]; then - upload_it "$pkg_type" "$cuda_ver" "$pkg_dir/$pkg" - else - echo "Could not find the package for $1. I looked for" - echo "python version $py_ver in $pkg_dir but couldn't" - echo "find anything" - exit 1 - fi - fi - shift - done -fi - -# Update wheel htmls -if [[ -n "$uploaded_a_wheel" ]]; then - "${NIGHTLIES_BUILDER_ROOT}/cron/update_s3_htmls.sh" -fi - -# Update the binary size list -"${NIGHTLIES_BUILDER_ROOT}/cron/upload_binary_sizes.sh" diff --git a/cron/upload_binary_sizes.sh b/cron/upload_binary_sizes.sh deleted file mode 100755 index 688e24ac0..000000000 --- a/cron/upload_binary_sizes.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/bin/bash - -# The hud at pytorch.org/builder populates its binary sizes from reading json -# files at s3://pytorch/nightly_logs/binary_sizes/cpu/2019_01_01.json (for all -# cpu versions and dates). This script populates those files by parsing conda -# info output or from s3 -# Usage: -# collect_binary_sizes.sh [date] -# -# This script needs a date to search for, which is either the first parameter -# or the variable $DATE -# This script assumes that the version string follows 1.1.0.dev20190101 format; -# specifically we construct the version as "*$DATE" -# The date should be in 2019_01_01 format, with underscores. -# -# N.B. this assumes that there is one version for each date. If you upload -# 1.1.0 *and* 1.2.0 binaries on the same date, then this will probably -# silently intermix the binary sizes. -# N.B. cuda versions are hardcoded into this file in the s3 section. -# -# If you look closely you'll notice that uploaded json files have underscores -# in their names like 2019_01_01 but the versions use 20190101. This is because -# we want the jsons to be more human readable, but need the version dates to -# match what `conda search` knows about - -set -ex -echo "collect_binary_sizes.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$" -SOURCE_DIR=$(cd $(dirname $0) && pwd) - -# Parse parameters, clean parameters -if [[ "$#" > 0 ]]; then - target_date="$1" -elif [[ -n "$DATE" ]]; then - target_date="$DATE" -elif [[ -n "$NIGHTLIES_DATE" ]]; then - target_date="$NIGHTLIES_DATE" -else - echo "Need a date in format 2019_01_01 as argument or in \$DATE" -fi - -# The docs say that this takes an underscore date but people don't read -# docs, so if it's wrong then we fix it. -target_date="$(echo $target_date | tr '-' '_')" -if [[ "${#target_date}" == 8 ]]; then - target_date="${target_date:0:4}_${target_date:4:2}_${target_date:6:2}" -fi - -# First write lines of "$platform $pkg_type $py_ver $cu_ver $size" to a log, -# then parse that into json at the end. Calls to `conda search` and s3 are -# handled in bash. Parsing `conda search` output is handled in Python. -# Converting all the final info to json is handled in Python. The .log stores -# lines of -# platform package_type python_version cuda_version size_in_bytes -# The .json stores a list of objects with those fields. -binary_sizes_log="$SOURCE_DIR/binary_sizes.log" -binary_sizes_json="$SOURCE_DIR/$target_date.json" -rm -f "$binary_sizes_log" -rm -f "$binary_sizes_json" -touch "$binary_sizes_log" - -# We always want to upload the binary sizes of the packages that do exist, so -# we collect the errors we come across and echo them later instead of failing -# fast. -# N.B. that we detect when `conda search` or `s3 ls` failed to fetch results, -# but we do not detect missing binaries. E.g. all macos wheels could be -# missing, but if `s3 ls` returns a single linux manywheel then we will not -# detect any error. -failed_binary_queries=() - - -############################################################################## -# Collect conda binary sizes -# This is read from `conda search`. - -# `conda search` takes a version string. We use *20190101* to catch -# 1.0.0.dev20190101 or 1.1.0.dev20190101+cu90 etc. All the nightly binaries -# have this general format of version string -conda_search_version="*$(echo $target_date | tr -d _)*" - -conda_platforms=('linux-64' 'osx-64') -conda_pkg_names=('pytorch') -tmp_json="_conda_search.json" -for pkg_name in "${conda_pkg_names[@]}"; do - for platform in "${conda_platforms[@]}"; do - # Read the info from conda-search - touch "$tmp_json" - set +e - conda search -c pytorch-nightly --json --platform "$platform" \ - "$pkg_name==$conda_search_version" > "$tmp_json" - if [[ "$?" != 0 ]]; then - set -e - echo "ERROR: Could not query conda for $platform" - failed_binary_queries+=("$platform conda $pkg_name") - continue - fi - set -e - - # Call Python to parse the json into 'log_name_form size_in_bytes' - python "$SOURCE_DIR/parse_conda_json.py" "$tmp_json" "$binary_sizes_log" - done -done -rm -f "$tmp_json" - - -############################################################################## -# Collect wheel binary sizes. These are read from s3 -aws_version="$(echo $target_date | tr -d _)" -cuda_versions=("cpu" "cu92" "cu100" "cu101") -for cu_ver in "${cuda_versions[@]}"; do - - # Read the info from s3 - s3_dir="s3://pytorch/whl/nightly/${cu_ver}/" - - # s3 ls output looks like lines of - # 2019-05-02 23:00:47 88928494 torch_nightly-1.1.0.dev20190503-cp36-none-macosx_10_7_x86_64.whl - # The grep command is - # --only-matching, only print out the string that matches - # \S* -- the numbers that should be the binary size, right before the - # package name - # \S*$target_date\S*\.whl -- some string that ends in .whl that has the - # date we're looking for in it - set +e - outputs=($(aws s3 ls "$s3_dir" | grep torch- | grep --only-matching "\S* \S*$aws_version\S*\.whl")) - - if [[ "$?" != 0 ]]; then - set -e - echo "ERROR: Could find no [many]wheels for $cu_ver" - failed_binary_queries+=("linux_and_macos [many]wheel $cu_ver") - continue - fi - set -e - - # outputs is now a list of [size whl size whl...] as different elements - # set +x so the echo of sizes is readable - set +x - for i in $(seq 0 2 $(( ${#outputs[@]} - 1 )) ); do - whl="${outputs[$(( $i + 1 ))]}" - size="${outputs[$i]}" - - # Parse the python version from the whl name. If the name is in format - # torch_nightly-1.0.0.dev20181113-cp35-cp35m-linux_x86_64.whl then it's - # linux, otherwise it should have a '-none' in the name and be for mac - if [[ "$whl" == *none* ]]; then - platform='macos' - pkg_type='wheel' - # The regex matches -cp27-none- , there is no 'mu' variant for mac - py_ver="$(echo $whl | grep --only-matching '\-cp..\-none\-')" - py_ver="${py_ver:3:1}.${py_ver:4:1}" - elif [[ "$whl" == *linux* ]]; then - platform='linux' - pkg_type='manywheel' - # The regex matches -cp27-cp27mu or -cp27-cp27m - py_ver="$(echo $whl | grep --only-matching '\-cp..\-cp..mu\?')" - py_ver="${py_ver:8:1}.${py_ver:9}" - elif [[ "$whl" == *win_amd64* ]]; then - platform='windows' - pkg_type='wheel' - # The regex matches -cp27-cp27mu or -cp27-cp27m - py_ver="$(echo $whl | grep --only-matching '\-cp..\-cp..mu\?')" - py_ver="${py_ver:8:1}.${py_ver:9}" - fi - - # Write to binary_sizes_log in log_name_form - echo "upload_binary_sizes.sh:: Size of $platform $pkg_type $py_ver $cu_ver is $size" - echo "$platform $pkg_type $py_ver $cu_ver $size" >> "$binary_sizes_log" - done - set -x -done - -# Convert the file of ' ' into a json for easy -# ingestion in the react HUD -python "$SOURCE_DIR/write_json.py" "$binary_sizes_log" "$binary_sizes_json" - -# Print all the types that failed. -if [[ -n "${failed_binary_queries[@]}" ]]; then - set +x - echo - echo "ERRORS: Failed to find sizes for all of:" - for failure in "${failed_binary_queries[@]}"; do - echo "$failure" - done - echo -fi -set -x - -# Upload the log to s3 -# N.B. if you want to change the name of this json file then you have to -# coordinate the change with the gh-pages-src branch -set +e -aws s3 cp "$binary_sizes_json" "s3://pytorch/nightly_logs/binary_sizes/" --acl public-read --quiet -if [[ "$?" != 0 ]]; then - set -e - echo "Upload to aws failed. Trying again loudly" - aws s3 cp "$binary_sizes_json" "s3://pytorch/nightly_logs/binary_sizes/" --acl public-read -fi -set -e - -# Surface the failure if anything went wrong -if [[ -n "${failed_binary_queries[@]}" ]]; then - exit 1 -fi diff --git a/cron/upload_logs.sh b/cron/upload_logs.sh deleted file mode 100755 index 7b5a62714..000000000 --- a/cron/upload_logs.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Uploads all logs to s3, takes no arguments. -# The log location is configured in cron/nightly_defaults.sh -# -# This is called by upload.sh, since upload.sh contains all the logic to set up -# conda and pip with credentials. This exists as a separate file so that you -# can manually just update the logs if you wish by just calling this file. -# -# If you call this yourself then you are responsible for your own credentials. -# N.B. PLEASE BE CAREFUL. In the past, it seems like manually logging in to -# anaconda has caused problems to cron's ability to login to Anaconda and -# prevented uploading. - -set -ex -echo "upload_logs.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$" -SOURCE_DIR=$(cd $(dirname $0) && pwd) -source "${SOURCE_DIR}/nightly_defaults.sh" - -# Uploads all of the logs -# N.B. do NOT include the master logs, as there are secrets in those - -pushd "$NIGHTLIES_FOLDER/logs" -all_logs=($(find . -name '*.log' -not -path '*master*')) -for log in "${all_logs[@]}"; do - echo "Copying $log to s3://pytorch/$LOGS_S3_DIR/" - aws s3 cp "$log" "s3://pytorch/$LOGS_S3_DIR/" --acl public-read -done -popd diff --git a/cron/write_json.py b/cron/write_json.py deleted file mode 100755 index 34be4e11e..000000000 --- a/cron/write_json.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -import sys - -# Usage: -# write_json.py input_file output_file -# Reads a file of ' ' into a json file - -inputfile = sys.argv[1] -outputfile = sys.argv[2] - -data = [] - -with open(inputfile, 'r') as infile: - for line in infile: - platform, pkg_type, py_ver, cu_ver, size = line.split() - data.append({ - 'os': platform, - 'pkgType': pkg_type, - 'pyVer': py_ver, - 'cuVer': cu_ver, - 'size': size, - }) - -with open(outputfile, 'w') as outfile: - json.dump(data, outfile) diff --git a/packaging/FIXME_THIS_IS_COPIED_FROM_VISION_REPO b/packaging/FIXME_THIS_IS_COPIED_FROM_VISION_REPO deleted file mode 100644 index e69de29bb..000000000 diff --git a/packaging/README.md b/packaging/README.md deleted file mode 100644 index 7d3c5f783..000000000 --- a/packaging/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Building torchvision packages for release - -## Anaconda packages - -### Linux - -```bash -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/conda-cuda bash -pushd remote/conda - -./build_vision.sh 9.0 -./build_vision.sh 10.0 -./build_vision.sh cpu - -# copy packages over to /remote -# exit docker -# anaconda upload -u pytorch torchvision*.bz2 -``` - -### OSX - -```bash -# create a fresh anaconda environment / install and activate it -conda install -y conda-build anaconda-client -./build_vision.sh cpu - -# copy packages over to /remote -# exit docker -# anaconda upload -u pytorch torchvision*.bz2 -``` - -### Windows - -```bash -# Open `Git Bash` and change dir to `conda` -./build_vision.sh 9.0 -./build_vision.sh 10.0 -./build_vision.sh cpu - -# copy packages to a output directory -# anaconda upload -u pytorch torchvision*.bz2 -``` - -## Wheels - -### Linux - -pushd wheel - -```bash -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda90:latest bash -cd remote -./linux_manywheel.sh cu90 - -rm -rf /usr/local/cuda* -./linux_manywheel.sh cpu -``` - -```bash -nvidia-docker run -it --ipc=host --rm -v $(pwd):/remote soumith/manylinux-cuda100:latest bash -cd remote -./linux_manywheel.sh cu100 -``` - -wheels are in the folders `cpu`, `cu90`, `cu100`. - -You can upload the `cu90` wheels to twine with `twine upload *.whl`. -Which wheels we upload depends on which wheels PyTorch uploads as default, and right now, it's `cu90`. - -### OSX - -```bash -pushd wheel -./osx_wheel.sh -``` - -### Windows - -```cmd -set PYTORCH_REPO=pytorch - -pushd windows -call build_vision.bat 90 0.3.0 1 -call build_vision.bat 100 0.3.0 1 -call build_vision.bat cpu 0.3.0 1 -``` - -wheels are in the current folder. - -You can upload them to twine with `twine upload *.whl` diff --git a/packaging/build_conda.sh b/packaging/build_conda.sh deleted file mode 100755 index 795377dcc..000000000 --- a/packaging/build_conda.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -xe - -COMMAND_TO_WRAP=$1 - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -. "$script_dir/pkg_helpers.bash" - -export BUILD_TYPE=conda -setup_env 0.5.0 -export SOURCE_ROOT_DIR="$PWD" -setup_conda_pytorch_constraint -setup_conda_cudatoolkit_constraint - - -echo "Printing python version BEFORE Conda activation" -python --version - -echo "About to run: source activate env$PYTHON_VERSION" -source activate "env$PYTHON_VERSION" - - -echo "Printing python version AFTER Conda activation" -python --version - - -if [ $CU_VERSION != 'cpu' ] -then - - if [ $CU_VERSION == '9.2' ] - then - conda install pytorch torchvision cudatoolkit=$CU_VERSION -c pytorch-nightly -c defaults -c numba/label/dev - else - conda install pytorch torchvision cudatoolkit=$CU_VERSION -c pytorch-nightly - fi -else - - conda install pytorch torchvision cpuonly -c pytorch-nightly -fi - - - -$COMMAND_TO_WRAP \ No newline at end of file diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh deleted file mode 100755 index 3ea954017..000000000 --- a/packaging/build_wheel.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -ex - -COMMAND_TO_WRAP=$1 - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -. "$script_dir/pkg_helpers.bash" - -export BUILD_TYPE=wheel -setup_env 0.5.0 -setup_wheel_python -pip_install numpy pyyaml future ninja -setup_pip_pytorch_version -#python setup.py clean -#IS_WHEEL=1 python setup.py bdist_wheel - -$COMMAND_TO_WRAP diff --git a/packaging/conda/build_vision.sh b/packaging/conda/build_vision.sh deleted file mode 100755 index cb50de4bd..000000000 --- a/packaging/conda/build_vision.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash -if [[ -x "/remote/anaconda_token" ]]; then - . /remote/anaconda_token || true -fi - -set -ex - -# Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters. Pass cuda version" - echo "CUDA version should be M.m with no dot, e.g. '8.0' or 'cpu'" - exit 1 -fi -desired_cuda="$1" - -export TORCHVISION_BUILD_VERSION="0.3.0" -export TORCHVISION_BUILD_NUMBER=1 - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -if [[ -z "$WIN_PACKAGE_WORK_DIR" ]]; then - WIN_PACKAGE_WORK_DIR="$(echo $(pwd -W) | tr '/' '\\')\\tmp_conda_$(date +%H%M%S)" -fi - -if [[ "$OSTYPE" == "msys" ]]; then - mkdir -p "$WIN_PACKAGE_WORK_DIR" || true - vision_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/torchvision-src" - git config --system core.longpaths true -else - vision_rootdir="$(pwd)/torchvision-src" -fi - -if [[ ! -d "$vision_rootdir" ]]; then - rm -rf "$vision_rootdir" - git clone "https://github.com/pytorch/vision" "$vision_rootdir" - pushd "$vision_rootdir" - git checkout v$TORCHVISION_BUILD_VERSION - popd -fi - -cd "$SOURCE_DIR" - -if [[ "$OSTYPE" == "msys" ]]; then - export tmp_conda="${WIN_PACKAGE_WORK_DIR}\\conda" - export miniconda_exe="${WIN_PACKAGE_WORK_DIR}\\miniconda.exe" - rm -rf "$tmp_conda" - rm -f "$miniconda_exe" - curl -sSk https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe" - "$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe" - pushd $tmp_conda - export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH" - popd - # We have to skip 3.17 because of the following bug. - # https://github.com/conda/conda-build/issues/3285 - retry conda install -yq conda-build -fi - -ANACONDA_USER=pytorch -conda config --set anaconda_upload no - - -export TORCHVISION_PACKAGE_SUFFIX="" -if [[ "$desired_cuda" == 'cpu' ]]; then - export CONDA_CUDATOOLKIT_CONSTRAINT="" - export CUDA_VERSION="None" - if [[ "$OSTYPE" != "darwin"* ]]; then - export TORCHVISION_PACKAGE_SUFFIX="-cpu" - fi -else - . ./switch_cuda_version.sh $desired_cuda - if [[ "$desired_cuda" == "10.0" ]]; then - export CONDA_CUDATOOLKIT_CONSTRAINT=" - cudatoolkit >=10.0,<10.1 # [not osx]" - elif [[ "$desired_cuda" == "9.0" ]]; then - export CONDA_CUDATOOLKIT_CONSTRAINT=" - cudatoolkit >=9.0,<9.1 # [not osx]" - else - echo "unhandled desired_cuda: $desired_cuda" - exit 1 - fi -fi - -if [[ "$OSTYPE" == "msys" ]]; then - time conda build -c $ANACONDA_USER --no-anaconda-upload vs2017 -else - time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchvision -fi -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchvision -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchvision -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchvision -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.8 torchvision - -set +e diff --git a/packaging/conda/install_conda.bat b/packaging/conda/install_conda.bat deleted file mode 100644 index 6052ad08b..000000000 --- a/packaging/conda/install_conda.bat +++ /dev/null @@ -1 +0,0 @@ -start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda% diff --git a/packaging/conda/switch_cuda_version.sh b/packaging/conda/switch_cuda_version.sh deleted file mode 100755 index 342def938..000000000 --- a/packaging/conda/switch_cuda_version.sh +++ /dev/null @@ -1,28 +0,0 @@ -if [[ "$OSTYPE" == "msys" ]]; then - CUDA_DIR="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v$1" -else - CUDA_DIR="/usr/local/cuda-$1" -fi - -if ! ls "$CUDA_DIR" -then - echo "folder $CUDA_DIR not found to switch" -fi - -echo "Switching symlink to $CUDA_DIR" -mkdir -p /usr/local -rm -fr /usr/local/cuda -ln -s "$CUDA_DIR" /usr/local/cuda - -if [[ "$OSTYPE" == "msys" ]]; then - export CUDA_VERSION=`ls /usr/local/cuda/bin/cudart64*.dll | head -1 | tr '._' ' ' | cut -d ' ' -f2` - export CUDNN_VERSION=`ls /usr/local/cuda/bin/cudnn64*.dll | head -1 | tr '._' ' ' | cut -d ' ' -f2` -else - export CUDA_VERSION=$(ls /usr/local/cuda/lib64/libcudart.so.*|sort|tac | head -1 | rev | cut -d"." -f -3 | rev) - export CUDNN_VERSION=$(ls /usr/local/cuda/lib64/libcudnn.so.*|sort|tac | head -1 | rev | cut -d"." -f -3 | rev) -fi - -ls -alh /usr/local/cuda - -echo "CUDA_VERSION=$CUDA_VERSION" -echo "CUDNN_VERSION=$CUDNN_VERSION" diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash deleted file mode 100644 index 2576e0802..000000000 --- a/packaging/pkg_helpers.bash +++ /dev/null @@ -1,249 +0,0 @@ -# A set of useful bash functions for common functionality we need to do in -# many build scripts - - -# Setup CUDA environment variables, based on CU_VERSION -# -# Inputs: -# CU_VERSION (cpu, cu92, cu100) -# NO_CUDA_PACKAGE (bool) -# BUILD_TYPE (conda, wheel) -# -# Outputs: -# VERSION_SUFFIX (e.g., "") -# PYTORCH_VERSION_SUFFIX (e.g., +cpu) -# WHEEL_DIR (e.g., cu100/) -# CUDA_HOME (e.g., /usr/local/cuda-9.2, respected by torch.utils.cpp_extension) -# FORCE_CUDA (respected by torchvision setup.py) -# NVCC_FLAGS (respected by torchvision setup.py) -# -# Precondition: CUDA versions are installed in their conventional locations in -# /usr/local/cuda-* -# -# NOTE: Why VERSION_SUFFIX versus PYTORCH_VERSION_SUFFIX? If you're building -# a package with CUDA on a platform we support CUDA on, VERSION_SUFFIX == -# PYTORCH_VERSION_SUFFIX and everyone is happy. However, if you are building a -# package with only CPU bits (e.g., torchaudio), then VERSION_SUFFIX is always -# empty, but PYTORCH_VERSION_SUFFIX is +cpu (because that's how you get a CPU -# version of a Python package. But that doesn't apply if you're on OS X, -# since the default CU_VERSION on OS X is cpu. -setup_cuda() { - - # First, compute version suffixes. By default, assume no version suffixes - export VERSION_SUFFIX="" - export PYTORCH_VERSION_SUFFIX="" - export WHEEL_DIR="" - # Wheel builds need suffixes (but not if they're on OS X, which never has suffix) - if [[ "$BUILD_TYPE" == "wheel" ]] && [[ "$(uname)" != Darwin ]]; then - # The default CUDA has no suffix - if [[ "$CU_VERSION" != "cu101" ]]; then - export PYTORCH_VERSION_SUFFIX="+$CU_VERSION" - fi - # Match the suffix scheme of pytorch, unless this package does not have - # CUDA builds (in which case, use default) - if [[ -z "$NO_CUDA_PACKAGE" ]]; then - export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX" - # If the suffix is non-empty, we will use a wheel subdirectory - if [[ -n "$PYTORCH_VERSION_SUFFIX" ]]; then - export WHEEL_DIR="$CU_VERSION/" - fi - fi - fi - - # Now work out the CUDA settings - case "$CU_VERSION" in - cu102) - export CUDA_HOME=/usr/local/cuda-10.2/ - export FORCE_CUDA=1 - # Hard-coding gencode flags is temporary situation until - # https://github.com/pytorch/pytorch/pull/23408 lands - export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" - ;; - cu101) - export CUDA_HOME=/usr/local/cuda-10.1/ - export FORCE_CUDA=1 - # Hard-coding gencode flags is temporary situation until - # https://github.com/pytorch/pytorch/pull/23408 lands - export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" - ;; - cu100) - export CUDA_HOME=/usr/local/cuda-10.0/ - export FORCE_CUDA=1 - # Hard-coding gencode flags is temporary situation until - # https://github.com/pytorch/pytorch/pull/23408 lands - export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" - ;; - cu92) - export CUDA_HOME=/usr/local/cuda-9.2/ - export FORCE_CUDA=1 - export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50" - ;; - cpu) - ;; - *) - echo "Unrecognized CU_VERSION=$CU_VERSION" - exit 1 - ;; - esac -} - -# Populate build version if necessary, and add version suffix -# -# Inputs: -# BUILD_VERSION (e.g., 0.2.0 or empty) -# VERSION_SUFFIX (e.g., +cpu) -# -# Outputs: -# BUILD_VERSION (e.g., 0.2.0.dev20190807+cpu) -# -# Fill BUILD_VERSION if it doesn't exist already with a nightly string -# Usage: setup_build_version 0.2.0 -setup_build_version() { - if [[ -z "$BUILD_VERSION" ]]; then - export BUILD_VERSION="$1.dev$(date "+%Y%m%d")$VERSION_SUFFIX" - else - export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX" - fi -} - -# Set some useful variables for OS X, if applicable -setup_macos() { - if [[ "$(uname)" == Darwin ]]; then - export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ - fi -} - -# Top-level entry point for things every package will need to do -# -# Usage: setup_env 0.2.0 -setup_env() { - setup_cuda - setup_build_version "$1" - setup_macos -} - -# Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -activate_proper_conda_python_version() { - conda env remove -n "env$PYTHON_VERSION" || true - conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION" - conda activate "env$PYTHON_VERSION" -} - -# Inputs: -# PYTHON_VERSION (2.7, 3.5, 3.6, 3.7) -# UNICODE_ABI (bool) -# -# Outputs: -# PATH modified to put correct Python version in PATH -# -# Precondition: If Linux, you are in a pytorch/manylinux-cuda* Docker image -setup_wheel_python() { - if [[ "$(uname)" == Darwin ]]; then - eval "$(conda shell.bash hook)" - - activate_proper_conda_python_version - - else - case "$PYTHON_VERSION" in - 2.7) - if [[ -n "$UNICODE_ABI" ]]; then - python_abi=cp27-cp27mu - else - python_abi=cp27-cp27m - fi - ;; - 3.5) python_abi=cp35-cp35m ;; - 3.6) python_abi=cp36-cp36m ;; - 3.7) python_abi=cp37-cp37m ;; - 3.8) python_abi=cp38-cp38 ;; - *) - echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" - exit 1 - ;; - esac - export PATH="/opt/python/$python_abi/bin:$PATH" - fi -} - -# Install with pip a bit more robustly than the default -pip_install() { - retry pip install --progress-bar off "$@" -} - -# Install torch with pip, respecting PYTORCH_VERSION, and record the installed -# version into PYTORCH_VERSION, if applicable -setup_pip_pytorch_version() { - if [[ -z "$PYTORCH_VERSION" ]]; then - # Install latest prerelease version of torch, per our nightlies, consistent - # with the requested cuda version - pip_install --pre torch -f "https://download.pytorch.org/whl/nightly/${WHEEL_DIR}torch_nightly.html" - if [[ "$CUDA_VERSION" == "cpu" ]]; then - # CUDA and CPU are ABI compatible on the CPU-only parts, so strip - # in this case - export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" - else - export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version: *//')" - fi - else - pip_install "torch==$PYTORCH_VERSION$CUDA_SUFFIX" \ - -f https://download.pytorch.org/whl/torch_stable.html \ - -f https://download.pytorch.org/whl/nightly/torch_nightly.html - fi -} - -# Fill PYTORCH_VERSION with the latest conda nightly version, and -# CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions -# -# You MUST have populated CUDA_SUFFIX before hand. -setup_conda_pytorch_constraint() { - if [[ -z "$PYTORCH_VERSION" ]]; then - export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" - export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | ./packaging/versionator.py)" - if [[ -z "$PYTORCH_VERSION" ]]; then - echo "PyTorch version auto detection failed" - echo "No package found for CU_VERSION=$CU_VERSION and PYTHON_VERSION=$PYTHON_VERSION" - exit 1 - fi - else - export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" - fi - if [[ "$CU_VERSION" == cpu ]]; then - export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" - export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" - else - export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}" - export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}" - fi -} - -# Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT -setup_conda_cudatoolkit_constraint() { - export CONDA_CPUONLY_FEATURE="" - if [[ "$(uname)" == Darwin ]]; then - export CONDA_CUDATOOLKIT_CONSTRAINT="" - else - case "$CU_VERSION" in - cu101) - export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.1,<10.2 # [not osx]" - ;; - cu100) - export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]" - ;; - cu92) - export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]" - ;; - cpu) - export CONDA_CUDATOOLKIT_CONSTRAINT="" - export CONDA_CPUONLY_FEATURE="- cpuonly" - ;; - *) - echo "Unrecognized CU_VERSION=$CU_VERSION" - exit 1 - ;; - esac - fi -} diff --git a/packaging/torchvision/bld.bat b/packaging/torchvision/bld.bat deleted file mode 100644 index 14f6935fb..000000000 --- a/packaging/torchvision/bld.bat +++ /dev/null @@ -1,24 +0,0 @@ -@echo on - -set TORCHVISION_BUILD_VERSION=%PKG_VERSION% -set TORCHVISION_BUILD_NUMBER=%PKG_BUILDNUM% - -if not "%CUDA_VERSION%" == "None" ( - set build_with_cuda=1 - set desired_cuda=%CUDA_VERSION:~0,-1%.%CUDA_VERSION:~-1,1% -) else ( - set build_with_cuda= -) - -if "%build_with_cuda%" == "" goto cuda_flags_end - -set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%desired_cuda% -set CUDA_BIN_PATH=%CUDA_PATH%\bin -set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -if "%desired_cuda%" == "9.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50 -if "%desired_cuda%" == "10.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50 - -:cuda_flags_end - -python setup.py install --single-version-externally-managed --record=record.txt -if errorlevel 1 exit /b 1 diff --git a/packaging/torchvision/conda_build_config.yaml b/packaging/torchvision/conda_build_config.yaml deleted file mode 100644 index 5188bb0eb..000000000 --- a/packaging/torchvision/conda_build_config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -blas_impl: - - mkl # [x86_64] -c_compiler: - - vs2017 # [win] -cxx_compiler: - - vs2017 # [win] -python: - - 3.5 - - 3.6 -# This differs from target_platform in that it determines what subdir the compiler -# will target, not what subdir the compiler package will be itself. -# For example, we need a win-64 vs2008_win-32 package, so that we compile win-32 -# code on win-64 miniconda. -cross_compiler_target_platform: - - win-64 # [win] -target_platform: - - win-64 # [win] -vc: - - 14 -zip_keys: - - # [win] - - vc # [win] - - c_compiler # [win] - - cxx_compiler # [win] diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml deleted file mode 100644 index fdcfbf78d..000000000 --- a/packaging/torchvision/meta.yaml +++ /dev/null @@ -1,57 +0,0 @@ -package: - name: torchvision - version: "{{ environ.get('BUILD_VERSION') }}" - -source: - path: "{{ environ.get('SOURCE_ROOT_DIR') }}" - -requirements: - build: - - {{ compiler('c') }} # [win] - - host: - - python - - setuptools - {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} - {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} - - run: - - python - - pillow >=4.1.1 - - numpy >=1.11 - - six - {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} - {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} - -build: - string: py{{py}}_{{ environ['CU_VERSION'] }} - script: python setup.py install --single-version-externally-managed --record=record.txt # [not win] - script_env: - - CUDA_HOME - - FORCE_CUDA - - NVCC_FLAGS - features: - {{ CONDA_CPUONLY_FEATURE }} - -test: - imports: - - torchvision - - torchvision.datasets - - torchvision.transforms - source_files: - - test - requires: - - pytest - - scipy - - mock - - av - - ca-certificates - commands: - pytest . - - -about: - home: https://github.com/pytorch/vision - license: BSD - license_file: LICENSE - summary: 'image and video datasets and models for torch deep learning' diff --git a/packaging/versionator.py b/packaging/versionator.py deleted file mode 100755 index 1a7460983..000000000 --- a/packaging/versionator.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python - -""" -TODO: This was hard to read in pkg_helpers.bash, so I've extracted it -to its own script. This script is not yet being called by -pkg_helpers.bash yet. -""" - - -import os -import sys -import json -import re - -cuver = os.environ.get('CU_VERSION') -cuver = (cuver[:-1] + '.' + cuver[-1]).replace('cu', 'cuda') if cuver != 'cpu' else cuver - -pytorch_entries = json.load(sys.stdin)['pytorch'] - -filtered_pytorch_entries_plat_cuda = list(filter( - lambda x: (x['platform'] == 'darwin' or cuver in x['fn']), pytorch_entries -)) - -filtered_pytorch_entries_py_ver = list(filter( - lambda x: 'py' + os.environ['PYTHON_VERSION'] in x['fn'], filtered_pytorch_entries_plat_cuda -)) - -versions = [x['version'] for x in filtered_pytorch_entries_py_ver] - -try: - last_entry = versions[-1] - print(re.sub(r'\\+.*$', '', last_entry)) - -except Exception as e: - - all_platforms = set([x['platform'] for x in pytorch_entries]) - all_fns = set([x['fn'] for x in pytorch_entries]) - - msg = "\n\t".join([ - "Exception was: " + str(e), - "Unfiltered entries count: " + str(len(pytorch_entries)), - "Filtered by platform count: " + str(len(filtered_pytorch_entries_plat_cuda)), - "Filtered by python version count: " + str(len(filtered_pytorch_entries_py_ver)), - "all_platforms:\n" + "".join(map(lambda x: "\t\t" + str(x) + "\n", all_platforms)), - "all_fns:\n" + "".join(map(lambda x: "\t\t" + str(x) + "\n", all_fns)), - ]) - - sys.exit(msg) diff --git a/packaging/wheel/linux_manywheel.sh b/packaging/wheel/linux_manywheel.sh deleted file mode 100644 index 79b181f55..000000000 --- a/packaging/wheel/linux_manywheel.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -set -ex - -if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters. Pass cuda version" - echo "CUDA version should be cu92, cu100, cu101 or cpu" - exit 1 -fi -export CUVER="$1" # cu92 cu100 cu101 cpu - -if [[ "$CUVER" == "cu101" ]]; then - cu_suffix="" -else - cu_suffix="+$CUVER" -fi - -export TORCHVISION_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")${cu_suffix}" -export TORCHVISION_BUILD_NUMBER="1" -export TORCHVISION_LOCAL_VERSION_LABEL="$CUVER" -export OUT_DIR="/remote/$CUVER" - -pushd /opt/python -DESIRED_PYTHON=(*/) -popd -for desired_py in "${DESIRED_PYTHON[@]}"; do - python_installations+=("/opt/python/$desired_py") -done - -OLD_PATH=$PATH -cd /tmp -rm -rf vision -git clone https://github.com/pytorch/vision - -cd /tmp/vision - -for PYDIR in "${python_installations[@]}"; do - export PATH=$PYDIR/bin:$OLD_PATH - pip install --upgrade pip - pip install numpy pyyaml future - - pip uninstall -y torch || true - pip uninstall -y torch_nightly || true - - export TORCHVISION_PYTORCH_DEPENDENCY_NAME=torch_nightly - pip install torch_nightly -f https://download.pytorch.org/whl/nightly/$CUVER/torch_nightly.html - # CPU/CUDA variants of PyTorch have ABI compatible PyTorch for - # the CPU only bits. Therefore, we - # strip off the local package qualifier, but ONLY if we're - # doing a CPU build. - if [[ "$CUVER" == "cpu" ]]; then - export TORCHVISION_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//' | sed 's/+.\+//')" - else - export TORCHVISION_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: \+//')" - fi - echo "Building against ${TORCHVISION_PYTORCH_DEPENDENCY_VERSION}" - - pip install ninja - python setup.py clean - python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done diff --git a/packaging/wheel/osx_wheel.sh b/packaging/wheel/osx_wheel.sh deleted file mode 100644 index 468936048..000000000 --- a/packaging/wheel/osx_wheel.sh +++ /dev/null @@ -1,52 +0,0 @@ -if [[ ":$PATH:" == *"conda"* ]]; then - echo "existing anaconda install in PATH, remove it and run script" - exit 1 -fi -# download and activate anaconda -rm -rf ~/minconda_wheel_env_tmp -wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh && \ - chmod +x Miniconda3-latest-MacOSX-x86_64.sh && \ - ./Miniconda3-latest-MacOSX-x86_64.sh -b -p ~/minconda_wheel_env_tmp && \ - rm Miniconda3-latest-MacOSX-x86_64.sh - -. ~/minconda_wheel_env_tmp/bin/activate - - -export TORCHVISION_BUILD_VERSION="0.4.0.dev$(date "+%Y%m%d")" -export TORCHVISION_BUILD_NUMBER="1" -export OUT_DIR=~/torchvision_wheels - -export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ - -pushd /tmp -rm -rf vision -git clone https://github.com/pytorch/vision -pushd vision - -desired_pythons=( "2.7" "3.5" "3.6" "3.7" "3.8" ) -# for each python -for desired_python in "${desired_pythons[@]}" -do - # create and activate python env - env_name="env$desired_python" - conda create -yn $env_name python="$desired_python" - conda activate $env_name - - pip uninstall -y torch || true - pip uninstall -y torch_nightly || true - - export TORCHVISION_PYTORCH_DEPENDENCY_NAME=torch_nightly - pip install torch_nightly -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - export TORCHVISION_PYTORCH_DEPENDENCY_VERSION="$(pip show torch_nightly | grep ^Version: | sed 's/Version: *//')" - echo "Building against ${TORCHAUDIO_PYTORCH_DEPENDENCY_VERSION}" - - # install torchvision dependencies - pip install ninja scipy pytest - - python setup.py clean - python setup.py bdist_wheel - mkdir -p $OUT_DIR - cp dist/*.whl $OUT_DIR/ -done -popd -popd diff --git a/packaging/windows/build_vision.bat b/packaging/windows/build_vision.bat deleted file mode 100644 index c7b65dc04..000000000 --- a/packaging/windows/build_vision.bat +++ /dev/null @@ -1,94 +0,0 @@ -@echo off - -:: This script parses args, installs required libraries (miniconda, MKL, -:: Magma), and then delegates to cpu.bat, cuda80.bat, etc. - -IF NOT "%CUDA_VERSION%" == "" IF NOT "%TORCHVISION_BUILD_VERSION%" == "" if NOT "%TORCHVISION_BUILD_NUMBER%" == "" goto env_end -if "%~1"=="" goto arg_error -if "%~2"=="" goto arg_error -if "%~3"=="" goto arg_error -if NOT "%~4"=="" goto arg_error -goto arg_end - -:arg_error - -echo Illegal number of parameters. Pass cuda version, pytorch version, build number -echo CUDA version should be Mm with no dot, e.g. '80' -echo DESIRED_PYTHON should be M.m, e.g. '2.7' -exit /b 1 - -:arg_end - -set CUDA_VERSION=%~1 -set TORCHVISION_BUILD_VERSION=%~2 -set TORCHVISION_BUILD_NUMBER=%~3 - -:env_end - -if NOT "%CUDA_VERSION%" == "cpu" ( - set CUDA_PREFIX=cuda%CUDA_VERSION% - set CUVER=cu%CUDA_VERSION% -) else ( - set CUDA_PREFIX=cpu - set CUVER=cpu -) - -set BUILD_VISION=1 -set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index - -IF "%DESIRED_PYTHON%" == "" set DESIRED_PYTHON=3.5;3.6;3.7 -set DESIRED_PYTHON_PREFIX=%DESIRED_PYTHON:.=% -set DESIRED_PYTHON_PREFIX=py%DESIRED_PYTHON_PREFIX:;=;py% - -set SRC_DIR=%~dp0 -pushd %SRC_DIR% - -:: Install Miniconda3 -set "CONDA_HOME=%CD%\conda" -set "tmp_conda=%CONDA_HOME%" -set "miniconda_exe=%CD%\miniconda.exe" -rmdir /s /q conda -del miniconda.exe -curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%" -call ..\conda\install_conda.bat -IF ERRORLEVEL 1 exit /b 1 -set "ORIG_PATH=%PATH%" -set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%" - -:: Create a new conda environment -setlocal EnableDelayedExpansion -FOR %%v IN (%DESIRED_PYTHON%) DO ( - set PYTHON_VERSION_STR=%%v - set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=! - conda remove -n py!PYTHON_VERSION_STR! --all -y || rmdir %CONDA_HOME%\envs\py!PYTHON_VERSION_STR! /s - conda create -n py!PYTHON_VERSION_STR! -y -q numpy>=1.11 mkl>=2018 python=%%v -) -endlocal - -if "%DEBUG%" == "1" ( - set BUILD_TYPE=debug -) ELSE ( - set BUILD_TYPE=release -) - -for %%v in (%DESIRED_PYTHON_PREFIX%) do ( - :: Activate Python Environment - set PYTHON_PREFIX=%%v - set "PATH=%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%" - pip install %TORCH_WHEEL% - @setlocal - :: Set Flags - if NOT "%CUDA_VERSION%"=="cpu" ( - set CUDNN_VERSION=7 - ) - call %CUDA_PREFIX%.bat - IF ERRORLEVEL 1 exit /b 1 - call internal\test.bat - IF ERRORLEVEL 1 exit /b 1 - @endlocal -) - -set "PATH=%ORIG_PATH%" -popd - -IF ERRORLEVEL 1 exit /b 1 diff --git a/packaging/windows/cpu.bat b/packaging/windows/cpu.bat deleted file mode 100644 index 392a687f9..000000000 --- a/packaging/windows/cpu.bat +++ /dev/null @@ -1,37 +0,0 @@ -@echo off - -IF NOT "%BUILD_VISION%" == "" ( - set MODULE_NAME=vision -) ELSE ( - set MODULE_NAME=pytorch -) - -IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( - call internal\clone.bat - cd .. - IF ERRORLEVEL 1 goto eof -) ELSE ( - call internal\clean.bat -) - -call internal\check_deps.bat -IF ERRORLEVEL 1 goto eof - -REM Check for optional components - -echo Disabling CUDA -set NO_CUDA=1 -set USE_CUDA=0 - -IF "%BUILD_VISION%" == "" ( - call internal\check_opts.bat - IF ERRORLEVEL 1 goto eof - - call internal\copy_cpu.bat - IF ERRORLEVEL 1 goto eof -) - -call internal\setup.bat -IF ERRORLEVEL 1 goto eof - -:eof diff --git a/packaging/windows/cuda100.bat b/packaging/windows/cuda100.bat deleted file mode 100644 index ac9be3c69..000000000 --- a/packaging/windows/cuda100.bat +++ /dev/null @@ -1,59 +0,0 @@ -@echo off - -IF NOT "%BUILD_VISION%" == "" ( - set MODULE_NAME=vision -) ELSE ( - set MODULE_NAME=pytorch -) - -IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( - call internal\clone.bat - cd .. - IF ERRORLEVEL 1 goto eof -) ELSE ( - call internal\clean.bat -) - -call internal\check_deps.bat -IF ERRORLEVEL 1 goto eof - -REM Check for optional components - -set NO_CUDA= -set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 - -IF "%NVTOOLSEXT_PATH%"=="" ( - echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing - exit /b 1 - goto optcheck -) - -IF "%CUDA_PATH_V10_0%"=="" ( - echo CUDA 10.0 not found, failing - exit /b 1 -) ELSE ( - IF "%BUILD_VISION%" == "" ( - set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5 - set TORCH_NVCC_FLAGS=-Xfatbin -compress-all - ) ELSE ( - set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50 - ) - - set "CUDA_PATH=%CUDA_PATH_V10_0%" - set "PATH=%CUDA_PATH_V10_0%\bin;%PATH%" -) - -:optcheck - -IF "%BUILD_VISION%" == "" ( - call internal\check_opts.bat - IF ERRORLEVEL 1 goto eof - - call internal\copy.bat - IF ERRORLEVEL 1 goto eof -) - -call internal\setup.bat -IF ERRORLEVEL 1 goto eof - -:eof diff --git a/packaging/windows/cuda90.bat b/packaging/windows/cuda90.bat deleted file mode 100644 index fe0294812..000000000 --- a/packaging/windows/cuda90.bat +++ /dev/null @@ -1,59 +0,0 @@ -@echo off - -IF NOT "%BUILD_VISION%" == "" ( - set MODULE_NAME=vision -) ELSE ( - set MODULE_NAME=pytorch -) - -IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( - call internal\clone.bat - cd .. - IF ERRORLEVEL 1 goto eof -) ELSE ( - call internal\clean.bat -) - -call internal\check_deps.bat -IF ERRORLEVEL 1 goto eof - -REM Check for optional components - -set NO_CUDA= -set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 - -IF "%NVTOOLSEXT_PATH%"=="" ( - echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing - exit /b 1 - goto optcheck -) - -IF "%CUDA_PATH_V9_0%"=="" ( - echo CUDA 9 not found, failing - exit /b 1 -) ELSE ( - IF "%BUILD_VISION%" == "" ( - set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;7.0 - set TORCH_NVCC_FLAGS=-Xfatbin -compress-all - ) ELSE ( - set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50 - ) - - set "CUDA_PATH=%CUDA_PATH_V9_0%" - set "PATH=%CUDA_PATH_V9_0%\bin;%PATH%" -) - -:optcheck - -IF "%BUILD_VISION%" == "" ( - call internal\check_opts.bat - IF ERRORLEVEL 1 goto eof - - call internal\copy.bat - IF ERRORLEVEL 1 goto eof -) - -call internal\setup.bat -IF ERRORLEVEL 1 goto eof - -:eof diff --git a/packaging/windows/internal/7z_install.bat b/packaging/windows/internal/7z_install.bat deleted file mode 100644 index d5a115636..000000000 --- a/packaging/windows/internal/7z_install.bat +++ /dev/null @@ -1,9 +0,0 @@ -@echo off - -curl -k https://www.7-zip.org/a/7z1805-x64.exe -O -if errorlevel 1 exit /b 1 - -start /wait 7z1805-x64.exe /S -if errorlevel 1 exit /b 1 - -set "PATH=%ProgramFiles%\7-Zip;%PATH%" diff --git a/packaging/windows/internal/auth.bat b/packaging/windows/internal/auth.bat deleted file mode 100644 index c874bce49..000000000 --- a/packaging/windows/internal/auth.bat +++ /dev/null @@ -1,46 +0,0 @@ -@echo off - -: From the following doc, the build won't be triggered if the users don't sign in daily. -: https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?tabs=yaml&view=vsts#my-build-didnt-run-what-happened -: To avoid this problem, we can just go through the sign in process using the following command. - -:auth_start - -if "%RETRY_TIMES%" == "" ( - set /a RETRY_TIMES=10 - set /a SLEEP_TIME=2 -) else ( - set /a RETRY_TIMES=%RETRY_TIMES%-1 - set /a SLEEP_TIME=%SLEEP_TIME%*2 -) - -for /f "usebackq tokens=*" %%i in (`curl -so NUL -w "%%{http_code}" -u %VSTS_AUTH% https://dev.azure.com/pytorch`) do ( - set STATUS_CODE=%%i -) - -IF NOT "%STATUS_CODE%" == "200" ( - echo Auth retry times remaining: %RETRY_TIMES% - echo Sleep time: %SLEEP_TIME% seconds - IF %RETRY_TIMES% EQU 0 ( - echo Auth failed - goto err - ) - waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul - goto auth_start -) ELSE ( - echo Login Attempt Succeeded - goto auth_end -) - -:err - -: Throw a warning if it fails -powershell -c "Write-Warning 'Login Attempt Failed'" - -:auth_end - -set RETRY_TIMES= -set SLEEP_TIME= -set STATUS_CODE= - -exit /b 0 diff --git a/packaging/windows/internal/check_deps.bat b/packaging/windows/internal/check_deps.bat deleted file mode 100644 index a159d4436..000000000 --- a/packaging/windows/internal/check_deps.bat +++ /dev/null @@ -1,67 +0,0 @@ -@echo off - -REM Check for necessary components - -IF NOT "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( - echo You should use 64 bits Windows to build and run PyTorch - exit /b 1 -) - -IF "%BUILD_VISION%" == "" ( - where /q cmake.exe - - IF ERRORLEVEL 1 ( - echo CMake is required to compile PyTorch on Windows - exit /b 1 - ) -) - -IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" ( - echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows - exit /b 1 -) - -for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do ( - if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( - set "VS15INSTALLDIR=%%i" - set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" - goto vswhere - ) -) - -:vswhere -IF "%VS15VCVARSALL%"=="" ( - echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows - exit /b 1 -) - -set MSSdk=1 -set DISTUTILS_USE_SDK=1 - -where /q python.exe - -IF ERRORLEVEL 1 ( - echo Python x64 3.5 or up is required to compile PyTorch on Windows - exit /b 1 -) - -for /F "usebackq delims=" %%i in (`python -c "import sys; print('{0[0]}{0[1]}'.format(sys.version_info))"`) do ( - set /a PYVER=%%i -) - -if %PYVER% LSS 35 ( - echo Warning: PyTorch for Python 2 under Windows is experimental. - echo Python x64 3.5 or up is recommended to compile PyTorch on Windows - echo Maybe you can create a virual environment if you have conda installed: - echo ^> conda create -n test python=3.6 pyyaml mkl numpy - echo ^> activate test -) - -for /F "usebackq delims=" %%i in (`python -c "import struct;print( 8 * struct.calcsize('P'))"`) do ( - set /a PYSIZE=%%i -) - -if %PYSIZE% NEQ 64 ( - echo Python x64 3.5 or up is required to compile PyTorch on Windows - exit /b 1 -) diff --git a/packaging/windows/internal/check_opts.bat b/packaging/windows/internal/check_opts.bat deleted file mode 100644 index 003ad9213..000000000 --- a/packaging/windows/internal/check_opts.bat +++ /dev/null @@ -1,33 +0,0 @@ -@echo off - -REM Check for optional components - -where /q ninja.exe - -IF NOT ERRORLEVEL 1 ( - echo Ninja found, using it to speed up builds - set CMAKE_GENERATOR=Ninja -) - -where /q clcache.exe - -IF NOT ERRORLEVEL 1 ( - echo clcache found, using it to speed up builds - set CC=clcache - set CXX=clcache -) - -where /q sccache.exe - -IF NOT ERRORLEVEL 1 ( - echo sccache found, using it to speed up builds - set CC=sccache cl - set CXX=sccache cl -) - -IF exist "%MKLProductDir%\mkl\lib\intel64_win" ( - echo MKL found, adding it to build - set "LIB=%MKLProductDir%\mkl\lib\intel64_win;%MKLProductDir%\compiler\lib\intel64_win;%LIB%"; -) - -exit /b 0 diff --git a/packaging/windows/internal/clean.bat b/packaging/windows/internal/clean.bat deleted file mode 100644 index 7489640f4..000000000 --- a/packaging/windows/internal/clean.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off - -cd %MODULE_NAME% -python setup.py clean -cd .. diff --git a/packaging/windows/internal/clone.bat b/packaging/windows/internal/clone.bat deleted file mode 100644 index f93271978..000000000 --- a/packaging/windows/internal/clone.bat +++ /dev/null @@ -1,60 +0,0 @@ -@echo off - -:: The conda and wheels jobs are seperated on Windows, so we don't need to clone again. -IF "%BUILD_VISION%" == "" ( - if exist "%NIGHTLIES_PYTORCH_ROOT%" ( - xcopy /E /Y /Q "%NIGHTLIES_PYTORCH_ROOT%" pytorch\ - cd pytorch - goto submodule - ) -) - -git clone https://github.com/%PYTORCH_REPO%/%MODULE_NAME% - -cd %MODULE_NAME% - -IF NOT "%BUILD_VISION%" == "" goto latest_end - -IF "%PYTORCH_BRANCH%" == "latest" ( goto latest_start ) else ( goto latest_end ) - -:latest_start - -if "%NIGHTLIES_DATE%" == "" ( goto date_start ) else ( goto date_end ) - -:date_start - -set "DATE_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyy_MM_dd'" -set "DATE_COMPACT_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyyMMdd'" - -FOR /F "delims=" %%i IN ('powershell -c "%DATE_CMD%"') DO set NIGHTLIES_DATE=%%i -FOR /F "delims=" %%i IN ('powershell -c "%DATE_COMPACT_CMD%"') DO set NIGHTLIES_DATE_COMPACT=%%i - -:date_end - -if "%NIGHTLIES_DATE_COMPACT%" == "" set NIGHTLIES_DATE_COMPACT=%NIGHTLIES_DATE:~0,4%%NIGHTLIES_DATE:~5,2%%NIGHTLIES_DATE:~8,2% - -:: Switch to the latest commit by 11:59 yesterday -echo PYTORCH_BRANCH is set to latest so I will find the last commit -echo before 0:00 midnight on %NIGHTLIES_DATE% -set git_date=%NIGHTLIES_DATE:_=-% -FOR /F "delims=" %%i IN ('git log --before %git_date% -n 1 "--pretty=%%H"') DO set last_commit=%%i -echo Setting PYTORCH_BRANCH to %last_commit% since that was the last -echo commit before %NIGHTLIES_DATE% -set PYTORCH_BRANCH=%last_commit% - -:latest_end - -IF "%BUILD_VISION%" == "" ( - IF "%PYTORCH_BRANCH%" == "" ( - set PYTORCH_BRANCH=v%PYTORCH_BUILD_VERSION% - ) - git checkout %PYTORCH_BRANCH% - IF ERRORLEVEL 1 git checkout tags/%PYTORCH_BRANCH% -) ELSE ( - git checkout v%TORCHVISION_BUILD_VERSION% -) - -:submodule - -git submodule update --init --recursive -IF ERRORLEVEL 1 exit /b 1 diff --git a/packaging/windows/internal/copy.bat b/packaging/windows/internal/copy.bat deleted file mode 100644 index b4aa397c6..000000000 --- a/packaging/windows/internal/copy.bat +++ /dev/null @@ -1,13 +0,0 @@ -copy "%CUDA_PATH%\bin\cusparse64_%CUDA_VERSION%.dll*" pytorch\torch\lib -copy "%CUDA_PATH%\bin\cublas64_%CUDA_VERSION%.dll*" pytorch\torch\lib -copy "%CUDA_PATH%\bin\cudart64_%CUDA_VERSION%.dll*" pytorch\torch\lib -copy "%CUDA_PATH%\bin\curand64_%CUDA_VERSION%.dll*" pytorch\torch\lib -copy "%CUDA_PATH%\bin\cufft64_%CUDA_VERSION%.dll*" pytorch\torch\lib -copy "%CUDA_PATH%\bin\cufftw64_%CUDA_VERSION%.dll*" pytorch\torch\lib - -copy "%CUDA_PATH%\bin\cudnn64_%CUDNN_VERSION%.dll*" pytorch\torch\lib -copy "%CUDA_PATH%\bin\nvrtc64_%CUDA_VERSION%*.dll*" pytorch\torch\lib -copy "%CUDA_PATH%\bin\nvrtc-builtins64_%CUDA_VERSION%.dll*" pytorch\torch\lib - -copy "C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64\nvToolsExt64_1.dll*" pytorch\torch\lib -copy "%CONDA_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib diff --git a/packaging/windows/internal/copy_cpu.bat b/packaging/windows/internal/copy_cpu.bat deleted file mode 100644 index f5b9d1151..000000000 --- a/packaging/windows/internal/copy_cpu.bat +++ /dev/null @@ -1 +0,0 @@ -copy "%CONDA_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib diff --git a/packaging/windows/internal/cuda_install.bat b/packaging/windows/internal/cuda_install.bat deleted file mode 100644 index b6ea9e237..000000000 --- a/packaging/windows/internal/cuda_install.bat +++ /dev/null @@ -1,122 +0,0 @@ -@echo off - -set SRC_DIR=%~dp0\.. - -if not exist "%SRC_DIR%\temp_build" mkdir "%SRC_DIR%\temp_build" - -set /a CUDA_VER=%CUDA_VERSION% -set CUDA_VER_MAJOR=%CUDA_VERSION:~0,-1% -set CUDA_VER_MINOR=%CUDA_VERSION:~-1,1% -set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR% - -IF %CUDA_VER% LEQ 90 ( - set "NVCC_PACKAGE=compiler_%CUDA_VERSION_STR%" -) ELSE ( - set "NVCC_PACKAGE=nvcc_%CUDA_VERSION_STR%" -) - -IF %CUDA_VER% EQU 80 goto cuda80 -IF %CUDA_VER% EQU 90 goto cuda90 -IF %CUDA_VER% EQU 91 goto cuda91 -IF %CUDA_VER% EQU 92 goto cuda92 -IF %CUDA_VER% EQU 100 goto cuda100 - -echo CUDA %CUDA_VERSION_STR% is not supported -exit /b 1 - -:cuda80 - -echo CUDA 8.0 is not supported -exit /b 1 - -:cuda90 -IF NOT EXIST "%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z" ( - curl -k -L https://www.dropbox.com/s/z5b7ryz0zrimntl/cuda_9.0.176_windows.7z?dl=1 --output "%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z" - if errorlevel 1 exit /b 1 - set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z" - set "NVCC_PACKAGE=compiler_%CUDA_VERSION_STR%" -) - -IF NOT EXIST "%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip" ( - curl -k -L https://www.dropbox.com/s/6p0xyqh472nu8m1/cudnn-9.0-windows7-x64-v7.zip?dl=1 --output "%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip" - if errorlevel 1 exit /b 1 - set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip" -) - -goto cuda_common - -:cuda91 - -IF NOT EXIST "%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z" ( - curl -k -L https://www.dropbox.com/s/7a4sbq0dln6v7t2/cuda_9.1.85_windows.7z?dl=1 --output "%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z" - if errorlevel 1 exit /b 1 - set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z" - set "NVCC_PACKAGE=nvcc_%CUDA_VERSION_STR%" -) - -IF NOT EXIST "%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip" ( - curl -k -L https://www.dropbox.com/s/e0prhgsrbyfi4ov/cudnn-9.1-windows7-x64-v7.zip?dl=1 --output "%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip" - if errorlevel 1 exit /b 1 - set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip" -) - -goto cuda_common - -:cuda92 - -echo CUDA 9.2 is not supported -exit /b 1 - -:cuda100 - -echo CUDA 10.0 is not supported -exit /b 1 - -:cuda_common - -set "CUDA_PREFIX=cuda%CUDA_VERSION%" - -IF NOT EXIST "%SRC_DIR%\temp_build\NvToolsExt.7z" ( - curl -k -L https://www.dropbox.com/s/9mcolalfdj4n979/NvToolsExt.7z?dl=1 --output "%SRC_DIR%\temp_build\NvToolsExt.7z" - if errorlevel 1 exit /b 1 -) - -echo Installing CUDA toolkit... - -7z x %CUDA_SETUP_FILE% -o"%SRC_DIR%\temp_build\cuda" -pushd "%SRC_DIR%\temp_build\cuda" -dir -start /wait setup.exe -s %NVCC_PACKAGE% cublas_%CUDA_VERSION_STR% cublas_dev_%CUDA_VERSION_STR% cudart_%CUDA_VERSION_STR% curand_%CUDA_VERSION_STR% curand_dev_%CUDA_VERSION_STR% cusparse_%CUDA_VERSION_STR% cusparse_dev_%CUDA_VERSION_STR% nvrtc_%CUDA_VERSION_STR% nvrtc_dev_%CUDA_VERSION_STR% cufft_%CUDA_VERSION_STR% cufft_dev_%CUDA_VERSION_STR% -popd -echo Installing VS integration... -xcopy /Y "%SRC_DIR%\temp_build\cuda\_vs\*.*" "c:\Program Files (x86)\MSBuild\Microsoft.Cpp\v4.0\V140\BuildCustomizations" - -echo Installing NvToolsExt... -7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt" -mkdir "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64" -mkdir "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\include" -mkdir "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\lib\x64" -xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\bin\x64\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64" -xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\include\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\include" -xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\lib\x64\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\lib\x64" - -echo Installing cuDNN... -7z x %CUDNN_SETUP_FILE% -o"%SRC_DIR%\temp_build\cudnn" -xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\bin\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin" -xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\lib\x64\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\lib\x64" -xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\include\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\include" - -echo Setting up environment... -set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\libnvvp;%PATH%" -set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%" -set "CUDA_PATH_V%CUDA_VER_MAJOR%_%CUDA_VER_MINOR%=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%" -set "NVTOOLSEXT_PATH=%ProgramFiles%\NVIDIA Corporation\NvToolsExt\" - -echo Cleaning temp files -rd /s /q "%SRC_DIR%\temp_build" -pushd %TEMP% -rd /s /q . -popd - -echo Using VS2015 as NVCC compiler -set "CUDAHOSTCXX=%VS140COMNTOOLS%\..\..\VC\bin\amd64\cl.exe" diff --git a/packaging/windows/internal/setup.bat b/packaging/windows/internal/setup.bat deleted file mode 100644 index fcf10f185..000000000 --- a/packaging/windows/internal/setup.bat +++ /dev/null @@ -1,91 +0,0 @@ -@echo off - -echo The flags after configuring: -echo NO_CUDA=%NO_CUDA% -echo CMAKE_GENERATOR=%CMAKE_GENERATOR% -if "%NO_CUDA%"=="" echo CUDA_PATH=%CUDA_PATH% -if NOT "%CC%"=="" echo CC=%CC% -if NOT "%CXX%"=="" echo CXX=%CXX% -if NOT "%DISTUTILS_USE_SDK%"=="" echo DISTUTILS_USE_SDK=%DISTUTILS_USE_SDK% - -set SRC_DIR=%~dp0\.. - -IF "%VSDEVCMD_ARGS%" == "" ( - call "%VS15VCVARSALL%" x64 -) ELSE ( - call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% -) - -pushd %SRC_DIR% - -IF NOT exist "setup.py" ( - cd %MODULE_NAME% -) - -if "%CXX%"=="sccache cl" ( - sccache --stop-server - sccache --start-server - sccache --zero-stats -) - - -if "%BUILD_PYTHONLESS%" == "" goto pytorch else goto libtorch - -:libtorch -set VARIANT=shared-with-deps - -mkdir libtorch -mkdir libtorch\bin -mkdir libtorch\cmake -mkdir libtorch\include -mkdir libtorch\lib -mkdir libtorch\share -mkdir libtorch\test - -mkdir build -pushd build -python ../tools/build_libtorch.py -popd - -IF ERRORLEVEL 1 exit /b 1 -IF NOT ERRORLEVEL 0 exit /b 1 - -move /Y torch\bin\*.* libtorch\bin\ -move /Y torch\cmake\*.* libtorch\cmake\ -robocopy /move /e torch\include\ libtorch\include\ -move /Y torch\lib\*.* libtorch\lib\ -robocopy /move /e torch\share\ libtorch\share\ -move /Y torch\test\*.* libtorch\test\ - -move /Y libtorch\bin\*.dll libtorch\lib\ - -git rev-parse HEAD > libtorch\build-hash - -IF "%DEBUG%" == "" ( - set LIBTORCH_PREFIX=libtorch-win-%VARIANT% -) ELSE ( - set LIBTORCH_PREFIX=libtorch-win-%VARIANT%-debug -) - -7z a -tzip %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip libtorch\* - -mkdir ..\output\%CUDA_PREFIX% -copy /Y %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip ..\output\%CUDA_PREFIX%\ -copy /Y %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip ..\output\%CUDA_PREFIX%\%LIBTORCH_PREFIX%-latest.zip - -goto build_end - -:pytorch -:: This stores in e.g. D:/_work/1/s/windows/output/cpu -pip wheel -e . --no-deps --wheel-dir ../output/%CUDA_PREFIX% - -:build_end -IF ERRORLEVEL 1 exit /b 1 -IF NOT ERRORLEVEL 0 exit /b 1 - -if "%CXX%"=="sccache cl" ( - taskkill /im sccache.exe /f /t || ver > nul - taskkill /im nvcc.exe /f /t || ver > nul -) - -cd .. diff --git a/packaging/windows/internal/test.bat b/packaging/windows/internal/test.bat deleted file mode 100644 index 1ad7d2ebe..000000000 --- a/packaging/windows/internal/test.bat +++ /dev/null @@ -1,75 +0,0 @@ -@echo off - -set SRC_DIR=%~dp0\.. -pushd %SRC_DIR% - -set PYTHON_VERSION=%PYTHON_PREFIX:py=cp% - -if "%BUILD_VISION%" == "" ( - pip install future pytest coverage hypothesis protobuf -) ELSE ( - pip install future pytest "pillow>=4.1.1" -) - - -for /F "delims=" %%i in ('where /R %SRC_DIR%\output\%CUDA_PREFIX% %MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i" - -if ERRORLEVEL 1 exit /b 1 - -if NOT "%BUILD_VISION%" == "" goto smoke_test_end - -echo Smoke testing imports -python -c "import torch" -if ERRORLEVEL 1 exit /b 1 - -python -c "from caffe2.python import core" -if ERRORLEVEL 1 exit /b 1 - -echo Checking that MKL is available -python -c "import torch; exit(0 if torch.backends.mkl.is_available() else 1)" -if ERRORLEVEL 1 exit /b 1 - -setlocal EnableDelayedExpansion -set NVIDIA_GPU_EXISTS=0 -for /F "delims=" %%i in ('wmic path win32_VideoController get name') do ( - set GPUS=%%i - if not "x!GPUS:NVIDIA=!" == "x!GPUS!" ( - SET NVIDIA_GPU_EXISTS=1 - goto gpu_check_end - ) -) -:gpu_check_end -endlocal & set NVIDIA_GPU_EXISTS=%NVIDIA_GPU_EXISTS% - -if NOT "%CUDA_PREFIX%" == "cpu" if "%NVIDIA_GPU_EXISTS%" == "1" ( - echo Checking that CUDA archs are setup correctly - python -c "import torch; torch.randn([3,5]).cuda()" - if ERRORLEVEL 1 exit /b 1 - - echo Checking that magma is available - python -c "import torch; torch.rand(1).cuda(); exit(0 if torch.cuda.has_magma else 1)" - if ERRORLEVEL 1 exit /b 1 - - echo Checking that CuDNN is available - python -c "import torch; exit(0 if torch.backends.cudnn.is_available() else 1)" - if ERRORLEVEL 1 exit /b 1 -) -:smoke_test_end - -echo Not running unit tests. Hopefully these problems are caught by CI -goto test_end - -if "%BUILD_VISION%" == "" ( - cd pytorch\test - python run_test.py -v -) else ( - cd vision - pytest . -) - -if ERRORLEVEL 1 exit /b 1 - -:test_end - -popd -exit /b 0 diff --git a/packaging/windows/internal/vs_install.bat b/packaging/windows/internal/vs_install.bat deleted file mode 100644 index 624227f0b..000000000 --- a/packaging/windows/internal/vs_install.bat +++ /dev/null @@ -1,28 +0,0 @@ -@echo off - -set VS_DOWNLOAD_LINK=https://aka.ms/vs/15/release/vs_buildtools.exe -IF "%VS_LATEST%" == "1" ( - set VS_INSTALL_ARGS= --nocache --norestart --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools - set VSDEVCMD_ARGS= -) ELSE ( - set VS_INSTALL_ARGS=--nocache --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools ^ - --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^ - --add Microsoft.Component.MSBuild ^ - --add Microsoft.VisualStudio.Component.Roslyn.Compiler ^ - --add Microsoft.VisualStudio.Component.TextTemplating ^ - --add Microsoft.VisualStudio.Component.VC.CoreIde ^ - --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest ^ - --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core ^ - --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^ - --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^ - --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 - set VSDEVCMD_ARGS=-vcvars_ver=14.11 -) - -curl -k -L %VS_DOWNLOAD_LINK% --output vs_installer.exe -if errorlevel 1 exit /b 1 - -start /wait .\vs_installer.exe %VS_INSTALL_ARGS% -if not errorlevel 0 exit /b 1 -if errorlevel 1 if not errorlevel 3010 exit /b 1 -if errorlevel 3011 exit /b 1 diff --git a/setup_and_test.sh b/setup_and_test.sh deleted file mode 100755 index d1c53a0b6..000000000 --- a/setup_and_test.sh +++ /dev/null @@ -1,47 +0,0 @@ -set -ex - -SOURCE_DIR=$(cd $(dirname $0) && pwd) - -# Written for circleci binary builds, to manage setting up the python env -# before calling run_tests.sh - -# Parameters -############################################################################# -if [[ -z "$PACKAGE_TYPE" || -z "$DESIRED_PYTHON" || -z "$PYVER_SHORT" || -z "$CUMAJMIN" || -z "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then - echo "The env variable PACKAGE_TYPE must be set to 'conda' or 'manywheel' or 'libtorch'" - echo "The env variable DESIRED_PYTHON must be set like 'cp27-cp27mu'" - echo "The env variable PYVER_SHORT must be set like '2.7'" - echo "The env variable CUMAJMIN must be set like 'cpu' or 'cu80' etc" - echo "The env variable PYTORCH_FINAL_PACKAGE_DIR must be set to a directory" - exit 1 -fi -py_nodot="$(echo $PYVER_SHORT | tr -d '.')" - -# Create Python env for testing -############################################################################# -if [[ "$PACKAGE_TYPE" == manywheel ]]; then - export PATH="/opt/python/$DESIRED_PYTHON/bin:$PATH" -elif [[ "$PACKAGE_TYPE" == conda ]]; then - source deactivate || true - conda create -yn "setuptest$py_nodot" python=$PYVER_SHORT - source activate "setuptest$py_nodot" -else - echo "This script does not handle $PACKAGE_TYPE packages" - exit 1 -fi -pip uninstall torch || true -pip uninstall torch || true - -# Install package -############################################################################# -package="$PYTORCH_FINAL_PACKAGE_DIR/$(ls $PYTORCH_FINAL_PACKAGE_DIR)" -pip uninstall "$package" || true -if [[ "$PACKAGE_TYPE" == manywheel ]]; then - pip install $package -else - conda install -y $package -fi - -# Test the package -############################################################################# -"$SOURCE_DIR/run_tests.sh" "$PACKAGE_TYPE" "$PYVER_SHORT" "$CUMAJMIN" diff --git a/test_community_repos/examples/.gitignore b/test_community_repos/examples/.gitignore deleted file mode 100644 index ea26ec23a..000000000 --- a/test_community_repos/examples/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.data diff --git a/test_community_repos/examples/dcgan/download-data.sh b/test_community_repos/examples/dcgan/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/dcgan/install-deps.sh b/test_community_repos/examples/dcgan/install-deps.sh deleted file mode 100755 index a4153384f..000000000 --- a/test_community_repos/examples/dcgan/install-deps.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -xe - -pip install subprocess32 - -pushd examples/dcgan -pip install -r requirements.txt -popd diff --git a/test_community_repos/examples/dcgan/run-script.sh b/test_community_repos/examples/dcgan/run-script.sh deleted file mode 100755 index 64c696f75..000000000 --- a/test_community_repos/examples/dcgan/run-script.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -xe - -CUDA_ARG=$1 - -pushd examples/dcgan -# smoke test -python main.py --dataset fake --dataroot . $CUDA_ARG --niter 100 -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/dcgan/run.sh b/test_community_repos/examples/dcgan/run.sh deleted file mode 100755 index 9f2e5a5fb..000000000 --- a/test_community_repos/examples/dcgan/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -xe - -if [ $CU_VERSION != 'cpu' ] -then - CUDA_ARG="--cuda" -fi - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh $CUDA_ARG -RETURN=$? -rm -rf examples -popd -exit $RETURN - diff --git a/test_community_repos/examples/fast_neural_style/download-data.sh b/test_community_repos/examples/fast_neural_style/download-data.sh deleted file mode 100755 index a199f77ca..000000000 --- a/test_community_repos/examples/fast_neural_style/download-data.sh +++ /dev/null @@ -1,7 +0,0 @@ -BASEDIR=$(dirname $0) -pushd $BASEDIR -curl -o cocotrain2014.zip http://images.cocodataset.org/zips/train2014.zip -# TODO: unzip isn't installed on the docker images, python is slow -python unzip.py cocotrain2014.zip -popd - diff --git a/test_community_repos/examples/fast_neural_style/install-deps.sh b/test_community_repos/examples/fast_neural_style/install-deps.sh deleted file mode 100755 index 50882b0d9..000000000 --- a/test_community_repos/examples/fast_neural_style/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -xe - -pip install -r requirements.txt \ No newline at end of file diff --git a/test_community_repos/examples/fast_neural_style/requirements.txt b/test_community_repos/examples/fast_neural_style/requirements.txt deleted file mode 100644 index bf54b654c..000000000 --- a/test_community_repos/examples/fast_neural_style/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy -torch -torchvision \ No newline at end of file diff --git a/test_community_repos/examples/fast_neural_style/run-script.sh b/test_community_repos/examples/fast_neural_style/run-script.sh deleted file mode 100755 index 93c44f0d9..000000000 --- a/test_community_repos/examples/fast_neural_style/run-script.sh +++ /dev/null @@ -1,5 +0,0 @@ -mkdir saved_models -python run.py -RETURN=$? -exit $RETURN - diff --git a/test_community_repos/examples/fast_neural_style/run.py b/test_community_repos/examples/fast_neural_style/run.py deleted file mode 100644 index 47aa1ead5..000000000 --- a/test_community_repos/examples/fast_neural_style/run.py +++ /dev/null @@ -1,67 +0,0 @@ -import re -import subprocess -import sys -import os - -PY3 = sys.version_info >= (3, 0) - - -def run(command, timeout): - """ - Returns (return-code, stdout, stderr) - """ - p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - output, err = p.communicate(timeout=timeout) - rc = p.returncode - if PY3: - output = output.decode("ascii") - err = err.decode("ascii") - return rc, output, err - - -# data lives in $BASEDIR/cocotrain2014/ -command_args = [ - 'python', - 'examples/fast_neural_style/neural_style/neural_style.py', - 'train', - '--dataset', - 'cocotrain2014', - '--style-image', - 'examples/fast_neural_style/images/style-images/mosaic.jpg', - '--save-model-dir', - './saved_models', - '--epochs', - '1', - '--image-size=128', - '--cuda', - '0' if os.environ.get("CU_VERSION") == 'cpu' else '1', -] - - -command = " ".join(command_args) - - -def main(): - # Test: run one epoch of fast neural style training. Warning: takes a while (half an hour?) - (rc, stdout, err) = subprocess.check_output(command, shell=True) - print("stdout:\n", stdout, "stderr:\n", err) - if rc is not 0: - sys.exit(rc) - - # Processes the output for losses - matches = re.findall('total: (\d+\.\d*)', stdout) - if len(matches) is 0: - print("error: unexpected output:", stdout) - sys.exit(1) - losses = [float(m) for m in matches] - - # Smoke test: assert losses are decreasing - prev = float('Inf') - for loss in losses: - if loss > prev: - print("error: non-decreasing loss:", losses) - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/test_community_repos/examples/fast_neural_style/run.sh b/test_community_repos/examples/fast_neural_style/run.sh deleted file mode 100755 index ff3e478a2..000000000 --- a/test_community_repos/examples/fast_neural_style/run.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN - diff --git a/test_community_repos/examples/fast_neural_style/unzip.py b/test_community_repos/examples/fast_neural_style/unzip.py deleted file mode 100644 index 99af59198..000000000 --- a/test_community_repos/examples/fast_neural_style/unzip.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python - -import zipfile -import re -import sys - - -def unzip(path): - """ - Unzips /path/to/some.zip to ./some - Doesn't work with - or _ in 'some' - """ - match = re.search("(\w+)\.zip", path) - if match is None: - print("Could not parse path") - return - - dest = match.group(1) - with zipfile.ZipFile(path) as z: - z.extractall(dest) - - -if __name__ == '__main__': - unzip(sys.argv[1]) - diff --git a/test_community_repos/examples/imagenet/download-data.sh b/test_community_repos/examples/imagenet/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/imagenet/install-deps.sh b/test_community_repos/examples/imagenet/install-deps.sh deleted file mode 100755 index 7e06421d4..000000000 --- a/test_community_repos/examples/imagenet/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -pushd examples/imagenet -pip install -r requirements.txt -popd diff --git a/test_community_repos/examples/imagenet/run-script.sh b/test_community_repos/examples/imagenet/run-script.sh deleted file mode 100755 index 8c59f104d..000000000 --- a/test_community_repos/examples/imagenet/run-script.sh +++ /dev/null @@ -1,11 +0,0 @@ -pushd examples/imagenet -if [ -z "$IMAGENET_ROOT" ]; then - popd - echo "IMAGENET_ROOT not set" - exit 1 -fi -python main.py -a resnet18 $IMAGENET_ROOT --epochs 1 -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/imagenet/run.sh b/test_community_repos/examples/imagenet/run.sh deleted file mode 100755 index 6730ac02c..000000000 --- a/test_community_repos/examples/imagenet/run.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN diff --git a/test_community_repos/examples/install-deps.sh b/test_community_repos/examples/install-deps.sh deleted file mode 100755 index 6b7df0209..000000000 --- a/test_community_repos/examples/install-deps.sh +++ /dev/null @@ -1,2 +0,0 @@ -git clone https://github.com/pytorch/examples.git -pushd examples diff --git a/test_community_repos/examples/mnist/download-data.sh b/test_community_repos/examples/mnist/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/mnist/install-deps.sh b/test_community_repos/examples/mnist/install-deps.sh deleted file mode 100755 index 547b1a4df..000000000 --- a/test_community_repos/examples/mnist/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -pushd examples/mnist -pip install -r requirements.txt -popd diff --git a/test_community_repos/examples/mnist/run-script.sh b/test_community_repos/examples/mnist/run-script.sh deleted file mode 100755 index bd60874b9..000000000 --- a/test_community_repos/examples/mnist/run-script.sh +++ /dev/null @@ -1,7 +0,0 @@ -pushd examples/mnist -# smoke test -python main.py --epochs 1 -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/mnist/run.sh b/test_community_repos/examples/mnist/run.sh deleted file mode 100755 index ff3e478a2..000000000 --- a/test_community_repos/examples/mnist/run.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN - diff --git a/test_community_repos/examples/mnist_hogwild/download-data.sh b/test_community_repos/examples/mnist_hogwild/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/mnist_hogwild/install-deps.sh b/test_community_repos/examples/mnist_hogwild/install-deps.sh deleted file mode 100755 index 16a60daef..000000000 --- a/test_community_repos/examples/mnist_hogwild/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -pushd examples/mnist_hogwild -pip install -r requirements.txt -popd diff --git a/test_community_repos/examples/mnist_hogwild/run-script.sh b/test_community_repos/examples/mnist_hogwild/run-script.sh deleted file mode 100755 index 6a52def0e..000000000 --- a/test_community_repos/examples/mnist_hogwild/run-script.sh +++ /dev/null @@ -1,7 +0,0 @@ -pushd examples/mnist_hogwild -# smoke tests -python main.py --epochs 1 -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/mnist_hogwild/run.sh b/test_community_repos/examples/mnist_hogwild/run.sh deleted file mode 100755 index ff3e478a2..000000000 --- a/test_community_repos/examples/mnist_hogwild/run.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN - diff --git a/test_community_repos/examples/regression/download-data.sh b/test_community_repos/examples/regression/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/regression/install-deps.sh b/test_community_repos/examples/regression/install-deps.sh deleted file mode 100755 index 50882b0d9..000000000 --- a/test_community_repos/examples/regression/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -xe - -pip install -r requirements.txt \ No newline at end of file diff --git a/test_community_repos/examples/regression/requirements.txt b/test_community_repos/examples/regression/requirements.txt deleted file mode 100644 index 12c6d5d5e..000000000 --- a/test_community_repos/examples/regression/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -torch diff --git a/test_community_repos/examples/regression/run-script.sh b/test_community_repos/examples/regression/run-script.sh deleted file mode 100755 index 221a72399..000000000 --- a/test_community_repos/examples/regression/run-script.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -xe - -pushd examples/regression -python main.py -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/regression/run.sh b/test_community_repos/examples/regression/run.sh deleted file mode 100755 index ff3e478a2..000000000 --- a/test_community_repos/examples/regression/run.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN - diff --git a/test_community_repos/examples/reinforcement_learning/download-data.sh b/test_community_repos/examples/reinforcement_learning/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/reinforcement_learning/install-deps.sh b/test_community_repos/examples/reinforcement_learning/install-deps.sh deleted file mode 100755 index 0edbe46b1..000000000 --- a/test_community_repos/examples/reinforcement_learning/install-deps.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -xe - -pip install subprocess32 - -pushd examples/reinforcement_learning -pip install -r requirements.txt -popd diff --git a/test_community_repos/examples/reinforcement_learning/run-script.sh b/test_community_repos/examples/reinforcement_learning/run-script.sh deleted file mode 100755 index 861e9b357..000000000 --- a/test_community_repos/examples/reinforcement_learning/run-script.sh +++ /dev/null @@ -1,4 +0,0 @@ -python run.py -RETURN_CODE=$? -exit $RETURN_CODE - diff --git a/test_community_repos/examples/reinforcement_learning/run.py b/test_community_repos/examples/reinforcement_learning/run.py deleted file mode 100644 index 2c140b79d..000000000 --- a/test_community_repos/examples/reinforcement_learning/run.py +++ /dev/null @@ -1,60 +0,0 @@ -import re -import subprocess32 -import sys - -PY3 = sys.version_info >= (3, 0) - -reinforce_cmd = 'python examples/reinforcement_learning/reinforce.py' -actor_critic_cmd = 'python examples/reinforcement_learning/actor_critic.py' - - -def run(command, timeout): - """ - Returns (return-code, stdout, stderr) - """ - p = subprocess32.Popen(command, stdout=subprocess32.PIPE, stderr=subprocess32.PIPE, shell=True) - output, err = p.communicate(timeout=timeout) - rc = p.returncode - if PY3: - output = output.decode("ascii") - err = err.decode("ascii") - return (rc, output, err) - - -def check_cartpole_example(command, seconds=30, baseline_iter=1000): - """ - Runs command. Checks that: - 1. the command exits within a timeout - 2. cartpole is solved - 3. the number of iters it takes to solve cartpole is less than baseline_iter - """ - (rc, stdout, stderr) = run(command, timeout=seconds) - print("stdout:\n", stdout) - print("stderr:\n", stderr) - if rc is not 0: - sys.exit(rc) - - # Reinforce should have solved cartpole - matches = re.search('Solved!', stdout) - if matches is None: - print("error: reinforce didn't solve cartpole") - sys.exit(1) - - matches = re.findall('Episode (\d+)', stdout) - if len(matches) is 0: - print("error: unexpected output: ", stdout) - sys.exit(1) - - losses = [int(m) for m in matches] - - if losses[-1] > baseline_iter: - print("error: too many iterations taken: {}".format(losses[-1])) - sys.exit(1) - - -if __name__ == '__main__': - check_cartpole_example(actor_critic_cmd, seconds=5*60, baseline_iter=4000) - - # NOTE: Times out after 60 seconds; changed to 3 minutes - check_cartpole_example(reinforce_cmd, seconds=3*60, baseline_iter=4000) - diff --git a/test_community_repos/examples/reinforcement_learning/run.sh b/test_community_repos/examples/reinforcement_learning/run.sh deleted file mode 100755 index 6730ac02c..000000000 --- a/test_community_repos/examples/reinforcement_learning/run.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN diff --git a/test_community_repos/examples/run.sh b/test_community_repos/examples/run.sh deleted file mode 100755 index dc5a4ba81..000000000 --- a/test_community_repos/examples/run.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR - -for file in */ ; do - echo "Testing $file"; - for script in $file/run.sh ; do - $script - done - echo "Test passed $file"; -done - -popd diff --git a/test_community_repos/examples/snli/download-data.sh b/test_community_repos/examples/snli/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/snli/install-deps.sh b/test_community_repos/examples/snli/install-deps.sh deleted file mode 100755 index b034249ae..000000000 --- a/test_community_repos/examples/snli/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -pushd examples/snli -pip install -r requirements.txt -popd diff --git a/test_community_repos/examples/snli/run-script.sh b/test_community_repos/examples/snli/run-script.sh deleted file mode 100755 index 0bf1296c0..000000000 --- a/test_community_repos/examples/snli/run-script.sh +++ /dev/null @@ -1,6 +0,0 @@ -pushd examples/snli -python train.py --epochs 1 -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/snli/run.sh b/test_community_repos/examples/snli/run.sh deleted file mode 100755 index 6730ac02c..000000000 --- a/test_community_repos/examples/snli/run.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN diff --git a/test_community_repos/examples/super_resolution/download-data.sh b/test_community_repos/examples/super_resolution/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/super_resolution/install-deps.sh b/test_community_repos/examples/super_resolution/install-deps.sh deleted file mode 100755 index 50882b0d9..000000000 --- a/test_community_repos/examples/super_resolution/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -xe - -pip install -r requirements.txt \ No newline at end of file diff --git a/test_community_repos/examples/super_resolution/requirements.txt b/test_community_repos/examples/super_resolution/requirements.txt deleted file mode 100644 index 37f700a78..000000000 --- a/test_community_repos/examples/super_resolution/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch -torchvision \ No newline at end of file diff --git a/test_community_repos/examples/super_resolution/run-script.sh b/test_community_repos/examples/super_resolution/run-script.sh deleted file mode 100755 index 8d676947a..000000000 --- a/test_community_repos/examples/super_resolution/run-script.sh +++ /dev/null @@ -1,6 +0,0 @@ -pushd examples/super_resolution -python main.py --upscale_factor 3 --batchSize 4 --testBatchSize 100 --nEpochs 1 --lr 0.001 -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/super_resolution/run.sh b/test_community_repos/examples/super_resolution/run.sh deleted file mode 100755 index ff3e478a2..000000000 --- a/test_community_repos/examples/super_resolution/run.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN - diff --git a/test_community_repos/examples/time_sequence_prediction/download-data.sh b/test_community_repos/examples/time_sequence_prediction/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/time_sequence_prediction/install-deps.sh b/test_community_repos/examples/time_sequence_prediction/install-deps.sh deleted file mode 100755 index b91f41641..000000000 --- a/test_community_repos/examples/time_sequence_prediction/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ - -# TODO don't install torch here -yes | pip install matplotlib torch diff --git a/test_community_repos/examples/time_sequence_prediction/run-script.sh b/test_community_repos/examples/time_sequence_prediction/run-script.sh deleted file mode 100755 index 59ecbbbe3..000000000 --- a/test_community_repos/examples/time_sequence_prediction/run-script.sh +++ /dev/null @@ -1,7 +0,0 @@ -pushd examples/time_sequence_prediction -python generate_sine_wave.py -python train.py -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/time_sequence_prediction/run.sh b/test_community_repos/examples/time_sequence_prediction/run.sh deleted file mode 100755 index ff3e478a2..000000000 --- a/test_community_repos/examples/time_sequence_prediction/run.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN - diff --git a/test_community_repos/examples/vae/download-data.sh b/test_community_repos/examples/vae/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/vae/install-deps.sh b/test_community_repos/examples/vae/install-deps.sh deleted file mode 100755 index 130b374e9..000000000 --- a/test_community_repos/examples/vae/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -pushd examples/vae -pip install -r requirements.txt -popd diff --git a/test_community_repos/examples/vae/run-script.sh b/test_community_repos/examples/vae/run-script.sh deleted file mode 100755 index 2a3b315ef..000000000 --- a/test_community_repos/examples/vae/run-script.sh +++ /dev/null @@ -1,6 +0,0 @@ -pushd examples/vae -python main.py --epochs 1 -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/vae/run.sh b/test_community_repos/examples/vae/run.sh deleted file mode 100755 index ff3e478a2..000000000 --- a/test_community_repos/examples/vae/run.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh -RETURN=$? -rm -rf examples -popd -exit $RETURN - diff --git a/test_community_repos/examples/word_language_model/download-data.sh b/test_community_repos/examples/word_language_model/download-data.sh deleted file mode 100755 index e69de29bb..000000000 diff --git a/test_community_repos/examples/word_language_model/install-deps.sh b/test_community_repos/examples/word_language_model/install-deps.sh deleted file mode 100755 index a79a550ba..000000000 --- a/test_community_repos/examples/word_language_model/install-deps.sh +++ /dev/null @@ -1,3 +0,0 @@ -pushd examples/word_language_model -pip install -r requirements.txt -popd diff --git a/test_community_repos/examples/word_language_model/run-script.sh b/test_community_repos/examples/word_language_model/run-script.sh deleted file mode 100755 index c28192bbc..000000000 --- a/test_community_repos/examples/word_language_model/run-script.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -xe - -CUDA_ARG=$1 - -pushd examples/word_language_model -# smoke tests -python main.py $CUDA_ARG --epochs 1 -python main.py $CUDA_ARG --epochs 1 --tied -RETURN_CODE=$? -popd -exit $RETURN_CODE - diff --git a/test_community_repos/examples/word_language_model/run.sh b/test_community_repos/examples/word_language_model/run.sh deleted file mode 100755 index 9f2e5a5fb..000000000 --- a/test_community_repos/examples/word_language_model/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -xe - -if [ $CU_VERSION != 'cpu' ] -then - CUDA_ARG="--cuda" -fi - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/pytorch/examples.git -./download-data.sh -./install-deps.sh -./run-script.sh $CUDA_ARG -RETURN=$? -rm -rf examples -popd -exit $RETURN - diff --git a/test_community_repos/external_projects/OpenNMT/install-deps.sh b/test_community_repos/external_projects/OpenNMT/install-deps.sh deleted file mode 100755 index 0e812b083..000000000 --- a/test_community_repos/external_projects/OpenNMT/install-deps.sh +++ /dev/null @@ -1,2 +0,0 @@ -pip install -r requirements.txt --ignore-installed - diff --git a/test_community_repos/external_projects/OpenNMT/run-script.sh b/test_community_repos/external_projects/OpenNMT/run-script.sh deleted file mode 100755 index 50147a0ca..000000000 --- a/test_community_repos/external_projects/OpenNMT/run-script.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -xe - - -if [ $CU_VERSION != 'cpu' ] -then - GPU_ARGS="-world_size 1 -gpu_ranks 0" -fi - - -python preprocess.py -train_src data/src-train.txt -train_tgt data/tgt-train.txt -valid_src data/src-val.txt -valid_tgt data/tgt-val.txt -save_data data/demo - -python train.py -data data/demo -save_model demo-model -train_steps 1 $GPU_ARGS - -python translate.py -model demo-model_*.pt -src data/src-test.txt -output pred.txt -replace_unk -verbose -gpu 0 -max_length 10 - diff --git a/test_community_repos/external_projects/OpenNMT/run.sh b/test_community_repos/external_projects/OpenNMT/run.sh deleted file mode 100755 index 5738728cb..000000000 --- a/test_community_repos/external_projects/OpenNMT/run.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR - -git clone https://github.com/OpenNMT/OpenNMT-py.git -pushd OpenNMT-py -../install-deps.sh -../run-script.sh -popd -rm -rf OpenNMT-py - -popd - diff --git a/test_community_repos/external_projects/advertorch/run.sh b/test_community_repos/external_projects/advertorch/run.sh deleted file mode 100755 index 8f8238a4b..000000000 --- a/test_community_repos/external_projects/advertorch/run.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -xe - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/BorealisAI/advertorch.git -pushd advertorch - -python setup.py install -python advertorch_examples/tutorial_train_mnist.py - -popd -rm -rf advertorch -popd - diff --git a/test_community_repos/external_projects/allennlp/run.sh b/test_community_repos/external_projects/allennlp/run.sh deleted file mode 100755 index 45295f1cc..000000000 --- a/test_community_repos/external_projects/allennlp/run.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -xe - -yes | pip install pytest-xdist -# pip install thinc breaks depending on gcc version -yes | conda install thinc - -TMPDIR=$RANDOM -mkdir /tmp/$TMPDIR -git clone https://github.com/allenai/allennlp /tmp/$TMPDIR -pushd /tmp/$TMPDIR - -pip install -r requirements.txt - -pytest -v -popd -rm -rf /tmp/$TMPDIR - diff --git a/test_community_repos/external_projects/block/run.sh b/test_community_repos/external_projects/block/run.sh deleted file mode 100755 index ac0eb797d..000000000 --- a/test_community_repos/external_projects/block/run.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -xe - -yes | pip install block nose scipy torch - - -TMPDIR=$RANDOM -mkdir /tmp/$TMPDIR -git clone https://github.com/bamos/block /tmp/$TMPDIR -pushd /tmp/$TMPDIR -nosetests test.py -popd -rm -rf /tmp/$TMPDIR diff --git a/test_community_repos/external_projects/botorch/run.sh b/test_community_repos/external_projects/botorch/run.sh deleted file mode 100755 index 5a4834f3d..000000000 --- a/test_community_repos/external_projects/botorch/run.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -xe - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/pytorch/botorch.git -pushd botorch - - -# ??? -#pip install -e .[dev] - -python setup.py test - - -popd -rm -rf botorch -popd - diff --git a/test_community_repos/external_projects/cyclegan/download_data.sh b/test_community_repos/external_projects/cyclegan/download_data.sh deleted file mode 100755 index 1bacef7a6..000000000 --- a/test_community_repos/external_projects/cyclegan/download_data.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -xe - -bash ./datasets/download_cyclegan_dataset.sh maps -bash ./datasets/download_pix2pix_dataset.sh facades - diff --git a/test_community_repos/external_projects/cyclegan/install-deps.sh b/test_community_repos/external_projects/cyclegan/install-deps.sh deleted file mode 100755 index 6e3dc97ac..000000000 --- a/test_community_repos/external_projects/cyclegan/install-deps.sh +++ /dev/null @@ -1,5 +0,0 @@ -pip install visdom -pip install dominate - -pip install torchvision - diff --git a/test_community_repos/external_projects/cyclegan/run-script.sh b/test_community_repos/external_projects/cyclegan/run-script.sh deleted file mode 100755 index f7e4cfb78..000000000 --- a/test_community_repos/external_projects/cyclegan/run-script.sh +++ /dev/null @@ -1,13 +0,0 @@ -set -e - -if [ $CU_VERSION == 'cpu' ] -then - FORCE_CPU_ARG="--gpu_ids -1" -fi - - -python -m visdom.server & -trap 'kill $(jobs -p)' EXIT -python train.py $FORCE_CPU_ARG --dataroot ./datasets/maps --name maps_cyclegan --model cycle_gan --no_dropout --niter 1 --niter_decay 0 --no_html --max_dataset_size 400 --batch_size 1 --print_freq 500 --display_freq 500 -python train.py $FORCE_CPU_ARG --dataroot ./datasets/facades --name facades_pix2pix --model pix2pix --which_model_netG unet_256 --which_direction BtoA --lambda_A 100 --dataset_mode aligned --no_lsgan --norm batch --pool_size 0 --niter 1 --niter_decay 0 --no_html --max_dataset_size 400 --batch_size 1 --print_freq 500 --display_freq 500 - diff --git a/test_community_repos/external_projects/cyclegan/run.sh b/test_community_repos/external_projects/cyclegan/run.sh deleted file mode 100755 index ed87a7950..000000000 --- a/test_community_repos/external_projects/cyclegan/run.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -xe - -# Hack to obtain "unzip" command -UNZIP_BINARY_PATH=$HOME/bin - -mkdir $UNZIP_BINARY_PATH -ln -s ~/project/test_community_repos/examples/fast_neural_style/unzip.py $UNZIP_BINARY_PATH/unzip -chmod +x $UNZIP_BINARY_PATH/unzip -export PATH=$PATH:$UNZIP_BINARY_PATH - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix.git -pushd pytorch-CycleGAN-and-pix2pix -../download_data.sh -../install-deps.sh -../run-script.sh -popd -rm -rf pytorch-CycleGAN-and-pix2pix -popd - -# Clean up hack -rm -r $UNZIP_BINARY_PATH diff --git a/test_community_repos/external_projects/fairseq/download_data.sh b/test_community_repos/external_projects/fairseq/download_data.sh deleted file mode 100755 index 5173374f2..000000000 --- a/test_community_repos/external_projects/fairseq/download_data.sh +++ /dev/null @@ -1,4 +0,0 @@ -pushd examples/translation -bash prepare-iwslt14.sh -popd - diff --git a/test_community_repos/external_projects/fairseq/install-deps.sh b/test_community_repos/external_projects/fairseq/install-deps.sh deleted file mode 100755 index b5d67c406..000000000 --- a/test_community_repos/external_projects/fairseq/install-deps.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -xe - -#pip install -r requirements.txt -python setup.py build - - -if [[ "$(uname)" != Darwin ]]; then - python setup.py develop - -else - # Potential workaround for probem on macos: - # fastBPE/fastBPE.hpp:17:10: fatal error: 'thread' file not found - # See: https://github.com/facebookresearch/XLM/issues/105 - - CFLAGS='-stdlib=libc++' python setup.py develop -fi - diff --git a/test_community_repos/external_projects/fairseq/run-script.sh b/test_community_repos/external_projects/fairseq/run-script.sh deleted file mode 100755 index 34415ffe6..000000000 --- a/test_community_repos/external_projects/fairseq/run-script.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -xe - -TEXT=examples/translation/iwslt14.tokenized.de-en -python preprocess.py --source-lang de --target-lang en \ - --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \ - --destdir data-bin/iwslt14.tokenized.de-en - -mkdir -p checkpoints/fconv -CUDA_VISIBLE_DEVICES=0 python train.py data-bin/iwslt14.tokenized.de-en \ - --lr 0.25 --clip-norm 0.1 --dropout 0.2 --max-tokens 4000 --max-epoch 1 \ - --arch fconv_iwslt_de_en --save-dir checkpoints/fconv - -python generate.py data-bin/iwslt14.tokenized.de-en \ - --path checkpoints/fconv/checkpoint_best.pt \ - --batch-size 128 --beam 5 - diff --git a/test_community_repos/external_projects/fairseq/run.sh b/test_community_repos/external_projects/fairseq/run.sh deleted file mode 100755 index 7fffbc48e..000000000 --- a/test_community_repos/external_projects/fairseq/run.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -xe - - -BASEDIR=$(dirname $0) -pushd $BASEDIR -git clone https://github.com/facebookresearch/fairseq-py.git -pushd fairseq-py -../download_data.sh -../install-deps.sh -../run-script.sh -popd -rm -rf fairseq-py -popd - diff --git a/test_community_repos/external_projects/fastai/run.sh b/test_community_repos/external_projects/fastai/run.sh deleted file mode 100755 index 2ed864427..000000000 --- a/test_community_repos/external_projects/fastai/run.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -xe - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/fastai/fastai.git -pushd fastai - -# See https://github.com/fastai/fastai#developer-install -tools/run-after-git-clone -pip install -e ".[dev]" - - - -pytest tests - - -popd -rm -rf fastai -popd - diff --git a/test_community_repos/external_projects/flair/run.sh b/test_community_repos/external_projects/flair/run.sh deleted file mode 100755 index d17dacd94..000000000 --- a/test_community_repos/external_projects/flair/run.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -xe - -yes | pip install pytest-xdist pipenv - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/zalandoresearch/flair.git -pushd flair - - -# Testing guidance taken from here: https://github.com/zalandoresearch/flair#running-unit-tests-locally -pipenv install --dev && pipenv shell -pytest --runintegration tests/ - - -popd -rm -rf flair -popd - diff --git a/test_community_repos/external_projects/geometric/run.sh b/test_community_repos/external_projects/geometric/run.sh deleted file mode 100755 index 2f935125a..000000000 --- a/test_community_repos/external_projects/geometric/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -xe - -# Install dependencies (see https://github.com/rusty1s/pytorch_geometric#installation) -pip install --verbose --no-cache-dir torch-scatter -pip install --verbose --no-cache-dir torch-sparse -pip install --verbose --no-cache-dir torch-cluster -pip install --verbose --no-cache-dir torch-spline-conv - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/rusty1s/pytorch_geometric.git -pushd pytorch_geometric - - -python setup.py test - - -popd -rm -rf pytorch_geometric -popd - diff --git a/test_community_repos/external_projects/gpytorch/run.sh b/test_community_repos/external_projects/gpytorch/run.sh deleted file mode 100755 index 7d9501d1d..000000000 --- a/test_community_repos/external_projects/gpytorch/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -xe - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/cornellius-gp/gpytorch.git -pushd gpytorch - - -python -m unittest - - -popd -rm -rf gpytorch -popd - diff --git a/test_community_repos/external_projects/ignite/run.sh b/test_community_repos/external_projects/ignite/run.sh deleted file mode 100755 index 8a512d55e..000000000 --- a/test_community_repos/external_projects/ignite/run.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -xe - -yes | pip install pytest sklearn mock - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/pytorch/ignite.git -pushd ignite - - -python setup.py test - - -popd -rm -rf ignite -popd - diff --git a/test_community_repos/external_projects/lightning/run.sh b/test_community_repos/external_projects/lightning/run.sh deleted file mode 100755 index c9876fe62..000000000 --- a/test_community_repos/external_projects/lightning/run.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -xe - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -# Testing instructions taken from here: -# https://github.com/williamFalcon/pytorch-lightning/tree/master/tests#pytorch-lightning-tests -git clone https://github.com/williamFalcon/pytorch-lightning.git -pushd pytorch-lightning - -# install module locally -pip install -e . - -# install dev deps -pip install -r requirements.txt - -# run tests -py.test -v - -popd -rm -rf pytorch-lightning -popd - diff --git a/test_community_repos/external_projects/parlai/run.sh b/test_community_repos/external_projects/parlai/run.sh deleted file mode 100755 index 5a616ff67..000000000 --- a/test_community_repos/external_projects/parlai/run.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -xe - -yes | pip install torchtext - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/facebookresearch/ParlAI.git -pushd ParlAI - - -python setup.py develop -#python examples/display_data.py -t squad -python examples/eval_model.py -m ir_baseline -t personachat -dt valid -python examples/train_model.py -t personachat -m transformer/ranker -mf /tmp/model_tr6 --n-layers 1 --embedding-size 300 --ffn-size 600 --n-heads 4 --num-epochs 2 -veps 0.25 -bs 64 -lr 0.001 --dropout 0.1 --embedding-type fasttext_cc --candidates batch - - -popd -rm -rf ParlAI -popd - diff --git a/test_community_repos/external_projects/pennylane/run.sh b/test_community_repos/external_projects/pennylane/run.sh deleted file mode 100755 index 1e3f22be7..000000000 --- a/test_community_repos/external_projects/pennylane/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -xe - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/XanaduAI/pennylane.git -pushd pennylane - - -python setup.py test - - -popd -rm -rf pennylane -popd - diff --git a/test_community_repos/external_projects/pix2pixHD/run.sh b/test_community_repos/external_projects/pix2pixHD/run.sh deleted file mode 100755 index 017e524c9..000000000 --- a/test_community_repos/external_projects/pix2pixHD/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -xe - - -yes | pip install dominate - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/NVIDIA/pix2pixHD.git -pushd pix2pixHD - - -# Testing guidance obtained from here: https://github.com/NVIDIA/pix2pixHD#testing - -# Download dataset -# TODO: Hosted on Google Drive as latest_net_G.pth; need somewhere programmatically accessible - -bash ./scripts/test_1024p.sh - - -popd -rm -rf pix2pixHD -popd - diff --git a/test_community_repos/external_projects/pyro/run.sh b/test_community_repos/external_projects/pyro/run.sh deleted file mode 100755 index d62c1c76b..000000000 --- a/test_community_repos/external_projects/pyro/run.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -xe - -yes | pip install pytest-xdist -yes | pip install sphinx -yes | pip install sphinx_rtd_theme -yes | pip install flake8 - -TMPDIR=$RANDOM -mkdir /tmp/$TMPDIR -git clone https://github.com/uber/pyro /tmp/$TMPDIR -pushd /tmp/$TMPDIR -yes | pip install . -make test -make test-cuda -make integration-test -make test-examples -make test-tutorials -popd -rm -rf /tmp/$TMPDIR - diff --git a/test_community_repos/external_projects/pysyft/run.sh b/test_community_repos/external_projects/pysyft/run.sh deleted file mode 100755 index cb16a9fbb..000000000 --- a/test_community_repos/external_projects/pysyft/run.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -xe - -yes | pip install pytest tblib websocket websockets lz4 msgpack zstd scipy torch - - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/OpenMined/PySyft.git -pushd PySyft - -pytest test/torch - - -popd -rm -rf PySyft -popd - diff --git a/test_community_repos/external_projects/skorch/run.sh b/test_community_repos/external_projects/skorch/run.sh deleted file mode 100755 index 8c9f8ee72..000000000 --- a/test_community_repos/external_projects/skorch/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR - - -git clone https://github.com/skorch-dev/skorch.git -pushd skorch - - -python setup.py develop -py.test - - -popd -rm -rf skorch -popd - diff --git a/test_community_repos/external_projects/tensorly/run.sh b/test_community_repos/external_projects/tensorly/run.sh deleted file mode 100755 index 7d8226e25..000000000 --- a/test_community_repos/external_projects/tensorly/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -xe - -yes | pip install git+https://github.com/tensorly/tensorly -yes | pip install pytest-xdist -yes | pip install nose - -yes | pip install torch - -TMPDIR=$RANDOM -mkdir /tmp/$TMPDIR -git clone https://github.com/tensorly/tensorly /tmp/$TMPDIR -pushd /tmp/$TMPDIR -TENSORLY_BACKEND='pytorch' pytest -v tensorly -popd -rm -rf /tmp/$TMPDIR - diff --git a/test_community_repos/run_all.py b/test_community_repos/run_all.py deleted file mode 100755 index e62162477..000000000 --- a/test_community_repos/run_all.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 - -import os.path -import unittest -import subprocess -import sys -import os - - -TIMEOUT = 2 * 60 * 60 # 2 hours - - -def run(command, timeout=None): - """ - Returns (return-code, stdout, stderr) - """ - completed = subprocess.run(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=True, - encoding="utf8", timeout=timeout) - - return completed.returncode, completed.stdout, completed.stderr - - -class TestRepos(unittest.TestCase): - pass - - -def _test(cls, directory): - command = os.path.join(directory, "run.sh") - (rc, out, err) = run(command, TIMEOUT) - cls.assertEqual(rc, 0, "Ran {}\nstdout:\n{}\nstderr:\n{}".format( - command, out, err)) - - -def generate_test_objects(target_directory): - """ - Generate the tests, one for each repo - """ - repos = sorted([os.path.normpath(os.path.join(target_directory, o)) for o in os.listdir(target_directory) if os.path.isdir(os.path.join(target_directory, o))]) - for f in repos: - print("found {}".format(f)) - setattr(TestRepos, "test_" + f, lambda cls, f=f: _test(cls, f)) - - -if __name__ == '__main__': - generate_test_objects('examples') - unittest.main() diff --git a/test_community_repos/run_all.sh b/test_community_repos/run_all.sh deleted file mode 100755 index cf16b3747..000000000 --- a/test_community_repos/run_all.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR - -./run_all.py - -popd diff --git a/test_community_repos/tutorials/run.py b/test_community_repos/tutorials/run.py deleted file mode 100644 index 8b18ee371..000000000 --- a/test_community_repos/tutorials/run.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import subprocess -import sys -import argparse - -PY3 = sys.version_info >= (3, 0) - - -blacklist = [ - "./advanced_source/super_resolution_with_caffe2.py", - # The docker image's python has some trouble with decoding unicode - "./intermediate_source/char_rnn_classification_tutorial.py", -] -visual = [ - "./advanced_source/neural_style_tutorial.py", - "./beginner_source/blitz/cifar10_tutorial.py", - "./beginner_source/data_loading_tutorial.py", - "./beginner_source/transfer_learning_tutorial.py", - "./intermediate_source/char_rnn_generation_tutorial.py", - "./intermediate_source/reinforcement_q_learning.py", - "./intermediate_source/seq2seq_translation_tutorial.py", - "./intermediate_source/spatial_transformer_tutorial.py", -] - - -def run(command): - """ - Returns (return-code, stdout, stderr) - """ - p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - output, err = p.communicate() - rc = p.returncode - if PY3: - output = output.decode("ascii") - err = err.decode("ascii") - return (rc, output, err) - - -def main(): - parser = argparse.ArgumentParser( - description="Run all pytorch tutorials") - parser.add_argument('--visual', dest='visual', - action='store_true', - default=False, - help='Run the tutorials that rely on a GUI. Default: False') - parser.add_argument('--py', dest='python', - action='store', - default='python', - help='the python binary. Default: python') - parser.add_argument('--all', dest='all', - action='store_true', - default=False, - help='Run all tutorials, include visual and blacklisted ones.') - - args = parser.parse_args() - run_visual = args.visual - - (rc, stdout, stderr) = run("find . -type f | grep -P 'source.+py$'") - if rc is not 0: - print("Couldn't execute find") - exit(1) - - files = stdout.split('\n') - files = [f for f in files if len(f) > 0] - failed = [] - warns = [] - - python = args.python - - for f in files: - if not args.all and f in blacklist: - print("skipping {}".format(f)) - continue - if not args.all and not run_visual and f in visual: - print("skipping {} b/c --visual was not set".format(f)) - continue - - (rc, out, err) = run("{} {}".format(python, f)) - fail_msg = "" - if rc is not 0: - failed.append((rc, out, err, f)) - fail_msg = " [FAILED]" - if rc is 0 and len(err) is not 0: - warns.append((rc, out, err, f)) - fail_msg = " [WARNINGS]" - print("testing {}{}".format(f, fail_msg)) - - if len(failed) is 0 and len(warns) is 0: - print("All tutorials ran successfully") - exit(0) - - for (rc, out, err, f) in warns: - print("-" * 50) - print("[WARNINGS] {} {} had warnings:".format(python, f)) - print("return code: {}\nstdout:\n{}\nstderr:\n{}\n".format( - rc, out, err)) - - if len(failed) is 0: - exit(0) - - for (rc, out, err, f) in failed: - print("-" * 50) - print("[FAILED] {} {} failed with the following:".format(python, f)) - print("return code: {}\nstdout:\n{}\nstderr:\n{}\n".format( - rc, out, err)) - - exit(1) - - -if __name__ == '__main__': - main() diff --git a/test_community_repos/tutorials/run.sh b/test_community_repos/tutorials/run.sh deleted file mode 100755 index fe3bd6df8..000000000 --- a/test_community_repos/tutorials/run.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -xe - -BASEDIR=$(dirname $0) -pushd $BASEDIR - -yes | pip install matplotlib gym pandas tensorflow scikit-image - -# The docker image doesn't come with these prebuilt -if [ -f /.dockerenv ]; then - apt-get -qq update - apt-get -qq -y install unzip - apt-get -qq -y install wget -fi - -git clone https://github.com/pytorch/tutorials.git - -pushd tutorials - -set -x -e - -# for seq2seq_translation_tutorial -wget https://download.pytorch.org/tutorial/data.zip -yes | unzip data.zip - -# The docker image can't display images -python ../run.py - -popd -popd diff --git a/test_imports_docker/import-cv2-pytorch.sh b/test_imports_docker/import-cv2-pytorch.sh deleted file mode 100755 index 0f6c0ee49..000000000 --- a/test_imports_docker/import-cv2-pytorch.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -python -c "import cv2; import torch; print(torch.__version__); print(cv2.__version__)" diff --git a/test_imports_docker/import-pytorch-cv2.sh b/test_imports_docker/import-pytorch-cv2.sh deleted file mode 100755 index 9cd9e7562..000000000 --- a/test_imports_docker/import-pytorch-cv2.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -python -c "import torch; import cv2; print(torch.__version__); print(cv2.__version__)" diff --git a/test_imports_docker/import-pytorch-scipy.sh b/test_imports_docker/import-pytorch-scipy.sh deleted file mode 100755 index d6d8c0d41..000000000 --- a/test_imports_docker/import-pytorch-scipy.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -python -c "import torch; import scipy; print(torch.__version__); print(scipy.__version__)" diff --git a/test_imports_docker/import-pytorch-tf.sh b/test_imports_docker/import-pytorch-tf.sh deleted file mode 100755 index 6f5508524..000000000 --- a/test_imports_docker/import-pytorch-tf.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -python -c "import torch; import tensorflow; print(torch.__version__); print(tensorflow.__version__)" diff --git a/test_imports_docker/import-scipy-pytorch.sh b/test_imports_docker/import-scipy-pytorch.sh deleted file mode 100755 index 014424a1b..000000000 --- a/test_imports_docker/import-scipy-pytorch.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -python -c "import scipy; import torch; print(torch.__version__); print(scipy.__version__)" diff --git a/test_imports_docker/import-tf-pytorch.sh b/test_imports_docker/import-tf-pytorch.sh deleted file mode 100755 index 06d024cd0..000000000 --- a/test_imports_docker/import-tf-pytorch.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -python -c "import tensorflow; import torch; print(torch.__version__); print(tensorflow.__version__)" diff --git a/test_imports_docker/import-torch.sh b/test_imports_docker/import-torch.sh deleted file mode 100755 index af060546e..000000000 --- a/test_imports_docker/import-torch.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -python -c "import torch as th; x = th.autograd.Variable(th.rand(1, 3, 2, 2)); l = th.nn.Upsample(scale_factor=2); print(l(x))" diff --git a/test_imports_docker/install-conda-centos.sh b/test_imports_docker/install-conda-centos.sh deleted file mode 100755 index c08eff7a2..000000000 --- a/test_imports_docker/install-conda-centos.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -e - -yum install -y -q wget - -export MV=3 -wget --quiet https://repo.continuum.io/miniconda/Miniconda$MV-latest-Linux-x86_64.sh && \ - chmod +x Miniconda$MV-latest-Linux-x86_64.sh && \ - ./Miniconda$MV-latest-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda$MV-latest-Linux-x86_64.sh - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - diff --git a/test_imports_docker/install-conda-ubuntu.sh b/test_imports_docker/install-conda-ubuntu.sh deleted file mode 100755 index 4d267bd42..000000000 --- a/test_imports_docker/install-conda-ubuntu.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e - -apt-get -qq update -apt-get -qq install -y wget git bzip2 2>&1 >/dev/null - -export MV=3 -wget --quiet https://repo.continuum.io/miniconda/Miniconda$MV-latest-Linux-x86_64.sh && \ - chmod +x Miniconda$MV-latest-Linux-x86_64.sh && \ - ./Miniconda$MV-latest-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda$MV-latest-Linux-x86_64.sh - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - diff --git a/test_imports_docker/install-cv2-conda.sh b/test_imports_docker/install-cv2-conda.sh deleted file mode 100755 index 2c423502b..000000000 --- a/test_imports_docker/install-cv2-conda.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -apt-get install -qq -y libgtk2.0-0 2>&1 >/dev/null -conda install -y -c menpo opencv3 - diff --git a/test_imports_docker/install-pytorch-conda.sh b/test_imports_docker/install-pytorch-conda.sh deleted file mode 100755 index 45202404a..000000000 --- a/test_imports_docker/install-pytorch-conda.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -conda install -y pytorch-nightly -c pytorch -python -c "import torch; print(torch.__version__)" diff --git a/test_imports_docker/install-pytorch-whl.sh b/test_imports_docker/install-pytorch-whl.sh deleted file mode 100755 index 61e43f689..000000000 --- a/test_imports_docker/install-pytorch-whl.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -pip install https://download.pytorch.org/whl/cu80/torch-0.3.0.post4-cp36-cp36m-linux_x86_64.whl -python -c "import torch; print(torch.__version__)" - diff --git a/test_imports_docker/install-scipy-conda.sh b/test_imports_docker/install-scipy-conda.sh deleted file mode 100755 index 0d8224a49..000000000 --- a/test_imports_docker/install-scipy-conda.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -conda install -y scipy -python -c "import scipy; print(scipy.__version__)" diff --git a/test_imports_docker/install-tf-conda.sh b/test_imports_docker/install-tf-conda.sh deleted file mode 100755 index ae7be6f1b..000000000 --- a/test_imports_docker/install-tf-conda.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -conda install -y tensorflow - diff --git a/test_imports_docker/install-tf-pip.sh b/test_imports_docker/install-tf-pip.sh deleted file mode 100755 index f6d0c8707..000000000 --- a/test_imports_docker/install-tf-pip.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -export PATH="/opt/conda/bin:$PATH" -export CMAKE_PREFIX_PATH="/opt/conda" - -pip install tensorflow - diff --git a/test_imports_docker/run.sh b/test_imports_docker/run.sh deleted file mode 100755 index 4f9d42ca8..000000000 --- a/test_imports_docker/run.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -set -ex - -nvidia-docker stop img || true - -# If any of these fail, they'll leave behind a running docker container -# named img. -# Remove it with `docker stop img` or attach it and see what's up with -# `docker attach img`. The following is needed to debug inside container: -# export PATH="/opt/conda/bin:$PATH" - -# UNCOMMENT WHAT YOU NEED: - -# pytorch installs -#./test-script.sh nvidia/cuda:9.0-base install-conda-ubuntu.sh install-pytorch-conda.sh import-torch.sh -#./test-script.sh nvidia/cuda:8.0-runtime-ubuntu14.04 install-conda-ubuntu.sh install-pytorch-conda.sh import-torch.sh - -# pytorch / tf -#./test-script.sh nvidia/cuda:9.0-base install-conda-ubuntu.sh install-tf-conda.sh install-pytorch-conda.sh import-pytorch-tf.sh - -# The following crashes: -# ./test-script.sh nvidia/cuda:9.0-base install-conda-ubuntu.sh install-tf-conda.sh install-pytorch-conda.sh import-tf-pytorch.sh - -# This is OK -# ./test-script.sh nvidia/cuda:8.0-runtime-ubuntu14.04 install-conda-ubuntu.sh install-tf-conda.sh install-pytorch-conda.sh import-pytorch-tf.sh - -# this also crashes: -# ./test-script.sh nvidia/cuda:8.0-runtime-ubuntu14.04 install-conda-ubuntu.sh install-tf-conda.sh install-pytorch-conda.sh import-tf-pytorch.sh - -# pytorch / scipy -# ./test-script.sh nvidia/cuda:9.0-base install-conda-ubuntu.sh install-scipy-conda.sh install-pytorch-conda.sh import-pytorch-scipy.sh -#./test-script.sh nvidia/cuda:9.0-base install-conda-ubuntu.sh install-scipy-conda.sh install-pytorch-conda.sh import-scipy-pytorch.sh -#./test-script.sh nvidia/cuda:8.0-runtime-ubuntu14.04 install-conda-ubuntu.sh install-scipy-conda.sh install-pytorch-conda.sh import-pytorch-scipy.sh -#./test-script.sh nvidia/cuda:8.0-runtime-ubuntu14.04 install-conda-ubuntu.sh install-scipy-conda.sh install-pytorch-conda.sh import-scipy-pytorch.sh - - -# pytorch / cv2 -# ./test-script.sh nvidia/cuda:9.0-base install-conda-ubuntu.sh install-cv2-conda.sh install-pytorch-conda.sh import-pytorch-cv2.sh - -# The following crashes: -# ./test-script.sh nvidia/cuda:9.0-base install-conda-ubuntu.sh install-cv2-conda.sh install-pytorch-conda.sh import-cv2-pytorch.sh - -# This is OK -# ./test-script.sh nvidia/cuda:8.0-runtime-ubuntu14.04 install-conda-ubuntu.sh install-cv2-conda.sh install-pytorch-conda.sh import-pytorch-cv2.sh -# ./test-script.sh nvidia/cuda:8.0-runtime-ubuntu14.04 install-conda-ubuntu.sh install-cv2-conda.sh install-pytorch-conda.sh import-cv2-pytorch.sh - -# Older scripts, that were used in 0.4 testing -# ./test-script.sh ubuntu:14.04 install-conda-ubuntu.sh install-pytorch-whl.sh -# ./test-script.sh ubuntu:16.04 install-conda-ubuntu.sh install-pytorch-whl.sh -#./test-script.sh centos:6 install-conda-centos.sh install-pytorch-conda.sh -# ./test-script.sh centos:6 install-conda-centos.sh install-pytorch-whl.sh - -# pytorch / tf compatability (cpu-only) -#./test-script.sh ubuntu:16.04 install-conda-ubuntu.sh install-tf-conda.sh install-pytorch-conda.sh import-tf-pytorch.sh import-pytorch-tf.sh -# ./test-script.sh ubuntu:14.04 install-conda-ubuntu.sh install-tf-conda.sh install-pytorch-whl.sh import-tf-pytorch.sh import-pytorch-tf.sh -#./test-script.sh ubuntu:16.04 install-conda-ubuntu.sh install-tf-pip.sh install-pytorch-conda.sh import-tf-pytorch.sh import-pytorch-tf.sh -# ./test-script.sh ubuntu:14.04 install-conda-ubuntu.sh install-tf-pip.sh install-pytorch-whl.sh import-tf-pytorch.sh import-pytorch-tf.sh - -# pytorch / cv2 (opencv3) compatibility (cpu-only) -# ./test-script.sh ubuntu:16.04 install-conda-ubuntu.sh install-cv2-conda.sh install-pytorch-whl.sh import-cv2-pytorch.sh import-pytorch-cv2.sh -#./test-script.sh ubuntu:16.04 install-conda-ubuntu.sh install-cv2-conda.sh install-pytorch-conda.sh import-cv2-pytorch.sh import-pytorch-cv2.sh - diff --git a/test_imports_docker/test-script.sh b/test_imports_docker/test-script.sh deleted file mode 100755 index 6d7786c22..000000000 --- a/test_imports_docker/test-script.sh +++ /dev/null @@ -1,13 +0,0 @@ -set -ex - -# Runs a script on a container with the image. - -IMAGE=$1 # e.g. ubuntu:12.04 -# The other arguments, 2-..., are script names, ie install-conda-ubuntu.sh -nvidia-docker run -d --name img -it --rm -v $(pwd):/remote $IMAGE /bin/bash -# Run the rest of the argument as scripts -for var in "${@:2}"; do - nvidia-docker exec img /remote/$var -done -nvidia-docker stop img -sleep 5 diff --git a/upgrade_gcc_abi.sh b/upgrade_gcc_abi.sh deleted file mode 100755 index f016ebd8a..000000000 --- a/upgrade_gcc_abi.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# This script has been renamed. This redirection only needs to be here for a -# few hours for current outstanding PRs. If you are reading this after July -# 24th then you can remove this safely. -SOURCE_DIR=$(cd $(dirname $0) && pwd) -source "${SOURCE_DIR}/update_compiler.sh"