From be243d635d3fcaee131fc338dd6680035549c27a Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Wed, 25 Mar 2020 14:59:45 +0200 Subject: [PATCH 01/39] Enable tests --- recipe/meta.yaml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index f5fd26e..1ffd294 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,6 +1,9 @@ {% set version = "0.16.0" %} {% set filename = "apache-arrow-" + version + ".tar.gz" %} {% set checksum = "261992de4029a1593195ff4000501503bd403146471b3168bd2cc414ad0fb7f5" %} +{% set number = "3" %} +{% set cuda_enabled = cuda_compiler_version is not undefined and cuda_compiler_version == '9.2' %} +{% set build_ext = "cuda" if cuda_enabled else "cpu" %} package: name: pyarrow @@ -12,7 +15,7 @@ source: sha256: {{ checksum }} build: - number: 2 + number: {{ number }} requirements: build: @@ -45,18 +48,25 @@ requirements: - six - snappy -test: - imports: - - pyarrow - - pyarrow.dataset # [not py==27] - - pyarrow.flight # [not py==27] - - pyarrow.gandiva # [(not py==27) and (not (aarch64 or ppc64le))] - - pyarrow.orc # [unix] - - pyarrow.parquet - - pyarrow.plasma # [unix] - - pyarrow.fs - - pyarrow._s3fs - - pyarrow._hdfs + test: + imports: + - pyarrow + - pyarrow.dataset # [not py==27] + - pyarrow.flight # [not py==27] + - pyarrow.gandiva # [(not py==27) and (not (aarch64 or ppc64le))] + - pyarrow.orc # [unix] + - pyarrow.parquet + - pyarrow.plasma # [unix] + - pyarrow.fs + - pyarrow._s3fs + - pyarrow._hdfs + - pyarrow.cuda # [linux and cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] + + about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: LICENSE.txt + summary: 'Python libraries for Apache Arrow' about: home: http://github.com/apache/arrow From 424860dd4aec8f6bc5edab242917f5f4d6657746 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Fri, 17 Apr 2020 11:52:56 +0300 Subject: [PATCH 02/39] rebase, use arrow-cpp-proc, add pearu to recipe-maintainers --- recipe/meta.yaml | 81 ++++++++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 1ffd294..40c2e06 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -6,7 +6,7 @@ {% set build_ext = "cuda" if cuda_enabled else "cpu" %} package: - name: pyarrow + name: pyarrow-ext version: {{ version }} source: @@ -15,38 +15,58 @@ source: sha256: {{ checksum }} build: + skip: true # [win32] + skip: true # [win and py<35] + skip: true # [linux and cuda_compiler_version is not undefined and cuda_compiler_version != '9.2' and cuda_compiler_version != 'None'] number: {{ number }} -requirements: - build: - - cmake - - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - host: - # directly pin boost-cpp as we also seem to directly include boost symbols - # in the Python modules. - - arrow-cpp {{ version }} - - boost-cpp - - cython - - numpy 1.16.* - - python - - setuptools - - setuptools_scm - - six +outputs: + - name: pyarrow + script: build-pyarrow.sh # [not win] + script: bld.bat # [win] + version: {{ version }} + build: + number: {{ number }} + string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" + skip: true # [win32] + skip: true # [win and py<35] + skip: true # [linux and cuda_compiler_version is not undefined and cuda_compiler_version != '9.2' and cuda_compiler_version != 'None'] + track_features: + - arrow-cuda # [linux and cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] - run: - - {{ pin_compatible('numpy', lower_bound='1.16') }} - - boost-cpp - - futures # [py==27] - - glog - - pandas - # empty parquet-cpp metapackage, force old versions to be uninstalled - - parquet-cpp 1.5.1.* - - python - - six - - snappy + requirements: + build: + - cmake + - ninja + - make # [unix] + - {{ compiler('c') }} + - {{ compiler('cxx') }} + # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda + - {{ compiler("cuda") }} # [linux and cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] + host: + # directly pin boost-cpp as we also seem to directly include boost symbols + # in the Python modules. + - arrow-cpp {{ version }} + - boost-cpp + - cython + - numpy 1.16.* + - python + - setuptools + - setuptools_scm + - six + run: + - {{ pin_compatible('numpy', lower_bound='1.16') }} + - boost-cpp + - futures # [py==27] + - glog + - pandas + # empty parquet-cpp metapackage, force old versions to be uninstalled + - parquet-cpp 1.5.1.* + - python + - six + - snappy + run_constrained: + - arrow-cpp-proc * {{ build_ext }} test: imports: @@ -86,3 +106,4 @@ extra: - kou - kszucs - pitrou + - pearu From 3929c18836f93c35922a2528b29ff48af2a523d4 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Fri, 17 Apr 2020 11:56:21 +0300 Subject: [PATCH 03/39] rebase (2) --- .../linux_aarch64_python3.6.____cpython.yaml | 30 -------- .../linux_aarch64_python3.7.____cpython.yaml | 30 -------- .../linux_aarch64_python3.8.____cpython.yaml | 30 -------- .../linux_ppc64le_python3.6.____cpython.yaml | 24 ------ .../linux_ppc64le_python3.7.____cpython.yaml | 24 ------ .../linux_ppc64le_python3.8.____cpython.yaml | 24 ------ .ci_support/linux_python3.6.____cpython.yaml | 24 ------ .ci_support/linux_python3.7.____cpython.yaml | 24 ------ .ci_support/linux_python3.8.____cpython.yaml | 24 ------ .ci_support/osx_python3.6.____cpython.yaml | 28 ------- .ci_support/osx_python3.7.____cpython.yaml | 28 ------- .ci_support/osx_python3.8.____cpython.yaml | 28 ------- .ci_support/win_python3.6.____cpython.yaml | 18 ----- .ci_support/win_python3.7.____cpython.yaml | 18 ----- .ci_support/win_python3.8.____cpython.yaml | 18 ----- .scripts/build_steps.sh | 39 ---------- .scripts/run_docker_build.sh | 75 ------------------- recipe/{build.sh => build-pyarrow.sh} | 12 ++- 18 files changed, 11 insertions(+), 487 deletions(-) delete mode 100644 .ci_support/linux_aarch64_python3.6.____cpython.yaml delete mode 100644 .ci_support/linux_aarch64_python3.7.____cpython.yaml delete mode 100644 .ci_support/linux_aarch64_python3.8.____cpython.yaml delete mode 100644 .ci_support/linux_ppc64le_python3.6.____cpython.yaml delete mode 100644 .ci_support/linux_ppc64le_python3.7.____cpython.yaml delete mode 100644 .ci_support/linux_ppc64le_python3.8.____cpython.yaml delete mode 100644 .ci_support/linux_python3.6.____cpython.yaml delete mode 100644 .ci_support/linux_python3.7.____cpython.yaml delete mode 100644 .ci_support/linux_python3.8.____cpython.yaml delete mode 100644 .ci_support/osx_python3.6.____cpython.yaml delete mode 100644 .ci_support/osx_python3.7.____cpython.yaml delete mode 100644 .ci_support/osx_python3.8.____cpython.yaml delete mode 100644 .ci_support/win_python3.6.____cpython.yaml delete mode 100644 .ci_support/win_python3.7.____cpython.yaml delete mode 100644 .ci_support/win_python3.8.____cpython.yaml delete mode 100755 .scripts/build_steps.sh delete mode 100755 .scripts/run_docker_build.sh rename recipe/{build.sh => build-pyarrow.sh} (69%) diff --git a/.ci_support/linux_aarch64_python3.6.____cpython.yaml b/.ci_support/linux_aarch64_python3.6.____cpython.yaml deleted file mode 100644 index 22a2d0f..0000000 --- a/.ci_support/linux_aarch64_python3.6.____cpython.yaml +++ /dev/null @@ -1,30 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-aarch64 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.6.* *_cpython diff --git a/.ci_support/linux_aarch64_python3.7.____cpython.yaml b/.ci_support/linux_aarch64_python3.7.____cpython.yaml deleted file mode 100644 index 92d7eb3..0000000 --- a/.ci_support/linux_aarch64_python3.7.____cpython.yaml +++ /dev/null @@ -1,30 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-aarch64 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.7.* *_cpython diff --git a/.ci_support/linux_aarch64_python3.8.____cpython.yaml b/.ci_support/linux_aarch64_python3.8.____cpython.yaml deleted file mode 100644 index 33d823c..0000000 --- a/.ci_support/linux_aarch64_python3.8.____cpython.yaml +++ /dev/null @@ -1,30 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-aarch64 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.8.* *_cpython diff --git a/.ci_support/linux_ppc64le_python3.6.____cpython.yaml b/.ci_support/linux_ppc64le_python3.6.____cpython.yaml deleted file mode 100644 index ed6da93..0000000 --- a/.ci_support/linux_ppc64le_python3.6.____cpython.yaml +++ /dev/null @@ -1,24 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '8' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '8' -docker_image: -- condaforge/linux-anvil-ppc64le -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.6.* *_cpython diff --git a/.ci_support/linux_ppc64le_python3.7.____cpython.yaml b/.ci_support/linux_ppc64le_python3.7.____cpython.yaml deleted file mode 100644 index d3a2888..0000000 --- a/.ci_support/linux_ppc64le_python3.7.____cpython.yaml +++ /dev/null @@ -1,24 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '8' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '8' -docker_image: -- condaforge/linux-anvil-ppc64le -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.7.* *_cpython diff --git a/.ci_support/linux_ppc64le_python3.8.____cpython.yaml b/.ci_support/linux_ppc64le_python3.8.____cpython.yaml deleted file mode 100644 index d62b168..0000000 --- a/.ci_support/linux_ppc64le_python3.8.____cpython.yaml +++ /dev/null @@ -1,24 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '8' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '8' -docker_image: -- condaforge/linux-anvil-ppc64le -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.8.* *_cpython diff --git a/.ci_support/linux_python3.6.____cpython.yaml b/.ci_support/linux_python3.6.____cpython.yaml deleted file mode 100644 index f49a027..0000000 --- a/.ci_support/linux_python3.6.____cpython.yaml +++ /dev/null @@ -1,24 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-comp7 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.6.* *_cpython diff --git a/.ci_support/linux_python3.7.____cpython.yaml b/.ci_support/linux_python3.7.____cpython.yaml deleted file mode 100644 index 9e9cedd..0000000 --- a/.ci_support/linux_python3.7.____cpython.yaml +++ /dev/null @@ -1,24 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-comp7 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.7.* *_cpython diff --git a/.ci_support/linux_python3.8.____cpython.yaml b/.ci_support/linux_python3.8.____cpython.yaml deleted file mode 100644 index c3e8bb6..0000000 --- a/.ci_support/linux_python3.8.____cpython.yaml +++ /dev/null @@ -1,24 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-comp7 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.8.* *_cpython diff --git a/.ci_support/osx_python3.6.____cpython.yaml b/.ci_support/osx_python3.6.____cpython.yaml deleted file mode 100644 index 053be7e..0000000 --- a/.ci_support/osx_python3.6.____cpython.yaml +++ /dev/null @@ -1,28 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -boost_cpp: -- '1.72' -c_compiler: -- clang -c_compiler_version: -- '9' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- clangxx -cxx_compiler_version: -- '9' -macos_machine: -- x86_64-apple-darwin13.4.0 -macos_min_version: -- '10.9' -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.6.* *_cpython diff --git a/.ci_support/osx_python3.7.____cpython.yaml b/.ci_support/osx_python3.7.____cpython.yaml deleted file mode 100644 index 0f0cde4..0000000 --- a/.ci_support/osx_python3.7.____cpython.yaml +++ /dev/null @@ -1,28 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -boost_cpp: -- '1.72' -c_compiler: -- clang -c_compiler_version: -- '9' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- clangxx -cxx_compiler_version: -- '9' -macos_machine: -- x86_64-apple-darwin13.4.0 -macos_min_version: -- '10.9' -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.7.* *_cpython diff --git a/.ci_support/osx_python3.8.____cpython.yaml b/.ci_support/osx_python3.8.____cpython.yaml deleted file mode 100644 index d3c3741..0000000 --- a/.ci_support/osx_python3.8.____cpython.yaml +++ /dev/null @@ -1,28 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -boost_cpp: -- '1.72' -c_compiler: -- clang -c_compiler_version: -- '9' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- clangxx -cxx_compiler_version: -- '9' -macos_machine: -- x86_64-apple-darwin13.4.0 -macos_min_version: -- '10.9' -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.8.* *_cpython diff --git a/.ci_support/win_python3.6.____cpython.yaml b/.ci_support/win_python3.6.____cpython.yaml deleted file mode 100644 index 6a14e22..0000000 --- a/.ci_support/win_python3.6.____cpython.yaml +++ /dev/null @@ -1,18 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- vs2015 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- vs2015 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.6.* *_cpython diff --git a/.ci_support/win_python3.7.____cpython.yaml b/.ci_support/win_python3.7.____cpython.yaml deleted file mode 100644 index ff9fd16..0000000 --- a/.ci_support/win_python3.7.____cpython.yaml +++ /dev/null @@ -1,18 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- vs2015 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- vs2015 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.7.* *_cpython diff --git a/.ci_support/win_python3.8.____cpython.yaml b/.ci_support/win_python3.8.____cpython.yaml deleted file mode 100644 index 6ef79ed..0000000 --- a/.ci_support/win_python3.8.____cpython.yaml +++ /dev/null @@ -1,18 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- vs2015 -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cxx_compiler: -- vs2015 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.8.* *_cpython diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh deleted file mode 100755 index 8a4af44..0000000 --- a/.scripts/build_steps.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here -# will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent -# changes to this script, consider a proposal to conda-smithy so that other feedstocks can also -# benefit from the improvement. - -set -xeuo pipefail -export PYTHONUNBUFFERED=1 -export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" -export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" -export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" -export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" - -cat >~/.condarc </dev/null && pwd )" -PROVIDER_DIR="$(basename $THISDIR)" - -FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) -RECIPE_ROOT="${FEEDSTOCK_ROOT}/recipe" - -docker info - -# In order for the conda-build process in the container to write to the mounted -# volumes, we need to run with the same id as the host machine, which is -# normally the owner of the mounted volumes, or at least has write permission -export HOST_USER_ID=$(id -u) -# Check if docker-machine is being used (normally on OSX) and get the uid from -# the VM -if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then - export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u) -fi - -ARTIFACTS="$FEEDSTOCK_ROOT/build_artifacts" - -if [ -z "$CONFIG" ]; then - set +x - FILES=`ls .ci_support/linux_*` - CONFIGS="" - for file in $FILES; do - CONFIGS="${CONFIGS}'${file:12:-5}' or "; - done - echo "Need to set CONFIG env variable. Value can be one of ${CONFIGS:0:-4}" - exit 1 -fi - -if [ -z "${DOCKER_IMAGE}" ]; then - SHYAML_INSTALLED="$(shyaml -h || echo NO)" - if [ "${SHYAML_INSTALLED}" == "NO" ]; then - echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Falling back to condaforge/linux-anvil-comp7" - DOCKER_IMAGE="condaforge/linux-anvil-comp7" - else - DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 condaforge/linux-anvil-comp7 )" - fi -fi - -mkdir -p "$ARTIFACTS" -DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}" -rm -f "$DONE_CANARY" - -if [ -z "${CI}" ]; then - DOCKER_RUN_ARGS="-it " -fi - -export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" -docker run ${DOCKER_RUN_ARGS} \ - -v "${RECIPE_ROOT}":/home/conda/recipe_root:ro,z \ - -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z \ - -e CONFIG \ - -e BINSTAR_TOKEN \ - -e HOST_USER_ID \ - -e UPLOAD_PACKAGES \ - -e GIT_BRANCH \ - -e UPLOAD_ON_BRANCH \ - -e CI \ - $DOCKER_IMAGE \ - bash \ - /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh - -# verify that the end of the script was reached -test -f "$DONE_CANARY" \ No newline at end of file diff --git a/recipe/build.sh b/recipe/build-pyarrow.sh similarity index 69% rename from recipe/build.sh rename to recipe/build-pyarrow.sh index 2dced50..b09898b 100644 --- a/recipe/build.sh +++ b/recipe/build-pyarrow.sh @@ -20,10 +20,20 @@ export PYARROW_WITH_ORC=1 export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_S3=1 +BUILD_EXT_FLAGS="" + +# Enable CUDA support +if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]] +then + export PYARROW_WITH_CUDA=1 + BUILD_EXT_FLAGS="${BUILD_EXT_FLAGS} --with-cuda" +else + export PYARROW_WITH_CUDA=0 +fi cd python $PYTHON setup.py \ - build_ext \ + build_ext $BUILD_EXT_FLAGS \ install --single-version-externally-managed \ --record=record.txt From f785cba46b2a43ebb1734a41bd968e54a4305de2 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Fri, 17 Apr 2020 11:56:49 +0300 Subject: [PATCH 04/39] MNT: Re-rendered with conda-build 3.18.11, conda-smithy 3.6.16, and conda-forge-pinning 2020.04.17.01.00.32 --- .azure-pipelines/azure-pipelines-linux.yml | 24 ++++-- .azure-pipelines/azure-pipelines-win.yml | 4 + .../linux_aarch64_python3.6.____cpython.yaml | 30 ++++++++ .../linux_aarch64_python3.7.____cpython.yaml | 30 ++++++++ .../linux_aarch64_python3.8.____cpython.yaml | 30 ++++++++ ...piler_version9.2python3.6.____cpython.yaml | 31 ++++++++ ...piler_version9.2python3.7.____cpython.yaml | 31 ++++++++ ...piler_version9.2python3.8.____cpython.yaml | 31 ++++++++ ...iler_versionNonepython3.6.____cpython.yaml | 31 ++++++++ ...iler_versionNonepython3.7.____cpython.yaml | 31 ++++++++ ...iler_versionNonepython3.8.____cpython.yaml | 31 ++++++++ .../linux_ppc64le_python3.6.____cpython.yaml | 24 ++++++ .../linux_ppc64le_python3.7.____cpython.yaml | 24 ++++++ .../linux_ppc64le_python3.8.____cpython.yaml | 24 ++++++ .ci_support/osx_python3.6.____cpython.yaml | 28 +++++++ .ci_support/osx_python3.7.____cpython.yaml | 28 +++++++ .ci_support/osx_python3.8.____cpython.yaml | 28 +++++++ .ci_support/win_python3.6.____cpython.yaml | 18 +++++ .ci_support/win_python3.7.____cpython.yaml | 18 +++++ .ci_support/win_python3.8.____cpython.yaml | 18 +++++ .github/CODEOWNERS | 2 +- .scripts/build_steps.sh | 39 ++++++++++ .scripts/run_docker_build.sh | 75 +++++++++++++++++++ README.md | 66 ++++++++++------ 24 files changed, 667 insertions(+), 29 deletions(-) create mode 100644 .ci_support/linux_aarch64_python3.6.____cpython.yaml create mode 100644 .ci_support/linux_aarch64_python3.7.____cpython.yaml create mode 100644 .ci_support/linux_aarch64_python3.8.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml create mode 100644 .ci_support/linux_ppc64le_python3.6.____cpython.yaml create mode 100644 .ci_support/linux_ppc64le_python3.7.____cpython.yaml create mode 100644 .ci_support/linux_ppc64le_python3.8.____cpython.yaml create mode 100644 .ci_support/osx_python3.6.____cpython.yaml create mode 100644 .ci_support/osx_python3.7.____cpython.yaml create mode 100644 .ci_support/osx_python3.8.____cpython.yaml create mode 100644 .ci_support/win_python3.6.____cpython.yaml create mode 100644 .ci_support/win_python3.7.____cpython.yaml create mode 100644 .ci_support/win_python3.8.____cpython.yaml create mode 100755 .scripts/build_steps.sh create mode 100755 .scripts/run_docker_build.sh diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index e5752e4..4681384 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -10,16 +10,28 @@ jobs: strategy: maxParallel: 8 matrix: - linux_python3.6.____cpython: - CONFIG: linux_python3.6.____cpython + linux_cuda_compiler_version9.2python3.6.____cpython: + CONFIG: linux_cuda_compiler_version9.2python3.6.____cpython + UPLOAD_PACKAGES: True + DOCKER_IMAGE: condaforge/linux-anvil-cuda:9.2 + linux_cuda_compiler_version9.2python3.7.____cpython: + CONFIG: linux_cuda_compiler_version9.2python3.7.____cpython + UPLOAD_PACKAGES: True + DOCKER_IMAGE: condaforge/linux-anvil-cuda:9.2 + linux_cuda_compiler_version9.2python3.8.____cpython: + CONFIG: linux_cuda_compiler_version9.2python3.8.____cpython + UPLOAD_PACKAGES: True + DOCKER_IMAGE: condaforge/linux-anvil-cuda:9.2 + linux_cuda_compiler_versionNonepython3.6.____cpython: + CONFIG: linux_cuda_compiler_versionNonepython3.6.____cpython UPLOAD_PACKAGES: True DOCKER_IMAGE: condaforge/linux-anvil-comp7 - linux_python3.7.____cpython: - CONFIG: linux_python3.7.____cpython + linux_cuda_compiler_versionNonepython3.7.____cpython: + CONFIG: linux_cuda_compiler_versionNonepython3.7.____cpython UPLOAD_PACKAGES: True DOCKER_IMAGE: condaforge/linux-anvil-comp7 - linux_python3.8.____cpython: - CONFIG: linux_python3.8.____cpython + linux_cuda_compiler_versionNonepython3.8.____cpython: + CONFIG: linux_cuda_compiler_versionNonepython3.8.____cpython UPLOAD_PACKAGES: True DOCKER_IMAGE: condaforge/linux-anvil-comp7 linux_aarch64_python3.6.____cpython: diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 06d5476..c6bed0d 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -77,6 +77,7 @@ jobs: # Configure the VM. - script: | set "CI=azure" + call activate base run_conda_forge_build_setup displayName: conda-forge build setup @@ -88,6 +89,7 @@ jobs: # Special cased version setting some more things! - script: | + call activate base conda.exe build "recipe" -m .ci_support\%CONFIG%.yaml displayName: Build recipe (vs2008) env: @@ -96,6 +98,7 @@ jobs: condition: contains(variables['CONFIG'], 'vs2008') - script: | + call activate base conda.exe build "recipe" -m .ci_support\%CONFIG%.yaml displayName: Build recipe env: @@ -104,6 +107,7 @@ jobs: - script: | set "GIT_BRANCH=%BUILD_SOURCEBRANCHNAME%" + call activate base upload_package .\ ".\recipe" .ci_support\%CONFIG%.yaml displayName: Upload package env: diff --git a/.ci_support/linux_aarch64_python3.6.____cpython.yaml b/.ci_support/linux_aarch64_python3.6.____cpython.yaml new file mode 100644 index 0000000..22a2d0f --- /dev/null +++ b/.ci_support/linux_aarch64_python3.6.____cpython.yaml @@ -0,0 +1,30 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-aarch64 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython diff --git a/.ci_support/linux_aarch64_python3.7.____cpython.yaml b/.ci_support/linux_aarch64_python3.7.____cpython.yaml new file mode 100644 index 0000000..92d7eb3 --- /dev/null +++ b/.ci_support/linux_aarch64_python3.7.____cpython.yaml @@ -0,0 +1,30 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-aarch64 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.7.* *_cpython diff --git a/.ci_support/linux_aarch64_python3.8.____cpython.yaml b/.ci_support/linux_aarch64_python3.8.____cpython.yaml new file mode 100644 index 0000000..33d823c --- /dev/null +++ b/.ci_support/linux_aarch64_python3.8.____cpython.yaml @@ -0,0 +1,30 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-aarch64 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython diff --git a/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml b/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml new file mode 100644 index 0000000..29ed7cc --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '9.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:9.2 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml b/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml new file mode 100644 index 0000000..8a38927 --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '9.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:9.2 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.7.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml b/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml new file mode 100644 index 0000000..f412ce0 --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '9.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:9.2 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml b/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml new file mode 100644 index 0000000..9aa691c --- /dev/null +++ b/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-comp7 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml b/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml new file mode 100644 index 0000000..f75dba7 --- /dev/null +++ b/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-comp7 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.7.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml b/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml new file mode 100644 index 0000000..b153ec7 --- /dev/null +++ b/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-comp7 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_ppc64le_python3.6.____cpython.yaml b/.ci_support/linux_ppc64le_python3.6.____cpython.yaml new file mode 100644 index 0000000..ed6da93 --- /dev/null +++ b/.ci_support/linux_ppc64le_python3.6.____cpython.yaml @@ -0,0 +1,24 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '8' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '8' +docker_image: +- condaforge/linux-anvil-ppc64le +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython diff --git a/.ci_support/linux_ppc64le_python3.7.____cpython.yaml b/.ci_support/linux_ppc64le_python3.7.____cpython.yaml new file mode 100644 index 0000000..d3a2888 --- /dev/null +++ b/.ci_support/linux_ppc64le_python3.7.____cpython.yaml @@ -0,0 +1,24 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '8' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '8' +docker_image: +- condaforge/linux-anvil-ppc64le +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.7.* *_cpython diff --git a/.ci_support/linux_ppc64le_python3.8.____cpython.yaml b/.ci_support/linux_ppc64le_python3.8.____cpython.yaml new file mode 100644 index 0000000..d62b168 --- /dev/null +++ b/.ci_support/linux_ppc64le_python3.8.____cpython.yaml @@ -0,0 +1,24 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '8' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '8' +docker_image: +- condaforge/linux-anvil-ppc64le +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython diff --git a/.ci_support/osx_python3.6.____cpython.yaml b/.ci_support/osx_python3.6.____cpython.yaml new file mode 100644 index 0000000..053be7e --- /dev/null +++ b/.ci_support/osx_python3.6.____cpython.yaml @@ -0,0 +1,28 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +boost_cpp: +- '1.72' +c_compiler: +- clang +c_compiler_version: +- '9' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '9' +macos_machine: +- x86_64-apple-darwin13.4.0 +macos_min_version: +- '10.9' +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython diff --git a/.ci_support/osx_python3.7.____cpython.yaml b/.ci_support/osx_python3.7.____cpython.yaml new file mode 100644 index 0000000..0f0cde4 --- /dev/null +++ b/.ci_support/osx_python3.7.____cpython.yaml @@ -0,0 +1,28 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +boost_cpp: +- '1.72' +c_compiler: +- clang +c_compiler_version: +- '9' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '9' +macos_machine: +- x86_64-apple-darwin13.4.0 +macos_min_version: +- '10.9' +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.7.* *_cpython diff --git a/.ci_support/osx_python3.8.____cpython.yaml b/.ci_support/osx_python3.8.____cpython.yaml new file mode 100644 index 0000000..d3c3741 --- /dev/null +++ b/.ci_support/osx_python3.8.____cpython.yaml @@ -0,0 +1,28 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +boost_cpp: +- '1.72' +c_compiler: +- clang +c_compiler_version: +- '9' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '9' +macos_machine: +- x86_64-apple-darwin13.4.0 +macos_min_version: +- '10.9' +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython diff --git a/.ci_support/win_python3.6.____cpython.yaml b/.ci_support/win_python3.6.____cpython.yaml new file mode 100644 index 0000000..ab19ca4 --- /dev/null +++ b/.ci_support/win_python3.6.____cpython.yaml @@ -0,0 +1,18 @@ +boost_cpp: +- '1.72' +c_compiler: +- vs2017 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- vs2017 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython diff --git a/.ci_support/win_python3.7.____cpython.yaml b/.ci_support/win_python3.7.____cpython.yaml new file mode 100644 index 0000000..d3f0623 --- /dev/null +++ b/.ci_support/win_python3.7.____cpython.yaml @@ -0,0 +1,18 @@ +boost_cpp: +- '1.72' +c_compiler: +- vs2017 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- vs2017 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.7.* *_cpython diff --git a/.ci_support/win_python3.8.____cpython.yaml b/.ci_support/win_python3.8.____cpython.yaml new file mode 100644 index 0000000..7adecc6 --- /dev/null +++ b/.ci_support/win_python3.8.____cpython.yaml @@ -0,0 +1,18 @@ +boost_cpp: +- '1.72' +c_compiler: +- vs2017 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- vs2017 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1c03536..38893ef 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @cpcloud @jreback @kou @kszucs @pcmoritz @pitrou @robertnishihara @siddharthteotia @wesm @xhochy \ No newline at end of file +* @cpcloud @jreback @kou @kszucs @pcmoritz @pearu @pitrou @robertnishihara @siddharthteotia @wesm @xhochy \ No newline at end of file diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh new file mode 100755 index 0000000..8a4af44 --- /dev/null +++ b/.scripts/build_steps.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here +# will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent +# changes to this script, consider a proposal to conda-smithy so that other feedstocks can also +# benefit from the improvement. + +set -xeuo pipefail +export PYTHONUNBUFFERED=1 +export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" +export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" +export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" +export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" + +cat >~/.condarc </dev/null && pwd )" +PROVIDER_DIR="$(basename $THISDIR)" + +FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) +RECIPE_ROOT="${FEEDSTOCK_ROOT}/recipe" + +docker info + +# In order for the conda-build process in the container to write to the mounted +# volumes, we need to run with the same id as the host machine, which is +# normally the owner of the mounted volumes, or at least has write permission +export HOST_USER_ID=$(id -u) +# Check if docker-machine is being used (normally on OSX) and get the uid from +# the VM +if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then + export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u) +fi + +ARTIFACTS="$FEEDSTOCK_ROOT/build_artifacts" + +if [ -z "$CONFIG" ]; then + set +x + FILES=`ls .ci_support/linux_*` + CONFIGS="" + for file in $FILES; do + CONFIGS="${CONFIGS}'${file:12:-5}' or "; + done + echo "Need to set CONFIG env variable. Value can be one of ${CONFIGS:0:-4}" + exit 1 +fi + +if [ -z "${DOCKER_IMAGE}" ]; then + SHYAML_INSTALLED="$(shyaml -h || echo NO)" + if [ "${SHYAML_INSTALLED}" == "NO" ]; then + echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Falling back to condaforge/linux-anvil-comp7" + DOCKER_IMAGE="condaforge/linux-anvil-comp7" + else + DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 condaforge/linux-anvil-comp7 )" + fi +fi + +mkdir -p "$ARTIFACTS" +DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}" +rm -f "$DONE_CANARY" + +if [ -z "${CI}" ]; then + DOCKER_RUN_ARGS="-it " +fi + +export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" +docker run ${DOCKER_RUN_ARGS} \ + -v "${RECIPE_ROOT}":/home/conda/recipe_root:ro,z \ + -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z \ + -e CONFIG \ + -e BINSTAR_TOKEN \ + -e HOST_USER_ID \ + -e UPLOAD_PACKAGES \ + -e GIT_BRANCH \ + -e UPLOAD_ON_BRANCH \ + -e CI \ + $DOCKER_IMAGE \ + bash \ + /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh + +# verify that the end of the script was reached +test -f "$DONE_CANARY" \ No newline at end of file diff --git a/README.md b/README.md index 87e0525..b0fcd5f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -About pyarrow -============= +About pyarrow-ext +================= Home: http://github.com/apache/arrow -Package license: Apache 2.0 +Package license: Apache-2.0 Feedstock license: BSD 3-Clause @@ -57,45 +57,66 @@ Current build status - linux_ppc64le_python3.6.____cpython + linux_cuda_compiler_version9.2python3.6.____cpython - variant + variant - linux_ppc64le_python3.7.____cpython + linux_cuda_compiler_version9.2python3.7.____cpython - variant + variant - linux_ppc64le_python3.8.____cpython + linux_cuda_compiler_version9.2python3.8.____cpython - variant + variant + + + + linux_cuda_compiler_versionNonepython3.6.____cpython + + + variant + + + + linux_cuda_compiler_versionNonepython3.7.____cpython + + + variant + + + + linux_cuda_compiler_versionNonepython3.8.____cpython + + + variant - linux_python3.6.____cpython + linux_ppc64le_python3.6.____cpython - variant + variant - linux_python3.7.____cpython + linux_ppc64le_python3.7.____cpython - variant + variant - linux_python3.8.____cpython + linux_ppc64le_python3.8.____cpython - variant + variant @@ -155,10 +176,10 @@ Current release info | --- | --- | --- | --- | | [![Conda Recipe](https://img.shields.io/badge/recipe-pyarrow-green.svg)](https://anaconda.org/conda-forge/pyarrow) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/pyarrow.svg)](https://anaconda.org/conda-forge/pyarrow) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/pyarrow.svg)](https://anaconda.org/conda-forge/pyarrow) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/pyarrow.svg)](https://anaconda.org/conda-forge/pyarrow) | -Installing pyarrow -================== +Installing pyarrow-ext +====================== -Installing `pyarrow` from the `conda-forge` channel can be achieved by adding `conda-forge` to your channels with: +Installing `pyarrow-ext` from the `conda-forge` channel can be achieved by adding `conda-forge` to your channels with: ``` conda config --add channels conda-forge @@ -215,17 +236,17 @@ Terminology produce the finished article (built conda distributions) -Updating pyarrow-feedstock -========================== +Updating pyarrow-ext-feedstock +============================== -If you would like to improve the pyarrow recipe or build a new +If you would like to improve the pyarrow-ext recipe or build a new package version, please fork this repository and submit a PR. Upon submission, your changes will be run on the appropriate platforms to give the reviewer an opportunity to confirm that the changes result in a successful build. Once merged, the recipe will be re-built and uploaded automatically to the `conda-forge` channel, whereupon the built conda packages will be available for everybody to install and use from the `conda-forge` channel. -Note that all branches in the conda-forge/pyarrow-feedstock are +Note that all branches in the conda-forge/pyarrow-ext-feedstock are immediately built and any created packages are uploaded, so PRs should be based on branches in forks and branches in the main repository should only be used to build distinct package versions. @@ -245,6 +266,7 @@ Feedstock Maintainers * [@kou](https://github.com/kou/) * [@kszucs](https://github.com/kszucs/) * [@pcmoritz](https://github.com/pcmoritz/) +* [@pearu](https://github.com/pearu/) * [@pitrou](https://github.com/pitrou/) * [@robertnishihara](https://github.com/robertnishihara/) * [@siddharthteotia](https://github.com/siddharthteotia/) From 662f537c36e08eac9ae46841402359fcd64ee0c6 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 12:23:48 +0300 Subject: [PATCH 05/39] Update --- recipe/meta.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 40c2e06..e7d6baa 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -17,7 +17,7 @@ source: build: skip: true # [win32] skip: true # [win and py<35] - skip: true # [linux and cuda_compiler_version is not undefined and cuda_compiler_version != '9.2' and cuda_compiler_version != 'None'] + skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] number: {{ number }} outputs: @@ -30,10 +30,9 @@ outputs: string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" skip: true # [win32] skip: true # [win and py<35] - skip: true # [linux and cuda_compiler_version is not undefined and cuda_compiler_version != '9.2' and cuda_compiler_version != 'None'] + skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] track_features: - - arrow-cuda # [linux and cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] - + - {{ "- arrow-cuda" if cuda_enabled else "" }} requirements: build: - cmake @@ -80,7 +79,7 @@ outputs: - pyarrow.fs - pyarrow._s3fs - pyarrow._hdfs - - pyarrow.cuda # [linux and cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] + {{ "- pyarrow.cuda" if cuda_enabled else "" }} about: home: http://github.com/apache/arrow From d1409fab3e93ac238abf54e94fb472bcd566bff7 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 12:25:03 +0300 Subject: [PATCH 06/39] MNT: Re-rendered with conda-build 3.18.11, conda-smithy 3.7.0, and conda-forge-pinning 2020.04.27.17.45.41 --- .azure-pipelines/azure-pipelines-osx.yml | 58 ++---------------------- .azure-pipelines/azure-pipelines-win.yml | 15 +++--- .scripts/build_steps.sh | 5 +- .scripts/run_docker_build.sh | 4 +- .scripts/run_osx_build.sh | 55 ++++++++++++++++++++++ 5 files changed, 72 insertions(+), 65 deletions(-) create mode 100755 .scripts/run_osx_build.sh diff --git a/.azure-pipelines/azure-pipelines-osx.yml b/.azure-pipelines/azure-pipelines-osx.yml index e864659..65502e5 100755 --- a/.azure-pipelines/azure-pipelines-osx.yml +++ b/.azure-pipelines/azure-pipelines-osx.yml @@ -23,60 +23,10 @@ jobs: steps: # TODO: Fast finish on azure pipelines? - script: | - echo "Fast Finish" - - - - script: | - echo "Removing homebrew from Azure to avoid conflicts." - curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > ~/uninstall_homebrew - chmod +x ~/uninstall_homebrew - ~/uninstall_homebrew -fq - rm ~/uninstall_homebrew - displayName: Remove homebrew - - - bash: | - echo "##vso[task.prependpath]$CONDA/bin" - sudo chown -R $USER $CONDA - displayName: Add conda to PATH - - - script: | - source activate base - conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=2 conda-build - displayName: 'Add conda-forge-ci-setup=2' - - - script: | - source activate base - echo "Configuring conda." - - setup_conda_rc ./ "./recipe" ./.ci_support/${CONFIG}.yaml export CI=azure - source run_conda_forge_build_setup - conda update --yes --quiet --override-channels -c conda-forge -c defaults --all - env: { - OSX_FORCE_SDK_DOWNLOAD: "1" - } - displayName: Configure conda and conda-build - - - script: | - source activate base - mangle_compiler ./ "./recipe" ./.ci_support/${CONFIG}.yaml - displayName: Mangle compiler - - - script: | - source activate base - make_build_number ./ "./recipe" ./.ci_support/${CONFIG}.yaml - displayName: Generate build number clobber file - - - script: | - source activate base - conda build "./recipe" -m ./.ci_support/${CONFIG}.yaml --clobber-file ./.ci_support/clobber_${CONFIG}.yaml - displayName: Build recipe - - - script: | - source activate base + export OSX_FORCE_SDK_DOWNLOAD="1" export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME - upload_package ./ "./recipe" ./.ci_support/${CONFIG}.yaml - displayName: Upload package + ./.scripts/run_osx_build.sh + displayName: Run OSX build env: - BINSTAR_TOKEN: $(BINSTAR_TOKEN) - condition: and(succeeded(), not(eq(variables['UPLOAD_PACKAGES'], 'False'))) \ No newline at end of file + BINSTAR_TOKEN: $(BINSTAR_TOKEN) \ No newline at end of file diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index c6bed0d..b0a337d 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -64,15 +64,19 @@ jobs: - task: CondaEnvironment@1 inputs: - packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=2' # Optional + packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=3 pip' # Optional installOptions: "-c conda-forge" updateConda: true displayName: Install conda-build and activate environment - script: set PYTHONUNBUFFERED=1 + displayName: Set PYTHONUNBUFFERED # Configure the VM - - script: setup_conda_rc .\ ".\recipe" .\.ci_support\%CONFIG%.yaml + - script: | + call activate base + setup_conda_rc .\ ".\recipe" .\.ci_support\%CONFIG%.yaml + displayName: conda-forge CI setup # Configure the VM. - script: | @@ -82,11 +86,6 @@ jobs: displayName: conda-forge build setup - - script: | - rmdir C:\strawberry /s /q - continueOnError: true - displayName: remove strawberryperl - # Special cased version setting some more things! - script: | call activate base @@ -108,7 +107,7 @@ jobs: - script: | set "GIT_BRANCH=%BUILD_SOURCEBRANCHNAME%" call activate base - upload_package .\ ".\recipe" .ci_support\%CONFIG%.yaml + upload_package .\ ".\recipe" .ci_support\%CONFIG%.yaml displayName: Upload package env: BINSTAR_TOKEN: $(BINSTAR_TOKEN) diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 8a4af44..5b14787 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -19,7 +19,8 @@ conda-build: CONDARC -conda install --yes --quiet conda-forge-ci-setup=2 conda-build -c conda-forge +conda install --yes --quiet conda-forge-ci-setup=3 conda-build pip -c conda-forge + # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" @@ -33,7 +34,7 @@ conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" + upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" fi touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" \ No newline at end of file diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 066a857..253723c 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -52,8 +52,10 @@ mkdir -p "$ARTIFACTS" DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}" rm -f "$DONE_CANARY" +# Allow people to specify extra default arguments to `docker run` (e.g. `--rm`) +DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}" if [ -z "${CI}" ]; then - DOCKER_RUN_ARGS="-it " + DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}" fi export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh new file mode 100755 index 0000000..775799b --- /dev/null +++ b/.scripts/run_osx_build.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -x + +echo -e "\n\nInstalling a fresh version of Miniforge." +if [[ ${CI} == "travis" ]]; then + echo -en 'travis_fold:start:install_miniforge\\r' +fi +MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download" +MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh" +curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" +bash $MINIFORGE_FILE -b +if [[ ${CI} == "travis" ]]; then + echo -en 'travis_fold:end:install_miniforge\\r' +fi + +echo -e "\n\nConfiguring conda." +if [[ ${CI} == "travis" ]]; then + echo -en 'travis_fold:start:configure_conda\\r' +fi + +source ${HOME}/miniforge3/etc/profile.d/conda.sh +conda activate base + +echo -e "\n\nInstalling conda-forge-ci-setup=3 and conda-build." +conda install -n base --quiet --yes conda-forge-ci-setup=3 conda-build pip + + + +echo -e "\n\nSetting up the condarc and mangling the compiler." +setup_conda_rc ./ ./recipe ./.ci_support/${CONFIG}.yaml +mangle_compiler ./ ./recipe .ci_support/${CONFIG}.yaml + +echo -e "\n\nMangling homebrew in the CI to avoid conflicts." +/usr/bin/sudo mangle_homebrew +/usr/bin/sudo -k + +echo -e "\n\nRunning the build setup script." +source run_conda_forge_build_setup + + +if [[ ${CI} == "travis" ]]; then + echo -en 'travis_fold:end:configure_conda\\r' +fi + +set -e + +echo -e "\n\nMaking the build clobber file and running the build." +make_build_number ./ ./recipe ./.ci_support/${CONFIG}.yaml +conda build ./recipe -m ./.ci_support/${CONFIG}.yaml --clobber-file ./.ci_support/clobber_${CONFIG}.yaml + +if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then + echo -e "\n\nUploading the packages." + upload_package ./ ./recipe ./.ci_support/${CONFIG}.yaml +fi \ No newline at end of file From f85475809e855dd86fca84213099a34e875112f1 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 12:36:53 +0300 Subject: [PATCH 07/39] Use build_ext on arrow-cpp --- recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index e7d6baa..730522d 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -41,11 +41,11 @@ outputs: - {{ compiler('c') }} - {{ compiler('cxx') }} # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda - - {{ compiler("cuda") }} # [linux and cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] + - {{ compiler("cuda") }} # [cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] host: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. - - arrow-cpp {{ version }} + - arrow-cpp {{ version }} {{ build_ext }} - boost-cpp - cython - numpy 1.16.* From 90418f5c1a37b49a2db6f27330551c8aff6e552f Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 12:41:50 +0300 Subject: [PATCH 08/39] Use build_ext on arrow-cpp (2nd try) --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 730522d..8e1a9d3 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -45,7 +45,7 @@ outputs: host: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. - - arrow-cpp {{ version }} {{ build_ext }} + - arrow-cpp {{ version }} *_{{ build_ext }} - boost-cpp - cython - numpy 1.16.* From b1c330e08b21907ac505bcb0fa66a225bbe40ba0 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 13:25:07 +0300 Subject: [PATCH 09/39] Fix typo. --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 8e1a9d3..377e817 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -32,7 +32,7 @@ outputs: skip: true # [win and py<35] skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] track_features: - - {{ "- arrow-cuda" if cuda_enabled else "" }} + {{ "- arrow-cuda" if cuda_enabled else "" }} requirements: build: - cmake From e4f7f6a9a5b2f0183c5c07b193f547bda5fda055 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 13:42:53 +0300 Subject: [PATCH 10/39] Use build_ext on arrow-cpp, also in run. --- recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 377e817..6ec5b2f 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -55,6 +55,7 @@ outputs: - six run: - {{ pin_compatible('numpy', lower_bound='1.16') }} + - arrow-cpp {{ version }} *_{{ build_ext }} - boost-cpp - futures # [py==27] - glog From f781817ee95c65e9f18e9c764ad7900938a4bb00 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 14:05:26 +0300 Subject: [PATCH 11/39] Add python, cudatoolkit deps. Allow newer numpy than 1.16. --- recipe/meta.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 6ec5b2f..759695d 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -47,8 +47,9 @@ outputs: # in the Python modules. - arrow-cpp {{ version }} *_{{ build_ext }} - boost-cpp + - python - cython - - numpy 1.16.* + - numpy >=1.16 - python - setuptools - setuptools_scm @@ -67,6 +68,7 @@ outputs: - snappy run_constrained: - arrow-cpp-proc * {{ build_ext }} + - cudatoolkit >=9.2 test: imports: From 414f2560a494222de549b6f4359490efbea67b61 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 14:25:43 +0300 Subject: [PATCH 12/39] Move python to build for win. Disable plasma test for cuda enabled builds --- recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 759695d..d4a96f6 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -38,6 +38,7 @@ outputs: - cmake - ninja - make # [unix] + - python # [win] - {{ compiler('c') }} - {{ compiler('cxx') }} # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda @@ -47,7 +48,6 @@ outputs: # in the Python modules. - arrow-cpp {{ version }} *_{{ build_ext }} - boost-cpp - - python - cython - numpy >=1.16 - python @@ -78,7 +78,7 @@ outputs: - pyarrow.gandiva # [(not py==27) and (not (aarch64 or ppc64le))] - pyarrow.orc # [unix] - pyarrow.parquet - - pyarrow.plasma # [unix] + - pyarrow.plasma # [unix and cuda_compiler_version == "None"] - pyarrow.fs - pyarrow._s3fs - pyarrow._hdfs From 780e457faba4a07433bde7277d27ab5cfa03d3ac Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 14:42:22 +0300 Subject: [PATCH 13/39] Disable plasma and cuda import tests for cuda enabled builds. --- recipe/meta.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index d4a96f6..e38b3e2 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -78,11 +78,12 @@ outputs: - pyarrow.gandiva # [(not py==27) and (not (aarch64 or ppc64le))] - pyarrow.orc # [unix] - pyarrow.parquet - - pyarrow.plasma # [unix and cuda_compiler_version == "None"] - pyarrow.fs - pyarrow._s3fs - pyarrow._hdfs - {{ "- pyarrow.cuda" if cuda_enabled else "" }} + # Importing pyarrow.plasma and pyarrow.cuda packages require libcuda.so.1 + - pyarrow.plasma # [unix and cuda_compiler_version == "None"] + # {{ "- pyarrow.cuda" if cuda_enabled else "" }} about: home: http://github.com/apache/arrow From c20e410fdc4d9ba8a900766cfeecb67883f1d295 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 15:00:58 +0300 Subject: [PATCH 14/39] Try python instead of %PYTHON% in bld.bat --- recipe/bld.bat | 2 +- recipe/meta.yaml | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/recipe/bld.bat b/recipe/bld.bat index dbea193..443055f 100644 --- a/recipe/bld.bat +++ b/recipe/bld.bat @@ -23,7 +23,7 @@ SET PYARROW_WITH_GANDIVA=1 SET PYARROW_WITH_PARQUET=1 SET PYARROW_CMAKE_GENERATOR=Ninja -"%PYTHON%" setup.py ^ +python setup.py ^ build_ext ^ install --single-version-externally-managed ^ --record=record.txt diff --git a/recipe/meta.yaml b/recipe/meta.yaml index e38b3e2..5b15266 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -38,7 +38,6 @@ outputs: - cmake - ninja - make # [unix] - - python # [win] - {{ compiler('c') }} - {{ compiler('cxx') }} # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda @@ -68,7 +67,6 @@ outputs: - snappy run_constrained: - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 test: imports: @@ -81,9 +79,9 @@ outputs: - pyarrow.fs - pyarrow._s3fs - pyarrow._hdfs - # Importing pyarrow.plasma and pyarrow.cuda packages require libcuda.so.1 - - pyarrow.plasma # [unix and cuda_compiler_version == "None"] - # {{ "- pyarrow.cuda" if cuda_enabled else "" }} + # When using cuda, importing pyarrow.plasma and pyarrow.cuda + # packages require libcuda.so.1 that is not available in CI + - pyarrow.plasma # [unix and cuda_compiler_version in (undefined, "None")] about: home: http://github.com/apache/arrow From ed8d21c1db9ee187cf6dd9786a6dcf0adbe52987 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 15:20:55 +0300 Subject: [PATCH 15/39] Restore %PYTHON% --- recipe/bld.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/bld.bat b/recipe/bld.bat index 443055f..8b4f48e 100644 --- a/recipe/bld.bat +++ b/recipe/bld.bat @@ -23,7 +23,7 @@ SET PYARROW_WITH_GANDIVA=1 SET PYARROW_WITH_PARQUET=1 SET PYARROW_CMAKE_GENERATOR=Ninja -python setup.py ^ +%PYTHON% setup.py ^ build_ext ^ install --single-version-externally-managed ^ --record=record.txt From 3e5a10ddc1c544caa6ec9384a85f17df5d7fa91f Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 15:45:34 +0300 Subject: [PATCH 16/39] Enable activate_in_script --- recipe/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 5b15266..8828a45 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -33,6 +33,8 @@ outputs: skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] track_features: {{ "- arrow-cuda" if cuda_enabled else "" }} + activate_in_script: True + requirements: build: - cmake From 945291710739e9c087c989df287121da5f3d6bd1 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 17:54:02 +0300 Subject: [PATCH 17/39] Rely on arrow-cpp-proc for selecting cuda or cpu builds. --- recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 8828a45..2289db6 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -47,7 +47,7 @@ outputs: host: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. - - arrow-cpp {{ version }} *_{{ build_ext }} + - arrow-cpp {{ version }} - boost-cpp - cython - numpy >=1.16 @@ -57,7 +57,7 @@ outputs: - six run: - {{ pin_compatible('numpy', lower_bound='1.16') }} - - arrow-cpp {{ version }} *_{{ build_ext }} + - arrow-cpp {{ version }} - boost-cpp - futures # [py==27] - glog From 1f05ba56c7072e7456a3188791dd33273cd07eaf Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 19:12:06 +0300 Subject: [PATCH 18/39] Restrict arrow-cpp with build_ext in host --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 2289db6..c3a10ea 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -47,7 +47,7 @@ outputs: host: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. - - arrow-cpp {{ version }} + - arrow-cpp {{ version }} *_{{ build_ext }} - boost-cpp - cython - numpy >=1.16 From ae69751fd42068a15d384fe3082ef2f925d54cf1 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 20:11:33 +0300 Subject: [PATCH 19/39] Use arrow-cpp-proc in host to restrict to build_ext --- recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c3a10ea..2e9c3ef 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -47,7 +47,8 @@ outputs: host: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. - - arrow-cpp {{ version }} *_{{ build_ext }} + - arrow-cpp {{ version }} + - arrow-cpp-proc * {{ build_ext }} - boost-cpp - cython - numpy >=1.16 @@ -57,7 +58,6 @@ outputs: - six run: - {{ pin_compatible('numpy', lower_bound='1.16') }} - - arrow-cpp {{ version }} - boost-cpp - futures # [py==27] - glog From f5d5bfa871d7bbd9801b14d1526ae1a9658cc860 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Tue, 28 Apr 2020 20:49:21 +0300 Subject: [PATCH 20/39] Restore arrow-cpp in run. --- recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 2e9c3ef..4a1c901 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -59,6 +59,7 @@ outputs: run: - {{ pin_compatible('numpy', lower_bound='1.16') }} - boost-cpp + - arrow-cpp {{ version }} - futures # [py==27] - glog - pandas From 52a2eded0ea88546b233a16eccef7cebd04ac146 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Thu, 30 Apr 2020 23:13:32 +0300 Subject: [PATCH 21/39] Test pyarrow.cuda - a temporary workaround. --- recipe/build-pyarrow.sh | 17 ++++++++++++++++- recipe/meta.yaml | 9 ++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/recipe/build-pyarrow.sh b/recipe/build-pyarrow.sh index b09898b..7d24558 100644 --- a/recipe/build-pyarrow.sh +++ b/recipe/build-pyarrow.sh @@ -10,7 +10,8 @@ export SETUPTOOLS_SCM_PRETEND_VERSION=$PKG_VERSION export PYARROW_BUILD_TYPE=release export PYARROW_WITH_DATASET=1 export PYARROW_WITH_FLIGHT=1 -if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then +if [[ "$(uname -m)" = "ppc64le" || "$(uname -m)" = "aarch64" ]] +then export PYARROW_WITH_GANDIVA=0 else export PYARROW_WITH_GANDIVA=1 @@ -37,3 +38,17 @@ $PYTHON setup.py \ build_ext $BUILD_EXT_FLAGS \ install --single-version-externally-managed \ --record=record.txt + +# Test CUDA support +if [[ "$PYARROW_WITH_CUDA" = "1" ]] +then + # move out from pyarrow source directory + mkdir tmp-test + cd tmp-test + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-compat-10-0_410.48-1_amd64.deb + ar x cuda-compat-10-0_410.48-1_amd64.deb + tar xvf data.tar.xz + export LD_LIBRARY_PATH=usr/local/cuda-10.0/compat/ + $PYTHON -c "import pyarrow.cuda" + $PYTHON -c "import pyarrow.plasma" +fi diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 4a1c901..aaa4021 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -15,8 +15,6 @@ source: sha256: {{ checksum }} build: - skip: true # [win32] - skip: true # [win and py<35] skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] number: {{ number }} @@ -28,8 +26,6 @@ outputs: build: number: {{ number }} string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" - skip: true # [win32] - skip: true # [win and py<35] skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] track_features: {{ "- arrow-cuda" if cuda_enabled else "" }} @@ -37,6 +33,7 @@ outputs: requirements: build: + - wget - cmake - ninja - make # [unix] @@ -82,10 +79,8 @@ outputs: - pyarrow.fs - pyarrow._s3fs - pyarrow._hdfs - # When using cuda, importing pyarrow.plasma and pyarrow.cuda - # packages require libcuda.so.1 that is not available in CI - pyarrow.plasma # [unix and cuda_compiler_version in (undefined, "None")] - + # CUDA dependent pyarrow.plasma and pyarrow.cuda are tested in build-arrow.sh about: home: http://github.com/apache/arrow license: Apache-2.0 From aa1184488a0926260f5378699bee7aac33b34adc Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Thu, 30 Apr 2020 23:20:21 +0300 Subject: [PATCH 22/39] Restrict wget for cuda enabled builds. --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index aaa4021..586719b 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -33,7 +33,7 @@ outputs: requirements: build: - - wget + - wget # [cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] - cmake - ninja - make # [unix] From 8d3e0a7854dd8d61ca58f2581fea1930f6575ef4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 14:09:41 +0200 Subject: [PATCH 23/39] remove outputs section this requires renaming build-pyarrow.sh back to build.sh --- recipe/{build-pyarrow.sh => build.sh} | 0 recipe/meta.yaml | 124 ++++++++++++-------------- 2 files changed, 55 insertions(+), 69 deletions(-) rename recipe/{build-pyarrow.sh => build.sh} (100%) diff --git a/recipe/build-pyarrow.sh b/recipe/build.sh similarity index 100% rename from recipe/build-pyarrow.sh rename to recipe/build.sh diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 586719b..01e7342 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -6,7 +6,7 @@ {% set build_ext = "cuda" if cuda_enabled else "cpu" %} package: - name: pyarrow-ext + name: pyarrow version: {{ version }} source: @@ -15,77 +15,63 @@ source: sha256: {{ checksum }} build: - skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] number: {{ number }} + string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" + skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] + track_features: + {{ "- arrow-cuda" if cuda_enabled else "" }} + activate_in_script: True -outputs: - - name: pyarrow - script: build-pyarrow.sh # [not win] - script: bld.bat # [win] - version: {{ version }} - build: - number: {{ number }} - string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" - skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] - track_features: - {{ "- arrow-cuda" if cuda_enabled else "" }} - activate_in_script: True +requirements: + build: + - wget # [cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] + - cmake + - ninja + - make # [unix] + - {{ compiler('c') }} + - {{ compiler('cxx') }} + # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda + - {{ compiler("cuda") }} # [cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] + host: + # directly pin boost-cpp as we also seem to directly include boost symbols + # in the Python modules. + - arrow-cpp {{ version }} + - arrow-cpp-proc * {{ build_ext }} + - boost-cpp + - cython + - numpy >=1.16 + - python + - setuptools + - setuptools_scm + - six + run: + - {{ pin_compatible('numpy', lower_bound='1.16') }} + - boost-cpp + - arrow-cpp {{ version }} + - futures # [py==27] + - glog + - pandas + # empty parquet-cpp metapackage, force old versions to be uninstalled + - parquet-cpp 1.5.1.* + - python + - six + - snappy + run_constrained: + - arrow-cpp-proc * {{ build_ext }} - requirements: - build: - - wget # [cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] - - cmake - - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] - host: - # directly pin boost-cpp as we also seem to directly include boost symbols - # in the Python modules. - - arrow-cpp {{ version }} - - arrow-cpp-proc * {{ build_ext }} - - boost-cpp - - cython - - numpy >=1.16 - - python - - setuptools - - setuptools_scm - - six - run: - - {{ pin_compatible('numpy', lower_bound='1.16') }} - - boost-cpp - - arrow-cpp {{ version }} - - futures # [py==27] - - glog - - pandas - # empty parquet-cpp metapackage, force old versions to be uninstalled - - parquet-cpp 1.5.1.* - - python - - six - - snappy - run_constrained: - - arrow-cpp-proc * {{ build_ext }} - - test: - imports: - - pyarrow - - pyarrow.dataset # [not py==27] - - pyarrow.flight # [not py==27] - - pyarrow.gandiva # [(not py==27) and (not (aarch64 or ppc64le))] - - pyarrow.orc # [unix] - - pyarrow.parquet - - pyarrow.fs - - pyarrow._s3fs - - pyarrow._hdfs - - pyarrow.plasma # [unix and cuda_compiler_version in (undefined, "None")] - # CUDA dependent pyarrow.plasma and pyarrow.cuda are tested in build-arrow.sh - about: - home: http://github.com/apache/arrow - license: Apache-2.0 - license_file: LICENSE.txt - summary: 'Python libraries for Apache Arrow' +test: + imports: + - pyarrow + - pyarrow.dataset # [not py==27] + - pyarrow.flight # [not py==27] + - pyarrow.gandiva # [(not py==27) and (not (aarch64 or ppc64le))] + - pyarrow.orc # [unix] + - pyarrow.parquet + - pyarrow.fs + - pyarrow._s3fs + - pyarrow._hdfs + - pyarrow.plasma # [unix and cuda_compiler_version in (undefined, "None")] + # CUDA dependent pyarrow.plasma and pyarrow.cuda are tested in build-arrow.sh about: home: http://github.com/apache/arrow From 3b950a9b35a3399e976e4e7deaed7bd2564fc0fb Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 14:13:44 +0200 Subject: [PATCH 24/39] move testing for cuda-build from build.sh to test-section in recipe --- recipe/build.sh | 14 -------------- recipe/meta.yaml | 4 ++-- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/recipe/build.sh b/recipe/build.sh index 7d24558..fa971db 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -38,17 +38,3 @@ $PYTHON setup.py \ build_ext $BUILD_EXT_FLAGS \ install --single-version-externally-managed \ --record=record.txt - -# Test CUDA support -if [[ "$PYARROW_WITH_CUDA" = "1" ]] -then - # move out from pyarrow source directory - mkdir tmp-test - cd tmp-test - wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-compat-10-0_410.48-1_amd64.deb - ar x cuda-compat-10-0_410.48-1_amd64.deb - tar xvf data.tar.xz - export LD_LIBRARY_PATH=usr/local/cuda-10.0/compat/ - $PYTHON -c "import pyarrow.cuda" - $PYTHON -c "import pyarrow.plasma" -fi diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 01e7342..bcb6c55 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -67,11 +67,11 @@ test: - pyarrow.gandiva # [(not py==27) and (not (aarch64 or ppc64le))] - pyarrow.orc # [unix] - pyarrow.parquet + - pyarrow.plasma # [unix] - pyarrow.fs - pyarrow._s3fs - pyarrow._hdfs - - pyarrow.plasma # [unix and cuda_compiler_version in (undefined, "None")] - # CUDA dependent pyarrow.plasma and pyarrow.cuda are tested in build-arrow.sh + - pyarrow.cuda # [linux and cuda_compiler_version != "None"] about: home: http://github.com/apache/arrow From aabb09b940cca01b4bc359ce8c0b6f2d95a1c382 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 14:21:07 +0200 Subject: [PATCH 25/39] add some cuda-relevant parts of the test-suite --- recipe/meta.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index bcb6c55..d7c5f0f 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -71,7 +71,15 @@ test: - pyarrow.fs - pyarrow._s3fs - pyarrow._hdfs - - pyarrow.cuda # [linux and cuda_compiler_version != "None"] + - pyarrow.cuda # [linux and cuda_compiler_version != "None"] + requires: # [linux and cuda_compiler_version != "None"] + - pytest # [linux and cuda_compiler_version != "None"] + - numba # [linux and cuda_compiler_version != "None"] + source_files: # [linux and cuda_compiler_version != "None"] + - python/pyarrow/tests/ # [linux and cuda_compiler_version != "None"] + commands: # [linux and cuda_compiler_version != "None"] + - python python/pyarrow/tests/test_cuda.py # [linux and cuda_compiler_version != "None"] + - python python/pyarrow/tests/test_cuda_numba_interop.py # [linux and cuda_compiler_version != "None"] about: home: http://github.com/apache/arrow From 99fbb55ef702bdf1cea439559c62e44fd7c0f59c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 14:26:12 +0200 Subject: [PATCH 26/39] Revert "Restore arrow-cpp in run." This reverts commit f5d5bfa871d7bbd9801b14d1526ae1a9658cc860. --- recipe/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index d7c5f0f..b809ed0 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -47,7 +47,6 @@ requirements: run: - {{ pin_compatible('numpy', lower_bound='1.16') }} - boost-cpp - - arrow-cpp {{ version }} - futures # [py==27] - glog - pandas From 7899377d728fa70948b1d20411c01e58a88c8ffc Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 14:27:08 +0200 Subject: [PATCH 27/39] minor cleanups (that also reduce diff to 0.16.x-branch) --- recipe/meta.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index b809ed0..f15923a 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -24,10 +24,9 @@ build: requirements: build: - - wget # [cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] - cmake - ninja - - make # [unix] + - make # [unix] - {{ compiler('c') }} - {{ compiler('cxx') }} # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda @@ -39,7 +38,7 @@ requirements: - arrow-cpp-proc * {{ build_ext }} - boost-cpp - cython - - numpy >=1.16 + - numpy 1.16.* - python - setuptools - setuptools_scm From c9bab51e187463a3b0f153b1b8d5aa2b2febb1a3 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 14:29:58 +0200 Subject: [PATCH 28/39] simplify requirement for arrow-cpp-dependency --- recipe/meta.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index f15923a..b564c6e 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -34,8 +34,7 @@ requirements: host: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. - - arrow-cpp {{ version }} - - arrow-cpp-proc * {{ build_ext }} + - arrow-cpp ={{ version }}=*{{ build_ext }} - boost-cpp - cython - numpy 1.16.* From 637415915937c62c75378da7dd0210bad40361f3 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 14:34:15 +0200 Subject: [PATCH 29/39] enable building for cuda 10.x --- recipe/meta.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index b564c6e..0a40dce 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -2,7 +2,7 @@ {% set filename = "apache-arrow-" + version + ".tar.gz" %} {% set checksum = "261992de4029a1593195ff4000501503bd403146471b3168bd2cc414ad0fb7f5" %} {% set number = "3" %} -{% set cuda_enabled = cuda_compiler_version is not undefined and cuda_compiler_version == '9.2' %} +{% set cuda_enabled = cuda_compiler_version is not undefined and cuda_compiler_version != "None" %} {% set build_ext = "cuda" if cuda_enabled else "cpu" %} package: @@ -17,7 +17,6 @@ source: build: number: {{ number }} string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" - skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] track_features: {{ "- arrow-cuda" if cuda_enabled else "" }} activate_in_script: True @@ -30,7 +29,7 @@ requirements: - {{ compiler('c') }} - {{ compiler('cxx') }} # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] + - {{ compiler("cuda") }} # [cuda_compiler_version is not undefined and cuda_compiler_version != "None"] host: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. From bec5f77d11bdc66e5a917cc794a5347e944ede6c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 14:52:19 +0200 Subject: [PATCH 30/39] remove activate_in_script --- recipe/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 0a40dce..d75750e 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -19,7 +19,6 @@ build: string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" track_features: {{ "- arrow-cuda" if cuda_enabled else "" }} - activate_in_script: True requirements: build: From a3ef86dad8aeb96baad6839575eb7822c5fb51eb Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 15:00:12 +0200 Subject: [PATCH 31/39] MNT: Re-rendered with conda-build 3.19.2, conda-smithy 3.7.1, and conda-forge-pinning 2020.05.14.17.14.20 --- .azure-pipelines/azure-pipelines-linux.yml | 77 +++++++++++++----- .azure-pipelines/azure-pipelines-osx.yml | 10 +-- .azure-pipelines/azure-pipelines-win.yml | 21 ++--- ...iler_version10.0python3.6.____cpython.yaml | 31 +++++++ ...iler_version10.0python3.7.____cpython.yaml | 31 +++++++ ...iler_version10.0python3.8.____cpython.yaml | 31 +++++++ ...iler_version10.1python3.6.____cpython.yaml | 31 +++++++ ...iler_version10.1python3.7.____cpython.yaml | 31 +++++++ ...iler_version10.1python3.8.____cpython.yaml | 31 +++++++ ...iler_version10.2python3.6.____cpython.yaml | 31 +++++++ ...iler_version10.2python3.7.____cpython.yaml | 31 +++++++ ...iler_version10.2python3.8.____cpython.yaml | 31 +++++++ .gitattributes | 2 +- .scripts/build_steps.sh | 3 +- .scripts/run_docker_build.sh | 2 +- README.md | 81 ++++++++++++++++--- 16 files changed, 424 insertions(+), 51 deletions(-) create mode 100644 .ci_support/linux_cuda_compiler_version10.0python3.6.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version10.0python3.7.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version10.0python3.8.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version10.1python3.6.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version10.1python3.7.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version10.1python3.8.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version10.2python3.6.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version10.2python3.7.____cpython.yaml create mode 100644 .ci_support/linux_cuda_compiler_version10.2python3.8.____cpython.yaml diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index 4681384..8b9a01a 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -6,46 +6,83 @@ jobs: - job: linux pool: vmImage: ubuntu-16.04 - timeoutInMinutes: 360 strategy: - maxParallel: 8 matrix: + linux_aarch64_python3.6.____cpython: + CONFIG: linux_aarch64_python3.6.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-aarch64 + linux_aarch64_python3.7.____cpython: + CONFIG: linux_aarch64_python3.7.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-aarch64 + linux_aarch64_python3.8.____cpython: + CONFIG: linux_aarch64_python3.8.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-aarch64 + linux_cuda_compiler_version10.0python3.6.____cpython: + CONFIG: linux_cuda_compiler_version10.0python3.6.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.0 + linux_cuda_compiler_version10.0python3.7.____cpython: + CONFIG: linux_cuda_compiler_version10.0python3.7.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.0 + linux_cuda_compiler_version10.0python3.8.____cpython: + CONFIG: linux_cuda_compiler_version10.0python3.8.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.0 + linux_cuda_compiler_version10.1python3.6.____cpython: + CONFIG: linux_cuda_compiler_version10.1python3.6.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.1 + linux_cuda_compiler_version10.1python3.7.____cpython: + CONFIG: linux_cuda_compiler_version10.1python3.7.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.1 + linux_cuda_compiler_version10.1python3.8.____cpython: + CONFIG: linux_cuda_compiler_version10.1python3.8.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.1 + linux_cuda_compiler_version10.2python3.6.____cpython: + CONFIG: linux_cuda_compiler_version10.2python3.6.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.2 + linux_cuda_compiler_version10.2python3.7.____cpython: + CONFIG: linux_cuda_compiler_version10.2python3.7.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.2 + linux_cuda_compiler_version10.2python3.8.____cpython: + CONFIG: linux_cuda_compiler_version10.2python3.8.____cpython + UPLOAD_PACKAGES: 'True' + DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.2 linux_cuda_compiler_version9.2python3.6.____cpython: CONFIG: linux_cuda_compiler_version9.2python3.6.____cpython - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: condaforge/linux-anvil-cuda:9.2 linux_cuda_compiler_version9.2python3.7.____cpython: CONFIG: linux_cuda_compiler_version9.2python3.7.____cpython - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: condaforge/linux-anvil-cuda:9.2 linux_cuda_compiler_version9.2python3.8.____cpython: CONFIG: linux_cuda_compiler_version9.2python3.8.____cpython - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: condaforge/linux-anvil-cuda:9.2 linux_cuda_compiler_versionNonepython3.6.____cpython: CONFIG: linux_cuda_compiler_versionNonepython3.6.____cpython - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: condaforge/linux-anvil-comp7 linux_cuda_compiler_versionNonepython3.7.____cpython: CONFIG: linux_cuda_compiler_versionNonepython3.7.____cpython - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: condaforge/linux-anvil-comp7 linux_cuda_compiler_versionNonepython3.8.____cpython: CONFIG: linux_cuda_compiler_versionNonepython3.8.____cpython - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: condaforge/linux-anvil-comp7 - linux_aarch64_python3.6.____cpython: - CONFIG: linux_aarch64_python3.6.____cpython - UPLOAD_PACKAGES: True - DOCKER_IMAGE: condaforge/linux-anvil-aarch64 - linux_aarch64_python3.7.____cpython: - CONFIG: linux_aarch64_python3.7.____cpython - UPLOAD_PACKAGES: True - DOCKER_IMAGE: condaforge/linux-anvil-aarch64 - linux_aarch64_python3.8.____cpython: - CONFIG: linux_aarch64_python3.8.____cpython - UPLOAD_PACKAGES: True - DOCKER_IMAGE: condaforge/linux-anvil-aarch64 + maxParallel: 8 + timeoutInMinutes: 360 + steps: # configure qemu binfmt-misc running. This allows us to run docker containers # embedded qemu-static diff --git a/.azure-pipelines/azure-pipelines-osx.yml b/.azure-pipelines/azure-pipelines-osx.yml index 65502e5..3fe530e 100755 --- a/.azure-pipelines/azure-pipelines-osx.yml +++ b/.azure-pipelines/azure-pipelines-osx.yml @@ -6,19 +6,19 @@ jobs: - job: osx pool: vmImage: macOS-10.14 - timeoutInMinutes: 360 strategy: - maxParallel: 8 matrix: osx_python3.6.____cpython: CONFIG: osx_python3.6.____cpython - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' osx_python3.7.____cpython: CONFIG: osx_python3.7.____cpython - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' osx_python3.8.____cpython: CONFIG: osx_python3.8.____cpython - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' + maxParallel: 8 + timeoutInMinutes: 360 steps: # TODO: Fast finish on azure pipelines? diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index b0a337d..86024b1 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -6,28 +6,23 @@ jobs: - job: win pool: vmImage: vs2017-win2016 - timeoutInMinutes: 360 strategy: - maxParallel: 4 matrix: win_python3.6.____cpython: CONFIG: win_python3.6.____cpython - CONDA_BLD_PATH: D:\\bld\\ - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' win_python3.7.____cpython: CONFIG: win_python3.7.____cpython - CONDA_BLD_PATH: D:\\bld\\ - UPLOAD_PACKAGES: True + UPLOAD_PACKAGES: 'True' win_python3.8.____cpython: CONFIG: win_python3.8.____cpython - CONDA_BLD_PATH: D:\\bld\\ - UPLOAD_PACKAGES: True - steps: - # TODO: Fast finish on azure pipelines? - - script: | - ECHO ON - + UPLOAD_PACKAGES: 'True' + maxParallel: 4 + timeoutInMinutes: 360 + variables: + CONDA_BLD_PATH: D:\\bld\\ + steps: - script: | choco install vcpython27 -fdv -y --debug condition: contains(variables['CONFIG'], 'vs2008') diff --git a/.ci_support/linux_cuda_compiler_version10.0python3.6.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.0python3.6.____cpython.yaml new file mode 100644 index 0000000..632f198 --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version10.0python3.6.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.0' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:10.0 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.0python3.7.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.0python3.7.____cpython.yaml new file mode 100644 index 0000000..0393d7f --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version10.0python3.7.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.0' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:10.0 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.7.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.0python3.8.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.0python3.8.____cpython.yaml new file mode 100644 index 0000000..bf59d15 --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version10.0python3.8.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.0' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:10.0 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.1python3.6.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.1python3.6.____cpython.yaml new file mode 100644 index 0000000..d7e33d7 --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version10.1python3.6.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.1' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:10.1 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.1python3.7.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.1python3.7.____cpython.yaml new file mode 100644 index 0000000..0707ac4 --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version10.1python3.7.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.1' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:10.1 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.7.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.1python3.8.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.1python3.8.____cpython.yaml new file mode 100644 index 0000000..af3ad5d --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version10.1python3.8.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.1' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:10.1 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.2python3.6.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.2python3.6.____cpython.yaml new file mode 100644 index 0000000..73c6037 --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version10.2python3.6.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:10.2 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.2python3.7.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.2python3.7.____cpython.yaml new file mode 100644 index 0000000..417becf --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version10.2python3.7.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:10.2 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.7.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.2python3.8.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.2python3.8.____cpython.yaml new file mode 100644 index 0000000..d3ac74f --- /dev/null +++ b/.ci_support/linux_cuda_compiler_version10.2python3.8.____cpython.yaml @@ -0,0 +1,31 @@ +boost_cpp: +- '1.72' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:10.2 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython +zip_keys: +- - cuda_compiler_version + - docker_image diff --git a/.gitattributes b/.gitattributes index ac943c1..9060b27 100644 --- a/.gitattributes +++ b/.gitattributes @@ -17,7 +17,7 @@ bld.bat text eol=crlf .gitattributes linguist-generated=true .gitignore linguist-generated=true .travis.yml linguist-generated=true -.scripts linguist-generated=true +.scripts/* linguist-generated=true LICENSE.txt linguist-generated=true README.md linguist-generated=true azure-pipelines.yml linguist-generated=true diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 5b14787..bef68a0 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -21,7 +21,6 @@ CONDARC conda install --yes --quiet conda-forge-ci-setup=3 conda-build pip -c conda-forge - # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" @@ -34,7 +33,7 @@ conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" + upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" fi touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" \ No newline at end of file diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 253723c..2e4fccc 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -60,7 +60,7 @@ fi export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ - -v "${RECIPE_ROOT}":/home/conda/recipe_root:ro,z \ + -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z \ -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z \ -e CONFIG \ -e BINSTAR_TOKEN \ diff --git a/README.md b/README.md index b0fcd5f..b65c9aa 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -About pyarrow-ext -================= +About pyarrow +============= Home: http://github.com/apache/arrow @@ -56,6 +56,69 @@ Current build status variant + + linux_cuda_compiler_version10.0python3.6.____cpython + + + variant + + + + linux_cuda_compiler_version10.0python3.7.____cpython + + + variant + + + + linux_cuda_compiler_version10.0python3.8.____cpython + + + variant + + + + linux_cuda_compiler_version10.1python3.6.____cpython + + + variant + + + + linux_cuda_compiler_version10.1python3.7.____cpython + + + variant + + + + linux_cuda_compiler_version10.1python3.8.____cpython + + + variant + + + + linux_cuda_compiler_version10.2python3.6.____cpython + + + variant + + + + linux_cuda_compiler_version10.2python3.7.____cpython + + + variant + + + + linux_cuda_compiler_version10.2python3.8.____cpython + + + variant + + linux_cuda_compiler_version9.2python3.6.____cpython @@ -176,10 +239,10 @@ Current release info | --- | --- | --- | --- | | [![Conda Recipe](https://img.shields.io/badge/recipe-pyarrow-green.svg)](https://anaconda.org/conda-forge/pyarrow) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/pyarrow.svg)](https://anaconda.org/conda-forge/pyarrow) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/pyarrow.svg)](https://anaconda.org/conda-forge/pyarrow) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/pyarrow.svg)](https://anaconda.org/conda-forge/pyarrow) | -Installing pyarrow-ext -====================== +Installing pyarrow +================== -Installing `pyarrow-ext` from the `conda-forge` channel can be achieved by adding `conda-forge` to your channels with: +Installing `pyarrow` from the `conda-forge` channel can be achieved by adding `conda-forge` to your channels with: ``` conda config --add channels conda-forge @@ -236,17 +299,17 @@ Terminology produce the finished article (built conda distributions) -Updating pyarrow-ext-feedstock -============================== +Updating pyarrow-feedstock +========================== -If you would like to improve the pyarrow-ext recipe or build a new +If you would like to improve the pyarrow recipe or build a new package version, please fork this repository and submit a PR. Upon submission, your changes will be run on the appropriate platforms to give the reviewer an opportunity to confirm that the changes result in a successful build. Once merged, the recipe will be re-built and uploaded automatically to the `conda-forge` channel, whereupon the built conda packages will be available for everybody to install and use from the `conda-forge` channel. -Note that all branches in the conda-forge/pyarrow-ext-feedstock are +Note that all branches in the conda-forge/pyarrow-feedstock are immediately built and any created packages are uploaded, so PRs should be based on branches in forks and branches in the main repository should only be used to build distinct package versions. From 2a7c224dd00b2654762675f3b44a3b285984499d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 15:23:39 +0200 Subject: [PATCH 32/39] Re-add arrow-cpp to runtime requirements Partially revert "Revert "Restore arrow-cpp in run."" This partially reverts commit 99fbb55ef702bdf1cea439559c62e44fd7c0f59c. --- recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index d75750e..13140d8 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -43,6 +43,7 @@ requirements: run: - {{ pin_compatible('numpy', lower_bound='1.16') }} - boost-cpp + - arrow-cpp ={{ version }}=*{{ build_ext }} - futures # [py==27] - glog - pandas From 13fc3131def8d5428eec15ce64a9ea84cc4c969b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 15:33:58 +0200 Subject: [PATCH 33/39] remove six as dependency --- recipe/meta.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 13140d8..dcc1712 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -39,7 +39,6 @@ requirements: - python - setuptools - setuptools_scm - - six run: - {{ pin_compatible('numpy', lower_bound='1.16') }} - boost-cpp @@ -50,7 +49,6 @@ requirements: # empty parquet-cpp metapackage, force old versions to be uninstalled - parquet-cpp 1.5.1.* - python - - six - snappy run_constrained: - arrow-cpp-proc * {{ build_ext }} From faa277b66005c7293b8686bb84c2bcb12236574f Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 15:37:48 +0200 Subject: [PATCH 34/39] ensure cuda-tests are only run on linux64 --- recipe/meta.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index dcc1712..4c3d091 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -28,7 +28,7 @@ requirements: - {{ compiler('c') }} - {{ compiler('cxx') }} # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version is not undefined and cuda_compiler_version != "None"] + - {{ compiler("cuda") }} # [cuda_compiler_version not in (undefined, "None")] host: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. @@ -65,15 +65,15 @@ test: - pyarrow.fs - pyarrow._s3fs - pyarrow._hdfs - - pyarrow.cuda # [linux and cuda_compiler_version != "None"] - requires: # [linux and cuda_compiler_version != "None"] - - pytest # [linux and cuda_compiler_version != "None"] - - numba # [linux and cuda_compiler_version != "None"] - source_files: # [linux and cuda_compiler_version != "None"] - - python/pyarrow/tests/ # [linux and cuda_compiler_version != "None"] - commands: # [linux and cuda_compiler_version != "None"] - - python python/pyarrow/tests/test_cuda.py # [linux and cuda_compiler_version != "None"] - - python python/pyarrow/tests/test_cuda_numba_interop.py # [linux and cuda_compiler_version != "None"] + - pyarrow.cuda # [cuda_compiler_version not in (undefined, "None")] + requires: # [cuda_compiler_version not in (undefined, "None")] + - pytest # [cuda_compiler_version not in (undefined, "None")] + - numba # [cuda_compiler_version not in (undefined, "None")] + source_files: # [cuda_compiler_version not in (undefined, "None")] + - python/pyarrow/tests/ # [cuda_compiler_version not in (undefined, "None")] + commands: # [cuda_compiler_version not in (undefined, "None")] + - python python/pyarrow/tests/test_cuda.py # [cuda_compiler_version not in (undefined, "None")] + - python python/pyarrow/tests/test_cuda_numba_interop.py # [cuda_compiler_version not in (undefined, "None")] about: home: http://github.com/apache/arrow From a77e93f8880f536b693b4143a182870421de6da0 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 15:55:35 +0200 Subject: [PATCH 35/39] Revert "enable building for cuda 10.x" This (partially) reverts commit 637415915937c62c75378da7dd0210bad40361f3. --- recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 4c3d091..0a03a82 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -17,6 +17,7 @@ source: build: number: {{ number }} string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" + skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] track_features: {{ "- arrow-cuda" if cuda_enabled else "" }} From 1c9916276ea90ea4ad1274f0dd9207b187af237b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 15 May 2020 16:02:29 +0200 Subject: [PATCH 36/39] MNT: Re-rendered with conda-build 3.19.2, conda-smithy 3.7.1, and conda-forge-pinning 2020.05.14.17.14.20 --- .azure-pipelines/azure-pipelines-linux.yml | 36 ----------- ...iler_version10.0python3.6.____cpython.yaml | 31 --------- ...iler_version10.0python3.7.____cpython.yaml | 31 --------- ...iler_version10.0python3.8.____cpython.yaml | 31 --------- ...iler_version10.1python3.6.____cpython.yaml | 31 --------- ...iler_version10.1python3.7.____cpython.yaml | 31 --------- ...iler_version10.1python3.8.____cpython.yaml | 31 --------- ...iler_version10.2python3.6.____cpython.yaml | 31 --------- ...iler_version10.2python3.7.____cpython.yaml | 31 --------- ...iler_version10.2python3.8.____cpython.yaml | 31 --------- README.md | 63 ------------------- 11 files changed, 378 deletions(-) delete mode 100644 .ci_support/linux_cuda_compiler_version10.0python3.6.____cpython.yaml delete mode 100644 .ci_support/linux_cuda_compiler_version10.0python3.7.____cpython.yaml delete mode 100644 .ci_support/linux_cuda_compiler_version10.0python3.8.____cpython.yaml delete mode 100644 .ci_support/linux_cuda_compiler_version10.1python3.6.____cpython.yaml delete mode 100644 .ci_support/linux_cuda_compiler_version10.1python3.7.____cpython.yaml delete mode 100644 .ci_support/linux_cuda_compiler_version10.1python3.8.____cpython.yaml delete mode 100644 .ci_support/linux_cuda_compiler_version10.2python3.6.____cpython.yaml delete mode 100644 .ci_support/linux_cuda_compiler_version10.2python3.7.____cpython.yaml delete mode 100644 .ci_support/linux_cuda_compiler_version10.2python3.8.____cpython.yaml diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index 8b9a01a..4226a25 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -20,42 +20,6 @@ jobs: CONFIG: linux_aarch64_python3.8.____cpython UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: condaforge/linux-anvil-aarch64 - linux_cuda_compiler_version10.0python3.6.____cpython: - CONFIG: linux_cuda_compiler_version10.0python3.6.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.0 - linux_cuda_compiler_version10.0python3.7.____cpython: - CONFIG: linux_cuda_compiler_version10.0python3.7.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.0 - linux_cuda_compiler_version10.0python3.8.____cpython: - CONFIG: linux_cuda_compiler_version10.0python3.8.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.0 - linux_cuda_compiler_version10.1python3.6.____cpython: - CONFIG: linux_cuda_compiler_version10.1python3.6.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.1 - linux_cuda_compiler_version10.1python3.7.____cpython: - CONFIG: linux_cuda_compiler_version10.1python3.7.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.1 - linux_cuda_compiler_version10.1python3.8.____cpython: - CONFIG: linux_cuda_compiler_version10.1python3.8.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.1 - linux_cuda_compiler_version10.2python3.6.____cpython: - CONFIG: linux_cuda_compiler_version10.2python3.6.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.2 - linux_cuda_compiler_version10.2python3.7.____cpython: - CONFIG: linux_cuda_compiler_version10.2python3.7.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.2 - linux_cuda_compiler_version10.2python3.8.____cpython: - CONFIG: linux_cuda_compiler_version10.2python3.8.____cpython - UPLOAD_PACKAGES: 'True' - DOCKER_IMAGE: condaforge/linux-anvil-cuda:10.2 linux_cuda_compiler_version9.2python3.6.____cpython: CONFIG: linux_cuda_compiler_version9.2python3.6.____cpython UPLOAD_PACKAGES: 'True' diff --git a/.ci_support/linux_cuda_compiler_version10.0python3.6.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.0python3.6.____cpython.yaml deleted file mode 100644 index 632f198..0000000 --- a/.ci_support/linux_cuda_compiler_version10.0python3.6.____cpython.yaml +++ /dev/null @@ -1,31 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.0' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-cuda:10.0 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.6.* *_cpython -zip_keys: -- - cuda_compiler_version - - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.0python3.7.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.0python3.7.____cpython.yaml deleted file mode 100644 index 0393d7f..0000000 --- a/.ci_support/linux_cuda_compiler_version10.0python3.7.____cpython.yaml +++ /dev/null @@ -1,31 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.0' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-cuda:10.0 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.7.* *_cpython -zip_keys: -- - cuda_compiler_version - - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.0python3.8.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.0python3.8.____cpython.yaml deleted file mode 100644 index bf59d15..0000000 --- a/.ci_support/linux_cuda_compiler_version10.0python3.8.____cpython.yaml +++ /dev/null @@ -1,31 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.0' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-cuda:10.0 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.8.* *_cpython -zip_keys: -- - cuda_compiler_version - - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.1python3.6.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.1python3.6.____cpython.yaml deleted file mode 100644 index d7e33d7..0000000 --- a/.ci_support/linux_cuda_compiler_version10.1python3.6.____cpython.yaml +++ /dev/null @@ -1,31 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.1' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-cuda:10.1 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.6.* *_cpython -zip_keys: -- - cuda_compiler_version - - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.1python3.7.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.1python3.7.____cpython.yaml deleted file mode 100644 index 0707ac4..0000000 --- a/.ci_support/linux_cuda_compiler_version10.1python3.7.____cpython.yaml +++ /dev/null @@ -1,31 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.1' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-cuda:10.1 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.7.* *_cpython -zip_keys: -- - cuda_compiler_version - - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.1python3.8.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.1python3.8.____cpython.yaml deleted file mode 100644 index af3ad5d..0000000 --- a/.ci_support/linux_cuda_compiler_version10.1python3.8.____cpython.yaml +++ /dev/null @@ -1,31 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.1' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-cuda:10.1 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.8.* *_cpython -zip_keys: -- - cuda_compiler_version - - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.2python3.6.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.2python3.6.____cpython.yaml deleted file mode 100644 index 73c6037..0000000 --- a/.ci_support/linux_cuda_compiler_version10.2python3.6.____cpython.yaml +++ /dev/null @@ -1,31 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-cuda:10.2 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.6.* *_cpython -zip_keys: -- - cuda_compiler_version - - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.2python3.7.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.2python3.7.____cpython.yaml deleted file mode 100644 index 417becf..0000000 --- a/.ci_support/linux_cuda_compiler_version10.2python3.7.____cpython.yaml +++ /dev/null @@ -1,31 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-cuda:10.2 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.7.* *_cpython -zip_keys: -- - cuda_compiler_version - - docker_image diff --git a/.ci_support/linux_cuda_compiler_version10.2python3.8.____cpython.yaml b/.ci_support/linux_cuda_compiler_version10.2python3.8.____cpython.yaml deleted file mode 100644 index d3ac74f..0000000 --- a/.ci_support/linux_cuda_compiler_version10.2python3.8.____cpython.yaml +++ /dev/null @@ -1,31 +0,0 @@ -boost_cpp: -- '1.72' -c_compiler: -- gcc -c_compiler_version: -- '7' -channel_sources: -- conda-forge,defaults -channel_targets: -- conda-forge main -cuda_compiler: -- nvcc -cuda_compiler_version: -- '10.2' -cxx_compiler: -- gxx -cxx_compiler_version: -- '7' -docker_image: -- condaforge/linux-anvil-cuda:10.2 -pin_run_as_build: - boost-cpp: - max_pin: x.x.x - python: - min_pin: x.x - max_pin: x.x -python: -- 3.8.* *_cpython -zip_keys: -- - cuda_compiler_version - - docker_image diff --git a/README.md b/README.md index b65c9aa..b948cca 100644 --- a/README.md +++ b/README.md @@ -56,69 +56,6 @@ Current build status variant - - linux_cuda_compiler_version10.0python3.6.____cpython - - - variant - - - - linux_cuda_compiler_version10.0python3.7.____cpython - - - variant - - - - linux_cuda_compiler_version10.0python3.8.____cpython - - - variant - - - - linux_cuda_compiler_version10.1python3.6.____cpython - - - variant - - - - linux_cuda_compiler_version10.1python3.7.____cpython - - - variant - - - - linux_cuda_compiler_version10.1python3.8.____cpython - - - variant - - - - linux_cuda_compiler_version10.2python3.6.____cpython - - - variant - - - - linux_cuda_compiler_version10.2python3.7.____cpython - - - variant - - - - linux_cuda_compiler_version10.2python3.8.____cpython - - - variant - - linux_cuda_compiler_version9.2python3.6.____cpython From 7c1174e43edfb84bcad3db53aebe01bfb7c5e324 Mon Sep 17 00:00:00 2001 From: Pearu Peterson Date: Sat, 16 May 2020 01:52:49 +0300 Subject: [PATCH 37/39] Fix re2 issue. Don't run pyarrow unittests. --- recipe/meta.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 0a03a82..c5a10a1 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -34,6 +34,9 @@ requirements: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. - arrow-cpp ={{ version }}=*{{ build_ext }} + # The version of re2 should match with the version that arrow-cpp + # was built against: + - re2 2020.04.* - boost-cpp - cython - numpy 1.16.* @@ -44,7 +47,7 @@ requirements: - {{ pin_compatible('numpy', lower_bound='1.16') }} - boost-cpp - arrow-cpp ={{ version }}=*{{ build_ext }} - - futures # [py==27] + - re2 2020.04.* - glog - pandas # empty parquet-cpp metapackage, force old versions to be uninstalled @@ -53,28 +56,25 @@ requirements: - snappy run_constrained: - arrow-cpp-proc * {{ build_ext }} + - cudatoolkit >=9.2 # [cuda_compiler_version not in (undefined, "None")] test: imports: - pyarrow - - pyarrow.dataset # [not py==27] - - pyarrow.flight # [not py==27] - - pyarrow.gandiva # [(not py==27) and (not (aarch64 or ppc64le))] + - pyarrow.dataset + - pyarrow.flight + - pyarrow.gandiva # [not (aarch64 or ppc64le)] - pyarrow.orc # [unix] - pyarrow.parquet - pyarrow.plasma # [unix] - pyarrow.fs - pyarrow._s3fs - pyarrow._hdfs - - pyarrow.cuda # [cuda_compiler_version not in (undefined, "None")] - requires: # [cuda_compiler_version not in (undefined, "None")] - - pytest # [cuda_compiler_version not in (undefined, "None")] - - numba # [cuda_compiler_version not in (undefined, "None")] - source_files: # [cuda_compiler_version not in (undefined, "None")] - - python/pyarrow/tests/ # [cuda_compiler_version not in (undefined, "None")] - commands: # [cuda_compiler_version not in (undefined, "None")] - - python python/pyarrow/tests/test_cuda.py # [cuda_compiler_version not in (undefined, "None")] - - python python/pyarrow/tests/test_cuda_numba_interop.py # [cuda_compiler_version not in (undefined, "None")] + # We can only test importing cuda package but cannot run when a + # CUDA device is not available, for instance, when building from a + # CI. That said, the pyarrow.cuda import test is carried out in + # the build script. + - pyarrow.cuda # [cuda_compiler_version not in (undefined, "None")] about: home: http://github.com/apache/arrow From c8f8ec4382d7750c00b58a7d6455fb04fc6b1dbd Mon Sep 17 00:00:00 2001 From: h-vetinari <33685575+h-vetinari@users.noreply.github.com> Date: Sat, 16 May 2020 13:28:22 +0200 Subject: [PATCH 38/39] Update recipe/meta.yaml --- recipe/meta.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c5a10a1..e95db52 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -71,9 +71,7 @@ test: - pyarrow._s3fs - pyarrow._hdfs # We can only test importing cuda package but cannot run when a - # CUDA device is not available, for instance, when building from a - # CI. That said, the pyarrow.cuda import test is carried out in - # the build script. + # CUDA device is not available, for instance, when building from CI. - pyarrow.cuda # [cuda_compiler_version not in (undefined, "None")] about: From be4c0c8b852180182f5c585ef95120bd3bfb12b7 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 18 May 2020 21:59:24 +0200 Subject: [PATCH 39/39] remove re2; align/minimize some selectors --- recipe/meta.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index e95db52..43e4189 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -17,7 +17,7 @@ source: build: number: {{ number }} string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" - skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] + skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] track_features: {{ "- arrow-cuda" if cuda_enabled else "" }} @@ -34,9 +34,6 @@ requirements: # directly pin boost-cpp as we also seem to directly include boost symbols # in the Python modules. - arrow-cpp ={{ version }}=*{{ build_ext }} - # The version of re2 should match with the version that arrow-cpp - # was built against: - - re2 2020.04.* - boost-cpp - cython - numpy 1.16.* @@ -47,7 +44,6 @@ requirements: - {{ pin_compatible('numpy', lower_bound='1.16') }} - boost-cpp - arrow-cpp ={{ version }}=*{{ build_ext }} - - re2 2020.04.* - glog - pandas # empty parquet-cpp metapackage, force old versions to be uninstalled @@ -56,7 +52,7 @@ requirements: - snappy run_constrained: - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version not in (undefined, "None")] + - cudatoolkit >=9.2 # [cuda_compiler_version not in (undefined, "None")] test: imports: @@ -72,7 +68,7 @@ test: - pyarrow._hdfs # We can only test importing cuda package but cannot run when a # CUDA device is not available, for instance, when building from CI. - - pyarrow.cuda # [cuda_compiler_version not in (undefined, "None")] + - pyarrow.cuda # [cuda_compiler_version not in (undefined, "None")] about: home: http://github.com/apache/arrow