From 573a4596f698ae70e9ef92b4756fb751e829125d Mon Sep 17 00:00:00 2001 From: regro-cf-autotick-bot <36490558+regro-cf-autotick-bot@users.noreply.github.com> Date: Fri, 18 Nov 2022 10:37:51 +0000 Subject: [PATCH 1/8] migration: ArchRebuild --- conda-forge.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/conda-forge.yml b/conda-forge.yml index 2f501ce8..23aa2f0f 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -1,6 +1,9 @@ +conda_build: + pkg_format: '2' conda_forge_output_validation: true github: branch_name: main tooling_branch_name: main -conda_build: - pkg_format: '2' +provider: + linux_aarch64: default + linux_ppc64le: default From d0ed46bf37b04aadb8c2e0aaa10b94607cd75281 Mon Sep 17 00:00:00 2001 From: regro-cf-autotick-bot <36490558+regro-cf-autotick-bot@users.noreply.github.com> Date: Fri, 18 Nov 2022 10:40:03 +0000 Subject: [PATCH 2/8] MNT: Re-rendered with conda-build 3.23.0, conda-smithy 3.21.3, and conda-forge-pinning 2022.11.18.09.29.52 --- .azure-pipelines/azure-pipelines-win.yml | 2 +- .ci_support/linux_aarch64_.yaml | 47 ++++++++++++++++++++++++ .ci_support/linux_ppc64le_.yaml | 43 ++++++++++++++++++++++ .circleci/config.yml | 2 +- .scripts/build_steps.sh | 4 +- .scripts/run_osx_build.sh | 5 +-- .travis.yml | 27 ++++++++++++++ README.md | 39 +++++++++++++++----- 8 files changed, 153 insertions(+), 16 deletions(-) create mode 100644 .ci_support/linux_aarch64_.yaml create mode 100644 .ci_support/linux_ppc64le_.yaml create mode 100644 .travis.yml diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 32f20e60..4e3cf11c 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -41,7 +41,7 @@ jobs: - script: | call activate base - mamba.exe install 'python=3.9' conda-build conda pip boa 'conda-forge-ci-setup=3' -c conda-forge --strict-channel-priority --yes + mamba.exe install "python=3.9" conda-build conda pip boa conda-forge-ci-setup=3 "py-lief<0.12" -c conda-forge --strict-channel-priority --yes displayName: Install conda-build - script: set PYTHONUNBUFFERED=1 diff --git a/.ci_support/linux_aarch64_.yaml b/.ci_support/linux_aarch64_.yaml new file mode 100644 index 00000000..2e017b99 --- /dev/null +++ b/.ci_support/linux_aarch64_.yaml @@ -0,0 +1,47 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '10' +docker_image: +- quay.io/condaforge/linux-anvil-aarch64 +libblas: +- 3.9 *netlib +liblapack: +- 3.9 *netlib +numpy: +- '1.21' +- '1.23' +- '1.20' +- '1.20' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +target_platform: +- linux-aarch64 +zip_keys: +- - cxx_compiler_version + - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy diff --git a/.ci_support/linux_ppc64le_.yaml b/.ci_support/linux_ppc64le_.yaml new file mode 100644 index 00000000..2d474489 --- /dev/null +++ b/.ci_support/linux_ppc64le_.yaml @@ -0,0 +1,43 @@ +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '10' +docker_image: +- quay.io/condaforge/linux-anvil-ppc64le +libblas: +- 3.9 *netlib +liblapack: +- 3.9 *netlib +numpy: +- '1.21' +- '1.23' +- '1.20' +- '1.20' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +target_platform: +- linux-ppc64le +zip_keys: +- - cxx_compiler_version + - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e61aa24..8b4ef2f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ # This file was generated automatically from conda-smithy. To update this configuration, # update the conda-forge.yml and/or the recipe/meta.yaml. -# -*- mode: yaml -*- +# -*- mode: jinja-yaml -*- version: 2 diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 595f8b5e..71a4242a 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -33,9 +33,9 @@ CONDARC mamba install --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 + conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" mamba update --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 + conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index 07de621f..caa788e6 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -23,11 +23,10 @@ bash $MINIFORGE_FILE -b -p ${MINIFORGE_HOME} source ${MINIFORGE_HOME}/etc/profile.d/conda.sh conda activate base -echo -e "\n\nInstalling ['conda-forge-ci-setup=3'] and conda-build." mamba install --update-specs --quiet --yes --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 + conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" mamba update --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 + conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..58a72eb8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,27 @@ +# This file was generated automatically from conda-smithy. To update this configuration, +# update the conda-forge.yml and/or the recipe/meta.yaml. + +language: generic + + + +matrix: + include: + - env: CONFIG=linux_aarch64_ UPLOAD_PACKAGES=True PLATFORM=linux-aarch64 DOCKER_IMAGE=quay.io/condaforge/linux-anvil-aarch64 + os: linux + arch: arm64 + dist: focal + + - env: CONFIG=linux_ppc64le_ UPLOAD_PACKAGES=True PLATFORM=linux-ppc64le DOCKER_IMAGE=quay.io/condaforge/linux-anvil-ppc64le + os: linux + arch: ppc64le + dist: focal + +script: + - export CI=travis + - export GIT_BRANCH="$TRAVIS_BRANCH" + - export FEEDSTOCK_NAME=$(basename ${TRAVIS_REPO_SLUG}) + - if [[ "${TRAVIS_PULL_REQUEST:-}" == "false" ]]; then export IS_PR_BUILD="False"; else export IS_PR_BUILD="True"; fi + + + - if [[ ${PLATFORM} =~ .*linux.* ]]; then CONDA_FORGE_DOCKER_RUN_ARGS="--network=host --security-opt=seccomp=unconfined" ./.scripts/run_docker_build.sh; fi \ No newline at end of file diff --git a/README.md b/README.md index 3aa7d731..bda39e9d 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,14 @@ Current build status ==================== - +
+ + + @@ -47,56 +54,70 @@ Current build status + + + + + + From 1e67bc4b9774b716968ad80703c6f0e883962f58 Mon Sep 17 00:00:00 2001 From: "conda-forge-webservices[bot]" <91080706+conda-forge-webservices[bot]@users.noreply.github.com> Date: Wed, 4 Jan 2023 22:12:42 +0000 Subject: [PATCH 3/8] MNT: Re-rendered with conda-build 3.23.3, conda-smithy 3.22.1, and conda-forge-pinning 2023.01.04.13.03.46 --- .azure-pipelines/azure-pipelines-linux.yml | 4 ++-- .azure-pipelines/azure-pipelines-win.yml | 4 ++++ ...x_64_cuda_compiler_versionNonecxx_compiler_version11.yaml} | 2 +- .ci_support/linux_aarch64_.yaml | 2 +- .ci_support/linux_ppc64le_.yaml | 2 +- README.md | 4 ++-- 6 files changed, 11 insertions(+), 7 deletions(-) rename .ci_support/{linux_64_cuda_compiler_versionNonecxx_compiler_version10.yaml => linux_64_cuda_compiler_versionNonecxx_compiler_version11.yaml} (98%) diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index 4c13f66a..c3e36d4b 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -20,8 +20,8 @@ jobs: CONFIG: linux_64_cuda_compiler_version11.2cxx_compiler_version10 UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 - linux_64_cuda_compiler_versionNonecxx_compiler_version10: - CONFIG: linux_64_cuda_compiler_versionNonecxx_compiler_version10 + linux_64_cuda_compiler_versionNonecxx_compiler_version11: + CONFIG: linux_64_cuda_compiler_versionNonecxx_compiler_version11 UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 timeoutInMinutes: 360 diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 4e3cf11c..1f309833 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -20,6 +20,7 @@ jobs: timeoutInMinutes: 360 variables: CONDA_BLD_PATH: D:\\bld\\ + UPLOAD_TEMP: D:\\tmp steps: - task: PythonScript@0 @@ -78,6 +79,9 @@ jobs: - script: | set "GIT_BRANCH=%BUILD_SOURCEBRANCHNAME%" set "FEEDSTOCK_NAME=%BUILD_REPOSITORY_NAME:*/=%" + set "TEMP=$(UPLOAD_TEMP)" + if not exist "%TEMP%\" md "%TEMP%" + set "TMP=%TEMP%" call activate base upload_package --validate --feedstock-name="%FEEDSTOCK_NAME%" .\ ".\recipe" .ci_support\%CONFIG%.yaml displayName: Upload package diff --git a/.ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version10.yaml b/.ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version11.yaml similarity index 98% rename from .ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version10.yaml rename to .ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version11.yaml index c18bb994..c1b75d1b 100644 --- a/.ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version10.yaml +++ b/.ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version11.yaml @@ -11,7 +11,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 libblas: diff --git a/.ci_support/linux_aarch64_.yaml b/.ci_support/linux_aarch64_.yaml index 2e017b99..7467bc0a 100644 --- a/.ci_support/linux_aarch64_.yaml +++ b/.ci_support/linux_aarch64_.yaml @@ -15,7 +15,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-aarch64 libblas: diff --git a/.ci_support/linux_ppc64le_.yaml b/.ci_support/linux_ppc64le_.yaml index 2d474489..d0f60e38 100644 --- a/.ci_support/linux_ppc64le_.yaml +++ b/.ci_support/linux_ppc64le_.yaml @@ -11,7 +11,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-ppc64le libblas: diff --git a/README.md b/README.md index bda39e9d..15126ff9 100644 --- a/README.md +++ b/README.md @@ -72,10 +72,10 @@ Current build status - + From eb76b403e7ddd256e0585f31571de4c923b58db4 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Wed, 4 Jan 2023 14:24:34 -0800 Subject: [PATCH 4/8] Use AVX2 on x86_64 architectures only --- recipe/meta.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c7d1be4e..5e707101 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -112,8 +112,10 @@ outputs: commands: - exit 0 - # build two separate C++ libs, one for generic x64, and one for AVX2 - {% for CF_FAISS_BUILD in ["avx2", "generic"] %} + # build two separate C++ libs, one generic, and one for AVX2 on x86_64 + {% set CF_FAISS_BUILD_OPTS = ["generic"] %} + {% set CF_FAISS_BUILD_OPTS = ["avx2"] + CF_FAISS_BUILD_OPTS %} # [x86_64] + {% for CF_FAISS_BUILD in CF_FAISS_BUILD_OPTS %} # order libfaiss last in loop due to conda/conda-build#4090; libfaiss-avx2 # is only used for faiss and not important enough to work-around for this bug {% if CF_FAISS_BUILD == "generic" %} @@ -191,13 +193,13 @@ outputs: - pip - numpy - libfaiss ={{ version }}=*_{{ faiss_proc_type }} - - libfaiss-avx2 ={{ version }}=*_{{ faiss_proc_type }} + - libfaiss-avx2 ={{ version }}=*_{{ faiss_proc_type }} # [x86_64] - libblas - liblapack run: - python - libfaiss ={{ version }}=*_{{ faiss_proc_type }} - - libfaiss-avx2 ={{ version }}=*_{{ faiss_proc_type }} + - libfaiss-avx2 ={{ version }}=*_{{ faiss_proc_type }} # [x86_64] - {{ pin_compatible('numpy') }} run_constrained: - faiss-cpu ==9999999999 # [cuda_compiler_version != "None"] @@ -223,7 +225,7 @@ outputs: # the linux & windows CI agents support AVX2 (OSX doesn't yet), so by default, # we expect faiss will load the library with AVX2-support, see # https://github.com/facebookresearch/faiss/blob/v1.7.1/faiss/python/loader.py#L52-L66 - - export HAS_AVX2=YES && ./test-pkg.sh # [linux] + - export HAS_AVX2=YES && ./test-pkg.sh # [linux64] - export HAS_AVX2=NO && ./test-pkg.sh # [osx] # skip test suite on win + cuda < 11.2 due to time outs (note: "None" >= "11.2") {% if cuda_compiler_version|string >= "11.2" %} From 193a5f92e079e7dd80da4424177204ae342f3399 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Wed, 4 Jan 2023 14:33:22 -0800 Subject: [PATCH 5/8] Constrain BLAS for testing on x86_64 only --- recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 5e707101..48d6f2e8 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -210,8 +210,8 @@ outputs: requires: # trying to test all blas-variants runs into conda/conda-build#3947 # - libblas =*=*{{ blas_impl }} - # testing with MKL, as upstream considers this the most important - - libblas =*=*mkl + # testing with MKL on x86_64, as upstream considers this the most important + - libblas =*=*mkl # [x86_64] - scipy - pytest files: From fe14caf51aed094be9c29f167076c150de5d1075 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Wed, 4 Jan 2023 14:45:53 -0800 Subject: [PATCH 6/8] Run generic package tests on other architectures --- recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 48d6f2e8..d5433754 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -225,8 +225,8 @@ outputs: # the linux & windows CI agents support AVX2 (OSX doesn't yet), so by default, # we expect faiss will load the library with AVX2-support, see # https://github.com/facebookresearch/faiss/blob/v1.7.1/faiss/python/loader.py#L52-L66 - - export HAS_AVX2=YES && ./test-pkg.sh # [linux64] - - export HAS_AVX2=NO && ./test-pkg.sh # [osx] + - export HAS_AVX2=YES && ./test-pkg.sh # [linux and x86_64] + - export HAS_AVX2=NO && ./test-pkg.sh # [osx or (linux and not x86_64)] # skip test suite on win + cuda < 11.2 due to time outs (note: "None" >= "11.2") {% if cuda_compiler_version|string >= "11.2" %} - set "HAS_AVX2=YES" && test-pkg.bat # [win] From 0a30f14a62df63865ed185340f8e8f890207b215 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Wed, 4 Jan 2023 15:09:37 -0800 Subject: [PATCH 7/8] Build AVX2 only on x86_64 --- recipe/build-pkg.sh | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/recipe/build-pkg.sh b/recipe/build-pkg.sh index 323f5085..6a10689d 100644 --- a/recipe/build-pkg.sh +++ b/recipe/build-pkg.sh @@ -19,25 +19,26 @@ cmake ${CMAKE_ARGS} \ cmake --build _build_python_generic --target swigfaiss -j $CPU_COUNT # Build version with avx2 support, see build-lib.sh -cmake ${CMAKE_ARGS} \ - -Dfaiss_ROOT=_libfaiss_avx2_stage/ \ - -DFAISS_OPT_LEVEL=avx2 \ - -DFAISS_ENABLE_GPU=${FAISS_ENABLE_GPU} \ - -DCMAKE_BUILD_TYPE=Release \ - -DPython_EXECUTABLE="${PYTHON}" \ - -B _build_python_avx2 \ - faiss/python -cmake --build _build_python_avx2 --target swigfaiss_avx2 -j $CPU_COUNT +if [[ "${target_platform}" == *-64 ]]; then + cmake ${CMAKE_ARGS} \ + -Dfaiss_ROOT=_libfaiss_avx2_stage/ \ + -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_ENABLE_GPU=${FAISS_ENABLE_GPU} \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE="${PYTHON}" \ + -B _build_python_avx2 \ + faiss/python + cmake --build _build_python_avx2 --target swigfaiss_avx2 -j $CPU_COUNT -# copy generated swig module with avx2-support to specifically named file, cf. -# https://github.com/facebookresearch/faiss/blob/v1.7.1/faiss/python/setup.py#L37-L40 -cp _build_python_avx2/swigfaiss_avx2.py _build_python_generic/swigfaiss_avx2.py -cp _build_python_avx2/_swigfaiss_avx2.so _build_python_generic/_swigfaiss_avx2.so + # copy generated swig module with avx2-support to specifically named file, cf. + # https://github.com/facebookresearch/faiss/blob/v1.7.1/faiss/python/setup.py#L37-L40 + cp _build_python_avx2/swigfaiss_avx2.py _build_python_generic/swigfaiss_avx2.py + cp _build_python_avx2/_swigfaiss_avx2.so _build_python_generic/_swigfaiss_avx2.so +fi # Build actual python module. pushd _build_python_generic $PYTHON setup.py install --single-version-externally-managed --record=record.txt --prefix=$PREFIX popd # clean up cmake-cache between builds -rm -r _build_python_generic -rm -r _build_python_avx2 +rm -r _build_python_* From 6926c550a3df4c8836f1ec42babae93398e208d0 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Wed, 4 Jan 2023 15:36:09 -0800 Subject: [PATCH 8/8] Bump `build/number` to `3` --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index d5433754..98507c2e 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -92,7 +92,7 @@ source: - patches/0006-add-bigobj-to-swigfaiss-compile-options-on-windows.patch build: - number: 2 + number: 3 skip: true # [linux and cuda_compiler_version == "10.2"] # always crashes skip: true # [win and cuda_compiler_version in ("11.0", "11.1")]
Travis + + linux + +
Azurelinux_64_cuda_compiler_version11.0cxx_compiler_version9 - variant + variant
linux_64_cuda_compiler_version11.1cxx_compiler_version10 - variant + variant
linux_64_cuda_compiler_version11.2cxx_compiler_version10 - variant + variant
linux_64_cuda_compiler_versionNonecxx_compiler_version10 - variant + variant + +
linux_aarch64 + + variant + +
linux_ppc64le + + variant
osx_64 - variant + variant
win_64_cuda_compiler_version10.2 - variant + variant
win_64_cuda_compiler_version11.2 - variant + variant
win_64_cuda_compiler_versionNone - variant + variant
linux_64_cuda_compiler_versionNonecxx_compiler_version10linux_64_cuda_compiler_versionNonecxx_compiler_version11 - variant + variant