diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index 4c13f66a..c3e36d4b 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -20,8 +20,8 @@ jobs: CONFIG: linux_64_cuda_compiler_version11.2cxx_compiler_version10 UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 - linux_64_cuda_compiler_versionNonecxx_compiler_version10: - CONFIG: linux_64_cuda_compiler_versionNonecxx_compiler_version10 + linux_64_cuda_compiler_versionNonecxx_compiler_version11: + CONFIG: linux_64_cuda_compiler_versionNonecxx_compiler_version11 UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 timeoutInMinutes: 360 diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 32f20e60..1f309833 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -20,6 +20,7 @@ jobs: timeoutInMinutes: 360 variables: CONDA_BLD_PATH: D:\\bld\\ + UPLOAD_TEMP: D:\\tmp steps: - task: PythonScript@0 @@ -41,7 +42,7 @@ jobs: - script: | call activate base - mamba.exe install 'python=3.9' conda-build conda pip boa 'conda-forge-ci-setup=3' -c conda-forge --strict-channel-priority --yes + mamba.exe install "python=3.9" conda-build conda pip boa conda-forge-ci-setup=3 "py-lief<0.12" -c conda-forge --strict-channel-priority --yes displayName: Install conda-build - script: set PYTHONUNBUFFERED=1 @@ -78,6 +79,9 @@ jobs: - script: | set "GIT_BRANCH=%BUILD_SOURCEBRANCHNAME%" set "FEEDSTOCK_NAME=%BUILD_REPOSITORY_NAME:*/=%" + set "TEMP=$(UPLOAD_TEMP)" + if not exist "%TEMP%\" md "%TEMP%" + set "TMP=%TEMP%" call activate base upload_package --validate --feedstock-name="%FEEDSTOCK_NAME%" .\ ".\recipe" .ci_support\%CONFIG%.yaml displayName: Upload package diff --git a/.ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version10.yaml b/.ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version11.yaml similarity index 98% rename from .ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version10.yaml rename to .ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version11.yaml index c18bb994..c1b75d1b 100644 --- a/.ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version10.yaml +++ b/.ci_support/linux_64_cuda_compiler_versionNonecxx_compiler_version11.yaml @@ -11,7 +11,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '10' +- '11' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 libblas: diff --git a/.ci_support/linux_aarch64_.yaml b/.ci_support/linux_aarch64_.yaml new file mode 100644 index 00000000..7467bc0a --- /dev/null +++ b/.ci_support/linux_aarch64_.yaml @@ -0,0 +1,47 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '11' +docker_image: +- quay.io/condaforge/linux-anvil-aarch64 +libblas: +- 3.9 *netlib +liblapack: +- 3.9 *netlib +numpy: +- '1.21' +- '1.23' +- '1.20' +- '1.20' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +target_platform: +- linux-aarch64 +zip_keys: +- - cxx_compiler_version + - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy diff --git a/.ci_support/linux_ppc64le_.yaml b/.ci_support/linux_ppc64le_.yaml new file mode 100644 index 00000000..d0f60e38 --- /dev/null +++ b/.ci_support/linux_ppc64le_.yaml @@ -0,0 +1,43 @@ +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '11' +docker_image: +- quay.io/condaforge/linux-anvil-ppc64le +libblas: +- 3.9 *netlib +liblapack: +- 3.9 *netlib +numpy: +- '1.21' +- '1.23' +- '1.20' +- '1.20' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +target_platform: +- linux-ppc64le +zip_keys: +- - cxx_compiler_version + - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e61aa24..8b4ef2f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ # This file was generated automatically from conda-smithy. To update this configuration, # update the conda-forge.yml and/or the recipe/meta.yaml. -# -*- mode: yaml -*- +# -*- mode: jinja-yaml -*- version: 2 diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 595f8b5e..71a4242a 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -33,9 +33,9 @@ CONDARC mamba install --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 + conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" mamba update --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 + conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index 07de621f..caa788e6 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -23,11 +23,10 @@ bash $MINIFORGE_FILE -b -p ${MINIFORGE_HOME} source ${MINIFORGE_HOME}/etc/profile.d/conda.sh conda activate base -echo -e "\n\nInstalling ['conda-forge-ci-setup=3'] and conda-build." mamba install --update-specs --quiet --yes --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 + conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" mamba update --update-specs --yes --quiet --channel conda-forge \ - conda-build pip boa conda-forge-ci-setup=3 + conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12" diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..58a72eb8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,27 @@ +# This file was generated automatically from conda-smithy. To update this configuration, +# update the conda-forge.yml and/or the recipe/meta.yaml. + +language: generic + + + +matrix: + include: + - env: CONFIG=linux_aarch64_ UPLOAD_PACKAGES=True PLATFORM=linux-aarch64 DOCKER_IMAGE=quay.io/condaforge/linux-anvil-aarch64 + os: linux + arch: arm64 + dist: focal + + - env: CONFIG=linux_ppc64le_ UPLOAD_PACKAGES=True PLATFORM=linux-ppc64le DOCKER_IMAGE=quay.io/condaforge/linux-anvil-ppc64le + os: linux + arch: ppc64le + dist: focal + +script: + - export CI=travis + - export GIT_BRANCH="$TRAVIS_BRANCH" + - export FEEDSTOCK_NAME=$(basename ${TRAVIS_REPO_SLUG}) + - if [[ "${TRAVIS_PULL_REQUEST:-}" == "false" ]]; then export IS_PR_BUILD="False"; else export IS_PR_BUILD="True"; fi + + + - if [[ ${PLATFORM} =~ .*linux.* ]]; then CONDA_FORGE_DOCKER_RUN_ARGS="--network=host --security-opt=seccomp=unconfined" ./.scripts/run_docker_build.sh; fi \ No newline at end of file diff --git a/README.md b/README.md index 3aa7d731..15126ff9 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,14 @@ Current build status ==================== - +
+ + + @@ -47,56 +54,70 @@ Current build status + + + + + + - + diff --git a/conda-forge.yml b/conda-forge.yml index 2f501ce8..23aa2f0f 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -1,6 +1,9 @@ +conda_build: + pkg_format: '2' conda_forge_output_validation: true github: branch_name: main tooling_branch_name: main -conda_build: - pkg_format: '2' +provider: + linux_aarch64: default + linux_ppc64le: default diff --git a/recipe/build-pkg.sh b/recipe/build-pkg.sh index 323f5085..6a10689d 100644 --- a/recipe/build-pkg.sh +++ b/recipe/build-pkg.sh @@ -19,25 +19,26 @@ cmake ${CMAKE_ARGS} \ cmake --build _build_python_generic --target swigfaiss -j $CPU_COUNT # Build version with avx2 support, see build-lib.sh -cmake ${CMAKE_ARGS} \ - -Dfaiss_ROOT=_libfaiss_avx2_stage/ \ - -DFAISS_OPT_LEVEL=avx2 \ - -DFAISS_ENABLE_GPU=${FAISS_ENABLE_GPU} \ - -DCMAKE_BUILD_TYPE=Release \ - -DPython_EXECUTABLE="${PYTHON}" \ - -B _build_python_avx2 \ - faiss/python -cmake --build _build_python_avx2 --target swigfaiss_avx2 -j $CPU_COUNT +if [[ "${target_platform}" == *-64 ]]; then + cmake ${CMAKE_ARGS} \ + -Dfaiss_ROOT=_libfaiss_avx2_stage/ \ + -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_ENABLE_GPU=${FAISS_ENABLE_GPU} \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE="${PYTHON}" \ + -B _build_python_avx2 \ + faiss/python + cmake --build _build_python_avx2 --target swigfaiss_avx2 -j $CPU_COUNT -# copy generated swig module with avx2-support to specifically named file, cf. -# https://github.com/facebookresearch/faiss/blob/v1.7.1/faiss/python/setup.py#L37-L40 -cp _build_python_avx2/swigfaiss_avx2.py _build_python_generic/swigfaiss_avx2.py -cp _build_python_avx2/_swigfaiss_avx2.so _build_python_generic/_swigfaiss_avx2.so + # copy generated swig module with avx2-support to specifically named file, cf. + # https://github.com/facebookresearch/faiss/blob/v1.7.1/faiss/python/setup.py#L37-L40 + cp _build_python_avx2/swigfaiss_avx2.py _build_python_generic/swigfaiss_avx2.py + cp _build_python_avx2/_swigfaiss_avx2.so _build_python_generic/_swigfaiss_avx2.so +fi # Build actual python module. pushd _build_python_generic $PYTHON setup.py install --single-version-externally-managed --record=record.txt --prefix=$PREFIX popd # clean up cmake-cache between builds -rm -r _build_python_generic -rm -r _build_python_avx2 +rm -r _build_python_* diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c7d1be4e..98507c2e 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -92,7 +92,7 @@ source: - patches/0006-add-bigobj-to-swigfaiss-compile-options-on-windows.patch build: - number: 2 + number: 3 skip: true # [linux and cuda_compiler_version == "10.2"] # always crashes skip: true # [win and cuda_compiler_version in ("11.0", "11.1")] @@ -112,8 +112,10 @@ outputs: commands: - exit 0 - # build two separate C++ libs, one for generic x64, and one for AVX2 - {% for CF_FAISS_BUILD in ["avx2", "generic"] %} + # build two separate C++ libs, one generic, and one for AVX2 on x86_64 + {% set CF_FAISS_BUILD_OPTS = ["generic"] %} + {% set CF_FAISS_BUILD_OPTS = ["avx2"] + CF_FAISS_BUILD_OPTS %} # [x86_64] + {% for CF_FAISS_BUILD in CF_FAISS_BUILD_OPTS %} # order libfaiss last in loop due to conda/conda-build#4090; libfaiss-avx2 # is only used for faiss and not important enough to work-around for this bug {% if CF_FAISS_BUILD == "generic" %} @@ -191,13 +193,13 @@ outputs: - pip - numpy - libfaiss ={{ version }}=*_{{ faiss_proc_type }} - - libfaiss-avx2 ={{ version }}=*_{{ faiss_proc_type }} + - libfaiss-avx2 ={{ version }}=*_{{ faiss_proc_type }} # [x86_64] - libblas - liblapack run: - python - libfaiss ={{ version }}=*_{{ faiss_proc_type }} - - libfaiss-avx2 ={{ version }}=*_{{ faiss_proc_type }} + - libfaiss-avx2 ={{ version }}=*_{{ faiss_proc_type }} # [x86_64] - {{ pin_compatible('numpy') }} run_constrained: - faiss-cpu ==9999999999 # [cuda_compiler_version != "None"] @@ -208,8 +210,8 @@ outputs: requires: # trying to test all blas-variants runs into conda/conda-build#3947 # - libblas =*=*{{ blas_impl }} - # testing with MKL, as upstream considers this the most important - - libblas =*=*mkl + # testing with MKL on x86_64, as upstream considers this the most important + - libblas =*=*mkl # [x86_64] - scipy - pytest files: @@ -223,8 +225,8 @@ outputs: # the linux & windows CI agents support AVX2 (OSX doesn't yet), so by default, # we expect faiss will load the library with AVX2-support, see # https://github.com/facebookresearch/faiss/blob/v1.7.1/faiss/python/loader.py#L52-L66 - - export HAS_AVX2=YES && ./test-pkg.sh # [linux] - - export HAS_AVX2=NO && ./test-pkg.sh # [osx] + - export HAS_AVX2=YES && ./test-pkg.sh # [linux and x86_64] + - export HAS_AVX2=NO && ./test-pkg.sh # [osx or (linux and not x86_64)] # skip test suite on win + cuda < 11.2 due to time outs (note: "None" >= "11.2") {% if cuda_compiler_version|string >= "11.2" %} - set "HAS_AVX2=YES" && test-pkg.bat # [win]
Travis + + linux + +
Azurelinux_64_cuda_compiler_version11.0cxx_compiler_version9 - variant + variant
linux_64_cuda_compiler_version11.1cxx_compiler_version10 - variant + variant
linux_64_cuda_compiler_version11.2cxx_compiler_version10 - variant + variant + +
linux_64_cuda_compiler_versionNonecxx_compiler_version11 + + variant + +
linux_aarch64 + + variant
linux_64_cuda_compiler_versionNonecxx_compiler_version10linux_ppc64le - variant + variant
osx_64 - variant + variant
win_64_cuda_compiler_version10.2 - variant + variant
win_64_cuda_compiler_version11.2 - variant + variant
win_64_cuda_compiler_versionNone - variant + variant