apacheGH-40079: [CI][Packaging] Enable Azure in more tests and builds (…

…apache#40080) ### Rationale for this change We want python side tests of `AzureFileSystem` to run in CI. ### What changes are included in this PR? - Add missing `export` to enable Azure pyarrow tests - Enable azure in sdist tests. - Enable Azure on macos python builds - Enable azure in conda builds and install dependencies (Azure C++ SDK and azurite) - Enable retries on C++ tests to mitigate apache#40121 Probably all of this should have been included in apache#39971 ### Are these changes tested? There is no new functionality to test ### Are there any user-facing changes? No * Closes: apache#40079 * GitHub Issue: apache#40079 Authored-by: Thomas Newton <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
kou · Feb 22, 2024 · b089c6a · b089c6a
1 parent f9995ac
commit b089c6a
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 0 deletions.
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -132,6 +132,7 @@ jobs:
     timeout-minutes: 60
     env:
       ARROW_HOME: /usr/local
+      ARROW_AZURE: ON
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
       ARROW_GANDIVA: ON

diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt
@@ -16,6 +16,11 @@
 # under the License.
 
 aws-sdk-cpp=1.11.68
+azure-core-cpp>=1.10.3
+azure-identity-cpp>=1.6.0
+azure-storage-blobs-cpp>=12.10.0
+azure-storage-common-cpp>=12.5.0
+azure-storage-files-datalake-cpp>=12.9.0
 benchmark>=1.6.0
 boost-cpp>=1.68.0
 brotli
@@ -34,6 +39,7 @@ libutf8proc
 lz4-c
 make
 ninja
+nodejs
 orc
 pkg-config
 python

diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile
@@ -42,6 +42,13 @@ RUN mamba install -q -y \
         valgrind && \
     mamba clean --all
 
+# Ensure npm, node and azurite are on path. npm and node are required to install azurite, which will then need to 
+# be on the path for the tests to run.  
+ENV PATH=/opt/conda/envs/arrow/bin:$PATH
+
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 # We want to install the GCS testbench using the same Python binary that the Conda code will use.
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
@@ -50,6 +57,7 @@ COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 
 ENV ARROW_ACERO=ON \
+    ARROW_AZURE=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_DATASET=ON \
     ARROW_DEPENDENCY_SOURCE=CONDA \

diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh
@@ -86,6 +86,7 @@ ctest \
     --label-regex unittest \
     --output-on-failure \
     --parallel ${n_jobs} \
+    --repeat until-pass:3 \
     --timeout ${ARROW_CTEST_TIMEOUT:-300} \
     "${ctest_options[@]}" \
     "$@"

diff --git a/ci/scripts/python_sdist_test.sh b/ci/scripts/python_sdist_test.sh
@@ -28,6 +28,7 @@ export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
 export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
 export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
 export PYARROW_WITH_ACERO=${ARROW_ACERO:-ON}
+export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
 export PYARROW_WITH_S3=${ARROW_S3:-OFF}
 export PYARROW_WITH_ORC=${ARROW_ORC:-OFF}
 export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}

diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh
@@ -52,6 +52,7 @@ fi
 : ${PYARROW_TEST_S3:=${ARROW_S3:-ON}}
 
 export PYARROW_TEST_ACERO
+export PYARROW_TEST_AZURE
 export PYARROW_TEST_CUDA
 export PYARROW_TEST_DATASET
 export PYARROW_TEST_FLIGHT