Merge remote-tracking branch 'upstream/main' into device-registry

.asf.yaml

@@ -19,7 +19,6 @@ github:
   description: "Apache Arrow is a multi-language toolbox for accelerated data interchange and in-memory processing"
   homepage: https://arrow.apache.org/
   collaborators:
-    - amoeba
     - anjakefala
     - benibus
     - danepitkin

.env

@@ -65,7 +65,7 @@ JDK=8
 KARTOTHEK=latest
 # LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
 LLVM=14
-MAVEN=3.6.3
+MAVEN=3.8.7
 NODE=18
 NUMBA=latest
 NUMPY=latest

.github/workflows/dev.yml

@@ -35,6 +35,7 @@ jobs:
     name: Lint C++, Python, R, Docker, RAT
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 15
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0

.github/workflows/java.yml

@@ -84,7 +84,12 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: archery docker run ${{ matrix.image }}
+          GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+        run: |
+          archery docker run \
+            -e CI=true \
+            -e "GRADLE_ENTERPRISE_ACCESS_KEY=$GRADLE_ENTERPRISE_ACCESS_KEY" \
+            ${{ matrix.image }}
       - name: Docker Push
         if: >-
           success() &&
@@ -119,9 +124,13 @@ jobs:
           submodules: recursive
       - name: Build
         shell: bash
+        env:
+          GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
         run: ci/scripts/java_build.sh $(pwd) $(pwd)/build
       - name: Test
         shell: bash
+        env:
+          GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
         run: ci/scripts/java_test.sh $(pwd) $(pwd)/build
 
   windows:
@@ -146,7 +155,11 @@ jobs:
           submodules: recursive
       - name: Build
         shell: bash
+        env:
+          GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
         run: ci/scripts/java_build.sh $(pwd) $(pwd)/build
       - name: Test
         shell: bash
+        env:
+          GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
         run: ci/scripts/java_test.sh $(pwd) $(pwd)/build

.github/workflows/java_jni.yml

@@ -118,7 +118,12 @@ jobs:
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: archery docker run conda-python-java-integration
+          GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+        run: |
+          archery docker run \
+            -e CI=true \
+            -e "GRADLE_ENTERPRISE_ACCESS_KEY=$GRADLE_ENTERPRISE_ACCESS_KEY" \
+            conda-python-java-integration
       - name: Docker Push
         if: >-
           success() &&

.gitignore

@@ -100,3 +100,6 @@ __debug_bin
 
 # direnv
 .envrc
+
+# Develocity
+.mvn/.gradle-enterprise/

.mvn/extensions.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<extensions xmlns="http://maven.apache.org/EXTENSIONS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
+    <extension>
+        <groupId>com.gradle</groupId>
+        <artifactId>gradle-enterprise-maven-extension</artifactId>
+        <version>1.20</version>
+    </extension>
+    <extension>
+        <groupId>com.gradle</groupId>
+        <artifactId>common-custom-user-data-maven-extension</artifactId>
+        <version>1.12.5</version>
+    </extension>
+</extensions>

.mvn/gradle-enterprise.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<gradleEnterprise xmlns="https://www.gradle.com/gradle-enterprise-maven" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://www.gradle.com/gradle-enterprise-maven https://www.gradle.com/schema/gradle-enterprise-maven.xsd">
+    <server>
+        <url>https://ge.apache.org</url>
+        <allowUntrusted>false</allowUntrusted>
+    </server>
+    <buildScan>
+        <capture>
+            <goalInputFiles>true</goalInputFiles>
+            <buildLogging>true</buildLogging>
+            <testLogging>true</testLogging>
+        </capture>
+        <backgroundBuildScanUpload>#{isFalse(env['CI'])}</backgroundBuildScanUpload>
+        <publish>ALWAYS</publish>
+        <publishIfAuthenticated>true</publishIfAuthenticated>
+        <obfuscation>
+            <ipAddresses>#{{'0.0.0.0'}}</ipAddresses>
+        </obfuscation>
+    </buildScan>
+    <buildCache>
+        <remote>
+            <enabled>false</enabled>
+        </remote>
+    </buildCache>
+</gradleEnterprise>

.pre-commit-config.yaml

@@ -29,12 +29,11 @@ repos:
         entry: bash -c "git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar && ./dev/release/run-rat.sh arrow-src.tar"
         always_run: true
         pass_filenames: false
-      - id: hadolint
+  - repo: https://github.com/hadolint/hadolint
+    rev: v2.12.0
+    hooks:
+      - id: hadolint-docker
         name: Docker Format
-        language: docker_image
-        types:
-          - dockerfile
-        entry: --entrypoint /bin/hadolint hadolint/hadolint:latest -
         exclude: ^dev/.*$
   - repo: https://github.com/pycqa/flake8
     rev: 6.1.0

ci/docker/conda-integration.dockerfile

@@ -20,7 +20,7 @@ ARG arch=amd64
 FROM ${repo}:${arch}-conda-cpp
 
 ARG arch=amd64
-ARG maven=3.5
+ARG maven=3.8.7
 ARG node=16
 ARG yarn=1.22
 ARG jdk=8

ci/docker/conda-python-hdfs.dockerfile

@@ -21,7 +21,7 @@ ARG python=3.8
 FROM ${repo}:${arch}-conda-python-${python}
 
 ARG jdk=8
-ARG maven=3.5
+ARG maven=3.8.7
 RUN mamba install -q -y \
         maven=${maven} \
         openjdk=${jdk} \

ci/docker/conda-python-jpype.dockerfile

@@ -21,7 +21,7 @@ ARG python=3.8
 FROM ${repo}:${arch}-conda-python-${python}
 
 ARG jdk=11
-ARG maven=3.6
+ARG maven=3.8.7
 RUN mamba install -q -y \
         maven=${maven} \
         openjdk=${jdk} \

ci/docker/conda-python-spark.dockerfile

@@ -21,7 +21,7 @@ ARG python=3.8
 FROM ${repo}:${arch}-conda-python-${python}
 
 ARG jdk=8
-ARG maven=3.5
+ARG maven=3.8.7
 
 ARG numpy=latest
 COPY ci/scripts/install_numpy.sh /arrow/ci/scripts/

ci/docker/linux-apt-docs.dockerfile

@@ -61,7 +61,7 @@ RUN apt-get update -y && \
 
 ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64
 
-ARG maven=3.6.3
+ARG maven=3.8.7
 COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/util_download_apache.sh \
     "maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" /opt

ci/scripts/java_build.sh

@@ -32,7 +32,7 @@ java_jni_dist_dir=${3}
 if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then
   # Since some files for s390_64 are not available at maven central,
   # download pre-build files from Artifactory and install them explicitly
-  mvn_install="mvn install:install-file"
+  mvn_install="mvn clean install:install-file"
   wget="wget"
   artifactory_base_url="https://apache.jfrog.io/artifactory/arrow"
 
@@ -89,12 +89,12 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
   mvn="${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni"
 fi
 
-${mvn} install
+${mvn} clean install
 
 if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then
   # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633
   mkdir -p ${build_dir}/docs/java/reference
-  ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false install site
+  ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false clean install site
   rsync -a ${arrow_dir}/java/target/site/apidocs/ ${build_dir}/docs/java/reference
 fi
 

ci/scripts/java_test.sh

@@ -35,7 +35,7 @@ mvn="${mvn} -T 2C"
 
 pushd ${source_dir}
 
-${mvn} test
+${mvn} clean test
 
 projects=()
 if [ "${ARROW_DATASET}" = "ON" ]; then
@@ -48,14 +48,14 @@ if [ "${ARROW_ORC}" = "ON" ]; then
   projects+=(adapter/orc)
 fi
 if [ "${#projects[@]}" -gt 0 ]; then
-  ${mvn} test \
+  ${mvn} clean test \
          -Parrow-jni \
          -pl $(IFS=,; echo "${projects[*]}") \
          -Darrow.cpp.build.dir=${java_jni_dist_dir}
 fi
 
 if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
-  ${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir=${java_jni_dist_dir}
+  ${mvn} clean test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir=${java_jni_dist_dir}
 fi
 
 popd

ci/scripts/python_sdist_test.sh

@@ -55,6 +55,11 @@ if [ -n "${PYARROW_VERSION:-}" ]; then
 else
   sdist=$(ls ${arrow_dir}/python/dist/pyarrow-*.tar.gz | sort -r | head -n1)
 fi
+
+if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
+  . "${ARROW_PYTHON_VENV}/bin/activate"
+fi
+
 ${PYTHON:-python} -m pip install ${sdist}
 
 pytest -r s ${PYTEST_ARGS:-} --pyargs pyarrow

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -1812,7 +1812,9 @@ if(ARROW_WITH_PROTOBUF)
   else()
     set(ARROW_PROTOBUF_REQUIRED_VERSION "2.6.1")
   endif()
-  if(ARROW_ORC OR ARROW_WITH_OPENTELEMETRY)
+  if(ARROW_ORC
+     OR ARROW_SUBSTRAIT
+     OR ARROW_WITH_OPENTELEMETRY)
     set(ARROW_PROTOBUF_ARROW_CMAKE_PACKAGE_NAME "Arrow")
     set(ARROW_PROTOBUF_ARROW_PC_PACKAGE_NAME "arrow")
   elseif(ARROW_FLIGHT)

cpp/src/arrow/CMakeLists.txt

@@ -1175,6 +1175,7 @@ add_arrow_benchmark(builder_benchmark)
 add_arrow_benchmark(compare_benchmark)
 add_arrow_benchmark(memory_pool_benchmark)
 add_arrow_benchmark(type_benchmark)
+add_arrow_benchmark(tensor_benchmark)
 
 #
 # Recurse into sub-directories