diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake
index 6d889f9f181..c0646e44f4b 100644
--- a/cpp/cmake/thirdparty/get_arrow.cmake
+++ b/cpp/cmake/thirdparty/get_arrow.cmake
@@ -32,7 +32,7 @@ function(find_libarrow_in_python_wheel PYARROW_VERSION)
   # `${MINOR_VERSION}${PATCH_VERSION}` is almost always equivalent to "00"),
   # the soname is not generated by concatenating the major, minor, and patch versions into a single
   # version number soname, just `${MAJOR_VERSION}00`
-  set(PYARROW_LIB "libarrow.${CMAKE_SHARED_LIBRARY_SUFFIX}.${PYARROW_SO_VER}00")
+  set(PYARROW_LIB "libarrow.so.${PYARROW_SO_VER}00")
 
   execute_process(
     COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_library_dirs()[0])"
@@ -451,7 +451,7 @@ if(${Python_FOUND})
       OUTPUT_VARIABLE _pyarrow_lib_dir
       OUTPUT_STRIP_TRAILING_WHITESPACE
     )
-    file(GLOB _pyarrow_libs "${_pyarrow_lib_dir}/libarrow.${CMAKE_SHARED_LIBRARY_SUFFIX}*")
+    file(GLOB _pyarrow_libs "${_pyarrow_lib_dir}/libarrow.so*")
     list(LENGTH _pyarrow_libs _pyarrow_libs_len)
     if(_pyarrow_libs_len GREATER 0)
       set(USE_LIBARROW_FROM_PYARROW ON)