GH-41910: [Python] Add support for Pyodide

python/CMakeLists.txt

@@ -1,5 +1,5 @@
 # Licensed to the Apache Software Foundation (ASF) under one
-# or more cod ntributor license agreements.  See the NOTICE file
+# or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
@@ -68,6 +68,25 @@ if(POLICY CMP0095)
   cmake_policy(SET CMP0095 NEW)
 endif()
 
+# this option is used to auto-set defaults for pyarrow build
+option(DUMP_ARROW_ARGUMENTS "Dump the arrow arguments then quit" OFF)
+
+if($ENV{PYODIDE})
+  # these variables are needed for building pyarrow on Emscripten
+  # if they aren't set, cmake cross compiling fails for python
+  # modules (at least under pyodide it does)
+  set(Python3_INCLUDE_DIR $ENV{PYTHONINCLUDE})
+  set(Python3_LIBRARY $ENV{CPYTHONLIB})
+  set(Python3_NumPy_INCLUDE_DIR $ENV{NUMPY_LIB}/core/include)
+  set(Python3_EXECUTABLE)
+  set(ENV{_PYTHON_SYSCONFIGDATA_NAME} $ENV{SYSCONFIG_NAME})
+
+  # remove any c++ standard settings from cflags and let cmake set it.
+  # n.b. this is here because pyodide sets -stdc++=14, which breaks pyarrow builds
+  string(REGEX REPLACE "-std=c\\+\\+[0-9]+" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+
+endif()
+
 # Use the first Python installation on PATH, not the newest one
 set(Python3_FIND_STRATEGY "LOCATION")
 # On Windows, use registry last, not first
@@ -260,6 +279,22 @@ include(GNUInstallDirs)
 
 find_package(Arrow REQUIRED)
 
+if(DUMP_ARROW_ARGUMENTS)
+  message(STATUS "----- ARROW_SETTINGS_DUMP -----")
+  get_cmake_property(_variableNames VARIABLES)
+  list(SORT _variableNames)
+  foreach(_variableName ${_variableNames})
+    unset(MATCHED)
+    string(REGEX MATCH ^ARROW_.* MATCHED ${_variableName})
+    if(NOT MATCHED)
+      continue()
+    endif()
+    message(STATUS "${_variableName}=${${_variableName}}")
+  endforeach()
+  message(STATUS "----- ARROW_SETTINGS_END -----")
+  return()
+endif()
+
 set(PYARROW_CPP_ROOT_DIR pyarrow/src)
 set(PYARROW_CPP_SOURCE_DIR ${PYARROW_CPP_ROOT_DIR}/arrow/python)
 set(PYARROW_CPP_SRCS
@@ -570,28 +605,42 @@ endif()
 
 # Acero
 if(PYARROW_BUILD_ACERO)
-  if(PYARROW_BUNDLE_ARROW_CPP)
-    bundle_arrow_lib(${ARROW_ACERO_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION})
-    if(MSVC)
-      bundle_arrow_import_lib(${ARROW_ACERO_IMPORT_LIB})
+  if(ARROW_BUILD_SHARED)
+
+    if(PYARROW_BUNDLE_ARROW_CPP)
+      bundle_arrow_lib(${ARROW_ACERO_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION})
+      if(MSVC)
+        bundle_arrow_import_lib(${ARROW_ACERO_IMPORT_LIB})
+      endif()
     endif()
-  endif()
 
-  set(ACERO_LINK_LIBS ArrowAcero::arrow_acero_shared)
-  list(APPEND CYTHON_EXTENSIONS _acero)
+    set(ACERO_LINK_LIBS ArrowAcero::arrow_acero_shared)
+    list(APPEND CYTHON_EXTENSIONS _acero)
+  else()
+    # ACERO is statically linked into libarrow_python already
+    set(ACERO_LINK_LIBS)
+    list(APPEND CYTHON_EXTENSIONS _acero)
+  endif()
 endif()
 
 # Dataset
 if(PYARROW_BUILD_DATASET)
-  if(PYARROW_BUNDLE_ARROW_CPP)
-    bundle_arrow_lib(${ARROW_DATASET_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION})
-    if(MSVC)
-      bundle_arrow_import_lib(${ARROW_DATASET_IMPORT_LIB})
+  if(ARROW_BUILD_SHARED)
+
+    if(PYARROW_BUNDLE_ARROW_CPP)
+      bundle_arrow_lib(${ARROW_DATASET_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION})
+      if(MSVC)
+        bundle_arrow_import_lib(${ARROW_DATASET_IMPORT_LIB})
+      endif()
     endif()
-  endif()
 
-  set(DATASET_LINK_LIBS ArrowDataset::arrow_dataset_shared)
+    set(DATASET_LINK_LIBS ArrowDataset::arrow_dataset_shared)
+  else()
+    # dataset is sttically linked into libarrow_python already
+    set(DATASET_LINK_LIBS)
+  endif()
   list(APPEND CYTHON_EXTENSIONS _dataset)
+
 endif()
 
 # Parquet
@@ -612,7 +661,9 @@ if(PYARROW_BUILD_PARQUET)
     endif()
     set(PARQUET_LINK_LIBS Parquet::parquet_shared)
   else()
-    set(PARQUET_LINK_LIBS Parquet::parquet_static)
+    # parquet is linked into libarrow_python already
+    # so isn't needed in the extension
+    set(PARQUET_LINK_LIBS "")
   endif()
   list(APPEND CYTHON_EXTENSIONS _parquet)
   if(PYARROW_BUILD_PARQUET_ENCRYPTION)

python/pyarrow/_dataset_parquet.pyx

@@ -27,7 +27,7 @@ import warnings
 
 import pyarrow as pa
 from pyarrow.lib cimport *
-from pyarrow.lib import frombytes, tobytes
+from pyarrow.lib import frombytes, tobytes, is_threading_enabled
 from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.libarrow_dataset cimport *
 from pyarrow.includes.libarrow_dataset_parquet cimport *
@@ -687,6 +687,8 @@ cdef class ParquetFragmentScanOptions(FragmentScanOptions):
             new CParquetFragmentScanOptions()))
         self.use_buffered_stream = use_buffered_stream
         self.buffer_size = buffer_size
+        if pre_buffer and not is_threading_enabled():
+            pre_buffer=False
         self.pre_buffer = pre_buffer
         if thrift_string_size_limit is not None:
             self.thrift_string_size_limit = thrift_string_size_limit
@@ -730,6 +732,8 @@ cdef class ParquetFragmentScanOptions(FragmentScanOptions):
 
     @pre_buffer.setter
     def pre_buffer(self, bint pre_buffer):
+        if pre_buffer and not is_threading_enabled():
+            return
         self.arrow_reader_properties().set_pre_buffer(pre_buffer)
 
     @property

python/pyarrow/_parquet.pyx

@@ -38,7 +38,7 @@ from pyarrow.lib cimport (_Weakrefable, Buffer, Schema,
 
 from pyarrow.lib import (ArrowException, NativeFile, BufferOutputStream,
                          _stringify_path,
-                         tobytes, frombytes)
+                         tobytes, frombytes, is_threading_enabled)
 
 cimport cpython as cp
 
@@ -1206,6 +1206,9 @@ cdef class ParquetReader(_Weakrefable):
                 default_arrow_reader_properties())
             FileReaderBuilder builder
 
+        if pre_buffer and not is_threading_enabled():
+            pre_buffer=False
+
         if metadata is not None:
             c_metadata = metadata.sp_metadata
 

python/pyarrow/includes/libarrow_python.pxd

@@ -301,3 +301,6 @@ cdef extern from "arrow/python/benchmark.h" namespace "arrow::py::benchmark":
 
 cdef extern from "arrow/python/gdb.h" namespace "arrow::gdb" nogil:
     void GdbTestSession "arrow::gdb::TestSession"()
+
+cdef extern from "arrow/python/helpers.h" namespace "arrow::py::internal":
+    c_bool IsThreadingEnabled()

python/pyarrow/io.pxi

@@ -32,7 +32,6 @@ from queue import Queue, Empty as QueueEmpty
 
 from pyarrow.util import _is_path_like, _stringify_path
 
-
 # 64K
 DEFAULT_BUFFER_SIZE = 2 ** 16
 
@@ -677,16 +676,19 @@ cdef class NativeFile(_Weakrefable):
 
         self.seek(0)
 
-        writer_thread = threading.Thread(target=bg_write)
+        if is_threading_enabled():
+            writer_thread = threading.Thread(target=bg_write)
+        else:
+            writer_thread = None
 
         # This isn't ideal -- PyBytes_FromStringAndSize copies the data from
         # the passed buffer, so it's hard for us to avoid doubling the memory
         buf = <uint8_t*> malloc(buffer_size)
         if buf == NULL:
             raise MemoryError("Failed to allocate {0} bytes"
                               .format(buffer_size))
-
-        writer_thread.start()
+        if writer_thread:
+            writer_thread.start()
 
         cdef int64_t total_bytes = 0
         cdef int32_t c_buffer_size = buffer_size
@@ -706,18 +708,23 @@ cdef class NativeFile(_Weakrefable):
                 pybuf = cp.PyBytes_FromStringAndSize(<const char*>buf,
                                                      bytes_read)
 
-                if writer_thread.is_alive():
-                    while write_queue.full():
-                        time.sleep(0.01)
-                else:
-                    break
+                if writer_thread is not None:
+                    if writer_thread.is_alive():
+                        while write_queue.full():
+                            time.sleep(0.01)
+                    else:
+                        break
 
-                write_queue.put_nowait(pybuf)
+                    write_queue.put_nowait(pybuf)
+
+                else:
+                    # no background thread
+                    stream.write(pybuf)
         finally:
             free(buf)
             done = True
-
-        writer_thread.join()
+        if writer_thread is not None:
+            writer_thread.join()
         if exc_info is not None:
             raise exc_info[0], exc_info[1], exc_info[2]
 
@@ -753,26 +760,33 @@ cdef class NativeFile(_Weakrefable):
             except Exception as e:
                 exc_info = sys.exc_info()
 
-        writer_thread = threading.Thread(target=bg_write)
-        writer_thread.start()
+        if is_threading_enabled():
+            writer_thread = threading.Thread(target=bg_write)
+            writer_thread.start()
+        else:
+            writer_thread = None
 
         try:
             while True:
                 buf = stream.read(buffer_size)
                 if not buf:
                     break
 
-                if writer_thread.is_alive():
-                    while write_queue.full():
-                        time.sleep(0.01)
-                else:
-                    break
+                if writer_thread is not None:
+                    if writer_thread.is_alive():
+                        while write_queue.full():
+                            time.sleep(0.01)
+                    else:
+                        break
 
-                write_queue.put_nowait(buf)
+                    write_queue.put_nowait(buf)
+                else:
+                    # no threading
+                    self.write(buf)
         finally:
             done = True
-
-        writer_thread.join()
+        if writer_thread is not None:
+            writer_thread.join()
         if exc_info is not None:
             raise exc_info[0], exc_info[1], exc_info[2]
 

python/pyarrow/lib.pyx

@@ -79,6 +79,10 @@ def set_cpu_count(int count):
     check_status(SetCpuThreadPoolCapacity(count))
 
 
+def is_threading_enabled():
+    return libarrow_python.IsThreadingEnabled()
+
+
 Type_NA = _Type_NA
 Type_BOOL = _Type_BOOL
 Type_UINT8 = _Type_UINT8

python/pyarrow/pandas_compat.py

@@ -33,7 +33,7 @@
 import numpy as np
 
 import pyarrow as pa
-from pyarrow.lib import _pandas_api, frombytes  # noqa
+from pyarrow.lib import _pandas_api, frombytes, is_threading_enabled  # noqa
 
 
 _logical_type_map = {}
@@ -608,7 +608,7 @@ def _can_definitely_zero_copy(arr):
                 arr.flags.contiguous and
                 issubclass(arr.dtype.type, np.integer))
 
-    if nthreads == 1:
+    if nthreads == 1 or not is_threading_enabled():
         arrays = [convert_column(c, f)
                   for c, f in zip(columns_to_convert, convert_fields)]
     else: