Remove Arrow dependency from the datasource.hpp public header

conda/recipes/libcudf/meta.yaml

@@ -194,6 +194,7 @@ outputs:
         - test -f $PREFIX/include/cudf/hashing.hpp
         - test -f $PREFIX/include/cudf/hashing/detail/hashing.hpp
         - test -f $PREFIX/include/cudf/interop.hpp
+        - test -f $PREFIX/include/cudf/io/arrow_io_source.hpp
         - test -f $PREFIX/include/cudf/io/avro.hpp
         - test -f $PREFIX/include/cudf/io/csv.hpp
         - test -f $PREFIX/include/cudf/io/data_sink.hpp

cpp/include/cudf/io/arrow_io_source.hpp

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2023, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "datasource.hpp"
+
+#include <arrow/buffer.h>
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/io/file.h>
+#include <arrow/io/memory.h>
+
+#include <memory>
+#include <string>
+
+namespace cudf::io {
+/**
+ * @addtogroup io_datasources
+ * @{
+ * @file
+ */
+
+/**
+ * @brief Implementation class for reading from an Apache Arrow file. The file
+ * could be a memory-mapped file or other implementation supported by Arrow.
+ */
+class arrow_io_source : public datasource {
+  /**
+   * @brief Implementation for an owning buffer where `arrow::Buffer` holds the data.
+   */
+  class arrow_io_buffer : public buffer {
+    std::shared_ptr<arrow::Buffer> arrow_buffer;
+
+   public:
+    explicit arrow_io_buffer(std::shared_ptr<arrow::Buffer> arrow_buffer)
+      : arrow_buffer(arrow_buffer)
+    {
+    }
+    [[nodiscard]] size_t size() const override { return arrow_buffer->size(); }
+    [[nodiscard]] uint8_t const* data() const override { return arrow_buffer->data(); }
+  };
+
+ public:
+  /**
+   * @brief Constructs an object from an Apache Arrow Filesystem URI
+   *
+   * @param arrow_uri Apache Arrow Filesystem URI
+   */
+  explicit arrow_io_source(std::string const& arrow_uri)
+  {
+    std::string const uri_start_delimiter = "//";
+    std::string const uri_end_delimiter   = "?";
+
+    auto const result = arrow::fs::FileSystemFromUri(arrow_uri);
+    CUDF_EXPECTS(result.ok(), "Failed to generate Arrow Filesystem instance from URI.");
+    filesystem = result.ValueOrDie();
+
+    // Parse the path from the URI
+    size_t start          = arrow_uri.find(uri_start_delimiter) == std::string::npos
+                              ? 0
+                              : arrow_uri.find(uri_start_delimiter) + uri_start_delimiter.size();
+    size_t end            = arrow_uri.find(uri_end_delimiter) - start;
+    std::string_view path = arrow_uri.substr(start, end);
+
+    auto const in_stream = filesystem->OpenInputFile(static_cast<std::string>(path).c_str());
+    CUDF_EXPECTS(in_stream.ok(), "Failed to open Arrow RandomAccessFile");
+    arrow_file = in_stream.ValueOrDie();
+  }
+
+  /**
+   * @brief Constructs an object from an `arrow` source object.
+   *
+   * @param file The `arrow` object from which the data is read
+   */
+  explicit arrow_io_source(std::shared_ptr<arrow::io::RandomAccessFile> file) : arrow_file(file) {}
+
+  /**
+   * @brief Returns a buffer with a subset of data from the `arrow` source.
+   *
+   * @param offset The offset in bytes from which to read
+   * @param size The number of bytes to read
+   * @return A buffer with the read data
+   */
+  std::unique_ptr<buffer> host_read(size_t offset, size_t size) override
+  {
+    auto const result = arrow_file->ReadAt(offset, size);
+    CUDF_EXPECTS(result.ok(), "Cannot read file data");
+    return std::make_unique<arrow_io_buffer>(result.ValueOrDie());
+  }
+
+  /**
+   * @brief Reads a selected range from the `arrow` source into a preallocated buffer.
+   *
+   * @param[in] offset The offset in bytes from which to read
+   * @param[in] size The number of bytes to read
+   * @param[out] dst The preallocated buffer to read into
+   * @return The number of bytes read
+   */
+  size_t host_read(size_t offset, size_t size, uint8_t* dst) override
+  {
+    auto const result = arrow_file->ReadAt(offset, size, dst);
+    CUDF_EXPECTS(result.ok(), "Cannot read file data");
+    return result.ValueOrDie();
+  }
+
+  /**
+   * @brief Returns the size of the data in the `arrow` source.
+   *
+   * @return The size of the data in the `arrow` source
+   */
+  [[nodiscard]] size_t size() const override
+  {
+    auto const result = arrow_file->GetSize();
+    CUDF_EXPECTS(result.ok(), "Cannot get file size");
+    return result.ValueOrDie();
+  }
+
+ private:
+  std::shared_ptr<arrow::fs::FileSystem> filesystem;
+  std::shared_ptr<arrow::io::RandomAccessFile> arrow_file;
+};
+
+/** @} */  // end of group
+}  // namespace cudf::io

cpp/include/cudf/io/datasource.hpp

@@ -22,35 +22,6 @@
 
 #include <rmm/cuda_stream_view.hpp>
 
-#include <arrow/buffer.h>
-
-// We disable warning 611 because some Arrow subclasses of
-// `arrow::fs::FileSystem` only partially override the `Equals` method,
-// triggering warning 611-D from nvcc.
-#ifdef __CUDACC__
-#pragma nv_diag_suppress 611
-#endif
-#include <arrow/filesystem/filesystem.h>
-#include <arrow/filesystem/s3fs.h>
-#ifdef __CUDACC__
-#pragma nv_diag_default 611
-#endif
-
-// We disable warning 2810 to workaround the compile issue (warning treated as error):
-// result.h(263): error #2810-D: ignoring return value type with "nodiscard" attribute
-#ifdef __CUDACC__
-#pragma nv_diag_suppress 2810
-#endif
-#include <arrow/result.h>
-#ifdef __CUDACC__
-#pragma nv_diag_default 2810
-#endif
-
-#include <arrow/io/file.h>
-#include <arrow/io/interfaces.h>
-#include <arrow/io/memory.h>
-#include <arrow/status.h>
-
 #include <future>
 #include <memory>
 
@@ -149,15 +120,6 @@ class datasource {
    */
   static std::unique_ptr<datasource> create(cudf::device_span<std::byte const> buffer);
 
-  /**
-   * @brief Creates a source from a from an Arrow file.
-   *
-   * @param[in] arrow_file RandomAccessFile to which the API calls are forwarded
-   * @return Constructed datasource object
-   */
-  static std::unique_ptr<datasource> create(
-    std::shared_ptr<arrow::io::RandomAccessFile> arrow_file);
-
   /**
    * @brief Creates a source from an user implemented datasource object.
    *
@@ -412,108 +374,6 @@ class datasource {
   };
 };
 
-/**
- * @brief Implementation class for reading from an Apache Arrow file. The file
- * could be a memory-mapped file or other implementation supported by Arrow.
- */
-class arrow_io_source : public datasource {
-  /**
-   * @brief Implementation for an owning buffer where `arrow::Buffer` holds the data.
-   */
-  class arrow_io_buffer : public buffer {
-    std::shared_ptr<arrow::Buffer> arrow_buffer;
-
-   public:
-    explicit arrow_io_buffer(std::shared_ptr<arrow::Buffer> arrow_buffer)
-      : arrow_buffer(arrow_buffer)
-    {
-    }
-    [[nodiscard]] size_t size() const override { return arrow_buffer->size(); }
-    [[nodiscard]] uint8_t const* data() const override { return arrow_buffer->data(); }
-  };
-
- public:
-  /**
-   * @brief Constructs an object from an Apache Arrow Filesystem URI
-   *
-   * @param arrow_uri Apache Arrow Filesystem URI
-   */
-  explicit arrow_io_source(std::string_view arrow_uri)
-  {
-    std::string const uri_start_delimiter = "//";
-    std::string const uri_end_delimiter   = "?";
-
-    arrow::Result<std::shared_ptr<arrow::fs::FileSystem>> result =
-      arrow::fs::FileSystemFromUri(static_cast<std::string>(arrow_uri));
-    CUDF_EXPECTS(result.ok(), "Failed to generate Arrow Filesystem instance from URI.");
-    filesystem = result.ValueOrDie();
-
-    // Parse the path from the URI
-    size_t start          = arrow_uri.find(uri_start_delimiter) == std::string::npos
-                              ? 0
-                              : arrow_uri.find(uri_start_delimiter) + uri_start_delimiter.size();
-    size_t end            = arrow_uri.find(uri_end_delimiter) - start;
-    std::string_view path = arrow_uri.substr(start, end);
-
-    arrow::Result<std::shared_ptr<arrow::io::RandomAccessFile>> in_stream =
-      filesystem->OpenInputFile(static_cast<std::string>(path).c_str());
-    CUDF_EXPECTS(in_stream.ok(), "Failed to open Arrow RandomAccessFile");
-    arrow_file = in_stream.ValueOrDie();
-  }
-
-  /**
-   * @brief Constructs an object from an `arrow` source object.
-   *
-   * @param file The `arrow` object from which the data is read
-   */
-  explicit arrow_io_source(std::shared_ptr<arrow::io::RandomAccessFile> file) : arrow_file(file) {}
-
-  /**
-   * @brief Returns a buffer with a subset of data from the `arrow` source.
-   *
-   * @param offset The offset in bytes from which to read
-   * @param size The number of bytes to read
-   * @return A buffer with the read data
-   */
-  std::unique_ptr<buffer> host_read(size_t offset, size_t size) override
-  {
-    auto result = arrow_file->ReadAt(offset, size);
-    CUDF_EXPECTS(result.ok(), "Cannot read file data");
-    return std::make_unique<arrow_io_buffer>(result.ValueOrDie());
-  }
-
-  /**
-   * @brief Reads a selected range from the `arrow` source into a preallocated buffer.
-   *
-   * @param[in] offset The offset in bytes from which to read
-   * @param[in] size The number of bytes to read
-   * @param[out] dst The preallocated buffer to read into
-   * @return The number of bytes read
-   */
-  size_t host_read(size_t offset, size_t size, uint8_t* dst) override
-  {
-    auto result = arrow_file->ReadAt(offset, size, dst);
-    CUDF_EXPECTS(result.ok(), "Cannot read file data");
-    return result.ValueOrDie();
-  }
-
-  /**
-   * @brief Returns the size of the data in the `arrow` source.
-   *
-   * @return The size of the data in the `arrow` source
-   */
-  [[nodiscard]] size_t size() const override
-  {
-    auto result = arrow_file->GetSize();
-    CUDF_EXPECTS(result.ok(), "Cannot get file size");
-    return result.ValueOrDie();
-  }
-
- private:
-  std::shared_ptr<arrow::fs::FileSystem> filesystem;
-  std::shared_ptr<arrow::io::RandomAccessFile> arrow_file;
-};
-
 /** @} */  // end of group
 }  // namespace io
 }  // namespace cudf

cpp/include/cudf/io/types.hpp

@@ -32,13 +32,6 @@
 #include <unordered_map>
 #include <vector>
 
-// Forward declarations
-namespace arrow {
-namespace io {
-class RandomAccessFile;
-}
-}  // namespace arrow
-
 namespace cudf {
 //! IO interfaces
 namespace io {
@@ -286,8 +279,6 @@ constexpr inline auto is_byte_like_type()
  * @brief Source information for read interfaces
  */
 struct source_info {
-  std::vector<std::shared_ptr<arrow::io::RandomAccessFile>> _files;  //!< Input files
-
   source_info() = default;
 
   /**
@@ -438,12 +429,6 @@ struct source_info {
    * @return The device buffers of the input
    */
   [[nodiscard]] auto const& device_buffers() const { return _device_buffers; }
-  /**
-   * @brief Get the input files
-   *
-   * @return The input files
-   */
-  [[nodiscard]] auto const& files() const { return _files; }
   /**
    * @brief Get the user sources of the input
    *

cpp/src/io/utilities/datasource.cpp

@@ -17,6 +17,7 @@
 #include "file_io_utilities.hpp"
 
 #include <cudf/detail/utilities/vector_factories.hpp>
+#include <cudf/io/arrow_io_source.hpp>
 #include <cudf/io/datasource.hpp>
 #include <cudf/utilities/error.hpp>
 #include <cudf/utilities/span.hpp>

cpp/tests/io/arrow_io_source_test.cpp

@@ -21,7 +21,7 @@
 #include <cudf_test/table_utilities.hpp>
 #include <cudf_test/type_lists.hpp>
 
-#include <cudf/io/datasource.hpp>
+#include <cudf/io/arrow_io_source.hpp>
 #include <cudf/io/json.hpp>
 #include <cudf/io/parquet.hpp>
 

cpp/tests/io/csv_test.cpp

@@ -23,8 +23,8 @@
 
 #include <cudf/detail/iterator.cuh>
 #include <cudf/fixed_point/fixed_point.hpp>
+#include <cudf/io/arrow_io_source.hpp>
 #include <cudf/io/csv.hpp>
-#include <cudf/io/datasource.hpp>
 #include <cudf/strings/convert/convert_datetime.hpp>
 #include <cudf/strings/convert/convert_fixed_point.hpp>
 #include <cudf/strings/strings_column_view.hpp>

cpp/tests/io/json_test.cpp

@@ -23,7 +23,7 @@
 #include <cudf_test/type_lists.hpp>
 
 #include <cudf/detail/iterator.cuh>
-#include <cudf/io/datasource.hpp>
+#include <cudf/io/arrow_io_source.hpp>
 #include <cudf/io/json.hpp>
 #include <cudf/strings/convert/convert_fixed_point.hpp>
 #include <cudf/strings/strings_column_view.hpp>

python/cudf/cudf/_lib/cpp/io/arrow_io_source.pxd

@@ -0,0 +1,15 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+from libcpp.memory cimport shared_ptr
+from libcpp.string cimport string
+from pyarrow.includes.libarrow cimport CRandomAccessFile
+
+cimport cudf._lib.cpp.io.datasource as cudf_io_datasource
+
+
+cdef extern from "cudf/io/arrow_io_source.hpp" \
+        namespace "cudf::io" nogil:
+
+    cdef cppclass arrow_io_source(cudf_io_datasource.datasource):
+        arrow_io_source(const string& arrow_uri) except +
+        arrow_io_source(shared_ptr[CRandomAccessFile]) except +