clean up

cmake RemoteHandle::read() remove StreamFuture(StreamFuture&&) python bindings and tests read to device memory dependencies: aws-sdk-cpp parse_s3_path cmake: adding AWSSDK to python built RemoteFile.from_url() benchmark benchmark: --api numpy ibenchmark: --api cudf cleanup benchmark: adding --api cudf-fsspec read(): use PushAndPopContext impl. pread ensure_aws_s3_api_is_initalized AwsS3Client S3Context clean up SameThreadExecutor use shared pointer cmake: AWSSDK COMPONENTS s3 transfer create_transfer_manager clean up BufferAsStream test_read: larger data size KVIKIO_NVTX_FUNC_RANGE benchmark: use random RemoteHandle::read_to_host(): print bandwidth benchmark clean up remove the use of the transfer module ci: some more aws-sdk-cpp benchmark: pytest.importorskip("moto") remote_handle.pyx remote_file.py make remote IO optional don't use typing_extensions dependencies: boto3 and moto more aws-sdk-cpp trigger CI error if remote module wasn't built
rapidsai · Aug 13, 2024 · 32130c2 · 32130c2
1 parent ef2a704
commit 32130c2
Show file tree

Hide file tree

Showing 19 changed files with 872 additions and 13 deletions.
diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml
@@ -6,6 +6,8 @@ channels:
 - conda-forge
 - nvidia
 dependencies:
+- aws-sdk-cpp
+- boto3>=1.21.21
 - c-compiler
 - cmake>=3.26.4,!=3.30.0
 - cuda-python>=11.7.1,<12.0a0
@@ -17,6 +19,7 @@ dependencies:
 - dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-aarch64=11.*
+- moto>=4.0.8
 - ninja
 - numcodecs <0.12.0
 - numpy>=1.23,<2.0a0

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -6,6 +6,8 @@ channels:
 - conda-forge
 - nvidia
 dependencies:
+- aws-sdk-cpp
+- boto3>=1.21.21
 - c-compiler
 - cmake>=3.26.4,!=3.30.0
 - cuda-python>=11.7.1,<12.0a0
@@ -19,6 +21,7 @@ dependencies:
 - gcc_linux-64=11.*
 - libcufile-dev=1.4.0.31
 - libcufile=1.4.0.31
+- moto>=4.0.8
 - ninja
 - numcodecs <0.12.0
 - numpy>=1.23,<2.0a0

diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml
@@ -6,6 +6,8 @@ channels:
 - conda-forge
 - nvidia
 dependencies:
+- aws-sdk-cpp
+- boto3>=1.21.21
 - c-compiler
 - cmake>=3.26.4,!=3.30.0
 - cuda-nvcc
@@ -18,6 +20,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-aarch64=11.*
 - libcufile-dev
+- moto>=4.0.8
 - ninja
 - numcodecs <0.12.0
 - numpy>=1.23,<2.0a0

diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml
@@ -6,6 +6,8 @@ channels:
 - conda-forge
 - nvidia
 dependencies:
+- aws-sdk-cpp
+- boto3>=1.21.21
 - c-compiler
 - cmake>=3.26.4,!=3.30.0
 - cuda-nvcc
@@ -18,6 +20,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - libcufile-dev
+- moto>=4.0.8
 - ninja
 - numcodecs <0.12.0
 - numpy>=1.23,<2.0a0

diff --git a/conda/recipes/kvikio/meta.yaml b/conda/recipes/kvikio/meta.yaml
@@ -52,6 +52,7 @@ requirements:
     - {{ compiler('cuda') }}
     {% endif %}
     - {{ stdlib("c") }}
+    - aws-sdk-cpp
   host:
     - python
     - pip
@@ -64,11 +65,13 @@ requirements:
     - rapids-build-backend >=0.3.0,<0.4.0.dev0
     - scikit-build-core >=0.10.0
     - libkvikio ={{ version }}
+    - aws-sdk-cpp
   run:
     - python
     - numpy >=1.23,<2.0a0
     - cupy >=12.0.0
     - zarr
+    - aws-sdk-cpp
     # See https://github.com/zarr-developers/numcodecs/pull/475
     - numcodecs <0.12.0
     - packaging

diff --git a/conda/recipes/libkvikio/meta.yaml b/conda/recipes/libkvikio/meta.yaml
@@ -43,6 +43,7 @@ requirements:
     {% endif %}
     - ninja
     - {{ stdlib("c") }}
+    - aws-sdk-cpp
   host:
     - cuda-version ={{ cuda_version }}
     {% if cuda_major == "11" %}
@@ -52,6 +53,7 @@ requirements:
     {% else %}
     - libcufile-dev  # [linux]
     {% endif %}
+    - aws-sdk-cpp
 
 outputs:
   - name: libkvikio
@@ -72,8 +74,10 @@ outputs:
     requirements:
       build:
         - cmake {{ cmake_version }}
+        - aws-sdk-cpp
       host:
         - cuda-version ={{ cuda_version }}
+        - aws-sdk-cpp
       run:
         - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
         {% if cuda_major == "11" %}
@@ -83,6 +87,7 @@ outputs:
         {% else %}
         - libcufile-dev  # [linux]
         {% endif %}
+        - aws-sdk-cpp
     test:
         commands:
           - test -f $PREFIX/include/kvikio/file_handle.hpp
@@ -106,6 +111,7 @@ outputs:
         - cuda-cudart-dev
         - libcufile-dev  # [linux]
         {% endif %}
+        - aws-sdk-cpp
     requirements:
       build:
         - cmake {{ cmake_version }}
@@ -118,6 +124,7 @@ outputs:
         - cuda-cudart-dev
         - libcufile-dev  # [linux]
         {% endif %}
+        - aws-sdk-cpp
       run:
         - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
         {% if cuda_major == "11" %}
@@ -127,6 +134,7 @@ outputs:
         - cuda-cudart
         - libcufile  # [linux]
         {% endif %}
+        - aws-sdk-cpp
     about:
       home: https://rapids.ai
       license: Apache-2.0

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -55,6 +55,12 @@ rapids_find_package(
   INSTALL_EXPORT_SET kvikio-exports
 )
 
+rapids_find_package(
+  AWSSDK COMPONENTS s3
+  BUILD_EXPORT_SET kvikio-exports
+  INSTALL_EXPORT_SET kvikio-exports
+)
+
 rapids_find_package(
   cuFile
   BUILD_EXPORT_SET kvikio-exports
@@ -130,6 +136,7 @@ target_include_directories(
 )
 target_link_libraries(
   kvikio INTERFACE Threads::Threads ${CMAKE_DL_LIBS} nvtx3::nvtx3-cpp BS::thread_pool
+                   ${AWSSDK_LINK_LIBRARIES}
 )
 target_compile_features(kvikio INTERFACE cxx_std_17)
 

diff --git a/cpp/examples/CMakeLists.txt b/cpp/examples/CMakeLists.txt
@@ -14,6 +14,8 @@
 
 set(TEST_INSTALL_PATH bin/tests/libkvikio)
 
+# Example: basic_io
+
 if(CUDAToolkit_FOUND)
   add_executable(BASIC_IO_TEST basic_io.cpp)
   set_target_properties(BASIC_IO_TEST PROPERTIES INSTALL_RPATH "\$ORIGIN/../../lib")
@@ -35,6 +37,8 @@ else()
   message(STATUS "Cannot build the basic_io example when CUDA is not found")
 endif()
 
+# Example: basic_no_cuda
+
 add_executable(BASIC_NO_CUDA_TEST basic_no_cuda.cpp)
 set_target_properties(BASIC_NO_CUDA_TEST PROPERTIES INSTALL_RPATH "\$ORIGIN/../../lib")
 target_include_directories(BASIC_NO_CUDA_TEST PRIVATE ../include)

diff --git a/cpp/include/kvikio/file_handle.hpp b/cpp/include/kvikio/file_handle.hpp
@@ -144,7 +144,7 @@ class FileHandle {
   bool _initialized{false};
   bool _compat_mode{false};
   mutable std::size_t _nbytes{0};  // The size of the underlying file, zero means unknown.
-  CUfileHandle_t _handle{};
+  CUfileHandle_t _handle{nullptr};
 
  public:
   static constexpr mode_t m644 = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH;
@@ -208,7 +208,7 @@ class FileHandle {
       _initialized{std::exchange(o._initialized, false)},
       _compat_mode{std::exchange(o._compat_mode, false)},
       _nbytes{std::exchange(o._nbytes, 0)},
-      _handle{std::exchange(o._handle, CUfileHandle_t{})}
+      _handle{std::exchange(o._handle, CUfileHandle_t{nullptr})}
   {
   }
   FileHandle& operator=(FileHandle&& o) noexcept
@@ -218,7 +218,7 @@ class FileHandle {
     _initialized   = std::exchange(o._initialized, false);
     _compat_mode   = std::exchange(o._compat_mode, false);
     _nbytes        = std::exchange(o._nbytes, 0);
-    _handle        = std::exchange(o._handle, CUfileHandle_t{});
+    _handle        = std::exchange(o._handle, CUfileHandle_t{nullptr});
     return *this;
   }
   ~FileHandle() noexcept { close(); }
@@ -232,8 +232,8 @@ class FileHandle {
   {
     if (closed()) { return; }
 
-    if (!_compat_mode) { cuFileAPI::instance().HandleDeregister(_handle); }
-    ::close(_fd_direct_off);
+    if (_handle != nullptr) { cuFileAPI::instance().HandleDeregister(_handle); }
+    if (_fd_direct_off != -1) { ::close(_fd_direct_off); }
     if (_fd_direct_on != -1) { ::close(_fd_direct_on); }
     _fd_direct_on  = -1;
     _fd_direct_off = -1;