Merge #265

265: Use list_objects_v2 to check isdir r=fchorney a=fchorney

isdir for an s3 directory currently calls `s3_list_buckets` to check if a top level bucket is a directory. I ran into an issue where the role I was using didn't have permissions to call `list_buckets`.

This change essentially calls `S3.list_objects_v2` with `max-keys` set to `0` as we don't actually need to retrieve any keys. If the bucket exists and you have permissions for it, it will return with no error, but if the bucket doesn't exist we return false or raise an exception if some other error occurred. 

Not too sure if this is exactly how we want to go about this (re: the exceptions), so if you have any suggestions let me know. I don't think there should be any issues for using `S3.list_objects_v2` instead of `s3_list_buckets`. 

Co-authored-by: Fernando Chorney <[email protected]>
Co-authored-by: Fernando Chorney <[email protected]>

Project.toml

@@ -1,6 +1,6 @@
 name = "AWSS3"
 uuid = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
-version = "0.9.8"
+version = "0.10.0"
 
 [deps]
 AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
@@ -11,6 +11,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 EzXML = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615"
 FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
 MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
+Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Retry = "20febd7b-183b-5ae2-ac4a-720e7ce64774"
 SymDict = "2da68c74-98d7-5633-99d6-8493888d7b1e"
@@ -26,7 +27,8 @@ EzXML = "0.9, 1"
 FilePathsBase = "=0.9.11, =0.9.12, =0.9.13, =0.9.14, =0.9.15"
 JSON3 = "1"
 MbedTLS = "0.6, 0.7, 1"
-Minio = "0.1"
+Minio = "0.1, 0.2"
+Mocking = "0.7"
 OrderedCollections = "1"
 Retry = "0.3, 0.4"
 SymDict = "0.3"

src/AWSS3.jl

@@ -39,18 +39,19 @@ export S3Path,
 using AWS
 using AWS.AWSServices: s3
 using ArrowTypes
+using Base64
+using Compat: @something
+using Dates
+using EzXML
 using FilePathsBase
 using FilePathsBase: /, join
+using Mocking
 using OrderedCollections: OrderedDict, LittleDict
-using SymDict
 using Retry
-using XMLDict
-using EzXML
-using Dates
-using Base64
-using UUIDs
+using SymDict
 using URIs
-using Compat: @something
+using UUIDs
+using XMLDict
 
 @service S3 use_response_type = true
 

src/s3path.jl

@@ -239,8 +239,18 @@ Base.isfile(fp::S3Path) = !fp.isdirectory && exists(fp)
 function Base.isdir(fp::S3Path)
     fp.isdirectory || return false
     if isempty(fp.segments)  # special handling of buckets themselves
-        # may not be super efficient for those with billions of buckets, but really our best option
-        fp.bucket ∈ s3_list_buckets(get_config(fp))
+        try
+            @mock S3.list_objects_v2(
+                fp.bucket, Dict("max-keys" => "0"); aws_config=get_config(fp)
+            )
+            return true
+        catch e
+            if ecode(e) == "NoSuchBucket"
+                return false
+            else
+                rethrow()
+            end
+        end
     else
         exists(fp)
     end

test/runtests.jl

@@ -2,14 +2,17 @@ using AWS
 using AWS.AWSExceptions: AWSException
 using AWSS3
 using Arrow
-using Test
 using Dates
-using Retry
 using FilePathsBase
-using FilePathsBase: /, join
 using FilePathsBase.TestPaths
-using UUIDs: uuid4
+using FilePathsBase: /, join
 using JSON3
+using Mocking
+using Retry
+using Test
+using UUIDs: uuid4
+
+Mocking.activate()
 
 @service S3 use_response_type = true
 

test/s3path.jl

@@ -421,6 +421,44 @@ function s3path_tests(config)
         end
     end
 
+    @testset "isdir" begin
+        function _generate_exception(code)
+            return AWSException(
+                code, "", nothing, AWS.HTTP.Exceptions.StatusError(404, "", "", ""), nothing
+            )
+        end
+
+        s3_path = S3Path("s3://$(bucket_name)"; config=config)
+
+        @testset "top level bucket" begin
+            @testset "success" begin
+                @test isdir(s3_path) == true
+            end
+
+            @testset "NoSuchBucket" begin
+                test_exception = _generate_exception("NoSuchBucket")
+                patch = @patch function AWSS3.S3.list_objects_v2(args...; kwargs...)
+                    throw(test_exception)
+                end
+
+                apply(patch) do
+                    @test isdir(s3_path) == false
+                end
+            end
+
+            @testset "Other Exception" begin
+                test_exception = _generate_exception("TestException")
+                patch = @patch function AWSS3.S3.list_objects_v2(args...; kwargs...)
+                    throw(test_exception)
+                end
+
+                apply(patch) do
+                    @test_throws AWSException isdir(s3_path)
+                end
+            end
+        end
+    end
+
     @testset "JSON roundtripping" begin
         json_path = S3Path("s3://$(bucket_name)/test_json"; config=config)
         my_dict = Dict("key" => "value", "key2" => 5.0)