Skip to content

Commit

Permalink
Set prefix, delimiter params even when empty
Browse files Browse the repository at this point in the history 
We have never set values which are empty on the request
because they are perhaps not useful in the List query,
but this assumption is wrong when there are restricted
policies for a given user, because empty is actually
a valid value in IAM or Bucket policy conditions.

For example following condition would never work with our
ListObjects call and AWS cli would work fine.

            "Condition": {
                "StringEquals": {
                    "s3:prefix": [
                        "",
                        "data/",
                        "data"
                    ],
                    "s3:delimiter": [
                        "/",
                        ""
                    ]
                }
            }

The reason is empty or not prefix and delimiter should be
added to the query param in List operation, such that server
can use the value to validate the policies for the incoming
request.

Refer minio/minio-go#1064
  • Loading branch information
@harshavardhana
harshavardhana committed Jan 31, 2019
1 parent db8313d commit a118b2c
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 110 deletions.
66 changes: 33 additions & 33 deletions Minio.Functional.Tests/FunctionalTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ private static String CreateFile(int size, string dataFileName = null)
File.WriteAllBytes(fileName, data);
return GetFilePath(fileName);
}

return GetFilePath(dataFileName);
}

Expand All @@ -94,8 +94,8 @@ public static String GetRandomName(int length = 5)
}
return "miniodotnet" + result.ToString();
}
// Return true if running in Mint mode
public static bool IsMintEnv()
// Return true if running in Mint mode
public static bool IsMintEnv()
{
return !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("MINT_DATA_DIR"));
}
Expand Down Expand Up @@ -146,13 +146,13 @@ public static void Main(string[] args)
else
minioClient = new MinioClient(endPoint, accessKey, secretKey);

// Assign parameters before starting the test
// Assign parameters before starting the test
string bucketName = GetRandomName();
string objectName = GetRandomName();
string destBucketName = GetRandomName();
string destObjectName = GetRandomName();

// Set app Info
// Set app Info
minioClient.SetAppInfo("app-name", "app-version");
// Set HTTP Tracing On
// minioClient.SetTraceOn(new JsonNetLogger());
Expand Down Expand Up @@ -277,7 +277,7 @@ private static void runCoreTests(MinioClient minioClient)
ListObjects_Test1(minioClient).Wait();
RemoveObject_Test1(minioClient).Wait();
CopyObject_Test1(minioClient).Wait();

// Test SetPolicyAsync function
SetBucketPolicy_Test1(minioClient).Wait();

Expand Down Expand Up @@ -313,7 +313,7 @@ private async static Task BucketExists_Test(MinioClient minio)
catch (MinioException ex)
{
new MintLogger("BucketExists_Test",bucketExistsSignature,"Tests whether BucketExists passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
}
}
}

private async static Task MakeBucket_Test1(MinioClient minio)
Expand All @@ -337,7 +337,7 @@ private async static Task MakeBucket_Test1(MinioClient minio)
catch (MinioException ex)
{
new MintLogger("MakeBucket_Test1",makeBucketSignature,"Tests whether MakeBucket passes",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message,ex.ToString(),args).Log();
}
}
}

private async static Task MakeBucket_Test2(MinioClient minio)
Expand All @@ -363,7 +363,7 @@ private async static Task MakeBucket_Test2(MinioClient minio)
{
new MintLogger("MakeBucket_Test2",makeBucketSignature,testType,TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
}

}
private async static Task MakeBucket_Test3(MinioClient minio, bool aws = false)
{
Expand Down Expand Up @@ -504,7 +504,7 @@ private async static Task PutObject_Test1(MinioClient minio)
}
}


private async static Task PutObject_Test2(MinioClient minio)
{
DateTime startTime = DateTime.Now;
Expand Down Expand Up @@ -671,7 +671,7 @@ private async static Task PutObject_Test7(MinioClient minio)
await Setup_Test(minio, bucketName);
using (System.IO.MemoryStream filestream = rsg.GenerateStreamFromSeed(1 * MB))
{

long size = -1;
long file_write_size = filestream.Length;

Expand Down Expand Up @@ -744,7 +744,7 @@ private async static Task PutGetStatEncryptedObject_Test1(MinioClient minio)
};
try
{
// Putobject with SSE-C encryption.
// Putobject with SSE-C encryption.
await Setup_Test(minio, bucketName);
Aes aesEncryption = Aes.Create();
aesEncryption.KeySize = 256;
Expand Down Expand Up @@ -1268,7 +1268,7 @@ await minio.PutObjectAsync(bucketName,
new MintLogger("CopyObject_Test5",copyObjectSignature,"Tests whether CopyObject multi-part copy upload for large files works",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
}
}

}

private async static Task CopyObject_Test6(MinioClient minio)
Expand Down Expand Up @@ -1702,7 +1702,7 @@ await minio.GetObjectAsync(bucketName, objectName,
{
new MintLogger("GetObject_Test1",getObjectSignature1,"Tests whether GetObject as stream works",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
}

}
private async static Task GetObject_Test2(MinioClient minio)
{
Expand Down Expand Up @@ -1736,7 +1736,7 @@ private async static Task GetObject_Test2(MinioClient minio)
{
new MintLogger("GetObject_Test2",getObjectSignature1,"Tests for non-existent GetObject",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
}

}
private async static Task GetObject_Test3(MinioClient minio)
{
Expand Down Expand Up @@ -1822,7 +1822,7 @@ await minio.PutObjectAsync(bucketName,
{
new MintLogger("FGetObject_Test1",getObjectSignature3,"Tests whether FGetObject passes for small upload",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
}

}

private async static Task FPutObject_Test1(MinioClient minio)
Expand Down Expand Up @@ -1913,8 +1913,8 @@ private async static Task ListObjects_Test1(MinioClient minio)
tasks[i]= PutObject_Task(minio, bucketName, objectName + i.ToString(), null, null, 0, null, rsg.GenerateStreamFromSeed(1*MB));
}
await Task.WhenAll(tasks);
ListObjects_Test(minio, bucketName, prefix, 2,false).Wait();

ListObjects_Test(minio, bucketName, prefix, 2, false).Wait();
System.Threading.Thread.Sleep(5000);
await minio.RemoveObjectAsync(bucketName, objectName + "0");
await minio.RemoveObjectAsync(bucketName, objectName + "1");
Expand All @@ -1939,7 +1939,7 @@ private async static Task ListObjects_Test2(MinioClient minio)
{
await Setup_Test(minio, bucketName);

ListObjects_Test(minio, bucketName, null, 0).Wait(5000);
ListObjects_Test(minio, bucketName, "", 0).Wait(5000);
await TearDown(minio, bucketName);
new MintLogger("ListObjects_Test2",listObjectsSignature,"Tests whether ListObjects passes when bucket is empty",TestStatus.PASS,(DateTime.Now - startTime),args:args).Log();
}
Expand Down Expand Up @@ -1971,7 +1971,7 @@ private async static Task ListObjects_Test3(MinioClient minio)
}
await Task.WhenAll(tasks);

ListObjects_Test(minio, bucketName, prefix, 2,true).Wait();
ListObjects_Test(minio, bucketName, prefix, 2, true).Wait();
System.Threading.Thread.Sleep(5000);
await minio.RemoveObjectAsync(bucketName, objectName + "0");
await minio.RemoveObjectAsync(bucketName, objectName + "1");
Expand Down Expand Up @@ -2003,8 +2003,8 @@ private async static Task ListObjects_Test4(MinioClient minio)
tasks[i]= PutObject_Task(minio, bucketName, objectName + i.ToString(), null, null, 0, null, rsg.GenerateStreamFromSeed(1*MB));
}
await Task.WhenAll(tasks);
ListObjects_Test(minio, bucketName, null, 2,false).Wait();

ListObjects_Test(minio, bucketName, "", 2, false).Wait();
System.Threading.Thread.Sleep(5000);
await minio.RemoveObjectAsync(bucketName, objectName + "0");
await minio.RemoveObjectAsync(bucketName, objectName + "1");
Expand Down Expand Up @@ -2040,8 +2040,8 @@ private async static Task ListObjects_Test5(MinioClient minio)
}
}
await Task.WhenAll(tasks);
ListObjects_Test(minio, bucketName, objectNamePrefix, numObjects,false).Wait();

ListObjects_Test(minio, bucketName, objectNamePrefix, numObjects, false).Wait();
System.Threading.Thread.Sleep(5000);
for(int index=1; index <= numObjects; index++)
{
Expand Down Expand Up @@ -2364,7 +2364,7 @@ await minio.PutObjectAsync(bucketName,
new MintLogger("PresignedPutObject_Test2",presignedPutObjectSignature,"Tests whether PresignedPutObject url retrieves object from bucket when invalid expiry is set.",TestStatus.FAIL,(DateTime.Now - startTime),"",ex.Message, ex.ToString(),args).Log();
}
}

private static async Task UploadObjectAsync(string url, string filePath)
{
HttpWebRequest httpRequest = WebRequest.Create(url) as HttpWebRequest;
Expand Down Expand Up @@ -2406,7 +2406,7 @@ private async static Task PresignedPostPolicy_Test1(MinioClient minio)
await Setup_Test(minio, bucketName);
await minio.PutObjectAsync(bucketName,
objectName,
fileName);
fileName);
var pairs = new List<KeyValuePair<string, string>>();
string url = "https://s3.amazonaws.com/" + bucketName;
Tuple<string, System.Collections.Generic.Dictionary<string, string>> policyTuple = await minio.PresignedPostPolicyAsync(form);
Expand Down Expand Up @@ -2477,7 +2477,7 @@ await minio.PutObjectAsync(bucketName,

IDisposable subscription = observable.Subscribe(
item => Assert.AreEqual(item.Key, objectName),
ex => Assert.Fail());
ex => Assert.Fail());

await minio.RemoveIncompleteUploadAsync(bucketName, objectName);
}
Expand All @@ -2498,7 +2498,7 @@ private async static Task ListIncompleteUpload_Test2(MinioClient minio)
{
DateTime startTime = DateTime.Now;
string bucketName = GetRandomName(15);
string prefix = "minioprefix/";
string prefix = "minioprefix/";
string objectName = prefix + GetRandomName(10);
string contentType = "gzip";
Dictionary<string,string> args = new Dictionary<string,string>
Expand Down Expand Up @@ -2527,11 +2527,11 @@ await minio.PutObjectAsync(bucketName,
}
catch (OperationCanceledException)
{
IObservable<Upload> observable = minio.ListIncompleteUploads(bucketName,"minioprefix",false);
IObservable<Upload> observable = minio.ListIncompleteUploads(bucketName, "minioprefix", false);

IDisposable subscription = observable.Subscribe(
item => Assert.AreEqual(item.Key, objectName),
ex => Assert.Fail());
ex => Assert.Fail());

await minio.RemoveIncompleteUploadAsync(bucketName, objectName);
}
Expand Down Expand Up @@ -2577,11 +2577,11 @@ await minio.PutObjectAsync(bucketName,
}
catch (OperationCanceledException)
{
IObservable<Upload> observable = minio.ListIncompleteUploads(bucketName,prefix,true);
IObservable<Upload> observable = minio.ListIncompleteUploads(bucketName, prefix, true);

IDisposable subscription = observable.Subscribe(
item => Assert.AreEqual(item.Key, objectName),
ex => Assert.Fail());
ex => Assert.Fail());

await minio.RemoveIncompleteUploadAsync(bucketName, objectName);
}
Expand Down Expand Up @@ -2631,7 +2631,7 @@ await minio.PutObjectAsync(bucketName,

IDisposable subscription = observable.Subscribe(
item => Assert.Fail(),
ex => Assert.Fail());
ex => Assert.Fail());
}
await TearDown(minio, bucketName);
new MintLogger("RemoveIncompleteUpload_Test",removeIncompleteUploadSignature,"Tests whether RemoveIncompleteUpload passes.",TestStatus.PASS,(DateTime.Now - startTime), args:args).Log();
Expand Down
46 changes: 24 additions & 22 deletions Minio/ApiEndpoints/BucketOperations.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public partial class MinioClient : IBucketOperations
location = this.Region;
}
}

// Set Target URL
Uri requestUrl = RequestUtil.MakeTargetURL(this.BaseUrl, this.Secure,location);
SetTargetURL(requestUrl);
Expand Down Expand Up @@ -134,20 +134,27 @@ public partial class MinioClient : IBucketOperations
/// List all objects non-recursively in a bucket with a given prefix, optionally emulating a directory
/// </summary>
/// <param name="bucketName">Bucket to list objects from</param>
/// <param name="prefix">Filters all objects not beginning with a given prefix</param>
/// <param name="recursive">Set to false to emulate a directory</param>
/// <param name="prefix">Filters all objects beginning with a given prefix</param>
/// <param name="recursive">Set to true to recursively list all objects</param>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
/// <returns>An observable of items that client can subscribe to</returns>
public IObservable<Item> ListObjectsAsync(string bucketName, string prefix = null, bool recursive = true, CancellationToken cancellationToken = default(CancellationToken))
public IObservable<Item> ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default(CancellationToken))
{
return Observable.Create<Item>(
async obs =>
{
bool isRunning = true;
string marker = null;
var delimiter = "/";
if (recursive)
{
delimiter = "";
}
while (isRunning)
{
Tuple<ListBucketResult, List<Item>> result = await GetObjectListAsync(bucketName, prefix, recursive, marker, cancellationToken).ConfigureAwait(false);
Tuple<ListBucketResult, List<Item>> result = await GetObjectListAsync(bucketName, prefix, delimiter, marker, cancellationToken).ConfigureAwait(false);
Item lastItem = null;
foreach (Item item in result.Item2)
{
Expand All @@ -171,35 +178,30 @@ public partial class MinioClient : IBucketOperations
/// Gets the list of objects in the bucket filtered by prefix
/// </summary>
/// <param name="bucketName">Bucket to list objects from</param>
/// <param name="prefix">Filters all objects not beginning with a given prefix</param>
/// <param name="recursive">Set to false to emulate a directory</param>
/// <param name="prefix">Filters all objects starting with a given prefix</param>
/// <param name="delimiter">Delimit the output upto this character</param>
/// <param name="marker">marks location in the iterator sequence</param>
/// <returns>Task with a tuple populated with objects</returns>
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>

private async Task<Tuple<ListBucketResult, List<Item>>> GetObjectListAsync(string bucketName, string prefix, bool recursive, string marker, CancellationToken cancellationToken = default(CancellationToken))
private async Task<Tuple<ListBucketResult, List<Item>>> GetObjectListAsync(string bucketName, string prefix, string delimiter, string marker, CancellationToken cancellationToken = default(CancellationToken))
{
var queries = new List<string>();
if (!recursive)
{
queries.Add("delimiter=%2F");
}
if (prefix != null)
{
queries.Add("prefix=" + Uri.EscapeDataString(prefix));
}
queries.Add("delimiter="+ Uri.EscapeDataString(delimiter));
queries.Add("prefix=" + Uri.EscapeDataString(prefix));
queries.Add("max-keys=1000");

if (marker != null)
{
queries.Add("marker=" + Uri.EscapeDataString(marker));
}
queries.Add("max-keys=1000");
string query = string.Join("&", queries);

var request = await this.CreateRequest(Method.GET,
bucketName,
resourcePath: "?" + query)
.ConfigureAwait(false);

var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false);

var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
Expand Down Expand Up @@ -292,7 +294,7 @@ public partial class MinioClient : IBucketOperations
resourcePath: "?notification")
.ConfigureAwait(false);
BucketNotification notification = null;

var response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false);
var contentBytes = System.Text.Encoding.UTF8.GetBytes(response.Content);
using (var stream = new MemoryStream(contentBytes))
Expand All @@ -314,11 +316,11 @@ public partial class MinioClient : IBucketOperations
var request = await this.CreateRequest(Method.PUT, bucketName,
resourcePath: "?notification")
.ConfigureAwait(false);

request.XmlSerializer = new RestSharp.Serializers.DotNetXmlSerializer();
request.RequestFormat = DataFormat.Xml;
request.AddBody(notification);

IRestResponse response = await this.ExecuteTaskAsync(this.NoErrorHandlers, request, cancellationToken).ConfigureAwait(false);
}

Expand All @@ -335,4 +337,4 @@ public partial class MinioClient : IBucketOperations
return SetBucketNotificationsAsync(bucketName, notification, cancellationToken);
}
}
}
}
Loading

0 comments on commit a118b2c

Please sign in to comment.