Skip to content

Commit

Permalink
Allow setting the acl for uploaded files (s3) (#204)
Browse files Browse the repository at this point in the history 
* Allow setting the acl for uploaded files (s3)

* rename option and set the acl when creating a bucket.

* add test

* (Not) Handle unknown pre-defined urls

* cleanup
  • Loading branch information
@Gitii
Gitii authored Oct 21, 2024
1 parent 610cb10 commit 751d665
Show file tree
Hide file tree
Showing 7 changed files with 94 additions and 24 deletions.
25 changes: 13 additions & 12 deletions doc/client-settings.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,18 +64,19 @@ More options can be found in the [Azure.Identity README](https://github.com/Azur

## Amazon s3 specific properties

| Property | Description |
| --- | ------ |
| profileName | AWS [credentials file](https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file) profile name. *[Cannot be used with accessKeyId or secretAccessKey]* |
| accessKeyId | Access key id *[Cannot be used with profileName]* |
| secretAccessKey | Secret access key *[Cannot be used with profileName]* |
| bucketName | S3 bucket name *[Required]* |
| region | S3 region *[Cannot be used with serviceURL]* |
| serviceURL | S3 service URL *[Cannot be used with region]* |
| path | Full URI of the storage bucket. If not specified a default URI will be used. |
| feedSubPath | Provides a sub directory path within the bucket where the feed should be added. This allows for multiple feeds within a single bucket. |
| serverSideEncryptionMethod | The encryption to use for uploaded objects. Only `AES256` and `None` are currently supported. Default is `None` |
| compress | Compress JSON files with GZIP before uploading. Default is *true* |
| Property | Description |
|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| profileName | AWS [credentials file](https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file) profile name. *[Cannot be used with accessKeyId or secretAccessKey]* |
| accessKeyId | Access key id *[Cannot be used with profileName]* |
| secretAccessKey | Secret access key *[Cannot be used with profileName]* |
| bucketName | S3 bucket name *[Required]* |
| region | S3 region *[Cannot be used with serviceURL]* |
| serviceURL | S3 service URL *[Cannot be used with region]* |
| path | Full URI of the storage bucket. If not specified a default URI will be used. |
| feedSubPath | Provides a sub directory path within the bucket where the feed should be added. This allows for multiple feeds within a single bucket. |
| serverSideEncryptionMethod | The encryption to use for uploaded objects. Only `AES256` and `None` are currently supported. Default is `None` |
| compress | Compress JSON files with GZIP before uploading. Default is *true* |
| acl | A acl can be set for uploaded files. By default, no specific canned acl is set and bucket defaults and/or policies are in effect. If the bucket is created by sleet and an acl is set, then the default bucket acl will be set to that acl. |

Either `region` or `serviceURL` should be specified but not both.

Expand Down
7 changes: 5 additions & 2 deletions src/SleetLib/FileSystem/AmazonS3File.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class AmazonS3File : FileBase
private readonly string key;
private readonly bool compress = true;
private readonly ServerSideEncryptionMethod serverSideEncryptionMethod;
private readonly S3CannedACL acl;

internal AmazonS3File(
AmazonS3FileSystem fileSystem,
Expand All @@ -26,14 +27,16 @@ internal AmazonS3File(
string bucketName,
string key,
ServerSideEncryptionMethod serverSideEncryptionMethod,
bool compress = true)
bool compress = true,
S3CannedACL acl = null)
: base(fileSystem, rootPath, displayPath, localCacheFile, fileSystem.LocalCache.PerfTracker)
{
this.client = client;
this.bucketName = bucketName;
this.key = key;
this.compress = compress;
this.serverSideEncryptionMethod = serverSideEncryptionMethod;
this.acl = acl;
}

protected override async Task CopyFromSource(ILogger log, CancellationToken token)
Expand Down Expand Up @@ -131,7 +134,7 @@ protected override async Task CopyToSource(ILogger log, CancellationToken token)
log.LogWarning($"Unknown file type: {absoluteUri}");
}

await UploadFileAsync(client, bucketName, key, contentType, contentEncoding, writeStream, serverSideEncryptionMethod, token)
await UploadFileAsync(client, bucketName, key, contentType, contentEncoding, writeStream, serverSideEncryptionMethod, acl, token)
.ConfigureAwait(false);

writeStream.Dispose();
Expand Down
32 changes: 26 additions & 6 deletions src/SleetLib/FileSystem/AmazonS3FileSystem.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,28 +20,30 @@ public class AmazonS3FileSystem : FileSystemBase
private readonly IAmazonS3 _client;
private readonly bool _compress;
private readonly ServerSideEncryptionMethod _serverSideEncryptionMethod;
private readonly S3CannedACL _acl;

private bool? _hasBucket;

public AmazonS3FileSystem(LocalCache cache, Uri root, IAmazonS3 client, string bucketName)
: this(cache, root, root, client, bucketName, ServerSideEncryptionMethod.None)
public AmazonS3FileSystem(LocalCache cache, Uri root, IAmazonS3 client, string bucketName, string acl)
: this(cache, root, root, client, bucketName, ServerSideEncryptionMethod.None, acl: acl)
{
}

public AmazonS3FileSystem(
LocalCache cache,
public AmazonS3FileSystem(LocalCache cache,
Uri root,
Uri baseUri,
IAmazonS3 client,
string bucketName,
ServerSideEncryptionMethod serverSideEncryptionMethod,
string feedSubPath = null,
bool compress = true)
bool compress = true,
S3CannedACL acl = null)
: base(cache, root, baseUri)
{
_client = client;
_bucketName = bucketName;
_serverSideEncryptionMethod = serverSideEncryptionMethod;
_acl = acl;

if (!string.IsNullOrEmpty(feedSubPath))
{
Expand Down Expand Up @@ -117,7 +119,7 @@ public override async Task<IReadOnlyList<ISleetFile>> GetFiles(ILogger log, Canc
private ISleetFile CreateAmazonS3File(SleetUriPair pair)
{
var key = GetRelativePath(pair.Root);
return new AmazonS3File(this, pair.Root, pair.BaseURI, LocalCache.GetNewTempPath(), _client, _bucketName, key, _serverSideEncryptionMethod, _compress);
return new AmazonS3File(this, pair.Root, pair.BaseURI, LocalCache.GetNewTempPath(), _client, _bucketName, key, _serverSideEncryptionMethod, _compress, _acl);
}

public override string GetRelativePath(Uri uri)
Expand Down Expand Up @@ -176,6 +178,12 @@ public override async Task CreateBucket(ILogger log, CancellationToken token)
// Set the public policy to public read-only
await Retry(SetBucketPolicy, log, token);

// Set the default acl of the bucket. Must not conflict with the public access policy.
if (_acl != null)
{
await Retry(SetBucketAcl, log, token);
}

// Get and release the lock to ensure that everything will work for the next operation.
// In the E2E tests there are often failures due to the bucket saying it is not available
// even though the above checks passed. To work around this wait until a file can be
Expand Down Expand Up @@ -223,6 +231,18 @@ private Task SetOwnership(ILogger log, CancellationToken token)
return _client.PutBucketOwnershipControlsAsync(ownerReq, token);
}

// Set the default acl of the bucket.
private Task SetBucketAcl(ILogger log, CancellationToken token)
{
var aclReq = new PutACLRequest()
{
BucketName = _bucketName,
CannedACL = _acl
};

return _client.PutACLAsync(aclReq, token);
}

// Remove public access blocks to allow public policies.
private Task SetPublicAccessBlocks(ILogger log, CancellationToken token)
{
Expand Down
6 changes: 6 additions & 0 deletions src/SleetLib/FileSystem/AmazonS3FileSystemAbstraction.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ public static async Task UploadFileAsync(
string contentEncoding,
Stream reader,
ServerSideEncryptionMethod serverSideEncryptionMethod,
S3CannedACL acl,
CancellationToken token)
{
var transferUtility = new TransferUtility(client);
Expand Down Expand Up @@ -158,6 +159,11 @@ public static async Task UploadFileAsync(
if (contentEncoding != null)
request.Headers.ContentEncoding = contentEncoding;

if (acl != null)
{
request.CannedACL = acl;
}

using (transferUtility)
await transferUtility.UploadAsync(request, token).ConfigureAwait(false);
}
Expand Down
13 changes: 11 additions & 2 deletions src/SleetLib/FileSystem/FileSystemFactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
var serviceURL = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serviceURL");
var serverSideEncryptionMethod = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serverSideEncryptionMethod") ?? "None";
var compress = JsonUtility.GetBoolCaseInsensitive(sourceEntry, "compress", true);
var acl = JsonUtility.GetValueCaseInsensitive(sourceEntry, "acl");

if (string.IsNullOrEmpty(bucketName))
{
Expand All @@ -120,6 +121,12 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
throw new ArgumentException("Only 'None' or 'AES256' are currently supported for serverSideEncryptionMethod");
}

S3CannedACL resolvedAcl = null;
if (acl != null)
{
resolvedAcl = S3CannedACL.FindValue(acl);
}

// Use the SDK value
var serverSideEncryptionMethodValue = ServerSideEncryptionMethod.None;
if (serverSideEncryptionMethod == "AES256")
Expand Down Expand Up @@ -183,7 +190,7 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
}
// Load credentials from an ECS docker container
// Check if the env var GenericContainerCredentials.RelativeURIEnvVariable exists
// Previously this used ECSTaskCredentials.RelativeURIEnvVariable but that was
// Previously this used ECSTaskCredentials.RelativeURIEnvVariable but that was
// deprecated and the property is now internal on GenericContainerCredentials
else if (
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")))
Expand Down Expand Up @@ -229,7 +236,9 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
bucketName,
serverSideEncryptionMethodValue,
feedSubPath,
compress);
compress,
resolvedAcl
);
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions test/Sleet.AmazonS3.Tests/AmazonS3TestContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ public class AmazonS3TestContext : IDisposable

private bool cleanupDone = false;

public AmazonS3TestContext()
public AmazonS3TestContext(string acl = null)
{
BucketName = $"sleet-test-{Guid.NewGuid().ToString()}";
LocalCache = new LocalCache();
Expand All @@ -35,7 +35,7 @@ public AmazonS3TestContext()
Client = new AmazonS3Client(accessKeyId, secretAccessKey, config);
Uri = AmazonS3Utility.GetBucketPath(BucketName, region);

FileSystem = new AmazonS3FileSystem(LocalCache, Uri, Client, BucketName);
FileSystem = new AmazonS3FileSystem(LocalCache, Uri, Client, BucketName, acl);
Logger = new TestLogger();
}

Expand Down
31 changes: 31 additions & 0 deletions test/Sleet.AmazonS3.Tests/BasicTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,37 @@ public async Task GivenAStorageAccountWithNoContainerVerifyPushSucceeds()
}
}

[EnvVarExistsFact(AmazonS3TestContext.EnvAccessKeyId)]
public async Task GivenAStorageAccountWithNoContainerPublicAclVerifyPushSucceeds()
{
using (var packagesFolder = new TestFolder())
using (var testContext = new AmazonS3TestContext(acl: "public-read"))
{
// Skip creation and allow it to be done during push.
testContext.CreateBucketOnInit = false;

await testContext.InitAsync();

var testPackage = new TestNupkg("packageA", "1.0.0");
var zipFile = testPackage.Save(packagesFolder.Root);

var result = await PushCommand.RunAsync(testContext.LocalSettings,
testContext.FileSystem,
new List<string>() { zipFile.FullName },
force: false,
skipExisting: false,
log: testContext.Logger);

result &= await ValidateCommand.RunAsync(testContext.LocalSettings,
testContext.FileSystem,
testContext.Logger);

result.Should().BeTrue();

await testContext.CleanupAsync();
}
}

[EnvVarExistsFact(AmazonS3TestContext.EnvAccessKeyId)]
public async Task GivenAStorageAccountWithNoInitVerifyPushSucceeds()
{
Expand Down

0 comments on commit 751d665

Please sign in to comment.