Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow setting the acl for uploaded files (s3) #204

Merged
merged 5 commits into from
Oct 21, 2024
Merged

Allow setting the acl for uploaded files (s3) #204

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1779d7c
Allow setting the acl for uploaded files (s3)
Gitii Oct 9, 2024
50ab57b
rename option and set the acl when creating a bucket.
Gitii Oct 10, 2024
897a0e5
add test
Gitii Oct 11, 2024
6ddfb8e
(Not) Handle unknown pre-defined urls
Gitii Oct 14, 2024
cb2afe0
cleanup
Gitii Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions doc/client-settings.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,18 +64,19 @@ More options can be found in the [Azure.Identity README](https://github.com/Azur

## Amazon s3 specific properties

| Property | Description |
| --- | ------ |
| profileName | AWS [credentials file](https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file) profile name. *[Cannot be used with accessKeyId or secretAccessKey]* |
| accessKeyId | Access key id *[Cannot be used with profileName]* |
| secretAccessKey | Secret access key *[Cannot be used with profileName]* |
| bucketName | S3 bucket name *[Required]* |
| region | S3 region *[Cannot be used with serviceURL]* |
| serviceURL | S3 service URL *[Cannot be used with region]* |
| path | Full URI of the storage bucket. If not specified a default URI will be used. |
| feedSubPath | Provides a sub directory path within the bucket where the feed should be added. This allows for multiple feeds within a single bucket. |
| serverSideEncryptionMethod | The encryption to use for uploaded objects. Only `AES256` and `None` are currently supported. Default is `None` |
| compress | Compress JSON files with GZIP before uploading. Default is *true* |
| Property | Description |
|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| profileName | AWS [credentials file](https://docs.aws.amazon.com/sdk-for-net/v2/developer-guide/net-dg-config-creds.html#creds-file) profile name. *[Cannot be used with accessKeyId or secretAccessKey]* |
| accessKeyId | Access key id *[Cannot be used with profileName]* |
| secretAccessKey | Secret access key *[Cannot be used with profileName]* |
| bucketName | S3 bucket name *[Required]* |
| region | S3 region *[Cannot be used with serviceURL]* |
| serviceURL | S3 service URL *[Cannot be used with region]* |
| path | Full URI of the storage bucket. If not specified a default URI will be used. |
| feedSubPath | Provides a sub directory path within the bucket where the feed should be added. This allows for multiple feeds within a single bucket. |
| serverSideEncryptionMethod | The encryption to use for uploaded objects. Only `AES256` and `None` are currently supported. Default is `None` |
| compress | Compress JSON files with GZIP before uploading. Default is *true* |
| acl | A acl can be set for uploaded files. By default, no specific canned acl is set and bucket defaults and/or policies are in effect. If the bucket is created by sleet and an acl is set, then the default bucket acl will be set to that acl. |

Either `region` or `serviceURL` should be specified but not both.

Expand Down
7 changes: 5 additions & 2 deletions src/SleetLib/FileSystem/AmazonS3File.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class AmazonS3File : FileBase
private readonly string key;
private readonly bool compress = true;
private readonly ServerSideEncryptionMethod serverSideEncryptionMethod;
private readonly S3CannedACL acl;

internal AmazonS3File(
AmazonS3FileSystem fileSystem,
Expand All @@ -26,14 +27,16 @@ internal AmazonS3File(
string bucketName,
string key,
ServerSideEncryptionMethod serverSideEncryptionMethod,
bool compress = true)
bool compress = true,
S3CannedACL acl = null)
: base(fileSystem, rootPath, displayPath, localCacheFile, fileSystem.LocalCache.PerfTracker)
{
this.client = client;
this.bucketName = bucketName;
this.key = key;
this.compress = compress;
this.serverSideEncryptionMethod = serverSideEncryptionMethod;
this.acl = acl;
}

protected override async Task CopyFromSource(ILogger log, CancellationToken token)
Expand Down Expand Up @@ -131,7 +134,7 @@ protected override async Task CopyToSource(ILogger log, CancellationToken token)
log.LogWarning($"Unknown file type: {absoluteUri}");
}

await UploadFileAsync(client, bucketName, key, contentType, contentEncoding, writeStream, serverSideEncryptionMethod, token)
await UploadFileAsync(client, bucketName, key, contentType, contentEncoding, writeStream, serverSideEncryptionMethod, acl, token)
.ConfigureAwait(false);

writeStream.Dispose();
Expand Down
32 changes: 26 additions & 6 deletions src/SleetLib/FileSystem/AmazonS3FileSystem.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,28 +20,30 @@ public class AmazonS3FileSystem : FileSystemBase
private readonly IAmazonS3 _client;
private readonly bool _compress;
private readonly ServerSideEncryptionMethod _serverSideEncryptionMethod;
private readonly S3CannedACL _acl;

private bool? _hasBucket;

public AmazonS3FileSystem(LocalCache cache, Uri root, IAmazonS3 client, string bucketName)
: this(cache, root, root, client, bucketName, ServerSideEncryptionMethod.None)
public AmazonS3FileSystem(LocalCache cache, Uri root, IAmazonS3 client, string bucketName, string acl)
: this(cache, root, root, client, bucketName, ServerSideEncryptionMethod.None, acl: acl)
{
}

public AmazonS3FileSystem(
LocalCache cache,
public AmazonS3FileSystem(LocalCache cache,
Uri root,
Uri baseUri,
IAmazonS3 client,
string bucketName,
ServerSideEncryptionMethod serverSideEncryptionMethod,
string feedSubPath = null,
bool compress = true)
bool compress = true,
S3CannedACL acl = null)
: base(cache, root, baseUri)
{
_client = client;
_bucketName = bucketName;
_serverSideEncryptionMethod = serverSideEncryptionMethod;
_acl = acl;

if (!string.IsNullOrEmpty(feedSubPath))
{
Expand Down Expand Up @@ -117,7 +119,7 @@ public override async Task<IReadOnlyList<ISleetFile>> GetFiles(ILogger log, Canc
private ISleetFile CreateAmazonS3File(SleetUriPair pair)
{
var key = GetRelativePath(pair.Root);
return new AmazonS3File(this, pair.Root, pair.BaseURI, LocalCache.GetNewTempPath(), _client, _bucketName, key, _serverSideEncryptionMethod, _compress);
return new AmazonS3File(this, pair.Root, pair.BaseURI, LocalCache.GetNewTempPath(), _client, _bucketName, key, _serverSideEncryptionMethod, _compress, _acl);
}

public override string GetRelativePath(Uri uri)
Expand Down Expand Up @@ -176,6 +178,12 @@ public override async Task CreateBucket(ILogger log, CancellationToken token)
// Set the public policy to public read-only
await Retry(SetBucketPolicy, log, token);

// Set the default acl of the bucket. Must not conflict with the public access policy.
if (_acl != null)
{
await Retry(SetBucketAcl, log, token);
}

// Get and release the lock to ensure that everything will work for the next operation.
// In the E2E tests there are often failures due to the bucket saying it is not available
// even though the above checks passed. To work around this wait until a file can be
Expand Down Expand Up @@ -223,6 +231,18 @@ private Task SetOwnership(ILogger log, CancellationToken token)
return _client.PutBucketOwnershipControlsAsync(ownerReq, token);
}

// Set the default acl of the bucket.
private Task SetBucketAcl(ILogger log, CancellationToken token)
{
var aclReq = new PutACLRequest()
{
BucketName = _bucketName,
CannedACL = _acl
};

return _client.PutACLAsync(aclReq, token);
}

// Remove public access blocks to allow public policies.
private Task SetPublicAccessBlocks(ILogger log, CancellationToken token)
{
Expand Down
6 changes: 6 additions & 0 deletions src/SleetLib/FileSystem/AmazonS3FileSystemAbstraction.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ public static async Task UploadFileAsync(
string contentEncoding,
Stream reader,
ServerSideEncryptionMethod serverSideEncryptionMethod,
S3CannedACL acl,
CancellationToken token)
{
var transferUtility = new TransferUtility(client);
Expand Down Expand Up @@ -158,6 +159,11 @@ public static async Task UploadFileAsync(
if (contentEncoding != null)
request.Headers.ContentEncoding = contentEncoding;

if (acl != null)
{
request.CannedACL = acl;
}

using (transferUtility)
await transferUtility.UploadAsync(request, token).ConfigureAwait(false);
}
Expand Down
13 changes: 11 additions & 2 deletions src/SleetLib/FileSystem/FileSystemFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
var serviceURL = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serviceURL");
var serverSideEncryptionMethod = JsonUtility.GetValueCaseInsensitive(sourceEntry, "serverSideEncryptionMethod") ?? "None";
var compress = JsonUtility.GetBoolCaseInsensitive(sourceEntry, "compress", true);
var acl = JsonUtility.GetValueCaseInsensitive(sourceEntry, "acl");

if (string.IsNullOrEmpty(bucketName))
{
Expand All @@ -120,6 +121,12 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
throw new ArgumentException("Only 'None' or 'AES256' are currently supported for serverSideEncryptionMethod");
}

S3CannedACL resolvedAcl = null;
if (acl != null)
{
resolvedAcl = S3CannedACL.FindValue(acl);
}

// Use the SDK value
var serverSideEncryptionMethodValue = ServerSideEncryptionMethod.None;
if (serverSideEncryptionMethod == "AES256")
Expand Down Expand Up @@ -183,7 +190,7 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
}
// Load credentials from an ECS docker container
// Check if the env var GenericContainerCredentials.RelativeURIEnvVariable exists
// Previously this used ECSTaskCredentials.RelativeURIEnvVariable but that was
// Previously this used ECSTaskCredentials.RelativeURIEnvVariable but that was
// deprecated and the property is now internal on GenericContainerCredentials
else if (
!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")))
Expand Down Expand Up @@ -229,7 +236,9 @@ public static async Task<ISleetFileSystem> CreateFileSystemAsync(LocalSettings s
bucketName,
serverSideEncryptionMethodValue,
feedSubPath,
compress);
compress,
resolvedAcl
);
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions test/Sleet.AmazonS3.Tests/AmazonS3TestContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ public class AmazonS3TestContext : IDisposable

private bool cleanupDone = false;

public AmazonS3TestContext()
public AmazonS3TestContext(string acl = null)
{
BucketName = $"sleet-test-{Guid.NewGuid().ToString()}";
LocalCache = new LocalCache();
Expand All @@ -35,7 +35,7 @@ public AmazonS3TestContext()
Client = new AmazonS3Client(accessKeyId, secretAccessKey, config);
Uri = AmazonS3Utility.GetBucketPath(BucketName, region);

FileSystem = new AmazonS3FileSystem(LocalCache, Uri, Client, BucketName);
FileSystem = new AmazonS3FileSystem(LocalCache, Uri, Client, BucketName, acl);
Logger = new TestLogger();
}

Expand Down
31 changes: 31 additions & 0 deletions test/Sleet.AmazonS3.Tests/BasicTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,37 @@ public async Task GivenAStorageAccountWithNoContainerVerifyPushSucceeds()
}
}

[EnvVarExistsFact(AmazonS3TestContext.EnvAccessKeyId)]
public async Task GivenAStorageAccountWithNoContainerPublicAclVerifyPushSucceeds()
{
using (var packagesFolder = new TestFolder())
using (var testContext = new AmazonS3TestContext(acl: "public-read"))
{
// Skip creation and allow it to be done during push.
testContext.CreateBucketOnInit = false;

await testContext.InitAsync();

var testPackage = new TestNupkg("packageA", "1.0.0");
var zipFile = testPackage.Save(packagesFolder.Root);

var result = await PushCommand.RunAsync(testContext.LocalSettings,
testContext.FileSystem,
new List<string>() { zipFile.FullName },
force: false,
skipExisting: false,
log: testContext.Logger);

result &= await ValidateCommand.RunAsync(testContext.LocalSettings,
testContext.FileSystem,
testContext.Logger);

result.Should().BeTrue();

await testContext.CleanupAsync();
}
}

[EnvVarExistsFact(AmazonS3TestContext.EnvAccessKeyId)]
public async Task GivenAStorageAccountWithNoInitVerifyPushSucceeds()
{
Expand Down
Loading