Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Amazon S3 Bucket (Independent Publisher) - Fix for issue #3702 #3731

Open
wants to merge 3 commits into
base: dev
Choose a base branch
from
Open

Amazon S3 Bucket (Independent Publisher) - Fix for issue #3702 #3731

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d1dca9d
- Fixed: base64 content was not converted into binary content to stor…
megel-mme2k Dec 8, 2024
56f0fa5
- `x-ms-summary` added for response fields
megel-mme2k Dec 8, 2024
b74025b
Readme updated
megel-mme2k Dec 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 43 additions & 20 deletions independent-publisher-connectors/AmazonS3Bucket/apiDefinition.swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,35 +30,43 @@
"type": "object",
"properties": {
"Name": {
"x-ms-summary": "Name",
"type": "string",
"description": "The bucket name."
},
"Prefix": {
"x-ms-summary": "Prefix",
"type": "string",
"description": "Keys that begin with the indicated prefix."
},
"MaxKeys": {
"x-ms-summary": "Max Keys",
"type": "string",
"description": "Sets the maximum number of keys returned in the response. By default, the action returns up to 1,000 key names. The response might contain fewer keys but will never contain more."
},
"IsTruncated": {
"x-ms-summary": "Is Truncated",
"type": "string",
"description": "Set to false if all of the results were returned. Set to true if more keys are available to return. If the number of results exceeds that specified by MaxKeys, all of the results might not be returned."
},
"KeyCount": {
"x-ms-summary": "Key Count",
"type": "string",
"description": "KeyCount is the number of keys returned with this request. KeyCount will always be less than or equal to the MaxKeys field. For example, if you ask for 50 keys, your result will include 50 keys or fewer."
},
"ContinuationToken": {
"x-ms-summary": "Continuation Token",
"type": "string",
"description": "If ContinuationToken was sent with the request, it is included in the response."
},
"NextContinuationToken": {
"x-ms-summary": "Next Continuation Token",
"type": "string",
"description": "NextContinuationToken is sent when isTruncated is true, which means there are more keys in the bucket that can be listed. The next list requests to Amazon S3 can be continued with this NextContinuationToken. NextContinuationToken is obfuscated and is not a real key"
},
"StartAfter": {
"type": "string",
"x-ms-summary": "Start After",
"description": "If StartAfter was sent with the request, it is included in the response."
},
"Contents": {
Expand All @@ -68,33 +76,40 @@
"properties": {
"Key": {
"type": "string",
"description": "The key of the object."
"description": "The key of the object.",
"x-ms-summary": "Key"
},
"LastModified": {
"type": "string",
"description": "The last modified date of the object"
"description": "The last modified date of the object",
"x-ms-summary": "Last Modified"
},
"Size": {
"type": "string",
"description": "The size of the object in bytes."
"description": "The size of the object in bytes.",
"x-ms-summary": "Size"
},
"Owner": {
"type": "object",
"x-ms-summary": "Owner",
"properties": {
"ID": {
"type": "string",
"description": "The ID of the owner."
"description": "The ID of the owner.",
"x-ms-summary": "ID"
},
"DisplayName": {
"type": "string",
"description": "The display name of the owner."
"description": "The display name of the owner.",
"x-ms-summary": "Display Name"
}
},
"description": "Owner"
},
"StorageClass": {
"type": "string",
"description": "The S3 Storage Class"
"description": "The S3 Storage Class",
"x-ms-summary": "Storage Class"
}
}
},
Expand Down Expand Up @@ -202,19 +217,19 @@
}
},
"/aws/s3/{region}/{bucket}/{key}": {
"delete": {
"get": {
"responses": {
"204": {
"200": {
"description": "success",
"schema": {}
}
},
"tags": [
"S3-Bucket"
],
"summary": "Delete Object",
"description": "Delete Object from S3 Bucket",
"operationId": "delete-object-s3",
"summary": "Get Object",
"description": "Get Object from S3 Bucket",
"operationId": "get-object-s3",
"parameters": [
{
"name": "region",
Expand All @@ -239,24 +254,25 @@
"in": "path",
"required": true,
"type": "string",
"default": "",
"description": "The key of the object.",
"x-ms-summary": "Key"
}
]
},
"get": {
"delete": {
"responses": {
"200": {
"204": {
"description": "success",
"schema": {}
}
},
"tags": [
"S3-Bucket"
],
"summary": "Get Object",
"description": "Get Object from S3 Bucket",
"operationId": "get-object-s3",
"summary": "Delete Object",
"description": "Delete Object from S3 Bucket",
"operationId": "delete-object-s3",
"parameters": [
{
"name": "region",
Expand All @@ -281,7 +297,6 @@
"in": "path",
"required": true,
"type": "string",
"default": "",
"description": "The key of the object.",
"x-ms-summary": "Key"
}
Expand Down Expand Up @@ -336,6 +351,14 @@
"format": "binary"
},
"x-ms-summary": "Content"
},
{
"name": "content-type",
"in": "header",
"required": false,
"type": "string",
"description": "A standard MIME type describing the format of the contents.",
"x-ms-summary": "Content Type"
}
]
}
Expand All @@ -358,15 +381,15 @@
"x-ms-connector-metadata": [
{
"propertyName": "Website",
"propertyValue": "https://aws.amazon.com//"
"propertyValue": "https://aws.amazon.com/"
},
{
"propertyName": "Privacy policy",
"propertyValue": "https://aws.amazon.com//"
"propertyValue": "https://aws.amazon.com/"
},
{
"propertyName": "Categories",
"propertyValue": "Collaboration;Content and Files"
}
]
}
}
4 changes: 4 additions & 0 deletions independent-publisher-connectors/AmazonS3Bucket/readme.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ This connector supports the following operations:
* The content is returned as a string.
* Put Object
* Large file requests might run into timeout issues such as HTTP 500 "Request to the backend service timed out". This is caused by custom connector script that creates the AWS Signature Version 4. The script must be finished within 5 seconds. For large files, the script might take longer than 5 seconds to finish. ([Microsoft FAQ: Script must be finished within 5 seconds](https://learn.microsoft.com/en-us/connectors/custom-connectors/write-code#custom-code-faq))
* Some characters in the filename might cause issues. For example, the connector might not support filenames with special characters such as `&`, `|`, `$`, and `?`.

### Fixed Issues

Expand All @@ -46,6 +47,9 @@ This connector supports the following operations:
* The connector now supports filenames with characters such as spaces (`folder 1/my file.csv`).
* The connector now supports binary content such as PDF files.
*Note: Large files might run into timeout issues.([Microsoft FAQ: Script must be finished within 5 seconds](https://learn.microsoft.com/en-us/connectors/custom-connectors/write-code#custom-code-faq))
* The connector now supports files with characters such as `(`, `)`, `{`, `}`, `[`, `]`, and `#` in the object key.
* The connector now supports specifying the `Content-Type` of the object as a parameter.
* The connector now supports new storing of binary content in the S3 bucket. This means, base64 encoded content is converted into binary content before storing it in the S3 bucket. This addresses [S3 PUT does not work for Office Files (Excel, Word, PowerPoint) #3702](https://github.com/microsoft/PowerPlatformConnectors/issues/3702).

## AWS Signature Version 4

Expand Down
46 changes: 35 additions & 11 deletions independent-publisher-connectors/AmazonS3Bucket/script.csx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
public class Script : ScriptBase
public class Script : ScriptBase
{
public override async Task<HttpResponseMessage> ExecuteAsync()
{
Expand Down Expand Up @@ -31,7 +31,7 @@
var path = pathAfterAws.Split('/');
string service = path.Skip(0).FirstOrDefault(); // Service;
string region = path.Skip(1).FirstOrDefault(); // Region;
string objectName = path.Skip(2).FirstOrDefault(); // Object Name;
string bucketName = path.Skip(2).FirstOrDefault(); // Bucket Name;
string objectKey = path.Skip(3).FirstOrDefault(); // Object Key;

// Object key can include / in the name, so we need to decode it
Expand All @@ -42,7 +42,7 @@

logger.LogInformation($"Service: {(service ?? "---")}");
logger.LogInformation($"Region: {(region ?? "---")}");
logger.LogInformation($"Object Name: {(objectName ?? "---")}");
logger.LogInformation($"Bucket Name: {(bucketName ?? "---")}");
logger.LogInformation($"Object Key: {(objectKey ?? "---")}");

var regionUrlPart = string.Empty;
Expand All @@ -55,17 +55,23 @@
var endpointUri = string.Format("https://{2}.{0}{1}.amazonaws.com",
service,
regionUrlPart,
objectName);
bucketName);
if (! string.IsNullOrEmpty(objectKey)) {
endpointUri = string.Format("{0}/{1}", endpointUri, objectKey);
}
}
foreach (var character in new[]{"(", ")", "[", "]", "{", "}", "#"}) {
if (endpointUri.Contains(character))
endpointUri = endpointUri.Replace(character, Uri.EscapeDataString(character));
}
var uri = new Uri($"{endpointUri}{requestUri.Query}");

logger.LogInformation($"Uri AbsolutePath: {uri.AbsolutePath}");

var request = new System.Net.Http.HttpRequestMessage(this.Context.Request.Method, uri)
{
Content = this.Context.Request.Content
Content = GetConvertedRequestContent()
};
SignRequest(request, service, region, accessKeyId, accessKeySecret);

SignRequest(request, service, region, accessKeyId, accessKeySecret, logger);
logger.LogInformation($"Call: {request.Method} {request.RequestUri!}");

// Use the context to forward/send an HTTP request
Expand All @@ -85,6 +91,20 @@
throw new Exception("Unexpected Authentication");
}

private System.Net.Http.HttpContent GetConvertedRequestContent()
{
var logger = this.Context.Logger;
try {
var binary = Convert.FromBase64String(this.Context.Request.Content.ReadAsStringAsync().Result);
logger.LogInformation($"Base64 content converted into binary (Binary content length: {binary.Length})");
return new System.Net.Http.ByteArrayContent(binary);

} catch {
logger.LogInformation("Content can't be converted from Base64 to binary, keep the content as it is.");
return this.Context.Request.Content;
}
}

private async Task<HttpResponseMessage> HandleTransformXML2Json(HttpResponseMessage response)
{
// Do the transformation if the response was successful, otherwise return error responses as-is
Expand Down Expand Up @@ -125,7 +145,8 @@
string service,
string region,
string awsAccessKey,
string awsSecretKey)
string awsSecretKey,
ILogger logger = null)
{
var signer = new AWS4SignerForAuthorizationHeader
{
Expand Down Expand Up @@ -156,7 +177,7 @@
headers.Add(AWS4SignerBase.X_Amz_Content_SHA256, bodyHash);
}

string signature = signer.ComputeSignature(headers, queryParameters, bodyHash, awsAccessKey, awsSecretKey);
string signature = signer.ComputeSignature(headers, queryParameters, bodyHash, awsAccessKey, awsSecretKey, logger);

foreach (var header in headers.Keys)
{
Expand Down Expand Up @@ -465,7 +486,8 @@
string queryParameters,
string bodyHash,
string awsAccessKey,
string awsSecretKey)
string awsSecretKey,
ILogger logger = null)
{
// first get the date and time for the subsequent request, and convert to ISO 8601 format
// for use in signature generation
Expand Down Expand Up @@ -515,6 +537,7 @@
canonicalizedHeaderNames,
canonicalizedHeaders,
bodyHash);
logger?.LogInformation($"Canonical Request:\n{canonicalRequest}");
Console.WriteLine("\nCanonicalRequest:\n{0}", canonicalRequest);

// generate a hash of the canonical request, to go into signature computation
Expand All @@ -534,6 +557,7 @@
stringToSign.AppendFormat("{0}-{1}\n{2}\n{3}\n", SCHEME, ALGORITHM, dateTimeStamp, scope);
stringToSign.Append(ToHexString(canonicalRequestHashBytes, true));

logger?.LogInformation($"String to Sign:\n{stringToSign}");
Console.WriteLine("\nStringToSign:\n{0}", stringToSign);

// compute the signing key
Expand Down