diff --git a/independent-publisher-connectors/AmazonS3Bucket/apiDefinition.swagger.json b/independent-publisher-connectors/AmazonS3Bucket/apiDefinition.swagger.json index 1ef592ce70..2057a4bd3b 100644 --- a/independent-publisher-connectors/AmazonS3Bucket/apiDefinition.swagger.json +++ b/independent-publisher-connectors/AmazonS3Bucket/apiDefinition.swagger.json @@ -30,35 +30,43 @@ "type": "object", "properties": { "Name": { + "x-ms-summary": "Name", "type": "string", "description": "The bucket name." }, "Prefix": { + "x-ms-summary": "Prefix", "type": "string", "description": "Keys that begin with the indicated prefix." }, "MaxKeys": { + "x-ms-summary": "Max Keys", "type": "string", "description": "Sets the maximum number of keys returned in the response. By default, the action returns up to 1,000 key names. The response might contain fewer keys but will never contain more." }, "IsTruncated": { + "x-ms-summary": "Is Truncated", "type": "string", "description": "Set to false if all of the results were returned. Set to true if more keys are available to return. If the number of results exceeds that specified by MaxKeys, all of the results might not be returned." }, "KeyCount": { + "x-ms-summary": "Key Count", "type": "string", "description": "KeyCount is the number of keys returned with this request. KeyCount will always be less than or equal to the MaxKeys field. For example, if you ask for 50 keys, your result will include 50 keys or fewer." }, "ContinuationToken": { + "x-ms-summary": "Continuation Token", "type": "string", "description": "If ContinuationToken was sent with the request, it is included in the response." }, "NextContinuationToken": { + "x-ms-summary": "Next Continuation Token", "type": "string", "description": "NextContinuationToken is sent when isTruncated is true, which means there are more keys in the bucket that can be listed. The next list requests to Amazon S3 can be continued with this NextContinuationToken. NextContinuationToken is obfuscated and is not a real key" }, "StartAfter": { "type": "string", + "x-ms-summary": "Start After", "description": "If StartAfter was sent with the request, it is included in the response." }, "Contents": { @@ -68,33 +76,40 @@ "properties": { "Key": { "type": "string", - "description": "The key of the object." + "description": "The key of the object.", + "x-ms-summary": "Key" }, "LastModified": { "type": "string", - "description": "The last modified date of the object" + "description": "The last modified date of the object", + "x-ms-summary": "Last Modified" }, "Size": { "type": "string", - "description": "The size of the object in bytes." + "description": "The size of the object in bytes.", + "x-ms-summary": "Size" }, "Owner": { "type": "object", + "x-ms-summary": "Owner", "properties": { "ID": { "type": "string", - "description": "The ID of the owner." + "description": "The ID of the owner.", + "x-ms-summary": "ID" }, "DisplayName": { "type": "string", - "description": "The display name of the owner." + "description": "The display name of the owner.", + "x-ms-summary": "Display Name" } }, "description": "Owner" }, "StorageClass": { "type": "string", - "description": "The S3 Storage Class" + "description": "The S3 Storage Class", + "x-ms-summary": "Storage Class" } } }, @@ -202,9 +217,9 @@ } }, "/aws/s3/{region}/{bucket}/{key}": { - "delete": { + "get": { "responses": { - "204": { + "200": { "description": "success", "schema": {} } @@ -212,9 +227,9 @@ "tags": [ "S3-Bucket" ], - "summary": "Delete Object", - "description": "Delete Object from S3 Bucket", - "operationId": "delete-object-s3", + "summary": "Get Object", + "description": "Get Object from S3 Bucket", + "operationId": "get-object-s3", "parameters": [ { "name": "region", @@ -239,14 +254,15 @@ "in": "path", "required": true, "type": "string", + "default": "", "description": "The key of the object.", "x-ms-summary": "Key" } ] }, - "get": { + "delete": { "responses": { - "200": { + "204": { "description": "success", "schema": {} } @@ -254,9 +270,9 @@ "tags": [ "S3-Bucket" ], - "summary": "Get Object", - "description": "Get Object from S3 Bucket", - "operationId": "get-object-s3", + "summary": "Delete Object", + "description": "Delete Object from S3 Bucket", + "operationId": "delete-object-s3", "parameters": [ { "name": "region", @@ -281,7 +297,6 @@ "in": "path", "required": true, "type": "string", - "default": "", "description": "The key of the object.", "x-ms-summary": "Key" } @@ -336,6 +351,14 @@ "format": "binary" }, "x-ms-summary": "Content" + }, + { + "name": "content-type", + "in": "header", + "required": false, + "type": "string", + "description": "A standard MIME type describing the format of the contents.", + "x-ms-summary": "Content Type" } ] } @@ -358,15 +381,15 @@ "x-ms-connector-metadata": [ { "propertyName": "Website", - "propertyValue": "https://aws.amazon.com//" + "propertyValue": "https://aws.amazon.com/" }, { "propertyName": "Privacy policy", - "propertyValue": "https://aws.amazon.com//" + "propertyValue": "https://aws.amazon.com/" }, { "propertyName": "Categories", "propertyValue": "Collaboration;Content and Files" } ] -} +} \ No newline at end of file diff --git a/independent-publisher-connectors/AmazonS3Bucket/readme.md b/independent-publisher-connectors/AmazonS3Bucket/readme.md index 04b54b5ee6..a93c0aa514 100644 --- a/independent-publisher-connectors/AmazonS3Bucket/readme.md +++ b/independent-publisher-connectors/AmazonS3Bucket/readme.md @@ -38,6 +38,7 @@ This connector supports the following operations: * The content is returned as a string. * Put Object * Large file requests might run into timeout issues such as HTTP 500 "Request to the backend service timed out". This is caused by custom connector script that creates the AWS Signature Version 4. The script must be finished within 5 seconds. For large files, the script might take longer than 5 seconds to finish. ([Microsoft FAQ: Script must be finished within 5 seconds](https://learn.microsoft.com/en-us/connectors/custom-connectors/write-code#custom-code-faq)) + * Some characters in the filename might cause issues. For example, the connector might not support filenames with special characters such as `&`, `|`, `$`, and `?`. ### Fixed Issues @@ -46,6 +47,9 @@ This connector supports the following operations: * The connector now supports filenames with characters such as spaces (`folder 1/my file.csv`). * The connector now supports binary content such as PDF files. *Note: Large files might run into timeout issues.([Microsoft FAQ: Script must be finished within 5 seconds](https://learn.microsoft.com/en-us/connectors/custom-connectors/write-code#custom-code-faq)) + * The connector now supports files with characters such as `(`, `)`, `{`, `}`, `[`, `]`, and `#` in the object key. + * The connector now supports specifying the `Content-Type` of the object as a parameter. + * The connector now supports new storing of binary content in the S3 bucket. This means, base64 encoded content is converted into binary content before storing it in the S3 bucket. This addresses [S3 PUT does not work for Office Files (Excel, Word, PowerPoint) #3702](https://github.com/microsoft/PowerPlatformConnectors/issues/3702). ## AWS Signature Version 4 diff --git a/independent-publisher-connectors/AmazonS3Bucket/script.csx b/independent-publisher-connectors/AmazonS3Bucket/script.csx index ee1733dd3c..2ada4d4d5f 100644 --- a/independent-publisher-connectors/AmazonS3Bucket/script.csx +++ b/independent-publisher-connectors/AmazonS3Bucket/script.csx @@ -1,4 +1,4 @@ -public class Script : ScriptBase +public class Script : ScriptBase { public override async Task ExecuteAsync() { @@ -31,7 +31,7 @@ var path = pathAfterAws.Split('/'); string service = path.Skip(0).FirstOrDefault(); // Service; string region = path.Skip(1).FirstOrDefault(); // Region; - string objectName = path.Skip(2).FirstOrDefault(); // Object Name; + string bucketName = path.Skip(2).FirstOrDefault(); // Bucket Name; string objectKey = path.Skip(3).FirstOrDefault(); // Object Key; // Object key can include / in the name, so we need to decode it @@ -42,7 +42,7 @@ logger.LogInformation($"Service: {(service ?? "---")}"); logger.LogInformation($"Region: {(region ?? "---")}"); - logger.LogInformation($"Object Name: {(objectName ?? "---")}"); + logger.LogInformation($"Bucket Name: {(bucketName ?? "---")}"); logger.LogInformation($"Object Key: {(objectKey ?? "---")}"); var regionUrlPart = string.Empty; @@ -55,17 +55,23 @@ var endpointUri = string.Format("https://{2}.{0}{1}.amazonaws.com", service, regionUrlPart, - objectName); + bucketName); if (! string.IsNullOrEmpty(objectKey)) { endpointUri = string.Format("{0}/{1}", endpointUri, objectKey); - } + } + foreach (var character in new[]{"(", ")", "[", "]", "{", "}", "#"}) { + if (endpointUri.Contains(character)) + endpointUri = endpointUri.Replace(character, Uri.EscapeDataString(character)); + } var uri = new Uri($"{endpointUri}{requestUri.Query}"); - + logger.LogInformation($"Uri AbsolutePath: {uri.AbsolutePath}"); + var request = new System.Net.Http.HttpRequestMessage(this.Context.Request.Method, uri) { - Content = this.Context.Request.Content + Content = GetConvertedRequestContent() }; - SignRequest(request, service, region, accessKeyId, accessKeySecret); + + SignRequest(request, service, region, accessKeyId, accessKeySecret, logger); logger.LogInformation($"Call: {request.Method} {request.RequestUri!}"); // Use the context to forward/send an HTTP request @@ -85,6 +91,20 @@ throw new Exception("Unexpected Authentication"); } + private System.Net.Http.HttpContent GetConvertedRequestContent() + { + var logger = this.Context.Logger; + try { + var binary = Convert.FromBase64String(this.Context.Request.Content.ReadAsStringAsync().Result); + logger.LogInformation($"Base64 content converted into binary (Binary content length: {binary.Length})"); + return new System.Net.Http.ByteArrayContent(binary); + + } catch { + logger.LogInformation("Content can't be converted from Base64 to binary, keep the content as it is."); + return this.Context.Request.Content; + } + } + private async Task HandleTransformXML2Json(HttpResponseMessage response) { // Do the transformation if the response was successful, otherwise return error responses as-is @@ -125,7 +145,8 @@ string service, string region, string awsAccessKey, - string awsSecretKey) + string awsSecretKey, + ILogger logger = null) { var signer = new AWS4SignerForAuthorizationHeader { @@ -156,7 +177,7 @@ headers.Add(AWS4SignerBase.X_Amz_Content_SHA256, bodyHash); } - string signature = signer.ComputeSignature(headers, queryParameters, bodyHash, awsAccessKey, awsSecretKey); + string signature = signer.ComputeSignature(headers, queryParameters, bodyHash, awsAccessKey, awsSecretKey, logger); foreach (var header in headers.Keys) { @@ -465,7 +486,8 @@ string queryParameters, string bodyHash, string awsAccessKey, - string awsSecretKey) + string awsSecretKey, + ILogger logger = null) { // first get the date and time for the subsequent request, and convert to ISO 8601 format // for use in signature generation @@ -515,6 +537,7 @@ canonicalizedHeaderNames, canonicalizedHeaders, bodyHash); + logger?.LogInformation($"Canonical Request:\n{canonicalRequest}"); Console.WriteLine("\nCanonicalRequest:\n{0}", canonicalRequest); // generate a hash of the canonical request, to go into signature computation @@ -534,6 +557,7 @@ stringToSign.AppendFormat("{0}-{1}\n{2}\n{3}\n", SCHEME, ALGORITHM, dateTimeStamp, scope); stringToSign.Append(ToHexString(canonicalRequestHashBytes, true)); + logger?.LogInformation($"String to Sign:\n{stringToSign}"); Console.WriteLine("\nStringToSign:\n{0}", stringToSign); // compute the signing key