StreamToString() does not corrupt images anymore. (Azure#10166)

* StreamToString() does not corrupt images anymore. The previous code corrupted files that aren't simple text files. The code corrupted all image files that I downloaded. The new code keeps the file in its original binary format. A string does not have the same encoding as an image. To preserve the file in its proper binary code you have to keep it as a buffer. Call .toString() at the end of the download. * Changed streamToString to streamToBuffer * Updated TS docs * Improved performance of streamToBuffer() ts * Improved performance of streamToBuffer() js * streamToBuffer Co-authored-by: Lin Jian <[email protected]> Co-authored-by: Lin Jian <[email protected]>
AzureSDKAutomation · Sep 7, 2020 · 8c2cf0b · 8c2cf0b
1 parent 89faf37
commit 8c2cf0b
Show file tree

Hide file tree

Showing 16 changed files with 118 additions and 103 deletions.
diff --git a/sdk/storage/storage-blob/README.md b/sdk/storage/storage-blob/README.md
@@ -434,18 +434,18 @@ async function main() {
   // Get blob content from position 0 to the end
   // In Node.js, get downloaded data by accessing downloadBlockBlobResponse.readableStreamBody
   const downloadBlockBlobResponse = await blobClient.download();
-  const downloaded = await streamToString(downloadBlockBlobResponse.readableStreamBody);
+  const downloaded = (await streamToBuffer(downloadBlockBlobResponse.readableStreamBody)).toString();
   console.log("Downloaded blob content:", downloaded);
 
-  // [Node.js only] A helper method used to read a Node.js readable stream into string
-  async function streamToString(readableStream) {
+  // [Node.js only] A helper method used to read a Node.js readable stream into a Buffer
+  async function streamToBuffer(readableStream) {
     return new Promise((resolve, reject) => {
       const chunks = [];
       readableStream.on("data", (data) => {
-        chunks.push(data.toString());
+        chunks.push(data instanceof Buffer ? data : Buffer.from(data));
       });
       readableStream.on("end", () => {
-        resolve(chunks.join(""));
+        resolve(Buffer.concat(chunks));
       });
       readableStream.on("error", reject);
     });

diff --git a/sdk/storage/storage-blob/samples/javascript/basic.js b/sdk/storage/storage-blob/samples/javascript/basic.js
@@ -76,7 +76,7 @@ async function main() {
   const downloadBlockBlobResponse = await blockBlobClient.download(0);
   console.log(
     "Downloaded blob content",
-    await streamToString(downloadBlockBlobResponse.readableStreamBody)
+    (await streamToBuffer(downloadBlockBlobResponse.readableStreamBody)).toString()
   );
 
   // Delete container
@@ -85,15 +85,15 @@ async function main() {
   console.log("deleted container");
 }
 
-// A helper method used to read a Node.js readable stream into string
-async function streamToString(readableStream) {
+// A helper method used to read a Node.js readable stream into a Buffer
+async function streamToBuffer(readableStream) {
   return new Promise((resolve, reject) => {
     const chunks = [];
     readableStream.on("data", (data) => {
-      chunks.push(data.toString());
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
     });
     readableStream.on("end", () => {
-      resolve(chunks.join(""));
+      resolve(Buffer.concat(chunks));
     });
     readableStream.on("error", reject);
   });

diff --git a/sdk/storage/storage-blob/samples/javascript/errorsAndResponses.js b/sdk/storage/storage-blob/samples/javascript/errorsAndResponses.js
@@ -90,9 +90,9 @@ async function main() {
     blockBlobClient = containerClient.getBlockBlobClient(blobName);
     const downloadBlockBlobResponse = await blockBlobClient.download();
     console.log(
-      `Downloaded blob content - ${await streamToString(
-        downloadBlockBlobResponse.readableStreamBody
-      )},`
+      `Downloaded blob content - ${(
+        await streamToBuffer(downloadBlockBlobResponse.readableStreamBody)
+      ).toString()},`
     );
     console.log(
       `requestId - ${downloadBlockBlobResponse.requestId}, statusCode - ${downloadBlockBlobResponse._response.status}\n`
@@ -137,15 +137,15 @@ async function main() {
   }
 }
 
-// A helper method used to read a Node.js readable stream into string
-async function streamToString(readableStream) {
+// A helper method used to read a Node.js readable stream into a Buffer
+async function streamToBuffer(readableStream) {
   return new Promise((resolve, reject) => {
     const chunks = [];
     readableStream.on("data", (data) => {
-      chunks.push(data.toString());
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
     });
     readableStream.on("end", () => {
-      resolve(chunks.join(""));
+      resolve(Buffer.concat(chunks));
     });
     readableStream.on("error", reject);
   });

diff --git a/sdk/storage/storage-blob/samples/javascript/readingSnapshot.js b/sdk/storage/storage-blob/samples/javascript/readingSnapshot.js
@@ -64,23 +64,26 @@ async function main() {
     (await blobSnapshotClient.getProperties()).contentLength
   );
 
-  console.log("Downloaded blob content", await streamToString(response.readableStreamBody));
+  console.log(
+    "Downloaded blob content",
+    (await streamToBuffer(response.readableStreamBody)).toString()
+  );
 
   // Delete container
   await containerClient.delete();
 
   console.log("deleted container");
 }
 
-// A helper method used to read a Node.js readable stream into string
-async function streamToString(readableStream) {
+// A helper method used to read a Node.js readable stream into a Buffer
+async function streamToBuffer(readableStream) {
   return new Promise((resolve, reject) => {
     const chunks = [];
     readableStream.on("data", (data) => {
-      chunks.push(data.toString());
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
     });
     readableStream.on("end", () => {
-      resolve(chunks.join(""));
+      resolve(Buffer.concat(chunks));
     });
     readableStream.on("error", reject);
   });

diff --git a/sdk/storage/storage-blob/samples/typescript/src/basic.ts b/sdk/storage/storage-blob/samples/typescript/src/basic.ts
@@ -81,7 +81,7 @@ export async function main() {
   const downloadBlockBlobResponse: BlobDownloadResponseModel = await blockBlobClient.download(0);
   console.log(
     "Downloaded blob content",
-    await streamToString(downloadBlockBlobResponse.readableStreamBody!)
+    (await streamToBuffer(downloadBlockBlobResponse.readableStreamBody!)).toString()
   );
 
   // Delete container
@@ -90,15 +90,15 @@ export async function main() {
   console.log("deleted container");
 }
 
-// A helper method used to read a Node.js readable stream into string
-async function streamToString(readableStream: NodeJS.ReadableStream) {
+// A helper method used to read a Node.js readable stream into a Buffer
+async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise<Buffer> {
   return new Promise((resolve, reject) => {
-    const chunks: string[] = [];
-    readableStream.on("data", (data) => {
-      chunks.push(data.toString());
+    const chunks: Buffer[] = [];
+    readableStream.on("data", (data: Buffer | string) => {
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
     });
     readableStream.on("end", () => {
-      resolve(chunks.join(""));
+      resolve(Buffer.concat(chunks));
     });
     readableStream.on("error", reject);
   });

diff --git a/sdk/storage/storage-blob/samples/typescript/src/errorsAndResponses.ts b/sdk/storage/storage-blob/samples/typescript/src/errorsAndResponses.ts
@@ -91,9 +91,9 @@ export async function main() {
     blockBlobClient = containerClient.getBlockBlobClient(blobName);
     const downloadBlockBlobResponse = await blockBlobClient.download();
     console.log(
-      `Downloaded blob content - ${await streamToString(
-        downloadBlockBlobResponse.readableStreamBody!
-      )},`
+      `Downloaded blob content - ${(
+        await streamToBuffer(downloadBlockBlobResponse.readableStreamBody!)
+      ).toString()},`
     );
     console.log(
       `requestId - ${downloadBlockBlobResponse.requestId}, statusCode - ${downloadBlockBlobResponse._response.status}\n`
@@ -138,15 +138,15 @@ export async function main() {
   }
 }
 
-// A helper method used to read a Node.js readable stream into string
-async function streamToString(readableStream: NodeJS.ReadableStream) {
+// A helper method used to read a Node.js readable stream into a Buffer
+async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise<Buffer> {
   return new Promise((resolve, reject) => {
-    const chunks: string[] = [];
-    readableStream.on("data", (data) => {
-      chunks.push(data.toString());
+    const chunks: Buffer[] = [];
+    readableStream.on("data", (data: Buffer | string) => {
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
     });
     readableStream.on("end", () => {
-      resolve(chunks.join(""));
+      resolve(Buffer.concat(chunks));
     });
     readableStream.on("error", reject);
   });

diff --git a/sdk/storage/storage-blob/samples/typescript/src/readingSnapshot.ts b/sdk/storage/storage-blob/samples/typescript/src/readingSnapshot.ts
@@ -64,23 +64,26 @@ export async function main() {
     (await blobSnapshotClient.getProperties()).contentLength
   );
 
-  console.log("Downloaded blob content", await streamToString(response.readableStreamBody!));
-
+  console.log(
+    "Downloaded blob content",
+    (await streamToBuffer(response.readableStreamBody!)).toString()
+  );
+
   // Delete container
   await containerClient.delete();
 
   console.log("deleted container");
 }
 
-// A helper method used to read a Node.js readable stream into string
-async function streamToString(readableStream: NodeJS.ReadableStream) {
+// A helper method used to read a Node.js readable stream into a Buffer
+async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise<Buffer> {
   return new Promise((resolve, reject) => {
-    const chunks: string[] = [];
-    readableStream.on("data", (data) => {
-      chunks.push(data.toString());
+    const chunks: Buffer[] = [];
+    readableStream.on("data", (data: Buffer | string) => {
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
     });
     readableStream.on("end", () => {
-      resolve(chunks.join(""));
+      resolve(Buffer.concat(chunks));
     });
     readableStream.on("error", reject);
   });

diff --git a/sdk/storage/storage-blob/src/Clients.ts b/sdk/storage/storage-blob/src/Clients.ts
@@ -1277,20 +1277,20 @@ export class BlobClient extends StorageClient {
    * ```js
    * // Download and convert a blob to a string
    * const downloadBlockBlobResponse = await blobClient.download();
-   * const downloaded = await streamToString(downloadBlockBlobResponse.readableStreamBody);
-   * console.log("Downloaded blob content:", downloaded);
+   * const downloaded = await streamToBuffer(downloadBlockBlobResponse.readableStreamBody);
+   * console.log("Downloaded blob content:", downloaded.toString());
    *
-   * async function streamToString(readableStream) {
-   *   return new Promise((resolve, reject) => {
-   *     const chunks = [];
-   *     readableStream.on("data", (data) => {
-   *       chunks.push(data.toString());
-   *     });
-   *     readableStream.on("end", () => {
-   *       resolve(chunks.join(""));
-   *     });
-   *     readableStream.on("error", reject);
-   *   });
+   * async function streamToBuffer(readableStream) {
+   * return new Promise((resolve, reject) => {
+   * const chunks = [];
+   * readableStream.on("data", (data) => {
+   * chunks.push(data instanceof Buffer ? data : Buffer.from(data));
+   * });
+   * readableStream.on("end", () => {
+   * resolve(Buffer.concat(chunks));
+   * });
+   * readableStream.on("error", reject);
+   * });
    * }
    * ```
    *

diff --git a/sdk/storage/storage-file-datalake/README.md b/sdk/storage/storage-file-datalake/README.md
@@ -482,18 +482,18 @@ async function main() {
   // Get file content from position 0 to the end
   // In Node.js, get downloaded data by accessing downloadResponse.readableStreamBody
   const downloadResponse = await fileClient.read();
-  const downloaded = await streamToString(downloadResponse.readableStreamBody);
-  console.log("Downloaded file content:", downloaded);
+  const downloaded = await streamToBuffer(downloadResponse.readableStreamBody);
+  console.log("Downloaded file content:", downloaded.toString());
 
-  // [Node.js only] A helper method used to read a Node.js readable stream into string
-  async function streamToString(readableStream) {
+  // [Node.js only] A helper method used to read a Node.js readable stream into a Buffer.
+  async function streamToBuffer(readableStream) {
     return new Promise((resolve, reject) => {
       const chunks = [];
       readableStream.on("data", (data) => {
-        chunks.push(data.toString());
+        chunks.push(data instanceof Buffer ? data : Buffer.from(data));
       });
       readableStream.on("end", () => {
-        resolve(chunks.join(""));
+        resolve(Buffer.concat(chunks));
       });
       readableStream.on("error", reject);
     });

diff --git a/sdk/storage/storage-file-datalake/samples/javascript/basic.js b/sdk/storage/storage-file-datalake/samples/javascript/basic.js
@@ -72,7 +72,7 @@ async function main() {
   const readFileResponse = await fileClient.read();
   console.log(
     "Downloaded file content",
-    await streamToString(readFileResponse.readableStreamBody)
+    (await streamToBuffer(readFileResponse.readableStreamBody)).toString()
   );
 
   // Delete filesystem
@@ -81,15 +81,15 @@ async function main() {
   console.log("Deleted filesystem");
 }
 
-// A helper method used to read a Node.js readable stream into string
-async function streamToString(readableStream) {
+// A helper method used to read a Node.js readable stream into a Buffer
+async function streamToBuffer(readableStream) {
   return new Promise((resolve, reject) => {
     const chunks = [];
     readableStream.on("data", (data) => {
-      chunks.push(data.toString());
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
     });
     readableStream.on("end", () => {
-      resolve(chunks.join(""));
+      resolve(Buffer.concat(chunks));
     });
     readableStream.on("error", reject);
   });

diff --git a/sdk/storage/storage-file-datalake/samples/typescript/basic.ts b/sdk/storage/storage-file-datalake/samples/typescript/basic.ts
@@ -73,23 +73,26 @@ async function main() {
   // In Node.js, get downloaded data by accessing downloadBlockBlobResponse.readableStreamBody
   // In browsers, get downloaded data by accessing downloadBlockBlobResponse.contentAsBlob
   const readFileResponse = await fileClient.read();
-  console.log("Downloaded file content", await streamToString(readFileResponse.readableStreamBody));
+  console.log(
+    "Downloaded file content",
+    (await streamToBuffer(readFileResponse.readableStreamBody)).toString()
+  );
 
   // Delete filesystem
   await fileSystemClient.delete();
 
   console.log("Deleted filesystem");
 }
 
-// A helper method used to read a Node.js readable stream into string
-async function streamToString(readableStream: NodeJS.ReadableStream) {
+// A helper method used to read a Node.js readable stream into a Buffer
+async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise<Buffer> {
   return new Promise((resolve, reject) => {
-    const chunks: string[] = [];
-    readableStream.on("data", (data) => {
-      chunks.push(data.toString());
+    const chunks: Buffer[] = [];
+    readableStream.on("data", (data: Buffer | string) => {
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
     });
     readableStream.on("end", () => {
-      resolve(chunks.join(""));
+      resolve(Buffer.concat(chunks));
     });
     readableStream.on("error", reject);
   });

diff --git a/sdk/storage/storage-file-datalake/src/clients.ts b/sdk/storage/storage-file-datalake/src/clients.ts
@@ -1134,17 +1134,17 @@ export class DataLakeFileClient extends DataLakePathClient {
    * ```js
    * // Download and convert a file to a string
    * const downloadResponse = await fileClient.read();
-   * const downloaded = await streamToString(downloadResponse.readableStreamBody);
-   * console.log("Downloaded file content:", downloaded);
+   * const downloaded = await streamToBuffer(downloadResponse.readableStreamBody);
+   * console.log("Downloaded file content:", downloaded.toString());
    *
-   * async function streamToString(readableStream) {
+   * async function streamToBuffer(readableStream) {
    *   return new Promise((resolve, reject) => {
    *     const chunks = [];
    *     readableStream.on("data", (data) => {
-   *       chunks.push(data.toString());
+   *       chunks.push(data instanceof Buffer ? data : Buffer.from(data));
    *     });
    *     readableStream.on("end", () => {
-   *       resolve(chunks.join(""));
+   *       resolve(Buffer.concat(chunks));
    *     });
    *     readableStream.on("error", reject);
    *   });

diff --git a/sdk/storage/storage-file-share/README.md b/sdk/storage/storage-file-share/README.md
@@ -402,15 +402,15 @@ const serviceClient = new ShareServiceClient(
 const shareName = "<share name>";
 const fileName = "<file name>";
 
-// [Node.js only] A helper method used to read a Node.js readable stream into string
-async function streamToString(readableStream) {
+// [Node.js only] A helper method used to read a Node.js readable stream into a Buffer
+async function streamToBuffer(readableStream) {
   return new Promise((resolve, reject) => {
     const chunks = [];
     readableStream.on("data", (data) => {
-      chunks.push(data.toString());
+      chunks.push(data instanceof Buffer ? data : Buffer.from(data));
     });
     readableStream.on("end", () => {
-      resolve(chunks.join(""));
+      resolve(Buffer.concat(chunks));
     });
     readableStream.on("error", reject);
   });
@@ -425,7 +425,9 @@ async function main() {
   // In Node.js, get downloaded data by accessing downloadFileResponse.readableStreamBody
   const downloadFileResponse = await fileClient.download();
   console.log(
-    `Downloaded file content: ${await streamToString(downloadFileResponse.readableStreamBody)}`
+    `Downloaded file content: ${(
+      await streamToBuffer(downloadFileResponse.readableStreamBody)
+    ).toString()}`
   );
 }