From 8c2cf0b4b223c210fa58023787350e454d2a127b Mon Sep 17 00:00:00 2001 From: Alex Spelt <39807948+AlexSpelt@users.noreply.github.com> Date: Mon, 7 Sep 2020 08:00:11 +0200 Subject: [PATCH] StreamToString() does not corrupt images anymore. (#10166) * StreamToString() does not corrupt images anymore. The previous code corrupted files that aren't simple text files. The code corrupted all image files that I downloaded. The new code keeps the file in its original binary format. A string does not have the same encoding as an image. To preserve the file in its proper binary code you have to keep it as a buffer. Call .toString() at the end of the download. * Changed streamToString to streamToBuffer * Updated TS docs * Improved performance of streamToBuffer() ts * Improved performance of streamToBuffer() js * streamToBuffer Co-authored-by: Lin Jian Co-authored-by: Lin Jian <1215122919@qq.com> --- sdk/storage/storage-blob/README.md | 10 +++---- .../storage-blob/samples/javascript/basic.js | 10 +++---- .../samples/javascript/errorsAndResponses.js | 14 +++++----- .../samples/javascript/readingSnapshot.js | 13 ++++++---- .../samples/typescript/src/basic.ts | 14 +++++----- .../typescript/src/errorsAndResponses.ts | 18 ++++++------- .../samples/typescript/src/readingSnapshot.ts | 19 ++++++++------ sdk/storage/storage-blob/src/Clients.ts | 26 +++++++++---------- sdk/storage/storage-file-datalake/README.md | 12 ++++----- .../samples/javascript/basic.js | 10 +++---- .../samples/typescript/basic.ts | 17 +++++++----- .../storage-file-datalake/src/clients.ts | 10 +++---- sdk/storage/storage-file-share/README.md | 12 +++++---- .../samples/javascript/basic.js | 12 +++++---- .../samples/typescript/src/basic.ts | 16 +++++++----- .../storage-file-share/src/ShareFileClient.ts | 8 +++--- 16 files changed, 118 insertions(+), 103 deletions(-) diff --git a/sdk/storage/storage-blob/README.md b/sdk/storage/storage-blob/README.md index ab0d1452600c..a592422b699b 100644 --- a/sdk/storage/storage-blob/README.md +++ b/sdk/storage/storage-blob/README.md @@ -434,18 +434,18 @@ async function main() { // Get blob content from position 0 to the end // In Node.js, get downloaded data by accessing downloadBlockBlobResponse.readableStreamBody const downloadBlockBlobResponse = await blobClient.download(); - const downloaded = await streamToString(downloadBlockBlobResponse.readableStreamBody); + const downloaded = (await streamToBuffer(downloadBlockBlobResponse.readableStreamBody)).toString(); console.log("Downloaded blob content:", downloaded); - // [Node.js only] A helper method used to read a Node.js readable stream into string - async function streamToString(readableStream) { + // [Node.js only] A helper method used to read a Node.js readable stream into a Buffer + async function streamToBuffer(readableStream) { return new Promise((resolve, reject) => { const chunks = []; readableStream.on("data", (data) => { - chunks.push(data.toString()); + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-blob/samples/javascript/basic.js b/sdk/storage/storage-blob/samples/javascript/basic.js index 629ec485b742..05b5fdd62b46 100644 --- a/sdk/storage/storage-blob/samples/javascript/basic.js +++ b/sdk/storage/storage-blob/samples/javascript/basic.js @@ -76,7 +76,7 @@ async function main() { const downloadBlockBlobResponse = await blockBlobClient.download(0); console.log( "Downloaded blob content", - await streamToString(downloadBlockBlobResponse.readableStreamBody) + (await streamToBuffer(downloadBlockBlobResponse.readableStreamBody)).toString() ); // Delete container @@ -85,15 +85,15 @@ async function main() { console.log("deleted container"); } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream) { return new Promise((resolve, reject) => { const chunks = []; readableStream.on("data", (data) => { - chunks.push(data.toString()); + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-blob/samples/javascript/errorsAndResponses.js b/sdk/storage/storage-blob/samples/javascript/errorsAndResponses.js index 8ccd4888f7a0..0b6b19e17b2e 100644 --- a/sdk/storage/storage-blob/samples/javascript/errorsAndResponses.js +++ b/sdk/storage/storage-blob/samples/javascript/errorsAndResponses.js @@ -90,9 +90,9 @@ async function main() { blockBlobClient = containerClient.getBlockBlobClient(blobName); const downloadBlockBlobResponse = await blockBlobClient.download(); console.log( - `Downloaded blob content - ${await streamToString( - downloadBlockBlobResponse.readableStreamBody - )},` + `Downloaded blob content - ${( + await streamToBuffer(downloadBlockBlobResponse.readableStreamBody) + ).toString()},` ); console.log( `requestId - ${downloadBlockBlobResponse.requestId}, statusCode - ${downloadBlockBlobResponse._response.status}\n` @@ -137,15 +137,15 @@ async function main() { } } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream) { return new Promise((resolve, reject) => { const chunks = []; readableStream.on("data", (data) => { - chunks.push(data.toString()); + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-blob/samples/javascript/readingSnapshot.js b/sdk/storage/storage-blob/samples/javascript/readingSnapshot.js index 294123931d6e..9bceeaf7880c 100644 --- a/sdk/storage/storage-blob/samples/javascript/readingSnapshot.js +++ b/sdk/storage/storage-blob/samples/javascript/readingSnapshot.js @@ -64,7 +64,10 @@ async function main() { (await blobSnapshotClient.getProperties()).contentLength ); - console.log("Downloaded blob content", await streamToString(response.readableStreamBody)); + console.log( + "Downloaded blob content", + (await streamToBuffer(response.readableStreamBody)).toString() + ); // Delete container await containerClient.delete(); @@ -72,15 +75,15 @@ async function main() { console.log("deleted container"); } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream) { return new Promise((resolve, reject) => { const chunks = []; readableStream.on("data", (data) => { - chunks.push(data.toString()); + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-blob/samples/typescript/src/basic.ts b/sdk/storage/storage-blob/samples/typescript/src/basic.ts index 85b16f1ed3a7..077d0a332c12 100644 --- a/sdk/storage/storage-blob/samples/typescript/src/basic.ts +++ b/sdk/storage/storage-blob/samples/typescript/src/basic.ts @@ -81,7 +81,7 @@ export async function main() { const downloadBlockBlobResponse: BlobDownloadResponseModel = await blockBlobClient.download(0); console.log( "Downloaded blob content", - await streamToString(downloadBlockBlobResponse.readableStreamBody!) + (await streamToBuffer(downloadBlockBlobResponse.readableStreamBody!)).toString() ); // Delete container @@ -90,15 +90,15 @@ export async function main() { console.log("deleted container"); } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream: NodeJS.ReadableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise { return new Promise((resolve, reject) => { - const chunks: string[] = []; - readableStream.on("data", (data) => { - chunks.push(data.toString()); + const chunks: Buffer[] = []; + readableStream.on("data", (data: Buffer | string) => { + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-blob/samples/typescript/src/errorsAndResponses.ts b/sdk/storage/storage-blob/samples/typescript/src/errorsAndResponses.ts index 0a58d5320633..211cd56b5956 100644 --- a/sdk/storage/storage-blob/samples/typescript/src/errorsAndResponses.ts +++ b/sdk/storage/storage-blob/samples/typescript/src/errorsAndResponses.ts @@ -91,9 +91,9 @@ export async function main() { blockBlobClient = containerClient.getBlockBlobClient(blobName); const downloadBlockBlobResponse = await blockBlobClient.download(); console.log( - `Downloaded blob content - ${await streamToString( - downloadBlockBlobResponse.readableStreamBody! - )},` + `Downloaded blob content - ${( + await streamToBuffer(downloadBlockBlobResponse.readableStreamBody!) + ).toString()},` ); console.log( `requestId - ${downloadBlockBlobResponse.requestId}, statusCode - ${downloadBlockBlobResponse._response.status}\n` @@ -138,15 +138,15 @@ export async function main() { } } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream: NodeJS.ReadableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise { return new Promise((resolve, reject) => { - const chunks: string[] = []; - readableStream.on("data", (data) => { - chunks.push(data.toString()); + const chunks: Buffer[] = []; + readableStream.on("data", (data: Buffer | string) => { + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-blob/samples/typescript/src/readingSnapshot.ts b/sdk/storage/storage-blob/samples/typescript/src/readingSnapshot.ts index a0dece1e0c22..1d3a35c447f8 100644 --- a/sdk/storage/storage-blob/samples/typescript/src/readingSnapshot.ts +++ b/sdk/storage/storage-blob/samples/typescript/src/readingSnapshot.ts @@ -64,23 +64,26 @@ export async function main() { (await blobSnapshotClient.getProperties()).contentLength ); - console.log("Downloaded blob content", await streamToString(response.readableStreamBody!)); - + console.log( + "Downloaded blob content", + (await streamToBuffer(response.readableStreamBody!)).toString() + ); + // Delete container await containerClient.delete(); console.log("deleted container"); } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream: NodeJS.ReadableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise { return new Promise((resolve, reject) => { - const chunks: string[] = []; - readableStream.on("data", (data) => { - chunks.push(data.toString()); + const chunks: Buffer[] = []; + readableStream.on("data", (data: Buffer | string) => { + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-blob/src/Clients.ts b/sdk/storage/storage-blob/src/Clients.ts index f811fc845f67..691574528bde 100644 --- a/sdk/storage/storage-blob/src/Clients.ts +++ b/sdk/storage/storage-blob/src/Clients.ts @@ -1277,20 +1277,20 @@ export class BlobClient extends StorageClient { * ```js * // Download and convert a blob to a string * const downloadBlockBlobResponse = await blobClient.download(); - * const downloaded = await streamToString(downloadBlockBlobResponse.readableStreamBody); - * console.log("Downloaded blob content:", downloaded); + * const downloaded = await streamToBuffer(downloadBlockBlobResponse.readableStreamBody); + * console.log("Downloaded blob content:", downloaded.toString()); * - * async function streamToString(readableStream) { - * return new Promise((resolve, reject) => { - * const chunks = []; - * readableStream.on("data", (data) => { - * chunks.push(data.toString()); - * }); - * readableStream.on("end", () => { - * resolve(chunks.join("")); - * }); - * readableStream.on("error", reject); - * }); + * async function streamToBuffer(readableStream) { + * return new Promise((resolve, reject) => { + * const chunks = []; + * readableStream.on("data", (data) => { + * chunks.push(data instanceof Buffer ? data : Buffer.from(data)); + * }); + * readableStream.on("end", () => { + * resolve(Buffer.concat(chunks)); + * }); + * readableStream.on("error", reject); + * }); * } * ``` * diff --git a/sdk/storage/storage-file-datalake/README.md b/sdk/storage/storage-file-datalake/README.md index 68d858a25817..77962be088fd 100644 --- a/sdk/storage/storage-file-datalake/README.md +++ b/sdk/storage/storage-file-datalake/README.md @@ -482,18 +482,18 @@ async function main() { // Get file content from position 0 to the end // In Node.js, get downloaded data by accessing downloadResponse.readableStreamBody const downloadResponse = await fileClient.read(); - const downloaded = await streamToString(downloadResponse.readableStreamBody); - console.log("Downloaded file content:", downloaded); + const downloaded = await streamToBuffer(downloadResponse.readableStreamBody); + console.log("Downloaded file content:", downloaded.toString()); - // [Node.js only] A helper method used to read a Node.js readable stream into string - async function streamToString(readableStream) { + // [Node.js only] A helper method used to read a Node.js readable stream into a Buffer. + async function streamToBuffer(readableStream) { return new Promise((resolve, reject) => { const chunks = []; readableStream.on("data", (data) => { - chunks.push(data.toString()); + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-file-datalake/samples/javascript/basic.js b/sdk/storage/storage-file-datalake/samples/javascript/basic.js index 343f25fec42c..9e5cc1393a31 100644 --- a/sdk/storage/storage-file-datalake/samples/javascript/basic.js +++ b/sdk/storage/storage-file-datalake/samples/javascript/basic.js @@ -72,7 +72,7 @@ async function main() { const readFileResponse = await fileClient.read(); console.log( "Downloaded file content", - await streamToString(readFileResponse.readableStreamBody) + (await streamToBuffer(readFileResponse.readableStreamBody)).toString() ); // Delete filesystem @@ -81,15 +81,15 @@ async function main() { console.log("Deleted filesystem"); } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream) { return new Promise((resolve, reject) => { const chunks = []; readableStream.on("data", (data) => { - chunks.push(data.toString()); + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-file-datalake/samples/typescript/basic.ts b/sdk/storage/storage-file-datalake/samples/typescript/basic.ts index cab7591e0bcc..aac063c942e0 100644 --- a/sdk/storage/storage-file-datalake/samples/typescript/basic.ts +++ b/sdk/storage/storage-file-datalake/samples/typescript/basic.ts @@ -73,7 +73,10 @@ async function main() { // In Node.js, get downloaded data by accessing downloadBlockBlobResponse.readableStreamBody // In browsers, get downloaded data by accessing downloadBlockBlobResponse.contentAsBlob const readFileResponse = await fileClient.read(); - console.log("Downloaded file content", await streamToString(readFileResponse.readableStreamBody)); + console.log( + "Downloaded file content", + (await streamToBuffer(readFileResponse.readableStreamBody)).toString() + ); // Delete filesystem await fileSystemClient.delete(); @@ -81,15 +84,15 @@ async function main() { console.log("Deleted filesystem"); } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream: NodeJS.ReadableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise { return new Promise((resolve, reject) => { - const chunks: string[] = []; - readableStream.on("data", (data) => { - chunks.push(data.toString()); + const chunks: Buffer[] = []; + readableStream.on("data", (data: Buffer | string) => { + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-file-datalake/src/clients.ts b/sdk/storage/storage-file-datalake/src/clients.ts index 20f0f199a90c..8462b1f6c305 100644 --- a/sdk/storage/storage-file-datalake/src/clients.ts +++ b/sdk/storage/storage-file-datalake/src/clients.ts @@ -1134,17 +1134,17 @@ export class DataLakeFileClient extends DataLakePathClient { * ```js * // Download and convert a file to a string * const downloadResponse = await fileClient.read(); - * const downloaded = await streamToString(downloadResponse.readableStreamBody); - * console.log("Downloaded file content:", downloaded); + * const downloaded = await streamToBuffer(downloadResponse.readableStreamBody); + * console.log("Downloaded file content:", downloaded.toString()); * - * async function streamToString(readableStream) { + * async function streamToBuffer(readableStream) { * return new Promise((resolve, reject) => { * const chunks = []; * readableStream.on("data", (data) => { - * chunks.push(data.toString()); + * chunks.push(data instanceof Buffer ? data : Buffer.from(data)); * }); * readableStream.on("end", () => { - * resolve(chunks.join("")); + * resolve(Buffer.concat(chunks)); * }); * readableStream.on("error", reject); * }); diff --git a/sdk/storage/storage-file-share/README.md b/sdk/storage/storage-file-share/README.md index cb3465c98204..fc5293936c6c 100644 --- a/sdk/storage/storage-file-share/README.md +++ b/sdk/storage/storage-file-share/README.md @@ -402,15 +402,15 @@ const serviceClient = new ShareServiceClient( const shareName = ""; const fileName = ""; -// [Node.js only] A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream) { +// [Node.js only] A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream) { return new Promise((resolve, reject) => { const chunks = []; readableStream.on("data", (data) => { - chunks.push(data.toString()); + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); @@ -425,7 +425,9 @@ async function main() { // In Node.js, get downloaded data by accessing downloadFileResponse.readableStreamBody const downloadFileResponse = await fileClient.download(); console.log( - `Downloaded file content: ${await streamToString(downloadFileResponse.readableStreamBody)}` + `Downloaded file content: ${( + await streamToBuffer(downloadFileResponse.readableStreamBody) + ).toString()}` ); } diff --git a/sdk/storage/storage-file-share/samples/javascript/basic.js b/sdk/storage/storage-file-share/samples/javascript/basic.js index c74efb10fcb1..c48f8c030086 100644 --- a/sdk/storage/storage-file-share/samples/javascript/basic.js +++ b/sdk/storage/storage-file-share/samples/javascript/basic.js @@ -75,7 +75,9 @@ async function main() { // In browsers, get downloaded data by accessing downloadFileResponse.contentAsBlob const downloadFileResponse = await fileClient.download(0); console.log( - `Downloaded file content: ${await streamToString(downloadFileResponse.readableStreamBody)}` + `Downloaded file content: ${( + await streamToBuffer(downloadFileResponse.readableStreamBody) + ).toString()}` ); // Delete share @@ -83,15 +85,15 @@ async function main() { console.log(`deleted share ${shareName}`); } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream) { return new Promise((resolve, reject) => { const chunks = []; readableStream.on("data", (data) => { - chunks.push(data.toString()); + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-file-share/samples/typescript/src/basic.ts b/sdk/storage/storage-file-share/samples/typescript/src/basic.ts index ef4cbcce45ff..2be656549054 100644 --- a/sdk/storage/storage-file-share/samples/typescript/src/basic.ts +++ b/sdk/storage/storage-file-share/samples/typescript/src/basic.ts @@ -77,7 +77,9 @@ export async function main() { // In browsers, get downloaded data by accessing downloadFileResponse.contentAsBlob const downloadFileResponse = await fileClient.download(0); console.log( - `Downloaded file content: ${await streamToString(downloadFileResponse.readableStreamBody!)}` + `Downloaded file content: ${( + await streamToBuffer(downloadFileResponse.readableStreamBody!) + ).toString()}` ); // Delete share @@ -85,15 +87,15 @@ export async function main() { console.log(`deleted share ${shareName}`); } -// A helper method used to read a Node.js readable stream into string -async function streamToString(readableStream: NodeJS.ReadableStream) { +// A helper method used to read a Node.js readable stream into a Buffer +async function streamToBuffer(readableStream: NodeJS.ReadableStream): Promise { return new Promise((resolve, reject) => { - const chunks: string[] = []; - readableStream.on("data", (data) => { - chunks.push(data.toString()); + const chunks: Buffer[] = []; + readableStream.on("data", (data: Buffer | string) => { + chunks.push(data instanceof Buffer ? data : Buffer.from(data)); }); readableStream.on("end", () => { - resolve(chunks.join("")); + resolve(Buffer.concat(chunks)); }); readableStream.on("error", reject); }); diff --git a/sdk/storage/storage-file-share/src/ShareFileClient.ts b/sdk/storage/storage-file-share/src/ShareFileClient.ts index 3c27d5e30d5b..86d55c274c0b 100644 --- a/sdk/storage/storage-file-share/src/ShareFileClient.ts +++ b/sdk/storage/storage-file-share/src/ShareFileClient.ts @@ -1102,18 +1102,18 @@ export class ShareFileClient extends StorageClient { * const downloadFileResponse = await fileClient.download(); * console.log( * "Downloaded file content:", - * await streamToString(downloadFileResponse.readableStreamBody)} + * (await streamToBuffer(downloadFileResponse.readableStreamBody)).toString()} * ); * * // A helper method used to read a Node.js readable stream into string - * async function streamToString(readableStream) { + * async function streamToBuffer(readableStream) { * return new Promise((resolve, reject) => { * const chunks = []; * readableStream.on("data", (data) => { - * chunks.push(data.toString()); + * chunks.push(data instanceof Buffer ? data : Buffer.from(data)); * }); * readableStream.on("end", () => { - * resolve(chunks.join("")); + * resolve(Buffer.concat(chunks)); * }); * readableStream.on("error", reject); * });