From 1d7e0e46fa64e1ff4aaf787a542f9378b1a75049 Mon Sep 17 00:00:00 2001 From: Harsha Nalluru Date: Thu, 21 Jan 2021 01:05:25 -0800 Subject: [PATCH] [Perf Tests] Storage blob perf tests - track 1 and track 2 (#12737) ### Depends on https://github.com/Azure/azure-sdk-for-js/pull/12662 ### Changes in the PR - Track 2 tests are part of the test folder, which would be compiled along with the regular tests and would require changes if the API is updated - Track 1 tests - a separate npm project, takes dependence on the perf package, doesn't get compiled along with the regular tests ### To run track 2 perf tests 1. Build the storage-blob package `rush build -t storage-blob`. 2. Navigate to `storage-blob` folder `cd sdk\storage\storage-blob\`. 3. Create a storage account and populate the .env file at `storage\storage-blob` folder with `ACCOUNT_NAME` and `ACCOUNT_KEY` variables. 4. Run the tests as follows - download - `npm run perfstress-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2` - upload - `npm run perfstress-test:node -- StorageBlobUploadTest --warmup 2 --duration 7 --iterations 2 --parallel 2` - upload file - `npm run perfstress-test:node -- StorageBlobUploadFileTest --warmup 2 --duration 7 --iterations 2 --parallel 2` - list blobs - `npm run perfstress-test:node -- StorageBlobListTest --warmup 2 --duration 7 --iterations 2 --parallel 2` - download using sas with storage-blob - `npm run perfstress-test:node -- StorageBlobDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2` - download using sas with node-fetch - `npm run perfstress-test:node -- NodeFetchDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2` - download using sas with core-http - `npm run perfstress-test:node -- CoreHTTPDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2` - download using sas with core-https - `npm run perfstress-test:node -- CoreHTTPSDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2` ### To run track 1 perf tests 1. Navigate to `test-utils\perfstress` folder `cd sdk\test-utils\perfstress\` 2. Build the package `rush update && rush build -t test-utils-perfstress` 3. Pack the perf package `rushx pack` 4. Navigate to `storage-blob\perfstress\track-1` folder `cd sdk\storage\storage-blob\perfstress\track-1`. 5. Install the perf package `npm i ..\..\..\..\..\test-utils\perfstress\azure-test-utils-perfstress-1.0.0.tgz` 6. Run `npm install` to get `storage-blob V10`. 7. Create a storage account and populate the .env file with `ACCOUNT_NAME` and `ACCOUNT_KEY` variables. 8. Run the tests as follows - download - `npm run perfstress-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2` - upload - `npm run perfstress-test:node -- StorageBlobUploadTest --warmup 2 --duration 7 --iterations 2 --parallel 2` - list blobs - `npm run perfstress-test:node -- StorageBlobListTest --warmup 2 --duration 7 --iterations 2 --parallel 2` --- eng/.docsettings.yml | 2 + sdk/storage/storage-blob/package.json | 5 ++ .../test/perfstress/track-1/README.md | 16 ++++ .../test/perfstress/track-1/download.spec.ts | 48 ++++++++++++ .../test/perfstress/track-1/index.spec.ts | 15 ++++ .../test/perfstress/track-1/listBlobs.spec.ts | 45 +++++++++++ .../test/perfstress/track-1/package.json | 18 +++++ .../perfstress/track-1/storageTest.spec.ts | 60 +++++++++++++++ .../test/perfstress/track-1/tsconfig.json | 5 ++ .../test/perfstress/track-1/upload.spec.ts | 41 ++++++++++ .../test/perfstress/track-2/README.md | 23 ++++++ .../test/perfstress/track-2/core-http.spec.ts | 35 +++++++++ .../perfstress/track-2/core-https.spec.ts | 25 ++++++ .../perfstress/track-2/dowloadWithSAS.spec.ts | 77 +++++++++++++++++++ .../test/perfstress/track-2/download.spec.ts | 48 ++++++++++++ .../test/perfstress/track-2/index.spec.ts | 29 +++++++ .../test/perfstress/track-2/listBlobs.spec.ts | 39 ++++++++++ .../perfstress/track-2/node-fetch.spec.ts | 19 +++++ .../perfstress/track-2/storageTest.spec.ts | 37 +++++++++ .../test/perfstress/track-2/upload.spec.ts | 38 +++++++++ .../perfstress/track-2/uploadFromFile.spec.ts | 37 +++++++++ sdk/storage/storage-blob/tsconfig.json | 3 +- 22 files changed, 664 insertions(+), 1 deletion(-) create mode 100644 sdk/storage/storage-blob/test/perfstress/track-1/README.md create mode 100644 sdk/storage/storage-blob/test/perfstress/track-1/download.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-1/index.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-1/listBlobs.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-1/package.json create mode 100644 sdk/storage/storage-blob/test/perfstress/track-1/storageTest.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-1/tsconfig.json create mode 100644 sdk/storage/storage-blob/test/perfstress/track-1/upload.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/README.md create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/core-http.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/core-https.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/dowloadWithSAS.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/download.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/index.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/listBlobs.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/node-fetch.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/storageTest.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/upload.spec.ts create mode 100644 sdk/storage/storage-blob/test/perfstress/track-2/uploadFromFile.spec.ts diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml index a38c550febd0..6968de7c2fd8 100644 --- a/eng/.docsettings.yml +++ b/eng/.docsettings.yml @@ -22,6 +22,8 @@ omitted_paths: - sdk/schemaregistry/README.md - sdk/storage/*/test/README.md - sdk/storage/storage-internal-avro/* + - sdk/storage/*/test/perfstress/track-1/* + - sdk/storage/*/test/perfstress/track-2/* - sdk/textanalytics/*/test/README.md - sdk/**/samples/* - sdk/**/samples-*/* diff --git a/sdk/storage/storage-blob/package.json b/sdk/storage/storage-blob/package.json index 106ebea4d2b6..998dab0e2d89 100644 --- a/sdk/storage/storage-blob/package.json +++ b/sdk/storage/storage-blob/package.json @@ -53,6 +53,7 @@ "lint:fix": "eslint package.json api-extractor.json src test --ext .ts --fix", "lint": "eslint package.json api-extractor.json src test --ext .ts -f html -o storage-blob-lintReport.html || exit 0", "pack": "npm pack 2>&1", + "perfstress-test:node": "cross-env TS_NODE_COMPILER_OPTIONS=\"{\\\"module\\\": \\\"commonjs\\\"}\" ts-node test/perfstress/track-2/index.spec.ts", "prebuild": "npm run clean", "test:browser": "npm run clean && npm run build:test && npm run unit-test:browser", "test:node": "npm run clean && npm run build:test && npm run unit-test:node", @@ -128,16 +129,19 @@ "tslib": "^2.0.0" }, "devDependencies": { + "@azure/core-https": "1.0.0-beta.1", "@azure/dev-tool": "^1.0.0", "@azure/eslint-plugin-azure-sdk": "^3.0.0", "@azure/identity": "^1.1.0", "@azure/test-utils-recorder": "^1.0.0", + "@azure/test-utils-perfstress": "^1.0.0", "@microsoft/api-extractor": "7.7.11", "@rollup/plugin-multi-entry": "^3.0.0", "@rollup/plugin-replace": "^2.2.0", "@rollup/plugin-json": "^4.0.0", "@types/mocha": "^7.0.2", "@types/node": "^8.0.0", + "@types/node-fetch": "^2.5.0", "assert": "^1.4.1", "cross-env": "^7.0.2", "dotenv": "^8.2.0", @@ -161,6 +165,7 @@ "karma-remap-istanbul": "^0.6.0", "mocha": "^7.1.1", "mocha-junit-reporter": "^1.18.0", + "node-fetch": "^2.6.0", "nyc": "^14.0.0", "prettier": "^1.16.4", "puppeteer": "^3.3.0", diff --git a/sdk/storage/storage-blob/test/perfstress/track-1/README.md b/sdk/storage/storage-blob/test/perfstress/track-1/README.md new file mode 100644 index 000000000000..57989bc19f05 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-1/README.md @@ -0,0 +1,16 @@ +### Guide + +1. Navigate to `test-utils\perfstress` folder `cd sdk\test-utils\perfstress\` +2. Build the package `rush update && rush build -t test-utils-perfstress` +3. Pack the perf package `rushx pack` +4. Navigate to `storage-blob\perfstress\track-1` folder `cd sdk\storage\storage-blob\perfstress\track-1`. +5. Install the perf package `npm i ..\..\..\..\..\test-utils\perfstress\azure-test-utils-perfstress-1.0.0.tgz` +6. Run `npm install` to get `storage-blob V10`. +7. Create a storage account and populate the .env file with `ACCOUNT_NAME` and `ACCOUNT_KEY` variables. +8. Run the tests as follows + - download + - `npm run perfstress-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2` + - upload + - `npm run perfstress-test:node -- StorageBlobUploadTest --warmup 2 --duration 7 --iterations 2 --parallel 2` + - list blobs + - `npm run perfstress-test:node -- StorageBlobListTest --warmup 2 --duration 7 --iterations 2 --parallel 2` diff --git a/sdk/storage/storage-blob/test/perfstress/track-1/download.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-1/download.spec.ts new file mode 100644 index 000000000000..826aab25b1a0 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-1/download.spec.ts @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { generateUuid } from "@azure/core-http"; +import { Aborter, BlobURL, BlockBlobURL } from "@azure/storage-blob"; +import { PerfStressOptionDictionary, drainStream } from "@azure/test-utils-perfstress"; +import { StorageBlobTest } from "./storageTest.spec"; + +interface StorageBlobDownloadTestOptions { + size: number; +} + +export class StorageBlobDownloadTest extends StorageBlobTest { + public options: PerfStressOptionDictionary = { + size: { + required: true, + description: "Size in bytes", + shortName: "sz", + longName: "size", + defaultValue: 10240 + } + }; + + static blobName = generateUuid(); + blockBlobClient: BlockBlobURL; + + constructor() { + super(); + this.blockBlobClient = BlockBlobURL.fromBlobURL( + BlobURL.fromContainerURL(this.containerClient, StorageBlobDownloadTest.blobName) + ); + } + + public async globalSetup() { + await super.globalSetup(); + // Create a blob + await this.blockBlobClient.upload( + Aborter.none, + Buffer.alloc(this.parsedOptions.size.value!), + this.parsedOptions.size.value! + ); + } + + async runAsync(): Promise { + const downloadResponse = await this.blockBlobClient.download(Aborter.none, 0); + await drainStream(downloadResponse.readableStreamBody!); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-1/index.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-1/index.spec.ts new file mode 100644 index 000000000000..9f84b9c18373 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-1/index.spec.ts @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { PerfStressProgram, selectPerfStressTest } from "@azure/test-utils-perfstress"; +import { StorageBlobDownloadTest } from "./download.spec"; +import { StorageBlobUploadTest } from "./upload.spec"; +import { StorageBlobListTest } from "./listBlobs.spec"; + +console.log("=== Starting the perfStress test ==="); + +const perfStressProgram = new PerfStressProgram( + selectPerfStressTest([StorageBlobDownloadTest, StorageBlobUploadTest, StorageBlobListTest]) +); + +perfStressProgram.run(); diff --git a/sdk/storage/storage-blob/test/perfstress/track-1/listBlobs.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-1/listBlobs.spec.ts new file mode 100644 index 000000000000..6ea5867ca65d --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-1/listBlobs.spec.ts @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +import { generateUuid } from "@azure/core-http"; +import { Aborter, BlockBlobURL } from "@azure/storage-blob"; +import { executeParallel, PerfStressOptionDictionary } from "@azure/test-utils-perfstress"; +import { StorageBlobTest } from "./storageTest.spec"; + +interface StorageBlobListTestOptions { + count: number; +} + +export class StorageBlobListTest extends StorageBlobTest { + public options: PerfStressOptionDictionary = { + count: { + required: true, + description: "Number of blobs to be listed", + longName: "count", + defaultValue: 10 + } + }; + + public async globalSetup() { + await super.globalSetup(); + await executeParallel( + async (count: number, parallelIndex: number) => { + const blockBlobClient = BlockBlobURL.fromContainerURL(this.containerClient, generateUuid()); + blockBlobClient.upload(Aborter.none, Buffer.alloc(0), 0); + console.log(`[` + parallelIndex + `] ` + count); + }, + this.parsedOptions.count.value!, + 32 + ); + } + + async runAsync(): Promise { + // List blobs + let marker = undefined; + do { + const segmentResponse = await this.containerClient.listBlobFlatSegment(Aborter.none, marker); + for (const _ of segmentResponse.segment.blobItems) { + } + marker = segmentResponse.nextMarker; + } while (marker); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-1/package.json b/sdk/storage/storage-blob/test/perfstress/track-1/package.json new file mode 100644 index 000000000000..860314dbcd95 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-1/package.json @@ -0,0 +1,18 @@ +{ + "name": "track-1", + "version": "1.0.0", + "description": "", + "main": "index.js", + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@azure/storage-blob": "^10.5.0", + "@azure/test-utils-perfstress": "file:../../../../../test-utils/perfstress/azure-test-utils-perfstress-1.0.0.tgz", + "@types/uuid": "^8.3.0", + "uuid": "^8.3.2" + }, + "scripts": { + "perfstress-test:node": "ts-node index.spec.ts" + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-1/storageTest.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-1/storageTest.spec.ts new file mode 100644 index 000000000000..2e3d32fc949c --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-1/storageTest.spec.ts @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { generateUuid } from "@azure/core-http"; +import { + Aborter, + ContainerURL, + ServiceURL, + SharedKeyCredential, + StorageURL +} from "@azure/storage-blob"; +import { PerfStressTest, getEnvVar } from "@azure/test-utils-perfstress"; + +// Expects the .env file at the same level as the "test" folder +import * as dotenv from "dotenv"; +dotenv.config({ path: "../../../.env" }); + +export abstract class StorageBlobTest extends PerfStressTest { + blobServiceClient: ServiceURL; + containerClient: ContainerURL; + static containerName = generateUuid(); + + constructor() { + super(); + const connectionString = getEnvVar("STORAGE_CONNECTION_STRING"); + const accountName = getValueInConnectionString(connectionString, "AccountName"); + const accountKey = getValueInConnectionString(connectionString, "AccountKey"); + + const sharedKeyCredential = new SharedKeyCredential(accountName, accountKey); + this.blobServiceClient = new ServiceURL( + `https://${accountName}.blob.core.windows.net`, + StorageURL.newPipeline(sharedKeyCredential) + ); + this.containerClient = ContainerURL.fromServiceURL( + this.blobServiceClient, + StorageBlobTest.containerName + ); + } + + public async globalSetup() { + await this.containerClient.create(Aborter.none); + } + + public async globalCleanup() { + await this.containerClient.delete(Aborter.none); + } +} + +export function getValueInConnectionString( + connectionString: string, + argument: "AccountName" | "AccountKey" +) { + const elements = connectionString.split(";"); + for (const element of elements) { + if (element.trim().startsWith(argument)) { + return element.trim().match(argument + "=(.*)")![1]; + } + } + return ""; +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-1/tsconfig.json b/sdk/storage/storage-blob/test/perfstress/track-1/tsconfig.json new file mode 100644 index 000000000000..391488ab17f4 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-1/tsconfig.json @@ -0,0 +1,5 @@ +{ + "compilerOptions": { + "module": "commonjs" + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-1/upload.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-1/upload.spec.ts new file mode 100644 index 000000000000..35866e97540c --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-1/upload.spec.ts @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { generateUuid } from "@azure/core-http"; +import { Aborter, BlockBlobURL } from "@azure/storage-blob"; +import { PerfStressOptionDictionary } from "@azure/test-utils-perfstress"; + +// Expects the .env file at the same level as the "test" folder +import * as dotenv from "dotenv"; +import { StorageBlobTest } from "./storageTest.spec"; +dotenv.config({ path: "../../../.env" }); + +interface StorageBlobUploadTestOptions { + size: number; +} + +export class StorageBlobUploadTest extends StorageBlobTest { + blobName: string; + blockBlobClient: BlockBlobURL; + buffer: Buffer; + public options: PerfStressOptionDictionary = { + size: { + required: true, + description: "Size in bytes", + shortName: "sz", + longName: "size", + defaultValue: 10240 + } + }; + + constructor() { + super(); + this.blobName = generateUuid(); + this.blockBlobClient = BlockBlobURL.fromContainerURL(this.containerClient, this.blobName); + this.buffer = Buffer.alloc(this.parsedOptions.size.value!); + } + + async runAsync(): Promise { + await this.blockBlobClient.upload(Aborter.none, this.buffer, this.parsedOptions.size.value!); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/README.md b/sdk/storage/storage-blob/test/perfstress/track-2/README.md new file mode 100644 index 000000000000..5658b122ccb2 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/README.md @@ -0,0 +1,23 @@ +### Guide + +1. Build the storage-blob package `rush build -t storage-blob`. +2. Navigate to `storage-blob` folder `cd sdk\storage\storage-blob\`. +3. Create a storage account and populate the .env file at `storage\storage-blob` folder with `STORAGE_CONNECTION_STRING` variables. +4. Run the tests as follows + + - download + - `npm run perfstress-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2` + - upload + - `npm run perfstress-test:node -- StorageBlobUploadTest --warmup 2 --duration 7 --iterations 2 --parallel 2` + - upload file + - `npm run perfstress-test:node -- StorageBlobUploadFileTest --warmup 2 --duration 7 --iterations 2 --parallel 2` + - list blobs + - `npm run perfstress-test:node -- StorageBlobListTest --warmup 2 --duration 7 --iterations 2 --parallel 2` + - download using sas with storage-blob + - `npm run perfstress-test:node -- StorageBlobDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2` + - download using sas with node-fetch + - `npm run perfstress-test:node -- NodeFetchDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2` + - download using sas with core-http + - `npm run perfstress-test:node -- CoreHTTPDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2` + - download using sas with core-https + - `npm run perfstress-test:node -- CoreHTTPSDownloadWithSASTest --warmup 2 --duration 7 --iterations 2 --parallel 2` diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/core-http.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/core-http.spec.ts new file mode 100644 index 000000000000..b46ecaf41f3c --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/core-http.spec.ts @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { StorageBlobDownloadWithSASTest } from "./dowloadWithSAS.spec"; +import { ServiceClient, WebResource } from "@azure/core-http"; +import { drainStream } from "@azure/test-utils-perfstress"; + +export class CoreHTTPDownloadWithSASTest extends StorageBlobDownloadWithSASTest { + client: ServiceClient; + webResource: WebResource; + constructor() { + super(); + this.client = new ServiceClient(); + this.webResource = new WebResource( + this.sasUrl, + undefined, + undefined, + undefined, + undefined, + true, // streamResponseBody + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + true // keepAlive + ); + } + + async runAsync(): Promise { + const response = await this.client.sendRequest(this.webResource); + await drainStream(response.readableStreamBody!); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/core-https.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/core-https.spec.ts new file mode 100644 index 000000000000..5714b0b056d1 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/core-https.spec.ts @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { StorageBlobDownloadWithSASTest } from "./dowloadWithSAS.spec"; +import { DefaultHttpsClient, createPipelineRequest, PipelineRequest } from "@azure/core-https"; +import { drainStream } from "@azure/test-utils-perfstress"; + +export class CoreHTTPSDownloadWithSASTest extends StorageBlobDownloadWithSASTest { + client: DefaultHttpsClient; + request: PipelineRequest; + constructor() { + super(); + this.client = new DefaultHttpsClient(); + this.request = createPipelineRequest({ + url: this.sasUrl, + streamResponseBody: true, + keepAlive: true + }); + } + + async runAsync(): Promise { + const response = await this.client.sendRequest(this.request); + await drainStream(response.readableStreamBody!); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/dowloadWithSAS.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/dowloadWithSAS.spec.ts new file mode 100644 index 000000000000..5bb9c3ff0f97 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/dowloadWithSAS.spec.ts @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { PerfStressOptionDictionary, getEnvVar, drainStream } from "@azure/test-utils-perfstress"; +import { StorageBlobTest } from "./storageTest.spec"; +import { + BlockBlobClient, + generateBlobSASQueryParameters, + BlobSASPermissions, + BlobClient +} from "../../../src"; +import { getValueInConnString } from "../../../src/utils/utils.common"; +import { generateUuid } from "@azure/core-http"; + +interface StorageBlobDownloadTestOptions { + size: number; +} + +export class StorageBlobDownloadWithSASTest extends StorageBlobTest< + StorageBlobDownloadTestOptions +> { + public options: PerfStressOptionDictionary = { + size: { + required: true, + description: "Size in bytes", + shortName: "sz", + longName: "size", + defaultValue: 10240 + } + }; + + static blobName = generateUuid(); + blockBlobClient: BlockBlobClient; + blobClientFromSAS: BlobClient; + sasUrl: string; + + constructor() { + super(); + this.blockBlobClient = this.containerClient.getBlockBlobClient( + StorageBlobDownloadWithSASTest.blobName + ); + const sasParams = generateBlobSASQueryParameters( + { + // Expires in a day + expiresOn: new Date(new Date().getTime() + 86400000), + containerName: StorageBlobDownloadWithSASTest.containerName, + blobName: StorageBlobDownloadWithSASTest.blobName, + permissions: BlobSASPermissions.parse("r") + }, + this.sharedKeyCredential + ).toString(); + + this.sasUrl = `https://${getValueInConnString( + getEnvVar("STORAGE_CONNECTION_STRING"), + "AccountName" + )}.blob.core.windows.net/${StorageBlobDownloadWithSASTest.containerName}/${ + StorageBlobDownloadWithSASTest.blobName + }?${sasParams}`; + + this.blobClientFromSAS = new BlobClient(this.sasUrl); + } + + public async globalSetup() { + await super.globalSetup(); + + // Create a blob + await this.blockBlobClient.upload( + Buffer.alloc(this.parsedOptions.size.value!), + this.parsedOptions.size.value! + ); + } + + async runAsync(): Promise { + const downloadResponse = await this.blobClientFromSAS.download(); + await drainStream(downloadResponse.readableStreamBody!); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/download.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/download.spec.ts new file mode 100644 index 000000000000..7acb9b3030da --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/download.spec.ts @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { drainStream, PerfStressOptionDictionary } from "@azure/test-utils-perfstress"; +import { StorageBlobTest } from "./storageTest.spec"; +import { BlockBlobClient } from "../../../src"; +import { generateUuid } from "@azure/core-http"; + +interface StorageBlobDownloadTestOptions { + size: number; +} + +export class StorageBlobDownloadTest extends StorageBlobTest { + public options: PerfStressOptionDictionary = { + size: { + required: true, + description: "Size in bytes", + shortName: "sz", + longName: "size", + defaultValue: 10240 + } + }; + + static blobName = generateUuid(); + blockBlobClient: BlockBlobClient; + + constructor() { + super(); + this.blockBlobClient = this.containerClient.getBlockBlobClient( + StorageBlobDownloadTest.blobName + ); + } + + public async globalSetup() { + await super.globalSetup(); + + // Create a blob + await this.blockBlobClient.upload( + Buffer.alloc(this.parsedOptions.size.value!), + this.parsedOptions.size.value! + ); + } + + async runAsync(): Promise { + const downloadResponse = await this.blockBlobClient.download(); + await drainStream(downloadResponse.readableStreamBody!); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/index.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/index.spec.ts new file mode 100644 index 000000000000..d447f2f9c24f --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/index.spec.ts @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { PerfStressProgram, selectPerfStressTest } from "@azure/test-utils-perfstress"; +import { StorageBlobDownloadTest } from "./download.spec"; +import { StorageBlobUploadTest } from "./upload.spec"; +import { StorageBlobUploadFileTest } from "./uploadFromFile.spec"; +import { StorageBlobListTest } from "./listBlobs.spec"; +import { StorageBlobDownloadWithSASTest } from "./dowloadWithSAS.spec"; +import { CoreHTTPDownloadWithSASTest } from "./core-http.spec"; +import { NodeFetchDownloadWithSASTest } from "./node-fetch.spec"; +import { CoreHTTPSDownloadWithSASTest } from "./core-https.spec"; + +console.log("=== Starting the perfStress test ==="); + +const perfStressProgram = new PerfStressProgram( + selectPerfStressTest([ + StorageBlobDownloadTest, + StorageBlobUploadTest, + StorageBlobUploadFileTest, + StorageBlobListTest, + StorageBlobDownloadWithSASTest, + CoreHTTPDownloadWithSASTest, + CoreHTTPSDownloadWithSASTest, + NodeFetchDownloadWithSASTest + ]) +); + +perfStressProgram.run(); diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/listBlobs.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/listBlobs.spec.ts new file mode 100644 index 000000000000..089dae59fd13 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/listBlobs.spec.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { generateUuid } from "@azure/core-http"; +import { PerfStressOptionDictionary, executeParallel } from "@azure/test-utils-perfstress"; +import { StorageBlobTest } from "./storageTest.spec"; +interface StorageBlobListTestOptions { + count: number; +} + +export class StorageBlobListTest extends StorageBlobTest { + public options: PerfStressOptionDictionary = { + count: { + required: true, + description: "Number of blobs to be listed", + longName: "count", + defaultValue: 10 + } + }; + + public async globalSetup() { + await super.globalSetup(); + await executeParallel( + async (count: number, parallelIndex: number) => { + await this.containerClient.uploadBlockBlob(generateUuid(), Buffer.alloc(0), 0); + console.log(`[` + parallelIndex + `] ` + count); + }, + this.parsedOptions.count.value!, + 32 + ); + } + + async runAsync(): Promise { + for await (const segmentResponse of this.containerClient.listBlobsFlat().byPage()) { + for (const _ of segmentResponse.segment.blobItems) { + } + } + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/node-fetch.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/node-fetch.spec.ts new file mode 100644 index 000000000000..b44859c657ab --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/node-fetch.spec.ts @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { StorageBlobDownloadWithSASTest } from "./dowloadWithSAS.spec"; +import node_fetch from "node-fetch"; +import https from "https"; +import { drainStream } from "@azure/test-utils-perfstress"; +export class NodeFetchDownloadWithSASTest extends StorageBlobDownloadWithSASTest { + agent: https.Agent; + constructor() { + super(); + this.agent = new https.Agent({ keepAlive: true }); + } + + async runAsync(): Promise { + const response = await node_fetch(this.sasUrl, { agent: this.agent }); + await drainStream(response.body); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/storageTest.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/storageTest.spec.ts new file mode 100644 index 000000000000..26797b5b0dcf --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/storageTest.spec.ts @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { generateUuid } from "@azure/core-http"; +import { PerfStressTest, getEnvVar } from "@azure/test-utils-perfstress"; +import { BlobServiceClient, ContainerClient, StorageSharedKeyCredential } from "../../../src"; +import { getValueInConnString } from "../../../src/utils/utils.common"; + +// Expects the .env file at the same level as the "test" folder +import * as dotenv from "dotenv"; +dotenv.config(); + +export abstract class StorageBlobTest extends PerfStressTest { + blobServiceClient: BlobServiceClient; + containerClient: ContainerClient; + sharedKeyCredential: StorageSharedKeyCredential; + static containerName = generateUuid(); + + constructor() { + super(); + const connectionString = getEnvVar("STORAGE_CONNECTION_STRING"); + this.sharedKeyCredential = new StorageSharedKeyCredential( + getValueInConnString(connectionString, "AccountName"), + getValueInConnString(connectionString, "AccountKey") + ); + this.blobServiceClient = BlobServiceClient.fromConnectionString(connectionString); + this.containerClient = this.blobServiceClient.getContainerClient(StorageBlobTest.containerName); + } + + public async globalSetup() { + await this.containerClient.create(); + } + + public async globalCleanup() { + await this.containerClient.delete(); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/upload.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/upload.spec.ts new file mode 100644 index 000000000000..3f057f9d618d --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/upload.spec.ts @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { generateUuid } from "@azure/core-http"; +import { PerfStressOptionDictionary } from "@azure/test-utils-perfstress"; +import { StorageBlobTest } from "./storageTest.spec"; + +interface StorageBlobUploadTestOptions { + size: number; +} + +export class StorageBlobUploadTest extends StorageBlobTest { + blobName: string; + buffer: Buffer; + public options: PerfStressOptionDictionary = { + size: { + required: true, + description: "Size in bytes", + shortName: "sz", + longName: "size", + defaultValue: 10240 + } + }; + + constructor() { + super(); + this.blobName = generateUuid(); + this.buffer = Buffer.alloc(this.parsedOptions.size.value!); + } + + async runAsync(): Promise { + await this.containerClient.uploadBlockBlob( + this.blobName, + this.buffer, + this.parsedOptions.size.value! + ); + } +} diff --git a/sdk/storage/storage-blob/test/perfstress/track-2/uploadFromFile.spec.ts b/sdk/storage/storage-blob/test/perfstress/track-2/uploadFromFile.spec.ts new file mode 100644 index 000000000000..18f69ab1d690 --- /dev/null +++ b/sdk/storage/storage-blob/test/perfstress/track-2/uploadFromFile.spec.ts @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +import fs from "fs"; +import util from "util"; +import { BlockBlobClient } from "../../../src"; +const writeFile = util.promisify(fs.writeFile); +const fileExists = util.promisify(fs.exists); +const mkdir = util.promisify(fs.mkdir); +const deleteFile = util.promisify(fs.unlink); + +import { StorageBlobUploadTest } from "./upload.spec"; + +const dirName = "temp"; +const fileName = `${dirName}/upload-from-test-temp-file.txt`; + +export class StorageBlobUploadFileTest extends StorageBlobUploadTest { + blockBlobClient: BlockBlobClient; + constructor() { + super(); + this.blockBlobClient = this.containerClient.getBlockBlobClient(this.blobName); + } + + public async globalSetup() { + await super.globalSetup(); + if (!(await fileExists(dirName))) await mkdir(dirName); + await writeFile(fileName, Buffer.alloc(this.parsedOptions.size.value!)); + } + + public async globalCleanup() { + await deleteFile(fileName); + await super.globalCleanup(); + } + + async runAsync(): Promise { + await this.blockBlobClient.uploadFile(fileName); + } +} diff --git a/sdk/storage/storage-blob/tsconfig.json b/sdk/storage/storage-blob/tsconfig.json index 497852003721..87bfa51f261a 100644 --- a/sdk/storage/storage-blob/tsconfig.json +++ b/sdk/storage/storage-blob/tsconfig.json @@ -12,7 +12,8 @@ "./typings/**/*.d.ts", "../storage-internal-avro/node_modules", "../storage-common/node_modules", - "./samples/**" + "./samples/**", + "test/perfstress/track-1/" ], "include": [ "./src/**/*.ts",