From 00e240185f5287ca58b00777962b1ccb660e7373 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 11 Jun 2020 17:37:38 +0800 Subject: [PATCH] fix minor bug in avro --- common/config/rush/pnpm-lock.yaml | 71 ++++++++++++++++++- .../.vscode/settings.json | 2 +- .../storage-blob-changefeed/package.json | 18 ++--- .../review/storage-blob-changefeed.api.md | 10 +-- .../samples/typscript/basic.ts | 26 +++---- .../samples/typscript/resume.ts | 42 +++++++++++ .../storage-internal-avro/package.json | 2 +- .../src/AvroReadableFromStream.ts | 10 +-- 8 files changed, 136 insertions(+), 45 deletions(-) create mode 100644 sdk/storage/storage-blob-changefeed/samples/typscript/resume.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 98af63ca439d..264f2107d6d1 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -25,6 +25,7 @@ dependencies: '@rush-temp/search-documents': 'file:projects/search-documents.tgz' '@rush-temp/service-bus': 'file:projects/service-bus.tgz' '@rush-temp/storage-blob': 'file:projects/storage-blob.tgz' + '@rush-temp/storage-blob-changefeed': 'file:projects/storage-blob-changefeed.tgz' '@rush-temp/storage-file-datalake': 'file:projects/storage-file-datalake.tgz' '@rush-temp/storage-file-share': 'file:projects/storage-file-share.tgz' '@rush-temp/storage-internal-avro': 'file:projects/storage-internal-avro.tgz' @@ -6741,14 +6742,14 @@ packages: node: '>=0.10.0' resolution: integrity: sha1-WIeWa7WCpFA6QetST301ARgVphM= - /ts-loader/6.2.2_typescript@3.8.3: + /ts-loader/6.2.2_typescript@3.9.5: dependencies: chalk: 2.4.2 enhanced-resolve: 4.1.1 loader-utils: 1.4.0 micromatch: 4.0.2 semver: 6.3.0 - typescript: 3.8.3 + typescript: 3.9.5 dev: false engines: node: '>=8.6' @@ -7838,7 +7839,7 @@ packages: shx: 0.3.2 sinon: 9.0.2 tough-cookie: 4.0.0 - ts-loader: 6.2.2_typescript@3.8.3 + ts-loader: 6.2.2_typescript@3.9.5 ts-node: 8.8.2_typescript@3.9.5 tslib: 2.0.0 tunnel: 0.0.6 @@ -8773,6 +8774,69 @@ packages: integrity: sha512-wvwki9VUk1OBa5vOKDNUtNv48hISwA4O1yQGklLVg3K/WdRmz/a7r4stTezbEKbjTk0mOYSU2hX3HgUSssIisA== tarball: 'file:projects/service-bus.tgz' version: 0.0.0 + 'file:projects/storage-blob-changefeed.tgz': + dependencies: + '@azure/core-tracing': 1.0.0-preview.8 + '@microsoft/api-extractor': 7.7.11 + '@opentelemetry/api': 0.6.1 + '@rollup/plugin-commonjs': 11.0.2_rollup@1.32.1 + '@rollup/plugin-multi-entry': 3.0.0_rollup@1.32.1 + '@rollup/plugin-node-resolve': 8.0.1_rollup@1.32.1 + '@rollup/plugin-replace': 2.3.1_rollup@1.32.1 + '@types/mocha': 7.0.2 + '@types/node': 8.10.59 + '@types/sinon': 9.0.4 + '@typescript-eslint/eslint-plugin': 2.27.0_2f26aa176a29afc2bd7e8bd79afb588b + '@typescript-eslint/parser': 2.27.0_eslint@6.8.0+typescript@3.9.5 + assert: 1.5.0 + cross-env: 7.0.2 + dotenv: 8.2.0 + downlevel-dts: 0.4.0 + es6-promise: 4.2.8 + eslint: 6.8.0 + eslint-config-prettier: 6.10.1_eslint@6.8.0 + eslint-plugin-no-null: 1.0.2_eslint@6.8.0 + eslint-plugin-no-only-tests: 2.4.0 + eslint-plugin-promise: 4.2.1 + esm: 3.2.25 + events: 3.1.0 + inherits: 2.0.4 + karma: 4.4.1 + karma-chrome-launcher: 3.1.0 + karma-coverage: 2.0.1 + karma-edge-launcher: 0.4.2_karma@4.4.1 + karma-env-preprocessor: 0.1.1 + karma-firefox-launcher: 1.3.0 + karma-ie-launcher: 1.0.0_karma@4.4.1 + karma-json-preprocessor: 0.3.3_karma@4.4.1 + karma-json-to-file-reporter: 1.0.1 + karma-junit-reporter: 2.0.1_karma@4.4.1 + karma-mocha: 1.3.0 + karma-mocha-reporter: 2.2.5_karma@4.4.1 + karma-remap-istanbul: 0.6.0_karma@4.4.1 + mocha: 7.1.1 + mocha-junit-reporter: 1.23.3_mocha@7.1.1 + nyc: 14.1.1 + prettier: 1.19.1 + puppeteer: 3.3.0 + rimraf: 3.0.2 + rollup: 1.32.1 + rollup-plugin-shim: 1.0.0 + rollup-plugin-sourcemaps: 0.4.2_rollup@1.32.1 + rollup-plugin-terser: 5.3.0_rollup@1.32.1 + rollup-plugin-visualizer: 4.0.4_rollup@1.32.1 + sinon: 9.0.2 + source-map-support: 0.5.16 + ts-node: 8.8.2_typescript@3.9.5 + tslib: 2.0.0 + typescript: 3.9.5 + util: 0.12.2 + dev: false + name: '@rush-temp/storage-blob-changefeed' + resolution: + integrity: sha512-DoG0fYwiYsjufvHaRJKJA6luLgv102q8ICSwVEyRnP2lE1eRfqLz2gPmVOABjRLIM4luom8WMNLV5OPo/FNXZQ== + tarball: 'file:projects/storage-blob-changefeed.tgz' + version: 0.0.0 'file:projects/storage-blob.tgz': dependencies: '@azure/core-tracing': 1.0.0-preview.8 @@ -9275,6 +9339,7 @@ specifiers: '@rush-temp/search-documents': 'file:./projects/search-documents.tgz' '@rush-temp/service-bus': 'file:./projects/service-bus.tgz' '@rush-temp/storage-blob': 'file:./projects/storage-blob.tgz' + '@rush-temp/storage-blob-changefeed': 'file:./projects/storage-blob-changefeed.tgz' '@rush-temp/storage-file-datalake': 'file:./projects/storage-file-datalake.tgz' '@rush-temp/storage-file-share': 'file:./projects/storage-file-share.tgz' '@rush-temp/storage-internal-avro': 'file:./projects/storage-internal-avro.tgz' diff --git a/sdk/storage/storage-blob-changefeed/.vscode/settings.json b/sdk/storage/storage-blob-changefeed/.vscode/settings.json index 7ceb5ace3e9d..71c65f9f433a 100644 --- a/sdk/storage/storage-blob-changefeed/.vscode/settings.json +++ b/sdk/storage/storage-blob-changefeed/.vscode/settings.json @@ -10,7 +10,7 @@ "editor.detectIndentation": false }, "[json]": { - "editor.formatOnSave": true, + "editor.formatOnSave": false, "editor.tabSize": 2, "editor.detectIndentation": false }, diff --git a/sdk/storage/storage-blob-changefeed/package.json b/sdk/storage/storage-blob-changefeed/package.json index 2b6172b9d42f..ca6e0a4ed34a 100644 --- a/sdk/storage/storage-blob-changefeed/package.json +++ b/sdk/storage/storage-blob-changefeed/package.json @@ -103,7 +103,7 @@ "@azure/logger": "^1.0.0", "@opentelemetry/api": "^0.6.1", "events": "^3.0.0", - "tslib": "^1.10.0" + "tslib": "^2.0.0" }, "devDependencies": { "@azure/identity": "^1.1.0-preview", @@ -113,11 +113,11 @@ "@rollup/plugin-replace": "^2.2.0", "@types/mocha": "^7.0.2", "@types/node": "^8.0.0", - "@types/sinon": "^7.0.13", + "@types/sinon": "^9.0.4", "@typescript-eslint/eslint-plugin": "^2.0.0", "@typescript-eslint/parser": "^2.0.0", "assert": "^1.4.1", - "cross-env": "^6.0.3", + "cross-env": "^7.0.2", "dotenv": "^8.2.0", "downlevel-dts": "~0.4.0", "es6-promise": "^4.2.5", @@ -145,19 +145,19 @@ "mocha-junit-reporter": "^1.18.0", "nyc": "^14.0.0", "prettier": "^1.16.4", - "puppeteer": "^2.0.0", + "puppeteer": "^3.3.0", "rimraf": "^3.0.0", "rollup": "^1.16.3", "@rollup/plugin-commonjs": "11.0.2", - "@rollup/plugin-node-resolve": "^7.0.0", + "@rollup/plugin-node-resolve": "^8.0.0", "rollup-plugin-shim": "^1.0.0", "rollup-plugin-sourcemaps": "^0.4.2", "rollup-plugin-terser": "^5.1.1", - "rollup-plugin-visualizer": "^3.1.1", + "rollup-plugin-visualizer": "^4.0.4", "source-map-support": "^0.5.9", "ts-node": "^8.3.0", - "typescript": "~3.8.3", + "typescript": "~3.9.3", "util": "^0.12.1", - "sinon": "^7.1.0" + "sinon": "^9.0.2" } -} +} \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md index d9da33e36272..d2784c73b7d3 100644 --- a/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md +++ b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md @@ -45,22 +45,14 @@ export interface BlobChangeFeedEventData { // (undocumented) contentLength: number; // (undocumented) - contentOffset?: number; - // (undocumented) contentType: string; // (undocumented) - destinationUrl?: string; - // (undocumented) etag: string; // (undocumented) - recursive?: string; - // (undocumented) requestId: string; // (undocumented) sequencer: string; // (undocumented) - sourceUrl?: string; - // (undocumented) url: string; } @@ -74,7 +66,7 @@ export class BlobChangeFeedEventPage { } // @public (undocumented) -export type BlobChangeFeedEventType = "BlobCreate" | "BlobDeleted"; +export type BlobChangeFeedEventType = "UnspecifiedEventType" | "BlobCreated" | "BlobDeleted" | "BlobPropertiesUpdated" | "BlobSnapshotCreated" | "Control" | "BlobTierChanged" | "BlobAsyncOperationInitiated" | "BlobMetadataUpdated"; // @public (undocumented) export type BlobType = "BlockBlob" | "AppendBlob" | "PageBlob"; diff --git a/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts b/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts index b981d16b0ae4..d60041802016 100644 --- a/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts +++ b/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts @@ -1,6 +1,5 @@ import { BlobServiceClient, StorageSharedKeyCredential } from "@azure/storage-blob"; -// import { BlobChangeFeedClient } from "@azure/storage-blob-changefeed"; -import { BlobChangeFeedClient } from "../../src"; +import { BlobChangeFeedClient, BlobChangeFeedEvent } from "../../src"; // Load the .env file if it exists import * as dotenv from "dotenv"; @@ -23,24 +22,15 @@ export async function main() { sharedKeyCredential ); - - const containerClient = blobServiceClient.getContainerClient("$blobchangefeed"); - console.log("List container.") - for await (const item of containerClient.listBlobsFlat()) { - console.log(`${item.name}: ${item.properties.contentLength}`); - } - const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); - let i = 0; - for await (const event of changeFeedClient.getChanges()) { - i++; - if (i <= 2) { - console.log(event); - } else { - break; - } + + const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be floor to 22:00 + const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be ceil to 22:00 + let changeFeedEvents: BlobChangeFeedEvent[] = []; + // You can also provide just a start or end time. + for await (const event of changeFeedClient.getChanges({ start, end })) { + changeFeedEvents.push(event); } - console.log(`event count: ${i}`); } main().catch((err) => { diff --git a/sdk/storage/storage-blob-changefeed/samples/typscript/resume.ts b/sdk/storage/storage-blob-changefeed/samples/typscript/resume.ts new file mode 100644 index 000000000000..37e1884e3699 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/samples/typscript/resume.ts @@ -0,0 +1,42 @@ +import { BlobServiceClient, StorageSharedKeyCredential } from "@azure/storage-blob"; +import { BlobChangeFeedClient, BlobChangeFeedEvent } from "../../src"; + +// Load the .env file if it exists +import * as dotenv from "dotenv"; +console.log(dotenv.config()); + +import { setLogLevel } from "@azure/logger"; +setLogLevel("info"); + +export async function main() { + // Enter your storage account name and shared key + const account = process.env.ACCOUNT_NAME || ""; + const accountKey = process.env.ACCOUNT_KEY || ""; + + // Use StorageSharedKeyCredential with storage account and account key + // StorageSharedKeyCredential is only available in Node.js runtime, not in browsers + const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey); + const blobServiceClient = new BlobServiceClient( + // When using AnonymousCredential, following url should include a valid SAS or support public access + `https://${account}.blob.core.windows.net`, + sharedKeyCredential + ); + + const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); + let changeFeedEvents: BlobChangeFeedEvent[] = []; + const firstPage = await changeFeedClient.getChanges().byPage({ maxPageSize: 10 }).next(); + for (const event of firstPage) { + changeFeedEvents.push(event); + } + + // Resume iterating from the pervious position with the continuationToken. + for await (const eventPage of changeFeedClient.getChanges().byPage({ continuationToken: firstPage.continuationToken })) { + for (const event of eventPage) { + changeFeedEvents.push(event); + } + } +} + +main().catch((err) => { + console.error("Error running sample:", err.message); +}); diff --git a/sdk/storage/storage-internal-avro/package.json b/sdk/storage/storage-internal-avro/package.json index 09b8b61300b7..8a1bb28f4acf 100644 --- a/sdk/storage/storage-internal-avro/package.json +++ b/sdk/storage/storage-internal-avro/package.json @@ -69,7 +69,7 @@ "rimraf": "^3.0.0", "rollup": "^1.16.3", "@rollup/plugin-commonjs": "11.0.2", - "@rollup/plugin-node-resolve": "^7.0.0", + "@rollup/plugin-node-resolve": "^8.0.0", "rollup-plugin-shim": "^1.0.0", "rollup-plugin-sourcemaps": "^0.4.2", "rollup-plugin-terser": "^5.1.1", diff --git a/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts index ac5412cacb47..0dd8e75b5a1d 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts @@ -56,12 +56,14 @@ export class AvroReadableFromStream extends AvroReadable { let chunk = this._readable.read(size); if (chunk) { this._position += chunk.length; + + this._readable.removeListener("readable", readableCallback); + this._readable.removeListener("error", rejectCallback); + this._readable.removeListener("end", rejectCallback); + this._readable.removeListener("close", rejectCallback); + // chunk.length maybe less than desired size if the stream ends. resolve(this.toUint8Array(chunk)); - this._readable.removeListener("readable", readableCallback); - this._readable.removeListener("error", reject); - this._readable.removeListener("end", reject); - this._readable.removeListener("close", reject); } };