From b032ef1b7ad71bced80179f0958f2467a0af2373 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 28 May 2020 22:04:09 +0800 Subject: [PATCH 01/17] fix --- sdk/storage/storage-internal-avro/package.json | 2 +- sdk/storage/storage-internal-avro/src/AvroParser.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/storage/storage-internal-avro/package.json b/sdk/storage/storage-internal-avro/package.json index 8a1bb28f4acf..09b8b61300b7 100644 --- a/sdk/storage/storage-internal-avro/package.json +++ b/sdk/storage/storage-internal-avro/package.json @@ -69,7 +69,7 @@ "rimraf": "^3.0.0", "rollup": "^1.16.3", "@rollup/plugin-commonjs": "11.0.2", - "@rollup/plugin-node-resolve": "^8.0.0", + "@rollup/plugin-node-resolve": "^7.0.0", "rollup-plugin-shim": "^1.0.0", "rollup-plugin-sourcemaps": "^0.4.2", "rollup-plugin-terser": "^5.1.1", diff --git a/sdk/storage/storage-internal-avro/src/AvroParser.ts b/sdk/storage/storage-internal-avro/src/AvroParser.ts index 4bb310068953..4a418593e1b4 100644 --- a/sdk/storage/storage-internal-avro/src/AvroParser.ts +++ b/sdk/storage/storage-internal-avro/src/AvroParser.ts @@ -46,6 +46,7 @@ export class AvroParser { if (haveMoreByte) { // Switch to float arithmetic // FIXME: this only works when zigZagEncoded is no more than Number.MAX_SAFE_INTEGER (2**53 - 1) + zigZagEncoded = zigZagEncoded; significanceInFloat = 268435456; // 2 ** 28. do { byte = await AvroParser.readByte(stream); From 33f56f4c023656fa1a3141027648094d89d79f44 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 14 May 2020 20:21:32 +0800 Subject: [PATCH 02/17] change feed with mininal live tests --- rush.json | 7 +- .../.vscode/extensions.json | 3 + .../.vscode/launch.json | 59 ++++++ .../.vscode/settings.json | 27 +++ sdk/storage/storage-blob-changefeed/LICENSE | 21 +++ .../api-extractor.json | 31 +++ .../storage-blob-changefeed/package.json | 161 ++++++++++++++++ .../review/storage-blob-change-feed.api.md | 91 +++++++++ .../rollup.base.config.js | 177 ++++++++++++++++++ .../storage-blob-changefeed/rollup.config.js | 17 ++ .../rollup.test.config.js | 8 + .../samples/tsconfig.json | 9 + .../samples/typscript/basic.ts | 48 +++++ .../samples/typscript/package.json | 44 +++++ .../samples/typscript/sample.env | 20 ++ .../src/AvroReaderFactory.ts | 23 +++ .../src/BlobChangeFeedClient.ts | 115 ++++++++++++ .../storage-blob-changefeed/src/ChangeFeed.ts | 131 +++++++++++++ .../src/ChangeFeedFactory.ts | 132 +++++++++++++ .../storage-blob-changefeed/src/Chunk.ts | 48 +++++ .../src/ChunkFactory.ts | 39 ++++ .../storage-blob-changefeed/src/Segment.ts | 83 ++++++++ .../src/SegmentFactory.ts | 55 ++++++ .../storage-blob-changefeed/src/Shard.ts | 59 ++++++ .../src/ShardFactory.ts | 47 +++++ .../storage-blob-changefeed/src/index.ts | 2 + .../storage-blob-changefeed/src/log.ts | 9 + .../src/models/BlobChangeFeedEvent.ts | 32 ++++ .../src/models/ChangeFeedCursor.ts | 20 ++ .../src/utils/constants.ts | 6 + .../src/utils/utils.browser.ts | 31 +++ .../src/utils/utils.common.ts | 94 ++++++++++ .../src/utils/utils.node.ts | 41 ++++ .../test/blobchangefeedclient.spec.ts | 132 +++++++++++++ .../storage-blob-changefeed/tsconfig.json | 26 +++ .../src/generated/src/models/parameters.ts | 2 +- .../src/generated/src/storageClientContext.ts | 2 +- .../src/AvroReadableFromStream.ts | 1 + 38 files changed, 1850 insertions(+), 3 deletions(-) create mode 100644 sdk/storage/storage-blob-changefeed/.vscode/extensions.json create mode 100644 sdk/storage/storage-blob-changefeed/.vscode/launch.json create mode 100644 sdk/storage/storage-blob-changefeed/.vscode/settings.json create mode 100644 sdk/storage/storage-blob-changefeed/LICENSE create mode 100644 sdk/storage/storage-blob-changefeed/api-extractor.json create mode 100644 sdk/storage/storage-blob-changefeed/package.json create mode 100644 sdk/storage/storage-blob-changefeed/review/storage-blob-change-feed.api.md create mode 100644 sdk/storage/storage-blob-changefeed/rollup.base.config.js create mode 100644 sdk/storage/storage-blob-changefeed/rollup.config.js create mode 100644 sdk/storage/storage-blob-changefeed/rollup.test.config.js create mode 100644 sdk/storage/storage-blob-changefeed/samples/tsconfig.json create mode 100644 sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts create mode 100644 sdk/storage/storage-blob-changefeed/samples/typscript/package.json create mode 100644 sdk/storage/storage-blob-changefeed/samples/typscript/sample.env create mode 100644 sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/Chunk.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/Segment.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/Shard.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/ShardFactory.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/index.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/log.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/utils/constants.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/utils/utils.browser.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts create mode 100644 sdk/storage/storage-blob-changefeed/src/utils/utils.node.ts create mode 100644 sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts create mode 100644 sdk/storage/storage-blob-changefeed/tsconfig.json diff --git a/rush.json b/rush.json index 75f4e6fe7b9d..1a88ae61de66 100644 --- a/rush.json +++ b/rush.json @@ -457,6 +457,11 @@ "projectFolder": "sdk/storage/storage-blob", "versionPolicyName": "client" }, + { + "packageName": "@azure/storage-blob-changefeed", + "projectFolder": "sdk/storage/storage-blob-changefeed", + "versionPolicyName": "client" + }, { "packageName": "@azure/storage-file-share", "projectFolder": "sdk/storage/storage-file-share", @@ -493,4 +498,4 @@ "versionPolicyName": "utility" } ] -} +} \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/.vscode/extensions.json b/sdk/storage/storage-blob-changefeed/.vscode/extensions.json new file mode 100644 index 000000000000..c83e26348e1f --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["esbenp.prettier-vscode"] +} diff --git a/sdk/storage/storage-blob-changefeed/.vscode/launch.json b/sdk/storage/storage-blob-changefeed/.vscode/launch.json new file mode 100644 index 000000000000..24dbfc9d74c4 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/.vscode/launch.json @@ -0,0 +1,59 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Debug Javascript Samples", + "program": "${workspaceFolder}/samples/javascript/basic.js", + "preLaunchTask": "npm: build:js-samples" + }, + { + "type": "node", + "request": "launch", + "name": "Debug Typescript Samples", + "program": "${workspaceFolder}/samples/typescript/basic.ts", + "preLaunchTask": "npm: build:ts-samples", + "outFiles": ["${workspaceFolder}/dist-esm/samples/typescript/*.js"] + }, + { + "type": "node", + "request": "launch", + "name": "Debug Mocha Test [Without Rollup]", + "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", + "args": [ + "-r", + "ts-node/register", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/test/*.spec.ts", + "${workspaceFolder}/test/node/*.spec.ts" + ], + "env": { "TS_NODE_COMPILER_OPTIONS": "{\"module\": \"commonjs\"}" }, + "envFile": "${workspaceFolder}/../.env", + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "protocol": "inspector" + }, + { + "type": "node", + "request": "launch", + "name": "Debug Unit Tests", + "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", + "args": [ + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/dist-test/index.node.js" + ], + "internalConsoleOptions": "openOnSessionStart", + "preLaunchTask": "npm: build:test" + } + ] +} diff --git a/sdk/storage/storage-blob-changefeed/.vscode/settings.json b/sdk/storage/storage-blob-changefeed/.vscode/settings.json new file mode 100644 index 000000000000..7ceb5ace3e9d --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/.vscode/settings.json @@ -0,0 +1,27 @@ +{ + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.DS_Store": true + }, + "[typescript]": { + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.detectIndentation": false + }, + "[json]": { + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.detectIndentation": false + }, + "[yaml]": { + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.detectIndentation": false + }, + "editor.rulers": [ + 100 + ], + "typescript.preferences.quoteStyle": "double", + "javascript.preferences.quoteStyle": "double" + } \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/LICENSE b/sdk/storage/storage-blob-changefeed/LICENSE new file mode 100644 index 000000000000..ea8fb1516028 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2020 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/sdk/storage/storage-blob-changefeed/api-extractor.json b/sdk/storage/storage-blob-changefeed/api-extractor.json new file mode 100644 index 000000000000..8df76318f270 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/api-extractor.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + "mainEntryPointFilePath": "typings/latest/storage-blob-changefeed/src/index.d.ts", + "docModel": { + "enabled": false + }, + "apiReport": { + "enabled": true, + "reportFolder": "./review" + }, + "dtsRollup": { + "enabled": true, + "untrimmedFilePath": "", + "publicTrimmedFilePath": "./typings/latest/storage-blob-changefeed.d.ts" + }, + "messages": { + "tsdocMessageReporting": { + "default": { + "logLevel": "none" + } + }, + "extractorMessageReporting": { + "ae-missing-release-tag": { + "logLevel": "none" + }, + "ae-unresolved-link": { + "logLevel": "none" + } + } + } +} \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/package.json b/sdk/storage/storage-blob-changefeed/package.json new file mode 100644 index 000000000000..dbe58ad04fff --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/package.json @@ -0,0 +1,161 @@ +{ + "name": "@azure/storage-blob-changefeed", + "sdk-type": "client", + "version": "12.0.0-preview.1", + "description": "Microsoft Azure Storage SDK for JavaScript - Blob Change Feed", + "main": "./dist/index.js", + "module": "./dist-esm/storage-blob-changefeed/src/index.js", + "browser": { + "./dist-esm/storage-blob-changefeed/src/utils/utils.node.js": "./dist-esm/storage-blob-changefeed/src/utils/utils.browser.js", + "./dist-esm/storage-blob-changefeed/test/utils/index.js": "./dist-esm/storage-blob-changefeed/test/utils/index.browser.js", + "fs": false, + "os": false, + "process": false + }, + "types": "./typings/latest/storage-blob-changefeed.d.ts", + "typesVersions": { + "<3.6": { + "*": [ + "./typings/3.1/storage-blob-changefeed.d.ts" + ] + } + }, + "engine": { + "node": ">=8.0.0" + }, + "scripts": { + "build:es6": "tsc -p tsconfig.json", + "build:nodebrowser": "rollup -c 2>&1", + "build:samples": "npm run clean && npm run build:es6 && cross-env ONLY_NODE=true rollup -c 2>&1 && npm run build:prep-samples", + "build:prep-samples": "node ../../../common/scripts/prep-samples.js && cd samples && tsc", + "build:test": "npm run build:es6 && rollup -c rollup.test.config.js 2>&1", + "build:types": "downlevel-dts typings/latest typings/3.1", + "build": "npm run build:es6 && npm run build:nodebrowser && api-extractor run --local && npm run build:types", + "check-format": "prettier --list-different --config ../../.prettierrc.json \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"", + "clean": "rimraf dist dist-esm dist-test typings temp dist-browser/*.js* dist-browser/*.zip statistics.html coverage coverage-browser .nyc_output *.tgz *.log test*.xml TEST*.xml", + "clean:samples": "rimraf samples/javascript/node_modules samples/typescript/node_modules samples/typescript/dist samples/typescript/package-lock.json samples/javascript/package-lock.json", + "extract-api": "tsc -p . && api-extractor run --local", + "execute:js-samples": "node ../../../common/scripts/run-samples.js samples/javascript/", + "execute:ts-samples": "node ../../../common/scripts/run-samples.js samples/typescript/dist/samples/typescript/src/", + "execute:samples": "npm run build:samples && npm run execute:js-samples && npm run execute:ts-samples", + "format": "prettier --write --config ../../.prettierrc.json \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"", + "integration-test:browser": "karma start --single-run", + "integration-test:node": "nyc mocha -r esm --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 300000 dist-esm/storage-blob-changefeed/test/*.spec.js dist-esm/storage-blob-changefeed/test/node/*.spec.js", + "integration-test": "npm run integration-test:node && npm run integration-test:browser", + "pack": "npm pack 2>&1", + "prebuild": "npm run clean", + "test:browser": "npm run clean && npm run build:test && npm run unit-test:browser", + "test:node": "npm run clean && npm run build:test && npm run unit-test:node", + "test": "npm run clean && npm run build:test && npm run unit-test", + "unit-test:browser": "karma start --single-run", + "unit-test:node": "mocha --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 120000 dist-test/index.node.js", + "unit-test": "npm run unit-test:node && npm run unit-test:browser", + "emulator-tests": "cross-env STORAGE_CONNECTION_STRING=UseDevelopmentStorage=true && npm run test:node" + }, + "files": [ + "BreakingChanges.md", + "types/", + "dist/", + "dist-browser/", + "dist-esm/storage-blob-changefeed/src/", + "dist-esm/storage-internal-avro/src/", + "typings/latest/storage-blob-changefeed.d.ts", + "typings/3.1/storage-blob-changefeed.d.ts", + "README.md", + "LICENSE" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/Azure/azure-sdk-for-js.git" + }, + "keywords": [ + "Azure", + "Storage", + "Blob", + "Change feed", + "Node.js", + "TypeScript", + "JavaScript", + "Browser" + ], + "author": "Microsoft Corporation", + "license": "MIT", + "bugs": { + "url": "https://github.com/Azure/azure-sdk-for-js/issues" + }, + "homepage": "https://github.com/Azure/azure-sdk-for-js#readme", + "sideEffects": false, + "//metadata": { + "constantPaths": [ + { + "path": "src/utils/constants.ts", + "prefix": "SDK_VERSION" + } + ] + }, + "dependencies": { + "@azure/storage-blob": "^12.1.2", + "@azure/abort-controller": "^1.0.0", + "@azure/core-http": "^1.1.1", + "@azure/core-lro": "^1.0.2", + "@azure/core-paging": "^1.1.1", + "@azure/core-tracing": "1.0.0-preview.8", + "@azure/logger": "^1.0.0", + "@opentelemetry/api": "^0.6.1", + "events": "^3.0.0", + "tslib": "^1.10.0" + }, + "devDependencies": { + "@azure/identity": "^1.1.0-preview", + "@azure/test-utils-recorder": "^1.0.0", + "@microsoft/api-extractor": "7.7.11", + "@rollup/plugin-multi-entry": "^3.0.0", + "@rollup/plugin-replace": "^2.2.0", + "@types/mocha": "^7.0.2", + "@types/node": "^8.0.0", + "@typescript-eslint/eslint-plugin": "^2.0.0", + "@typescript-eslint/parser": "^2.0.0", + "assert": "^1.4.1", + "cross-env": "^6.0.3", + "dotenv": "^8.2.0", + "downlevel-dts": "~0.4.0", + "es6-promise": "^4.2.5", + "eslint": "^6.1.0", + "eslint-config-prettier": "^6.0.0", + "eslint-plugin-no-null": "^1.0.2", + "eslint-plugin-no-only-tests": "^2.3.0", + "eslint-plugin-promise": "^4.1.1", + "esm": "^3.2.18", + "inherits": "^2.0.3", + "karma": "^4.0.1", + "karma-chrome-launcher": "^3.0.0", + "karma-coverage": "^2.0.0", + "karma-edge-launcher": "^0.4.2", + "karma-env-preprocessor": "^0.1.1", + "karma-firefox-launcher": "^1.1.0", + "karma-ie-launcher": "^1.0.0", + "karma-json-preprocessor": "^0.3.3", + "karma-json-to-file-reporter": "^1.0.1", + "karma-junit-reporter": "^2.0.1", + "karma-mocha": "^1.3.0", + "karma-mocha-reporter": "^2.2.5", + "karma-remap-istanbul": "^0.6.0", + "mocha": "^7.1.1", + "mocha-junit-reporter": "^1.18.0", + "nyc": "^14.0.0", + "prettier": "^1.16.4", + "puppeteer": "^2.0.0", + "rimraf": "^3.0.0", + "rollup": "^1.16.3", + "@rollup/plugin-commonjs": "11.0.2", + "@rollup/plugin-node-resolve": "^7.0.0", + "rollup-plugin-shim": "^1.0.0", + "rollup-plugin-sourcemaps": "^0.4.2", + "rollup-plugin-terser": "^5.1.1", + "rollup-plugin-visualizer": "^3.1.1", + "source-map-support": "^0.5.9", + "ts-node": "^8.3.0", + "typescript": "~3.8.3", + "util": "^0.12.1" + } +} \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/review/storage-blob-change-feed.api.md b/sdk/storage/storage-blob-changefeed/review/storage-blob-change-feed.api.md new file mode 100644 index 000000000000..6a7f4c6c3c9e --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/review/storage-blob-change-feed.api.md @@ -0,0 +1,91 @@ +## API Report File for "@azure/storage-blob-changefeed" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { BlobServiceClient } from '@azure/storage-blob'; +import { PagedAsyncIterableIterator } from '@azure/core-paging'; + +// @public (undocumented) +export class BlobChangeFeedClient { + constructor(blobServiceClient: BlobServiceClient); + // (undocumented) + getChanges(options?: ChangeFeedGetChangesOptions): PagedAsyncIterableIterator; + } + +// @public (undocumented) +export interface BlobChangeFeedEvent { + // (undocumented) + data: BlobChangeFeedEventData; + // (undocumented) + dataVersion?: string; + // (undocumented) + eventTime: string; + // (undocumented) + eventType: BlobChangeFeedEventType; + // (undocumented) + id: string; + // (undocumented) + metadataVersion: string; + // (undocumented) + subject: string; + // (undocumented) + topic: string; +} + +// @public (undocumented) +export interface BlobChangeFeedEventData { + // (undocumented) + api: string; + // (undocumented) + blobType: BlobType; + // (undocumented) + clientRequestId: string; + // (undocumented) + contentLength: number; + // (undocumented) + contentType: string; + // (undocumented) + destinationUrl?: string; + // (undocumented) + eTag: string; + // (undocumented) + recursive?: string; + // (undocumented) + requestId: string; + // (undocumented) + sequencer: string; + // (undocumented) + sourceUrl?: string; + // (undocumented) + url: string; +} + +// @public (undocumented) +export class BlobChangeFeedEventPage { + constructor(); + // (undocumented) + continuationToken: string; + // (undocumented) + events: BlobChangeFeedEvent[]; +} + +// @public (undocumented) +export type BlobChangeFeedEventType = "BlobCreate" | "BlobDeleted"; + +// @public (undocumented) +export type BlobType = "BlockBlob" | "AppendBlob" | "PageBlob"; + +// @public (undocumented) +export interface ChangeFeedGetChangesOptions { + // (undocumented) + end?: Date; + // (undocumented) + start?: Date; +} + + +// (No @packageDocumentation comment for this package) + +``` diff --git a/sdk/storage/storage-blob-changefeed/rollup.base.config.js b/sdk/storage/storage-blob-changefeed/rollup.base.config.js new file mode 100644 index 000000000000..abd11444d821 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/rollup.base.config.js @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import nodeResolve from "@rollup/plugin-node-resolve"; +import multiEntry from "@rollup/plugin-multi-entry"; +import cjs from "@rollup/plugin-commonjs"; +import replace from "@rollup/plugin-replace"; +import { terser } from "rollup-plugin-terser"; +import sourcemaps from "rollup-plugin-sourcemaps"; +import shim from "rollup-plugin-shim"; +// import visualizer from "rollup-plugin-visualizer"; + +const version = require("./package.json").version; +const banner = [ + "/*!", + ` * Azure Storage SDK for JavaScript - Blob, ${version}`, + " * Copyright (c) Microsoft and contributors. All rights reserved.", + " */" +].join("\n"); + +const pkg = require("./package.json"); +const depNames = Object.keys(pkg.dependencies); +const production = process.env.NODE_ENV === "production"; + +export function nodeConfig(test = false) { + const externalNodeBuiltins = [ + "@azure/core-http", + "crypto", + "fs", + "events", + "os", + "stream", + "util" + ]; + const baseConfig = { + input: "dist-esm/storage-blob-changefeed/src/index.js", + external: depNames.concat(externalNodeBuiltins), + output: { + file: "dist/index.js", + format: "cjs", + sourcemap: true + }, + preserveSymlinks: false, + plugins: [ + sourcemaps(), + replace({ + delimiters: ["", ""], + values: { + // replace dynamic checks with if (true) since this is for node only. + // Allows rollup's dead code elimination to be more aggressive. + "if (isNode)": "if (true)" + } + }), + nodeResolve({ preferBuiltins: true }), + cjs() + ], + onwarn(warning, warn) { + if (warning.code === "CIRCULAR_DEPENDENCY") { + throw new Error(warning.message); + } + warn(warning); + } + }; + + if (test) { + // entry point is every test file + baseConfig.input = [ + "dist-esm/storage-blob-changefeed/test/*.spec.js", + "dist-esm/storage-blob-changefeed/test/node/*.spec.js", + "dist-esm/storage-blob-changefeed/src/index.js" + ]; + baseConfig.plugins.unshift(multiEntry()); + + // different output file + baseConfig.output.file = "dist-test/index.node.js"; + + // mark assert as external + baseConfig.external.push("assert", "fs", "path", "buffer", "zlib"); + + baseConfig.context = "null"; + + // Disable tree-shaking of test code. In rollup-plugin-node-resolve@5.0.0, rollup started respecting + // the "sideEffects" field in package.json. Since our package.json sets "sideEffects=false", this also + // applies to test code, which causes all tests to be removed by tree-shaking. + baseConfig.treeshake = false; + } else if (production) { + baseConfig.plugins.push(terser()); + } + + return baseConfig; +} + +export function browserConfig(test = false) { + const baseConfig = { + input: "dist-esm/storage-blob-changefeed/src/index.browser.js", + output: { + file: "dist-browser/azure-storage-blob-changefeed.js", + banner: banner, + format: "umd", + name: "azblob", + sourcemap: true + }, + preserveSymlinks: false, + plugins: [ + sourcemaps(), + replace({ + delimiters: ["", ""], + values: { + // replace dynamic checks with if (false) since this is for + // browser only. Rollup's dead code elimination will remove + // any code guarded by if (isNode) { ... } + "if (isNode)": "if (false)" + } + }), + // fs and os are not used by the browser bundle, so just shim it + // dotenv doesn't work in the browser, so replace it with a no-op function + shim({ + dotenv: `export function config() { }`, + fs: ` + export function stat() { } + export function createReadStream() { } + export function createWriteStream() { } + `, + os: ` + export const type = 1; + export const release = 1; + `, + util: ` + export function promisify() { } + ` + }), + nodeResolve({ + mainFields: ["module", "browser"], + preferBuiltins: false + }), + cjs({ + namedExports: { + events: ["EventEmitter"], + assert: [ + "ok", + "deepEqual", + "equal", + "fail", + "strictEqual", + "deepStrictEqual", + "notDeepEqual", + "notDeepStrictEqual" + ], + "@opentelemetry/api": ["CanonicalCode", "SpanKind", "TraceFlags"] + } + }) + ], + onwarn(warning, warn) { + if (warning.code === "CIRCULAR_DEPENDENCY") { + throw new Error(warning.message); + } + warn(warning); + } + }; + + if (test) { + baseConfig.input = ["dist-esm/storage-blob-changefeed/test/*.spec.js", "dist-esm/storage-blob-changefeed/test/browser/*.spec.js"]; + baseConfig.plugins.unshift(multiEntry({ exports: false })); + baseConfig.output.file = "dist-test/index.browser.js"; + // mark fs-extra as external + baseConfig.external = ["fs-extra"]; + + baseConfig.context = "null"; + + // Disable tree-shaking of test code. In rollup-plugin-node-resolve@5.0.0, rollup started respecting + // the "sideEffects" field in package.json. Since our package.json sets "sideEffects=false", this also + // applies to test code, which causes all tests to be removed by tree-shaking. + baseConfig.treeshake = false; + } + + return baseConfig; +} diff --git a/sdk/storage/storage-blob-changefeed/rollup.config.js b/sdk/storage/storage-blob-changefeed/rollup.config.js new file mode 100644 index 000000000000..a62dabd573b4 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/rollup.config.js @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as base from "./rollup.base.config"; + +const inputs = []; + +if (!process.env.ONLY_BROWSER) { + inputs.push(base.nodeConfig()); +} + +// Disable this until we are ready to run rollup for the browser. +// if (!process.env.ONLY_NODE) { +// inputs.push(base.browserConfig()); +// } + +export default inputs; diff --git a/sdk/storage/storage-blob-changefeed/rollup.test.config.js b/sdk/storage/storage-blob-changefeed/rollup.test.config.js new file mode 100644 index 000000000000..b147ec721e5c --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/rollup.test.config.js @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as base from "./rollup.base.config"; + +export default [base.nodeConfig(true), + // base.browserConfig(true) +]; diff --git a/sdk/storage/storage-blob-changefeed/samples/tsconfig.json b/sdk/storage/storage-blob-changefeed/samples/tsconfig.json new file mode 100644 index 000000000000..3a37abdb0902 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/samples/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "outDir": "typescript/dist" + }, + "include": ["typescript/src/**.ts"], + "exclude": ["typescript/*.json", "**/node_modules/", "../node_modules", "../typings"] +} diff --git a/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts b/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts new file mode 100644 index 000000000000..b981d16b0ae4 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts @@ -0,0 +1,48 @@ +import { BlobServiceClient, StorageSharedKeyCredential } from "@azure/storage-blob"; +// import { BlobChangeFeedClient } from "@azure/storage-blob-changefeed"; +import { BlobChangeFeedClient } from "../../src"; + +// Load the .env file if it exists +import * as dotenv from "dotenv"; +console.log(dotenv.config()); + +import { setLogLevel } from "@azure/logger"; +setLogLevel("info"); + +export async function main() { + // Enter your storage account name and shared key + const account = process.env.ACCOUNT_NAME || ""; + const accountKey = process.env.ACCOUNT_KEY || ""; + + // Use StorageSharedKeyCredential with storage account and account key + // StorageSharedKeyCredential is only available in Node.js runtime, not in browsers + const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey); + const blobServiceClient = new BlobServiceClient( + // When using AnonymousCredential, following url should include a valid SAS or support public access + `https://${account}.blob.core.windows.net`, + sharedKeyCredential + ); + + + const containerClient = blobServiceClient.getContainerClient("$blobchangefeed"); + console.log("List container.") + for await (const item of containerClient.listBlobsFlat()) { + console.log(`${item.name}: ${item.properties.contentLength}`); + } + + const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); + let i = 0; + for await (const event of changeFeedClient.getChanges()) { + i++; + if (i <= 2) { + console.log(event); + } else { + break; + } + } + console.log(`event count: ${i}`); +} + +main().catch((err) => { + console.error("Error running sample:", err.message); +}); diff --git a/sdk/storage/storage-blob-changefeed/samples/typscript/package.json b/sdk/storage/storage-blob-changefeed/samples/typscript/package.json new file mode 100644 index 000000000000..f749fce30875 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/samples/typscript/package.json @@ -0,0 +1,44 @@ +{ + "name": "azure-storage-blob-changefeed-samples-ts", + "private": true, + "version": "0.1.0", + "description": "Azure Storage Blob Change Feed client library samples for TypeScript", + "engine": { + "node": ">=8.0.0" + }, + "scripts": { + "build": "tsc", + "prebuild": "rimraf dist/" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/Azure/azure-sdk-for-js.git" + }, + "keywords": [ + "Azure", + "Storage", + "Blob", + "Change Feed", + "Node.js", + "TypeScript" + ], + "author": "Microsoft Corporation", + "license": "MIT", + "bugs": { + "url": "https://github.com/Azure/azure-sdk-for-js/issues" + }, + "homepage": "https://github.com/Azure/azure-sdk-for-js#readme", + "sideEffects": false, + "dependencies": { + "@azure/abort-controller": "latest", + "@azure/identity": "latest", + "@azure/storage-blob": "latest", + "@azure/storage-blob-changefeed": "latest", + "dotenv": "^8.2.0" + }, + "devDependencies": { + "@types/node": "^8.0.0", + "rimraf": "^3.0.0", + "typescript": "~3.6.4" + } +} \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/samples/typscript/sample.env b/sdk/storage/storage-blob-changefeed/samples/typscript/sample.env new file mode 100644 index 000000000000..92a81cac6547 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/samples/typscript/sample.env @@ -0,0 +1,20 @@ +# Used in most samples. Retrieve these values from a storage account in the Azure Portal. +ACCOUNT_NAME= +ACCOUNT_KEY= + +# Used for withConnString +STORAGE_CONNECTION_STRING= + +# Used for the advanced and anonymousCred tests. Create a SAS token for a storage account in the Azure Portal. +ACCOUNT_SAS= + +# Used to authenticate using Azure AD as a service principal for role-based authentication. +# +# See the documentation for `EnvironmentCredential` at the following link: +# https://docs.microsoft.com/javascript/api/@azure/identity/environmentcredential +AZURE_TENANT_ID= +AZURE_CLIENT_ID= +AZURE_CLIENT_SECRET= + +# To run the proxyAuth sample, set up an HTTP proxy and enter your information: +# HTTP_PROXY=http://localhost:3128 diff --git a/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts b/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts new file mode 100644 index 000000000000..9e4d58a9966a --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts @@ -0,0 +1,23 @@ +import { AvroReadable, AvroReader } from '../../storage-internal-avro/src'; + +export class AvroReaderFactory { + public buildAvroReader(dataStream: AvroReadable): AvroReader; + public buildAvroReader( + dataStream: AvroReadable, + headerStream: AvroReadable, + blockOffset: number, + eventIndex: number): AvroReader; + + public buildAvroReader( + dataStream: AvroReadable, + headerStream?: AvroReadable, + blockOffset?: number, + eventIndex?: number + ): AvroReader { + if (headerStream) { + return new AvroReader(dataStream, headerStream, blockOffset!, eventIndex!); + } else { + return new AvroReader(dataStream); + } + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts b/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts new file mode 100644 index 000000000000..c72b6c6e59e1 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts @@ -0,0 +1,115 @@ +import { BlobServiceClient } from "@azure/storage-blob"; +import { PagedAsyncIterableIterator, PageSettings } from "@azure/core-paging"; +import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; +import { ChangeFeedFactory } from "./ChangeFeedFactory"; +import { ChangeFeed } from "./ChangeFeed"; +import { CHANGE_FEED_DEFAULT_PAGE_SIZE } from "./utils/constants"; + +export interface ChangeFeedGetChangesOptions { + start?: Date; + end?: Date; +} + +export class BlobChangeFeedEventPage { + public events: BlobChangeFeedEvent[]; + public continuationToken: string; + + constructor() { + this.events = []; + this.continuationToken = ""; + } +} + +export class BlobChangeFeedClient { + /** + * blobServiceClient provided by @azure/storage-blob package. + * + * @private + * @type {BlobServiceClient} + * @memberof DataLakeServiceClient + */ + private _blobServiceClient: BlobServiceClient; + private _changeFeedFactory: ChangeFeedFactory; + + public constructor(blobServiceClient: BlobServiceClient) { + this._blobServiceClient = blobServiceClient; + this._changeFeedFactory = new ChangeFeedFactory(); + } + + public getChanges(options: ChangeFeedGetChangesOptions = {}) + : PagedAsyncIterableIterator { + const iter = this.getChange(options); + return { + /** + * @member {Promise} [next] The next method, part of the iteration protocol + */ + async next() { + return iter.next(); + }, + /** + * @member {Symbol} [asyncIterator] The connection to the async iterator, part of the iteration protocol + */ + [Symbol.asyncIterator]() { + return this; + }, + /** + * @member {Function} [byPage] Return an AsyncIterableIterator that works a page at a time + */ + byPage: (settings: PageSettings = {}) => { + return this.getPage(settings.continuationToken, settings.maxPageSize, options); + } + }; + } + + private async *getChange(options: ChangeFeedGetChangesOptions = {}) + : AsyncIterableIterator { + const changeFeed: ChangeFeed = await this._changeFeedFactory.buildChangeFeed( + this._blobServiceClient, + undefined, + options.start, + options.end + ); + + while (changeFeed.hasNext()) { + const event = await changeFeed.getChange(); + if (event) { + yield event; + } else { + return; + } + } + } + + // start in ChangeFeedGetChangesOptions will be ignored when continuationToken is specified. + private async *getPage(continuationToken?: string, maxPageSize?: number, options: ChangeFeedGetChangesOptions = {}) + : AsyncIterableIterator { + const changeFeed: ChangeFeed = await this._changeFeedFactory.buildChangeFeed( + this._blobServiceClient, + continuationToken, + options.start, + options.end + ); + + if (!maxPageSize || maxPageSize > CHANGE_FEED_DEFAULT_PAGE_SIZE) { + maxPageSize = CHANGE_FEED_DEFAULT_PAGE_SIZE; + } + while (changeFeed.hasNext()) { + let eventPage = new BlobChangeFeedEventPage(); + while (changeFeed.hasNext() && eventPage.events.length < maxPageSize) { + const event = await changeFeed.getChange(); + if (event) { + eventPage.events.push(event); + } + } + if (changeFeed.hasNext()) { + eventPage.continuationToken = JSON.stringify(changeFeed.getCursor()); + } + if (eventPage.events.length > 0) { + yield eventPage; + } + else { + return; + } + } + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts b/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts new file mode 100644 index 000000000000..88dc075c88ee --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts @@ -0,0 +1,131 @@ +import { ContainerClient } from '@azure/storage-blob'; +import { Segment } from './Segment'; +import { SegmentFactory } from './SegmentFactory'; +import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; +import { ChangeFeedCursor } from "./models/ChangeFeedCursor"; +import { getURLPath, hashString, getSegmentsInYear, minDate } from "./utils/utils.common"; + +export class ChangeFeed { + /** + * BlobContainerClient for making List Blob requests and creating Segments. + * + * @private + * @type {ContainerClient} + * @memberof ChangeFeed + */ + private readonly _containerClient?: ContainerClient; + + private readonly _segmentFactory?: SegmentFactory; + + private readonly _years: number[]; + + private _segments: string[]; + + private _currentSegment?: Segment; + + private _lastConsumable?: Date; + + private _startTime?: Date; + + private _endTime?: Date; + + private _end?: Date; + + constructor(); + constructor(containerClient: ContainerClient, + segmentFactory: SegmentFactory, + years: number[], + segments: string[], + currentSegment: Segment, + lastConsumable: Date, + startTime?: Date, + endTime?: Date + ); + + constructor(containerClient?: ContainerClient, + segmentFactory?: SegmentFactory, + years?: number[], + segments?: string[], + currentSegment?: Segment, + lastConsumable?: Date, + startTime?: Date, + endTime?: Date + ) { + this._containerClient = containerClient; + this._segmentFactory = segmentFactory; + this._years = years || []; + this._segments = segments || []; + this._currentSegment = currentSegment; + this._lastConsumable = lastConsumable; + this._startTime = startTime; + this._endTime = endTime; + if (this._lastConsumable) { + this._end = minDate(this._lastConsumable, this._endTime); + } + } + + public hasNext(): boolean { + // Empty ChangeFeed, using _currentSegment as the indicator. + if (!this._currentSegment) { + return false; + } + + if (this._segments.length === 0 && this._years.length === 0 && !this._currentSegment.hasNext()) { + return false; + } + + return this._currentSegment.finalized && this._currentSegment.dateTime < this._end!; + } + + public async getChange(): Promise { + if (!this.hasNext()) { + throw new Error("Change feed doesn't have any more events"); + } + + let event: BlobChangeFeedEvent | undefined = undefined; + while (!event && this.hasNext()) { + event = await this._currentSegment!.getChange(); + await this.advanceSegmentIfNecessary(); + } + return event; + } + + public getCursor(): ChangeFeedCursor { + if (!this._currentSegment) { + throw new Error("Empty Change Feed shouldn't call this function."); + } + + return { + urlHash: hashString(getURLPath(this._containerClient!.url)!), + endTime: this._endTime?.toJSON(), + currentSegmentCursor: this._currentSegment!.getCursor() + }; + } + + private async advanceSegmentIfNecessary(): Promise { + if (!this._currentSegment) { + throw new Error("Empty Change Feed shouldn't call this function."); + } + + // If the current segment has more Events, we don't need to do anything. + if (this._currentSegment.hasNext()) { + return; + } + + // If the current segment is completed, remove it + if (this._segments.length > 0) { + this._currentSegment = await this._segmentFactory!.buildSegment(this._containerClient!, this._segments.shift()!); + } + // If _segments is empty, refill it + else if (this._segments.length === 0 && this._years.length > 0) { + const year = this._years.shift(); + this._segments = await getSegmentsInYear(this._containerClient!, year!, this._startTime, this._end); + + if (this._segments.length > 0) { + this._currentSegment = await this._segmentFactory!.buildSegment(this._containerClient!, this._segments.shift()!); + } else { + throw new Error(`Year ${year} in the middle should have returned valid segments.`); + } + } + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts new file mode 100644 index 000000000000..00e6a2b2587e --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts @@ -0,0 +1,132 @@ +import { BlobServiceClient, ContainerClient } from "@azure/storage-blob"; +import { ChangeFeed } from "./ChangeFeed"; +import { ChangeFeedCursor } from "./models/ChangeFeedCursor"; +import { + CHANGE_FEED_CONTAINER_NAME, + CHANGE_FEED_META_SEGMENT_PATH +} from './utils/constants'; +import { + ceilToNearestHour, + floorToNearestHour, + getURLPath, + hashString, + getYearsPaths, + getSegmentsInYear, + minDate +} from './utils/utils.common'; +import { + bodyToString +} from './utils/utils.node'; +import { SegmentFactory } from "./SegmentFactory"; +import { ShardFactory } from "./ShardFactory"; +import { ChunkFactory } from "./ChunkFactory"; +import { AvroReaderFactory } from "./AvroReaderFactory"; +import { Segment } from "./Segment"; + +interface MetaSegments { + version?: number; + lastConsumable: string; +} + +export class ChangeFeedFactory { + private readonly _segmentFactory: SegmentFactory; + + constructor(); + constructor(segmentFactory: SegmentFactory); + constructor(segmentFactory?: SegmentFactory) { + if (segmentFactory) { + this._segmentFactory = segmentFactory; + } + else { + this._segmentFactory = new SegmentFactory( + new ShardFactory( + new ChunkFactory( + new AvroReaderFactory()))); + } + } + + public async buildChangeFeed( + blobServiceClient: BlobServiceClient, + continuationToken?: string, + startTime?: Date, + endTime?: Date + ): Promise { + const containerClient = blobServiceClient.getContainerClient(CHANGE_FEED_CONTAINER_NAME); + let cursor: ChangeFeedCursor | undefined = undefined; + // Create cursor. + if (continuationToken) { + cursor = JSON.parse(continuationToken); + ChangeFeedFactory.validateCursor(containerClient, cursor!); + // startTime passed in is ignored + startTime = new Date(cursor!.currentSegmentCursor.segmentTime); + if (cursor!.endTime) { + endTime = new Date(cursor!.endTime!); + } + } + // Round start and end time if we are not using the cursor. + else { + startTime = floorToNearestHour(startTime); + endTime = ceilToNearestHour(endTime); + } + + // Check if Change Feed has been enabled for this account. + let changeFeedContainerExists = await containerClient.exists(); + if (!changeFeedContainerExists) { + throw new Error("Change Feed hasn't been enabled on this account, or is currently being enabled."); + } + + // Get last consumable. + const blobClient = containerClient.getBlobClient(CHANGE_FEED_META_SEGMENT_PATH); + const blobDownloadRes = await blobClient.download(); + const lastConsumable = new Date((JSON.parse(await bodyToString(blobDownloadRes)) as MetaSegments).lastConsumable); + + // Get year paths + const years: number[] = await getYearsPaths(containerClient); + + // Dequeue any years that occur before start time. + if (startTime) { + let startYear = startTime.getUTCFullYear(); + while (years.length > 0 && years[0] < startYear) { + years.shift(); + } + } + if (years.length === 0) { + return new ChangeFeed(); + } + + let segments: string[] = []; + while (segments.length === 0 && years.length !== 0) { + segments = await getSegmentsInYear( + containerClient, + years.shift()!, + startTime, + minDate(lastConsumable, endTime)); + } + if (segments.length === 0) { + return new ChangeFeed(); + } + const currentSegment: Segment = await this._segmentFactory.buildSegment( + containerClient, + segments.shift()!, + cursor?.currentSegmentCursor); + + return new ChangeFeed( + containerClient, + this._segmentFactory, + years, + segments, + currentSegment, + lastConsumable, + startTime, + endTime); + } + + private static validateCursor( + containerClient: ContainerClient, + cursor: ChangeFeedCursor + ): void { + if (hashString(getURLPath(containerClient.url)!) !== cursor.urlHash) { + throw new Error("Cursor URL does not match container URL."); + } + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/Chunk.ts b/sdk/storage/storage-blob-changefeed/src/Chunk.ts new file mode 100644 index 000000000000..467a8cf7a786 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/Chunk.ts @@ -0,0 +1,48 @@ +import { AvroReader } from '../../storage-internal-avro/src'; +import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; + +export class Chunk { + private readonly _avroReader: AvroReader; + private readonly _iter: AsyncIterableIterator; + + private _blockOffset: number; + public get blockOffset(): number { + return this._blockOffset; + } + + private _eventIndex: number; + public get eventIndex(): number { + return this._eventIndex; + } + + constructor( + avroReader: AvroReader, + blockOffset: number, + eventIndex: number + ) { + this._avroReader = avroReader; + this._blockOffset = blockOffset; + this._eventIndex = eventIndex; + + this._iter = this._avroReader.parseObjects(); + } + + public hasNext(): boolean { + return this._avroReader.hasNext(); + } + + public async getChange(): Promise { + if (!this.hasNext()) { + return undefined; + } + + const next = await this._iter.next(); + this._eventIndex = this._avroReader.objectIndex; + this._blockOffset = this._avroReader.blockOffset; + if (next.done) { + return undefined; + } else { + return next.value as BlobChangeFeedEvent; + } + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts b/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts new file mode 100644 index 000000000000..216f8a74849c --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts @@ -0,0 +1,39 @@ +import { AvroReaderFactory } from "./AvroReaderFactory"; +import { ContainerClient } from '@azure/storage-blob'; +import { Chunk } from "./Chunk"; +import { AvroReader } from "../../storage-internal-avro/src" +import { bodyToAvroReadable } from "./utils/utils.node"; + + +export class ChunkFactory { + private readonly _avroReaderFactory: AvroReaderFactory; + + constructor(avroReaderFactory: AvroReaderFactory) { + this._avroReaderFactory = avroReaderFactory; + } + + public async buildChunk( + containerClient: ContainerClient, + chunkPath: string, + blockOffset?: number, + eventIndex?: number + ): Promise { + const blobClient = containerClient.getBlobClient(chunkPath); + blockOffset = blockOffset || 0; + eventIndex = eventIndex || 0; + + const downloadRes = await blobClient.download(blockOffset); + + const dataStream = bodyToAvroReadable(downloadRes); + let avroReader: AvroReader; + if (blockOffset !== 0) { + const headerDownloadRes = await blobClient.download(0); + const headerStream = bodyToAvroReadable(headerDownloadRes); + avroReader = this._avroReaderFactory.buildAvroReader(dataStream, headerStream, blockOffset, eventIndex); + } else { + avroReader = this._avroReaderFactory.buildAvroReader(dataStream); + } + + return new Chunk(avroReader, blockOffset, eventIndex); + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/Segment.ts b/sdk/storage/storage-blob-changefeed/src/Segment.ts new file mode 100644 index 000000000000..bc6d11b08d32 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/Segment.ts @@ -0,0 +1,83 @@ +import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; +import { Shard } from './Shard'; +import { SegmentCursor, ShardCursor } from './models/ChangeFeedCursor'; + +export class Segment { + private readonly _shards: Shard[]; + + // Track shards that we have finished reading from. + private _shardDone: boolean[]; + private _shardDoneCount: number; + + private _shardIndex: number; + + private _finalized: boolean; + public get finalized(): boolean { + return this._finalized; + } + + // Assuming the dateTime of segments is rounded to hour. If not, our logic for fetching + // change events between a time range would be incorrect. + private _dateTime: Date; + public get dateTime(): Date { + return this._dateTime; + } + + constructor( + shards: Shard[], + shardIndex: number, + dateTime: Date, + finalized: boolean + ) { + this._shards = shards; + this._shardIndex = shardIndex; + this._dateTime = dateTime; + this._finalized = finalized; + + // TODO: add polyfill for Array.prototype.fill for IE11 + this._shardDone = Array(shards.length).fill(false); + this._shardDoneCount = 0; + } + + public hasNext(): boolean { + return this._shards.length > this._shardDoneCount; + } + + public async getChange(): Promise { + if (this._shardIndex >= this._shards.length || this._shardIndex < 0) { + throw new Error("shardIndex invalid."); + } + + let event: BlobChangeFeedEvent | undefined = undefined; + while (!event && this.hasNext()) { + if (this._shardDone[this._shardIndex]) { + this._shardIndex = (this._shardIndex + 1) % this._shards.length; // find next available shard + continue; + } + + const currentShard = this._shards[this._shardIndex]; + event = await currentShard.getChange(); + + if (!currentShard.hasNext()) { + this._shardDone[this._shardIndex] = true; + this._shardDoneCount++; + } + // Round robin with shards + this._shardIndex = (this._shardIndex + 1) % this._shards.length; + } + return event; + } + + public getCursor(): SegmentCursor { + let shardCursors: ShardCursor[] = []; + for (const shard of this._shards) { + shardCursors.push(shard.getCursor()); + } + + return { + shardCursors, + shardIndex: this._shardIndex, + segmentTime: this._dateTime.toJSON() + } + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts b/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts new file mode 100644 index 000000000000..bcf64634c44e --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts @@ -0,0 +1,55 @@ +import { ShardFactory } from "./ShardFactory"; +import { ContainerClient } from "@azure/storage-blob"; +import { CHANGE_FEED_STATUS_FINALIZED, CHANGE_FEED_CONTAINER_NAME } from './utils/constants'; +import { Shard } from './Shard'; +import { Segment } from './Segment'; +import { SegmentCursor } from './models/ChangeFeedCursor'; +import { bodyToString } from "./utils/utils.node"; +import { parseDateFromSegmentPath } from "./utils/utils.common"; + +export interface SegmentManifest { + version?: number; + begin?: Date; + intervalSecs?: number; + status: string; + config?: any; + chunkFilePaths: string[]; +} + +export class SegmentFactory { + private readonly _shardFactory?: ShardFactory; + + // FIXME: remove this empty constructor is not used. + constructor(); + constructor(shardFactory: ShardFactory); + constructor(shardFactory?: ShardFactory) { + this._shardFactory = shardFactory; + } + + public async buildSegment(containerClient: ContainerClient, + manifestPath: string, + cursor?: SegmentCursor + ): Promise { + let shards: Shard[] = []; + const dateTime: Date = parseDateFromSegmentPath(manifestPath); + const shardIndex = cursor?.shardIndex || 0; + + const blobClient = containerClient.getBlobClient(manifestPath); + const blobDownloadRes = await blobClient.download(); + const blobContent: string = await bodyToString(blobDownloadRes); + + const segmentManifest = JSON.parse(blobContent) as SegmentManifest; + const finalized = segmentManifest.status === CHANGE_FEED_STATUS_FINALIZED; + + if (finalized) { + let i = 0; + const containerPrefixLength = CHANGE_FEED_CONTAINER_NAME.length + 1; // "$blobchangefeed/" + for (const shardPath of segmentManifest.chunkFilePaths) { + const shard: Shard = await this._shardFactory!.buildShard(containerClient, shardPath.substring(containerPrefixLength), cursor?.shardCursors[i++]); + shards.push(shard); + } + } + + return new Segment(shards, shardIndex, dateTime, finalized); + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/Shard.ts b/sdk/storage/storage-blob-changefeed/src/Shard.ts new file mode 100644 index 000000000000..b53aaacc6f49 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/Shard.ts @@ -0,0 +1,59 @@ +import { ContainerClient } from "@azure/storage-blob"; +import { ChunkFactory } from "./ChunkFactory"; +import { Chunk } from "./Chunk"; +import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; +import { ShardCursor } from "./models/ChangeFeedCursor"; + +export class Shard { + private readonly _containerClient: ContainerClient; + + private readonly _chunkFactory: ChunkFactory; + + private readonly _chunks: string[]; + + private _currentChunk: Chunk; + + private _chunkIndex: number; + + constructor( + containerClient: ContainerClient, + chunkFactory: ChunkFactory, + chunks: string[], + currentChunk: Chunk, + chunkIndex: number) { + this._containerClient = containerClient; + this._chunkFactory = chunkFactory; + this._chunks = chunks; + this._currentChunk = currentChunk; + this._chunkIndex = chunkIndex; + } + + public hasNext(): boolean { + return this._chunks.length > 0 || this._currentChunk.hasNext(); + } + + public async getChange(): Promise { + let event: BlobChangeFeedEvent | undefined = undefined; + while (!event && this.hasNext()) { + event = await this._currentChunk.getChange(); + + // Remove currentChunk if it doesn't have more events. + if (!this._currentChunk.hasNext() && this._chunks.length > 0) { + this._currentChunk = await this._chunkFactory.buildChunk( + this._containerClient, + this._chunks.shift()! + ); + this._chunkIndex++; + } + } + return event; + } + + public getCursor(): ShardCursor { + return { + chunkIndex: this._chunkIndex, + blockOffset: this._currentChunk.blockOffset, + eventIndex: this._currentChunk.eventIndex + }; + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts b/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts new file mode 100644 index 000000000000..0cbc1155536d --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts @@ -0,0 +1,47 @@ +import { ChunkFactory } from './ChunkFactory'; +import { ShardCursor } from './models/ChangeFeedCursor'; +import { Shard } from "./Shard"; +import { ContainerClient } from "@azure/storage-blob"; + +export class ShardFactory { + private readonly _chunkFactory: ChunkFactory; + + constructor(chunkFactory: ChunkFactory) { + this._chunkFactory = chunkFactory; + } + + public async buildShard( + containerClient: ContainerClient, + shardPath: string, + shardCursor?: ShardCursor + ) { + let chunks: string[] = []; + const chunkIndex: number = shardCursor?.chunkIndex || 0; + const blockOffset: number = shardCursor?.blockOffset || 0; + const eventIndex: number = shardCursor?.eventIndex || 0; + + for await (const blobItem of containerClient.listBlobsFlat({ prefix: shardPath })) { + chunks.push(blobItem.name); + } + + if (chunks.length === 0) { + throw new Error(`No chunk under directory ${shardPath}.`); + } + + if (chunkIndex < 0 || chunkIndex >= chunks.length) { + throw new Error(`Invalid chunkIndex for ${shardPath}.`); + } + + // Fast forward to current Chunk. + if (chunkIndex > 0) { + chunks.splice(0, chunkIndex); + } + + const currentChunk = await this._chunkFactory.buildChunk( + containerClient, + chunks.shift()!, + blockOffset, + eventIndex); + return new Shard(containerClient, this._chunkFactory, chunks, currentChunk, chunkIndex); + } +} diff --git a/sdk/storage/storage-blob-changefeed/src/index.ts b/sdk/storage/storage-blob-changefeed/src/index.ts new file mode 100644 index 000000000000..efab53b3d462 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/index.ts @@ -0,0 +1,2 @@ +export * from './BlobChangeFeedClient'; +export * from './models/BlobChangeFeedEvent'; diff --git a/sdk/storage/storage-blob-changefeed/src/log.ts b/sdk/storage/storage-blob-changefeed/src/log.ts new file mode 100644 index 000000000000..11eb733d3095 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/log.ts @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createClientLogger } from "@azure/logger"; + +/** + * The @azure/logger configuration for this package. + */ +export const logger = createClientLogger("storage-blob-changefeed"); diff --git a/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts new file mode 100644 index 000000000000..3fd67f76ca06 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts @@ -0,0 +1,32 @@ +export type BlobChangeFeedEventType = "BlobCreate" | "BlobDeleted"; + +export interface BlobChangeFeedEvent { + topic: string; + subject: string; + eventType: BlobChangeFeedEventType; + eventTime: string; + id: string; // GUID + data: BlobChangeFeedEventData; + dataVersion?: string; + metadataVersion: string; +} + + +export type BlobType = "BlockBlob" | "AppendBlob" | "PageBlob"; + +export interface BlobChangeFeedEventData { + api: string; + clientRequestId: string; // GUID + requestId: string; // GUID + eTag: string; + contentType: string; + contentLength: number; + blobType: BlobType; + url: string; + sequencer: string; + + // For HNS only. + destinationUrl?: string; + sourceUrl?: string; + recursive?: string; +} diff --git a/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts b/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts new file mode 100644 index 000000000000..3ec4a626c9d7 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts @@ -0,0 +1,20 @@ + +export interface ChangeFeedCursor { + urlHash: number; + endTime?: string; + currentSegmentCursor: SegmentCursor; +} + + +export interface SegmentCursor { + shardCursors: ShardCursor[]; + shardIndex: number; + segmentTime: string; +} + + +export interface ShardCursor { + chunkIndex: number; + blockOffset: number; + eventIndex: number; +} diff --git a/sdk/storage/storage-blob-changefeed/src/utils/constants.ts b/sdk/storage/storage-blob-changefeed/src/utils/constants.ts new file mode 100644 index 000000000000..9838de65066c --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/utils/constants.ts @@ -0,0 +1,6 @@ +export const CHANGE_FEED_CONTAINER_NAME: string = "$blobchangefeed"; +export const CHANGE_FEED_META_SEGMENT_PATH: string = "meta/segments.json"; +export const CHANGE_FEED_DEFAULT_PAGE_SIZE: number = 5000; // align with rest API list operations +export const CHANGE_FEED_STATUS_FINALIZED: string = "Finalized"; +export const CHANGE_FEED_SEGMENT_PREFIX: string = "idx/segments/"; +export const CHANGE_FEED_INITIALIZATION_SEGMENT: string = "1601"; diff --git a/sdk/storage/storage-blob-changefeed/src/utils/utils.browser.ts b/sdk/storage/storage-blob-changefeed/src/utils/utils.browser.ts new file mode 100644 index 000000000000..0f5d38eedfe8 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/utils/utils.browser.ts @@ -0,0 +1,31 @@ +/** + * Read body from downloading operation methods to string. + * Work on both Node.js and browser environment. + * + * @param response Convenience layer methods response with downloaded body + * @param length Length of Readable stream, needed for Node.js environment + */ +export async function bodyToString( + response: { + readableStreamBody?: NodeJS.ReadableStream; + blobBody?: Promise; + }, + // tslint:disable-next-line:variable-name + _length?: number +): Promise { + const blob = await response.blobBody!; + return blobToString(blob); +} + +export async function blobToString(blob: Blob): Promise { + const fileReader = new FileReader(); + return new Promise((resolve, reject) => { + fileReader.onloadend = (ev: any) => { + resolve(ev.target!.result); + }; + fileReader.onerror = reject; + fileReader.readAsText(blob); + }); +} + +export function bodyToAvroReadable() { } diff --git a/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts b/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts new file mode 100644 index 000000000000..996fd5a529b7 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts @@ -0,0 +1,94 @@ +import { URLBuilder } from "@azure/core-http"; +import { ContainerClient } from "@azure/storage-blob"; +import { CHANGE_FEED_SEGMENT_PREFIX, CHANGE_FEED_INITIALIZATION_SEGMENT } from "./constants"; + +const millisecondsInAnHour = 60 * 60 * 1000; +export function ceilToNearestHour(date: Date | undefined): Date | undefined { + if (date === undefined) { + return undefined; + } + return new Date(Math.ceil(date.getTime() / millisecondsInAnHour) * millisecondsInAnHour); +} + +export function floorToNearestHour(date: Date | undefined): Date | undefined { + if (date === undefined) { + return undefined; + } + return new Date(Math.floor(date.getTime() / millisecondsInAnHour) * millisecondsInAnHour); +} + +/** + * Get URL path from an URL string. + * + * @export + * @param {string} url Source URL string + * @returns {(string | undefined)} + */ +export function getURLPath(url: string): string | undefined { + const urlParsed = URLBuilder.parse(url); + return urlParsed.getPath(); +} + +// s[0]*31^(n - 1) + s[1]*31^(n - 2) + ... + s[n - 1] +export function hashString(str: string): number { + let hash = 0; + for (let i = 0; i < str.length; i++) { + hash = ((hash << 5) - hash) + str.charCodeAt(i); + hash |= 0;; // Bit operation converts operands to 32-bit integers + } + return hash; +} + +export async function getYearsPaths(containerClient: ContainerClient): Promise { + let years: number[] = []; + for await (const item of containerClient.listBlobsByHierarchy("/", { prefix: CHANGE_FEED_SEGMENT_PREFIX })) { + // TODO: add String.prototype.includes polyfill for IE11 + if (item.kind === "prefix" && !item.name.includes(CHANGE_FEED_INITIALIZATION_SEGMENT)) { + let yearStr = item.name.slice(CHANGE_FEED_SEGMENT_PREFIX.length, -1); + years.push(parseInt(yearStr)); + } + } + return years.sort((a, b) => a - b); +} + +export async function getSegmentsInYear(containerClient: ContainerClient, year: number, startTime?: Date, endTime?: Date): Promise { + let segments: string[] = []; + const prefix = `${CHANGE_FEED_SEGMENT_PREFIX}${year}/` + for await (const item of containerClient.listBlobsFlat({ prefix })) { + const segmentTime = parseDateFromSegmentPath(item.name); + if (startTime && segmentTime < startTime + || endTime && segmentTime >= endTime) { + continue; + } + segments.push(item.name); + } + return segments; +} + +export function parseDateFromSegmentPath(segmentPath: string): Date { + const splitPath = segmentPath.split('/'); + if (splitPath.length < 3) { + throw new Error(`${segmentPath} is not a valid segment path.`); + } + + let segmentTime = new Date(0); + segmentTime.setUTCFullYear(parseInt(splitPath[2])); + + if (splitPath.length >= 4) { + segmentTime.setUTCMonth(parseInt(splitPath[3]) - 1); + } + if (splitPath.length >= 5) { + segmentTime.setUTCDate(parseInt(splitPath[4])); + } + if (splitPath.length >= 6) { + segmentTime.setUTCHours(parseInt(splitPath[5]) / 100); + } + return segmentTime; +} + +export function minDate(dateA: Date, dateB?: Date): Date { + if (dateB && dateB < dateA) { + return dateB; + } + return dateA; +} diff --git a/sdk/storage/storage-blob-changefeed/src/utils/utils.node.ts b/sdk/storage/storage-blob-changefeed/src/utils/utils.node.ts new file mode 100644 index 000000000000..f0d33bf0446c --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/src/utils/utils.node.ts @@ -0,0 +1,41 @@ +import { AvroReadable, AvroReadableFromStream } from '../../../storage-internal-avro/src'; + +/** + * Read body from downloading operation methods to string. + * Work on both Node.js and browser environment. + * + * @param response Convenience layer methods response with downloaded body + * @param length Length of Readable stream, needed for Node.js environment + */ +export async function bodyToString( + response: { + readableStreamBody?: NodeJS.ReadableStream; + blobBody?: Promise; + }, + length?: number +): Promise { + return new Promise((resolve, reject) => { + response.readableStreamBody!.on("readable", () => { + let chunk; + chunk = response.readableStreamBody!.read(length); + if (chunk) { + resolve(chunk.toString()); + } + }); + + response.readableStreamBody!.on("error", reject); + response.readableStreamBody!.on("end", () => { + resolve(""); + }); + }); +} + + +export function bodyToAvroReadable( + response: { + readableStreamBody?: NodeJS.ReadableStream; + blobBody?: Promise; + } +): AvroReadable { + return new AvroReadableFromStream(response.readableStreamBody!); +} diff --git a/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts new file mode 100644 index 000000000000..95a99fd83230 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts @@ -0,0 +1,132 @@ +import * as assert from "assert"; +import { BlobServiceClient, StorageSharedKeyCredential } from "@azure/storage-blob"; +import { BlobChangeFeedClient, BlobChangeFeedEvent, BlobChangeFeedEventPage } from "../src"; + +import * as dotenv from "dotenv"; +dotenv.config(); + +// import { setLogLevel } from "@azure/logger"; +// setLogLevel("info"); + +describe("BlobChangeFeedClient", async () => { + const account = process.env.ACCOUNT_NAME || ""; + const accountKey = process.env.ACCOUNT_KEY || ""; + const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey); + const blobServiceClient = new BlobServiceClient( + `https://${account}.blob.core.windows.net`, + sharedKeyCredential + ); + const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); + + before(async function () { + if (process.env.CHANGE_FEED_ENABLED !== "1") { + this.skip(); + } + }); + + it("next(): fetch all events", async () => { + let i = 0; + for await (const event of changeFeedClient.getChanges()) { + if (i++ === 0) { + assert.ok(event.eventType); + assert.ok(event.data.blobType); + } + } + }); + + it("next(): with start and end time", async () => { + let i = 0; + let lastEvent: BlobChangeFeedEvent | undefined; + const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be rounded to 22:00 + const startRounded = new Date(Date.UTC(2020, 1, 21, 22, 0, 0)); + const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be rounded to 22:00 + const endRounded = new Date(Date.UTC(2020, 4, 8, 22, 0, 0)); + for await (const event of changeFeedClient.getChanges({ start, end })) { + if (i++ === 0) { + assert.ok(event.eventType); + assert.ok(event.data.blobType); + assert.ok(new Date(event.eventTime) >= startRounded); + } + lastEvent = event; + } + + if (lastEvent) { + assert.ok(new Date(lastEvent.eventTime) < endRounded); + } + }); + + it("byPage()", async () => { + const maxPageSize = 2 + const iter = changeFeedClient.getChanges().byPage({ maxPageSize }); + const nextPage = await iter.next(); + if (nextPage.done) { + return; + } + assert.equal(nextPage.value.events.length, maxPageSize); + const event = nextPage.value.events[0]; + assert.ok(event.eventType); + assert.ok(event.data.blobType); + + // continuationToken + const iter1 = changeFeedClient.getChanges().byPage({ continuationToken: nextPage.value.continuationToken, maxPageSize }); + const nextPage1 = await iter1.next(); + if (nextPage1.done) { + return; + } + assert.equal(nextPage1.value.events.length, maxPageSize); + const event1 = nextPage1.value.events[0]; + assert.ok(event1.eventType); + assert.ok(event1.data.blobType); + assert.notEqual(event1.id, event.id); + + // fetch between time range + const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be ignored + const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be rounded to 22:00 + const endRounded = new Date(Date.UTC(2020, 4, 8, 22, 0, 0)); + const iter2 = changeFeedClient.getChanges({ start, end }).byPage({ continuationToken: nextPage1.value.continuationToken }); + let i = 0; + let lastEventPage: BlobChangeFeedEventPage | undefined; + for await (const eventPage of iter2) { + if (i++ === 0) { + const firstEvent = eventPage.events[0]; + assert.ok(firstEvent.eventType); + assert.ok(firstEvent.data.blobType); + assert.notEqual(firstEvent.id, event.id); + } + lastEventPage = eventPage; + } + + if (lastEventPage) { + const lastEvent = lastEventPage.events[lastEventPage.events.length - 1]; + assert.ok(new Date(lastEvent.eventTime) < endRounded); + } + }); +}); + + +describe("BlobChangeFeedClient: Change Feed not configured", async () => { + const account = process.env.ACCOUNT_NAME || ""; + const accountKey = process.env.ACCOUNT_KEY || ""; + const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey); + const blobServiceClient = new BlobServiceClient( + `https://${account}.blob.core.windows.net`, + sharedKeyCredential + ); + const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); + + before(async function () { + if (process.env.CHANGE_FEED_ENABLED === "1") { + this.skip(); + } + }); + + it("should throw when fetching changes", async () => { + let exceptionCaught = false; + try { + await changeFeedClient.getChanges().next(); + } catch (err) { + exceptionCaught = true; + } + assert.ok(exceptionCaught); + }); +}); diff --git a/sdk/storage/storage-blob-changefeed/tsconfig.json b/sdk/storage/storage-blob-changefeed/tsconfig.json new file mode 100644 index 000000000000..f89af540f58e --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "alwaysStrict": true, + "noImplicitAny": true, + "preserveConstEnums": true, + "sourceMap": true, + "inlineSources": true, + "newLine": "LF", + "target": "es5", + "moduleResolution": "node", + "noUnusedLocals": true, + "noUnusedParameters": true, + "strict": true, + "module": "esNext", + "outDir": "./dist-esm", + "declaration": true, + "declarationMap": true, + "importHelpers": true, + "declarationDir": "./typings/latest", + "lib": ["dom", "es5", "es6", "es7", "esnext"], + "esModuleInterop": true + }, + "compileOnSave": true, + "exclude": ["node_modules", "../storage-internal-avro/node_modules", "./samples/**"], + "include": ["./src/**/*.ts", "./test/**/*.ts", "../storage-internal-avro/**/*.ts"] +} diff --git a/sdk/storage/storage-blob/src/generated/src/models/parameters.ts b/sdk/storage/storage-blob/src/generated/src/models/parameters.ts index c985e05682ff..9445287c3509 100644 --- a/sdk/storage/storage-blob/src/generated/src/models/parameters.ts +++ b/sdk/storage/storage-blob/src/generated/src/models/parameters.ts @@ -1680,7 +1680,7 @@ export const version: coreHttp.OperationParameter = { required: true, isConstant: true, serializedName: "x-ms-version", - defaultValue: '2019-12-12', + defaultValue: '2019-10-10', type: { name: "String" } diff --git a/sdk/storage/storage-blob/src/generated/src/storageClientContext.ts b/sdk/storage/storage-blob/src/generated/src/storageClientContext.ts index 31b47553f19f..e01d6d8f9982 100644 --- a/sdk/storage/storage-blob/src/generated/src/storageClientContext.ts +++ b/sdk/storage/storage-blob/src/generated/src/storageClientContext.ts @@ -39,7 +39,7 @@ export class StorageClientContext extends coreHttp.ServiceClient { super(undefined, options); - this.version = '2019-12-12'; + this.version = '2019-10-10'; this.baseUri = "{url}"; this.requestContentType = "application/json; charset=utf-8"; this.url = url; diff --git a/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts index ab8dfea45a49..ac5412cacb47 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts @@ -29,6 +29,7 @@ export class AvroReadableFromStream extends AvroReadable { return this._position; } public async read(size: number): Promise { + // console.log(`reading stream for size ${size} at position ${this._position}`); if (size < 0) { throw new Error(`size parameter should be positive: ${size}`); } From 21be39dce59752e1326ed25977f1343b99e0bb43 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Fri, 5 Jun 2020 20:12:42 +0800 Subject: [PATCH 03/17] mocking tests --- .../storage-blob-changefeed/package.json | 4 +- .../storage-blob-changefeed/src/ChangeFeed.ts | 13 +- .../src/ChangeFeedFactory.ts | 4 + .../storage-blob-changefeed/src/Segment.ts | 2 +- .../src/SegmentFactory.ts | 10 +- .../storage-blob-changefeed/src/Shard.ts | 2 +- .../src/models/ChangeFeedCursor.ts | 1 + .../src/utils/utils.common.ts | 11 +- .../test/blobchangefeedclient.spec.ts | 3 - .../test/changefeed.spec.ts | 223 ++++++++++++++++++ .../test/chunk.spec.ts | 66 ++++++ .../test/resources/ChangeFeedManifest.json | 12 + .../test/resources/SegmentManifest.json | 26 ++ .../test/segment.spec.ts | 90 +++++++ .../test/shard.spec.ts | 75 ++++++ 15 files changed, 519 insertions(+), 23 deletions(-) create mode 100644 sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts create mode 100644 sdk/storage/storage-blob-changefeed/test/chunk.spec.ts create mode 100644 sdk/storage/storage-blob-changefeed/test/resources/ChangeFeedManifest.json create mode 100644 sdk/storage/storage-blob-changefeed/test/resources/SegmentManifest.json create mode 100644 sdk/storage/storage-blob-changefeed/test/segment.spec.ts create mode 100644 sdk/storage/storage-blob-changefeed/test/shard.spec.ts diff --git a/sdk/storage/storage-blob-changefeed/package.json b/sdk/storage/storage-blob-changefeed/package.json index dbe58ad04fff..da23bd4f0ae4 100644 --- a/sdk/storage/storage-blob-changefeed/package.json +++ b/sdk/storage/storage-blob-changefeed/package.json @@ -113,6 +113,7 @@ "@rollup/plugin-replace": "^2.2.0", "@types/mocha": "^7.0.2", "@types/node": "^8.0.0", + "@types/sinon": "^7.0.13", "@typescript-eslint/eslint-plugin": "^2.0.0", "@typescript-eslint/parser": "^2.0.0", "assert": "^1.4.1", @@ -156,6 +157,7 @@ "source-map-support": "^0.5.9", "ts-node": "^8.3.0", "typescript": "~3.8.3", - "util": "^0.12.1" + "util": "^0.12.1", + "sinon": "^7.1.0" } } \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts b/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts index 88dc075c88ee..69ef7dcea89d 100644 --- a/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts +++ b/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts @@ -3,7 +3,7 @@ import { Segment } from './Segment'; import { SegmentFactory } from './SegmentFactory'; import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; import { ChangeFeedCursor } from "./models/ChangeFeedCursor"; -import { getURLPath, hashString, getSegmentsInYear, minDate } from "./utils/utils.common"; +import { getURI, hashString, getSegmentsInYear, minDate } from "./utils/utils.common"; export class ChangeFeed { /** @@ -78,12 +78,8 @@ export class ChangeFeed { } public async getChange(): Promise { - if (!this.hasNext()) { - throw new Error("Change feed doesn't have any more events"); - } - let event: BlobChangeFeedEvent | undefined = undefined; - while (!event && this.hasNext()) { + while (event === undefined && this.hasNext()) { event = await this._currentSegment!.getChange(); await this.advanceSegmentIfNecessary(); } @@ -96,7 +92,8 @@ export class ChangeFeed { } return { - urlHash: hashString(getURLPath(this._containerClient!.url)!), + cursorVersion: 1, + urlHash: hashString(getURI(this._containerClient!.url)!), endTime: this._endTime?.toJSON(), currentSegmentCursor: this._currentSegment!.getCursor() }; @@ -124,7 +121,7 @@ export class ChangeFeed { if (this._segments.length > 0) { this._currentSegment = await this._segmentFactory!.buildSegment(this._containerClient!, this._segments.shift()!); } else { - throw new Error(`Year ${year} in the middle should have returned valid segments.`); + this._currentSegment = undefined; } } } diff --git a/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts index 00e6a2b2587e..7f39531776c2 100644 --- a/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts @@ -75,6 +75,10 @@ export class ChangeFeedFactory { throw new Error("Change Feed hasn't been enabled on this account, or is currently being enabled."); } + if (startTime && endTime && startTime >= endTime) { + return new ChangeFeed(); + } + // Get last consumable. const blobClient = containerClient.getBlobClient(CHANGE_FEED_META_SEGMENT_PATH); const blobDownloadRes = await blobClient.download(); diff --git a/sdk/storage/storage-blob-changefeed/src/Segment.ts b/sdk/storage/storage-blob-changefeed/src/Segment.ts index bc6d11b08d32..fc1cbad689c2 100644 --- a/sdk/storage/storage-blob-changefeed/src/Segment.ts +++ b/sdk/storage/storage-blob-changefeed/src/Segment.ts @@ -49,7 +49,7 @@ export class Segment { } let event: BlobChangeFeedEvent | undefined = undefined; - while (!event && this.hasNext()) { + while (event === undefined && this.hasNext()) { if (this._shardDone[this._shardIndex]) { this._shardIndex = (this._shardIndex + 1) % this._shards.length; // find next available shard continue; diff --git a/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts b/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts index bcf64634c44e..f9a25450abb8 100644 --- a/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts @@ -17,12 +17,9 @@ export interface SegmentManifest { } export class SegmentFactory { - private readonly _shardFactory?: ShardFactory; + private readonly _shardFactory: ShardFactory; - // FIXME: remove this empty constructor is not used. - constructor(); - constructor(shardFactory: ShardFactory); - constructor(shardFactory?: ShardFactory) { + constructor(shardFactory: ShardFactory) { this._shardFactory = shardFactory; } @@ -43,9 +40,10 @@ export class SegmentFactory { if (finalized) { let i = 0; + const containerPrefixLength = CHANGE_FEED_CONTAINER_NAME.length + 1; // "$blobchangefeed/" for (const shardPath of segmentManifest.chunkFilePaths) { - const shard: Shard = await this._shardFactory!.buildShard(containerClient, shardPath.substring(containerPrefixLength), cursor?.shardCursors[i++]); + const shard: Shard = await this._shardFactory.buildShard(containerClient, shardPath.substring(containerPrefixLength), cursor?.shardCursors[i++]); shards.push(shard); } } diff --git a/sdk/storage/storage-blob-changefeed/src/Shard.ts b/sdk/storage/storage-blob-changefeed/src/Shard.ts index b53aaacc6f49..c37d09a4827f 100644 --- a/sdk/storage/storage-blob-changefeed/src/Shard.ts +++ b/sdk/storage/storage-blob-changefeed/src/Shard.ts @@ -34,7 +34,7 @@ export class Shard { public async getChange(): Promise { let event: BlobChangeFeedEvent | undefined = undefined; - while (!event && this.hasNext()) { + while (event === undefined && this.hasNext()) { event = await this._currentChunk.getChange(); // Remove currentChunk if it doesn't have more events. diff --git a/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts b/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts index 3ec4a626c9d7..7e424af4ff13 100644 --- a/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts +++ b/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts @@ -1,5 +1,6 @@ export interface ChangeFeedCursor { + cursorVersion: number; urlHash: number; endTime?: string; currentSegmentCursor: SegmentCursor; diff --git a/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts b/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts index 996fd5a529b7..3fc836404f12 100644 --- a/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts +++ b/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts @@ -18,15 +18,15 @@ export function floorToNearestHour(date: Date | undefined): Date | undefined { } /** - * Get URL path from an URL string. + * Get URI from an URL string. * * @export * @param {string} url Source URL string * @returns {(string | undefined)} */ -export function getURLPath(url: string): string | undefined { +export function getURI(url: string): string | undefined { const urlParsed = URLBuilder.parse(url); - return urlParsed.getPath(); + return `${urlParsed.getHost()}${urlParsed.getPort()}${urlParsed.getPath()}`; } // s[0]*31^(n - 1) + s[1]*31^(n - 2) + ... + s[n - 1] @@ -53,6 +53,11 @@ export async function getYearsPaths(containerClient: ContainerClient): Promise { let segments: string[] = []; + const yearBeginTime = new Date(Date.UTC(year, 0)); + if (endTime && yearBeginTime >= endTime) { + return segments; + } + const prefix = `${CHANGE_FEED_SEGMENT_PREFIX}${year}/` for await (const item of containerClient.listBlobsFlat({ prefix })) { const segmentTime = parseDateFromSegmentPath(item.name); diff --git a/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts index 95a99fd83230..c6615db3b256 100644 --- a/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts @@ -5,9 +5,6 @@ import { BlobChangeFeedClient, BlobChangeFeedEvent, BlobChangeFeedEventPage } fr import * as dotenv from "dotenv"; dotenv.config(); -// import { setLogLevel } from "@azure/logger"; -// setLogLevel("info"); - describe("BlobChangeFeedClient", async () => { const account = process.env.ACCOUNT_NAME || ""; const accountKey = process.env.ACCOUNT_KEY || ""; diff --git a/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts b/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts new file mode 100644 index 000000000000..af37c960b79f --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts @@ -0,0 +1,223 @@ +import * as assert from "assert"; +import * as fs from "fs"; +import * as path from "path"; +import * as sinon from "sinon"; +import { BlobServiceClient, ContainerClient, BlobClient } from "@azure/storage-blob"; +import { SegmentFactory } from "../src/SegmentFactory"; +import { Segment } from "../src/Segment"; +import { ChangeFeedFactory } from "../src/ChangeFeedFactory"; +import { hashString, getURI } from "../src/utils/utils.common"; + +describe("Change Feed", async () => { + const manifestFilePath = path.join("test", "resources", "ChangeFeedManifest.json"); + const lastConsumable = new Date("2020-05-04T19:10:00.000Z"); + const segmentCount = 5; + const yearPaths = [ + { kind: "prefix", name: "idx/segments/1601/" }, + { kind: "prefix", name: "idx/segments/2019/" }, + { kind: "prefix", name: "idx/segments/2020/" } + ]; + const segmentsIn2019 = [ + { name: "idx/segments/2019/03/02/2000/meta.json" }, + { name: "idx/segments/2019/04/03/2200/meta.json" }, + { name: "idx/segments/2019/05/03/2200/meta.json" } + ]; + const segmentsIn2020 = [ + { name: "idx/segments/2020/03/02/2000/meta.json" }, + { name: "idx/segments/2020/05/04/1900/meta.json" } + ]; + const segmentTimes = [ + new Date(Date.UTC(2019, 2, 2, 20)), + new Date(Date.UTC(2019, 3, 3, 22)), + new Date(Date.UTC(2019, 4, 3, 22)), + new Date(Date.UTC(2020, 2, 2, 20)), + new Date(Date.UTC(2020, 4, 4, 19)) + ]; + let serviceClientStub: sinon.SinonStubbedInstance; + let segmentFactoryStub: sinon.SinonStubbedInstance; + let containerClientStub: sinon.SinonStubbedInstance; + let segmentStubs: sinon.SinonStubbedInstance[]; + let changeFeedFactory: ChangeFeedFactory; + + async function* fakeList(items: any[]) { + for (const item of items) { + yield item; + } + } + + async function* listTwoArray(itemsA: any[], itemsB: any[]) { + for (const item of itemsA) { + yield item; + } + for (const item of itemsB) { + yield item; + } + } + + beforeEach(async () => { + serviceClientStub = sinon.createStubInstance(BlobServiceClient); + containerClientStub = sinon.createStubInstance(ContainerClient); + const blobClientStub = sinon.createStubInstance(BlobClient); + segmentFactoryStub = sinon.createStubInstance(SegmentFactory); + changeFeedFactory = new ChangeFeedFactory(segmentFactoryStub); + + serviceClientStub.getContainerClient.returns(containerClientStub as any); + containerClientStub.exists.resolves(true); + containerClientStub.getBlobClient.returns(blobClientStub as any); + containerClientStub.listBlobsByHierarchy.withArgs("/").callsFake(() => (fakeList(yearPaths) as any)); + containerClientStub.listBlobsFlat.withArgs({ prefix: "idx/segments/2019/" }).callsFake(() => (fakeList(segmentsIn2019) as any)); + containerClientStub.listBlobsFlat.withArgs({ prefix: "idx/segments/2020/" }).callsFake(() => (fakeList(segmentsIn2020) as any)); + // TODO: rewrite for browser + blobClientStub.download.callsFake(() => { + return new Promise((resolve) => { resolve({ readableStreamBody: fs.createReadStream(manifestFilePath) } as any) }); + }); + + segmentStubs = []; + const segmentIter = listTwoArray(segmentsIn2019, segmentsIn2020); + for (let i = 0; i < segmentCount; i++) { + segmentStubs.push(sinon.createStubInstance(Segment)); + segmentFactoryStub.buildSegment.withArgs(sinon.match.any, (await segmentIter.next()).value.name).resolves(segmentStubs[i] as any); + } + for (let i = 0; i < segmentCount; i++) { + sinon.stub(segmentStubs[i], "dateTime").value(segmentTimes[i]); + sinon.stub(segmentStubs[i], "finalized").value(i < segmentCount - 1); + segmentStubs[i].hasNext.returns(true); + segmentStubs[i].getChange.resolves(i as any); + } + }); + + afterEach(() => { + sinon.restore(); + }); + + it("no valid years in change feed container", async () => { + const yearPaths = [{ kind: "prefix", name: "idx/segments/1601/" }]; + containerClientStub.listBlobsByHierarchy.withArgs("/").returns(fakeList(yearPaths) as any); + const changeFeed = await changeFeedFactory.buildChangeFeed(serviceClientStub as any); + assert.ok(!changeFeed.hasNext()); + }); + + it("no years after start time", async () => { + const yearPaths = [ + { kind: "prefix", name: "idx/segments/1601/" }, + { kind: "prefix", name: "idx/segments/2019/" } + ]; + containerClientStub.listBlobsByHierarchy.withArgs("/").returns(fakeList(yearPaths) as any); + const changeFeed = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, undefined, new Date(Date.UTC(2020, 0))); + assert.ok(!changeFeed.hasNext()); + }); + + it("no segments remaining in start year", async () => { + const yearPaths = [ + { kind: "prefix", name: "idx/segments/1601/" }, + { kind: "prefix", name: "idx/segments/2019/" } + ]; + containerClientStub.listBlobsByHierarchy.withArgs("/").returns(fakeList(yearPaths) as any); + + const segments = [ + { name: "idx/segments/2019/03/02/2000/meta.json" }, + { name: "idx/segments/2019/04/03/2200/meta.json" } + ]; + containerClientStub.listBlobsFlat.returns(fakeList(segments) as any); + + const changeFeed = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, undefined, new Date(Date.UTC(2019, 5))); + assert.ok(!changeFeed.hasNext()); + }); + + it("getChange", async () => { + const changeFeed = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, undefined, new Date(Date.UTC(2019, 0))); + assert.ok(changeFeed.hasNext()); + + const event = await changeFeed.getChange(); + assert.equal(event, 0); + + // advance to next non-empty segment + for (let i = 0; i < 2; i++) { + segmentStubs[i].hasNext.returns(false); + segmentStubs[i].getChange.resolves(undefined); + } + assert.ok(changeFeed.hasNext()); + const event2 = await changeFeed.getChange(); + assert.equal(event2, 2); + + // advanced to next year + segmentStubs[2].hasNext.returns(false); + segmentStubs[2].getChange.resolves(undefined); + assert.ok(changeFeed.hasNext()); + const event3 = await changeFeed.getChange(); + assert.equal(event3, 3); + + // stop when segment not finalized + segmentStubs[3].hasNext.returns(false); + segmentStubs[3].getChange.resolves(undefined); + const event4 = await changeFeed.getChange(); + assert.equal(event4, undefined); + assert.ok(!changeFeed.hasNext()); + }); + + it("with start and end time", async () => { + // no valid segment between start and end + const changeFeed = await changeFeedFactory.buildChangeFeed( + serviceClientStub as any, + undefined, + new Date(Date.UTC(2019, 2, 2, 21)), + new Date(Date.UTC(2019, 3, 3, 22)) + ); + assert.ok(!changeFeed.hasNext()); + + // end earlier than lastConsumable + const changeFeed2 = await changeFeedFactory.buildChangeFeed( + serviceClientStub as any, + undefined, + new Date(Date.UTC(2019, 3, 3, 22)), + new Date(Date.UTC(2019, 4, 3, 22)), + ); + assert.ok(changeFeed2.hasNext()); + const event = await changeFeed2.getChange(); + assert.equal(event, 1); + + segmentStubs[1].hasNext.returns(false); + segmentStubs[1].getChange.resolves(undefined); + const event2 = await changeFeed2.getChange(); + assert.equal(event2, undefined); + + //end later than lastConsumable + const changeFeed3 = await changeFeedFactory.buildChangeFeed( + serviceClientStub as any, + undefined, + lastConsumable, + new Date(lastConsumable.getTime() + 1) + ); + assert.ok(!changeFeed3.hasNext()); + }); + + it("with continuation token", async () => { + const changeFeed = await changeFeedFactory.buildChangeFeed( + serviceClientStub as any, + undefined, + new Date(Date.UTC(2020, 2, 2, 20)) + ); + assert.ok(changeFeed.hasNext()); + + const containerUri = "https://account.blob.core.windows.net/$blobchangefeed"; + (containerClientStub as any).url = containerUri; + const cursor = changeFeed.getCursor(); + assert.deepStrictEqual(cursor.urlHash, hashString(getURI(containerUri)!)); + + segmentStubs[3].getCursor.returns({ shardCursors: [], shardIndex: 0, segmentTime: (new Date(Date.UTC(2020, 2, 2, 20))).toJSON() }); + const continuation = JSON.stringify(changeFeed.getCursor()); + const changeFeed2 = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, continuation); + assert.ok(changeFeed2.hasNext()); + const event = await changeFeed.getChange(); + assert.equal(event, 3); + + // finalized changed + sinon.stub(segmentStubs[4], "finalized").value(true); + segmentStubs[3].hasNext.returns(false); + segmentStubs[3].getChange.resolves(undefined); + const changeFeed3 = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, continuation); + assert.ok(changeFeed3.hasNext()); + const event2 = await changeFeed.getChange(); + assert.equal(event2, 4); + }); +}); diff --git a/sdk/storage/storage-blob-changefeed/test/chunk.spec.ts b/sdk/storage/storage-blob-changefeed/test/chunk.spec.ts new file mode 100644 index 000000000000..279e1b62acac --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/test/chunk.spec.ts @@ -0,0 +1,66 @@ +import * as assert from "assert"; +import { Chunk } from "../src/Chunk"; +import * as sinon from "sinon"; +import { AvroReader } from "../../storage-internal-avro/src"; + +class FakeAvroReader { + constructor( + public blockOffset: number, + public objectIndex: number, + public hasNext: boolean, + private _record: any, + public blockSize?: number + ) { } + + public async *parseObjects(): AsyncIterableIterator { + while (this.hasNext) { + this.blockOffset += this.blockSize || 1000; + this.objectIndex++; + yield this._record; + } + } +} + + +describe("Chunk", async () => { + afterEach(() => { + sinon.restore(); + }); + + it("hasNext()", async () => { + const avroReaderStub = sinon.createStubInstance(AvroReader); + avroReaderStub.hasNext.returns(true); + + const chunk = new Chunk(avroReaderStub as any, 0, 0); + assert.equal(chunk.hasNext(), true); + + avroReaderStub.hasNext.returns(false); + assert.equal(chunk.hasNext(), false); + + }); + + it("getChange", async () => { + // set up + const record = { a: 1 } + const fakeAvroReader = new FakeAvroReader(0, 0, true, record); + const avroReaderStub = sinon.createStubInstance(AvroReader); + avroReaderStub.hasNext.callsFake(() => fakeAvroReader.hasNext); + avroReaderStub.parseObjects.returns(fakeAvroReader.parseObjects()); + sinon.stub(avroReaderStub, "blockOffset").get(() => { return fakeAvroReader.blockOffset }); + sinon.stub(avroReaderStub, "objectIndex").get(() => { return fakeAvroReader.objectIndex }); + + const chunk = new Chunk(avroReaderStub as any, avroReaderStub.blockOffset, avroReaderStub.objectIndex); + + // act and verify + const change = await chunk.getChange(); + assert.deepStrictEqual(change, record); + assert.equal(chunk.blockOffset, avroReaderStub.blockOffset); + assert.equal(chunk.eventIndex, avroReaderStub.objectIndex); + + fakeAvroReader.hasNext = false; + const change2 = await chunk.getChange(); + assert.deepStrictEqual(change2, undefined); + assert.equal(chunk.blockOffset, avroReaderStub.blockOffset); + assert.equal(chunk.eventIndex, avroReaderStub.objectIndex); + }); +}); \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/test/resources/ChangeFeedManifest.json b/sdk/storage/storage-blob-changefeed/test/resources/ChangeFeedManifest.json new file mode 100644 index 000000000000..fcca218d3f25 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/test/resources/ChangeFeedManifest.json @@ -0,0 +1,12 @@ +{ + "version": 0, + "lastConsumable": "2020-05-04T19:10:00.000Z", + "storageDiagnostics": { + "version": 0, + "lastModifiedTime": "2020-05-04T19:25:09.594Z", + "data": { + "aid": "a6b895a0-7006-0041-0049-22cadf06029a", + "lfz": "2020-04-29T06:00:00.000Z" + } + } +} \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/test/resources/SegmentManifest.json b/sdk/storage/storage-blob-changefeed/test/resources/SegmentManifest.json new file mode 100644 index 000000000000..21b93ea966a3 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/test/resources/SegmentManifest.json @@ -0,0 +1,26 @@ +{ + "version": 0, + "begin": "2020-03-25T02:00:00.000Z", + "intervalSecs": 3600, + "status": "Finalized", + "config": { + "version": 0, + "configVersionEtag": "0x8d7d063fb40542c", + "numShards": 1, + "recordsFormat": "avro", + "formatSchemaVersion": 3, + "shardDistFnVersion": 1 + }, + "chunkFilePaths": [ + "$blobchangefeed/log/00/2020/03/25/0200/", + "$blobchangefeed/log/01/2020/03/25/0200/", + "$blobchangefeed/log/02/2020/03/25/0200/" + ], + "storageDiagnostics": { + "version": 0, + "lastModifiedTime": "2020-03-25T02:26:53.186Z", + "data": { + "aid": "61410c64-2006-0001-004c-02cde706e9dc" + } + } +} \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/test/segment.spec.ts b/sdk/storage/storage-blob-changefeed/test/segment.spec.ts new file mode 100644 index 000000000000..185ab44bbea8 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/test/segment.spec.ts @@ -0,0 +1,90 @@ +import * as assert from "assert"; +import * as fs from "fs"; +import * as path from "path"; +import * as sinon from "sinon"; +import { ContainerClient, BlobClient } from "@azure/storage-blob"; +import { Shard } from "../src/Shard"; +import { SegmentFactory } from "../src/SegmentFactory"; +import { ShardFactory } from "../src/ShardFactory"; + +describe("Shard", async () => { + const manifestPath = "idx/segments/2020/03/25/0200/meta.json"; + const dateTime = new Date(Date.UTC(2020, 2, 25, 2)); + const shardCount = 3; + const segmentManifestFilePath = path.join("test", "resources", "SegmentManifest.json"); + let containerClientStub: any; + let shardFactoryStub: any; + let shardStubs: any[]; + + beforeEach(() => { + containerClientStub = sinon.createStubInstance(ContainerClient); + const blobClientStub = sinon.createStubInstance(BlobClient); + containerClientStub.getBlobClient.returns(blobClientStub); + // TODO: rewrite for browser + blobClientStub.download.resolves({ readableStreamBody: fs.createReadStream(segmentManifestFilePath) } as any); + + shardFactoryStub = sinon.createStubInstance(ShardFactory); + shardStubs = []; + for (let i = 0; i < shardCount; i++) { + shardStubs.push(sinon.createStubInstance(Shard)); + shardFactoryStub.buildShard.onCall(i).returns(shardStubs[i]); + + shardStubs[i].hasNext.returns(true); + shardStubs[i].getChange.returns(i); + } + }); + + afterEach(() => { + sinon.restore(); + }); + + it("getChange round robin in shards", async () => { + const segmentFactory = new SegmentFactory(shardFactoryStub); + const segment = await segmentFactory.buildSegment(containerClientStub, manifestPath); + assert.ok(segment.hasNext()); + assert.equal(segment.dateTime.getTime(), dateTime.getTime()); + assert.ok(segment.finalized); + + // round robin + for (let i = 0; i < shardCount * 2 + 1; i++) { + const event = await segment.getChange(); + assert.equal(shardStubs[i % shardCount].getChange.callCount, Math.floor(i / shardCount) + 1); + assert.equal(event, i % shardCount); + } + + // skip finished shard + shardStubs[1].hasNext.returns(false); + shardStubs[1].getChange(undefined); + const event = await segment.getChange(); + assert.equal(event, 1); + + const shardRemainingCount = shardCount - 1; + for (let i = 0; i < shardRemainingCount; i++) { + const event = await segment.getChange(); + assert.equal(event, (i + 2) % shardCount); + } + const event2 = await segment.getChange(); + assert.equal(event2, 2); + + // all shards done, return undefined + for (let i = 0; i < shardCount; i++) { + shardStubs[i].hasNext.returns(false); + shardStubs[i].getChange.returns(undefined); + } + const lastEvent = await segment.getChange(); + assert.deepStrictEqual(lastEvent, undefined); + }); + + it("init with non-zero shardIndex", async () => { + const shardIndex = 1; + const segmentFactory = new SegmentFactory(shardFactoryStub); + const segment = await segmentFactory.buildSegment(containerClientStub, manifestPath, { shardIndex, shardCursors: [] } as any); + assert.ok(segment.hasNext()); + assert.equal(segment.dateTime.getTime(), dateTime.getTime()); + assert.ok(segment.finalized); + assert.equal(segment.getCursor().shardIndex, shardIndex); + + const event = await segment.getChange(); + assert.equal(event, shardIndex); + }); +}); diff --git a/sdk/storage/storage-blob-changefeed/test/shard.spec.ts b/sdk/storage/storage-blob-changefeed/test/shard.spec.ts new file mode 100644 index 000000000000..5e7fffef79cd --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/test/shard.spec.ts @@ -0,0 +1,75 @@ +import * as assert from "assert"; +import * as sinon from "sinon"; +import { ShardFactory } from "../src/ShardFactory"; +import { ContainerClient } from "@azure/storage-blob"; +import { ChunkFactory } from "../src/ChunkFactory"; +import { ShardCursor } from "../src/models/ChangeFeedCursor"; +import { Chunk } from "../src/Chunk"; + +describe("Shard", async () => { + let chunkFactoryStub: any; + let containerClientSub: any; + let chunkStub: any; + + async function* fakeListBlobsFlat(option: { prefix: string }) { + for (let i = 0; i < 5; i++) { + yield { name: `${option.prefix}000${i}.avro` }; + } + } + + beforeEach(() => { + chunkStub = sinon.createStubInstance(Chunk); + containerClientSub = sinon.createStubInstance(ContainerClient); + containerClientSub.listBlobsFlat.callsFake(fakeListBlobsFlat); + chunkFactoryStub = sinon.createStubInstance(ChunkFactory); + chunkFactoryStub.buildChunk.returns(chunkStub); + }); + + afterEach(() => { + sinon.restore(); + }); + + it("build shard with none-zero chunkIndex", async () => { + const shardPath = "$blobchangefeed/log/00/2019/02/22/1810/"; + const chunkIndex = 2; + const shardCursor: ShardCursor = { + chunkIndex, + blockOffset: 0, + eventIndex: 0, + } + + // build shard correctly + const shardFactory = new ShardFactory(chunkFactoryStub as any); + const shard = await shardFactory.buildShard(containerClientSub as any, shardPath, shardCursor); + assert.ok(chunkFactoryStub.buildChunk.calledWith(containerClientSub, `${shardPath}000${chunkIndex}.avro`)); + const cursor = shard.getCursor(); + assert.deepStrictEqual(cursor.chunkIndex, shardCursor.chunkIndex); + + // shift to next chunk when currentChunk is done + chunkStub.hasNext.returns(false); + const nextChunkStub = sinon.createStubInstance(Chunk); + nextChunkStub.hasNext.returns(true); + const event = { id: "a" }; + nextChunkStub.getChange.resolves(event as any); + chunkFactoryStub.buildChunk.returns(nextChunkStub); + + const change = await shard.getChange(); + assert.ok(chunkFactoryStub.buildChunk.calledWith(containerClientSub, `${shardPath}000${chunkIndex + 1}.avro`)); + assert.deepStrictEqual(change, event); + const cursor2 = shard.getCursor(); + assert.deepStrictEqual(cursor2.chunkIndex, shardCursor.chunkIndex + 1); + + // chunks used up + nextChunkStub.hasNext.returns(false); + nextChunkStub.getChange.resolves(undefined); + const lastChunkStub = sinon.createStubInstance(Chunk); + lastChunkStub.hasNext.returns(false); + chunkFactoryStub.buildChunk.returns(lastChunkStub); + + const change2 = await shard.getChange(); + assert.ok(chunkFactoryStub.buildChunk.calledWith(containerClientSub, `${shardPath}000${chunkIndex + 2}.avro`)); + assert.equal(change2, undefined); + const cursor3 = shard.getCursor(); + assert.deepStrictEqual(cursor3.chunkIndex, shardCursor.chunkIndex + 2); + }); +}); From 53d14c6115dc7c27697e8e902e35e9f83e8411ef Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Fri, 5 Jun 2020 20:41:23 +0800 Subject: [PATCH 04/17] format code & rename buildAvroReader parameter name --- .../api-extractor.json | 2 +- .../storage-blob-changefeed/package.json | 2 +- .../rollup.base.config.js | 5 +- .../rollup.test.config.js | 5 +- .../src/AvroReaderFactory.ts | 5 +- .../src/BlobChangeFeedClient.ts | 26 +++++---- .../storage-blob-changefeed/src/ChangeFeed.ts | 35 +++++++++--- .../src/ChangeFeedFactory.ts | 41 +++++++------- .../storage-blob-changefeed/src/Chunk.ts | 8 +-- .../src/ChunkFactory.ts | 12 ++-- .../storage-blob-changefeed/src/Segment.ts | 13 ++--- .../src/SegmentFactory.ts | 17 ++++-- .../storage-blob-changefeed/src/Shard.ts | 3 +- .../src/ShardFactory.ts | 7 ++- .../storage-blob-changefeed/src/index.ts | 4 +- .../src/models/BlobChangeFeedEvent.ts | 1 - .../src/models/ChangeFeedCursor.ts | 3 - .../src/utils/utils.browser.ts | 2 +- .../src/utils/utils.common.ts | 22 +++++--- .../src/utils/utils.node.ts | 13 ++--- .../test/blobchangefeedclient.spec.ts | 17 +++--- .../test/changefeed.spec.ts | 56 +++++++++++++++---- .../test/chunk.spec.ts | 22 +++++--- .../test/segment.spec.ts | 9 ++- .../test/shard.spec.ts | 25 +++++++-- 25 files changed, 220 insertions(+), 135 deletions(-) diff --git a/sdk/storage/storage-blob-changefeed/api-extractor.json b/sdk/storage/storage-blob-changefeed/api-extractor.json index 8df76318f270..ae67e7ef139f 100644 --- a/sdk/storage/storage-blob-changefeed/api-extractor.json +++ b/sdk/storage/storage-blob-changefeed/api-extractor.json @@ -28,4 +28,4 @@ } } } -} \ No newline at end of file +} diff --git a/sdk/storage/storage-blob-changefeed/package.json b/sdk/storage/storage-blob-changefeed/package.json index da23bd4f0ae4..2b6172b9d42f 100644 --- a/sdk/storage/storage-blob-changefeed/package.json +++ b/sdk/storage/storage-blob-changefeed/package.json @@ -160,4 +160,4 @@ "util": "^0.12.1", "sinon": "^7.1.0" } -} \ No newline at end of file +} diff --git a/sdk/storage/storage-blob-changefeed/rollup.base.config.js b/sdk/storage/storage-blob-changefeed/rollup.base.config.js index abd11444d821..3627e5de4354 100644 --- a/sdk/storage/storage-blob-changefeed/rollup.base.config.js +++ b/sdk/storage/storage-blob-changefeed/rollup.base.config.js @@ -159,7 +159,10 @@ export function browserConfig(test = false) { }; if (test) { - baseConfig.input = ["dist-esm/storage-blob-changefeed/test/*.spec.js", "dist-esm/storage-blob-changefeed/test/browser/*.spec.js"]; + baseConfig.input = [ + "dist-esm/storage-blob-changefeed/test/*.spec.js", + "dist-esm/storage-blob-changefeed/test/browser/*.spec.js" + ]; baseConfig.plugins.unshift(multiEntry({ exports: false })); baseConfig.output.file = "dist-test/index.browser.js"; // mark fs-extra as external diff --git a/sdk/storage/storage-blob-changefeed/rollup.test.config.js b/sdk/storage/storage-blob-changefeed/rollup.test.config.js index b147ec721e5c..5ebf5220d5e3 100644 --- a/sdk/storage/storage-blob-changefeed/rollup.test.config.js +++ b/sdk/storage/storage-blob-changefeed/rollup.test.config.js @@ -3,6 +3,7 @@ import * as base from "./rollup.base.config"; -export default [base.nodeConfig(true), - // base.browserConfig(true) +export default [ + base.nodeConfig(true) + // base.browserConfig(true) ]; diff --git a/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts b/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts index 9e4d58a9966a..ffef8de435c5 100644 --- a/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts @@ -1,7 +1,8 @@ -import { AvroReadable, AvroReader } from '../../storage-internal-avro/src'; +import { AvroReadable, AvroReader } from "../../storage-internal-avro/src"; export class AvroReaderFactory { - public buildAvroReader(dataStream: AvroReadable): AvroReader; + public buildAvroReader(headerAndDataStream: AvroReadable): AvroReader; + public buildAvroReader( dataStream: AvroReadable, headerStream: AvroReadable, diff --git a/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts b/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts index c72b6c6e59e1..7b6f0f56a629 100644 --- a/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts +++ b/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts @@ -26,7 +26,7 @@ export class BlobChangeFeedClient { * * @private * @type {BlobServiceClient} - * @memberof DataLakeServiceClient + * @memberof BlobChangeFeedClient */ private _blobServiceClient: BlobServiceClient; private _changeFeedFactory: ChangeFeedFactory; @@ -36,13 +36,14 @@ export class BlobChangeFeedClient { this._changeFeedFactory = new ChangeFeedFactory(); } - public getChanges(options: ChangeFeedGetChangesOptions = {}) - : PagedAsyncIterableIterator { + public getChanges( + options: ChangeFeedGetChangesOptions = {} + ): PagedAsyncIterableIterator { const iter = this.getChange(options); return { /** - * @member {Promise} [next] The next method, part of the iteration protocol - */ + * @member {Promise} [next] The next method, part of the iteration protocol + */ async next() { return iter.next(); }, @@ -61,8 +62,9 @@ export class BlobChangeFeedClient { }; } - private async *getChange(options: ChangeFeedGetChangesOptions = {}) - : AsyncIterableIterator { + private async *getChange( + options: ChangeFeedGetChangesOptions = {} + ): AsyncIterableIterator { const changeFeed: ChangeFeed = await this._changeFeedFactory.buildChangeFeed( this._blobServiceClient, undefined, @@ -81,8 +83,11 @@ export class BlobChangeFeedClient { } // start in ChangeFeedGetChangesOptions will be ignored when continuationToken is specified. - private async *getPage(continuationToken?: string, maxPageSize?: number, options: ChangeFeedGetChangesOptions = {}) - : AsyncIterableIterator { + private async *getPage( + continuationToken?: string, + maxPageSize?: number, + options: ChangeFeedGetChangesOptions = {} + ): AsyncIterableIterator { const changeFeed: ChangeFeed = await this._changeFeedFactory.buildChangeFeed( this._blobServiceClient, continuationToken, @@ -106,8 +111,7 @@ export class BlobChangeFeedClient { } if (eventPage.events.length > 0) { yield eventPage; - } - else { + } else { return; } } diff --git a/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts b/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts index 69ef7dcea89d..17ced578ada9 100644 --- a/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts +++ b/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts @@ -1,6 +1,6 @@ -import { ContainerClient } from '@azure/storage-blob'; -import { Segment } from './Segment'; -import { SegmentFactory } from './SegmentFactory'; +import { ContainerClient } from "@azure/storage-blob"; +import { Segment } from "./Segment"; +import { SegmentFactory } from "./SegmentFactory"; import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; import { ChangeFeedCursor } from "./models/ChangeFeedCursor"; import { getURI, hashString, getSegmentsInYear, minDate } from "./utils/utils.common"; @@ -32,7 +32,8 @@ export class ChangeFeed { private _end?: Date; constructor(); - constructor(containerClient: ContainerClient, + constructor( + containerClient: ContainerClient, segmentFactory: SegmentFactory, years: number[], segments: string[], @@ -42,7 +43,8 @@ export class ChangeFeed { endTime?: Date ); - constructor(containerClient?: ContainerClient, + constructor( + containerClient?: ContainerClient, segmentFactory?: SegmentFactory, years?: number[], segments?: string[], @@ -70,7 +72,11 @@ export class ChangeFeed { return false; } - if (this._segments.length === 0 && this._years.length === 0 && !this._currentSegment.hasNext()) { + if ( + this._segments.length === 0 && + this._years.length === 0 && + !this._currentSegment.hasNext() + ) { return false; } @@ -111,15 +117,26 @@ export class ChangeFeed { // If the current segment is completed, remove it if (this._segments.length > 0) { - this._currentSegment = await this._segmentFactory!.buildSegment(this._containerClient!, this._segments.shift()!); + this._currentSegment = await this._segmentFactory!.buildSegment( + this._containerClient!, + this._segments.shift()! + ); } // If _segments is empty, refill it else if (this._segments.length === 0 && this._years.length > 0) { const year = this._years.shift(); - this._segments = await getSegmentsInYear(this._containerClient!, year!, this._startTime, this._end); + this._segments = await getSegmentsInYear( + this._containerClient!, + year!, + this._startTime, + this._end + ); if (this._segments.length > 0) { - this._currentSegment = await this._segmentFactory!.buildSegment(this._containerClient!, this._segments.shift()!); + this._currentSegment = await this._segmentFactory!.buildSegment( + this._containerClient!, + this._segments.shift()! + ); } else { this._currentSegment = undefined; } diff --git a/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts index 7f39531776c2..27d81d6a70e7 100644 --- a/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts @@ -1,10 +1,7 @@ import { BlobServiceClient, ContainerClient } from "@azure/storage-blob"; import { ChangeFeed } from "./ChangeFeed"; import { ChangeFeedCursor } from "./models/ChangeFeedCursor"; -import { - CHANGE_FEED_CONTAINER_NAME, - CHANGE_FEED_META_SEGMENT_PATH -} from './utils/constants'; +import { CHANGE_FEED_CONTAINER_NAME, CHANGE_FEED_META_SEGMENT_PATH } from "./utils/constants"; import { ceilToNearestHour, floorToNearestHour, @@ -13,10 +10,8 @@ import { getYearsPaths, getSegmentsInYear, minDate -} from './utils/utils.common'; -import { - bodyToString -} from './utils/utils.node'; +} from "./utils/utils.common"; +import { bodyToString } from "./utils/utils.node"; import { SegmentFactory } from "./SegmentFactory"; import { ShardFactory } from "./ShardFactory"; import { ChunkFactory } from "./ChunkFactory"; @@ -36,12 +31,10 @@ export class ChangeFeedFactory { constructor(segmentFactory?: SegmentFactory) { if (segmentFactory) { this._segmentFactory = segmentFactory; - } - else { + } else { this._segmentFactory = new SegmentFactory( - new ShardFactory( - new ChunkFactory( - new AvroReaderFactory()))); + new ShardFactory(new ChunkFactory(new AvroReaderFactory())) + ); } } @@ -72,7 +65,9 @@ export class ChangeFeedFactory { // Check if Change Feed has been enabled for this account. let changeFeedContainerExists = await containerClient.exists(); if (!changeFeedContainerExists) { - throw new Error("Change Feed hasn't been enabled on this account, or is currently being enabled."); + throw new Error( + "Change Feed hasn't been enabled on this account, or is currently being enabled." + ); } if (startTime && endTime && startTime >= endTime) { @@ -82,7 +77,9 @@ export class ChangeFeedFactory { // Get last consumable. const blobClient = containerClient.getBlobClient(CHANGE_FEED_META_SEGMENT_PATH); const blobDownloadRes = await blobClient.download(); - const lastConsumable = new Date((JSON.parse(await bodyToString(blobDownloadRes)) as MetaSegments).lastConsumable); + const lastConsumable = new Date( + (JSON.parse(await bodyToString(blobDownloadRes)) as MetaSegments).lastConsumable + ); // Get year paths const years: number[] = await getYearsPaths(containerClient); @@ -104,7 +101,8 @@ export class ChangeFeedFactory { containerClient, years.shift()!, startTime, - minDate(lastConsumable, endTime)); + minDate(lastConsumable, endTime) + ); } if (segments.length === 0) { return new ChangeFeed(); @@ -112,7 +110,8 @@ export class ChangeFeedFactory { const currentSegment: Segment = await this._segmentFactory.buildSegment( containerClient, segments.shift()!, - cursor?.currentSegmentCursor); + cursor?.currentSegmentCursor + ); return new ChangeFeed( containerClient, @@ -122,13 +121,11 @@ export class ChangeFeedFactory { currentSegment, lastConsumable, startTime, - endTime); + endTime + ); } - private static validateCursor( - containerClient: ContainerClient, - cursor: ChangeFeedCursor - ): void { + private static validateCursor(containerClient: ContainerClient, cursor: ChangeFeedCursor): void { if (hashString(getURLPath(containerClient.url)!) !== cursor.urlHash) { throw new Error("Cursor URL does not match container URL."); } diff --git a/sdk/storage/storage-blob-changefeed/src/Chunk.ts b/sdk/storage/storage-blob-changefeed/src/Chunk.ts index 467a8cf7a786..a2d14d0bbf71 100644 --- a/sdk/storage/storage-blob-changefeed/src/Chunk.ts +++ b/sdk/storage/storage-blob-changefeed/src/Chunk.ts @@ -1,4 +1,4 @@ -import { AvroReader } from '../../storage-internal-avro/src'; +import { AvroReader } from "../../storage-internal-avro/src"; import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; export class Chunk { @@ -15,11 +15,7 @@ export class Chunk { return this._eventIndex; } - constructor( - avroReader: AvroReader, - blockOffset: number, - eventIndex: number - ) { + constructor(avroReader: AvroReader, blockOffset: number, eventIndex: number) { this._avroReader = avroReader; this._blockOffset = blockOffset; this._eventIndex = eventIndex; diff --git a/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts b/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts index 216f8a74849c..faa86327ae03 100644 --- a/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts @@ -1,10 +1,9 @@ import { AvroReaderFactory } from "./AvroReaderFactory"; -import { ContainerClient } from '@azure/storage-blob'; +import { ContainerClient } from "@azure/storage-blob"; import { Chunk } from "./Chunk"; -import { AvroReader } from "../../storage-internal-avro/src" +import { AvroReader } from "../../storage-internal-avro/src"; import { bodyToAvroReadable } from "./utils/utils.node"; - export class ChunkFactory { private readonly _avroReaderFactory: AvroReaderFactory; @@ -29,7 +28,12 @@ export class ChunkFactory { if (blockOffset !== 0) { const headerDownloadRes = await blobClient.download(0); const headerStream = bodyToAvroReadable(headerDownloadRes); - avroReader = this._avroReaderFactory.buildAvroReader(dataStream, headerStream, blockOffset, eventIndex); + avroReader = this._avroReaderFactory.buildAvroReader( + dataStream, + headerStream, + blockOffset, + eventIndex + ); } else { avroReader = this._avroReaderFactory.buildAvroReader(dataStream); } diff --git a/sdk/storage/storage-blob-changefeed/src/Segment.ts b/sdk/storage/storage-blob-changefeed/src/Segment.ts index fc1cbad689c2..527887630c22 100644 --- a/sdk/storage/storage-blob-changefeed/src/Segment.ts +++ b/sdk/storage/storage-blob-changefeed/src/Segment.ts @@ -1,6 +1,6 @@ import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; -import { Shard } from './Shard'; -import { SegmentCursor, ShardCursor } from './models/ChangeFeedCursor'; +import { Shard } from "./Shard"; +import { SegmentCursor, ShardCursor } from "./models/ChangeFeedCursor"; export class Segment { private readonly _shards: Shard[]; @@ -23,12 +23,7 @@ export class Segment { return this._dateTime; } - constructor( - shards: Shard[], - shardIndex: number, - dateTime: Date, - finalized: boolean - ) { + constructor(shards: Shard[], shardIndex: number, dateTime: Date, finalized: boolean) { this._shards = shards; this._shardIndex = shardIndex; this._dateTime = dateTime; @@ -78,6 +73,6 @@ export class Segment { shardCursors, shardIndex: this._shardIndex, segmentTime: this._dateTime.toJSON() - } + }; } } diff --git a/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts b/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts index f9a25450abb8..de56bc1f9f85 100644 --- a/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts @@ -1,9 +1,9 @@ import { ShardFactory } from "./ShardFactory"; import { ContainerClient } from "@azure/storage-blob"; -import { CHANGE_FEED_STATUS_FINALIZED, CHANGE_FEED_CONTAINER_NAME } from './utils/constants'; -import { Shard } from './Shard'; -import { Segment } from './Segment'; -import { SegmentCursor } from './models/ChangeFeedCursor'; +import { CHANGE_FEED_STATUS_FINALIZED, CHANGE_FEED_CONTAINER_NAME } from "./utils/constants"; +import { Shard } from "./Shard"; +import { Segment } from "./Segment"; +import { SegmentCursor } from "./models/ChangeFeedCursor"; import { bodyToString } from "./utils/utils.node"; import { parseDateFromSegmentPath } from "./utils/utils.common"; @@ -23,7 +23,8 @@ export class SegmentFactory { this._shardFactory = shardFactory; } - public async buildSegment(containerClient: ContainerClient, + public async buildSegment( + containerClient: ContainerClient, manifestPath: string, cursor?: SegmentCursor ): Promise { @@ -43,7 +44,11 @@ export class SegmentFactory { const containerPrefixLength = CHANGE_FEED_CONTAINER_NAME.length + 1; // "$blobchangefeed/" for (const shardPath of segmentManifest.chunkFilePaths) { - const shard: Shard = await this._shardFactory.buildShard(containerClient, shardPath.substring(containerPrefixLength), cursor?.shardCursors[i++]); + const shard: Shard = await this._shardFactory.buildShard( + containerClient, + shardPath.substring(containerPrefixLength), + cursor?.shardCursors[i++] + ); shards.push(shard); } } diff --git a/sdk/storage/storage-blob-changefeed/src/Shard.ts b/sdk/storage/storage-blob-changefeed/src/Shard.ts index c37d09a4827f..286b7cb56c6a 100644 --- a/sdk/storage/storage-blob-changefeed/src/Shard.ts +++ b/sdk/storage/storage-blob-changefeed/src/Shard.ts @@ -20,7 +20,8 @@ export class Shard { chunkFactory: ChunkFactory, chunks: string[], currentChunk: Chunk, - chunkIndex: number) { + chunkIndex: number + ) { this._containerClient = containerClient; this._chunkFactory = chunkFactory; this._chunks = chunks; diff --git a/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts b/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts index 0cbc1155536d..04027988962b 100644 --- a/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts @@ -1,5 +1,5 @@ -import { ChunkFactory } from './ChunkFactory'; -import { ShardCursor } from './models/ChangeFeedCursor'; +import { ChunkFactory } from "./ChunkFactory"; +import { ShardCursor } from "./models/ChangeFeedCursor"; import { Shard } from "./Shard"; import { ContainerClient } from "@azure/storage-blob"; @@ -41,7 +41,8 @@ export class ShardFactory { containerClient, chunks.shift()!, blockOffset, - eventIndex); + eventIndex + ); return new Shard(containerClient, this._chunkFactory, chunks, currentChunk, chunkIndex); } } diff --git a/sdk/storage/storage-blob-changefeed/src/index.ts b/sdk/storage/storage-blob-changefeed/src/index.ts index efab53b3d462..45b2282e86be 100644 --- a/sdk/storage/storage-blob-changefeed/src/index.ts +++ b/sdk/storage/storage-blob-changefeed/src/index.ts @@ -1,2 +1,2 @@ -export * from './BlobChangeFeedClient'; -export * from './models/BlobChangeFeedEvent'; +export * from "./BlobChangeFeedClient"; +export * from "./models/BlobChangeFeedEvent"; diff --git a/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts index 3fd67f76ca06..cf2e3379750f 100644 --- a/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts +++ b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts @@ -11,7 +11,6 @@ export interface BlobChangeFeedEvent { metadataVersion: string; } - export type BlobType = "BlockBlob" | "AppendBlob" | "PageBlob"; export interface BlobChangeFeedEventData { diff --git a/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts b/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts index 7e424af4ff13..f4796dafac0f 100644 --- a/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts +++ b/sdk/storage/storage-blob-changefeed/src/models/ChangeFeedCursor.ts @@ -1,4 +1,3 @@ - export interface ChangeFeedCursor { cursorVersion: number; urlHash: number; @@ -6,14 +5,12 @@ export interface ChangeFeedCursor { currentSegmentCursor: SegmentCursor; } - export interface SegmentCursor { shardCursors: ShardCursor[]; shardIndex: number; segmentTime: string; } - export interface ShardCursor { chunkIndex: number; blockOffset: number; diff --git a/sdk/storage/storage-blob-changefeed/src/utils/utils.browser.ts b/sdk/storage/storage-blob-changefeed/src/utils/utils.browser.ts index 0f5d38eedfe8..f12b8ea0cc7b 100644 --- a/sdk/storage/storage-blob-changefeed/src/utils/utils.browser.ts +++ b/sdk/storage/storage-blob-changefeed/src/utils/utils.browser.ts @@ -28,4 +28,4 @@ export async function blobToString(blob: Blob): Promise { }); } -export function bodyToAvroReadable() { } +export function bodyToAvroReadable() {} diff --git a/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts b/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts index 3fc836404f12..182253e510ad 100644 --- a/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts +++ b/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts @@ -33,15 +33,17 @@ export function getURI(url: string): string | undefined { export function hashString(str: string): number { let hash = 0; for (let i = 0; i < str.length; i++) { - hash = ((hash << 5) - hash) + str.charCodeAt(i); - hash |= 0;; // Bit operation converts operands to 32-bit integers + hash = (hash << 5) - hash + str.charCodeAt(i); + hash |= 0; // Bit operation converts operands to 32-bit integers } return hash; } export async function getYearsPaths(containerClient: ContainerClient): Promise { let years: number[] = []; - for await (const item of containerClient.listBlobsByHierarchy("/", { prefix: CHANGE_FEED_SEGMENT_PREFIX })) { + for await (const item of containerClient.listBlobsByHierarchy("/", { + prefix: CHANGE_FEED_SEGMENT_PREFIX + })) { // TODO: add String.prototype.includes polyfill for IE11 if (item.kind === "prefix" && !item.name.includes(CHANGE_FEED_INITIALIZATION_SEGMENT)) { let yearStr = item.name.slice(CHANGE_FEED_SEGMENT_PREFIX.length, -1); @@ -51,18 +53,22 @@ export async function getYearsPaths(containerClient: ContainerClient): Promise a - b); } -export async function getSegmentsInYear(containerClient: ContainerClient, year: number, startTime?: Date, endTime?: Date): Promise { +export async function getSegmentsInYear( + containerClient: ContainerClient, + year: number, + startTime?: Date, + endTime?: Date +): Promise { let segments: string[] = []; const yearBeginTime = new Date(Date.UTC(year, 0)); if (endTime && yearBeginTime >= endTime) { return segments; } - const prefix = `${CHANGE_FEED_SEGMENT_PREFIX}${year}/` + const prefix = `${CHANGE_FEED_SEGMENT_PREFIX}${year}/`; for await (const item of containerClient.listBlobsFlat({ prefix })) { const segmentTime = parseDateFromSegmentPath(item.name); - if (startTime && segmentTime < startTime - || endTime && segmentTime >= endTime) { + if ((startTime && segmentTime < startTime) || (endTime && segmentTime >= endTime)) { continue; } segments.push(item.name); @@ -71,7 +77,7 @@ export async function getSegmentsInYear(containerClient: ContainerClient, year: } export function parseDateFromSegmentPath(segmentPath: string): Date { - const splitPath = segmentPath.split('/'); + const splitPath = segmentPath.split("/"); if (splitPath.length < 3) { throw new Error(`${segmentPath} is not a valid segment path.`); } diff --git a/sdk/storage/storage-blob-changefeed/src/utils/utils.node.ts b/sdk/storage/storage-blob-changefeed/src/utils/utils.node.ts index f0d33bf0446c..4668d88857ac 100644 --- a/sdk/storage/storage-blob-changefeed/src/utils/utils.node.ts +++ b/sdk/storage/storage-blob-changefeed/src/utils/utils.node.ts @@ -1,4 +1,4 @@ -import { AvroReadable, AvroReadableFromStream } from '../../../storage-internal-avro/src'; +import { AvroReadable, AvroReadableFromStream } from "../../../storage-internal-avro/src"; /** * Read body from downloading operation methods to string. @@ -30,12 +30,9 @@ export async function bodyToString( }); } - -export function bodyToAvroReadable( - response: { - readableStreamBody?: NodeJS.ReadableStream; - blobBody?: Promise; - } -): AvroReadable { +export function bodyToAvroReadable(response: { + readableStreamBody?: NodeJS.ReadableStream; + blobBody?: Promise; +}): AvroReadable { return new AvroReadableFromStream(response.readableStreamBody!); } diff --git a/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts index c6615db3b256..870f127eca8c 100644 --- a/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts @@ -5,7 +5,7 @@ import { BlobChangeFeedClient, BlobChangeFeedEvent, BlobChangeFeedEventPage } fr import * as dotenv from "dotenv"; dotenv.config(); -describe("BlobChangeFeedClient", async () => { +describe.only("BlobChangeFeedClient", async () => { const account = process.env.ACCOUNT_NAME || ""; const accountKey = process.env.ACCOUNT_KEY || ""; const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey); @@ -15,7 +15,7 @@ describe("BlobChangeFeedClient", async () => { ); const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); - before(async function () { + before(async function() { if (process.env.CHANGE_FEED_ENABLED !== "1") { this.skip(); } @@ -53,7 +53,7 @@ describe("BlobChangeFeedClient", async () => { }); it("byPage()", async () => { - const maxPageSize = 2 + const maxPageSize = 2; const iter = changeFeedClient.getChanges().byPage({ maxPageSize }); const nextPage = await iter.next(); if (nextPage.done) { @@ -65,7 +65,9 @@ describe("BlobChangeFeedClient", async () => { assert.ok(event.data.blobType); // continuationToken - const iter1 = changeFeedClient.getChanges().byPage({ continuationToken: nextPage.value.continuationToken, maxPageSize }); + const iter1 = changeFeedClient + .getChanges() + .byPage({ continuationToken: nextPage.value.continuationToken, maxPageSize }); const nextPage1 = await iter1.next(); if (nextPage1.done) { return; @@ -80,7 +82,9 @@ describe("BlobChangeFeedClient", async () => { const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be ignored const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be rounded to 22:00 const endRounded = new Date(Date.UTC(2020, 4, 8, 22, 0, 0)); - const iter2 = changeFeedClient.getChanges({ start, end }).byPage({ continuationToken: nextPage1.value.continuationToken }); + const iter2 = changeFeedClient + .getChanges({ start, end }) + .byPage({ continuationToken: nextPage1.value.continuationToken }); let i = 0; let lastEventPage: BlobChangeFeedEventPage | undefined; for await (const eventPage of iter2) { @@ -100,7 +104,6 @@ describe("BlobChangeFeedClient", async () => { }); }); - describe("BlobChangeFeedClient: Change Feed not configured", async () => { const account = process.env.ACCOUNT_NAME || ""; const accountKey = process.env.ACCOUNT_KEY || ""; @@ -111,7 +114,7 @@ describe("BlobChangeFeedClient: Change Feed not configured", async () => { ); const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); - before(async function () { + before(async function() { if (process.env.CHANGE_FEED_ENABLED === "1") { this.skip(); } diff --git a/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts b/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts index af37c960b79f..37fc31b85bd7 100644 --- a/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts @@ -64,19 +64,29 @@ describe("Change Feed", async () => { serviceClientStub.getContainerClient.returns(containerClientStub as any); containerClientStub.exists.resolves(true); containerClientStub.getBlobClient.returns(blobClientStub as any); - containerClientStub.listBlobsByHierarchy.withArgs("/").callsFake(() => (fakeList(yearPaths) as any)); - containerClientStub.listBlobsFlat.withArgs({ prefix: "idx/segments/2019/" }).callsFake(() => (fakeList(segmentsIn2019) as any)); - containerClientStub.listBlobsFlat.withArgs({ prefix: "idx/segments/2020/" }).callsFake(() => (fakeList(segmentsIn2020) as any)); + containerClientStub.listBlobsByHierarchy + .withArgs("/") + .callsFake(() => fakeList(yearPaths) as any); + containerClientStub.listBlobsFlat + .withArgs({ prefix: "idx/segments/2019/" }) + .callsFake(() => fakeList(segmentsIn2019) as any); + containerClientStub.listBlobsFlat + .withArgs({ prefix: "idx/segments/2020/" }) + .callsFake(() => fakeList(segmentsIn2020) as any); // TODO: rewrite for browser blobClientStub.download.callsFake(() => { - return new Promise((resolve) => { resolve({ readableStreamBody: fs.createReadStream(manifestFilePath) } as any) }); + return new Promise((resolve) => { + resolve({ readableStreamBody: fs.createReadStream(manifestFilePath) } as any); + }); }); segmentStubs = []; const segmentIter = listTwoArray(segmentsIn2019, segmentsIn2020); for (let i = 0; i < segmentCount; i++) { segmentStubs.push(sinon.createStubInstance(Segment)); - segmentFactoryStub.buildSegment.withArgs(sinon.match.any, (await segmentIter.next()).value.name).resolves(segmentStubs[i] as any); + segmentFactoryStub.buildSegment + .withArgs(sinon.match.any, (await segmentIter.next()).value.name) + .resolves(segmentStubs[i] as any); } for (let i = 0; i < segmentCount; i++) { sinon.stub(segmentStubs[i], "dateTime").value(segmentTimes[i]); @@ -103,7 +113,11 @@ describe("Change Feed", async () => { { kind: "prefix", name: "idx/segments/2019/" } ]; containerClientStub.listBlobsByHierarchy.withArgs("/").returns(fakeList(yearPaths) as any); - const changeFeed = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, undefined, new Date(Date.UTC(2020, 0))); + const changeFeed = await changeFeedFactory.buildChangeFeed( + serviceClientStub as any, + undefined, + new Date(Date.UTC(2020, 0)) + ); assert.ok(!changeFeed.hasNext()); }); @@ -120,12 +134,20 @@ describe("Change Feed", async () => { ]; containerClientStub.listBlobsFlat.returns(fakeList(segments) as any); - const changeFeed = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, undefined, new Date(Date.UTC(2019, 5))); + const changeFeed = await changeFeedFactory.buildChangeFeed( + serviceClientStub as any, + undefined, + new Date(Date.UTC(2019, 5)) + ); assert.ok(!changeFeed.hasNext()); }); it("getChange", async () => { - const changeFeed = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, undefined, new Date(Date.UTC(2019, 0))); + const changeFeed = await changeFeedFactory.buildChangeFeed( + serviceClientStub as any, + undefined, + new Date(Date.UTC(2019, 0)) + ); assert.ok(changeFeed.hasNext()); const event = await changeFeed.getChange(); @@ -170,7 +192,7 @@ describe("Change Feed", async () => { serviceClientStub as any, undefined, new Date(Date.UTC(2019, 3, 3, 22)), - new Date(Date.UTC(2019, 4, 3, 22)), + new Date(Date.UTC(2019, 4, 3, 22)) ); assert.ok(changeFeed2.hasNext()); const event = await changeFeed2.getChange(); @@ -204,9 +226,16 @@ describe("Change Feed", async () => { const cursor = changeFeed.getCursor(); assert.deepStrictEqual(cursor.urlHash, hashString(getURI(containerUri)!)); - segmentStubs[3].getCursor.returns({ shardCursors: [], shardIndex: 0, segmentTime: (new Date(Date.UTC(2020, 2, 2, 20))).toJSON() }); + segmentStubs[3].getCursor.returns({ + shardCursors: [], + shardIndex: 0, + segmentTime: new Date(Date.UTC(2020, 2, 2, 20)).toJSON() + }); const continuation = JSON.stringify(changeFeed.getCursor()); - const changeFeed2 = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, continuation); + const changeFeed2 = await changeFeedFactory.buildChangeFeed( + serviceClientStub as any, + continuation + ); assert.ok(changeFeed2.hasNext()); const event = await changeFeed.getChange(); assert.equal(event, 3); @@ -215,7 +244,10 @@ describe("Change Feed", async () => { sinon.stub(segmentStubs[4], "finalized").value(true); segmentStubs[3].hasNext.returns(false); segmentStubs[3].getChange.resolves(undefined); - const changeFeed3 = await changeFeedFactory.buildChangeFeed(serviceClientStub as any, continuation); + const changeFeed3 = await changeFeedFactory.buildChangeFeed( + serviceClientStub as any, + continuation + ); assert.ok(changeFeed3.hasNext()); const event2 = await changeFeed.getChange(); assert.equal(event2, 4); diff --git a/sdk/storage/storage-blob-changefeed/test/chunk.spec.ts b/sdk/storage/storage-blob-changefeed/test/chunk.spec.ts index 279e1b62acac..94463095bcf7 100644 --- a/sdk/storage/storage-blob-changefeed/test/chunk.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/chunk.spec.ts @@ -10,7 +10,7 @@ class FakeAvroReader { public hasNext: boolean, private _record: any, public blockSize?: number - ) { } + ) {} public async *parseObjects(): AsyncIterableIterator { while (this.hasNext) { @@ -21,7 +21,6 @@ class FakeAvroReader { } } - describe("Chunk", async () => { afterEach(() => { sinon.restore(); @@ -36,20 +35,27 @@ describe("Chunk", async () => { avroReaderStub.hasNext.returns(false); assert.equal(chunk.hasNext(), false); - }); it("getChange", async () => { // set up - const record = { a: 1 } + const record = { a: 1 }; const fakeAvroReader = new FakeAvroReader(0, 0, true, record); const avroReaderStub = sinon.createStubInstance(AvroReader); avroReaderStub.hasNext.callsFake(() => fakeAvroReader.hasNext); avroReaderStub.parseObjects.returns(fakeAvroReader.parseObjects()); - sinon.stub(avroReaderStub, "blockOffset").get(() => { return fakeAvroReader.blockOffset }); - sinon.stub(avroReaderStub, "objectIndex").get(() => { return fakeAvroReader.objectIndex }); + sinon.stub(avroReaderStub, "blockOffset").get(() => { + return fakeAvroReader.blockOffset; + }); + sinon.stub(avroReaderStub, "objectIndex").get(() => { + return fakeAvroReader.objectIndex; + }); - const chunk = new Chunk(avroReaderStub as any, avroReaderStub.blockOffset, avroReaderStub.objectIndex); + const chunk = new Chunk( + avroReaderStub as any, + avroReaderStub.blockOffset, + avroReaderStub.objectIndex + ); // act and verify const change = await chunk.getChange(); @@ -63,4 +69,4 @@ describe("Chunk", async () => { assert.equal(chunk.blockOffset, avroReaderStub.blockOffset); assert.equal(chunk.eventIndex, avroReaderStub.objectIndex); }); -}); \ No newline at end of file +}); diff --git a/sdk/storage/storage-blob-changefeed/test/segment.spec.ts b/sdk/storage/storage-blob-changefeed/test/segment.spec.ts index 185ab44bbea8..ab99ed1ed108 100644 --- a/sdk/storage/storage-blob-changefeed/test/segment.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/segment.spec.ts @@ -21,7 +21,9 @@ describe("Shard", async () => { const blobClientStub = sinon.createStubInstance(BlobClient); containerClientStub.getBlobClient.returns(blobClientStub); // TODO: rewrite for browser - blobClientStub.download.resolves({ readableStreamBody: fs.createReadStream(segmentManifestFilePath) } as any); + blobClientStub.download.resolves({ + readableStreamBody: fs.createReadStream(segmentManifestFilePath) + } as any); shardFactoryStub = sinon.createStubInstance(ShardFactory); shardStubs = []; @@ -78,7 +80,10 @@ describe("Shard", async () => { it("init with non-zero shardIndex", async () => { const shardIndex = 1; const segmentFactory = new SegmentFactory(shardFactoryStub); - const segment = await segmentFactory.buildSegment(containerClientStub, manifestPath, { shardIndex, shardCursors: [] } as any); + const segment = await segmentFactory.buildSegment(containerClientStub, manifestPath, { + shardIndex, + shardCursors: [] + } as any); assert.ok(segment.hasNext()); assert.equal(segment.dateTime.getTime(), dateTime.getTime()); assert.ok(segment.finalized); diff --git a/sdk/storage/storage-blob-changefeed/test/shard.spec.ts b/sdk/storage/storage-blob-changefeed/test/shard.spec.ts index 5e7fffef79cd..2613e54aeb8d 100644 --- a/sdk/storage/storage-blob-changefeed/test/shard.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/shard.spec.ts @@ -35,13 +35,18 @@ describe("Shard", async () => { const shardCursor: ShardCursor = { chunkIndex, blockOffset: 0, - eventIndex: 0, - } + eventIndex: 0 + }; // build shard correctly const shardFactory = new ShardFactory(chunkFactoryStub as any); const shard = await shardFactory.buildShard(containerClientSub as any, shardPath, shardCursor); - assert.ok(chunkFactoryStub.buildChunk.calledWith(containerClientSub, `${shardPath}000${chunkIndex}.avro`)); + assert.ok( + chunkFactoryStub.buildChunk.calledWith( + containerClientSub, + `${shardPath}000${chunkIndex}.avro` + ) + ); const cursor = shard.getCursor(); assert.deepStrictEqual(cursor.chunkIndex, shardCursor.chunkIndex); @@ -54,7 +59,12 @@ describe("Shard", async () => { chunkFactoryStub.buildChunk.returns(nextChunkStub); const change = await shard.getChange(); - assert.ok(chunkFactoryStub.buildChunk.calledWith(containerClientSub, `${shardPath}000${chunkIndex + 1}.avro`)); + assert.ok( + chunkFactoryStub.buildChunk.calledWith( + containerClientSub, + `${shardPath}000${chunkIndex + 1}.avro` + ) + ); assert.deepStrictEqual(change, event); const cursor2 = shard.getCursor(); assert.deepStrictEqual(cursor2.chunkIndex, shardCursor.chunkIndex + 1); @@ -67,7 +77,12 @@ describe("Shard", async () => { chunkFactoryStub.buildChunk.returns(lastChunkStub); const change2 = await shard.getChange(); - assert.ok(chunkFactoryStub.buildChunk.calledWith(containerClientSub, `${shardPath}000${chunkIndex + 2}.avro`)); + assert.ok( + chunkFactoryStub.buildChunk.calledWith( + containerClientSub, + `${shardPath}000${chunkIndex + 2}.avro` + ) + ); assert.equal(change2, undefined); const cursor3 = shard.getCursor(); assert.deepStrictEqual(cursor3.chunkIndex, shardCursor.chunkIndex + 2); From 427930d0af43c9c28960c94281c31244d0dafe86 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Sun, 7 Jun 2020 14:22:59 +0800 Subject: [PATCH 05/17] add README.md --- .../storage-blob-changefeed/CHANGELOG.md | 4 + sdk/storage/storage-blob-changefeed/README.md | 170 ++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 sdk/storage/storage-blob-changefeed/CHANGELOG.md create mode 100644 sdk/storage/storage-blob-changefeed/README.md diff --git a/sdk/storage/storage-blob-changefeed/CHANGELOG.md b/sdk/storage/storage-blob-changefeed/CHANGELOG.md new file mode 100644 index 000000000000..4f22a19715c2 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/CHANGELOG.md @@ -0,0 +1,4 @@ +# Release History + +## 12.0.0-preview.1 (2020.6) +- This preview is the first release supporting Azure Storage Blob Change Feed. diff --git a/sdk/storage/storage-blob-changefeed/README.md b/sdk/storage/storage-blob-changefeed/README.md new file mode 100644 index 000000000000..f9e780c3a9b1 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/README.md @@ -0,0 +1,170 @@ +# Azure Storage Blob Change Feed client library for JavaScript + +> Server Version: 2019-12-12 + +The purpose of the change feed is to provide transaction logs of all the changes that occur to the blobs and the blob metadata in your storage account. The change feed provides ordered, guaranteed, durable, immutable, read-only log of these changes. Client applications can read these logs at any time. The change feed enables you to build efficient and scalable solutions that process change events that occur in your Blob Storage account at a low cost. + +This project provides a client library in JavaScript that makes it easy to consume the change feed. + +Use the client libararies in this package to: + - Reading change feed events, all or within a time range + - Resuming reading events from a saved position + +[Source code](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed) | +[Package (npm)](https://www.npmjs.com/package/@azure/storage-blob-changefeed/) | +[API Reference Documentation](https://docs.microsoft.com/javascript/api/@azure/storage-blob-changefeed) | +[Product documentation](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-change-feed) | +[Samples](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/samples) | + +## Getting started + +**Prerequisites**: You must have an [Azure subscription](https://azure.microsoft.com/free/) and a [Storage Account](https://docs.microsoft.com/azure/storage/blobs/storage-quickstart-blobs-portal) to use this package. If you are using this package in a Node.js application, then Node.js version 8.0.0 or higher is required. + +### Install the package + +The preferred way to install the Azure Storage Blob client library for JavaScript is to use the npm package manager. Type the following into a terminal window: + +```bash +npm install @azure/storage-blob-changefeed +``` + +### Authenticate the client + +This library use a `BlobServiceClient` to initialize, refer to [storage-blob](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob#authenticate-the-client) for how to authenticate a `BlobServiceClient`. + +### Compatibility + +For this perview, this library is only compatible with Node.js. + +### CORS + +You need to set up [Cross-Origin Resource Sharing (CORS)](https://docs.microsoft.com/rest/api/storageservices/cross-origin-resource-sharing--cors--support-for-the-azure-storage-services) rules for your storage account if you need to develop for browsers. Go to Azure portal and Azure Storage Explorer, find your storage account, create new CORS rules for blob/queue/file/table service(s). + +For example, you can create following CORS settings for debugging. But please customize the settings carefully according to your requirements in production environment. + +- Allowed origins: \* +- Allowed verbs: DELETE,GET,HEAD,MERGE,POST,OPTIONS,PUT +- Allowed headers: \* +- Exposed headers: \* +- Maximum age (seconds): 86400 + +## Key concepts + +The change feed is stored as blobs in a special container in your storage account at standard blob pricing cost. You can control the retention period of these files based on your requirements. Change events are appended to the change feed as records in the Apache Avro format specification: a compact, fast, binary format that provides rich data structures with inline schema. This format is widely used in the Hadoop ecosystem, Stream Analytics, and Azure Data +Factory. + +This library offers a client you can use to fetch the change events. + +## Examples + +### Initialize the change feed client + +The `BlobChangeFeedClient` requires a `BlobServiceClient` to initialize. Refer to [storage-blob](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob#create-the-blob-service-client) for how to create the blob service client. Here is an example using `StorageSharedKeyCredential`. + + ```javascript + const { BlobServiceClient, StorageSharedKeyCredential } = require("@azure/storage-blob"); + const { BlobChangeFeedClient } = require("@azure/storage-blob-changefeed"); + + // Enter your storage account name and shared key + const account = ""; + const accountKey = ""; + // Use StorageSharedKeyCredential with storage account and account key + // StorageSharedKeyCredential is only avaiable in Node.js runtime, not in browsers + const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey); + const blobServiceClient = new BlobServiceClient( + `https://${account}.blob.core.windows.net`, + sharedKeyCredential + ); + + const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); + ``` + +### Reading all events in the Change Feed + +Use `BlobChangeFeedClient.getChanges()` to get iterators to iterate through the change events. + +```javascript +const { BlobChangeFeedEvent } = require("@azure/storage-blob-changefeed"); + +let changeFeedEvents : BlobChangeFeedEvent[] = []; +for await (const event of changeFeedClient.getChanges()) { + changeFeedEvents.push(event); +} +``` + +By page. + +```javascript +const { BlobChangeFeedEvent } = require("@azure/storage-blob-changefeed"); + +let changeFeedEvents : BlobChangeFeedEvent[] = []; +for await (const eventPage of changeFeedClient.getChanges().byPage()) { + for (const event of eventPage) { + changeFeedEvents.push(event); + } +} +``` + +### Resuming reading events with a cursor + +```javascript +const { BlobChangeFeedEvent } = require("@azure/storage-blob-changefeed"); + +let changeFeedEvents : BlobChangeFeedEvent[] = []; +const firstPage = await changeFeedClient.getChanges().byPage({maxPageSize: 10}).next(); +for (const event of firstPage) { + changeFeedEvents.push(event); +} + +// Resume iterating from the pervious position with the continuationToken. +for await (const eventPage of changeFeedClient.getChanges().byPage({continuationToken: firstPage.continuationToken})) { + for (const event of eventPage) { + changeFeedEvents.push(event); + } +} +``` + +### Reading events within a time range + +Pass start time and end time to `BlobChangeFeedClient.getChanges()` to fetch events within a time range. + +Note that for preview release, the change feed client will round start time down to the nearest hour, and round endTime up to the next hour. + +```javascript +const { BlobChangeFeedEvent } = require("@azure/storage-blob-changefeed"); + +const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be floor to 22:00 +const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be ceil to 22:00 + +let changeFeedEvents : BlobChangeFeedEvent[] = []; +// You can also provide just a start or end time. +for await (const event of changeFeedClient.getChanges({ start, end })) { + changeFeedEvents.push(event); +} +``` + +## Troubleshooting + +Enabling logging may help uncover useful information about failures. In order to see a log of HTTP requests and responses, set the `AZURE_LOG_LEVEL` environment variable to `info`. Alternatively, logging can be enabled at runtime by calling `setLogLevel` in the `@azure/logger`: + +```javascript +import { setLogLevel } from "@azure/logger"; + +setLogLevel("info"); +``` + +## Next steps + +More code samples: + +- [Blob Storage Samples (JavaScript)](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/samples/javascript) +- [Blob Storage Samples (TypeScript)](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/samples/typescript) +- [Blob Storage Test Cases](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/test/) + +## Contributing + +If you'd like to contribute to this library, please read the [contributing guide](https://github.com/Azure/azure-sdk-for-js/blob/master/CONTRIBUTING.md) to learn more about how to build and test the code. + +Also refer to [Storage specific guide](https://github.com/Azure/azure-sdk-for-js/blob/master/sdk/storage/CONTRIBUTING.md) for additional information on setting up the test environment for storage libraries. + +![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-js%2Fsdk%2Fstorage%2Fstorage-blob-changefeed%2FREADME.png) From 279d010dc150e2159beae3b450b8e751c82659cb Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Tue, 9 Jun 2020 14:16:28 +0800 Subject: [PATCH 06/17] change eventTime type from string to Date --- .../review/storage-blob-changefeed.api.md | 93 +++++++++++++++++++ .../src/AvroReaderFactory.ts | 3 +- .../src/ChangeFeedFactory.ts | 4 +- .../storage-blob-changefeed/src/Chunk.ts | 6 +- .../src/models/BlobChangeFeedEvent.ts | 3 +- .../test/blobchangefeedclient.spec.ts | 8 +- .../test/changefeed.spec.ts | 2 +- 7 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md diff --git a/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md new file mode 100644 index 000000000000..1cc4ef1a162e --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md @@ -0,0 +1,93 @@ +## API Report File for "@azure/storage-blob-changefeed" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { BlobServiceClient } from '@azure/storage-blob'; +import { PagedAsyncIterableIterator } from '@azure/core-paging'; + +// @public (undocumented) +export class BlobChangeFeedClient { + constructor(blobServiceClient: BlobServiceClient); + // (undocumented) + getChanges(options?: ChangeFeedGetChangesOptions): PagedAsyncIterableIterator; + } + +// @public (undocumented) +export interface BlobChangeFeedEvent { + // (undocumented) + data: BlobChangeFeedEventData; + // (undocumented) + dataVersion?: string; + // (undocumented) + eventTime: Date; + // (undocumented) + eventType: BlobChangeFeedEventType; + // (undocumented) + id: string; + // (undocumented) + metadataVersion: string; + // (undocumented) + subject: string; + // (undocumented) + topic: string; +} + +// @public (undocumented) +export interface BlobChangeFeedEventData { + // (undocumented) + api: string; + // (undocumented) + blobType: BlobType; + // (undocumented) + clientRequestId: string; + // (undocumented) + contentLength: number; + // (undocumented) + contentOffset?: number; + // (undocumented) + contentType: string; + // (undocumented) + destinationUrl?: string; + // (undocumented) + eTag: string; + // (undocumented) + recursive?: string; + // (undocumented) + requestId: string; + // (undocumented) + sequencer: string; + // (undocumented) + sourceUrl?: string; + // (undocumented) + url: string; +} + +// @public (undocumented) +export class BlobChangeFeedEventPage { + constructor(); + // (undocumented) + continuationToken: string; + // (undocumented) + events: BlobChangeFeedEvent[]; +} + +// @public (undocumented) +export type BlobChangeFeedEventType = "BlobCreate" | "BlobDeleted"; + +// @public (undocumented) +export type BlobType = "BlockBlob" | "AppendBlob" | "PageBlob"; + +// @public (undocumented) +export interface ChangeFeedGetChangesOptions { + // (undocumented) + end?: Date; + // (undocumented) + start?: Date; +} + + +// (No @packageDocumentation comment for this package) + +``` diff --git a/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts b/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts index ffef8de435c5..d2dc198ae591 100644 --- a/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts @@ -7,7 +7,8 @@ export class AvroReaderFactory { dataStream: AvroReadable, headerStream: AvroReadable, blockOffset: number, - eventIndex: number): AvroReader; + eventIndex: number + ): AvroReader; public buildAvroReader( dataStream: AvroReadable, diff --git a/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts index 27d81d6a70e7..f98a67f73f6c 100644 --- a/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts @@ -5,7 +5,7 @@ import { CHANGE_FEED_CONTAINER_NAME, CHANGE_FEED_META_SEGMENT_PATH } from "./uti import { ceilToNearestHour, floorToNearestHour, - getURLPath, + getURI, hashString, getYearsPaths, getSegmentsInYear, @@ -126,7 +126,7 @@ export class ChangeFeedFactory { } private static validateCursor(containerClient: ContainerClient, cursor: ChangeFeedCursor): void { - if (hashString(getURLPath(containerClient.url)!) !== cursor.urlHash) { + if (hashString(getURI(containerClient.url)!) !== cursor.urlHash) { throw new Error("Cursor URL does not match container URL."); } } diff --git a/sdk/storage/storage-blob-changefeed/src/Chunk.ts b/sdk/storage/storage-blob-changefeed/src/Chunk.ts index a2d14d0bbf71..416e9dd627ac 100644 --- a/sdk/storage/storage-blob-changefeed/src/Chunk.ts +++ b/sdk/storage/storage-blob-changefeed/src/Chunk.ts @@ -38,7 +38,11 @@ export class Chunk { if (next.done) { return undefined; } else { - return next.value as BlobChangeFeedEvent; + let eventRaw = next.value as BlobChangeFeedEvent; + if (eventRaw.eventTime) { + eventRaw.eventTime = new Date(eventRaw.eventTime); + } + return eventRaw; } } } diff --git a/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts index cf2e3379750f..83ea80c866f7 100644 --- a/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts +++ b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts @@ -4,7 +4,7 @@ export interface BlobChangeFeedEvent { topic: string; subject: string; eventType: BlobChangeFeedEventType; - eventTime: string; + eventTime: Date; id: string; // GUID data: BlobChangeFeedEventData; dataVersion?: string; @@ -25,6 +25,7 @@ export interface BlobChangeFeedEventData { sequencer: string; // For HNS only. + contentOffset?: number; destinationUrl?: string; sourceUrl?: string; recursive?: string; diff --git a/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts index 870f127eca8c..a1c574ccf13b 100644 --- a/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts @@ -5,7 +5,7 @@ import { BlobChangeFeedClient, BlobChangeFeedEvent, BlobChangeFeedEventPage } fr import * as dotenv from "dotenv"; dotenv.config(); -describe.only("BlobChangeFeedClient", async () => { +describe("BlobChangeFeedClient", async () => { const account = process.env.ACCOUNT_NAME || ""; const accountKey = process.env.ACCOUNT_KEY || ""; const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey); @@ -42,13 +42,13 @@ describe.only("BlobChangeFeedClient", async () => { if (i++ === 0) { assert.ok(event.eventType); assert.ok(event.data.blobType); - assert.ok(new Date(event.eventTime) >= startRounded); + assert.ok(event.eventTime >= startRounded); } lastEvent = event; } if (lastEvent) { - assert.ok(new Date(lastEvent.eventTime) < endRounded); + assert.ok(lastEvent.eventTime < endRounded); } }); @@ -99,7 +99,7 @@ describe.only("BlobChangeFeedClient", async () => { if (lastEventPage) { const lastEvent = lastEventPage.events[lastEventPage.events.length - 1]; - assert.ok(new Date(lastEvent.eventTime) < endRounded); + assert.ok(lastEvent.eventTime < endRounded); } }); }); diff --git a/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts b/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts index 37fc31b85bd7..af6ac21f08e0 100644 --- a/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts @@ -59,7 +59,7 @@ describe("Change Feed", async () => { containerClientStub = sinon.createStubInstance(ContainerClient); const blobClientStub = sinon.createStubInstance(BlobClient); segmentFactoryStub = sinon.createStubInstance(SegmentFactory); - changeFeedFactory = new ChangeFeedFactory(segmentFactoryStub); + changeFeedFactory = new ChangeFeedFactory(segmentFactoryStub as any); serviceClientStub.getContainerClient.returns(containerClientStub as any); containerClientStub.exists.resolves(true); From ae92a2641d74d362184e928c931c619087d1ba62 Mon Sep 17 00:00:00 2001 From: Lin Jian <1215122919@qq.com> Date: Wed, 10 Jun 2020 01:34:00 +0000 Subject: [PATCH 07/17] Update sdk/storage/storage-blob-changefeed/README.md Co-authored-by: Jeremy Meng --- sdk/storage/storage-blob-changefeed/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/storage-blob-changefeed/README.md b/sdk/storage/storage-blob-changefeed/README.md index f9e780c3a9b1..e1a7740c5ba9 100644 --- a/sdk/storage/storage-blob-changefeed/README.md +++ b/sdk/storage/storage-blob-changefeed/README.md @@ -22,7 +22,7 @@ Use the client libararies in this package to: ### Install the package -The preferred way to install the Azure Storage Blob client library for JavaScript is to use the npm package manager. Type the following into a terminal window: +The preferred way to install the Azure Storage Blob Change Feed client library for JavaScript is to use the npm package manager. Type the following into a terminal window: ```bash npm install @azure/storage-blob-changefeed From 7bb62fafa09bbbc4c55aa2b6a1f9c69f810f8438 Mon Sep 17 00:00:00 2001 From: Lin Jian <1215122919@qq.com> Date: Wed, 10 Jun 2020 01:34:31 +0000 Subject: [PATCH 08/17] Update sdk/storage/storage-blob-changefeed/README.md Co-authored-by: Jeremy Meng --- sdk/storage/storage-blob-changefeed/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/storage-blob-changefeed/README.md b/sdk/storage/storage-blob-changefeed/README.md index e1a7740c5ba9..d69a9bf2b10a 100644 --- a/sdk/storage/storage-blob-changefeed/README.md +++ b/sdk/storage/storage-blob-changefeed/README.md @@ -30,7 +30,7 @@ npm install @azure/storage-blob-changefeed ### Authenticate the client -This library use a `BlobServiceClient` to initialize, refer to [storage-blob](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob#authenticate-the-client) for how to authenticate a `BlobServiceClient`. +This library uses an authenticated `BlobServiceClient` to initialize. Refer to [storage-blob](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob#authenticate-the-client) for how to authenticate a `BlobServiceClient`. ### Compatibility From 2d7fcc320ab7333515d431d323b4e193b00a7af1 Mon Sep 17 00:00:00 2001 From: Lin Jian <1215122919@qq.com> Date: Wed, 10 Jun 2020 01:35:40 +0000 Subject: [PATCH 09/17] Update sdk/storage/storage-blob-changefeed/README.md Co-authored-by: Jeremy Meng --- sdk/storage/storage-blob-changefeed/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/storage-blob-changefeed/README.md b/sdk/storage/storage-blob-changefeed/README.md index d69a9bf2b10a..240d54c1356d 100644 --- a/sdk/storage/storage-blob-changefeed/README.md +++ b/sdk/storage/storage-blob-changefeed/README.md @@ -128,7 +128,7 @@ for await (const eventPage of changeFeedClient.getChanges().byPage({continuation Pass start time and end time to `BlobChangeFeedClient.getChanges()` to fetch events within a time range. -Note that for preview release, the change feed client will round start time down to the nearest hour, and round endTime up to the next hour. +Note that for this preview release, the change feed client will round start time down to the nearest hour, and round end time up to the next hour. ```javascript const { BlobChangeFeedEvent } = require("@azure/storage-blob-changefeed"); From 2a961fb1eb65c271d0bf700c9ead22f88d906a52 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Wed, 10 Jun 2020 11:40:19 +0800 Subject: [PATCH 10/17] PR comments --- sdk/storage/storage-blob-changefeed/README.md | 24 ++--- .../review/storage-blob-change-feed.api.md | 91 ------------------- .../src/AvroReaderFactory.ts | 6 +- .../src/BlobChangeFeedClient.ts | 10 +- .../storage-blob-changefeed/src/ChangeFeed.ts | 6 +- .../src/ChangeFeedFactory.ts | 6 +- .../src/ChunkFactory.ts | 6 +- .../src/SegmentFactory.ts | 4 +- .../storage-blob-changefeed/src/Shard.ts | 2 +- .../src/ShardFactory.ts | 4 +- .../src/utils/constants.ts | 2 +- .../src/utils/utils.common.ts | 2 +- .../test/changefeed.spec.ts | 24 ++--- .../test/segment.spec.ts | 6 +- .../test/shard.spec.ts | 14 +-- 15 files changed, 52 insertions(+), 155 deletions(-) delete mode 100644 sdk/storage/storage-blob-changefeed/review/storage-blob-change-feed.api.md diff --git a/sdk/storage/storage-blob-changefeed/README.md b/sdk/storage/storage-blob-changefeed/README.md index 240d54c1356d..6dae63a2da72 100644 --- a/sdk/storage/storage-blob-changefeed/README.md +++ b/sdk/storage/storage-blob-changefeed/README.md @@ -6,7 +6,7 @@ The purpose of the change feed is to provide transaction logs of all the changes This project provides a client library in JavaScript that makes it easy to consume the change feed. -Use the client libararies in this package to: +Use the client libraries in this package to: - Reading change feed events, all or within a time range - Resuming reading events from a saved position @@ -34,19 +34,7 @@ This library uses an authenticated `BlobServiceClient` to initialize. Refer to [ ### Compatibility -For this perview, this library is only compatible with Node.js. - -### CORS - -You need to set up [Cross-Origin Resource Sharing (CORS)](https://docs.microsoft.com/rest/api/storageservices/cross-origin-resource-sharing--cors--support-for-the-azure-storage-services) rules for your storage account if you need to develop for browsers. Go to Azure portal and Azure Storage Explorer, find your storage account, create new CORS rules for blob/queue/file/table service(s). - -For example, you can create following CORS settings for debugging. But please customize the settings carefully according to your requirements in production environment. - -- Allowed origins: \* -- Allowed verbs: DELETE,GET,HEAD,MERGE,POST,OPTIONS,PUT -- Allowed headers: \* -- Exposed headers: \* -- Maximum age (seconds): 86400 +For this preview, this library is only compatible with Node.js. ## Key concepts @@ -69,7 +57,7 @@ The `BlobChangeFeedClient` requires a `BlobServiceClient` to initialize. Refer t const account = ""; const accountKey = ""; // Use StorageSharedKeyCredential with storage account and account key - // StorageSharedKeyCredential is only avaiable in Node.js runtime, not in browsers + // StorageSharedKeyCredential is only available in Node.js runtime, not in browsers const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey); const blobServiceClient = new BlobServiceClient( `https://${account}.blob.core.windows.net`, @@ -157,9 +145,9 @@ setLogLevel("info"); More code samples: -- [Blob Storage Samples (JavaScript)](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/samples/javascript) -- [Blob Storage Samples (TypeScript)](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/samples/typescript) -- [Blob Storage Test Cases](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/test/) +- [Blob Storage Change Feed Samples (JavaScript)](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/samples/javascript) +- [Blob Storage Change Feed Samples (TypeScript)](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/samples/typescript) +- [Blob Storage Change Feed Test Cases](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/storage/storage-blob-changefeed/test/) ## Contributing diff --git a/sdk/storage/storage-blob-changefeed/review/storage-blob-change-feed.api.md b/sdk/storage/storage-blob-changefeed/review/storage-blob-change-feed.api.md deleted file mode 100644 index 6a7f4c6c3c9e..000000000000 --- a/sdk/storage/storage-blob-changefeed/review/storage-blob-change-feed.api.md +++ /dev/null @@ -1,91 +0,0 @@ -## API Report File for "@azure/storage-blob-changefeed" - -> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). - -```ts - -import { BlobServiceClient } from '@azure/storage-blob'; -import { PagedAsyncIterableIterator } from '@azure/core-paging'; - -// @public (undocumented) -export class BlobChangeFeedClient { - constructor(blobServiceClient: BlobServiceClient); - // (undocumented) - getChanges(options?: ChangeFeedGetChangesOptions): PagedAsyncIterableIterator; - } - -// @public (undocumented) -export interface BlobChangeFeedEvent { - // (undocumented) - data: BlobChangeFeedEventData; - // (undocumented) - dataVersion?: string; - // (undocumented) - eventTime: string; - // (undocumented) - eventType: BlobChangeFeedEventType; - // (undocumented) - id: string; - // (undocumented) - metadataVersion: string; - // (undocumented) - subject: string; - // (undocumented) - topic: string; -} - -// @public (undocumented) -export interface BlobChangeFeedEventData { - // (undocumented) - api: string; - // (undocumented) - blobType: BlobType; - // (undocumented) - clientRequestId: string; - // (undocumented) - contentLength: number; - // (undocumented) - contentType: string; - // (undocumented) - destinationUrl?: string; - // (undocumented) - eTag: string; - // (undocumented) - recursive?: string; - // (undocumented) - requestId: string; - // (undocumented) - sequencer: string; - // (undocumented) - sourceUrl?: string; - // (undocumented) - url: string; -} - -// @public (undocumented) -export class BlobChangeFeedEventPage { - constructor(); - // (undocumented) - continuationToken: string; - // (undocumented) - events: BlobChangeFeedEvent[]; -} - -// @public (undocumented) -export type BlobChangeFeedEventType = "BlobCreate" | "BlobDeleted"; - -// @public (undocumented) -export type BlobType = "BlockBlob" | "AppendBlob" | "PageBlob"; - -// @public (undocumented) -export interface ChangeFeedGetChangesOptions { - // (undocumented) - end?: Date; - // (undocumented) - start?: Date; -} - - -// (No @packageDocumentation comment for this package) - -``` diff --git a/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts b/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts index d2dc198ae591..59b59bd07233 100644 --- a/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/AvroReaderFactory.ts @@ -1,16 +1,16 @@ import { AvroReadable, AvroReader } from "../../storage-internal-avro/src"; export class AvroReaderFactory { - public buildAvroReader(headerAndDataStream: AvroReadable): AvroReader; + public create(headerAndDataStream: AvroReadable): AvroReader; - public buildAvroReader( + public create( dataStream: AvroReadable, headerStream: AvroReadable, blockOffset: number, eventIndex: number ): AvroReader; - public buildAvroReader( + public create( dataStream: AvroReadable, headerStream?: AvroReadable, blockOffset?: number, diff --git a/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts b/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts index 7b6f0f56a629..d42d96405d02 100644 --- a/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts +++ b/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts @@ -3,7 +3,7 @@ import { PagedAsyncIterableIterator, PageSettings } from "@azure/core-paging"; import { BlobChangeFeedEvent } from "./models/BlobChangeFeedEvent"; import { ChangeFeedFactory } from "./ChangeFeedFactory"; import { ChangeFeed } from "./ChangeFeed"; -import { CHANGE_FEED_DEFAULT_PAGE_SIZE } from "./utils/constants"; +import { CHANGE_FEED_MAX_PAGE_SIZE } from "./utils/constants"; export interface ChangeFeedGetChangesOptions { start?: Date; @@ -65,7 +65,7 @@ export class BlobChangeFeedClient { private async *getChange( options: ChangeFeedGetChangesOptions = {} ): AsyncIterableIterator { - const changeFeed: ChangeFeed = await this._changeFeedFactory.buildChangeFeed( + const changeFeed: ChangeFeed = await this._changeFeedFactory.create( this._blobServiceClient, undefined, options.start, @@ -88,15 +88,15 @@ export class BlobChangeFeedClient { maxPageSize?: number, options: ChangeFeedGetChangesOptions = {} ): AsyncIterableIterator { - const changeFeed: ChangeFeed = await this._changeFeedFactory.buildChangeFeed( + const changeFeed: ChangeFeed = await this._changeFeedFactory.create( this._blobServiceClient, continuationToken, options.start, options.end ); - if (!maxPageSize || maxPageSize > CHANGE_FEED_DEFAULT_PAGE_SIZE) { - maxPageSize = CHANGE_FEED_DEFAULT_PAGE_SIZE; + if (!maxPageSize || maxPageSize > CHANGE_FEED_MAX_PAGE_SIZE) { + maxPageSize = CHANGE_FEED_MAX_PAGE_SIZE; } while (changeFeed.hasNext()) { let eventPage = new BlobChangeFeedEventPage(); diff --git a/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts b/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts index 17ced578ada9..e68643e46693 100644 --- a/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts +++ b/sdk/storage/storage-blob-changefeed/src/ChangeFeed.ts @@ -99,7 +99,7 @@ export class ChangeFeed { return { cursorVersion: 1, - urlHash: hashString(getURI(this._containerClient!.url)!), + urlHash: hashString(getURI(this._containerClient!.url)), endTime: this._endTime?.toJSON(), currentSegmentCursor: this._currentSegment!.getCursor() }; @@ -117,7 +117,7 @@ export class ChangeFeed { // If the current segment is completed, remove it if (this._segments.length > 0) { - this._currentSegment = await this._segmentFactory!.buildSegment( + this._currentSegment = await this._segmentFactory!.create( this._containerClient!, this._segments.shift()! ); @@ -133,7 +133,7 @@ export class ChangeFeed { ); if (this._segments.length > 0) { - this._currentSegment = await this._segmentFactory!.buildSegment( + this._currentSegment = await this._segmentFactory!.create( this._containerClient!, this._segments.shift()! ); diff --git a/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts index f98a67f73f6c..9c28ccc55b1c 100644 --- a/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/ChangeFeedFactory.ts @@ -38,7 +38,7 @@ export class ChangeFeedFactory { } } - public async buildChangeFeed( + public async create( blobServiceClient: BlobServiceClient, continuationToken?: string, startTime?: Date, @@ -107,7 +107,7 @@ export class ChangeFeedFactory { if (segments.length === 0) { return new ChangeFeed(); } - const currentSegment: Segment = await this._segmentFactory.buildSegment( + const currentSegment: Segment = await this._segmentFactory.create( containerClient, segments.shift()!, cursor?.currentSegmentCursor @@ -126,7 +126,7 @@ export class ChangeFeedFactory { } private static validateCursor(containerClient: ContainerClient, cursor: ChangeFeedCursor): void { - if (hashString(getURI(containerClient.url)!) !== cursor.urlHash) { + if (hashString(getURI(containerClient.url)) !== cursor.urlHash) { throw new Error("Cursor URL does not match container URL."); } } diff --git a/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts b/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts index faa86327ae03..9f98201ebcf3 100644 --- a/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/ChunkFactory.ts @@ -11,7 +11,7 @@ export class ChunkFactory { this._avroReaderFactory = avroReaderFactory; } - public async buildChunk( + public async create( containerClient: ContainerClient, chunkPath: string, blockOffset?: number, @@ -28,14 +28,14 @@ export class ChunkFactory { if (blockOffset !== 0) { const headerDownloadRes = await blobClient.download(0); const headerStream = bodyToAvroReadable(headerDownloadRes); - avroReader = this._avroReaderFactory.buildAvroReader( + avroReader = this._avroReaderFactory.create( dataStream, headerStream, blockOffset, eventIndex ); } else { - avroReader = this._avroReaderFactory.buildAvroReader(dataStream); + avroReader = this._avroReaderFactory.create(dataStream); } return new Chunk(avroReader, blockOffset, eventIndex); diff --git a/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts b/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts index de56bc1f9f85..58a77fe52045 100644 --- a/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/SegmentFactory.ts @@ -23,7 +23,7 @@ export class SegmentFactory { this._shardFactory = shardFactory; } - public async buildSegment( + public async create( containerClient: ContainerClient, manifestPath: string, cursor?: SegmentCursor @@ -44,7 +44,7 @@ export class SegmentFactory { const containerPrefixLength = CHANGE_FEED_CONTAINER_NAME.length + 1; // "$blobchangefeed/" for (const shardPath of segmentManifest.chunkFilePaths) { - const shard: Shard = await this._shardFactory.buildShard( + const shard: Shard = await this._shardFactory.create( containerClient, shardPath.substring(containerPrefixLength), cursor?.shardCursors[i++] diff --git a/sdk/storage/storage-blob-changefeed/src/Shard.ts b/sdk/storage/storage-blob-changefeed/src/Shard.ts index 286b7cb56c6a..aee0a6be31e1 100644 --- a/sdk/storage/storage-blob-changefeed/src/Shard.ts +++ b/sdk/storage/storage-blob-changefeed/src/Shard.ts @@ -40,7 +40,7 @@ export class Shard { // Remove currentChunk if it doesn't have more events. if (!this._currentChunk.hasNext() && this._chunks.length > 0) { - this._currentChunk = await this._chunkFactory.buildChunk( + this._currentChunk = await this._chunkFactory.create( this._containerClient, this._chunks.shift()! ); diff --git a/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts b/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts index 04027988962b..069dd94b3c59 100644 --- a/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts +++ b/sdk/storage/storage-blob-changefeed/src/ShardFactory.ts @@ -10,7 +10,7 @@ export class ShardFactory { this._chunkFactory = chunkFactory; } - public async buildShard( + public async create( containerClient: ContainerClient, shardPath: string, shardCursor?: ShardCursor @@ -37,7 +37,7 @@ export class ShardFactory { chunks.splice(0, chunkIndex); } - const currentChunk = await this._chunkFactory.buildChunk( + const currentChunk = await this._chunkFactory.create( containerClient, chunks.shift()!, blockOffset, diff --git a/sdk/storage/storage-blob-changefeed/src/utils/constants.ts b/sdk/storage/storage-blob-changefeed/src/utils/constants.ts index 9838de65066c..5b4a0fae3965 100644 --- a/sdk/storage/storage-blob-changefeed/src/utils/constants.ts +++ b/sdk/storage/storage-blob-changefeed/src/utils/constants.ts @@ -1,6 +1,6 @@ export const CHANGE_FEED_CONTAINER_NAME: string = "$blobchangefeed"; export const CHANGE_FEED_META_SEGMENT_PATH: string = "meta/segments.json"; -export const CHANGE_FEED_DEFAULT_PAGE_SIZE: number = 5000; // align with rest API list operations +export const CHANGE_FEED_MAX_PAGE_SIZE: number = 5000; // align with rest API list operations export const CHANGE_FEED_STATUS_FINALIZED: string = "Finalized"; export const CHANGE_FEED_SEGMENT_PREFIX: string = "idx/segments/"; export const CHANGE_FEED_INITIALIZATION_SEGMENT: string = "1601"; diff --git a/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts b/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts index 182253e510ad..33908d6c3b7b 100644 --- a/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts +++ b/sdk/storage/storage-blob-changefeed/src/utils/utils.common.ts @@ -24,7 +24,7 @@ export function floorToNearestHour(date: Date | undefined): Date | undefined { * @param {string} url Source URL string * @returns {(string | undefined)} */ -export function getURI(url: string): string | undefined { +export function getURI(url: string): string { const urlParsed = URLBuilder.parse(url); return `${urlParsed.getHost()}${urlParsed.getPort()}${urlParsed.getPath()}`; } diff --git a/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts b/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts index af6ac21f08e0..e6cfcd321587 100644 --- a/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/changefeed.spec.ts @@ -84,7 +84,7 @@ describe("Change Feed", async () => { const segmentIter = listTwoArray(segmentsIn2019, segmentsIn2020); for (let i = 0; i < segmentCount; i++) { segmentStubs.push(sinon.createStubInstance(Segment)); - segmentFactoryStub.buildSegment + segmentFactoryStub.create .withArgs(sinon.match.any, (await segmentIter.next()).value.name) .resolves(segmentStubs[i] as any); } @@ -103,7 +103,7 @@ describe("Change Feed", async () => { it("no valid years in change feed container", async () => { const yearPaths = [{ kind: "prefix", name: "idx/segments/1601/" }]; containerClientStub.listBlobsByHierarchy.withArgs("/").returns(fakeList(yearPaths) as any); - const changeFeed = await changeFeedFactory.buildChangeFeed(serviceClientStub as any); + const changeFeed = await changeFeedFactory.create(serviceClientStub as any); assert.ok(!changeFeed.hasNext()); }); @@ -113,7 +113,7 @@ describe("Change Feed", async () => { { kind: "prefix", name: "idx/segments/2019/" } ]; containerClientStub.listBlobsByHierarchy.withArgs("/").returns(fakeList(yearPaths) as any); - const changeFeed = await changeFeedFactory.buildChangeFeed( + const changeFeed = await changeFeedFactory.create( serviceClientStub as any, undefined, new Date(Date.UTC(2020, 0)) @@ -134,7 +134,7 @@ describe("Change Feed", async () => { ]; containerClientStub.listBlobsFlat.returns(fakeList(segments) as any); - const changeFeed = await changeFeedFactory.buildChangeFeed( + const changeFeed = await changeFeedFactory.create( serviceClientStub as any, undefined, new Date(Date.UTC(2019, 5)) @@ -143,7 +143,7 @@ describe("Change Feed", async () => { }); it("getChange", async () => { - const changeFeed = await changeFeedFactory.buildChangeFeed( + const changeFeed = await changeFeedFactory.create( serviceClientStub as any, undefined, new Date(Date.UTC(2019, 0)) @@ -179,7 +179,7 @@ describe("Change Feed", async () => { it("with start and end time", async () => { // no valid segment between start and end - const changeFeed = await changeFeedFactory.buildChangeFeed( + const changeFeed = await changeFeedFactory.create( serviceClientStub as any, undefined, new Date(Date.UTC(2019, 2, 2, 21)), @@ -188,7 +188,7 @@ describe("Change Feed", async () => { assert.ok(!changeFeed.hasNext()); // end earlier than lastConsumable - const changeFeed2 = await changeFeedFactory.buildChangeFeed( + const changeFeed2 = await changeFeedFactory.create( serviceClientStub as any, undefined, new Date(Date.UTC(2019, 3, 3, 22)), @@ -204,7 +204,7 @@ describe("Change Feed", async () => { assert.equal(event2, undefined); //end later than lastConsumable - const changeFeed3 = await changeFeedFactory.buildChangeFeed( + const changeFeed3 = await changeFeedFactory.create( serviceClientStub as any, undefined, lastConsumable, @@ -214,7 +214,7 @@ describe("Change Feed", async () => { }); it("with continuation token", async () => { - const changeFeed = await changeFeedFactory.buildChangeFeed( + const changeFeed = await changeFeedFactory.create( serviceClientStub as any, undefined, new Date(Date.UTC(2020, 2, 2, 20)) @@ -224,7 +224,7 @@ describe("Change Feed", async () => { const containerUri = "https://account.blob.core.windows.net/$blobchangefeed"; (containerClientStub as any).url = containerUri; const cursor = changeFeed.getCursor(); - assert.deepStrictEqual(cursor.urlHash, hashString(getURI(containerUri)!)); + assert.deepStrictEqual(cursor.urlHash, hashString(getURI(containerUri))); segmentStubs[3].getCursor.returns({ shardCursors: [], @@ -232,7 +232,7 @@ describe("Change Feed", async () => { segmentTime: new Date(Date.UTC(2020, 2, 2, 20)).toJSON() }); const continuation = JSON.stringify(changeFeed.getCursor()); - const changeFeed2 = await changeFeedFactory.buildChangeFeed( + const changeFeed2 = await changeFeedFactory.create( serviceClientStub as any, continuation ); @@ -244,7 +244,7 @@ describe("Change Feed", async () => { sinon.stub(segmentStubs[4], "finalized").value(true); segmentStubs[3].hasNext.returns(false); segmentStubs[3].getChange.resolves(undefined); - const changeFeed3 = await changeFeedFactory.buildChangeFeed( + const changeFeed3 = await changeFeedFactory.create( serviceClientStub as any, continuation ); diff --git a/sdk/storage/storage-blob-changefeed/test/segment.spec.ts b/sdk/storage/storage-blob-changefeed/test/segment.spec.ts index ab99ed1ed108..657a527ec7f7 100644 --- a/sdk/storage/storage-blob-changefeed/test/segment.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/segment.spec.ts @@ -29,7 +29,7 @@ describe("Shard", async () => { shardStubs = []; for (let i = 0; i < shardCount; i++) { shardStubs.push(sinon.createStubInstance(Shard)); - shardFactoryStub.buildShard.onCall(i).returns(shardStubs[i]); + shardFactoryStub.create.onCall(i).returns(shardStubs[i]); shardStubs[i].hasNext.returns(true); shardStubs[i].getChange.returns(i); @@ -42,7 +42,7 @@ describe("Shard", async () => { it("getChange round robin in shards", async () => { const segmentFactory = new SegmentFactory(shardFactoryStub); - const segment = await segmentFactory.buildSegment(containerClientStub, manifestPath); + const segment = await segmentFactory.create(containerClientStub, manifestPath); assert.ok(segment.hasNext()); assert.equal(segment.dateTime.getTime(), dateTime.getTime()); assert.ok(segment.finalized); @@ -80,7 +80,7 @@ describe("Shard", async () => { it("init with non-zero shardIndex", async () => { const shardIndex = 1; const segmentFactory = new SegmentFactory(shardFactoryStub); - const segment = await segmentFactory.buildSegment(containerClientStub, manifestPath, { + const segment = await segmentFactory.create(containerClientStub, manifestPath, { shardIndex, shardCursors: [] } as any); diff --git a/sdk/storage/storage-blob-changefeed/test/shard.spec.ts b/sdk/storage/storage-blob-changefeed/test/shard.spec.ts index 2613e54aeb8d..f001b69fe2d8 100644 --- a/sdk/storage/storage-blob-changefeed/test/shard.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/shard.spec.ts @@ -22,7 +22,7 @@ describe("Shard", async () => { containerClientSub = sinon.createStubInstance(ContainerClient); containerClientSub.listBlobsFlat.callsFake(fakeListBlobsFlat); chunkFactoryStub = sinon.createStubInstance(ChunkFactory); - chunkFactoryStub.buildChunk.returns(chunkStub); + chunkFactoryStub.create.returns(chunkStub); }); afterEach(() => { @@ -40,9 +40,9 @@ describe("Shard", async () => { // build shard correctly const shardFactory = new ShardFactory(chunkFactoryStub as any); - const shard = await shardFactory.buildShard(containerClientSub as any, shardPath, shardCursor); + const shard = await shardFactory.create(containerClientSub as any, shardPath, shardCursor); assert.ok( - chunkFactoryStub.buildChunk.calledWith( + chunkFactoryStub.create.calledWith( containerClientSub, `${shardPath}000${chunkIndex}.avro` ) @@ -56,11 +56,11 @@ describe("Shard", async () => { nextChunkStub.hasNext.returns(true); const event = { id: "a" }; nextChunkStub.getChange.resolves(event as any); - chunkFactoryStub.buildChunk.returns(nextChunkStub); + chunkFactoryStub.create.returns(nextChunkStub); const change = await shard.getChange(); assert.ok( - chunkFactoryStub.buildChunk.calledWith( + chunkFactoryStub.create.calledWith( containerClientSub, `${shardPath}000${chunkIndex + 1}.avro` ) @@ -74,11 +74,11 @@ describe("Shard", async () => { nextChunkStub.getChange.resolves(undefined); const lastChunkStub = sinon.createStubInstance(Chunk); lastChunkStub.hasNext.returns(false); - chunkFactoryStub.buildChunk.returns(lastChunkStub); + chunkFactoryStub.create.returns(lastChunkStub); const change2 = await shard.getChange(); assert.ok( - chunkFactoryStub.buildChunk.calledWith( + chunkFactoryStub.create.calledWith( containerClientSub, `${shardPath}000${chunkIndex + 2}.avro` ) From 8eb61f350dc5e8323f89930d493ebde2bf897b23 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Wed, 10 Jun 2020 21:46:56 +0800 Subject: [PATCH 11/17] Rename eTag to etag in BlobChangeFeedEventData --- .../review/storage-blob-changefeed.api.md | 2 +- sdk/storage/storage-blob-changefeed/src/Chunk.ts | 6 +++++- .../src/models/BlobChangeFeedEvent.ts | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md index 1cc4ef1a162e..d9da33e36272 100644 --- a/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md +++ b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md @@ -51,7 +51,7 @@ export interface BlobChangeFeedEventData { // (undocumented) destinationUrl?: string; // (undocumented) - eTag: string; + etag: string; // (undocumented) recursive?: string; // (undocumented) diff --git a/sdk/storage/storage-blob-changefeed/src/Chunk.ts b/sdk/storage/storage-blob-changefeed/src/Chunk.ts index 416e9dd627ac..d23f79574111 100644 --- a/sdk/storage/storage-blob-changefeed/src/Chunk.ts +++ b/sdk/storage/storage-blob-changefeed/src/Chunk.ts @@ -38,10 +38,14 @@ export class Chunk { if (next.done) { return undefined; } else { - let eventRaw = next.value as BlobChangeFeedEvent; + let eventRaw = next.value as any; if (eventRaw.eventTime) { eventRaw.eventTime = new Date(eventRaw.eventTime); } + if (eventRaw.eTag) { + eventRaw.etag = eventRaw.eTag; + delete eventRaw.eTag; + } return eventRaw; } } diff --git a/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts index 83ea80c866f7..8d3f2440dd86 100644 --- a/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts +++ b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts @@ -17,7 +17,7 @@ export interface BlobChangeFeedEventData { api: string; clientRequestId: string; // GUID requestId: string; // GUID - eTag: string; + etag: string; contentType: string; contentLength: number; blobType: BlobType; From eac19bb35b587d4b01c17b124fd4cc0b7bbfa8db Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 11 Jun 2020 16:41:44 +0800 Subject: [PATCH 12/17] remove hns only properties in change event record --- .../src/models/BlobChangeFeedEvent.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts index 8d3f2440dd86..2b475a956210 100644 --- a/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts +++ b/sdk/storage/storage-blob-changefeed/src/models/BlobChangeFeedEvent.ts @@ -1,4 +1,6 @@ -export type BlobChangeFeedEventType = "BlobCreate" | "BlobDeleted"; +// https://msazure.visualstudio.com/One/_git/Storage-XStore?path=%2Fsrc%2FXTable%2FNotifications%2Flib%2FBlobChangeEventv4.json&version=GBmaster + +export type BlobChangeFeedEventType = "UnspecifiedEventType" | "BlobCreated" | "BlobDeleted" | "BlobPropertiesUpdated" | "BlobSnapshotCreated" | "Control" | "BlobTierChanged" | "BlobAsyncOperationInitiated" | "BlobMetadataUpdated"; export interface BlobChangeFeedEvent { topic: string; @@ -23,10 +25,4 @@ export interface BlobChangeFeedEventData { blobType: BlobType; url: string; sequencer: string; - - // For HNS only. - contentOffset?: number; - destinationUrl?: string; - sourceUrl?: string; - recursive?: string; } From 00e240185f5287ca58b00777962b1ccb660e7373 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 11 Jun 2020 17:37:38 +0800 Subject: [PATCH 13/17] fix minor bug in avro --- common/config/rush/pnpm-lock.yaml | 71 ++++++++++++++++++- .../.vscode/settings.json | 2 +- .../storage-blob-changefeed/package.json | 18 ++--- .../review/storage-blob-changefeed.api.md | 10 +-- .../samples/typscript/basic.ts | 26 +++---- .../samples/typscript/resume.ts | 42 +++++++++++ .../storage-internal-avro/package.json | 2 +- .../src/AvroReadableFromStream.ts | 10 +-- 8 files changed, 136 insertions(+), 45 deletions(-) create mode 100644 sdk/storage/storage-blob-changefeed/samples/typscript/resume.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 98af63ca439d..264f2107d6d1 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -25,6 +25,7 @@ dependencies: '@rush-temp/search-documents': 'file:projects/search-documents.tgz' '@rush-temp/service-bus': 'file:projects/service-bus.tgz' '@rush-temp/storage-blob': 'file:projects/storage-blob.tgz' + '@rush-temp/storage-blob-changefeed': 'file:projects/storage-blob-changefeed.tgz' '@rush-temp/storage-file-datalake': 'file:projects/storage-file-datalake.tgz' '@rush-temp/storage-file-share': 'file:projects/storage-file-share.tgz' '@rush-temp/storage-internal-avro': 'file:projects/storage-internal-avro.tgz' @@ -6741,14 +6742,14 @@ packages: node: '>=0.10.0' resolution: integrity: sha1-WIeWa7WCpFA6QetST301ARgVphM= - /ts-loader/6.2.2_typescript@3.8.3: + /ts-loader/6.2.2_typescript@3.9.5: dependencies: chalk: 2.4.2 enhanced-resolve: 4.1.1 loader-utils: 1.4.0 micromatch: 4.0.2 semver: 6.3.0 - typescript: 3.8.3 + typescript: 3.9.5 dev: false engines: node: '>=8.6' @@ -7838,7 +7839,7 @@ packages: shx: 0.3.2 sinon: 9.0.2 tough-cookie: 4.0.0 - ts-loader: 6.2.2_typescript@3.8.3 + ts-loader: 6.2.2_typescript@3.9.5 ts-node: 8.8.2_typescript@3.9.5 tslib: 2.0.0 tunnel: 0.0.6 @@ -8773,6 +8774,69 @@ packages: integrity: sha512-wvwki9VUk1OBa5vOKDNUtNv48hISwA4O1yQGklLVg3K/WdRmz/a7r4stTezbEKbjTk0mOYSU2hX3HgUSssIisA== tarball: 'file:projects/service-bus.tgz' version: 0.0.0 + 'file:projects/storage-blob-changefeed.tgz': + dependencies: + '@azure/core-tracing': 1.0.0-preview.8 + '@microsoft/api-extractor': 7.7.11 + '@opentelemetry/api': 0.6.1 + '@rollup/plugin-commonjs': 11.0.2_rollup@1.32.1 + '@rollup/plugin-multi-entry': 3.0.0_rollup@1.32.1 + '@rollup/plugin-node-resolve': 8.0.1_rollup@1.32.1 + '@rollup/plugin-replace': 2.3.1_rollup@1.32.1 + '@types/mocha': 7.0.2 + '@types/node': 8.10.59 + '@types/sinon': 9.0.4 + '@typescript-eslint/eslint-plugin': 2.27.0_2f26aa176a29afc2bd7e8bd79afb588b + '@typescript-eslint/parser': 2.27.0_eslint@6.8.0+typescript@3.9.5 + assert: 1.5.0 + cross-env: 7.0.2 + dotenv: 8.2.0 + downlevel-dts: 0.4.0 + es6-promise: 4.2.8 + eslint: 6.8.0 + eslint-config-prettier: 6.10.1_eslint@6.8.0 + eslint-plugin-no-null: 1.0.2_eslint@6.8.0 + eslint-plugin-no-only-tests: 2.4.0 + eslint-plugin-promise: 4.2.1 + esm: 3.2.25 + events: 3.1.0 + inherits: 2.0.4 + karma: 4.4.1 + karma-chrome-launcher: 3.1.0 + karma-coverage: 2.0.1 + karma-edge-launcher: 0.4.2_karma@4.4.1 + karma-env-preprocessor: 0.1.1 + karma-firefox-launcher: 1.3.0 + karma-ie-launcher: 1.0.0_karma@4.4.1 + karma-json-preprocessor: 0.3.3_karma@4.4.1 + karma-json-to-file-reporter: 1.0.1 + karma-junit-reporter: 2.0.1_karma@4.4.1 + karma-mocha: 1.3.0 + karma-mocha-reporter: 2.2.5_karma@4.4.1 + karma-remap-istanbul: 0.6.0_karma@4.4.1 + mocha: 7.1.1 + mocha-junit-reporter: 1.23.3_mocha@7.1.1 + nyc: 14.1.1 + prettier: 1.19.1 + puppeteer: 3.3.0 + rimraf: 3.0.2 + rollup: 1.32.1 + rollup-plugin-shim: 1.0.0 + rollup-plugin-sourcemaps: 0.4.2_rollup@1.32.1 + rollup-plugin-terser: 5.3.0_rollup@1.32.1 + rollup-plugin-visualizer: 4.0.4_rollup@1.32.1 + sinon: 9.0.2 + source-map-support: 0.5.16 + ts-node: 8.8.2_typescript@3.9.5 + tslib: 2.0.0 + typescript: 3.9.5 + util: 0.12.2 + dev: false + name: '@rush-temp/storage-blob-changefeed' + resolution: + integrity: sha512-DoG0fYwiYsjufvHaRJKJA6luLgv102q8ICSwVEyRnP2lE1eRfqLz2gPmVOABjRLIM4luom8WMNLV5OPo/FNXZQ== + tarball: 'file:projects/storage-blob-changefeed.tgz' + version: 0.0.0 'file:projects/storage-blob.tgz': dependencies: '@azure/core-tracing': 1.0.0-preview.8 @@ -9275,6 +9339,7 @@ specifiers: '@rush-temp/search-documents': 'file:./projects/search-documents.tgz' '@rush-temp/service-bus': 'file:./projects/service-bus.tgz' '@rush-temp/storage-blob': 'file:./projects/storage-blob.tgz' + '@rush-temp/storage-blob-changefeed': 'file:./projects/storage-blob-changefeed.tgz' '@rush-temp/storage-file-datalake': 'file:./projects/storage-file-datalake.tgz' '@rush-temp/storage-file-share': 'file:./projects/storage-file-share.tgz' '@rush-temp/storage-internal-avro': 'file:./projects/storage-internal-avro.tgz' diff --git a/sdk/storage/storage-blob-changefeed/.vscode/settings.json b/sdk/storage/storage-blob-changefeed/.vscode/settings.json index 7ceb5ace3e9d..71c65f9f433a 100644 --- a/sdk/storage/storage-blob-changefeed/.vscode/settings.json +++ b/sdk/storage/storage-blob-changefeed/.vscode/settings.json @@ -10,7 +10,7 @@ "editor.detectIndentation": false }, "[json]": { - "editor.formatOnSave": true, + "editor.formatOnSave": false, "editor.tabSize": 2, "editor.detectIndentation": false }, diff --git a/sdk/storage/storage-blob-changefeed/package.json b/sdk/storage/storage-blob-changefeed/package.json index 2b6172b9d42f..ca6e0a4ed34a 100644 --- a/sdk/storage/storage-blob-changefeed/package.json +++ b/sdk/storage/storage-blob-changefeed/package.json @@ -103,7 +103,7 @@ "@azure/logger": "^1.0.0", "@opentelemetry/api": "^0.6.1", "events": "^3.0.0", - "tslib": "^1.10.0" + "tslib": "^2.0.0" }, "devDependencies": { "@azure/identity": "^1.1.0-preview", @@ -113,11 +113,11 @@ "@rollup/plugin-replace": "^2.2.0", "@types/mocha": "^7.0.2", "@types/node": "^8.0.0", - "@types/sinon": "^7.0.13", + "@types/sinon": "^9.0.4", "@typescript-eslint/eslint-plugin": "^2.0.0", "@typescript-eslint/parser": "^2.0.0", "assert": "^1.4.1", - "cross-env": "^6.0.3", + "cross-env": "^7.0.2", "dotenv": "^8.2.0", "downlevel-dts": "~0.4.0", "es6-promise": "^4.2.5", @@ -145,19 +145,19 @@ "mocha-junit-reporter": "^1.18.0", "nyc": "^14.0.0", "prettier": "^1.16.4", - "puppeteer": "^2.0.0", + "puppeteer": "^3.3.0", "rimraf": "^3.0.0", "rollup": "^1.16.3", "@rollup/plugin-commonjs": "11.0.2", - "@rollup/plugin-node-resolve": "^7.0.0", + "@rollup/plugin-node-resolve": "^8.0.0", "rollup-plugin-shim": "^1.0.0", "rollup-plugin-sourcemaps": "^0.4.2", "rollup-plugin-terser": "^5.1.1", - "rollup-plugin-visualizer": "^3.1.1", + "rollup-plugin-visualizer": "^4.0.4", "source-map-support": "^0.5.9", "ts-node": "^8.3.0", - "typescript": "~3.8.3", + "typescript": "~3.9.3", "util": "^0.12.1", - "sinon": "^7.1.0" + "sinon": "^9.0.2" } -} +} \ No newline at end of file diff --git a/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md index d9da33e36272..d2784c73b7d3 100644 --- a/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md +++ b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md @@ -45,22 +45,14 @@ export interface BlobChangeFeedEventData { // (undocumented) contentLength: number; // (undocumented) - contentOffset?: number; - // (undocumented) contentType: string; // (undocumented) - destinationUrl?: string; - // (undocumented) etag: string; // (undocumented) - recursive?: string; - // (undocumented) requestId: string; // (undocumented) sequencer: string; // (undocumented) - sourceUrl?: string; - // (undocumented) url: string; } @@ -74,7 +66,7 @@ export class BlobChangeFeedEventPage { } // @public (undocumented) -export type BlobChangeFeedEventType = "BlobCreate" | "BlobDeleted"; +export type BlobChangeFeedEventType = "UnspecifiedEventType" | "BlobCreated" | "BlobDeleted" | "BlobPropertiesUpdated" | "BlobSnapshotCreated" | "Control" | "BlobTierChanged" | "BlobAsyncOperationInitiated" | "BlobMetadataUpdated"; // @public (undocumented) export type BlobType = "BlockBlob" | "AppendBlob" | "PageBlob"; diff --git a/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts b/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts index b981d16b0ae4..d60041802016 100644 --- a/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts +++ b/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts @@ -1,6 +1,5 @@ import { BlobServiceClient, StorageSharedKeyCredential } from "@azure/storage-blob"; -// import { BlobChangeFeedClient } from "@azure/storage-blob-changefeed"; -import { BlobChangeFeedClient } from "../../src"; +import { BlobChangeFeedClient, BlobChangeFeedEvent } from "../../src"; // Load the .env file if it exists import * as dotenv from "dotenv"; @@ -23,24 +22,15 @@ export async function main() { sharedKeyCredential ); - - const containerClient = blobServiceClient.getContainerClient("$blobchangefeed"); - console.log("List container.") - for await (const item of containerClient.listBlobsFlat()) { - console.log(`${item.name}: ${item.properties.contentLength}`); - } - const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); - let i = 0; - for await (const event of changeFeedClient.getChanges()) { - i++; - if (i <= 2) { - console.log(event); - } else { - break; - } + + const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be floor to 22:00 + const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be ceil to 22:00 + let changeFeedEvents: BlobChangeFeedEvent[] = []; + // You can also provide just a start or end time. + for await (const event of changeFeedClient.getChanges({ start, end })) { + changeFeedEvents.push(event); } - console.log(`event count: ${i}`); } main().catch((err) => { diff --git a/sdk/storage/storage-blob-changefeed/samples/typscript/resume.ts b/sdk/storage/storage-blob-changefeed/samples/typscript/resume.ts new file mode 100644 index 000000000000..37e1884e3699 --- /dev/null +++ b/sdk/storage/storage-blob-changefeed/samples/typscript/resume.ts @@ -0,0 +1,42 @@ +import { BlobServiceClient, StorageSharedKeyCredential } from "@azure/storage-blob"; +import { BlobChangeFeedClient, BlobChangeFeedEvent } from "../../src"; + +// Load the .env file if it exists +import * as dotenv from "dotenv"; +console.log(dotenv.config()); + +import { setLogLevel } from "@azure/logger"; +setLogLevel("info"); + +export async function main() { + // Enter your storage account name and shared key + const account = process.env.ACCOUNT_NAME || ""; + const accountKey = process.env.ACCOUNT_KEY || ""; + + // Use StorageSharedKeyCredential with storage account and account key + // StorageSharedKeyCredential is only available in Node.js runtime, not in browsers + const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey); + const blobServiceClient = new BlobServiceClient( + // When using AnonymousCredential, following url should include a valid SAS or support public access + `https://${account}.blob.core.windows.net`, + sharedKeyCredential + ); + + const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); + let changeFeedEvents: BlobChangeFeedEvent[] = []; + const firstPage = await changeFeedClient.getChanges().byPage({ maxPageSize: 10 }).next(); + for (const event of firstPage) { + changeFeedEvents.push(event); + } + + // Resume iterating from the pervious position with the continuationToken. + for await (const eventPage of changeFeedClient.getChanges().byPage({ continuationToken: firstPage.continuationToken })) { + for (const event of eventPage) { + changeFeedEvents.push(event); + } + } +} + +main().catch((err) => { + console.error("Error running sample:", err.message); +}); diff --git a/sdk/storage/storage-internal-avro/package.json b/sdk/storage/storage-internal-avro/package.json index 09b8b61300b7..8a1bb28f4acf 100644 --- a/sdk/storage/storage-internal-avro/package.json +++ b/sdk/storage/storage-internal-avro/package.json @@ -69,7 +69,7 @@ "rimraf": "^3.0.0", "rollup": "^1.16.3", "@rollup/plugin-commonjs": "11.0.2", - "@rollup/plugin-node-resolve": "^7.0.0", + "@rollup/plugin-node-resolve": "^8.0.0", "rollup-plugin-shim": "^1.0.0", "rollup-plugin-sourcemaps": "^0.4.2", "rollup-plugin-terser": "^5.1.1", diff --git a/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts index ac5412cacb47..0dd8e75b5a1d 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts @@ -56,12 +56,14 @@ export class AvroReadableFromStream extends AvroReadable { let chunk = this._readable.read(size); if (chunk) { this._position += chunk.length; + + this._readable.removeListener("readable", readableCallback); + this._readable.removeListener("error", rejectCallback); + this._readable.removeListener("end", rejectCallback); + this._readable.removeListener("close", rejectCallback); + // chunk.length maybe less than desired size if the stream ends. resolve(this.toUint8Array(chunk)); - this._readable.removeListener("readable", readableCallback); - this._readable.removeListener("error", reject); - this._readable.removeListener("end", reject); - this._readable.removeListener("close", reject); } }; From 4e60aeadb1b40b2d4f90dd51ea43fe4b6763b34d Mon Sep 17 00:00:00 2001 From: Lin Jian <1215122919@qq.com> Date: Fri, 12 Jun 2020 00:58:13 +0000 Subject: [PATCH 14/17] Update sdk/storage/storage-blob-changefeed/README.md Co-authored-by: Brian Terlson --- sdk/storage/storage-blob-changefeed/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/storage-blob-changefeed/README.md b/sdk/storage/storage-blob-changefeed/README.md index 6dae63a2da72..77263b248a8c 100644 --- a/sdk/storage/storage-blob-changefeed/README.md +++ b/sdk/storage/storage-blob-changefeed/README.md @@ -2,7 +2,7 @@ > Server Version: 2019-12-12 -The purpose of the change feed is to provide transaction logs of all the changes that occur to the blobs and the blob metadata in your storage account. The change feed provides ordered, guaranteed, durable, immutable, read-only log of these changes. Client applications can read these logs at any time. The change feed enables you to build efficient and scalable solutions that process change events that occur in your Blob Storage account at a low cost. +The change feed provides an ordered, guaranteed, durable, immutable, read-only transaction log of all the changes that occur to blobs and blob metadata in your storage account. Client applications can read these logs at any time. The change feed enables you to build efficient and scalable solutions that process change events that occur in your Blob Storage account at a low cost. This project provides a client library in JavaScript that makes it easy to consume the change feed. From f90b04a8084a116f1ad7dbde5bfbcff894015bd1 Mon Sep 17 00:00:00 2001 From: Lin Jian <1215122919@qq.com> Date: Fri, 12 Jun 2020 00:58:39 +0000 Subject: [PATCH 15/17] Update sdk/storage/storage-blob-changefeed/README.md Co-authored-by: Brian Terlson --- sdk/storage/storage-blob-changefeed/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/storage-blob-changefeed/README.md b/sdk/storage/storage-blob-changefeed/README.md index 77263b248a8c..fe0cfbe9ae3e 100644 --- a/sdk/storage/storage-blob-changefeed/README.md +++ b/sdk/storage/storage-blob-changefeed/README.md @@ -104,7 +104,7 @@ for (const event of firstPage) { changeFeedEvents.push(event); } -// Resume iterating from the pervious position with the continuationToken. +// Resume iterating from the previous position with the continuationToken. for await (const eventPage of changeFeedClient.getChanges().byPage({continuationToken: firstPage.continuationToken})) { for (const event of eventPage) { changeFeedEvents.push(event); From 999f4467d2a439b6b2e18c27344f026250908edc Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Fri, 12 Jun 2020 10:51:49 +0800 Subject: [PATCH 16/17] rename to BlobChangeFeedGetChangesOptions and fix comments --- sdk/storage/storage-blob-changefeed/README.md | 4 ++-- .../storage-blob-changefeed/samples/typscript/basic.ts | 4 ++-- .../storage-blob-changefeed/src/BlobChangeFeedClient.ts | 8 ++++---- .../test/blobchangefeedclient.spec.ts | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sdk/storage/storage-blob-changefeed/README.md b/sdk/storage/storage-blob-changefeed/README.md index fe0cfbe9ae3e..e48eaa0a9d8e 100644 --- a/sdk/storage/storage-blob-changefeed/README.md +++ b/sdk/storage/storage-blob-changefeed/README.md @@ -121,8 +121,8 @@ Note that for this preview release, the change feed client will round start time ```javascript const { BlobChangeFeedEvent } = require("@azure/storage-blob-changefeed"); -const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be floor to 22:00 -const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be ceil to 22:00 +const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be rounded down to 22:00 +const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be rounded up to 22:00 let changeFeedEvents : BlobChangeFeedEvent[] = []; // You can also provide just a start or end time. diff --git a/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts b/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts index d60041802016..9093a82e09a2 100644 --- a/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts +++ b/sdk/storage/storage-blob-changefeed/samples/typscript/basic.ts @@ -24,8 +24,8 @@ export async function main() { const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); - const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be floor to 22:00 - const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be ceil to 22:00 + const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be rounded down to 22:00 + const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be rounded up to 22:00 let changeFeedEvents: BlobChangeFeedEvent[] = []; // You can also provide just a start or end time. for await (const event of changeFeedClient.getChanges({ start, end })) { diff --git a/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts b/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts index d42d96405d02..8664d1f235bc 100644 --- a/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts +++ b/sdk/storage/storage-blob-changefeed/src/BlobChangeFeedClient.ts @@ -5,7 +5,7 @@ import { ChangeFeedFactory } from "./ChangeFeedFactory"; import { ChangeFeed } from "./ChangeFeed"; import { CHANGE_FEED_MAX_PAGE_SIZE } from "./utils/constants"; -export interface ChangeFeedGetChangesOptions { +export interface BlobChangeFeedGetChangesOptions { start?: Date; end?: Date; } @@ -37,7 +37,7 @@ export class BlobChangeFeedClient { } public getChanges( - options: ChangeFeedGetChangesOptions = {} + options: BlobChangeFeedGetChangesOptions = {} ): PagedAsyncIterableIterator { const iter = this.getChange(options); return { @@ -63,7 +63,7 @@ export class BlobChangeFeedClient { } private async *getChange( - options: ChangeFeedGetChangesOptions = {} + options: BlobChangeFeedGetChangesOptions = {} ): AsyncIterableIterator { const changeFeed: ChangeFeed = await this._changeFeedFactory.create( this._blobServiceClient, @@ -86,7 +86,7 @@ export class BlobChangeFeedClient { private async *getPage( continuationToken?: string, maxPageSize?: number, - options: ChangeFeedGetChangesOptions = {} + options: BlobChangeFeedGetChangesOptions = {} ): AsyncIterableIterator { const changeFeed: ChangeFeed = await this._changeFeedFactory.create( this._blobServiceClient, diff --git a/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts index a1c574ccf13b..e100620634e0 100644 --- a/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts +++ b/sdk/storage/storage-blob-changefeed/test/blobchangefeedclient.spec.ts @@ -15,7 +15,7 @@ describe("BlobChangeFeedClient", async () => { ); const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); - before(async function() { + before(async function () { if (process.env.CHANGE_FEED_ENABLED !== "1") { this.skip(); } @@ -34,9 +34,9 @@ describe("BlobChangeFeedClient", async () => { it("next(): with start and end time", async () => { let i = 0; let lastEvent: BlobChangeFeedEvent | undefined; - const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be rounded to 22:00 + const start = new Date(Date.UTC(2020, 1, 21, 22, 30, 0)); // will be rounded down to 22:00 const startRounded = new Date(Date.UTC(2020, 1, 21, 22, 0, 0)); - const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be rounded to 22:00 + const end = new Date(Date.UTC(2020, 4, 8, 21, 10, 0)); // will be rounded up to 22:00 const endRounded = new Date(Date.UTC(2020, 4, 8, 22, 0, 0)); for await (const event of changeFeedClient.getChanges({ start, end })) { if (i++ === 0) { @@ -114,7 +114,7 @@ describe("BlobChangeFeedClient: Change Feed not configured", async () => { ); const changeFeedClient = new BlobChangeFeedClient(blobServiceClient); - before(async function() { + before(async function () { if (process.env.CHANGE_FEED_ENABLED === "1") { this.skip(); } From 718c061cf52cf71fbc01002934f97e29e0b1cd19 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Fri, 12 Jun 2020 10:52:57 +0800 Subject: [PATCH 17/17] update api --- .../review/storage-blob-changefeed.api.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md index d2784c73b7d3..afd6db2f2c96 100644 --- a/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md +++ b/sdk/storage/storage-blob-changefeed/review/storage-blob-changefeed.api.md @@ -11,7 +11,7 @@ import { PagedAsyncIterableIterator } from '@azure/core-paging'; export class BlobChangeFeedClient { constructor(blobServiceClient: BlobServiceClient); // (undocumented) - getChanges(options?: ChangeFeedGetChangesOptions): PagedAsyncIterableIterator; + getChanges(options?: BlobChangeFeedGetChangesOptions): PagedAsyncIterableIterator; } // @public (undocumented) @@ -69,16 +69,16 @@ export class BlobChangeFeedEventPage { export type BlobChangeFeedEventType = "UnspecifiedEventType" | "BlobCreated" | "BlobDeleted" | "BlobPropertiesUpdated" | "BlobSnapshotCreated" | "Control" | "BlobTierChanged" | "BlobAsyncOperationInitiated" | "BlobMetadataUpdated"; // @public (undocumented) -export type BlobType = "BlockBlob" | "AppendBlob" | "PageBlob"; - -// @public (undocumented) -export interface ChangeFeedGetChangesOptions { +export interface BlobChangeFeedGetChangesOptions { // (undocumented) end?: Date; // (undocumented) start?: Date; } +// @public (undocumented) +export type BlobType = "BlockBlob" | "AppendBlob" | "PageBlob"; + // (No @packageDocumentation comment for this package)