From 3262acb3cbf1fdc89dc65e1891c1aa92ce439193 Mon Sep 17 00:00:00 2001 From: Lin Jian <1215122919@qq.com> Date: Thu, 21 May 2020 15:34:08 +0800 Subject: [PATCH] storage-internal-avro parser (#8893) * IReadable * Avro compiles * use storage-internal-avro as shared source in storage-blob * local build config * test and bug fix * rename IReadable to AvroReadable, separate the stream implementation with the interface declaration we need different implementation in browsers * add "private": true to pacakge.json * use Record * me * fix * tentative: use string enum instead of string union to avoid two copies of constant string Co-authored-by: Lin Jian --- common/config/rush/pnpm-lock.yaml | 91 ++++- rush.json | 5 + sdk/storage/storage-blob/api-extractor.json | 4 +- sdk/storage/storage-blob/package.json | 21 +- .../storage-blob/rollup.base.config.js | 12 +- sdk/storage/storage-blob/tsconfig.json | 4 +- .../.vscode/extensions.json | 3 + .../storage-internal-avro/.vscode/launch.json | 59 +++ .../.vscode/settings.json | 27 ++ sdk/storage/storage-internal-avro/README.md | 20 + .../storage-internal-avro/package.json | 82 ++++ .../rollup.base.config.js | 177 ++++++++ .../storage-internal-avro/rollup.config.js | 17 + .../rollup.test.config.js | 6 + .../src/AvroConstants.ts | 4 + .../storage-internal-avro/src/AvroParser.ts | 381 ++++++++++++++++++ .../storage-internal-avro/src/AvroReadable.ts | 4 + .../src/AvroReadableFromStream.ts | 59 +++ .../storage-internal-avro/src/AvroReader.ts | 142 +++++++ .../storage-internal-avro/src/index.ts | 3 + .../src/utils/utils.common.ts | 15 + .../test/avroreadable.spec.ts | 25 ++ .../test/avroreader.spec.ts | 59 +++ .../test/resources/test_null_0.avro | Bin 0 -> 75 bytes .../test/resources/test_null_1.avro | Bin 0 -> 88 bytes .../test/resources/test_null_10.avro | Bin 0 -> 153 bytes .../test/resources/test_null_11.avro | Bin 0 -> 213 bytes .../test/resources/test_null_12.avro | Bin 0 -> 105 bytes .../test/resources/test_null_13.avro | Bin 0 -> 157 bytes .../test/resources/test_null_14.avro | Bin 0 -> 358 bytes .../test/resources/test_null_2.avro | Bin 0 -> 308 bytes .../test/resources/test_null_3.avro | Bin 0 -> 177 bytes .../test/resources/test_null_4.avro | Bin 0 -> 94 bytes .../test/resources/test_null_5.avro | Bin 0 -> 95 bytes .../test/resources/test_null_6.avro | Bin 0 -> 116 bytes .../test/resources/test_null_7.avro | Bin 0 -> 158 bytes .../test/resources/test_null_8.avro | Bin 0 -> 123 bytes .../test/resources/test_null_9.avro | Bin 0 -> 134 bytes .../storage-internal-avro/tsconfig.json | 26 ++ 39 files changed, 1212 insertions(+), 34 deletions(-) create mode 100644 sdk/storage/storage-internal-avro/.vscode/extensions.json create mode 100644 sdk/storage/storage-internal-avro/.vscode/launch.json create mode 100644 sdk/storage/storage-internal-avro/.vscode/settings.json create mode 100644 sdk/storage/storage-internal-avro/README.md create mode 100644 sdk/storage/storage-internal-avro/package.json create mode 100644 sdk/storage/storage-internal-avro/rollup.base.config.js create mode 100644 sdk/storage/storage-internal-avro/rollup.config.js create mode 100644 sdk/storage/storage-internal-avro/rollup.test.config.js create mode 100644 sdk/storage/storage-internal-avro/src/AvroConstants.ts create mode 100644 sdk/storage/storage-internal-avro/src/AvroParser.ts create mode 100644 sdk/storage/storage-internal-avro/src/AvroReadable.ts create mode 100644 sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts create mode 100644 sdk/storage/storage-internal-avro/src/AvroReader.ts create mode 100644 sdk/storage/storage-internal-avro/src/index.ts create mode 100644 sdk/storage/storage-internal-avro/src/utils/utils.common.ts create mode 100644 sdk/storage/storage-internal-avro/test/avroreadable.spec.ts create mode 100644 sdk/storage/storage-internal-avro/test/avroreader.spec.ts create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_0.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_1.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_10.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_11.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_12.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_13.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_14.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_2.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_3.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_4.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_5.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_6.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_7.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_8.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_9.avro create mode 100644 sdk/storage/storage-internal-avro/tsconfig.json diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index e719193d48ce..c9cfa0167905 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -27,6 +27,7 @@ dependencies: '@rush-temp/storage-blob': 'file:projects/storage-blob.tgz' '@rush-temp/storage-file-datalake': 'file:projects/storage-file-datalake.tgz' '@rush-temp/storage-file-share': 'file:projects/storage-file-share.tgz' + '@rush-temp/storage-internal-avro': 'file:projects/storage-internal-avro.tgz' '@rush-temp/storage-queue': 'file:projects/storage-queue.tgz' '@rush-temp/template': 'file:projects/template.tgz' '@rush-temp/test-utils-perfstress': 'file:projects/test-utils-perfstress.tgz' @@ -7346,12 +7347,14 @@ packages: karma-mocha: 1.3.0 karma-mocha-reporter: 2.2.5_karma@4.4.1 karma-remap-istanbul: 0.6.0_karma@4.4.1 + karma-sourcemap-loader: 0.3.7 mocha: 7.1.1 mocha-junit-reporter: 1.23.3_mocha@7.1.1 nyc: 14.1.1 prettier: 1.19.1 rimraf: 3.0.2 rollup: 1.32.1 + rollup-plugin-shim: 1.0.0 rollup-plugin-sourcemaps: 0.4.2_rollup@1.32.1 rollup-plugin-terser: 5.3.0_rollup@1.32.1 rollup-plugin-visualizer: 3.3.2_rollup@1.32.1 @@ -7362,7 +7365,7 @@ packages: dev: false name: '@rush-temp/ai-form-recognizer' resolution: - integrity: sha512-vPtrVCxruasMOV692Byx4Hc1n9WdZRNrJwSCIlVhxaGMrdgI/sETYrnF23OqFgjg8nXJtNh19xaPwMTHOk4PJw== + integrity: sha512-drftp8AmeOVBMm6hfpBJnt/OziXXLqhNWfZo1oLqfQSBISJxghr/apNc4CKrcSp9qURbWaJkeANWuHxIVeYndQ== tarball: 'file:projects/ai-form-recognizer.tgz' version: 0.0.0 'file:projects/ai-text-analytics.tgz': @@ -7466,7 +7469,7 @@ packages: dev: false name: '@rush-temp/app-configuration' resolution: - integrity: sha512-SQNvkmhpg4EcfyYBmi7l3n60gHtqbUDk3mjTBDSBM33F/01q+CjQiagcClrN98g62mlxhFaK+N+T6q7Ex3g1Ng== + integrity: sha512-QesKkz6aJwUt+oVkUzm4ESihvMHTjl1/Ims68Ue1i9H7jevsWAMDX+U1G+7rb0xCtx013PZdoedC9Qb6dsj/fg== tarball: 'file:projects/app-configuration.tgz' version: 0.0.0 'file:projects/core-amqp.tgz': @@ -7627,7 +7630,7 @@ packages: dev: false name: '@rush-temp/core-auth' resolution: - integrity: sha512-FEhKPyW6XNIk4vlVBgYt+gwtDiSmOhAxTq9YBQxpLaqiyMVjZn25WfcD/zj2/ZLnE5VKQRTxt+cKF52iETQdPw== + integrity: sha512-a8h+Fir7HdvM06N5A+BKxvig7SmvI2pQPUnVNoL2HWpuW5wdcelVyVpODYRiA43kNPsodP0AIlqWJSZOPnwtiw== tarball: 'file:projects/core-auth.tgz' version: 0.0.0 'file:projects/core-http.tgz': @@ -7871,7 +7874,7 @@ packages: dev: false name: '@rush-temp/core-tracing' resolution: - integrity: sha512-GfFzTI4+ug82g0/v2qF8bQlPaoUFvHYrHMnn3qg+dkoJe523z5ppsclkIGwkC5nabXHHNE/nzxxM09dYQU99gA== + integrity: sha512-6lNv0IiPu0TlqcRff8jI141AEY7u/AQxmVK0+H8R+sFgB6T+ZW80B1sqOAW3GjHnKKtNQ8M8l+2SHtzHtF87PQ== tarball: 'file:projects/core-tracing.tgz' version: 0.0.0 'file:projects/cosmos.tgz': @@ -8003,6 +8006,7 @@ packages: cross-env: 6.0.3 debug: 4.1.1 dotenv: 8.2.0 + downlevel-dts: 0.4.0 eslint: 6.8.0 eslint-config-prettier: 6.10.1_eslint@6.8.0 eslint-plugin-no-null: 1.0.2_eslint@6.8.0 @@ -8044,7 +8048,7 @@ packages: dev: false name: '@rush-temp/event-hubs' resolution: - integrity: sha512-bW/dj5xOrcog/7Gw/922ikLp5WOES4Et87oiwjVTzEG9U2LfhstUBDahsAR0f6n6nr6A28xNA0B7h/H0G8qNkA== + integrity: sha512-SvD3McUNYWtff0uFqaogeUbmRCdqLiwVUsRO7SrkIjdVazSydRCU5cl5IUGRKCQ4IMPzGkbXN/yyN1tmG45uGw== tarball: 'file:projects/event-hubs.tgz' version: 0.0.0 'file:projects/event-processor-host.tgz': @@ -8288,7 +8292,7 @@ packages: dev: false name: '@rush-temp/keyvault-certificates' resolution: - integrity: sha512-+H4N1yRRmhJgOU/nza/+W6YVctvTvYZkw7+S+oBQ/WYeOLZR+WHAc6/b9tR3hxiAhri9wxvpb+voD6/DduXS+Q== + integrity: sha512-Jn9KFNDBX87QBJ1Lcjp78dYcE+dsMg5rYpio/JeY0V2mWlNdh7qTKLvCNYDBGbLA74vByV3WeKwIQryHAi+05g== tarball: 'file:projects/keyvault-certificates.tgz' version: 0.0.0 'file:projects/keyvault-keys.tgz': @@ -8354,7 +8358,7 @@ packages: dev: false name: '@rush-temp/keyvault-keys' resolution: - integrity: sha512-sgYATU21SAy92H4+m2jbkwp26PdeRYHMZWmQZ1Gj9+5ZFbLCEcHm4KYukZ66EwBdmIwwodVRg65hROm17zBzfg== + integrity: sha512-eoe4wq30U9rGPBp+IAHTibqqR8RCZQ0ijWtJutDZ8WaiHbmIKaSFbQRY1A0+N/xSYHSKrcZDwTZC3cEpeWHa9A== tarball: 'file:projects/keyvault-keys.tgz' version: 0.0.0 'file:projects/keyvault-secrets.tgz': @@ -8420,7 +8424,7 @@ packages: dev: false name: '@rush-temp/keyvault-secrets' resolution: - integrity: sha512-CRFOhh1AMS5pvaXG3ZM5NboFAkQ5UF21UvbWpdajYCPg5J8s1znnWn01Tc+0b24brxyyUEWDQo6Yzqx8lG/35Q== + integrity: sha512-PjKc2VsK3yhM2nLsw1SEAqNOeKw3mK5APQ/6AJC2QyHWP/lDYghPHH6ep/78LBThcm5K9uLFMwM49ZLnYsnY+A== tarball: 'file:projects/keyvault-secrets.tgz' version: 0.0.0 'file:projects/logger.tgz': @@ -8533,7 +8537,7 @@ packages: dev: false name: '@rush-temp/search-documents' resolution: - integrity: sha512-GYl0ZrFxMq1Qfkfihi1vHdOuSzQp3Q+Z3c2nd+u2RzXcVTiTRE4t0KnON8OUdyXYJvsjrHeLpBdOrwVuY2He/Q== + integrity: sha512-dSUxmkUpg7gC0P73FvSLK5Fr4E8WGpdgaycSvR4ZSA69VpYIodX65aU7uyfkHrpuFCkydY7fbBbaXYiWfmvLTA== tarball: 'file:projects/search-documents.tgz' version: 0.0.0 'file:projects/service-bus.tgz': @@ -8612,7 +8616,7 @@ packages: dev: false name: '@rush-temp/service-bus' resolution: - integrity: sha512-/sInDnVXuGr13OJ6o0ybAgSwdTi6MBF20+2cjGTjcLcHfjuYoVDKqZQOGQCCtUDobnIsx3KyN3sASXIrCv15wQ== + integrity: sha512-ySmiDTgbTBpMYgXNSDGpIzuHs+5raDZenJw/jEBXhioOYpNXL+oDYK5EZQ5S+BKMCmGXOD7QvrpQortMrf3IrA== tarball: 'file:projects/service-bus.tgz' version: 0.0.0 'file:projects/storage-blob.tgz': @@ -8674,7 +8678,7 @@ packages: dev: false name: '@rush-temp/storage-blob' resolution: - integrity: sha512-eR/qeWB9CgdJYsGNIepqVhQamsJILdvpwcQWeSUeQmytJ5yJ8V1fQ5KHDpww195EpgIufovJIQVrJflKdVLxvw== + integrity: sha512-qNcuXw81uBfRLvrBeMcFs/hCXl6FCAglnFkswE98KtSV3Pj2rtxYPQWJxG4/O2GkzCRyVkgxYtfb9uDTYJXZ8w== tarball: 'file:projects/storage-blob.tgz' version: 0.0.0 'file:projects/storage-file-datalake.tgz': @@ -8744,7 +8748,7 @@ packages: dev: false name: '@rush-temp/storage-file-datalake' resolution: - integrity: sha512-Tugmx2vtJKOUGPUbXkCh1Oj/9W0bJ0otjt8Wf0t8Yf/4QzkCfvHzKEFOBT6xUWALbNoSRnvRoxremV2yZrRLrw== + integrity: sha512-ax+/3IKbvusjZpXmloEUa+6cQrULHB6eHFpd6dNzQmQ0a1QOIg3n5uHIEY2X/1kQCOkcMWVITX+imkz6b9Gv2A== tarball: 'file:projects/storage-file-datalake.tgz' version: 0.0.0 'file:projects/storage-file-share.tgz': @@ -8806,9 +8810,67 @@ packages: dev: false name: '@rush-temp/storage-file-share' resolution: - integrity: sha512-u8LJYzcyLwixe5WEH9sDJx4DSChUybX6sHKuFTfyKC98299ipDbcnk7KlDlQkkpMA+4YC2vTE7Zj9jySMF9ezQ== + integrity: sha512-Kzv6N1reNbnI8qxNbwPIXTbMwBaDP9S43wPHoL8akm3ooY8ftesQtF6qCj39Tfw1HozCojH1QRdyahlqIAnTRg== tarball: 'file:projects/storage-file-share.tgz' version: 0.0.0 + 'file:projects/storage-internal-avro.tgz': + dependencies: + '@microsoft/api-extractor': 7.7.11 + '@rollup/plugin-commonjs': 11.0.2_rollup@1.32.1 + '@rollup/plugin-multi-entry': 3.0.0_rollup@1.32.1 + '@rollup/plugin-node-resolve': 7.1.1_rollup@1.32.1 + '@rollup/plugin-replace': 2.3.1_rollup@1.32.1 + '@types/mocha': 7.0.2 + '@types/node': 8.10.59 + '@typescript-eslint/eslint-plugin': 2.27.0_1b16601d03b675251fc711870248ce8d + '@typescript-eslint/parser': 2.27.0_eslint@6.8.0+typescript@3.8.3 + assert: 1.5.0 + cross-env: 6.0.3 + dotenv: 8.2.0 + downlevel-dts: 0.4.0 + es6-promise: 4.2.8 + eslint: 6.8.0 + eslint-config-prettier: 6.10.1_eslint@6.8.0 + eslint-plugin-no-null: 1.0.2_eslint@6.8.0 + eslint-plugin-no-only-tests: 2.4.0 + eslint-plugin-promise: 4.2.1 + esm: 3.2.25 + inherits: 2.0.4 + karma: 4.4.1 + karma-chrome-launcher: 3.1.0 + karma-coverage: 2.0.1 + karma-edge-launcher: 0.4.2_karma@4.4.1 + karma-env-preprocessor: 0.1.1 + karma-firefox-launcher: 1.3.0 + karma-ie-launcher: 1.0.0_karma@4.4.1 + karma-json-preprocessor: 0.3.3_karma@4.4.1 + karma-json-to-file-reporter: 1.0.1 + karma-junit-reporter: 2.0.1_karma@4.4.1 + karma-mocha: 1.3.0 + karma-mocha-reporter: 2.2.5_karma@4.4.1 + karma-remap-istanbul: 0.6.0_karma@4.4.1 + mocha: 7.1.1 + mocha-junit-reporter: 1.23.3_mocha@7.1.1 + nyc: 14.1.1 + prettier: 1.19.1 + puppeteer: 2.1.1 + rimraf: 3.0.2 + rollup: 1.32.1 + rollup-plugin-shim: 1.0.0 + rollup-plugin-sourcemaps: 0.4.2_rollup@1.32.1 + rollup-plugin-terser: 5.3.0_rollup@1.32.1 + rollup-plugin-visualizer: 3.3.2_rollup@1.32.1 + source-map-support: 0.5.16 + ts-node: 8.8.2_typescript@3.8.3 + tslib: 1.11.1 + typescript: 3.8.3 + util: 0.12.2 + dev: false + name: '@rush-temp/storage-internal-avro' + resolution: + integrity: sha512-xQ/8G7hl1NT0N067xLF7XdeD/Fhh+tiMlwEenqOHiUIZ8eMexdCJ5+MhnFOq33oqjMD8dcKku4BjyKQrXBFERg== + tarball: 'file:projects/storage-internal-avro.tgz' + version: 0.0.0 'file:projects/storage-queue.tgz': dependencies: '@azure/core-tracing': 1.0.0-preview.7 @@ -8867,7 +8929,7 @@ packages: dev: false name: '@rush-temp/storage-queue' resolution: - integrity: sha512-7fZ/BCFfKLRNtNOpIkszzJXjIxRDj15LDFPHlynzc02Wc1mxS0NDD7eEptC37QCaBbZzaBqjKnYIC6/O+T2lCA== + integrity: sha512-WcfDHWEOKo17l6dpW4UFbHJgEzQd438F+F61uequ6uc6OH1XAQ/QUU5dWGYiJz+/SU11vyfIW81ZiRsiihQMdg== tarball: 'file:projects/storage-queue.tgz' version: 0.0.0 'file:projects/template.tgz': @@ -9061,6 +9123,7 @@ specifiers: '@rush-temp/storage-blob': 'file:./projects/storage-blob.tgz' '@rush-temp/storage-file-datalake': 'file:./projects/storage-file-datalake.tgz' '@rush-temp/storage-file-share': 'file:./projects/storage-file-share.tgz' + '@rush-temp/storage-internal-avro': 'file:./projects/storage-internal-avro.tgz' '@rush-temp/storage-queue': 'file:./projects/storage-queue.tgz' '@rush-temp/template': 'file:./projects/template.tgz' '@rush-temp/test-utils-perfstress': 'file:./projects/test-utils-perfstress.tgz' diff --git a/rush.json b/rush.json index 003220b9bbef..6e3a14fa0877 100644 --- a/rush.json +++ b/rush.json @@ -447,6 +447,11 @@ "projectFolder": "sdk/servicebus/service-bus", "versionPolicyName": "client" }, + { + "packageName": "@azure/storage-internal-avro", + "projectFolder": "sdk/storage/storage-internal-avro", + "versionPolicyName": "utility" + }, { "packageName": "@azure/storage-blob", "projectFolder": "sdk/storage/storage-blob", diff --git a/sdk/storage/storage-blob/api-extractor.json b/sdk/storage/storage-blob/api-extractor.json index 9d52d2a63383..ede224adc3de 100644 --- a/sdk/storage/storage-blob/api-extractor.json +++ b/sdk/storage/storage-blob/api-extractor.json @@ -1,6 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", - "mainEntryPointFilePath": "typings/latest/src/index.d.ts", + "mainEntryPointFilePath": "typings/latest/storage-blob/src/index.d.ts", "docModel": { "enabled": false }, @@ -28,4 +28,4 @@ } } } -} +} \ No newline at end of file diff --git a/sdk/storage/storage-blob/package.json b/sdk/storage/storage-blob/package.json index 4ff1955e951c..c5b673095689 100644 --- a/sdk/storage/storage-blob/package.json +++ b/sdk/storage/storage-blob/package.json @@ -4,14 +4,14 @@ "version": "12.2.0", "description": "Microsoft Azure Storage SDK for JavaScript - Blob", "main": "./dist/index.js", - "module": "./dist-esm/src/index.js", + "module": "./dist-esm/storage-blob/src/index.js", "browser": { - "./dist-esm/src/index.js": "./dist-esm/src/index.browser.js", - "./dist-esm/src/credentials/StorageSharedKeyCredential.js": "./dist-esm/src/credentials/StorageSharedKeyCredential.browser.js", - "./dist-esm/src/utils/utils.node.js": "./dist-esm/src/utils/utils.browser.js", - "./dist-esm/test/utils/index.js": "./dist-esm/test/utils/index.browser.js", - "./dist-esm/src/BatchUtils.js": "./dist-esm/src/BatchUtils.browser.js", - "./dist-esm/src/BlobDownloadResponse.js": "./dist-esm/src/BlobDownloadResponse.browser.js", + "./dist-esm/storage-blob/src/index.js": "./dist-esm/storage-blob/src/index.browser.js", + "./dist-esm/storage-blob/src/credentials/StorageSharedKeyCredential.js": "./dist-esm/storage-blob/src/credentials/StorageSharedKeyCredential.browser.js", + "./dist-esm/storage-blob/src/utils/utils.node.js": "./dist-esm/storage-blob/src/utils/utils.browser.js", + "./dist-esm/storage-blob/test/utils/index.js": "./dist-esm/storage-blob/test/utils/index.browser.js", + "./dist-esm/storage-blob/src/BatchUtils.js": "./dist-esm/storage-blob/src/BatchUtils.browser.js", + "./dist-esm/storage-blob/src/BlobDownloadResponse.js": "./dist-esm/storage-blob/src/BlobDownloadResponse.browser.js", "fs": false, "os": false, "process": false @@ -46,7 +46,7 @@ "execute:samples": "npm run build:samples && npm run execute:js-samples && npm run execute:ts-samples", "format": "prettier --write --config ../../.prettierrc.json \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"", "integration-test:browser": "karma start --single-run", - "integration-test:node": "nyc mocha -r esm --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 300000 dist-esm/test/*.spec.js dist-esm/test/node/*.spec.js", + "integration-test:node": "nyc mocha -r esm --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 300000 dist-esm/storage-blob/test/*.spec.js dist-esm/storage-blob/test/node/*.spec.js", "integration-test": "npm run integration-test:node && npm run integration-test:browser", "lint:fix": "eslint -c ../../.eslintrc.old.json src test samples --ext .ts --fix", "lint": "eslint -c ../../.eslintrc.old.json src test samples --ext .ts -f html -o storage-blob-lintReport.html || exit 0", @@ -63,7 +63,8 @@ "files": [ "BreakingChanges.md", "dist/", - "dist-esm/src/", + "dist-esm/storage-blob/src/", + "dist-esm/storage-internal-avro/src/", "typings/latest/storage-blob.d.ts", "typings/3.1/storage-blob.d.ts", "README.md", @@ -161,4 +162,4 @@ "typescript": "~3.8.3", "util": "^0.12.1" } -} +} \ No newline at end of file diff --git a/sdk/storage/storage-blob/rollup.base.config.js b/sdk/storage/storage-blob/rollup.base.config.js index f0a687a6f317..d3ff2d0f3dfc 100644 --- a/sdk/storage/storage-blob/rollup.base.config.js +++ b/sdk/storage/storage-blob/rollup.base.config.js @@ -33,7 +33,7 @@ export function nodeConfig(test = false) { "util" ]; const baseConfig = { - input: "dist-esm/src/index.js", + input: "dist-esm/storage-blob/src/index.js", external: depNames.concat(externalNodeBuiltins), output: { file: "dist/index.js", @@ -65,9 +65,9 @@ export function nodeConfig(test = false) { if (test) { // entry point is every test file baseConfig.input = [ - "dist-esm/test/*.spec.js", - "dist-esm/test/node/*.spec.js", - "dist-esm/src/index.js" + "dist-esm/storage-blob/test/*.spec.js", + "dist-esm/storage-blob/test/node/*.spec.js", + "dist-esm/storage-blob/src/index.js" ]; baseConfig.plugins.unshift(multiEntry()); @@ -92,7 +92,7 @@ export function nodeConfig(test = false) { export function browserConfig(test = false) { const baseConfig = { - input: "dist-esm/src/index.browser.js", + input: "dist-esm/storage-blob/src/index.browser.js", output: { file: "dist-browser/azure-storage-blob.js", banner: banner, @@ -159,7 +159,7 @@ export function browserConfig(test = false) { }; if (test) { - baseConfig.input = ["dist-esm/test/*.spec.js", "dist-esm/test/browser/*.spec.js"]; + baseConfig.input = ["dist-esm/storage-blob/test/*.spec.js", "dist-esm/storage-blob/test/browser/*.spec.js"]; baseConfig.plugins.unshift(multiEntry({ exports: false })); baseConfig.output.file = "dist-test/index.browser.js"; // mark fs-extra as external diff --git a/sdk/storage/storage-blob/tsconfig.json b/sdk/storage/storage-blob/tsconfig.json index 7d0f438f54f6..f89af540f58e 100644 --- a/sdk/storage/storage-blob/tsconfig.json +++ b/sdk/storage/storage-blob/tsconfig.json @@ -21,6 +21,6 @@ "esModuleInterop": true }, "compileOnSave": true, - "exclude": ["node_modules", "./samples/**"], - "include": ["./src/**/*.ts", "./test/**/*.ts"] + "exclude": ["node_modules", "../storage-internal-avro/node_modules", "./samples/**"], + "include": ["./src/**/*.ts", "./test/**/*.ts", "../storage-internal-avro/**/*.ts"] } diff --git a/sdk/storage/storage-internal-avro/.vscode/extensions.json b/sdk/storage/storage-internal-avro/.vscode/extensions.json new file mode 100644 index 000000000000..c83e26348e1f --- /dev/null +++ b/sdk/storage/storage-internal-avro/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["esbenp.prettier-vscode"] +} diff --git a/sdk/storage/storage-internal-avro/.vscode/launch.json b/sdk/storage/storage-internal-avro/.vscode/launch.json new file mode 100644 index 000000000000..24dbfc9d74c4 --- /dev/null +++ b/sdk/storage/storage-internal-avro/.vscode/launch.json @@ -0,0 +1,59 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Debug Javascript Samples", + "program": "${workspaceFolder}/samples/javascript/basic.js", + "preLaunchTask": "npm: build:js-samples" + }, + { + "type": "node", + "request": "launch", + "name": "Debug Typescript Samples", + "program": "${workspaceFolder}/samples/typescript/basic.ts", + "preLaunchTask": "npm: build:ts-samples", + "outFiles": ["${workspaceFolder}/dist-esm/samples/typescript/*.js"] + }, + { + "type": "node", + "request": "launch", + "name": "Debug Mocha Test [Without Rollup]", + "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", + "args": [ + "-r", + "ts-node/register", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/test/*.spec.ts", + "${workspaceFolder}/test/node/*.spec.ts" + ], + "env": { "TS_NODE_COMPILER_OPTIONS": "{\"module\": \"commonjs\"}" }, + "envFile": "${workspaceFolder}/../.env", + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "protocol": "inspector" + }, + { + "type": "node", + "request": "launch", + "name": "Debug Unit Tests", + "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", + "args": [ + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/dist-test/index.node.js" + ], + "internalConsoleOptions": "openOnSessionStart", + "preLaunchTask": "npm: build:test" + } + ] +} diff --git a/sdk/storage/storage-internal-avro/.vscode/settings.json b/sdk/storage/storage-internal-avro/.vscode/settings.json new file mode 100644 index 000000000000..7ceb5ace3e9d --- /dev/null +++ b/sdk/storage/storage-internal-avro/.vscode/settings.json @@ -0,0 +1,27 @@ +{ + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.DS_Store": true + }, + "[typescript]": { + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.detectIndentation": false + }, + "[json]": { + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.detectIndentation": false + }, + "[yaml]": { + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.detectIndentation": false + }, + "editor.rulers": [ + 100 + ], + "typescript.preferences.quoteStyle": "double", + "javascript.preferences.quoteStyle": "double" + } \ No newline at end of file diff --git a/sdk/storage/storage-internal-avro/README.md b/sdk/storage/storage-internal-avro/README.md new file mode 100644 index 000000000000..72ce26939d78 --- /dev/null +++ b/sdk/storage/storage-internal-avro/README.md @@ -0,0 +1,20 @@ +# Azure Storage Internal Avro client library for JavaScript +- For internal use only. + +## Getting started +- For internal use only. + +## Key concepts +- For internal use only. + +## Examples +- For internal use only. + +## Troubleshooting +- For internal use only. + +## Next steps +- For internal use only. + +## Contributing +- For internal use only. \ No newline at end of file diff --git a/sdk/storage/storage-internal-avro/package.json b/sdk/storage/storage-internal-avro/package.json new file mode 100644 index 000000000000..9e628455afe0 --- /dev/null +++ b/sdk/storage/storage-internal-avro/package.json @@ -0,0 +1,82 @@ +{ + "name": "@azure/storage-internal-avro", + "sideEffect": false, + "private": true, + "author": "Microsoft Corporation", + "version": "1.0.0-preview.1", + "description": "internal avro parser", + "license": "MIT", + "repository": "github:Azure/azure-sdk-for-js", + "main": "./srt/index.ts", + "module": "dist-esm/index.js", + "types": "./types/index.d.ts", + "engines": { + "node": ">=8.0.0" + }, + "scripts": { + "build:es6": "tsc -p tsconfig.json", + "build:nodebrowser": "rollup -c 2>&1", + "build": "npm run build:es6 && npm run build:nodebrowser", + "build:test": "npm run build:es6 && rollup -c rollup.test.config.js 2>&1", + "clean": "rimraf dist dist-esm dist-test typings temp dist-browser/*.js* dist-browser/*.zip statistics.html coverage coverage-browser .nyc_output *.tgz *.log test*.xml TEST*.xml", + "unit-test:node": "mocha --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 120000 dist-test/index.node.js", + "test:node": "npm run clean && npm run build:test && npm run unit-test:node", + "format": "prettier --write --config ../../.prettierrc.json \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"" + }, + "dependencies": { + "tslib": "^1.10.0" + }, + "devDependencies": { + "@azure/identity": "^1.1.0-preview", + "@azure/test-utils-recorder": "^1.0.0", + "@microsoft/api-extractor": "7.7.11", + "@rollup/plugin-multi-entry": "^3.0.0", + "@rollup/plugin-replace": "^2.2.0", + "@types/mocha": "^7.0.2", + "@types/node": "^8.0.0", + "@typescript-eslint/eslint-plugin": "^2.0.0", + "@typescript-eslint/parser": "^2.0.0", + "assert": "^1.4.1", + "cross-env": "^6.0.3", + "dotenv": "^8.2.0", + "downlevel-dts": "~0.4.0", + "es6-promise": "^4.2.5", + "eslint": "^6.1.0", + "eslint-config-prettier": "^6.0.0", + "eslint-plugin-no-null": "^1.0.2", + "eslint-plugin-no-only-tests": "^2.3.0", + "eslint-plugin-promise": "^4.1.1", + "esm": "^3.2.18", + "inherits": "^2.0.3", + "karma": "^4.0.1", + "karma-chrome-launcher": "^3.0.0", + "karma-coverage": "^2.0.0", + "karma-edge-launcher": "^0.4.2", + "karma-env-preprocessor": "^0.1.1", + "karma-firefox-launcher": "^1.1.0", + "karma-ie-launcher": "^1.0.0", + "karma-json-preprocessor": "^0.3.3", + "karma-json-to-file-reporter": "^1.0.1", + "karma-junit-reporter": "^2.0.1", + "karma-mocha": "^1.3.0", + "karma-mocha-reporter": "^2.2.5", + "karma-remap-istanbul": "^0.6.0", + "mocha": "^7.1.1", + "mocha-junit-reporter": "^1.18.0", + "nyc": "^14.0.0", + "prettier": "^1.16.4", + "puppeteer": "^2.0.0", + "rimraf": "^3.0.0", + "rollup": "^1.16.3", + "@rollup/plugin-commonjs": "^11.0.1", + "@rollup/plugin-node-resolve": "^7.0.0", + "rollup-plugin-shim": "^1.0.0", + "rollup-plugin-sourcemaps": "^0.4.2", + "rollup-plugin-terser": "^5.1.1", + "rollup-plugin-visualizer": "^3.1.1", + "source-map-support": "^0.5.9", + "ts-node": "^8.3.0", + "typescript": "~3.8.3", + "util": "^0.12.1" + } +} \ No newline at end of file diff --git a/sdk/storage/storage-internal-avro/rollup.base.config.js b/sdk/storage/storage-internal-avro/rollup.base.config.js new file mode 100644 index 000000000000..6c638d082ff5 --- /dev/null +++ b/sdk/storage/storage-internal-avro/rollup.base.config.js @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import nodeResolve from "@rollup/plugin-node-resolve"; +import multiEntry from "@rollup/plugin-multi-entry"; +import cjs from "@rollup/plugin-commonjs"; +import replace from "@rollup/plugin-replace"; +import { terser } from "rollup-plugin-terser"; +import sourcemaps from "rollup-plugin-sourcemaps"; +import shim from "rollup-plugin-shim"; +// import visualizer from "rollup-plugin-visualizer"; + +const version = require("./package.json").version; +const banner = [ + "/*!", + ` * Azure Storage SDK for JavaScript - Blob, ${version}`, + " * Copyright (c) Microsoft and contributors. All rights reserved.", + " */" +].join("\n"); + +const pkg = require("./package.json"); +const depNames = Object.keys(pkg.dependencies); +const production = process.env.NODE_ENV === "production"; + +export function nodeConfig(test = false) { + const externalNodeBuiltins = [ + "@azure/core-http", + "crypto", + "fs", + "events", + "os", + "stream", + "util" + ]; + const baseConfig = { + input: "dist-esm/src/index.js", + external: depNames.concat(externalNodeBuiltins), + output: { + file: "dist/index.js", + format: "cjs", + sourcemap: true + }, + preserveSymlinks: false, + plugins: [ + sourcemaps(), + replace({ + delimiters: ["", ""], + values: { + // replace dynamic checks with if (true) since this is for node only. + // Allows rollup's dead code elimination to be more aggressive. + "if (isNode)": "if (true)" + } + }), + nodeResolve({ preferBuiltins: true }), + cjs() + ], + onwarn(warning, warn) { + if (warning.code === "CIRCULAR_DEPENDENCY") { + throw new Error(warning.message); + } + warn(warning); + } + }; + + if (test) { + // entry point is every test file + baseConfig.input = [ + "dist-esm/test/*.spec.js", + "dist-esm/test/node/*.spec.js", + "dist-esm/src/index.js" + ]; + baseConfig.plugins.unshift(multiEntry()); + + // different output file + baseConfig.output.file = "dist-test/index.node.js"; + + // mark assert as external + baseConfig.external.push("assert", "fs", "path", "buffer", "zlib"); + + baseConfig.context = "null"; + + // Disable tree-shaking of test code. In rollup-plugin-node-resolve@5.0.0, rollup started respecting + // the "sideEffects" field in package.json. Since our package.json sets "sideEffects=false", this also + // applies to test code, which causes all tests to be removed by tree-shaking. + baseConfig.treeshake = false; + } else if (production) { + baseConfig.plugins.push(terser()); + } + + return baseConfig; +} + +export function browserConfig(test = false) { + const baseConfig = { + input: "dist-esm/src/index.browser.js", + output: { + file: "dist-browser/azure-internal-avro.js", + banner: banner, + format: "umd", + name: "azblob", + sourcemap: true + }, + preserveSymlinks: false, + plugins: [ + sourcemaps(), + replace({ + delimiters: ["", ""], + values: { + // replace dynamic checks with if (false) since this is for + // browser only. Rollup's dead code elimination will remove + // any code guarded by if (isNode) { ... } + "if (isNode)": "if (false)" + } + }), + // fs and os are not used by the browser bundle, so just shim it + // dotenv doesn't work in the browser, so replace it with a no-op function + shim({ + dotenv: `export function config() { }`, + fs: ` + export function stat() { } + export function createReadStream() { } + export function createWriteStream() { } + `, + os: ` + export const type = 1; + export const release = 1; + `, + util: ` + export function promisify() { } + ` + }), + nodeResolve({ + mainFields: ["module", "browser"], + preferBuiltins: false + }), + cjs({ + namedExports: { + events: ["EventEmitter"], + assert: [ + "ok", + "deepEqual", + "equal", + "fail", + "strictEqual", + "deepStrictEqual", + "notDeepEqual", + "notDeepStrictEqual" + ], + "@opentelemetry/api": ["CanonicalCode", "SpanKind", "TraceFlags"] + } + }) + ], + onwarn(warning, warn) { + if (warning.code === "CIRCULAR_DEPENDENCY") { + throw new Error(warning.message); + } + warn(warning); + } + }; + + if (test) { + baseConfig.input = ["dist-esm/test/*.spec.js", "dist-esm/test/browser/*.spec.js"]; + baseConfig.plugins.unshift(multiEntry({ exports: false })); + baseConfig.output.file = "dist-test/index.browser.js"; + // mark fs-extra as external + baseConfig.external = ["fs-extra"]; + + baseConfig.context = "null"; + + // Disable tree-shaking of test code. In rollup-plugin-node-resolve@5.0.0, rollup started respecting + // the "sideEffects" field in package.json. Since our package.json sets "sideEffects=false", this also + // applies to test code, which causes all tests to be removed by tree-shaking. + baseConfig.treeshake = false; + } + + return baseConfig; +} diff --git a/sdk/storage/storage-internal-avro/rollup.config.js b/sdk/storage/storage-internal-avro/rollup.config.js new file mode 100644 index 000000000000..a62dabd573b4 --- /dev/null +++ b/sdk/storage/storage-internal-avro/rollup.config.js @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as base from "./rollup.base.config"; + +const inputs = []; + +if (!process.env.ONLY_BROWSER) { + inputs.push(base.nodeConfig()); +} + +// Disable this until we are ready to run rollup for the browser. +// if (!process.env.ONLY_NODE) { +// inputs.push(base.browserConfig()); +// } + +export default inputs; diff --git a/sdk/storage/storage-internal-avro/rollup.test.config.js b/sdk/storage/storage-internal-avro/rollup.test.config.js new file mode 100644 index 000000000000..ad98718cce46 --- /dev/null +++ b/sdk/storage/storage-internal-avro/rollup.test.config.js @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as base from "./rollup.base.config"; + +export default [base.nodeConfig(true), base.browserConfig(true)]; diff --git a/sdk/storage/storage-internal-avro/src/AvroConstants.ts b/sdk/storage/storage-internal-avro/src/AvroConstants.ts new file mode 100644 index 000000000000..c5368c596246 --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/AvroConstants.ts @@ -0,0 +1,4 @@ +export const AVRO_SYNC_MARKER_SIZE: number = 16; +export const AVRO_INIT_BYTES: Uint8Array = new Uint8Array([79, 98, 106, 1]); +export const AVRO_CODEC_KEY: string = "avro.codec"; +export const AVRO_SCHEMA_KEY: string = "avro.schema"; diff --git a/sdk/storage/storage-internal-avro/src/AvroParser.ts b/sdk/storage/storage-internal-avro/src/AvroParser.ts new file mode 100644 index 000000000000..b63a347b4375 --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/AvroParser.ts @@ -0,0 +1,381 @@ +import { AvroReadable } from "./AvroReadable"; +import { KeyValuePair } from "./utils/utils.common"; + +export class AvroParser { + /** + * Reads a fixed number of bytes from the stream. + * + * @static + * @param stream + * @param length + */ + public static async readFixedBytes(stream: AvroReadable, length: number): Promise { + const bytes = await stream.read(length); + if (bytes.length != length) { + throw new Error("Hit stream end."); + } + return bytes; + } + + /** + * Reads a single byte from the stream. + * + * @static + * @param stream + */ + private static async readByte(stream: AvroReadable): Promise { + const buf = await AvroParser.readFixedBytes(stream, 1); + return buf[0]; + } + + private static async readZigZagLong(stream: AvroReadable): Promise { + // copied from https://github.com/apache/avro/blob/master/lang/js/lib/utils.js#L321 + let n = 0; + let k = 0; + let b, h, f, fk; + + do { + b = await AvroParser.readByte(stream); + h = b & 0x80; + n |= (b & 0x7f) << k; + k += 7; + } while (h && k < 28); + + if (h) { + // Switch to float arithmetic, otherwise we might overflow. + f = n; + fk = 268435456; // 2 ** 28. + do { + b = await AvroParser.readByte(stream); + f += (b & 0x7f) * fk; + fk *= 128; + } while (b & 0x80); + return (f % 2 ? -(f + 1) : f) / 2; + } + + return (n >> 1) ^ -(n & 1); + } + + public static async readLong(stream: AvroReadable): Promise { + return AvroParser.readZigZagLong(stream); + } + + public static async readInt(stream: AvroReadable): Promise { + return AvroParser.readZigZagLong(stream); + } + + public static async readNull(): Promise { + return null; + } + + public static async readBoolean(stream: AvroReadable): Promise { + const b = await AvroParser.readByte(stream); + if (b == 1) { + return true; + } else if (b == 0) { + return false; + } else { + throw new Error("Byte was not a boolean."); + } + } + + public static async readFloat(stream: AvroReadable): Promise { + const u8arr = await AvroParser.readFixedBytes(stream, 4); + const view = new DataView(u8arr.buffer, u8arr.byteOffset, u8arr.byteLength); + return view.getFloat32(0, true); // littleEndian = true + } + + public static async readDouble(stream: AvroReadable): Promise { + const u8arr = await AvroParser.readFixedBytes(stream, 8); + const view = new DataView(u8arr.buffer, u8arr.byteOffset, u8arr.byteLength); + return view.getFloat64(0, true); // littleEndian = true + } + + public static async readBytes(stream: AvroReadable): Promise { + const size = await AvroParser.readLong(stream); + if (size < 0) { + throw new Error("Bytes size was negative."); + } + + return await stream.read(size); + } + + public static async readString(stream: AvroReadable): Promise { + const u8arr = await AvroParser.readBytes(stream); + + // FIXME: need TextDecoder polyfill for IE + let utf8decoder = new TextDecoder(); + return utf8decoder.decode(u8arr); + } + + private static async readMapPair( + stream: AvroReadable, + readItemMethod: (s: AvroReadable) => Promise + ): Promise> { + const key = await AvroParser.readString(stream); + // FIXME: what about readFixed which need a length as parameter. + const value = await readItemMethod(stream); + return { key, value }; + } + + public static async readMap( + stream: AvroReadable, + readItemMethod: (s: AvroReadable) => Promise + ): Promise> { + const readPairMethod = async (stream: AvroReadable): Promise> => { + return await AvroParser.readMapPair(stream, readItemMethod); + }; + + const pairs: KeyValuePair[] = await AvroParser.readArray(stream, readPairMethod); + let dict: Record = {}; + for (const pair of pairs) { + dict[pair.key] = pair.value; + } + return dict; + } + + private static async readArray( + stream: AvroReadable, + readItemMethod: (s: AvroReadable) => Promise + ): Promise { + let items: T[] = []; + for ( + let count = await AvroParser.readLong(stream); + count != 0; + count = await AvroParser.readLong(stream) + ) { + if (count < 0) { + // Ignore block sizes + await AvroParser.readLong(stream); + count = -count; + } + + while (count--) { + const item: T = await readItemMethod(stream); + items.push(item); + } + } + return items; + } +} + +interface RecordField { + name: string; + type: string | ObjectSchema | (string | ObjectSchema)[]; // Unions may not immediately contain other unions. +} + +enum AvroComplex { + RECORD = 'record', + ENUM = 'enum', + ARRAY = 'array', + MAP = 'map', + UNION = 'union', + FIXED = 'fixed', +} + +interface ObjectSchema { + type: Exclude; + name?: string; + aliases?: string; + fields?: RecordField[]; + symbols?: string[]; + values?: string; + size?: number; +} + +export abstract class AvroType { + /** + * Reads an object from the stream. + * + * @param stream + */ + public abstract read(stream: AvroReadable): Promise; + + /** + * Determines the AvroType from the Avro Schema. + */ + public static fromSchema(schema: string | Object): AvroType { + if (typeof schema == "string") { + return AvroType.fromStringSchema(schema); + } else if (Array.isArray(schema)) { + return AvroType.fromArraySchema(schema); + } else { + return AvroType.fromObjectSchema(schema as ObjectSchema); + } + } + + private static fromStringSchema(schema: string): AvroType { + // FIXME: simpler way to tell if schema is of type AvroPrimitive? + switch (schema) { + case AvroPrimitive.NULL: + case AvroPrimitive.BOOLEAN: + case AvroPrimitive.INT: + case AvroPrimitive.LONG: + case AvroPrimitive.FLOAT: + case AvroPrimitive.DOUBLE: + case AvroPrimitive.BYTES: + case AvroPrimitive.STRING: + return new AvroPrimitiveType(schema as AvroPrimitive); + default: + throw new Error(`Unexpected Avro type ${schema}`); + } + } + + private static fromArraySchema(schema: any[]): AvroType { + return new AvroUnionType(schema.map(AvroType.fromSchema)); + } + + private static fromObjectSchema(schema: ObjectSchema): AvroType { + const type = schema.type; + // Primitives can be defined as strings or objects + try { + return AvroType.fromStringSchema(type); + } catch (err) { } + + switch (type) { + case AvroComplex.RECORD: + if (schema.aliases) { + throw new Error(`aliases currently is not supported, schema: ${schema}`); + } + if (!schema.name) { + throw new Error(`Required attribute 'name' doesn't exist on schema: ${schema}`); + } + + let fields: Record = {}; + if (!schema.fields) { + throw new Error(`Required attribute 'fields' doesn't exist on schema: ${schema}`); + } + for (const field of schema.fields) { + fields[field.name] = AvroType.fromSchema(field.type); + } + return new AvroRecordType(fields, schema.name); + case AvroComplex.ENUM: + if (schema.aliases) { + throw new Error(`aliases currently is not supported, schema: ${schema}`); + } + if (!schema.symbols) { + throw new Error(`Required attribute 'symbols' doesn't exist on schema: ${schema}`); + } + return new AvroEnumType(schema.symbols); + case AvroComplex.MAP: + if (!schema.values) { + throw new Error(`Required attribute 'values' doesn't exist on schema: ${schema}`); + } + return new AvroMapType(AvroType.fromSchema(schema.values)); + case AvroComplex.ARRAY: // Unused today + case AvroComplex.FIXED: // Unused today + default: + throw new Error(`Unexpected Avro type ${type} in ${schema}`); + } + } +} + +enum AvroPrimitive { + NULL = "null", + BOOLEAN = 'boolean', + INT = 'int', + LONG = 'long', + FLOAT = 'float', + DOUBLE = 'double', + BYTES = 'bytes', + STRING = 'string' +} + +class AvroPrimitiveType extends AvroType { + private _primitive: AvroPrimitive; + + constructor(primitive: AvroPrimitive) { + super(); + this._primitive = primitive; + } + + public async read(stream: AvroReadable): Promise { + switch (this._primitive) { + case AvroPrimitive.NULL: + return await AvroParser.readNull(); + case AvroPrimitive.BOOLEAN: + return await AvroParser.readBoolean(stream); + case AvroPrimitive.INT: + return await AvroParser.readInt(stream); + case AvroPrimitive.LONG: + return await AvroParser.readLong(stream); + case AvroPrimitive.FLOAT: + return await AvroParser.readFloat(stream); + case AvroPrimitive.DOUBLE: + return await AvroParser.readDouble(stream); + case AvroPrimitive.BYTES: + return await AvroParser.readBytes(stream); + case AvroPrimitive.STRING: + return await AvroParser.readString(stream); + default: + throw new Error("Unknown Avro Primitive"); + } + } +} + +class AvroEnumType extends AvroType { + private readonly _symbols: string[]; + + constructor(symbols: string[]) { + super(); + this._symbols = symbols; + } + + public async read(stream: AvroReadable): Promise { + const value = await AvroParser.readInt(stream); + return this._symbols[value]; + } +} + +class AvroUnionType extends AvroType { + private readonly _types: AvroType[]; + + constructor(types: AvroType[]) { + super(); + this._types = types; + } + + public async read(stream: AvroReadable): Promise { + const typeIndex = await AvroParser.readInt(stream); + return await this._types[typeIndex].read(stream); + } +} + +class AvroMapType extends AvroType { + private readonly _itemType: AvroType; + + constructor(itemType: AvroType) { + super(); + this._itemType = itemType; + } + + public async read(stream: AvroReadable): Promise { + const readItemMethod = async (s: AvroReadable): Promise => { + return await this._itemType.read(s); + }; + return await AvroParser.readMap(stream, readItemMethod); + } +} + +class AvroRecordType extends AvroType { + private readonly _name: string; + private readonly _fields: Record; + + constructor(fields: Record, name: string) { + super(); + this._fields = fields; + this._name = name; + } + + public async read(stream: AvroReadable): Promise { + let record: Record = {}; + // FIXME: what for? + record["$schema"] = this._name; + for (const key in this._fields) { + if (this._fields.hasOwnProperty(key)) { + record[key] = await this._fields[key].read(stream); + } + } + return record; + } +} diff --git a/sdk/storage/storage-internal-avro/src/AvroReadable.ts b/sdk/storage/storage-internal-avro/src/AvroReadable.ts new file mode 100644 index 000000000000..36151b8fa453 --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/AvroReadable.ts @@ -0,0 +1,4 @@ +export abstract class AvroReadable { + public abstract get position(): number; + public abstract async read(size: number): Promise; +} diff --git a/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts new file mode 100644 index 000000000000..afc3352c859f --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts @@ -0,0 +1,59 @@ +import { Readable } from "stream"; +import { AvroReadable } from './AvroReadable'; + +export class AvroReadableFromStream extends AvroReadable { + private _position: number; + private _readable: Readable; + // private _stillReadable: boolean; + constructor(readable: Readable) { + super(); + this._readable = readable; + this._position = 0; + // workaround due to Readable.readable only availabe after Node.js v11.4 + // this._stillReadable = true; + // this._readable.on("end", () => { + // this._stillReadable = false; + // }); + // this._readable.on("error", () => { + // this._stillReadable = false; + // }); + } + public get position(): number { + return this._position; + } + public async read(size: number): Promise { + if (size <= 0) { + throw new Error(`size parameter should be positive: ${size}`); + } + // readable is true if it is safe to call readable.read(), which means the stream has not been destroyed or emitted 'error' or 'end'. + // if (!this._stillReadable || this._readable.destroyed) { + if (!this._readable.readable) { + throw new Error("Stream no longer readable."); + } + // See if there is already enough data, note that "Only after readable.read() returns null, 'readable' will be emitted." + let chunk = this._readable.read(size); + if (chunk) { + this._position += chunk.length; + // chunk.lenght maybe less than desired size if the stream ends. + return chunk; + } + else { + // register callback to wait for enough data to read + return new Promise((resolve, reject) => { + const callback = () => { + let chunk = this._readable.read(size); + if (chunk) { + this._position += chunk.length; + // chunk.lenght maybe less than desired size if the stream ends. + resolve(chunk); + this._readable.removeListener("readable", callback); + } + }; + this._readable.on("readable", callback); + this._readable.once("error", reject); + this._readable.once("end", reject); + this._readable.once("close", reject); + }); + } + } +} diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts new file mode 100644 index 000000000000..b8a2caef1124 --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -0,0 +1,142 @@ +import { AvroReadable } from "./AvroReadable"; +import { AVRO_SYNC_MARKER_SIZE, AVRO_INIT_BYTES, AVRO_CODEC_KEY, AVRO_SCHEMA_KEY } from "./AvroConstants"; +import { arraysEqual } from "./utils/utils.common"; +import { AvroType, AvroParser } from "./AvroParser"; + +export class AvroReader { + private readonly _dataStream: AvroReadable; + + private readonly _headerStream: AvroReadable; + + private _syncMarker?: Uint8Array; + + private _metadata?: Record; + + private _itemType?: AvroType; + + private _itemsRemainingInBlock?: number; + + /// The byte offset within the Avro file (both header and data) + /// of the start of the current block. + private _blockOffset: number; + public get blockOffset(): number { + return this._blockOffset; + } + + private _objectIndex: number; + public get objectIndex(): number { + return this._objectIndex; + } + + private _initialized: boolean; + + constructor(dataStream: AvroReadable); + + constructor( + dataStream: AvroReadable, + headerStream: AvroReadable, + currentBlockOffset: number, + indexWithinCurrentBlock: number + ); + + constructor( + dataStream: AvroReadable, + headerStream?: AvroReadable, + currentBlockOffset?: number, + indexWithinCurrentBlock?: number + ) { + this._dataStream = dataStream; + this._headerStream = headerStream || dataStream; + this._initialized = false; + this._blockOffset = currentBlockOffset || 0; + this._objectIndex = indexWithinCurrentBlock || 0; + } + + // FUTURE: cancellation / aborter? + private async initialize() { + const header = await AvroParser.readFixedBytes( + this._headerStream, + AVRO_INIT_BYTES.length + ); + if (!arraysEqual(header, AVRO_INIT_BYTES)) { + throw new Error("Stream is not an Avro file."); + } + + // File metadata is written as if defined by the following map schema: + // { "type": "map", "values": "bytes"} + this._metadata = await AvroParser.readMap(this._headerStream, AvroParser.readString); + + // Validate codec + const codec = this._metadata![AVRO_CODEC_KEY]; + if (!(codec == undefined || codec == "null")) { + throw new Error("Codecs are not supported"); + } + + // The 16-byte, randomly-generated sync marker for this file. + this._syncMarker = await AvroParser.readFixedBytes( + this._headerStream, + AVRO_SYNC_MARKER_SIZE + ); + + // Parse the schema + const schema = JSON.parse(this._metadata![AVRO_SCHEMA_KEY]); + this._itemType = AvroType.fromSchema(schema); + + if (this._blockOffset == 0) { + this._blockOffset = this._dataStream.position; + } + + this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream); + // skip block length + await AvroParser.readLong(this._dataStream); + + this._initialized = true; + if (this._objectIndex && this._objectIndex > 0) { + for (let i = 0; i < this._objectIndex; i++) { + await this._itemType.read(this._dataStream); + this._itemsRemainingInBlock!--; + } + } + } + + public hasNext(): boolean { + return !this._initialized || this._itemsRemainingInBlock! > 0; + } + + public async *parseObjects(): AsyncIterableIterator { + if (!this._initialized) { + await this.initialize(); + } + + while (this.hasNext()) { + const result = await this._itemType!.read(this._dataStream); + + this._itemsRemainingInBlock!--; + this._objectIndex!++; + + if (this._itemsRemainingInBlock == 0) { + const marker = await AvroParser.readFixedBytes(this._dataStream, AVRO_SYNC_MARKER_SIZE); + + this._blockOffset = this._dataStream.position; + this._objectIndex = 0; + + if (!arraysEqual(this._syncMarker!, marker)) { + throw new Error("Stream is not a valid Avro file."); + } + + try { + this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream); + } catch (err) { + // We hit the end of the stream. + this._itemsRemainingInBlock = 0; + } + + if (this._itemsRemainingInBlock! > 0) { + // Ignore block size + await AvroParser.readLong(this._dataStream); + } + } + yield result; + } + } +} diff --git a/sdk/storage/storage-internal-avro/src/index.ts b/sdk/storage/storage-internal-avro/src/index.ts new file mode 100644 index 000000000000..b8e8047dc0f9 --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/index.ts @@ -0,0 +1,3 @@ +export { AvroReader } from "./AvroReader"; +export { AvroReadable } from "./AvroReadable"; +export { AvroReadableFromStream } from "./AvroReadableFromStream"; diff --git a/sdk/storage/storage-internal-avro/src/utils/utils.common.ts b/sdk/storage/storage-internal-avro/src/utils/utils.common.ts new file mode 100644 index 000000000000..ace1939dd47a --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/utils/utils.common.ts @@ -0,0 +1,15 @@ +export interface KeyValuePair { + key: string; + value: T; +} + +export function arraysEqual(a: Uint8Array, b: Uint8Array) : boolean { + if (a === b) return true; + if (a == null || b == null) return false; + if (a.length != b.length) return false; + + for (let i = 0; i < a.length; ++i) { + if (a[i] !== b[i]) return false; + } + return true; +} diff --git a/sdk/storage/storage-internal-avro/test/avroreadable.spec.ts b/sdk/storage/storage-internal-avro/test/avroreadable.spec.ts new file mode 100644 index 000000000000..d2e382cabc10 --- /dev/null +++ b/sdk/storage/storage-internal-avro/test/avroreadable.spec.ts @@ -0,0 +1,25 @@ +import * as fs from "fs"; +import * as assert from "assert"; +import { AvroReadableFromStream } from "../src"; + +describe("AvroReadableFromStream", () => { + it("read pass end should throw", async () => { + let rs = fs.createReadStream("../README.md"); + + let rfs = new AvroReadableFromStream(rs); + assert.equal(rfs.position, 0); + + await rfs.read(10); + assert.equal(rfs.position, 10); + await rfs.read(100000); + + let exceptionCaught = false; + try { + await rfs.read(10); + } catch (err) { + assert.equal(err.message, "Stream no longer readable."); + exceptionCaught = true; + } + assert.ok(exceptionCaught); + }); +}); diff --git a/sdk/storage/storage-internal-avro/test/avroreader.spec.ts b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts new file mode 100644 index 000000000000..e7e51c180bcb --- /dev/null +++ b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts @@ -0,0 +1,59 @@ +import * as fs from "fs"; +import * as assert from "assert"; +import { AvroReader, AvroReadableFromStream } from "../src"; +import { arraysEqual } from "../src/utils/utils.common"; + +describe("AvroReader", () => { + it("test with local avro files", async () => { + const testCases: TestCase[] = [ + new TestCase("test_null_0.avro", (o) => assert.strictEqual(null, o)), // null + new TestCase("test_null_1.avro", (o) => assert.strictEqual(true, o)), // boolean + new TestCase("test_null_2.avro", (o) => assert.strictEqual("adsfasdf09809dsf-=adsf", o)), // string + new TestCase("test_null_3.avro", (o) => + assert.ok(arraysEqual(new TextEncoder().encode("12345abcd"), o as Uint8Array)) + ), // byte[] + new TestCase("test_null_4.avro", (o) => assert.strictEqual(1234, o)), // int + new TestCase("test_null_5.avro", (o) => assert.strictEqual(1234, o)), // long + new TestCase("test_null_6.avro", (o) => assert.strictEqual(1234.0, o)), // float + new TestCase("test_null_7.avro", (o) => assert.strictEqual(1234.0, o)), // double + // Not supported today. + // new TestCase("test_null_8.avro", o => assert.ok(arraysEqual(new TextEncoder().encode("B"), o as Uint8Array))), // fixed + new TestCase("test_null_9.avro", (o) => assert.strictEqual("B", o)), // enum + // Not supported today. + // new TestCase("test_null_10.avro", o => assert.deepStrictEqual([1, 2, 3], o)), // array + new TestCase("test_null_11.avro", (o) => assert.deepStrictEqual({ a: 1, b: 3, c: 2 }, o)), // map + new TestCase("test_null_12.avro", (o) => assert.strictEqual(null, o)), // union + new TestCase("test_null_13.avro", (o) => { + const expected = { $schema: "Test", f: 5 }; + const expectedEntries = Object.entries(expected); + const actualEntries = Object.entries(o!); + const actualMap = new Map(actualEntries); + assert.strictEqual(expectedEntries.length, actualEntries.length); + for (const [key, value] of expectedEntries) { + assert.deepStrictEqual(actualMap.get(key), value); + } + }) // record + ]; + + for (const testcase of testCases) { + const rs = fs.createReadStream(`./test/resources/${testcase.path}`); + const rfs = new AvroReadableFromStream(rs); + + const avroReader = new AvroReader(rfs); + const iter = avroReader.parseObjects(); + for await (const o of iter) { + testcase.predict(o); + } + } + }); +}); + +type Action = (o: Object | null) => void; +class TestCase { + public path: string; + public predict: Action; + constructor(path: string, action: Action) { + this.path = path; + this.predict = action; + } +} diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_0.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_0.avro new file mode 100644 index 0000000000000000000000000000000000000000..91c2b2469e5432eb9ec2390151bc9ff3e90ceaa0 GIT binary patch literal 75 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCd6YmRN(`NUQtMNu2+Y1`d^mr~ K=3Z$L3=seW)g9yj literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_1.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_1.avro new file mode 100644 index 0000000000000000000000000000000000000000..01371934eba3764a31d53dd3f9bc1e461702d1ab GIT binary patch literal 88 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCg_M%=^K()Y^OP9s8P`rr>EQjK TbNPM6yO#l9L_`?j09_6MP7xss literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_10.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_10.avro new file mode 100644 index 0000000000000000000000000000000000000000..97aaaa0bb91a78930168294a4d9b4a235d6dd92a GIT binary patch literal 153 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCU8@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCovM{eDhpDTtQ3@T6AP4d6qL#m zb4pW-K>|7XdFe{E49};o`JAs(f6}V2H`n@m#e0!EjBHGaOiW2^Ovx+^bYP-8005xm BHPQe8 literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_12.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_12.avro new file mode 100644 index 0000000000000000000000000000000000000000..ddf42625f4f320290e6da4136c343b800d139419 GIT binary patch literal 105 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCO`?^GONuh{(v@@+lt7XoIwv2< fk7c+TwR{ENa=xFFChYuieom>MhzJuLpvwUOZ!acI literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_13.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_13.avro new file mode 100644 index 0000000000000000000000000000000000000000..277376ae1aa5801191c8824813be8f020324fbae GIT binary patch literal 157 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCXE9bQl~fj_Dp@Hg6{RNU7o{la zC@AG6=7L2+Qj1GK{Itx}oRngqnrMXTocz3WWVLBZwXwAfaYu7sT72Vj$vnXPB|UzL Mf`|wg9H7eq0E1I9(f|Me literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_14.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_14.avro new file mode 100644 index 0000000000000000000000000000000000000000..3c34ec843837039174125c7b6d7b86554bd35374 GIT binary patch literal 358 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC=P_3+l~fj_Dp@Hg6{RNU7o{la zC@AG6=7L3hGK&j9{Itx}oRngqnrOJ{XeE$QAj#sAqRhN>APX*s#U$taykZ{B$gLMxCiYvk+gsN+Yg$~W$O1+dCXS5M M#1sZ<(dbqH03+gO_W%F@ literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_2.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_2.avro new file mode 100644 index 0000000000000000000000000000000000000000..bf119d9e16f55f99c06d203079cd7b35812f2744 GIT binary patch literal 308 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC1(b?QiZb)kl^8Z(S$pN@?6xH= opWCzEF8(bga)wzaF{L;yu{b5oz|z9N63EuI1&ItZVRVlJ04jEH;{X5v literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_3.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_3.avro new file mode 100644 index 0000000000000000000000000000000000000000..d542117f7f6e950c5858d2f5242d03db1142d117 GIT binary patch literal 177 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC`IM3>OHzxK7|d0}Qa-liyMB4a cZm;{t@4v_iMj=BZV-wTFq~sLZvCypr0FtpdUjP6A literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_4.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_4.avro new file mode 100644 index 0000000000000000000000000000000000000000..b514fd8218419e4dd2b5dfe9bda9f4b678a1581f GIT binary patch literal 94 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCxs)>VN|YGx{5Z8JF?q=kk$qY} QI;waJMKqQOV?uOQ0FbsOYybcN literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_5.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_5.avro new file mode 100644 index 0000000000000000000000000000000000000000..29e8ca4d5f3559887e1628238dff6d8499f315a1 GIT binary patch literal 95 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCd6aVU^U{?VDi@v#HHgi6KXqZr S;X}KQ-W1VTB8&;qRRI97RwwrW literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_6.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_6.avro new file mode 100644 index 0000000000000000000000000000000000000000..df22b0f901a3004e75a9a922eeaf6c61b942fa5f GIT binary patch literal 116 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC`IORf@)Jvx81%y!RhK2p%#Hf7 U#^f4vcdkePgTpKrVlcW+0GJvkApigX literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_7.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_7.avro new file mode 100644 index 0000000000000000000000000000000000000000..1168f99d0d1977ba6a446b5b4599a96efd6f072f GIT binary patch literal 158 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC1(Z_qOOtX^l^6>5J}b~GkAB(O YA*We={AY~F0!9W9@R;mCEgIbl0PIvMivR!s literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_8.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_8.avro new file mode 100644 index 0000000000000000000000000000000000000000..b4136af69b603bc2695bcdffdcd69e9abb650888 GIT binary patch literal 123 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCBdV23DhpDTtQ3^eGAmM3lynr7 u@)C2w0wJlzB_MurW)+BUSj#Y1s# literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_9.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_9.avro new file mode 100644 index 0000000000000000000000000000000000000000..90abc062240449b5df26a9aeeebf602cb9f38d73 GIT binary patch literal 134 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC^Qx6fDhpDTtQ3?|^Gb7-bQF~G z5_7@)kksN55Wl!GHz_}-7^oy#$q^*rq!e4rpyG4uSOkOUm+zatwOxp?(H9Y6f&+9p E09Ilw2LJ#7 literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/tsconfig.json b/sdk/storage/storage-internal-avro/tsconfig.json new file mode 100644 index 000000000000..7d0f438f54f6 --- /dev/null +++ b/sdk/storage/storage-internal-avro/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "alwaysStrict": true, + "noImplicitAny": true, + "preserveConstEnums": true, + "sourceMap": true, + "inlineSources": true, + "newLine": "LF", + "target": "es5", + "moduleResolution": "node", + "noUnusedLocals": true, + "noUnusedParameters": true, + "strict": true, + "module": "esNext", + "outDir": "./dist-esm", + "declaration": true, + "declarationMap": true, + "importHelpers": true, + "declarationDir": "./typings/latest", + "lib": ["dom", "es5", "es6", "es7", "esnext"], + "esModuleInterop": true + }, + "compileOnSave": true, + "exclude": ["node_modules", "./samples/**"], + "include": ["./src/**/*.ts", "./test/**/*.ts"] +}