From 2adbb0e817385ffe3e4fccf6e062ef0a2e2bf8fe Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Sat, 9 May 2020 21:58:16 +0800 Subject: [PATCH 01/14] IReadable --- sdk/storage/storage-internal-avro/README.md | 20 +++ .../storage-internal-avro/package.json | 22 +++ .../storage-internal-avro/src/AvroReader.ts | 144 ++++++++++++++++++ .../storage-internal-avro/src/IReadable.ts | 92 +++++++++++ 4 files changed, 278 insertions(+) create mode 100644 sdk/storage/storage-internal-avro/README.md create mode 100644 sdk/storage/storage-internal-avro/package.json create mode 100644 sdk/storage/storage-internal-avro/src/AvroReader.ts create mode 100644 sdk/storage/storage-internal-avro/src/IReadable.ts diff --git a/sdk/storage/storage-internal-avro/README.md b/sdk/storage/storage-internal-avro/README.md new file mode 100644 index 000000000000..72ce26939d78 --- /dev/null +++ b/sdk/storage/storage-internal-avro/README.md @@ -0,0 +1,20 @@ +# Azure Storage Internal Avro client library for JavaScript +- For internal use only. + +## Getting started +- For internal use only. + +## Key concepts +- For internal use only. + +## Examples +- For internal use only. + +## Troubleshooting +- For internal use only. + +## Next steps +- For internal use only. + +## Contributing +- For internal use only. \ No newline at end of file diff --git a/sdk/storage/storage-internal-avro/package.json b/sdk/storage/storage-internal-avro/package.json new file mode 100644 index 000000000000..d9a2ab5b43e3 --- /dev/null +++ b/sdk/storage/storage-internal-avro/package.json @@ -0,0 +1,22 @@ +{ + "name": "storage-internal-avro", + "version": "1.0.0", + "description": "internal avro parser", + "main": "index.js", + "directories": { + "test": "tests" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Microsoft Corporation", + "license": "MIT", + "dependencies": { + "tslib": "^1.10.0" + }, + "devDependencies": { + "@types/node": "^8.0.0", + "typescript": "~3.8.3", + "ts-node": "^8.3.0" + } +} diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts new file mode 100644 index 000000000000..160f0ee320a4 --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -0,0 +1,144 @@ +import { IReadable } from "./IReadable"; + +interface Metadata { + /** + * A name-value pair. + */ + [propertyName: string]: string; +} + +export class AvroReader { + private readonly _dataStream: IReadable; + + private readonly _headerStream: IReadable; + + private _syncMarker: Uint8Array | undefined; + + private _metadata: Metadata | undefined; + + private _itemType: AvroType | undefined; + + private _itemsRemainingInBlock: number | undefined; + + private _blockOffset: number; + public get blockOffset(): number { + return this._blockOffset; + } + + private _objectIndex: number; + public get objectIndex(): number { + return this._objectIndex; + } + + private _initialized: boolean; + + constructor( + dataStream: Readable + ); + + constructor( + dataStream: Readable, + headerStream: Readable, + currentBlockOffset: number, + indexWithinCurrentBlock: number + ); + + constructor( + dataStream: Readable, + headerStream?: Readable, + currentBlockOffset?: number, + indexWithinCurrentBlock?: number + ) { + this._dataStream = dataStream; + this._headerStream = headerStream || dataStream; + this._initialized = false; + this._blockOffset = currentBlockOffset || 0; + this._objectIndex = indexWithinCurrentBlock || 0; + } + + // TODO: cancellation / aborter? + private async initialize() { + const header = await AvroParser.readFixedBytes(this._headerStream, AvroConstants.InitBytes.Length); + if (header == AvroConstants.InitBytes) { + throw new Error("Stream is not an Avro file."); + } + + // File metadata is written as if defined by the following map schema: + // { "type": "map", "values": "bytes"} + this._metadata = await AvroParser.readMap(this._headerStream, AvroParser.readString); + + // Validate codec + const codec = this._metadata![AvroConstants.CodecKey]; + if (!(codec == undefined || codec == "null")) { + throw new Error("Codecs are not supported"); + } + + // The 16-byte, randomly-generated sync marker for this file. + this._syncMarker = await AvroParser.readFixedBytes(this._headerStream, AvroConstants.SyncMarkerSize); + + // Parse the schema + const schema = JSON.parse(this._metadata![AvroConstants.SchemaKey]); + this._itemType = AvroType.fromSchema(schema.RootElement); + + if (this._blockOffset == 0) { + this._blockOffset = this._dataStream. + } + + // Populate _itemsRemainingInCurrentBlock + this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream); + + // skip block length + await AvroParser.readLong(this._dataStream); + + this._initialized = true; + + if (this._objectIndex && this._objectIndex > 0) { + for (let i = 0; i < this._objectIndex; i++) { + await this._itemType.Read(this._dataStream); + this._itemsRemainingInBlock!--; + } + } + } + + public hasNext(): boolean { + return !this._initialized || this._itemsRemainingInBlock! > 0; + } + + public async *parseObjects(): AsyncIterableIterator { + if (!this._initialized) { + await this.initialize(); + } + + while (this.hasNext) { + const result = await this._itemType.read(this._dataStream); + + this._itemsRemainingInBlock!--; + this._objectIndex!++; + + if (this._itemsRemainingInBlock == 0) { + const marker = await AvroParser.readFixedBytes(this._dataStream, 16); + + BlockOffset = _dataStream.Position; + this._objectIndex = 0; + + if (!this._syncMarker == marker) { + throw new Error("Stream is not a valid Avro file."); + } + + try { + this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream); + } + catch (err) { + // We hit the end of the stream. + this._itemsRemainingInBlock = 0; + } + + if (this._itemsRemainingInBlock! > 0) { + // Ignore block size + await AvroParser.readLong(this._dataStream); + } + } + yield result; + } + } +} diff --git a/sdk/storage/storage-internal-avro/src/IReadable.ts b/sdk/storage/storage-internal-avro/src/IReadable.ts new file mode 100644 index 000000000000..5f76a9016387 --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/IReadable.ts @@ -0,0 +1,92 @@ +export abstract class IReadable { + public abstract get position(): number; + public abstract async read(size: number): Promise; +} + + +import { Readable } from "stream"; +export class ReadableFromStream extends IReadable { + private _position: number; + private _readable: Readable; + // private _stillReadable: boolean; + + constructor(readable: Readable) { + super(); + this._readable = readable; + this._position = 0; + + // workaround due to Readable.readable only availabe after Node.js v11.4 + // this._stillReadable = true; + // this._readable.on("end", () => { + // this._stillReadable = false; + // }); + // this._readable.on("error", () => { + // this._stillReadable = false; + // }); + } + + public get position(): number { + return this._position; + } + + public async read(size: number): Promise { + // readable is true if it is safe to call readable.read(), which means the stream has not been destroyed or emitted 'error' or 'end'. + // if (!this._stillReadable || this._readable.destroyed) { + if (!this._readable.readable) { + throw Error("Stream no longer readable."); + } + + // See if there is already enough data, note that "Only after readable.read() returns null, 'readable' will be emitted." + let chunk = this._readable.read(size); + if (chunk) { + this._position += chunk.length; + // chunk.lenght maybe less than desired size if the stream ends. + return chunk; + } else { + // register callback to wait for enough data to read + return new Promise((resolve, reject) => { + const callback = () => { + let chunk = this._readable.read(size); + if (chunk) { + this._position += chunk.length; + // chunk.lenght maybe less than desired size if the stream ends. + resolve(chunk); + this._readable.removeListener('readable', callback); + } + } + this._readable.on('readable', callback); + this._readable.once('error', reject); + this._readable.once('end', reject); + this._readable.once('close', reject); + }); + } + } +} + + +/* TEST CODE */ +import * as fs from "fs"; + +async function main() { + let rs = fs.createReadStream("README.md"); + + let rfs = new ReadableFromStream(rs); + console.log(rfs.position); + + const buf = await rfs.read(10); + console.log(buf.toString()); + console.log(rs.readable); + console.log(rs.readableLength); + + const buf2 = await rfs.read(100000); + console.log(buf2.toString()); + console.log(rfs.position); + + const buf3 = await rfs.read(10); + console.log(buf3.toString()); + console.log(rfs.position); +} + +main().catch((err) => { + console.error("Error running test:", err.message); +}); From 264dce7b494a6b3a5997c08ca9db83f1170a8d36 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Mon, 11 May 2020 13:38:21 +0800 Subject: [PATCH 02/14] Avro compiles --- .../storage-internal-avro/package.json | 16 +- .../src/AvroConstants.ts | 50 +++ .../storage-internal-avro/src/AvroParser.ts | 357 ++++++++++++++++++ .../storage-internal-avro/src/AvroReader.ts | 47 +-- .../storage-internal-avro/src/IReadable.ts | 9 +- .../storage-internal-avro/src/index.ts | 3 + .../src/utils/utils.common.ts | 26 ++ 7 files changed, 472 insertions(+), 36 deletions(-) create mode 100644 sdk/storage/storage-internal-avro/src/AvroConstants.ts create mode 100644 sdk/storage/storage-internal-avro/src/AvroParser.ts create mode 100644 sdk/storage/storage-internal-avro/src/index.ts create mode 100644 sdk/storage/storage-internal-avro/src/utils/utils.common.ts diff --git a/sdk/storage/storage-internal-avro/package.json b/sdk/storage/storage-internal-avro/package.json index d9a2ab5b43e3..9562511887ec 100644 --- a/sdk/storage/storage-internal-avro/package.json +++ b/sdk/storage/storage-internal-avro/package.json @@ -1,16 +1,18 @@ { "name": "storage-internal-avro", - "version": "1.0.0", + "author": "Microsoft Corporation", + "version": "1.0.0-preview.1", "description": "internal avro parser", - "main": "index.js", - "directories": { - "test": "tests" + "license": "MIT", + "main": "./srt/index.ts", + "module": "dist-esm/index.js", + "types": "./types/index.d.ts", + "engines": { + "node": ">=8.0.0" }, "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "build:es6": "tsc -p tsconfig.json" }, - "author": "Microsoft Corporation", - "license": "MIT", "dependencies": { "tslib": "^1.10.0" }, diff --git a/sdk/storage/storage-internal-avro/src/AvroConstants.ts b/sdk/storage/storage-internal-avro/src/AvroConstants.ts new file mode 100644 index 000000000000..43a56d3c9b8c --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/AvroConstants.ts @@ -0,0 +1,50 @@ +export const AvroConstants = { + SYNC_MARKER_SIZE: 16, + + // 'O', 'b', 'j', 1 + INIT_BYTES: new Uint8Array([79, 98, 106, 1]), + + CODEC_KEY: "avro.codec", + + SCHEMA_KEY: "avro.schema", + + NULL: "null", + + BOOLEAN: "boolean", + + INT: "int", + + LONG: "long", + + FLOAT: "float", + + DOUBLE: "double", + + BYTES: "bytes", + + STRING: "string", + + RECORD: "record", + + ENUM: "enum", + + MAP: "map", + + ARRAY: "array", + + UNION: "union", + + FIXED: "fixed", + + ALIASES: "aliases", + + NAME: "name", + + FIELDS: "fields", + + TYPE: "type", + + SYMBOLS: "symbols", + + VALUES: "values", +}; diff --git a/sdk/storage/storage-internal-avro/src/AvroParser.ts b/sdk/storage/storage-internal-avro/src/AvroParser.ts new file mode 100644 index 000000000000..ef3216f15efa --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/AvroParser.ts @@ -0,0 +1,357 @@ +import { IReadable } from "./IReadable"; +import { Dictionary, KeyValuePair } from "./utils/utils.common"; +import { AvroConstants } from "./AvroConstants"; + +export class AvroParser { + /** + * Reads a fixed number of bytes from the stream. + * + * @static + * @param stream + * @param length + */ + public static async readFixedBytes(stream: IReadable, length: number): Promise { + const bytes = await stream.read(length); + if (bytes.length != length) { + throw new Error("Hit stream end.") + } + return bytes; + } + + /** + * Reads a single byte from the stream. + * + * @static + * @param stream + */ + private static async readByte(stream: IReadable): Promise { + const buf = await this.readFixedBytes(stream, 1); + return buf[0]; + } + + private static async readZigZagLong(stream: IReadable): Promise { + // copied from https://github.com/apache/avro/blob/master/lang/js/lib/utils.js#L321 + let n = 0; + let k = 0; + let b, h, f, fk; + + do { + b = await this.readByte(stream); + h = b & 0x80; + n |= (b & 0x7f) << k; + k += 7; + } while (h && k < 28); + + if (h) { + // Switch to float arithmetic, otherwise we might overflow. + f = n; + fk = 268435456; // 2 ** 28. + do { + b = await this.readByte(stream); + f += (b & 0x7f) * fk; + fk *= 128; + } while (b & 0x80); + return (f % 2 ? -(f + 1) : f) / 2; + } + + return (n >> 1) ^ -(n & 1); + } + + public static async readLong(stream: IReadable): Promise { + return this.readZigZagLong(stream); + } + + public static async readInt(stream: IReadable): Promise { + return this.readZigZagLong(stream); + } + + public static async readNull(): Promise { + return null; + } + + public static async readBoolean(stream: IReadable): Promise { + const b = await this.readByte(stream); + if (b == 1) { + return true; + } else if (b == 0) { + return false; + } else { + throw new Error("Byte was not a boolean."); + } + } + + public static async readFloat(stream: IReadable): Promise { + const u8arr = await this.readFixedBytes(stream, 4); + const view = new DataView(u8arr.buffer); + return view.getFloat32(0, true); // littleEndian = true + } + + public static async readDouble(stream: IReadable): Promise { + const u8arr = await this.readFixedBytes(stream, 8); + const view = new DataView(u8arr.buffer); + return view.getFloat64(0, true); // littleEndian = true + } + + public static async readBytes(stream: IReadable): Promise { + const size = await this.readLong(stream); + if (size < 0) { + throw new Error("Bytes size was negative."); + } + + return await stream.read(size); + } + + public static async readString(stream: IReadable): Promise { + const u8arr = await this.readBytes(stream); + + // FIXME: need TextDecoder polyfill for IE + let utf8decoder = new TextDecoder(); + return utf8decoder.decode(u8arr); + } + + private static async readMapPair( + stream: IReadable, + readItemMethod: (s: IReadable) => Promise + ): Promise> { + const key = await this.readString(stream); + // FIXME: what about readFixed which need a length as parameter. + const value = await readItemMethod(stream); + return { key, value }; + } + + public static async readMap( + stream: IReadable, + readItemMethod: (s: IReadable) => Promise + ): Promise> { + const readPairMethod = async (stream: IReadable): Promise> => { + return await this.readMapPair(stream, readItemMethod); + } + + const pairs: KeyValuePair[] = await this.readArray(stream, readPairMethod); + let dict: Dictionary = {}; + for (const pair of pairs) { + dict[pair.key] = pair.value; + } + return dict; + } + + private static async readArray( + stream: IReadable, + readItemMethod: (s: IReadable) => Promise + ): Promise { + let items: T[] = []; + for (let count = await this.readLong(stream); + count != 0; + count = await this.readLong(stream)) { + if (count < 0) { + // Ignore block sizes + await this.readLong(stream); + count = - count; + } + + while (count--) { + const item: T = await readItemMethod(stream); + items.push(item); + } + } + return items; + } +} + + +interface RecordField { + name: string; + type: string | ObjectSchema | (string | ObjectSchema)[]; // Unions may not immediately contain other unions. +} + +interface ObjectSchema { + type: 'record' | 'enum' | 'array' | 'map' | 'fixed'; + name?: string; + aliases?: string; + fields?: RecordField[]; + symbols?: string[]; + values?: string; + size?: number; +}; + +export abstract class AvroType { + /** + * Reads an object from the stream. + * + * @param stream + */ + public abstract read(stream: IReadable): Promise; + + /** + * Determinds the AvroType from the Avro Schema. + */ + public static fromSchema(schema: string | Object): AvroType { + if (typeof schema == 'string') { + return this.fromStringSchema(schema); + } else if (Array.isArray(schema)) { + return this.fromArraySchema(schema); + } else { + return this.fromObjectSchema(schema as ObjectSchema); + } + } + + private static fromStringSchema(schema: string): AvroType { + // FIXME: simpler way to tell if schema is of type AvroPrimitive? + switch (schema) { + case AvroConstants.NULL: + case AvroConstants.BOOLEAN: + case AvroConstants.INT: + case AvroConstants.LONG: + case AvroConstants.FLOAT: + case AvroConstants.DOUBLE: + case AvroConstants.BYTES: + case AvroConstants.STRING: return new AvroPrimitiveType(schema as AvroPrimitive); + default: throw new Error(`Unexpected Avro type ${schema}`); + } + } + + private static fromArraySchema(schema: any[]): AvroType { + return new AvroUnionType(schema.map(this.fromSchema)); + } + + private static fromObjectSchema(schema: ObjectSchema): AvroType { + const type = schema.type; + // Primitives can be defined as strings or objects + try { + return this.fromStringSchema(type); + } catch (err) { } + + switch (type) { + case AvroConstants.RECORD: + if (schema.aliases) { + throw new Error(`aliases currently is not supported, schema: ${schema}`); + } + if (!schema.name) { + throw new Error(`Required attribute 'name' doesn't exist on schema: ${schema}`); + } + + let fields: Dictionary = {}; + if (!schema.fields) { + throw new Error(`Required attribute 'fields' doesn't exist on schema: ${schema}`); + } + for (const field of schema.fields) { + fields[field.name] = this.fromSchema(field.type); + } + return new AvroRecordType(fields, schema.name); + case AvroConstants.ENUM: + if (schema.aliases) { + throw new Error(`aliases currently is not supported, schema: ${schema}`); + } + if (!schema.symbols) { + throw new Error(`Required attribute 'symbols' doesn't exist on schema: ${schema}`); + } + return new AvroEnumType(schema.symbols); + case AvroConstants.MAP: + if (!schema.values) { + throw new Error(`Required attribute 'values' doesn't exist on schema: ${schema}`); + } + return new AvroMapType(this.fromSchema(schema.values)); + case AvroConstants.ARRAY: // Unused today + case AvroConstants.UNION: // Unused today + case AvroConstants.FIXED: // Unused today + default: + throw new Error(`Unexpected Avro type ${type} in ${schema}`); + } + } +} + + +type AvroPrimitive = 'null' | 'boolean' | 'int ' | 'long' | 'float' | 'double' | 'bytes' | 'string'; + +class AvroPrimitiveType extends AvroType { + private _primitive: AvroPrimitive; + + constructor(primitive: AvroPrimitive) { + super(); + this._primitive = primitive; + } + + public async read(stream: IReadable): Promise { + switch (this._primitive) { + case AvroConstants.NULL: return await AvroParser.readNull(); + case AvroConstants.BOOLEAN: return await AvroParser.readBoolean(stream); + case AvroConstants.INT: return await AvroParser.readInt(stream); + case AvroConstants.LONG: return await AvroParser.readLong(stream); + case AvroConstants.FLOAT: return await AvroParser.readFloat(stream); + case AvroConstants.DOUBLE: return await AvroParser.readDouble(stream); + case AvroConstants.BYTES: return await AvroParser.readBytes(stream); + case AvroConstants.STRING: return await AvroParser.readString(stream); + default: throw new Error("Unknown Avro Primitive"); + } + } +} + +class AvroEnumType extends AvroType { + private readonly _symbols: string[]; + + constructor(symbols: string[]) { + super(); + this._symbols = symbols; + } + + public async read(stream: IReadable): Promise { + const value = await AvroParser.readInt(stream); + return this._symbols[value]; + } +} + + +class AvroUnionType extends AvroType { + private readonly _types: AvroType[]; + + constructor(types: AvroType[]) { + super(); + this._types = types; + } + + public async read(stream: IReadable): Promise { + const typeIndex = await AvroParser.readInt(stream); + return await this._types[typeIndex].read(stream); + } +} + + +class AvroMapType extends AvroType { + private readonly _itemType: AvroType; + + constructor(itemType: AvroType) { + super(); + this._itemType = itemType; + } + + public async read(stream: IReadable): Promise { + const readItemMethod = async (s: IReadable): Promise => { + return await this._itemType.read(s); + } + return await AvroParser.readMap(stream, readItemMethod); + } +} + + +class AvroRecordType extends AvroType { + private readonly _name: string; + private readonly _fields: Dictionary; + + constructor(fields: Dictionary, name: string) { + super(); + this._fields = fields; + this._name = name; + } + + public async read(stream: IReadable): Promise { + let record: Dictionary = {}; + // FIXME: what for? + record["$schema"] = this._name; + for (const key in this._fields) { + if (this._fields.hasOwnProperty(key)) { + record[key] = await this._fields[key].read(stream); + } + } + return record; + } +} diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts index 160f0ee320a4..42a99e1cf4c2 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReader.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -1,11 +1,7 @@ import { IReadable } from "./IReadable"; - -interface Metadata { - /** - * A name-value pair. - */ - [propertyName: string]: string; -} +import { AvroConstants } from "./AvroConstants"; +import { Metadata, arraysEqual } from "./utils/utils.common"; +import { AvroType, AvroParser } from "./AvroParser" export class AvroReader { private readonly _dataStream: IReadable; @@ -33,19 +29,19 @@ export class AvroReader { private _initialized: boolean; constructor( - dataStream: Readable + dataStream: IReadable ); constructor( - dataStream: Readable, - headerStream: Readable, + dataStream: IReadable, + headerStream: IReadable, currentBlockOffset: number, indexWithinCurrentBlock: number ); constructor( - dataStream: Readable, - headerStream?: Readable, + dataStream: IReadable, + headerStream?: IReadable, currentBlockOffset?: number, indexWithinCurrentBlock?: number ) { @@ -56,10 +52,10 @@ export class AvroReader { this._objectIndex = indexWithinCurrentBlock || 0; } - // TODO: cancellation / aborter? + // FUTURE: cancellation / aborter? private async initialize() { - const header = await AvroParser.readFixedBytes(this._headerStream, AvroConstants.InitBytes.Length); - if (header == AvroConstants.InitBytes) { + const header = await AvroParser.readFixedBytes(this._headerStream, AvroConstants.INIT_BYTES.length); + if (!arraysEqual(header, AvroConstants.INIT_BYTES)) { throw new Error("Stream is not an Avro file."); } @@ -68,33 +64,30 @@ export class AvroReader { this._metadata = await AvroParser.readMap(this._headerStream, AvroParser.readString); // Validate codec - const codec = this._metadata![AvroConstants.CodecKey]; + const codec = this._metadata![AvroConstants.CODEC_KEY]; if (!(codec == undefined || codec == "null")) { throw new Error("Codecs are not supported"); } // The 16-byte, randomly-generated sync marker for this file. - this._syncMarker = await AvroParser.readFixedBytes(this._headerStream, AvroConstants.SyncMarkerSize); + this._syncMarker = await AvroParser.readFixedBytes(this._headerStream, AvroConstants.SYNC_MARKER_SIZE); // Parse the schema - const schema = JSON.parse(this._metadata![AvroConstants.SchemaKey]); + const schema = JSON.parse(this._metadata![AvroConstants.SCHEMA_KEY]); this._itemType = AvroType.fromSchema(schema.RootElement); if (this._blockOffset == 0) { - this._blockOffset = this._dataStream. + this._blockOffset = this._dataStream.position; } - // Populate _itemsRemainingInCurrentBlock this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream); - // skip block length await AvroParser.readLong(this._dataStream); this._initialized = true; - if (this._objectIndex && this._objectIndex > 0) { for (let i = 0; i < this._objectIndex; i++) { - await this._itemType.Read(this._dataStream); + await this._itemType.read(this._dataStream); this._itemsRemainingInBlock!--; } } @@ -104,13 +97,13 @@ export class AvroReader { return !this._initialized || this._itemsRemainingInBlock! > 0; } - public async *parseObjects(): AsyncIterableIterator { + public async *parseObjects(): AsyncIterableIterator { if (!this._initialized) { await this.initialize(); } while (this.hasNext) { - const result = await this._itemType.read(this._dataStream); + const result = await this._itemType!.read(this._dataStream); this._itemsRemainingInBlock!--; this._objectIndex!++; @@ -118,10 +111,10 @@ export class AvroReader { if (this._itemsRemainingInBlock == 0) { const marker = await AvroParser.readFixedBytes(this._dataStream, 16); - BlockOffset = _dataStream.Position; + this._blockOffset = this._dataStream.position; this._objectIndex = 0; - if (!this._syncMarker == marker) { + if (!arraysEqual(this._syncMarker!, marker)) { throw new Error("Stream is not a valid Avro file."); } diff --git a/sdk/storage/storage-internal-avro/src/IReadable.ts b/sdk/storage/storage-internal-avro/src/IReadable.ts index 5f76a9016387..d54e323bf88e 100644 --- a/sdk/storage/storage-internal-avro/src/IReadable.ts +++ b/sdk/storage/storage-internal-avro/src/IReadable.ts @@ -30,10 +30,14 @@ export class ReadableFromStream extends IReadable { } public async read(size: number): Promise { + if (size <= 0) { + throw new Error(`size parameter should be positive: ${size}`); + } + // readable is true if it is safe to call readable.read(), which means the stream has not been destroyed or emitted 'error' or 'end'. // if (!this._stillReadable || this._readable.destroyed) { if (!this._readable.readable) { - throw Error("Stream no longer readable."); + throw new Error("Stream no longer readable."); } // See if there is already enough data, note that "Only after readable.read() returns null, 'readable' will be emitted." @@ -69,14 +73,15 @@ import * as fs from "fs"; async function main() { let rs = fs.createReadStream("README.md"); + console.log(rs.read(0)); let rfs = new ReadableFromStream(rs); console.log(rfs.position); + console.log(rs.readable); const buf = await rfs.read(10); console.log(buf.toString()); console.log(rs.readable); - console.log(rs.readableLength); const buf2 = await rfs.read(100000); console.log(buf2.toString()); diff --git a/sdk/storage/storage-internal-avro/src/index.ts b/sdk/storage/storage-internal-avro/src/index.ts new file mode 100644 index 000000000000..71e41f76c2ed --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/index.ts @@ -0,0 +1,3 @@ +import { AvroReader } from "./AvroReader"; +import { IReadable } from "./IReadable"; +export { AvroReader, IReadable } diff --git a/sdk/storage/storage-internal-avro/src/utils/utils.common.ts b/sdk/storage/storage-internal-avro/src/utils/utils.common.ts new file mode 100644 index 000000000000..d561e54e7166 --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/utils/utils.common.ts @@ -0,0 +1,26 @@ +export interface Metadata { + /** + * A name-value pair. + */ + [propertyName: string]: string; +} + +export interface Dictionary { + [key: string]: T; +} + +export interface KeyValuePair { + key: string; + value: T; +} + +export function arraysEqual(a: Uint8Array, b : Uint8Array) { + if (a === b) return true; + if (a == null || b == null) return false; + if (a.length != b.length) return false; + + for (let i = 0; i < a.length; ++i) { + if (a[i] !== b[i]) return false; + } + return true; +} \ No newline at end of file From d9d34480d0407f4756755ddecf542753b391976c Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Tue, 12 May 2020 19:19:28 +0800 Subject: [PATCH 03/14] use storage-internal-avro as shared source in storage-blob --- sdk/storage/storage-blob/api-extractor.json | 4 ++-- sdk/storage/storage-blob/package.json | 20 +++++++++---------- .../storage-blob/rollup.base.config.js | 12 +++++------ sdk/storage/storage-blob/src/index.ts | 3 +++ sdk/storage/storage-blob/tsconfig.json | 4 ++-- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/sdk/storage/storage-blob/api-extractor.json b/sdk/storage/storage-blob/api-extractor.json index 9d52d2a63383..ede224adc3de 100644 --- a/sdk/storage/storage-blob/api-extractor.json +++ b/sdk/storage/storage-blob/api-extractor.json @@ -1,6 +1,6 @@ { "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", - "mainEntryPointFilePath": "typings/latest/src/index.d.ts", + "mainEntryPointFilePath": "typings/latest/storage-blob/src/index.d.ts", "docModel": { "enabled": false }, @@ -28,4 +28,4 @@ } } } -} +} \ No newline at end of file diff --git a/sdk/storage/storage-blob/package.json b/sdk/storage/storage-blob/package.json index 4ff1955e951c..17bfddd4584d 100644 --- a/sdk/storage/storage-blob/package.json +++ b/sdk/storage/storage-blob/package.json @@ -4,14 +4,14 @@ "version": "12.2.0", "description": "Microsoft Azure Storage SDK for JavaScript - Blob", "main": "./dist/index.js", - "module": "./dist-esm/src/index.js", + "module": "./dist-esm/storage-blob/src/index.js", "browser": { - "./dist-esm/src/index.js": "./dist-esm/src/index.browser.js", - "./dist-esm/src/credentials/StorageSharedKeyCredential.js": "./dist-esm/src/credentials/StorageSharedKeyCredential.browser.js", - "./dist-esm/src/utils/utils.node.js": "./dist-esm/src/utils/utils.browser.js", - "./dist-esm/test/utils/index.js": "./dist-esm/test/utils/index.browser.js", - "./dist-esm/src/BatchUtils.js": "./dist-esm/src/BatchUtils.browser.js", - "./dist-esm/src/BlobDownloadResponse.js": "./dist-esm/src/BlobDownloadResponse.browser.js", + "./dist-esm/storage-blob/src/index.js": "./dist-esm/storage-blob/src/index.browser.js", + "./dist-esm/storage-blob/src/credentials/StorageSharedKeyCredential.js": "./dist-esm/storage-blob/src/credentials/StorageSharedKeyCredential.browser.js", + "./dist-esm/storage-blob/src/utils/utils.node.js": "./dist-esm/storage-blob/src/utils/utils.browser.js", + "./dist-esm/storage-blob/test/utils/index.js": "./dist-esm/storage-blob/test/utils/index.browser.js", + "./dist-esm/storage-blob/src/BatchUtils.js": "./dist-esm/storage-blob/src/BatchUtils.browser.js", + "./dist-esm/storage-blob/src/BlobDownloadResponse.js": "./dist-esm/storage-blob/src/BlobDownloadResponse.browser.js", "fs": false, "os": false, "process": false @@ -46,7 +46,7 @@ "execute:samples": "npm run build:samples && npm run execute:js-samples && npm run execute:ts-samples", "format": "prettier --write --config ../../.prettierrc.json \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"", "integration-test:browser": "karma start --single-run", - "integration-test:node": "nyc mocha -r esm --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 300000 dist-esm/test/*.spec.js dist-esm/test/node/*.spec.js", + "integration-test:node": "nyc mocha -r esm --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 300000 dist-esm/storage-blob/test/*.spec.js dist-esm/storage-blob/test/node/*.spec.js", "integration-test": "npm run integration-test:node && npm run integration-test:browser", "lint:fix": "eslint -c ../../.eslintrc.old.json src test samples --ext .ts --fix", "lint": "eslint -c ../../.eslintrc.old.json src test samples --ext .ts -f html -o storage-blob-lintReport.html || exit 0", @@ -63,7 +63,7 @@ "files": [ "BreakingChanges.md", "dist/", - "dist-esm/src/", + "dist-esm/storage-blob/src/", "typings/latest/storage-blob.d.ts", "typings/3.1/storage-blob.d.ts", "README.md", @@ -161,4 +161,4 @@ "typescript": "~3.8.3", "util": "^0.12.1" } -} +} \ No newline at end of file diff --git a/sdk/storage/storage-blob/rollup.base.config.js b/sdk/storage/storage-blob/rollup.base.config.js index f0a687a6f317..d3ff2d0f3dfc 100644 --- a/sdk/storage/storage-blob/rollup.base.config.js +++ b/sdk/storage/storage-blob/rollup.base.config.js @@ -33,7 +33,7 @@ export function nodeConfig(test = false) { "util" ]; const baseConfig = { - input: "dist-esm/src/index.js", + input: "dist-esm/storage-blob/src/index.js", external: depNames.concat(externalNodeBuiltins), output: { file: "dist/index.js", @@ -65,9 +65,9 @@ export function nodeConfig(test = false) { if (test) { // entry point is every test file baseConfig.input = [ - "dist-esm/test/*.spec.js", - "dist-esm/test/node/*.spec.js", - "dist-esm/src/index.js" + "dist-esm/storage-blob/test/*.spec.js", + "dist-esm/storage-blob/test/node/*.spec.js", + "dist-esm/storage-blob/src/index.js" ]; baseConfig.plugins.unshift(multiEntry()); @@ -92,7 +92,7 @@ export function nodeConfig(test = false) { export function browserConfig(test = false) { const baseConfig = { - input: "dist-esm/src/index.browser.js", + input: "dist-esm/storage-blob/src/index.browser.js", output: { file: "dist-browser/azure-storage-blob.js", banner: banner, @@ -159,7 +159,7 @@ export function browserConfig(test = false) { }; if (test) { - baseConfig.input = ["dist-esm/test/*.spec.js", "dist-esm/test/browser/*.spec.js"]; + baseConfig.input = ["dist-esm/storage-blob/test/*.spec.js", "dist-esm/storage-blob/test/browser/*.spec.js"]; baseConfig.plugins.unshift(multiEntry({ exports: false })); baseConfig.output.file = "dist-test/index.browser.js"; // mark fs-extra as external diff --git a/sdk/storage/storage-blob/src/index.ts b/sdk/storage/storage-blob/src/index.ts index 6cb9eee5aa44..50e0c51576ce 100644 --- a/sdk/storage/storage-blob/src/index.ts +++ b/sdk/storage/storage-blob/src/index.ts @@ -48,3 +48,6 @@ export { BlobBeginCopyFromUrlPollState, CopyPollerBlobClient } from "./pollers/BlobStartCopyFromUrlPoller"; + +import { IReadable } from '../../storage-internal-avro/src' +export { IReadable } diff --git a/sdk/storage/storage-blob/tsconfig.json b/sdk/storage/storage-blob/tsconfig.json index 7d0f438f54f6..f89af540f58e 100644 --- a/sdk/storage/storage-blob/tsconfig.json +++ b/sdk/storage/storage-blob/tsconfig.json @@ -21,6 +21,6 @@ "esModuleInterop": true }, "compileOnSave": true, - "exclude": ["node_modules", "./samples/**"], - "include": ["./src/**/*.ts", "./test/**/*.ts"] + "exclude": ["node_modules", "../storage-internal-avro/node_modules", "./samples/**"], + "include": ["./src/**/*.ts", "./test/**/*.ts", "../storage-internal-avro/**/*.ts"] } From 6bbd5d55656a562da4bffaa94cf11e3aa43a6b12 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Tue, 12 May 2020 19:20:16 +0800 Subject: [PATCH 04/14] local build config --- rush.json | 10 +++++++ .../storage-internal-avro/tsconfig.json | 27 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 sdk/storage/storage-internal-avro/tsconfig.json diff --git a/rush.json b/rush.json index 003220b9bbef..30ab6b6876b3 100644 --- a/rush.json +++ b/rush.json @@ -447,6 +447,16 @@ "projectFolder": "sdk/servicebus/service-bus", "versionPolicyName": "client" }, + { + "packageName": "@azure/keyvault-common", + "projectFolder": "sdk/keyvault/keyvault-common", + "versionPolicyName": "utility" + }, + { + "packageName": "@azure/storage-internal-avro", + "projectFolder": "sdk/storage/storage-internal-avro", + "versionPolicyName": "utility" + }, { "packageName": "@azure/storage-blob", "projectFolder": "sdk/storage/storage-blob", diff --git a/sdk/storage/storage-internal-avro/tsconfig.json b/sdk/storage/storage-internal-avro/tsconfig.json new file mode 100644 index 000000000000..2a3bed613cbe --- /dev/null +++ b/sdk/storage/storage-internal-avro/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "alwaysStrict": true, + "noImplicitAny": true, + "preserveConstEnums": true, + "sourceMap": true, + "inlineSources": true, + "newLine": "LF", + "target": "es5", + "moduleResolution": "node", + "noUnusedLocals": true, + "noUnusedParameters": true, + "strict": true, + "module": "esNext", + "outDir": "./dist-esm", + "declaration": true, + "declarationMap": true, + "importHelpers": true, + "declarationDir": "./typings/latest", + "lib": ["dom", "es5", "es6", "es7", "esnext"], + "esModuleInterop": true + }, + "compileOnSave": true, + "exclude": ["node_modules", "./samples/**"], + "include": ["./src/**/*.ts", "./test/**/*.ts"] + } + \ No newline at end of file From 8a99a012ddaa6d864cae2f21ac3ad4f3016a7a60 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Tue, 12 May 2020 21:24:42 +0800 Subject: [PATCH 05/14] test and bug fix --- common/config/rush/pnpm-lock.yaml | 91 ++- rush.json | 5 - .../.vscode/extensions.json | 3 + .../storage-internal-avro/.vscode/launch.json | 59 ++ .../.vscode/settings.json | 27 + .../storage-internal-avro/package.json | 61 +- .../rollup.base.config.js | 177 +++++ .../storage-internal-avro/rollup.config.js | 17 + .../rollup.test.config.js | 6 + .../src/AvroConstants.ts | 50 +- .../storage-internal-avro/src/AvroParser.ts | 624 +++++++++--------- .../storage-internal-avro/src/AvroReader.ts | 23 +- .../storage-internal-avro/src/IReadable.ts | 42 +- .../storage-internal-avro/src/index.ts | 4 +- .../src/utils/utils.common.ts | 32 +- .../test/avroreader.spec.ts | 59 ++ .../test/readable.spec.ts | 25 + .../test/resources/test_null_0.avro | Bin 0 -> 75 bytes .../test/resources/test_null_1.avro | Bin 0 -> 88 bytes .../test/resources/test_null_10.avro | Bin 0 -> 153 bytes .../test/resources/test_null_11.avro | Bin 0 -> 213 bytes .../test/resources/test_null_12.avro | Bin 0 -> 105 bytes .../test/resources/test_null_13.avro | Bin 0 -> 157 bytes .../test/resources/test_null_14.avro | Bin 0 -> 358 bytes .../test/resources/test_null_2.avro | Bin 0 -> 308 bytes .../test/resources/test_null_3.avro | Bin 0 -> 177 bytes .../test/resources/test_null_4.avro | Bin 0 -> 94 bytes .../test/resources/test_null_5.avro | Bin 0 -> 95 bytes .../test/resources/test_null_6.avro | Bin 0 -> 116 bytes .../test/resources/test_null_7.avro | Bin 0 -> 158 bytes .../test/resources/test_null_8.avro | Bin 0 -> 123 bytes .../test/resources/test_null_9.avro | Bin 0 -> 134 bytes .../storage-internal-avro/tsconfig.json | 51 +- 33 files changed, 911 insertions(+), 445 deletions(-) create mode 100644 sdk/storage/storage-internal-avro/.vscode/extensions.json create mode 100644 sdk/storage/storage-internal-avro/.vscode/launch.json create mode 100644 sdk/storage/storage-internal-avro/.vscode/settings.json create mode 100644 sdk/storage/storage-internal-avro/rollup.base.config.js create mode 100644 sdk/storage/storage-internal-avro/rollup.config.js create mode 100644 sdk/storage/storage-internal-avro/rollup.test.config.js create mode 100644 sdk/storage/storage-internal-avro/test/avroreader.spec.ts create mode 100644 sdk/storage/storage-internal-avro/test/readable.spec.ts create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_0.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_1.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_10.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_11.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_12.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_13.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_14.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_2.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_3.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_4.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_5.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_6.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_7.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_8.avro create mode 100644 sdk/storage/storage-internal-avro/test/resources/test_null_9.avro diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index e719193d48ce..c9cfa0167905 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -27,6 +27,7 @@ dependencies: '@rush-temp/storage-blob': 'file:projects/storage-blob.tgz' '@rush-temp/storage-file-datalake': 'file:projects/storage-file-datalake.tgz' '@rush-temp/storage-file-share': 'file:projects/storage-file-share.tgz' + '@rush-temp/storage-internal-avro': 'file:projects/storage-internal-avro.tgz' '@rush-temp/storage-queue': 'file:projects/storage-queue.tgz' '@rush-temp/template': 'file:projects/template.tgz' '@rush-temp/test-utils-perfstress': 'file:projects/test-utils-perfstress.tgz' @@ -7346,12 +7347,14 @@ packages: karma-mocha: 1.3.0 karma-mocha-reporter: 2.2.5_karma@4.4.1 karma-remap-istanbul: 0.6.0_karma@4.4.1 + karma-sourcemap-loader: 0.3.7 mocha: 7.1.1 mocha-junit-reporter: 1.23.3_mocha@7.1.1 nyc: 14.1.1 prettier: 1.19.1 rimraf: 3.0.2 rollup: 1.32.1 + rollup-plugin-shim: 1.0.0 rollup-plugin-sourcemaps: 0.4.2_rollup@1.32.1 rollup-plugin-terser: 5.3.0_rollup@1.32.1 rollup-plugin-visualizer: 3.3.2_rollup@1.32.1 @@ -7362,7 +7365,7 @@ packages: dev: false name: '@rush-temp/ai-form-recognizer' resolution: - integrity: sha512-vPtrVCxruasMOV692Byx4Hc1n9WdZRNrJwSCIlVhxaGMrdgI/sETYrnF23OqFgjg8nXJtNh19xaPwMTHOk4PJw== + integrity: sha512-drftp8AmeOVBMm6hfpBJnt/OziXXLqhNWfZo1oLqfQSBISJxghr/apNc4CKrcSp9qURbWaJkeANWuHxIVeYndQ== tarball: 'file:projects/ai-form-recognizer.tgz' version: 0.0.0 'file:projects/ai-text-analytics.tgz': @@ -7466,7 +7469,7 @@ packages: dev: false name: '@rush-temp/app-configuration' resolution: - integrity: sha512-SQNvkmhpg4EcfyYBmi7l3n60gHtqbUDk3mjTBDSBM33F/01q+CjQiagcClrN98g62mlxhFaK+N+T6q7Ex3g1Ng== + integrity: sha512-QesKkz6aJwUt+oVkUzm4ESihvMHTjl1/Ims68Ue1i9H7jevsWAMDX+U1G+7rb0xCtx013PZdoedC9Qb6dsj/fg== tarball: 'file:projects/app-configuration.tgz' version: 0.0.0 'file:projects/core-amqp.tgz': @@ -7627,7 +7630,7 @@ packages: dev: false name: '@rush-temp/core-auth' resolution: - integrity: sha512-FEhKPyW6XNIk4vlVBgYt+gwtDiSmOhAxTq9YBQxpLaqiyMVjZn25WfcD/zj2/ZLnE5VKQRTxt+cKF52iETQdPw== + integrity: sha512-a8h+Fir7HdvM06N5A+BKxvig7SmvI2pQPUnVNoL2HWpuW5wdcelVyVpODYRiA43kNPsodP0AIlqWJSZOPnwtiw== tarball: 'file:projects/core-auth.tgz' version: 0.0.0 'file:projects/core-http.tgz': @@ -7871,7 +7874,7 @@ packages: dev: false name: '@rush-temp/core-tracing' resolution: - integrity: sha512-GfFzTI4+ug82g0/v2qF8bQlPaoUFvHYrHMnn3qg+dkoJe523z5ppsclkIGwkC5nabXHHNE/nzxxM09dYQU99gA== + integrity: sha512-6lNv0IiPu0TlqcRff8jI141AEY7u/AQxmVK0+H8R+sFgB6T+ZW80B1sqOAW3GjHnKKtNQ8M8l+2SHtzHtF87PQ== tarball: 'file:projects/core-tracing.tgz' version: 0.0.0 'file:projects/cosmos.tgz': @@ -8003,6 +8006,7 @@ packages: cross-env: 6.0.3 debug: 4.1.1 dotenv: 8.2.0 + downlevel-dts: 0.4.0 eslint: 6.8.0 eslint-config-prettier: 6.10.1_eslint@6.8.0 eslint-plugin-no-null: 1.0.2_eslint@6.8.0 @@ -8044,7 +8048,7 @@ packages: dev: false name: '@rush-temp/event-hubs' resolution: - integrity: sha512-bW/dj5xOrcog/7Gw/922ikLp5WOES4Et87oiwjVTzEG9U2LfhstUBDahsAR0f6n6nr6A28xNA0B7h/H0G8qNkA== + integrity: sha512-SvD3McUNYWtff0uFqaogeUbmRCdqLiwVUsRO7SrkIjdVazSydRCU5cl5IUGRKCQ4IMPzGkbXN/yyN1tmG45uGw== tarball: 'file:projects/event-hubs.tgz' version: 0.0.0 'file:projects/event-processor-host.tgz': @@ -8288,7 +8292,7 @@ packages: dev: false name: '@rush-temp/keyvault-certificates' resolution: - integrity: sha512-+H4N1yRRmhJgOU/nza/+W6YVctvTvYZkw7+S+oBQ/WYeOLZR+WHAc6/b9tR3hxiAhri9wxvpb+voD6/DduXS+Q== + integrity: sha512-Jn9KFNDBX87QBJ1Lcjp78dYcE+dsMg5rYpio/JeY0V2mWlNdh7qTKLvCNYDBGbLA74vByV3WeKwIQryHAi+05g== tarball: 'file:projects/keyvault-certificates.tgz' version: 0.0.0 'file:projects/keyvault-keys.tgz': @@ -8354,7 +8358,7 @@ packages: dev: false name: '@rush-temp/keyvault-keys' resolution: - integrity: sha512-sgYATU21SAy92H4+m2jbkwp26PdeRYHMZWmQZ1Gj9+5ZFbLCEcHm4KYukZ66EwBdmIwwodVRg65hROm17zBzfg== + integrity: sha512-eoe4wq30U9rGPBp+IAHTibqqR8RCZQ0ijWtJutDZ8WaiHbmIKaSFbQRY1A0+N/xSYHSKrcZDwTZC3cEpeWHa9A== tarball: 'file:projects/keyvault-keys.tgz' version: 0.0.0 'file:projects/keyvault-secrets.tgz': @@ -8420,7 +8424,7 @@ packages: dev: false name: '@rush-temp/keyvault-secrets' resolution: - integrity: sha512-CRFOhh1AMS5pvaXG3ZM5NboFAkQ5UF21UvbWpdajYCPg5J8s1znnWn01Tc+0b24brxyyUEWDQo6Yzqx8lG/35Q== + integrity: sha512-PjKc2VsK3yhM2nLsw1SEAqNOeKw3mK5APQ/6AJC2QyHWP/lDYghPHH6ep/78LBThcm5K9uLFMwM49ZLnYsnY+A== tarball: 'file:projects/keyvault-secrets.tgz' version: 0.0.0 'file:projects/logger.tgz': @@ -8533,7 +8537,7 @@ packages: dev: false name: '@rush-temp/search-documents' resolution: - integrity: sha512-GYl0ZrFxMq1Qfkfihi1vHdOuSzQp3Q+Z3c2nd+u2RzXcVTiTRE4t0KnON8OUdyXYJvsjrHeLpBdOrwVuY2He/Q== + integrity: sha512-dSUxmkUpg7gC0P73FvSLK5Fr4E8WGpdgaycSvR4ZSA69VpYIodX65aU7uyfkHrpuFCkydY7fbBbaXYiWfmvLTA== tarball: 'file:projects/search-documents.tgz' version: 0.0.0 'file:projects/service-bus.tgz': @@ -8612,7 +8616,7 @@ packages: dev: false name: '@rush-temp/service-bus' resolution: - integrity: sha512-/sInDnVXuGr13OJ6o0ybAgSwdTi6MBF20+2cjGTjcLcHfjuYoVDKqZQOGQCCtUDobnIsx3KyN3sASXIrCv15wQ== + integrity: sha512-ySmiDTgbTBpMYgXNSDGpIzuHs+5raDZenJw/jEBXhioOYpNXL+oDYK5EZQ5S+BKMCmGXOD7QvrpQortMrf3IrA== tarball: 'file:projects/service-bus.tgz' version: 0.0.0 'file:projects/storage-blob.tgz': @@ -8674,7 +8678,7 @@ packages: dev: false name: '@rush-temp/storage-blob' resolution: - integrity: sha512-eR/qeWB9CgdJYsGNIepqVhQamsJILdvpwcQWeSUeQmytJ5yJ8V1fQ5KHDpww195EpgIufovJIQVrJflKdVLxvw== + integrity: sha512-qNcuXw81uBfRLvrBeMcFs/hCXl6FCAglnFkswE98KtSV3Pj2rtxYPQWJxG4/O2GkzCRyVkgxYtfb9uDTYJXZ8w== tarball: 'file:projects/storage-blob.tgz' version: 0.0.0 'file:projects/storage-file-datalake.tgz': @@ -8744,7 +8748,7 @@ packages: dev: false name: '@rush-temp/storage-file-datalake' resolution: - integrity: sha512-Tugmx2vtJKOUGPUbXkCh1Oj/9W0bJ0otjt8Wf0t8Yf/4QzkCfvHzKEFOBT6xUWALbNoSRnvRoxremV2yZrRLrw== + integrity: sha512-ax+/3IKbvusjZpXmloEUa+6cQrULHB6eHFpd6dNzQmQ0a1QOIg3n5uHIEY2X/1kQCOkcMWVITX+imkz6b9Gv2A== tarball: 'file:projects/storage-file-datalake.tgz' version: 0.0.0 'file:projects/storage-file-share.tgz': @@ -8806,9 +8810,67 @@ packages: dev: false name: '@rush-temp/storage-file-share' resolution: - integrity: sha512-u8LJYzcyLwixe5WEH9sDJx4DSChUybX6sHKuFTfyKC98299ipDbcnk7KlDlQkkpMA+4YC2vTE7Zj9jySMF9ezQ== + integrity: sha512-Kzv6N1reNbnI8qxNbwPIXTbMwBaDP9S43wPHoL8akm3ooY8ftesQtF6qCj39Tfw1HozCojH1QRdyahlqIAnTRg== tarball: 'file:projects/storage-file-share.tgz' version: 0.0.0 + 'file:projects/storage-internal-avro.tgz': + dependencies: + '@microsoft/api-extractor': 7.7.11 + '@rollup/plugin-commonjs': 11.0.2_rollup@1.32.1 + '@rollup/plugin-multi-entry': 3.0.0_rollup@1.32.1 + '@rollup/plugin-node-resolve': 7.1.1_rollup@1.32.1 + '@rollup/plugin-replace': 2.3.1_rollup@1.32.1 + '@types/mocha': 7.0.2 + '@types/node': 8.10.59 + '@typescript-eslint/eslint-plugin': 2.27.0_1b16601d03b675251fc711870248ce8d + '@typescript-eslint/parser': 2.27.0_eslint@6.8.0+typescript@3.8.3 + assert: 1.5.0 + cross-env: 6.0.3 + dotenv: 8.2.0 + downlevel-dts: 0.4.0 + es6-promise: 4.2.8 + eslint: 6.8.0 + eslint-config-prettier: 6.10.1_eslint@6.8.0 + eslint-plugin-no-null: 1.0.2_eslint@6.8.0 + eslint-plugin-no-only-tests: 2.4.0 + eslint-plugin-promise: 4.2.1 + esm: 3.2.25 + inherits: 2.0.4 + karma: 4.4.1 + karma-chrome-launcher: 3.1.0 + karma-coverage: 2.0.1 + karma-edge-launcher: 0.4.2_karma@4.4.1 + karma-env-preprocessor: 0.1.1 + karma-firefox-launcher: 1.3.0 + karma-ie-launcher: 1.0.0_karma@4.4.1 + karma-json-preprocessor: 0.3.3_karma@4.4.1 + karma-json-to-file-reporter: 1.0.1 + karma-junit-reporter: 2.0.1_karma@4.4.1 + karma-mocha: 1.3.0 + karma-mocha-reporter: 2.2.5_karma@4.4.1 + karma-remap-istanbul: 0.6.0_karma@4.4.1 + mocha: 7.1.1 + mocha-junit-reporter: 1.23.3_mocha@7.1.1 + nyc: 14.1.1 + prettier: 1.19.1 + puppeteer: 2.1.1 + rimraf: 3.0.2 + rollup: 1.32.1 + rollup-plugin-shim: 1.0.0 + rollup-plugin-sourcemaps: 0.4.2_rollup@1.32.1 + rollup-plugin-terser: 5.3.0_rollup@1.32.1 + rollup-plugin-visualizer: 3.3.2_rollup@1.32.1 + source-map-support: 0.5.16 + ts-node: 8.8.2_typescript@3.8.3 + tslib: 1.11.1 + typescript: 3.8.3 + util: 0.12.2 + dev: false + name: '@rush-temp/storage-internal-avro' + resolution: + integrity: sha512-xQ/8G7hl1NT0N067xLF7XdeD/Fhh+tiMlwEenqOHiUIZ8eMexdCJ5+MhnFOq33oqjMD8dcKku4BjyKQrXBFERg== + tarball: 'file:projects/storage-internal-avro.tgz' + version: 0.0.0 'file:projects/storage-queue.tgz': dependencies: '@azure/core-tracing': 1.0.0-preview.7 @@ -8867,7 +8929,7 @@ packages: dev: false name: '@rush-temp/storage-queue' resolution: - integrity: sha512-7fZ/BCFfKLRNtNOpIkszzJXjIxRDj15LDFPHlynzc02Wc1mxS0NDD7eEptC37QCaBbZzaBqjKnYIC6/O+T2lCA== + integrity: sha512-WcfDHWEOKo17l6dpW4UFbHJgEzQd438F+F61uequ6uc6OH1XAQ/QUU5dWGYiJz+/SU11vyfIW81ZiRsiihQMdg== tarball: 'file:projects/storage-queue.tgz' version: 0.0.0 'file:projects/template.tgz': @@ -9061,6 +9123,7 @@ specifiers: '@rush-temp/storage-blob': 'file:./projects/storage-blob.tgz' '@rush-temp/storage-file-datalake': 'file:./projects/storage-file-datalake.tgz' '@rush-temp/storage-file-share': 'file:./projects/storage-file-share.tgz' + '@rush-temp/storage-internal-avro': 'file:./projects/storage-internal-avro.tgz' '@rush-temp/storage-queue': 'file:./projects/storage-queue.tgz' '@rush-temp/template': 'file:./projects/template.tgz' '@rush-temp/test-utils-perfstress': 'file:./projects/test-utils-perfstress.tgz' diff --git a/rush.json b/rush.json index 30ab6b6876b3..6e3a14fa0877 100644 --- a/rush.json +++ b/rush.json @@ -447,11 +447,6 @@ "projectFolder": "sdk/servicebus/service-bus", "versionPolicyName": "client" }, - { - "packageName": "@azure/keyvault-common", - "projectFolder": "sdk/keyvault/keyvault-common", - "versionPolicyName": "utility" - }, { "packageName": "@azure/storage-internal-avro", "projectFolder": "sdk/storage/storage-internal-avro", diff --git a/sdk/storage/storage-internal-avro/.vscode/extensions.json b/sdk/storage/storage-internal-avro/.vscode/extensions.json new file mode 100644 index 000000000000..c83e26348e1f --- /dev/null +++ b/sdk/storage/storage-internal-avro/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["esbenp.prettier-vscode"] +} diff --git a/sdk/storage/storage-internal-avro/.vscode/launch.json b/sdk/storage/storage-internal-avro/.vscode/launch.json new file mode 100644 index 000000000000..24dbfc9d74c4 --- /dev/null +++ b/sdk/storage/storage-internal-avro/.vscode/launch.json @@ -0,0 +1,59 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Debug Javascript Samples", + "program": "${workspaceFolder}/samples/javascript/basic.js", + "preLaunchTask": "npm: build:js-samples" + }, + { + "type": "node", + "request": "launch", + "name": "Debug Typescript Samples", + "program": "${workspaceFolder}/samples/typescript/basic.ts", + "preLaunchTask": "npm: build:ts-samples", + "outFiles": ["${workspaceFolder}/dist-esm/samples/typescript/*.js"] + }, + { + "type": "node", + "request": "launch", + "name": "Debug Mocha Test [Without Rollup]", + "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", + "args": [ + "-r", + "ts-node/register", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/test/*.spec.ts", + "${workspaceFolder}/test/node/*.spec.ts" + ], + "env": { "TS_NODE_COMPILER_OPTIONS": "{\"module\": \"commonjs\"}" }, + "envFile": "${workspaceFolder}/../.env", + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "protocol": "inspector" + }, + { + "type": "node", + "request": "launch", + "name": "Debug Unit Tests", + "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", + "args": [ + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/dist-test/index.node.js" + ], + "internalConsoleOptions": "openOnSessionStart", + "preLaunchTask": "npm: build:test" + } + ] +} diff --git a/sdk/storage/storage-internal-avro/.vscode/settings.json b/sdk/storage/storage-internal-avro/.vscode/settings.json new file mode 100644 index 000000000000..7ceb5ace3e9d --- /dev/null +++ b/sdk/storage/storage-internal-avro/.vscode/settings.json @@ -0,0 +1,27 @@ +{ + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.DS_Store": true + }, + "[typescript]": { + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.detectIndentation": false + }, + "[json]": { + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.detectIndentation": false + }, + "[yaml]": { + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.detectIndentation": false + }, + "editor.rulers": [ + 100 + ], + "typescript.preferences.quoteStyle": "double", + "javascript.preferences.quoteStyle": "double" + } \ No newline at end of file diff --git a/sdk/storage/storage-internal-avro/package.json b/sdk/storage/storage-internal-avro/package.json index 9562511887ec..4f769a338284 100644 --- a/sdk/storage/storage-internal-avro/package.json +++ b/sdk/storage/storage-internal-avro/package.json @@ -1,5 +1,5 @@ { - "name": "storage-internal-avro", + "name": "@azure/storage-internal-avro", "author": "Microsoft Corporation", "version": "1.0.0-preview.1", "description": "internal avro parser", @@ -11,14 +11,69 @@ "node": ">=8.0.0" }, "scripts": { - "build:es6": "tsc -p tsconfig.json" + "build:es6": "tsc -p tsconfig.json", + "build:nodebrowser": "rollup -c 2>&1", + "build": "npm run build:es6 && npm run build:nodebrowser", + "build:test": "npm run build:es6 && rollup -c rollup.test.config.js 2>&1", + "clean": "rimraf dist dist-esm dist-test typings temp dist-browser/*.js* dist-browser/*.zip statistics.html coverage coverage-browser .nyc_output *.tgz *.log test*.xml TEST*.xml", + "unit-test:node": "mocha --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 120000 dist-test/index.node.js", + "test:node": "npm run clean && npm run build:test && npm run unit-test:node", + "format": "prettier --write --config ../../.prettierrc.json \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"" }, "dependencies": { "tslib": "^1.10.0" }, "devDependencies": { + "@azure/identity": "^1.1.0-preview", + "@azure/test-utils-recorder": "^1.0.0", + "@microsoft/api-extractor": "7.7.11", + "@rollup/plugin-multi-entry": "^3.0.0", + "@rollup/plugin-replace": "^2.2.0", + "@types/mocha": "^7.0.2", "@types/node": "^8.0.0", + "@typescript-eslint/eslint-plugin": "^2.0.0", + "@typescript-eslint/parser": "^2.0.0", + "assert": "^1.4.1", + "cross-env": "^6.0.3", + "dotenv": "^8.2.0", + "downlevel-dts": "~0.4.0", + "es6-promise": "^4.2.5", + "eslint": "^6.1.0", + "eslint-config-prettier": "^6.0.0", + "eslint-plugin-no-null": "^1.0.2", + "eslint-plugin-no-only-tests": "^2.3.0", + "eslint-plugin-promise": "^4.1.1", + "esm": "^3.2.18", + "inherits": "^2.0.3", + "karma": "^4.0.1", + "karma-chrome-launcher": "^3.0.0", + "karma-coverage": "^2.0.0", + "karma-edge-launcher": "^0.4.2", + "karma-env-preprocessor": "^0.1.1", + "karma-firefox-launcher": "^1.1.0", + "karma-ie-launcher": "^1.0.0", + "karma-json-preprocessor": "^0.3.3", + "karma-json-to-file-reporter": "^1.0.1", + "karma-junit-reporter": "^2.0.1", + "karma-mocha": "^1.3.0", + "karma-mocha-reporter": "^2.2.5", + "karma-remap-istanbul": "^0.6.0", + "mocha": "^7.1.1", + "mocha-junit-reporter": "^1.18.0", + "nyc": "^14.0.0", + "prettier": "^1.16.4", + "puppeteer": "^2.0.0", + "rimraf": "^3.0.0", + "rollup": "^1.16.3", + "@rollup/plugin-commonjs": "^11.0.1", + "@rollup/plugin-node-resolve": "^7.0.0", + "rollup-plugin-shim": "^1.0.0", + "rollup-plugin-sourcemaps": "^0.4.2", + "rollup-plugin-terser": "^5.1.1", + "rollup-plugin-visualizer": "^3.1.1", + "source-map-support": "^0.5.9", + "ts-node": "^8.3.0", "typescript": "~3.8.3", - "ts-node": "^8.3.0" + "util": "^0.12.1" } } diff --git a/sdk/storage/storage-internal-avro/rollup.base.config.js b/sdk/storage/storage-internal-avro/rollup.base.config.js new file mode 100644 index 000000000000..6c638d082ff5 --- /dev/null +++ b/sdk/storage/storage-internal-avro/rollup.base.config.js @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import nodeResolve from "@rollup/plugin-node-resolve"; +import multiEntry from "@rollup/plugin-multi-entry"; +import cjs from "@rollup/plugin-commonjs"; +import replace from "@rollup/plugin-replace"; +import { terser } from "rollup-plugin-terser"; +import sourcemaps from "rollup-plugin-sourcemaps"; +import shim from "rollup-plugin-shim"; +// import visualizer from "rollup-plugin-visualizer"; + +const version = require("./package.json").version; +const banner = [ + "/*!", + ` * Azure Storage SDK for JavaScript - Blob, ${version}`, + " * Copyright (c) Microsoft and contributors. All rights reserved.", + " */" +].join("\n"); + +const pkg = require("./package.json"); +const depNames = Object.keys(pkg.dependencies); +const production = process.env.NODE_ENV === "production"; + +export function nodeConfig(test = false) { + const externalNodeBuiltins = [ + "@azure/core-http", + "crypto", + "fs", + "events", + "os", + "stream", + "util" + ]; + const baseConfig = { + input: "dist-esm/src/index.js", + external: depNames.concat(externalNodeBuiltins), + output: { + file: "dist/index.js", + format: "cjs", + sourcemap: true + }, + preserveSymlinks: false, + plugins: [ + sourcemaps(), + replace({ + delimiters: ["", ""], + values: { + // replace dynamic checks with if (true) since this is for node only. + // Allows rollup's dead code elimination to be more aggressive. + "if (isNode)": "if (true)" + } + }), + nodeResolve({ preferBuiltins: true }), + cjs() + ], + onwarn(warning, warn) { + if (warning.code === "CIRCULAR_DEPENDENCY") { + throw new Error(warning.message); + } + warn(warning); + } + }; + + if (test) { + // entry point is every test file + baseConfig.input = [ + "dist-esm/test/*.spec.js", + "dist-esm/test/node/*.spec.js", + "dist-esm/src/index.js" + ]; + baseConfig.plugins.unshift(multiEntry()); + + // different output file + baseConfig.output.file = "dist-test/index.node.js"; + + // mark assert as external + baseConfig.external.push("assert", "fs", "path", "buffer", "zlib"); + + baseConfig.context = "null"; + + // Disable tree-shaking of test code. In rollup-plugin-node-resolve@5.0.0, rollup started respecting + // the "sideEffects" field in package.json. Since our package.json sets "sideEffects=false", this also + // applies to test code, which causes all tests to be removed by tree-shaking. + baseConfig.treeshake = false; + } else if (production) { + baseConfig.plugins.push(terser()); + } + + return baseConfig; +} + +export function browserConfig(test = false) { + const baseConfig = { + input: "dist-esm/src/index.browser.js", + output: { + file: "dist-browser/azure-internal-avro.js", + banner: banner, + format: "umd", + name: "azblob", + sourcemap: true + }, + preserveSymlinks: false, + plugins: [ + sourcemaps(), + replace({ + delimiters: ["", ""], + values: { + // replace dynamic checks with if (false) since this is for + // browser only. Rollup's dead code elimination will remove + // any code guarded by if (isNode) { ... } + "if (isNode)": "if (false)" + } + }), + // fs and os are not used by the browser bundle, so just shim it + // dotenv doesn't work in the browser, so replace it with a no-op function + shim({ + dotenv: `export function config() { }`, + fs: ` + export function stat() { } + export function createReadStream() { } + export function createWriteStream() { } + `, + os: ` + export const type = 1; + export const release = 1; + `, + util: ` + export function promisify() { } + ` + }), + nodeResolve({ + mainFields: ["module", "browser"], + preferBuiltins: false + }), + cjs({ + namedExports: { + events: ["EventEmitter"], + assert: [ + "ok", + "deepEqual", + "equal", + "fail", + "strictEqual", + "deepStrictEqual", + "notDeepEqual", + "notDeepStrictEqual" + ], + "@opentelemetry/api": ["CanonicalCode", "SpanKind", "TraceFlags"] + } + }) + ], + onwarn(warning, warn) { + if (warning.code === "CIRCULAR_DEPENDENCY") { + throw new Error(warning.message); + } + warn(warning); + } + }; + + if (test) { + baseConfig.input = ["dist-esm/test/*.spec.js", "dist-esm/test/browser/*.spec.js"]; + baseConfig.plugins.unshift(multiEntry({ exports: false })); + baseConfig.output.file = "dist-test/index.browser.js"; + // mark fs-extra as external + baseConfig.external = ["fs-extra"]; + + baseConfig.context = "null"; + + // Disable tree-shaking of test code. In rollup-plugin-node-resolve@5.0.0, rollup started respecting + // the "sideEffects" field in package.json. Since our package.json sets "sideEffects=false", this also + // applies to test code, which causes all tests to be removed by tree-shaking. + baseConfig.treeshake = false; + } + + return baseConfig; +} diff --git a/sdk/storage/storage-internal-avro/rollup.config.js b/sdk/storage/storage-internal-avro/rollup.config.js new file mode 100644 index 000000000000..a62dabd573b4 --- /dev/null +++ b/sdk/storage/storage-internal-avro/rollup.config.js @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as base from "./rollup.base.config"; + +const inputs = []; + +if (!process.env.ONLY_BROWSER) { + inputs.push(base.nodeConfig()); +} + +// Disable this until we are ready to run rollup for the browser. +// if (!process.env.ONLY_NODE) { +// inputs.push(base.browserConfig()); +// } + +export default inputs; diff --git a/sdk/storage/storage-internal-avro/rollup.test.config.js b/sdk/storage/storage-internal-avro/rollup.test.config.js new file mode 100644 index 000000000000..ad98718cce46 --- /dev/null +++ b/sdk/storage/storage-internal-avro/rollup.test.config.js @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as base from "./rollup.base.config"; + +export default [base.nodeConfig(true), base.browserConfig(true)]; diff --git a/sdk/storage/storage-internal-avro/src/AvroConstants.ts b/sdk/storage/storage-internal-avro/src/AvroConstants.ts index 43a56d3c9b8c..68de046cd77e 100644 --- a/sdk/storage/storage-internal-avro/src/AvroConstants.ts +++ b/sdk/storage/storage-internal-avro/src/AvroConstants.ts @@ -1,50 +1,50 @@ export const AvroConstants = { - SYNC_MARKER_SIZE: 16, + SYNC_MARKER_SIZE: 16, - // 'O', 'b', 'j', 1 - INIT_BYTES: new Uint8Array([79, 98, 106, 1]), + // 'O', 'b', 'j', 1 + INIT_BYTES: new Uint8Array([79, 98, 106, 1]), - CODEC_KEY: "avro.codec", + CODEC_KEY: "avro.codec", - SCHEMA_KEY: "avro.schema", + SCHEMA_KEY: "avro.schema", - NULL: "null", + NULL: "null", - BOOLEAN: "boolean", + BOOLEAN: "boolean", - INT: "int", + INT: "int", - LONG: "long", + LONG: "long", - FLOAT: "float", + FLOAT: "float", - DOUBLE: "double", + DOUBLE: "double", - BYTES: "bytes", + BYTES: "bytes", - STRING: "string", + STRING: "string", - RECORD: "record", + RECORD: "record", - ENUM: "enum", + ENUM: "enum", - MAP: "map", + MAP: "map", - ARRAY: "array", + ARRAY: "array", - UNION: "union", + UNION: "union", - FIXED: "fixed", + FIXED: "fixed", - ALIASES: "aliases", + ALIASES: "aliases", - NAME: "name", + NAME: "name", - FIELDS: "fields", + FIELDS: "fields", - TYPE: "type", + TYPE: "type", - SYMBOLS: "symbols", + SYMBOLS: "symbols", - VALUES: "values", + VALUES: "values" }; diff --git a/sdk/storage/storage-internal-avro/src/AvroParser.ts b/sdk/storage/storage-internal-avro/src/AvroParser.ts index ef3216f15efa..df6cca50173b 100644 --- a/sdk/storage/storage-internal-avro/src/AvroParser.ts +++ b/sdk/storage/storage-internal-avro/src/AvroParser.ts @@ -3,355 +3,363 @@ import { Dictionary, KeyValuePair } from "./utils/utils.common"; import { AvroConstants } from "./AvroConstants"; export class AvroParser { - /** - * Reads a fixed number of bytes from the stream. - * - * @static - * @param stream - * @param length - */ - public static async readFixedBytes(stream: IReadable, length: number): Promise { - const bytes = await stream.read(length); - if (bytes.length != length) { - throw new Error("Hit stream end.") - } - return bytes; - } - - /** - * Reads a single byte from the stream. - * - * @static - * @param stream - */ - private static async readByte(stream: IReadable): Promise { - const buf = await this.readFixedBytes(stream, 1); - return buf[0]; - } - - private static async readZigZagLong(stream: IReadable): Promise { - // copied from https://github.com/apache/avro/blob/master/lang/js/lib/utils.js#L321 - let n = 0; - let k = 0; - let b, h, f, fk; - - do { - b = await this.readByte(stream); - h = b & 0x80; - n |= (b & 0x7f) << k; - k += 7; - } while (h && k < 28); - - if (h) { - // Switch to float arithmetic, otherwise we might overflow. - f = n; - fk = 268435456; // 2 ** 28. - do { - b = await this.readByte(stream); - f += (b & 0x7f) * fk; - fk *= 128; - } while (b & 0x80); - return (f % 2 ? -(f + 1) : f) / 2; - } - - return (n >> 1) ^ -(n & 1); - } - - public static async readLong(stream: IReadable): Promise { - return this.readZigZagLong(stream); - } - - public static async readInt(stream: IReadable): Promise { - return this.readZigZagLong(stream); - } - - public static async readNull(): Promise { - return null; - } - - public static async readBoolean(stream: IReadable): Promise { - const b = await this.readByte(stream); - if (b == 1) { - return true; - } else if (b == 0) { - return false; - } else { - throw new Error("Byte was not a boolean."); - } + /** + * Reads a fixed number of bytes from the stream. + * + * @static + * @param stream + * @param length + */ + public static async readFixedBytes(stream: IReadable, length: number): Promise { + const bytes = await stream.read(length); + if (bytes.length != length) { + throw new Error("Hit stream end."); } - - public static async readFloat(stream: IReadable): Promise { - const u8arr = await this.readFixedBytes(stream, 4); - const view = new DataView(u8arr.buffer); - return view.getFloat32(0, true); // littleEndian = true - } - - public static async readDouble(stream: IReadable): Promise { - const u8arr = await this.readFixedBytes(stream, 8); - const view = new DataView(u8arr.buffer); - return view.getFloat64(0, true); // littleEndian = true - } - - public static async readBytes(stream: IReadable): Promise { - const size = await this.readLong(stream); - if (size < 0) { - throw new Error("Bytes size was negative."); - } - - return await stream.read(size); + return bytes; + } + + /** + * Reads a single byte from the stream. + * + * @static + * @param stream + */ + private static async readByte(stream: IReadable): Promise { + const buf = await AvroParser.readFixedBytes(stream, 1); + return buf[0]; + } + + private static async readZigZagLong(stream: IReadable): Promise { + // copied from https://github.com/apache/avro/blob/master/lang/js/lib/utils.js#L321 + let n = 0; + let k = 0; + let b, h, f, fk; + + do { + b = await AvroParser.readByte(stream); + h = b & 0x80; + n |= (b & 0x7f) << k; + k += 7; + } while (h && k < 28); + + if (h) { + // Switch to float arithmetic, otherwise we might overflow. + f = n; + fk = 268435456; // 2 ** 28. + do { + b = await AvroParser.readByte(stream); + f += (b & 0x7f) * fk; + fk *= 128; + } while (b & 0x80); + return (f % 2 ? -(f + 1) : f) / 2; } - public static async readString(stream: IReadable): Promise { - const u8arr = await this.readBytes(stream); - - // FIXME: need TextDecoder polyfill for IE - let utf8decoder = new TextDecoder(); - return utf8decoder.decode(u8arr); + return (n >> 1) ^ -(n & 1); + } + + public static async readLong(stream: IReadable): Promise { + return AvroParser.readZigZagLong(stream); + } + + public static async readInt(stream: IReadable): Promise { + return AvroParser.readZigZagLong(stream); + } + + public static async readNull(): Promise { + return null; + } + + public static async readBoolean(stream: IReadable): Promise { + const b = await AvroParser.readByte(stream); + if (b == 1) { + return true; + } else if (b == 0) { + return false; + } else { + throw new Error("Byte was not a boolean."); } - - private static async readMapPair( - stream: IReadable, - readItemMethod: (s: IReadable) => Promise - ): Promise> { - const key = await this.readString(stream); - // FIXME: what about readFixed which need a length as parameter. - const value = await readItemMethod(stream); - return { key, value }; + } + + public static async readFloat(stream: IReadable): Promise { + const u8arr = await AvroParser.readFixedBytes(stream, 4); + const view = new DataView(u8arr.buffer, u8arr.byteOffset, u8arr.byteLength); + return view.getFloat32(0, true); // littleEndian = true + } + + public static async readDouble(stream: IReadable): Promise { + const u8arr = await AvroParser.readFixedBytes(stream, 8); + const view = new DataView(u8arr.buffer, u8arr.byteOffset, u8arr.byteLength); + return view.getFloat64(0, true); // littleEndian = true + } + + public static async readBytes(stream: IReadable): Promise { + const size = await AvroParser.readLong(stream); + if (size < 0) { + throw new Error("Bytes size was negative."); } - public static async readMap( - stream: IReadable, - readItemMethod: (s: IReadable) => Promise - ): Promise> { - const readPairMethod = async (stream: IReadable): Promise> => { - return await this.readMapPair(stream, readItemMethod); - } - - const pairs: KeyValuePair[] = await this.readArray(stream, readPairMethod); - let dict: Dictionary = {}; - for (const pair of pairs) { - dict[pair.key] = pair.value; - } - return dict; + return await stream.read(size); + } + + public static async readString(stream: IReadable): Promise { + const u8arr = await AvroParser.readBytes(stream); + + // FIXME: need TextDecoder polyfill for IE + let utf8decoder = new TextDecoder(); + return utf8decoder.decode(u8arr); + } + + private static async readMapPair( + stream: IReadable, + readItemMethod: (s: IReadable) => Promise + ): Promise> { + const key = await AvroParser.readString(stream); + // FIXME: what about readFixed which need a length as parameter. + const value = await readItemMethod(stream); + return { key, value }; + } + + public static async readMap( + stream: IReadable, + readItemMethod: (s: IReadable) => Promise + ): Promise> { + const readPairMethod = async (stream: IReadable): Promise> => { + return await AvroParser.readMapPair(stream, readItemMethod); + }; + + const pairs: KeyValuePair[] = await AvroParser.readArray(stream, readPairMethod); + let dict: Dictionary = {}; + for (const pair of pairs) { + dict[pair.key] = pair.value; } - - private static async readArray( - stream: IReadable, - readItemMethod: (s: IReadable) => Promise - ): Promise { - let items: T[] = []; - for (let count = await this.readLong(stream); - count != 0; - count = await this.readLong(stream)) { - if (count < 0) { - // Ignore block sizes - await this.readLong(stream); - count = - count; - } - - while (count--) { - const item: T = await readItemMethod(stream); - items.push(item); - } - } - return items; + return dict; + } + + private static async readArray( + stream: IReadable, + readItemMethod: (s: IReadable) => Promise + ): Promise { + let items: T[] = []; + for ( + let count = await AvroParser.readLong(stream); + count != 0; + count = await AvroParser.readLong(stream) + ) { + if (count < 0) { + // Ignore block sizes + await AvroParser.readLong(stream); + count = -count; + } + + while (count--) { + const item: T = await readItemMethod(stream); + items.push(item); + } } + return items; + } } - interface RecordField { - name: string; - type: string | ObjectSchema | (string | ObjectSchema)[]; // Unions may not immediately contain other unions. + name: string; + type: string | ObjectSchema | (string | ObjectSchema)[]; // Unions may not immediately contain other unions. } interface ObjectSchema { - type: 'record' | 'enum' | 'array' | 'map' | 'fixed'; - name?: string; - aliases?: string; - fields?: RecordField[]; - symbols?: string[]; - values?: string; - size?: number; -}; + type: "record" | "enum" | "array" | "map" | "fixed"; + name?: string; + aliases?: string; + fields?: RecordField[]; + symbols?: string[]; + values?: string; + size?: number; +} export abstract class AvroType { - /** - * Reads an object from the stream. - * - * @param stream - */ - public abstract read(stream: IReadable): Promise; - - /** - * Determinds the AvroType from the Avro Schema. - */ - public static fromSchema(schema: string | Object): AvroType { - if (typeof schema == 'string') { - return this.fromStringSchema(schema); - } else if (Array.isArray(schema)) { - return this.fromArraySchema(schema); - } else { - return this.fromObjectSchema(schema as ObjectSchema); - } - } - - private static fromStringSchema(schema: string): AvroType { - // FIXME: simpler way to tell if schema is of type AvroPrimitive? - switch (schema) { - case AvroConstants.NULL: - case AvroConstants.BOOLEAN: - case AvroConstants.INT: - case AvroConstants.LONG: - case AvroConstants.FLOAT: - case AvroConstants.DOUBLE: - case AvroConstants.BYTES: - case AvroConstants.STRING: return new AvroPrimitiveType(schema as AvroPrimitive); - default: throw new Error(`Unexpected Avro type ${schema}`); - } + /** + * Reads an object from the stream. + * + * @param stream + */ + public abstract read(stream: IReadable): Promise; + + /** + * Determinds the AvroType from the Avro Schema. + */ + public static fromSchema(schema: string | Object): AvroType { + if (typeof schema == "string") { + return AvroType.fromStringSchema(schema); + } else if (Array.isArray(schema)) { + return AvroType.fromArraySchema(schema); + } else { + return AvroType.fromObjectSchema(schema as ObjectSchema); } - - private static fromArraySchema(schema: any[]): AvroType { - return new AvroUnionType(schema.map(this.fromSchema)); + } + + private static fromStringSchema(schema: string): AvroType { + // FIXME: simpler way to tell if schema is of type AvroPrimitive? + switch (schema) { + case AvroConstants.NULL: + case AvroConstants.BOOLEAN: + case AvroConstants.INT: + case AvroConstants.LONG: + case AvroConstants.FLOAT: + case AvroConstants.DOUBLE: + case AvroConstants.BYTES: + case AvroConstants.STRING: + return new AvroPrimitiveType(schema as AvroPrimitive); + default: + throw new Error(`Unexpected Avro type ${schema}`); } + } + + private static fromArraySchema(schema: any[]): AvroType { + return new AvroUnionType(schema.map(AvroType.fromSchema)); + } + + private static fromObjectSchema(schema: ObjectSchema): AvroType { + const type = schema.type; + // Primitives can be defined as strings or objects + try { + return AvroType.fromStringSchema(type); + } catch (err) {} + + switch (type) { + case AvroConstants.RECORD: + if (schema.aliases) { + throw new Error(`aliases currently is not supported, schema: ${schema}`); + } + if (!schema.name) { + throw new Error(`Required attribute 'name' doesn't exist on schema: ${schema}`); + } - private static fromObjectSchema(schema: ObjectSchema): AvroType { - const type = schema.type; - // Primitives can be defined as strings or objects - try { - return this.fromStringSchema(type); - } catch (err) { } - - switch (type) { - case AvroConstants.RECORD: - if (schema.aliases) { - throw new Error(`aliases currently is not supported, schema: ${schema}`); - } - if (!schema.name) { - throw new Error(`Required attribute 'name' doesn't exist on schema: ${schema}`); - } - - let fields: Dictionary = {}; - if (!schema.fields) { - throw new Error(`Required attribute 'fields' doesn't exist on schema: ${schema}`); - } - for (const field of schema.fields) { - fields[field.name] = this.fromSchema(field.type); - } - return new AvroRecordType(fields, schema.name); - case AvroConstants.ENUM: - if (schema.aliases) { - throw new Error(`aliases currently is not supported, schema: ${schema}`); - } - if (!schema.symbols) { - throw new Error(`Required attribute 'symbols' doesn't exist on schema: ${schema}`); - } - return new AvroEnumType(schema.symbols); - case AvroConstants.MAP: - if (!schema.values) { - throw new Error(`Required attribute 'values' doesn't exist on schema: ${schema}`); - } - return new AvroMapType(this.fromSchema(schema.values)); - case AvroConstants.ARRAY: // Unused today - case AvroConstants.UNION: // Unused today - case AvroConstants.FIXED: // Unused today - default: - throw new Error(`Unexpected Avro type ${type} in ${schema}`); + let fields: Dictionary = {}; + if (!schema.fields) { + throw new Error(`Required attribute 'fields' doesn't exist on schema: ${schema}`); + } + for (const field of schema.fields) { + fields[field.name] = AvroType.fromSchema(field.type); } + return new AvroRecordType(fields, schema.name); + case AvroConstants.ENUM: + if (schema.aliases) { + throw new Error(`aliases currently is not supported, schema: ${schema}`); + } + if (!schema.symbols) { + throw new Error(`Required attribute 'symbols' doesn't exist on schema: ${schema}`); + } + return new AvroEnumType(schema.symbols); + case AvroConstants.MAP: + if (!schema.values) { + throw new Error(`Required attribute 'values' doesn't exist on schema: ${schema}`); + } + return new AvroMapType(AvroType.fromSchema(schema.values)); + case AvroConstants.ARRAY: // Unused today + case AvroConstants.UNION: // Unused today + case AvroConstants.FIXED: // Unused today + default: + throw new Error(`Unexpected Avro type ${type} in ${schema}`); } + } } - -type AvroPrimitive = 'null' | 'boolean' | 'int ' | 'long' | 'float' | 'double' | 'bytes' | 'string'; +type AvroPrimitive = "null" | "boolean" | "int " | "long" | "float" | "double" | "bytes" | "string"; class AvroPrimitiveType extends AvroType { - private _primitive: AvroPrimitive; - - constructor(primitive: AvroPrimitive) { - super(); - this._primitive = primitive; - } - - public async read(stream: IReadable): Promise { - switch (this._primitive) { - case AvroConstants.NULL: return await AvroParser.readNull(); - case AvroConstants.BOOLEAN: return await AvroParser.readBoolean(stream); - case AvroConstants.INT: return await AvroParser.readInt(stream); - case AvroConstants.LONG: return await AvroParser.readLong(stream); - case AvroConstants.FLOAT: return await AvroParser.readFloat(stream); - case AvroConstants.DOUBLE: return await AvroParser.readDouble(stream); - case AvroConstants.BYTES: return await AvroParser.readBytes(stream); - case AvroConstants.STRING: return await AvroParser.readString(stream); - default: throw new Error("Unknown Avro Primitive"); - } + private _primitive: AvroPrimitive; + + constructor(primitive: AvroPrimitive) { + super(); + this._primitive = primitive; + } + + public async read(stream: IReadable): Promise { + switch (this._primitive) { + case AvroConstants.NULL: + return await AvroParser.readNull(); + case AvroConstants.BOOLEAN: + return await AvroParser.readBoolean(stream); + case AvroConstants.INT: + return await AvroParser.readInt(stream); + case AvroConstants.LONG: + return await AvroParser.readLong(stream); + case AvroConstants.FLOAT: + return await AvroParser.readFloat(stream); + case AvroConstants.DOUBLE: + return await AvroParser.readDouble(stream); + case AvroConstants.BYTES: + return await AvroParser.readBytes(stream); + case AvroConstants.STRING: + return await AvroParser.readString(stream); + default: + throw new Error("Unknown Avro Primitive"); } + } } class AvroEnumType extends AvroType { - private readonly _symbols: string[]; + private readonly _symbols: string[]; - constructor(symbols: string[]) { - super(); - this._symbols = symbols; - } + constructor(symbols: string[]) { + super(); + this._symbols = symbols; + } - public async read(stream: IReadable): Promise { - const value = await AvroParser.readInt(stream); - return this._symbols[value]; - } + public async read(stream: IReadable): Promise { + const value = await AvroParser.readInt(stream); + return this._symbols[value]; + } } - class AvroUnionType extends AvroType { - private readonly _types: AvroType[]; + private readonly _types: AvroType[]; - constructor(types: AvroType[]) { - super(); - this._types = types; - } + constructor(types: AvroType[]) { + super(); + this._types = types; + } - public async read(stream: IReadable): Promise { - const typeIndex = await AvroParser.readInt(stream); - return await this._types[typeIndex].read(stream); - } + public async read(stream: IReadable): Promise { + const typeIndex = await AvroParser.readInt(stream); + return await this._types[typeIndex].read(stream); + } } - class AvroMapType extends AvroType { - private readonly _itemType: AvroType; - - constructor(itemType: AvroType) { - super(); - this._itemType = itemType; - } - - public async read(stream: IReadable): Promise { - const readItemMethod = async (s: IReadable): Promise => { - return await this._itemType.read(s); - } - return await AvroParser.readMap(stream, readItemMethod); - } + private readonly _itemType: AvroType; + + constructor(itemType: AvroType) { + super(); + this._itemType = itemType; + } + + public async read(stream: IReadable): Promise { + const readItemMethod = async (s: IReadable): Promise => { + return await this._itemType.read(s); + }; + return await AvroParser.readMap(stream, readItemMethod); + } } - class AvroRecordType extends AvroType { - private readonly _name: string; - private readonly _fields: Dictionary; - - constructor(fields: Dictionary, name: string) { - super(); - this._fields = fields; - this._name = name; - } - - public async read(stream: IReadable): Promise { - let record: Dictionary = {}; - // FIXME: what for? - record["$schema"] = this._name; - for (const key in this._fields) { - if (this._fields.hasOwnProperty(key)) { - record[key] = await this._fields[key].read(stream); - } - } - return record; + private readonly _name: string; + private readonly _fields: Dictionary; + + constructor(fields: Dictionary, name: string) { + super(); + this._fields = fields; + this._name = name; + } + + public async read(stream: IReadable): Promise { + let record: Dictionary = {}; + // FIXME: what for? + record["$schema"] = this._name; + for (const key in this._fields) { + if (this._fields.hasOwnProperty(key)) { + record[key] = await this._fields[key].read(stream); + } } + return record; + } } diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts index 42a99e1cf4c2..13e29bb41dd0 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReader.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -1,7 +1,7 @@ import { IReadable } from "./IReadable"; import { AvroConstants } from "./AvroConstants"; import { Metadata, arraysEqual } from "./utils/utils.common"; -import { AvroType, AvroParser } from "./AvroParser" +import { AvroType, AvroParser } from "./AvroParser"; export class AvroReader { private readonly _dataStream: IReadable; @@ -28,9 +28,7 @@ export class AvroReader { private _initialized: boolean; - constructor( - dataStream: IReadable - ); + constructor(dataStream: IReadable); constructor( dataStream: IReadable, @@ -54,7 +52,10 @@ export class AvroReader { // FUTURE: cancellation / aborter? private async initialize() { - const header = await AvroParser.readFixedBytes(this._headerStream, AvroConstants.INIT_BYTES.length); + const header = await AvroParser.readFixedBytes( + this._headerStream, + AvroConstants.INIT_BYTES.length + ); if (!arraysEqual(header, AvroConstants.INIT_BYTES)) { throw new Error("Stream is not an Avro file."); } @@ -70,11 +71,14 @@ export class AvroReader { } // The 16-byte, randomly-generated sync marker for this file. - this._syncMarker = await AvroParser.readFixedBytes(this._headerStream, AvroConstants.SYNC_MARKER_SIZE); + this._syncMarker = await AvroParser.readFixedBytes( + this._headerStream, + AvroConstants.SYNC_MARKER_SIZE + ); // Parse the schema const schema = JSON.parse(this._metadata![AvroConstants.SCHEMA_KEY]); - this._itemType = AvroType.fromSchema(schema.RootElement); + this._itemType = AvroType.fromSchema(schema); if (this._blockOffset == 0) { this._blockOffset = this._dataStream.position; @@ -102,7 +106,7 @@ export class AvroReader { await this.initialize(); } - while (this.hasNext) { + while (this.hasNext()) { const result = await this._itemType!.read(this._dataStream); this._itemsRemainingInBlock!--; @@ -120,8 +124,7 @@ export class AvroReader { try { this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream); - } - catch (err) { + } catch (err) { // We hit the end of the stream. this._itemsRemainingInBlock = 0; } diff --git a/sdk/storage/storage-internal-avro/src/IReadable.ts b/sdk/storage/storage-internal-avro/src/IReadable.ts index d54e323bf88e..ab07960c2989 100644 --- a/sdk/storage/storage-internal-avro/src/IReadable.ts +++ b/sdk/storage/storage-internal-avro/src/IReadable.ts @@ -3,7 +3,6 @@ export abstract class IReadable { public abstract async read(size: number): Promise; } - import { Readable } from "stream"; export class ReadableFromStream extends IReadable { private _position: number; @@ -55,43 +54,14 @@ export class ReadableFromStream extends IReadable { this._position += chunk.length; // chunk.lenght maybe less than desired size if the stream ends. resolve(chunk); - this._readable.removeListener('readable', callback); + this._readable.removeListener("readable", callback); } - } - this._readable.on('readable', callback); - this._readable.once('error', reject); - this._readable.once('end', reject); - this._readable.once('close', reject); + }; + this._readable.on("readable", callback); + this._readable.once("error", reject); + this._readable.once("end", reject); + this._readable.once("close", reject); }); } } } - - -/* TEST CODE */ -import * as fs from "fs"; - -async function main() { - let rs = fs.createReadStream("README.md"); - console.log(rs.read(0)); - - let rfs = new ReadableFromStream(rs); - console.log(rfs.position); - console.log(rs.readable); - - const buf = await rfs.read(10); - console.log(buf.toString()); - console.log(rs.readable); - - const buf2 = await rfs.read(100000); - console.log(buf2.toString()); - console.log(rfs.position); - - const buf3 = await rfs.read(10); - console.log(buf3.toString()); - console.log(rfs.position); -} - -main().catch((err) => { - console.error("Error running test:", err.message); -}); diff --git a/sdk/storage/storage-internal-avro/src/index.ts b/sdk/storage/storage-internal-avro/src/index.ts index 71e41f76c2ed..585f225f9b48 100644 --- a/sdk/storage/storage-internal-avro/src/index.ts +++ b/sdk/storage/storage-internal-avro/src/index.ts @@ -1,3 +1,3 @@ import { AvroReader } from "./AvroReader"; -import { IReadable } from "./IReadable"; -export { AvroReader, IReadable } +import { IReadable, ReadableFromStream } from "./IReadable"; +export { AvroReader, IReadable, ReadableFromStream }; diff --git a/sdk/storage/storage-internal-avro/src/utils/utils.common.ts b/sdk/storage/storage-internal-avro/src/utils/utils.common.ts index d561e54e7166..fcb56aa17b23 100644 --- a/sdk/storage/storage-internal-avro/src/utils/utils.common.ts +++ b/sdk/storage/storage-internal-avro/src/utils/utils.common.ts @@ -1,26 +1,26 @@ export interface Metadata { - /** - * A name-value pair. - */ - [propertyName: string]: string; + /** + * A name-value pair. + */ + [propertyName: string]: string; } export interface Dictionary { - [key: string]: T; + [key: string]: T; } export interface KeyValuePair { - key: string; - value: T; + key: string; + value: T; } -export function arraysEqual(a: Uint8Array, b : Uint8Array) { - if (a === b) return true; - if (a == null || b == null) return false; - if (a.length != b.length) return false; +export function arraysEqual(a: Uint8Array, b: Uint8Array) { + if (a === b) return true; + if (a == null || b == null) return false; + if (a.length != b.length) return false; - for (let i = 0; i < a.length; ++i) { - if (a[i] !== b[i]) return false; - } - return true; -} \ No newline at end of file + for (let i = 0; i < a.length; ++i) { + if (a[i] !== b[i]) return false; + } + return true; +} diff --git a/sdk/storage/storage-internal-avro/test/avroreader.spec.ts b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts new file mode 100644 index 000000000000..442be6b71b2a --- /dev/null +++ b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts @@ -0,0 +1,59 @@ +import * as fs from "fs"; +import * as assert from "assert"; +import { AvroReader, ReadableFromStream } from "../src"; +import { arraysEqual } from "../src/utils/utils.common"; + +describe.only("AvroReader", () => { + it("test with local avro files", async () => { + const testCases: TestCase[] = [ + new TestCase("test_null_0.avro", (o) => assert.strictEqual(null, o)), // null + new TestCase("test_null_1.avro", (o) => assert.strictEqual(true, o)), // boolean + new TestCase("test_null_2.avro", (o) => assert.strictEqual("adsfasdf09809dsf-=adsf", o)), // string + new TestCase("test_null_3.avro", (o) => + assert.ok(arraysEqual(new TextEncoder().encode("12345abcd"), o as Uint8Array)) + ), // byte[] + new TestCase("test_null_4.avro", (o) => assert.strictEqual(1234, o)), // int + new TestCase("test_null_5.avro", (o) => assert.strictEqual(1234, o)), // long + new TestCase("test_null_6.avro", (o) => assert.strictEqual(1234.0, o)), // float + new TestCase("test_null_7.avro", (o) => assert.strictEqual(1234.0, o)), // double + // Not supported today. + // new TestCase("test_null_8.avro", o => assert.ok(arraysEqual(new TextEncoder().encode("B"), o as Uint8Array))), // fixed + new TestCase("test_null_9.avro", (o) => assert.strictEqual("B", o)), // enum + // Not supported today. + // new TestCase("test_null_10.avro", o => assert.deepStrictEqual([1, 2, 3], o)), // array + new TestCase("test_null_11.avro", (o) => assert.deepStrictEqual({ a: 1, b: 3, c: 2 }, o)), // map + new TestCase("test_null_12.avro", (o) => assert.strictEqual(null, o)), // union + new TestCase("test_null_13.avro", (o) => { + const expected = { $schema: "Test", f: 5 }; + const expectedEntries = Object.entries(expected); + const actualEntries = Object.entries(o!); + const actualMap = new Map(actualEntries); + assert.strictEqual(expectedEntries.length, actualEntries.length); + for (const [key, value] of expectedEntries) { + assert.deepStrictEqual(actualMap.get(key), value); + } + }) // record + ]; + + for (const testcase of testCases) { + const rs = fs.createReadStream(`./test/resources/${testcase.path}`); + const rfs = new ReadableFromStream(rs); + + const avroReader = new AvroReader(rfs); + const iter = avroReader.parseObjects(); + + let o = await iter.next(); + testcase.predict(o.value); + } + }); +}); + +type Action = (o: Object | null) => void; +class TestCase { + public path: string; + public predict: Action; + constructor(path: string, action: Action) { + this.path = path; + this.predict = action; + } +} diff --git a/sdk/storage/storage-internal-avro/test/readable.spec.ts b/sdk/storage/storage-internal-avro/test/readable.spec.ts new file mode 100644 index 000000000000..4a96f3330d4f --- /dev/null +++ b/sdk/storage/storage-internal-avro/test/readable.spec.ts @@ -0,0 +1,25 @@ +import * as fs from "fs"; +import * as assert from "assert"; +import { ReadableFromStream } from "../src"; + +describe("ReadableFromStream", () => { + it("read pass end should throw", async () => { + let rs = fs.createReadStream("../README.md"); + + let rfs = new ReadableFromStream(rs); + assert.equal(rfs.position, 0); + + await rfs.read(10); + assert.equal(rfs.position, 10); + await rfs.read(100000); + + let exceptionCaught = false; + try { + await rfs.read(10); + } catch (err) { + assert.equal(err.message, "Stream no longer readable."); + exceptionCaught = true; + } + assert.ok(exceptionCaught); + }); +}); diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_0.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_0.avro new file mode 100644 index 0000000000000000000000000000000000000000..91c2b2469e5432eb9ec2390151bc9ff3e90ceaa0 GIT binary patch literal 75 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCd6YmRN(`NUQtMNu2+Y1`d^mr~ K=3Z$L3=seW)g9yj literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_1.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_1.avro new file mode 100644 index 0000000000000000000000000000000000000000..01371934eba3764a31d53dd3f9bc1e461702d1ab GIT binary patch literal 88 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCg_M%=^K()Y^OP9s8P`rr>EQjK TbNPM6yO#l9L_`?j09_6MP7xss literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_10.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_10.avro new file mode 100644 index 0000000000000000000000000000000000000000..97aaaa0bb91a78930168294a4d9b4a235d6dd92a GIT binary patch literal 153 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCU8@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCovM{eDhpDTtQ3@T6AP4d6qL#m zb4pW-K>|7XdFe{E49};o`JAs(f6}V2H`n@m#e0!EjBHGaOiW2^Ovx+^bYP-8005xm BHPQe8 literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_12.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_12.avro new file mode 100644 index 0000000000000000000000000000000000000000..ddf42625f4f320290e6da4136c343b800d139419 GIT binary patch literal 105 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCO`?^GONuh{(v@@+lt7XoIwv2< fk7c+TwR{ENa=xFFChYuieom>MhzJuLpvwUOZ!acI literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_13.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_13.avro new file mode 100644 index 0000000000000000000000000000000000000000..277376ae1aa5801191c8824813be8f020324fbae GIT binary patch literal 157 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCXE9bQl~fj_Dp@Hg6{RNU7o{la zC@AG6=7L2+Qj1GK{Itx}oRngqnrMXTocz3WWVLBZwXwAfaYu7sT72Vj$vnXPB|UzL Mf`|wg9H7eq0E1I9(f|Me literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_14.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_14.avro new file mode 100644 index 0000000000000000000000000000000000000000..3c34ec843837039174125c7b6d7b86554bd35374 GIT binary patch literal 358 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC=P_3+l~fj_Dp@Hg6{RNU7o{la zC@AG6=7L3hGK&j9{Itx}oRngqnrOJ{XeE$QAj#sAqRhN>APX*s#U$taykZ{B$gLMxCiYvk+gsN+Yg$~W$O1+dCXS5M M#1sZ<(dbqH03+gO_W%F@ literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_2.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_2.avro new file mode 100644 index 0000000000000000000000000000000000000000..bf119d9e16f55f99c06d203079cd7b35812f2744 GIT binary patch literal 308 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC1(b?QiZb)kl^8Z(S$pN@?6xH= opWCzEF8(bga)wzaF{L;yu{b5oz|z9N63EuI1&ItZVRVlJ04jEH;{X5v literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_3.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_3.avro new file mode 100644 index 0000000000000000000000000000000000000000..d542117f7f6e950c5858d2f5242d03db1142d117 GIT binary patch literal 177 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC`IM3>OHzxK7|d0}Qa-liyMB4a cZm;{t@4v_iMj=BZV-wTFq~sLZvCypr0FtpdUjP6A literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_4.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_4.avro new file mode 100644 index 0000000000000000000000000000000000000000..b514fd8218419e4dd2b5dfe9bda9f4b678a1581f GIT binary patch literal 94 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCxs)>VN|YGx{5Z8JF?q=kk$qY} QI;waJMKqQOV?uOQ0FbsOYybcN literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_5.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_5.avro new file mode 100644 index 0000000000000000000000000000000000000000..29e8ca4d5f3559887e1628238dff6d8499f315a1 GIT binary patch literal 95 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCd6aVU^U{?VDi@v#HHgi6KXqZr S;X}KQ-W1VTB8&;qRRI97RwwrW literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_6.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_6.avro new file mode 100644 index 0000000000000000000000000000000000000000..df22b0f901a3004e75a9a922eeaf6c61b942fa5f GIT binary patch literal 116 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC`IORf@)Jvx81%y!RhK2p%#Hf7 U#^f4vcdkePgTpKrVlcW+0GJvkApigX literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_7.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_7.avro new file mode 100644 index 0000000000000000000000000000000000000000..1168f99d0d1977ba6a446b5b4599a96efd6f072f GIT binary patch literal 158 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC1(Z_qOOtX^l^6>5J}b~GkAB(O YA*We={AY~F0!9W9@R;mCEgIbl0PIvMivR!s literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_8.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_8.avro new file mode 100644 index 0000000000000000000000000000000000000000..b4136af69b603bc2695bcdffdcd69e9abb650888 GIT binary patch literal 123 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCBdV23DhpDTtQ3^eGAmM3lynr7 u@)C2w0wJlzB_MurW)+BUSj#Y1s# literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/test/resources/test_null_9.avro b/sdk/storage/storage-internal-avro/test/resources/test_null_9.avro new file mode 100644 index 0000000000000000000000000000000000000000..90abc062240449b5df26a9aeeebf602cb9f38d73 GIT binary patch literal 134 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC^Qx6fDhpDTtQ3?|^Gb7-bQF~G z5_7@)kksN55Wl!GHz_}-7^oy#$q^*rq!e4rpyG4uSOkOUm+zatwOxp?(H9Y6f&+9p E09Ilw2LJ#7 literal 0 HcmV?d00001 diff --git a/sdk/storage/storage-internal-avro/tsconfig.json b/sdk/storage/storage-internal-avro/tsconfig.json index 2a3bed613cbe..7d0f438f54f6 100644 --- a/sdk/storage/storage-internal-avro/tsconfig.json +++ b/sdk/storage/storage-internal-avro/tsconfig.json @@ -1,27 +1,26 @@ { - "compilerOptions": { - "alwaysStrict": true, - "noImplicitAny": true, - "preserveConstEnums": true, - "sourceMap": true, - "inlineSources": true, - "newLine": "LF", - "target": "es5", - "moduleResolution": "node", - "noUnusedLocals": true, - "noUnusedParameters": true, - "strict": true, - "module": "esNext", - "outDir": "./dist-esm", - "declaration": true, - "declarationMap": true, - "importHelpers": true, - "declarationDir": "./typings/latest", - "lib": ["dom", "es5", "es6", "es7", "esnext"], - "esModuleInterop": true - }, - "compileOnSave": true, - "exclude": ["node_modules", "./samples/**"], - "include": ["./src/**/*.ts", "./test/**/*.ts"] - } - \ No newline at end of file + "compilerOptions": { + "alwaysStrict": true, + "noImplicitAny": true, + "preserveConstEnums": true, + "sourceMap": true, + "inlineSources": true, + "newLine": "LF", + "target": "es5", + "moduleResolution": "node", + "noUnusedLocals": true, + "noUnusedParameters": true, + "strict": true, + "module": "esNext", + "outDir": "./dist-esm", + "declaration": true, + "declarationMap": true, + "importHelpers": true, + "declarationDir": "./typings/latest", + "lib": ["dom", "es5", "es6", "es7", "esnext"], + "esModuleInterop": true + }, + "compileOnSave": true, + "exclude": ["node_modules", "./samples/**"], + "include": ["./src/**/*.ts", "./test/**/*.ts"] +} From 6d9c153f015c1bf87527ad2afe988b08b6038398 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Wed, 20 May 2020 10:58:23 +0800 Subject: [PATCH 06/14] rename IReadable to AvroReadable, separate the stream implementation with the interface declaration we need different implementation in browsers --- .../storage-internal-avro/src/AvroParser.ts | 50 +++++++++---------- .../storage-internal-avro/src/AvroReadable.ts | 4 ++ ...IReadable.ts => AvroReadableFromStream.ts} | 18 ++----- .../storage-internal-avro/src/AvroReader.ts | 16 +++--- .../storage-internal-avro/src/index.ts | 5 +- ...{readable.spec.ts => avroreadable.spec.ts} | 6 +-- .../test/avroreader.spec.ts | 4 +- 7 files changed, 50 insertions(+), 53 deletions(-) create mode 100644 sdk/storage/storage-internal-avro/src/AvroReadable.ts rename sdk/storage/storage-internal-avro/src/{IReadable.ts => AvroReadableFromStream.ts} (90%) rename sdk/storage/storage-internal-avro/test/{readable.spec.ts => avroreadable.spec.ts} (78%) diff --git a/sdk/storage/storage-internal-avro/src/AvroParser.ts b/sdk/storage/storage-internal-avro/src/AvroParser.ts index df6cca50173b..5f209289e56e 100644 --- a/sdk/storage/storage-internal-avro/src/AvroParser.ts +++ b/sdk/storage/storage-internal-avro/src/AvroParser.ts @@ -1,4 +1,4 @@ -import { IReadable } from "./IReadable"; +import { AvroReadable } from "./AvroReadable"; import { Dictionary, KeyValuePair } from "./utils/utils.common"; import { AvroConstants } from "./AvroConstants"; @@ -10,7 +10,7 @@ export class AvroParser { * @param stream * @param length */ - public static async readFixedBytes(stream: IReadable, length: number): Promise { + public static async readFixedBytes(stream: AvroReadable, length: number): Promise { const bytes = await stream.read(length); if (bytes.length != length) { throw new Error("Hit stream end."); @@ -24,12 +24,12 @@ export class AvroParser { * @static * @param stream */ - private static async readByte(stream: IReadable): Promise { + private static async readByte(stream: AvroReadable): Promise { const buf = await AvroParser.readFixedBytes(stream, 1); return buf[0]; } - private static async readZigZagLong(stream: IReadable): Promise { + private static async readZigZagLong(stream: AvroReadable): Promise { // copied from https://github.com/apache/avro/blob/master/lang/js/lib/utils.js#L321 let n = 0; let k = 0; @@ -57,11 +57,11 @@ export class AvroParser { return (n >> 1) ^ -(n & 1); } - public static async readLong(stream: IReadable): Promise { + public static async readLong(stream: AvroReadable): Promise { return AvroParser.readZigZagLong(stream); } - public static async readInt(stream: IReadable): Promise { + public static async readInt(stream: AvroReadable): Promise { return AvroParser.readZigZagLong(stream); } @@ -69,7 +69,7 @@ export class AvroParser { return null; } - public static async readBoolean(stream: IReadable): Promise { + public static async readBoolean(stream: AvroReadable): Promise { const b = await AvroParser.readByte(stream); if (b == 1) { return true; @@ -80,19 +80,19 @@ export class AvroParser { } } - public static async readFloat(stream: IReadable): Promise { + public static async readFloat(stream: AvroReadable): Promise { const u8arr = await AvroParser.readFixedBytes(stream, 4); const view = new DataView(u8arr.buffer, u8arr.byteOffset, u8arr.byteLength); return view.getFloat32(0, true); // littleEndian = true } - public static async readDouble(stream: IReadable): Promise { + public static async readDouble(stream: AvroReadable): Promise { const u8arr = await AvroParser.readFixedBytes(stream, 8); const view = new DataView(u8arr.buffer, u8arr.byteOffset, u8arr.byteLength); return view.getFloat64(0, true); // littleEndian = true } - public static async readBytes(stream: IReadable): Promise { + public static async readBytes(stream: AvroReadable): Promise { const size = await AvroParser.readLong(stream); if (size < 0) { throw new Error("Bytes size was negative."); @@ -101,7 +101,7 @@ export class AvroParser { return await stream.read(size); } - public static async readString(stream: IReadable): Promise { + public static async readString(stream: AvroReadable): Promise { const u8arr = await AvroParser.readBytes(stream); // FIXME: need TextDecoder polyfill for IE @@ -110,8 +110,8 @@ export class AvroParser { } private static async readMapPair( - stream: IReadable, - readItemMethod: (s: IReadable) => Promise + stream: AvroReadable, + readItemMethod: (s: AvroReadable) => Promise ): Promise> { const key = await AvroParser.readString(stream); // FIXME: what about readFixed which need a length as parameter. @@ -120,10 +120,10 @@ export class AvroParser { } public static async readMap( - stream: IReadable, - readItemMethod: (s: IReadable) => Promise + stream: AvroReadable, + readItemMethod: (s: AvroReadable) => Promise ): Promise> { - const readPairMethod = async (stream: IReadable): Promise> => { + const readPairMethod = async (stream: AvroReadable): Promise> => { return await AvroParser.readMapPair(stream, readItemMethod); }; @@ -136,8 +136,8 @@ export class AvroParser { } private static async readArray( - stream: IReadable, - readItemMethod: (s: IReadable) => Promise + stream: AvroReadable, + readItemMethod: (s: AvroReadable) => Promise ): Promise { let items: T[] = []; for ( @@ -181,7 +181,7 @@ export abstract class AvroType { * * @param stream */ - public abstract read(stream: IReadable): Promise; + public abstract read(stream: AvroReadable): Promise; /** * Determinds the AvroType from the Avro Schema. @@ -273,7 +273,7 @@ class AvroPrimitiveType extends AvroType { this._primitive = primitive; } - public async read(stream: IReadable): Promise { + public async read(stream: AvroReadable): Promise { switch (this._primitive) { case AvroConstants.NULL: return await AvroParser.readNull(); @@ -305,7 +305,7 @@ class AvroEnumType extends AvroType { this._symbols = symbols; } - public async read(stream: IReadable): Promise { + public async read(stream: AvroReadable): Promise { const value = await AvroParser.readInt(stream); return this._symbols[value]; } @@ -319,7 +319,7 @@ class AvroUnionType extends AvroType { this._types = types; } - public async read(stream: IReadable): Promise { + public async read(stream: AvroReadable): Promise { const typeIndex = await AvroParser.readInt(stream); return await this._types[typeIndex].read(stream); } @@ -333,8 +333,8 @@ class AvroMapType extends AvroType { this._itemType = itemType; } - public async read(stream: IReadable): Promise { - const readItemMethod = async (s: IReadable): Promise => { + public async read(stream: AvroReadable): Promise { + const readItemMethod = async (s: AvroReadable): Promise => { return await this._itemType.read(s); }; return await AvroParser.readMap(stream, readItemMethod); @@ -351,7 +351,7 @@ class AvroRecordType extends AvroType { this._name = name; } - public async read(stream: IReadable): Promise { + public async read(stream: AvroReadable): Promise { let record: Dictionary = {}; // FIXME: what for? record["$schema"] = this._name; diff --git a/sdk/storage/storage-internal-avro/src/AvroReadable.ts b/sdk/storage/storage-internal-avro/src/AvroReadable.ts new file mode 100644 index 000000000000..36151b8fa453 --- /dev/null +++ b/sdk/storage/storage-internal-avro/src/AvroReadable.ts @@ -0,0 +1,4 @@ +export abstract class AvroReadable { + public abstract get position(): number; + public abstract async read(size: number): Promise; +} diff --git a/sdk/storage/storage-internal-avro/src/IReadable.ts b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts similarity index 90% rename from sdk/storage/storage-internal-avro/src/IReadable.ts rename to sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts index ab07960c2989..afc3352c859f 100644 --- a/sdk/storage/storage-internal-avro/src/IReadable.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReadableFromStream.ts @@ -1,19 +1,14 @@ -export abstract class IReadable { - public abstract get position(): number; - public abstract async read(size: number): Promise; -} - import { Readable } from "stream"; -export class ReadableFromStream extends IReadable { +import { AvroReadable } from './AvroReadable'; + +export class AvroReadableFromStream extends AvroReadable { private _position: number; private _readable: Readable; // private _stillReadable: boolean; - constructor(readable: Readable) { super(); this._readable = readable; this._position = 0; - // workaround due to Readable.readable only availabe after Node.js v11.4 // this._stillReadable = true; // this._readable.on("end", () => { @@ -23,29 +18,26 @@ export class ReadableFromStream extends IReadable { // this._stillReadable = false; // }); } - public get position(): number { return this._position; } - public async read(size: number): Promise { if (size <= 0) { throw new Error(`size parameter should be positive: ${size}`); } - // readable is true if it is safe to call readable.read(), which means the stream has not been destroyed or emitted 'error' or 'end'. // if (!this._stillReadable || this._readable.destroyed) { if (!this._readable.readable) { throw new Error("Stream no longer readable."); } - // See if there is already enough data, note that "Only after readable.read() returns null, 'readable' will be emitted." let chunk = this._readable.read(size); if (chunk) { this._position += chunk.length; // chunk.lenght maybe less than desired size if the stream ends. return chunk; - } else { + } + else { // register callback to wait for enough data to read return new Promise((resolve, reject) => { const callback = () => { diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts index 13e29bb41dd0..4fbd9eec8816 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReader.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -1,12 +1,12 @@ -import { IReadable } from "./IReadable"; +import { AvroReadable } from "./AvroReadable"; import { AvroConstants } from "./AvroConstants"; import { Metadata, arraysEqual } from "./utils/utils.common"; import { AvroType, AvroParser } from "./AvroParser"; export class AvroReader { - private readonly _dataStream: IReadable; + private readonly _dataStream: AvroReadable; - private readonly _headerStream: IReadable; + private readonly _headerStream: AvroReadable; private _syncMarker: Uint8Array | undefined; @@ -28,18 +28,18 @@ export class AvroReader { private _initialized: boolean; - constructor(dataStream: IReadable); + constructor(dataStream: AvroReadable); constructor( - dataStream: IReadable, - headerStream: IReadable, + dataStream: AvroReadable, + headerStream: AvroReadable, currentBlockOffset: number, indexWithinCurrentBlock: number ); constructor( - dataStream: IReadable, - headerStream?: IReadable, + dataStream: AvroReadable, + headerStream?: AvroReadable, currentBlockOffset?: number, indexWithinCurrentBlock?: number ) { diff --git a/sdk/storage/storage-internal-avro/src/index.ts b/sdk/storage/storage-internal-avro/src/index.ts index 585f225f9b48..a43b307a484b 100644 --- a/sdk/storage/storage-internal-avro/src/index.ts +++ b/sdk/storage/storage-internal-avro/src/index.ts @@ -1,3 +1,4 @@ import { AvroReader } from "./AvroReader"; -import { IReadable, ReadableFromStream } from "./IReadable"; -export { AvroReader, IReadable, ReadableFromStream }; +import { AvroReadable } from "./AvroReadable"; +import { AvroReadableFromStream } from "./AvroReadableFromStream"; +export { AvroReader, AvroReadable, AvroReadableFromStream }; diff --git a/sdk/storage/storage-internal-avro/test/readable.spec.ts b/sdk/storage/storage-internal-avro/test/avroreadable.spec.ts similarity index 78% rename from sdk/storage/storage-internal-avro/test/readable.spec.ts rename to sdk/storage/storage-internal-avro/test/avroreadable.spec.ts index 4a96f3330d4f..d2e382cabc10 100644 --- a/sdk/storage/storage-internal-avro/test/readable.spec.ts +++ b/sdk/storage/storage-internal-avro/test/avroreadable.spec.ts @@ -1,12 +1,12 @@ import * as fs from "fs"; import * as assert from "assert"; -import { ReadableFromStream } from "../src"; +import { AvroReadableFromStream } from "../src"; -describe("ReadableFromStream", () => { +describe("AvroReadableFromStream", () => { it("read pass end should throw", async () => { let rs = fs.createReadStream("../README.md"); - let rfs = new ReadableFromStream(rs); + let rfs = new AvroReadableFromStream(rs); assert.equal(rfs.position, 0); await rfs.read(10); diff --git a/sdk/storage/storage-internal-avro/test/avroreader.spec.ts b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts index 442be6b71b2a..eb7f9f89c45d 100644 --- a/sdk/storage/storage-internal-avro/test/avroreader.spec.ts +++ b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts @@ -1,6 +1,6 @@ import * as fs from "fs"; import * as assert from "assert"; -import { AvroReader, ReadableFromStream } from "../src"; +import { AvroReader, AvroReadableFromStream } from "../src"; import { arraysEqual } from "../src/utils/utils.common"; describe.only("AvroReader", () => { @@ -37,7 +37,7 @@ describe.only("AvroReader", () => { for (const testcase of testCases) { const rs = fs.createReadStream(`./test/resources/${testcase.path}`); - const rfs = new ReadableFromStream(rs); + const rfs = new AvroReadableFromStream(rs); const avroReader = new AvroReader(rfs); const iter = avroReader.parseObjects(); From 411ec260fbf8630367d627fdd86d565c423db183 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Wed, 20 May 2020 11:12:59 +0800 Subject: [PATCH 07/14] add "private": true to pacakge.json --- sdk/storage/storage-internal-avro/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/storage/storage-internal-avro/package.json b/sdk/storage/storage-internal-avro/package.json index 4f769a338284..94db763533fb 100644 --- a/sdk/storage/storage-internal-avro/package.json +++ b/sdk/storage/storage-internal-avro/package.json @@ -4,6 +4,7 @@ "version": "1.0.0-preview.1", "description": "internal avro parser", "license": "MIT", + "private": true, "main": "./srt/index.ts", "module": "dist-esm/index.js", "types": "./types/index.d.ts", @@ -76,4 +77,4 @@ "typescript": "~3.8.3", "util": "^0.12.1" } -} +} \ No newline at end of file From c885b32de622284ea2ac9987661722f5e18754cf Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Wed, 20 May 2020 11:37:52 +0800 Subject: [PATCH 08/14] use Record --- .../storage-internal-avro/src/AvroParser.ts | 14 +++++++------- .../storage-internal-avro/src/AvroReader.ts | 4 ++-- .../src/utils/utils.common.ts | 13 +------------ 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/sdk/storage/storage-internal-avro/src/AvroParser.ts b/sdk/storage/storage-internal-avro/src/AvroParser.ts index 5f209289e56e..a8e1d4068fd2 100644 --- a/sdk/storage/storage-internal-avro/src/AvroParser.ts +++ b/sdk/storage/storage-internal-avro/src/AvroParser.ts @@ -1,5 +1,5 @@ import { AvroReadable } from "./AvroReadable"; -import { Dictionary, KeyValuePair } from "./utils/utils.common"; +import { KeyValuePair } from "./utils/utils.common"; import { AvroConstants } from "./AvroConstants"; export class AvroParser { @@ -122,13 +122,13 @@ export class AvroParser { public static async readMap( stream: AvroReadable, readItemMethod: (s: AvroReadable) => Promise - ): Promise> { + ): Promise> { const readPairMethod = async (stream: AvroReadable): Promise> => { return await AvroParser.readMapPair(stream, readItemMethod); }; const pairs: KeyValuePair[] = await AvroParser.readArray(stream, readPairMethod); - let dict: Dictionary = {}; + let dict: Record = {}; for (const pair of pairs) { dict[pair.key] = pair.value; } @@ -233,7 +233,7 @@ export abstract class AvroType { throw new Error(`Required attribute 'name' doesn't exist on schema: ${schema}`); } - let fields: Dictionary = {}; + let fields: Record = {}; if (!schema.fields) { throw new Error(`Required attribute 'fields' doesn't exist on schema: ${schema}`); } @@ -343,16 +343,16 @@ class AvroMapType extends AvroType { class AvroRecordType extends AvroType { private readonly _name: string; - private readonly _fields: Dictionary; + private readonly _fields: Record; - constructor(fields: Dictionary, name: string) { + constructor(fields: Record, name: string) { super(); this._fields = fields; this._name = name; } public async read(stream: AvroReadable): Promise { - let record: Dictionary = {}; + let record: Record = {}; // FIXME: what for? record["$schema"] = this._name; for (const key in this._fields) { diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts index 4fbd9eec8816..4eddd5c14508 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReader.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -1,6 +1,6 @@ import { AvroReadable } from "./AvroReadable"; import { AvroConstants } from "./AvroConstants"; -import { Metadata, arraysEqual } from "./utils/utils.common"; +import { arraysEqual } from "./utils/utils.common"; import { AvroType, AvroParser } from "./AvroParser"; export class AvroReader { @@ -10,7 +10,7 @@ export class AvroReader { private _syncMarker: Uint8Array | undefined; - private _metadata: Metadata | undefined; + private _metadata: Record | undefined; private _itemType: AvroType | undefined; diff --git a/sdk/storage/storage-internal-avro/src/utils/utils.common.ts b/sdk/storage/storage-internal-avro/src/utils/utils.common.ts index fcb56aa17b23..ace1939dd47a 100644 --- a/sdk/storage/storage-internal-avro/src/utils/utils.common.ts +++ b/sdk/storage/storage-internal-avro/src/utils/utils.common.ts @@ -1,20 +1,9 @@ -export interface Metadata { - /** - * A name-value pair. - */ - [propertyName: string]: string; -} - -export interface Dictionary { - [key: string]: T; -} - export interface KeyValuePair { key: string; value: T; } -export function arraysEqual(a: Uint8Array, b: Uint8Array) { +export function arraysEqual(a: Uint8Array, b: Uint8Array) : boolean { if (a === b) return true; if (a == null || b == null) return false; if (a.length != b.length) return false; From 696d5d1547ecdba32fcfec81140109e1d8138d2b Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Wed, 20 May 2020 14:44:20 +0800 Subject: [PATCH 09/14] me --- sdk/storage/storage-blob/package.json | 1 + sdk/storage/storage-blob/src/index.ts | 4 ++-- sdk/storage/storage-internal-avro/package.json | 4 +++- sdk/storage/storage-internal-avro/test/avroreader.spec.ts | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/storage/storage-blob/package.json b/sdk/storage/storage-blob/package.json index 17bfddd4584d..c5b673095689 100644 --- a/sdk/storage/storage-blob/package.json +++ b/sdk/storage/storage-blob/package.json @@ -64,6 +64,7 @@ "BreakingChanges.md", "dist/", "dist-esm/storage-blob/src/", + "dist-esm/storage-internal-avro/src/", "typings/latest/storage-blob.d.ts", "typings/3.1/storage-blob.d.ts", "README.md", diff --git a/sdk/storage/storage-blob/src/index.ts b/sdk/storage/storage-blob/src/index.ts index 50e0c51576ce..2d6249684b5d 100644 --- a/sdk/storage/storage-blob/src/index.ts +++ b/sdk/storage/storage-blob/src/index.ts @@ -49,5 +49,5 @@ export { CopyPollerBlobClient } from "./pollers/BlobStartCopyFromUrlPoller"; -import { IReadable } from '../../storage-internal-avro/src' -export { IReadable } +import { AvroReadable } from '../../storage-internal-avro/src' +export { AvroReadable } diff --git a/sdk/storage/storage-internal-avro/package.json b/sdk/storage/storage-internal-avro/package.json index 94db763533fb..9e628455afe0 100644 --- a/sdk/storage/storage-internal-avro/package.json +++ b/sdk/storage/storage-internal-avro/package.json @@ -1,10 +1,12 @@ { "name": "@azure/storage-internal-avro", + "sideEffect": false, + "private": true, "author": "Microsoft Corporation", "version": "1.0.0-preview.1", "description": "internal avro parser", "license": "MIT", - "private": true, + "repository": "github:Azure/azure-sdk-for-js", "main": "./srt/index.ts", "module": "dist-esm/index.js", "types": "./types/index.d.ts", diff --git a/sdk/storage/storage-internal-avro/test/avroreader.spec.ts b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts index eb7f9f89c45d..379014a5c541 100644 --- a/sdk/storage/storage-internal-avro/test/avroreader.spec.ts +++ b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts @@ -3,7 +3,7 @@ import * as assert from "assert"; import { AvroReader, AvroReadableFromStream } from "../src"; import { arraysEqual } from "../src/utils/utils.common"; -describe.only("AvroReader", () => { +describe("AvroReader", () => { it("test with local avro files", async () => { const testCases: TestCase[] = [ new TestCase("test_null_0.avro", (o) => assert.strictEqual(null, o)), // null From 6487153547f4cb751308e25fd2c8343f4145c912 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Wed, 20 May 2020 17:41:57 +0800 Subject: [PATCH 10/14] fix --- sdk/storage/storage-blob/src/index.ts | 3 --- sdk/storage/storage-internal-avro/src/AvroReader.ts | 8 ++++---- sdk/storage/storage-internal-avro/src/index.ts | 7 +++---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/sdk/storage/storage-blob/src/index.ts b/sdk/storage/storage-blob/src/index.ts index 2d6249684b5d..6cb9eee5aa44 100644 --- a/sdk/storage/storage-blob/src/index.ts +++ b/sdk/storage/storage-blob/src/index.ts @@ -48,6 +48,3 @@ export { BlobBeginCopyFromUrlPollState, CopyPollerBlobClient } from "./pollers/BlobStartCopyFromUrlPoller"; - -import { AvroReadable } from '../../storage-internal-avro/src' -export { AvroReadable } diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts index 4eddd5c14508..497754797dbb 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReader.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -8,13 +8,13 @@ export class AvroReader { private readonly _headerStream: AvroReadable; - private _syncMarker: Uint8Array | undefined; + private _syncMarker?: Uint8Array; - private _metadata: Record | undefined; + private _metadata?: Record; - private _itemType: AvroType | undefined; + private _itemType?: AvroType; - private _itemsRemainingInBlock: number | undefined; + private _itemsRemainingInBlock?: number; private _blockOffset: number; public get blockOffset(): number { diff --git a/sdk/storage/storage-internal-avro/src/index.ts b/sdk/storage/storage-internal-avro/src/index.ts index a43b307a484b..b8e8047dc0f9 100644 --- a/sdk/storage/storage-internal-avro/src/index.ts +++ b/sdk/storage/storage-internal-avro/src/index.ts @@ -1,4 +1,3 @@ -import { AvroReader } from "./AvroReader"; -import { AvroReadable } from "./AvroReadable"; -import { AvroReadableFromStream } from "./AvroReadableFromStream"; -export { AvroReader, AvroReadable, AvroReadableFromStream }; +export { AvroReader } from "./AvroReader"; +export { AvroReadable } from "./AvroReadable"; +export { AvroReadableFromStream } from "./AvroReadableFromStream"; From 18e8c2bcd9a98752775b56a765ceb9f82008a74e Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 21 May 2020 13:57:28 +0800 Subject: [PATCH 11/14] tentative: use string enum instead of string union to avoid two copies of constant string --- .../src/AvroConstants.ts | 54 ++------------ .../storage-internal-avro/src/AvroParser.ts | 70 ++++++++++++------- .../storage-internal-avro/src/AvroReader.ts | 12 ++-- 3 files changed, 53 insertions(+), 83 deletions(-) diff --git a/sdk/storage/storage-internal-avro/src/AvroConstants.ts b/sdk/storage/storage-internal-avro/src/AvroConstants.ts index 68de046cd77e..c5368c596246 100644 --- a/sdk/storage/storage-internal-avro/src/AvroConstants.ts +++ b/sdk/storage/storage-internal-avro/src/AvroConstants.ts @@ -1,50 +1,4 @@ -export const AvroConstants = { - SYNC_MARKER_SIZE: 16, - - // 'O', 'b', 'j', 1 - INIT_BYTES: new Uint8Array([79, 98, 106, 1]), - - CODEC_KEY: "avro.codec", - - SCHEMA_KEY: "avro.schema", - - NULL: "null", - - BOOLEAN: "boolean", - - INT: "int", - - LONG: "long", - - FLOAT: "float", - - DOUBLE: "double", - - BYTES: "bytes", - - STRING: "string", - - RECORD: "record", - - ENUM: "enum", - - MAP: "map", - - ARRAY: "array", - - UNION: "union", - - FIXED: "fixed", - - ALIASES: "aliases", - - NAME: "name", - - FIELDS: "fields", - - TYPE: "type", - - SYMBOLS: "symbols", - - VALUES: "values" -}; +export const AVRO_SYNC_MARKER_SIZE: number = 16; +export const AVRO_INIT_BYTES: Uint8Array = new Uint8Array([79, 98, 106, 1]); +export const AVRO_CODEC_KEY: string = "avro.codec"; +export const AVRO_SCHEMA_KEY: string = "avro.schema"; diff --git a/sdk/storage/storage-internal-avro/src/AvroParser.ts b/sdk/storage/storage-internal-avro/src/AvroParser.ts index a8e1d4068fd2..b63a347b4375 100644 --- a/sdk/storage/storage-internal-avro/src/AvroParser.ts +++ b/sdk/storage/storage-internal-avro/src/AvroParser.ts @@ -1,6 +1,5 @@ import { AvroReadable } from "./AvroReadable"; import { KeyValuePair } from "./utils/utils.common"; -import { AvroConstants } from "./AvroConstants"; export class AvroParser { /** @@ -165,8 +164,17 @@ interface RecordField { type: string | ObjectSchema | (string | ObjectSchema)[]; // Unions may not immediately contain other unions. } +enum AvroComplex { + RECORD = 'record', + ENUM = 'enum', + ARRAY = 'array', + MAP = 'map', + UNION = 'union', + FIXED = 'fixed', +} + interface ObjectSchema { - type: "record" | "enum" | "array" | "map" | "fixed"; + type: Exclude; name?: string; aliases?: string; fields?: RecordField[]; @@ -184,7 +192,7 @@ export abstract class AvroType { public abstract read(stream: AvroReadable): Promise; /** - * Determinds the AvroType from the Avro Schema. + * Determines the AvroType from the Avro Schema. */ public static fromSchema(schema: string | Object): AvroType { if (typeof schema == "string") { @@ -199,14 +207,14 @@ export abstract class AvroType { private static fromStringSchema(schema: string): AvroType { // FIXME: simpler way to tell if schema is of type AvroPrimitive? switch (schema) { - case AvroConstants.NULL: - case AvroConstants.BOOLEAN: - case AvroConstants.INT: - case AvroConstants.LONG: - case AvroConstants.FLOAT: - case AvroConstants.DOUBLE: - case AvroConstants.BYTES: - case AvroConstants.STRING: + case AvroPrimitive.NULL: + case AvroPrimitive.BOOLEAN: + case AvroPrimitive.INT: + case AvroPrimitive.LONG: + case AvroPrimitive.FLOAT: + case AvroPrimitive.DOUBLE: + case AvroPrimitive.BYTES: + case AvroPrimitive.STRING: return new AvroPrimitiveType(schema as AvroPrimitive); default: throw new Error(`Unexpected Avro type ${schema}`); @@ -222,10 +230,10 @@ export abstract class AvroType { // Primitives can be defined as strings or objects try { return AvroType.fromStringSchema(type); - } catch (err) {} + } catch (err) { } switch (type) { - case AvroConstants.RECORD: + case AvroComplex.RECORD: if (schema.aliases) { throw new Error(`aliases currently is not supported, schema: ${schema}`); } @@ -241,7 +249,7 @@ export abstract class AvroType { fields[field.name] = AvroType.fromSchema(field.type); } return new AvroRecordType(fields, schema.name); - case AvroConstants.ENUM: + case AvroComplex.ENUM: if (schema.aliases) { throw new Error(`aliases currently is not supported, schema: ${schema}`); } @@ -249,21 +257,29 @@ export abstract class AvroType { throw new Error(`Required attribute 'symbols' doesn't exist on schema: ${schema}`); } return new AvroEnumType(schema.symbols); - case AvroConstants.MAP: + case AvroComplex.MAP: if (!schema.values) { throw new Error(`Required attribute 'values' doesn't exist on schema: ${schema}`); } return new AvroMapType(AvroType.fromSchema(schema.values)); - case AvroConstants.ARRAY: // Unused today - case AvroConstants.UNION: // Unused today - case AvroConstants.FIXED: // Unused today + case AvroComplex.ARRAY: // Unused today + case AvroComplex.FIXED: // Unused today default: throw new Error(`Unexpected Avro type ${type} in ${schema}`); } } } -type AvroPrimitive = "null" | "boolean" | "int " | "long" | "float" | "double" | "bytes" | "string"; +enum AvroPrimitive { + NULL = "null", + BOOLEAN = 'boolean', + INT = 'int', + LONG = 'long', + FLOAT = 'float', + DOUBLE = 'double', + BYTES = 'bytes', + STRING = 'string' +} class AvroPrimitiveType extends AvroType { private _primitive: AvroPrimitive; @@ -275,21 +291,21 @@ class AvroPrimitiveType extends AvroType { public async read(stream: AvroReadable): Promise { switch (this._primitive) { - case AvroConstants.NULL: + case AvroPrimitive.NULL: return await AvroParser.readNull(); - case AvroConstants.BOOLEAN: + case AvroPrimitive.BOOLEAN: return await AvroParser.readBoolean(stream); - case AvroConstants.INT: + case AvroPrimitive.INT: return await AvroParser.readInt(stream); - case AvroConstants.LONG: + case AvroPrimitive.LONG: return await AvroParser.readLong(stream); - case AvroConstants.FLOAT: + case AvroPrimitive.FLOAT: return await AvroParser.readFloat(stream); - case AvroConstants.DOUBLE: + case AvroPrimitive.DOUBLE: return await AvroParser.readDouble(stream); - case AvroConstants.BYTES: + case AvroPrimitive.BYTES: return await AvroParser.readBytes(stream); - case AvroConstants.STRING: + case AvroPrimitive.STRING: return await AvroParser.readString(stream); default: throw new Error("Unknown Avro Primitive"); diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts index 497754797dbb..224312a46b59 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReader.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -1,5 +1,5 @@ import { AvroReadable } from "./AvroReadable"; -import { AvroConstants } from "./AvroConstants"; +import { AVRO_SYNC_MARKER_SIZE, AVRO_INIT_BYTES, AVRO_CODEC_KEY, AVRO_SCHEMA_KEY } from "./AvroConstants"; import { arraysEqual } from "./utils/utils.common"; import { AvroType, AvroParser } from "./AvroParser"; @@ -54,9 +54,9 @@ export class AvroReader { private async initialize() { const header = await AvroParser.readFixedBytes( this._headerStream, - AvroConstants.INIT_BYTES.length + AVRO_INIT_BYTES.length ); - if (!arraysEqual(header, AvroConstants.INIT_BYTES)) { + if (!arraysEqual(header, AVRO_INIT_BYTES)) { throw new Error("Stream is not an Avro file."); } @@ -65,7 +65,7 @@ export class AvroReader { this._metadata = await AvroParser.readMap(this._headerStream, AvroParser.readString); // Validate codec - const codec = this._metadata![AvroConstants.CODEC_KEY]; + const codec = this._metadata![AVRO_CODEC_KEY]; if (!(codec == undefined || codec == "null")) { throw new Error("Codecs are not supported"); } @@ -73,11 +73,11 @@ export class AvroReader { // The 16-byte, randomly-generated sync marker for this file. this._syncMarker = await AvroParser.readFixedBytes( this._headerStream, - AvroConstants.SYNC_MARKER_SIZE + AVRO_SYNC_MARKER_SIZE ); // Parse the schema - const schema = JSON.parse(this._metadata![AvroConstants.SCHEMA_KEY]); + const schema = JSON.parse(this._metadata![AVRO_SCHEMA_KEY]); this._itemType = AvroType.fromSchema(schema); if (this._blockOffset == 0) { From 1331017ecbfedc98414396db7f1798d2318bac94 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 21 May 2020 15:03:25 +0800 Subject: [PATCH 12/14] avroReader test iterate through all records --- sdk/storage/storage-internal-avro/test/avroreader.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/storage/storage-internal-avro/test/avroreader.spec.ts b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts index 379014a5c541..e7e51c180bcb 100644 --- a/sdk/storage/storage-internal-avro/test/avroreader.spec.ts +++ b/sdk/storage/storage-internal-avro/test/avroreader.spec.ts @@ -41,9 +41,9 @@ describe("AvroReader", () => { const avroReader = new AvroReader(rfs); const iter = avroReader.parseObjects(); - - let o = await iter.next(); - testcase.predict(o.value); + for await (const o of iter) { + testcase.predict(o); + } } }); }); From 389c567a118e0c73a58d18229ee04251d8b04685 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 21 May 2020 15:18:27 +0800 Subject: [PATCH 13/14] me --- sdk/storage/storage-internal-avro/src/AvroReader.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts index 224312a46b59..d4f1197c296a 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReader.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -16,6 +16,8 @@ export class AvroReader { private _itemsRemainingInBlock?: number; + /// The byte offset within the Avro file (both header and data) + /// of the start of the current block. private _blockOffset: number; public get blockOffset(): number { return this._blockOffset; From 76ec430c3343c264f8ffe6be3fcd49936d95c9be Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 21 May 2020 15:20:06 +0800 Subject: [PATCH 14/14] me --- sdk/storage/storage-internal-avro/src/AvroReader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/storage-internal-avro/src/AvroReader.ts b/sdk/storage/storage-internal-avro/src/AvroReader.ts index d4f1197c296a..b8a2caef1124 100644 --- a/sdk/storage/storage-internal-avro/src/AvroReader.ts +++ b/sdk/storage/storage-internal-avro/src/AvroReader.ts @@ -115,7 +115,7 @@ export class AvroReader { this._objectIndex!++; if (this._itemsRemainingInBlock == 0) { - const marker = await AvroParser.readFixedBytes(this._dataStream, 16); + const marker = await AvroParser.readFixedBytes(this._dataStream, AVRO_SYNC_MARKER_SIZE); this._blockOffset = this._dataStream.position; this._objectIndex = 0;