From 58a959db857dfffa79ebf56b510251671475b314 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Thu, 24 Sep 2020 18:40:53 +0800 Subject: [PATCH] [storage][stg74] quick query new output format "arrow" (#11423) * quick query new outputSerialization "arrow" * for datalake, and remove preprod --- ...d_work_with_arrow_output_configurations.js | 123 +++++++++++ .../storage-blob/review/storage-blob.api.md | 19 +- sdk/storage/storage-blob/src/Clients.ts | 34 ++- sdk/storage/storage-blob/src/index.ts | 4 +- sdk/storage/storage-blob/src/models.ts | 56 ++++- .../storage-blob/src/utils/utils.common.ts | 23 ++- .../storage-blob/src/utils/utils.node.ts | 24 +++ .../storage-blob/test/node/blobclient.spec.ts | 24 +++ ...ld_work_with_arrow_output_configuration.js | 193 ++++++++++++++++++ .../review/storage-file-datalake.api.md | 9 +- .../storage-file-datalake/src/models.ts | 34 ++- .../test/node/pathclient.spec.ts | 22 ++ 12 files changed, 552 insertions(+), 13 deletions(-) create mode 100644 sdk/storage/storage-blob/recordings/node/blobclient_nodejs_only/recording_query_should_work_with_arrow_output_configurations.js create mode 100644 sdk/storage/storage-file-datalake/recordings/node/datalakepathclient_nodejs_only/recording_quick_query_should_work_with_arrow_output_configuration.js diff --git a/sdk/storage/storage-blob/recordings/node/blobclient_nodejs_only/recording_query_should_work_with_arrow_output_configurations.js b/sdk/storage/storage-blob/recordings/node/blobclient_nodejs_only/recording_query_should_work_with_arrow_output_configurations.js new file mode 100644 index 000000000000..c413bec53011 --- /dev/null +++ b/sdk/storage/storage-blob/recordings/node/blobclient_nodejs_only/recording_query_should_work_with_arrow_output_configurations.js @@ -0,0 +1,123 @@ +let nock = require('nock'); + +module.exports.hash = "e29af77e299520ce36fecae9203c86a0"; + +module.exports.testInfo = {"uniqueName":{"container":"container160084451581307559","blob":"blob160084451748308902"},"newDate":{}} + +nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true}) + .put('/container160084451581307559') + .query(true) + .reply(201, "", [ + 'Transfer-Encoding', + 'chunked', + 'Last-Modified', + 'Wed, 23 Sep 2020 07:01:57 GMT', + 'ETag', + '"0x8D85F8E8FAC3266"', + 'x-ms-request-id', + 'ff7752d8-501e-0005-2177-91eaf9000000', + 'x-ms-client-request-id', + 'c6105029-9e60-4acf-81e0-3d9eb4c8b8ba', + 'x-ms-version', + '2020-02-10', + 'Date', + 'Wed, 23 Sep 2020 07:01:57 GMT' +]); + +nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true}) + .put('/container160084451581307559/blob160084451748308902', "Hello World") + .reply(201, "", [ + 'Transfer-Encoding', + 'chunked', + 'Content-MD5', + 'sQqNsWTgdUEFt6mb5y4/5Q==', + 'Last-Modified', + 'Wed, 23 Sep 2020 07:01:57 GMT', + 'ETag', + '"0x8D85F8E8FD825FA"', + 'x-ms-request-id', + 'ff7752dc-501e-0005-2277-91eaf9000000', + 'x-ms-client-request-id', + 'ae929343-963f-4594-a532-341e47cc78a7', + 'x-ms-version', + '2020-02-10', + 'x-ms-content-crc64', + 'YeJLfssylmU=', + 'x-ms-request-server-encrypted', + 'false', + 'Date', + 'Wed, 23 Sep 2020 07:01:57 GMT' +]); + +nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true}) + .put('/container160084451581307559/blob160084451748308902', "100,200,300,400\n150,250,350,450\n") + .reply(201, "", [ + 'Transfer-Encoding', + 'chunked', + 'Content-MD5', + 'v9C7YWQTetukQaGSOQcgRQ==', + 'Last-Modified', + 'Wed, 23 Sep 2020 07:01:57 GMT', + 'ETag', + '"0x8D85F8E9003549C"', + 'x-ms-request-id', + 'ff7752dd-501e-0005-2377-91eaf9000000', + 'x-ms-client-request-id', + '1b596356-6e7b-465c-9ee8-c15725386c19', + 'x-ms-version', + '2020-02-10', + 'x-ms-content-crc64', + 'gema9E3+zEY=', + 'x-ms-request-server-encrypted', + 'false', + 'Date', + 'Wed, 23 Sep 2020 07:01:57 GMT' +]); + +nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true}) + .post('/container160084451581307559/blob160084451748308902', "SQLselect * from BlobStoragearrowdecimalname42") + .query(true) + .reply(200, Buffer.from("4f626a0102166176726f2e736368656d61be1e5b0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e726573756c7444617461222c0a2020202022646f63223a2022486f6c647320726573756c74206461746120696e2074686520666f726d61742073706563696669656420666f72207468697320717565727920284353562c204a534f4e2c206574632e292e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a202264617461222c0a20202020202020202274797065223a20226279746573220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e6572726f72222c0a2020202022646f63223a2022416e206572726f722074686174206f63637572726564207768696c652070726f63657373696e67207468652071756572792e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a2022666174616c222c0a20202020202020202274797065223a2022626f6f6c65616e222c0a202020202020202022646f63223a2022496620747275652c2074686973206572726f722070726576656e747320667572746865722071756572792070726f63657373696e672e20204d6f726520726573756c742064617461206d61792062652072657475726e65642c20627574207468657265206973206e6f2067756172616e746565207468617420616c6c206f6620746865206f726967696e616c20646174612077696c6c2062652070726f6365737365642e202049662066616c73652c2074686973206572726f7220646f6573206e6f742070726576656e7420667572746865722071756572792070726f63657373696e672e220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a20226e616d65222c0a20202020202020202274797065223a2022737472696e67222c0a202020202020202022646f63223a2022546865206e616d65206f6620746865206572726f72220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a20226465736372697074696f6e222c0a20202020202020202274797065223a2022737472696e67222c0a202020202020202022646f63223a202241206465736372697074696f6e206f6620746865206572726f72220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a2022706f736974696f6e222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520626c6f62206f666673657420617420776869636820746865206572726f72206f63637572726564220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e70726f6772657373222c0a2020202022646f63223a2022496e666f726d6174696f6e2061626f7574207468652070726f6772657373206f6620746865207175657279222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a202262797465735363616e6e6564222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a2022546865206e756d626572206f6620627974657320746861742068617665206265656e207363616e6e6564220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a2022746f74616c4279746573222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520746f74616c206e756d626572206f6620627974657320746f206265207363616e6e656420696e2074686973207175657279220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e656e64222c0a2020202022646f63223a202253656e74206173207468652066696e616c206d657373616765206f662074686520726573706f6e73652c20696e6469636174696e67207468617420616c6c20726573756c74732068617665206265656e2073656e742e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a2022746f74616c4279746573222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520746f74616c206e756d626572206f6620627974657320746f206265207363616e6e656420696e2074686973207175657279220a2020202020207d0a202020205d0a20207d0a5d0a00ede8894823947843a9db686b39a1f2ae028604008004ffffffff800000001000000000000a000c000600050008000a000000000103000c000000080008000000040008000000040000000100000014000000100014000800060007000c000000100010000000000001072400000014000000040000000000000008000c0004000800080000000400000002000000040000006e616d650000000000000000ffffffff700000001000000000000a000e000600050008000a000000000303001000000000000a000c000000040008000a0000003000000004000000020000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000ede8894823947843a9db686b39a1f2ae02040000ede8894823947843a9db686b39a1f2ae028e01020116436c69656e744572726f726e4e756d626572206f6620636f6c756d6e7320696e2074686520726f7720646f6573206e6f74206d617463682074686520736368656d612e00ede8894823947843a9db686b39a1f2ae0206044040ede8894823947843a9db686b39a1f2ae02040640ede8894823947843a9db686b39a1f2ae", "hex"), [ + 'Transfer-Encoding', + 'chunked', + 'Content-Type', + 'avro/binary', + 'Last-Modified', + 'Wed, 23 Sep 2020 07:01:57 GMT', + 'Accept-Ranges', + 'bytes', + 'ETag', + '"0x8D85F8E9003549C"', + 'x-ms-creation-time', + 'Wed, 23 Sep 2020 07:01:57 GMT', + 'x-ms-lease-state', + 'available', + 'x-ms-lease-status', + 'unlocked', + 'x-ms-blob-type', + 'BlockBlob', + 'x-ms-request-id', + 'ff7752de-501e-0005-2477-91eaf9000000', + 'x-ms-version', + '2020-02-10', + 'x-ms-client-request-id', + '3d3df40b-b46c-4834-9cc2-92acf74c1dd1', + 'Date', + 'Wed, 23 Sep 2020 07:01:57 GMT' +]); + +nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true}) + .delete('/container160084451581307559') + .query(true) + .reply(202, "", [ + 'Transfer-Encoding', + 'chunked', + 'x-ms-request-id', + 'ff7752df-501e-0005-2577-91eaf9000000', + 'x-ms-client-request-id', + 'dd172217-6cf1-4bb8-82c1-2f17bceecefc', + 'x-ms-version', + '2020-02-10', + 'Date', + 'Wed, 23 Sep 2020 07:01:58 GMT' +]); diff --git a/sdk/storage/storage-blob/review/storage-blob.api.md b/sdk/storage/storage-blob/review/storage-blob.api.md index 1736483c5198..2fc23eec9775 100644 --- a/sdk/storage/storage-blob/review/storage-blob.api.md +++ b/sdk/storage/storage-blob/review/storage-blob.api.md @@ -885,6 +885,23 @@ export interface BlobProperties { tagCount?: number; } +// @public +export interface BlobQueryArrowConfiguration { + kind: "arrow"; + schema: BlobQueryArrowField[]; +} + +// @public +export interface BlobQueryArrowField { + name?: string; + precision?: number; + scale?: number; + type: BlobQueryArrowFieldType; +} + +// @public +export type BlobQueryArrowFieldType = "int64" | "bool" | "timestamp[ms]" | "string" | "double" | "decimal"; + // @public export interface BlobQueryCsvTextConfiguration { columnSeparator?: string; @@ -1363,7 +1380,7 @@ export interface BlockBlobQueryOptions extends CommonOptions { inputTextConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration; onError?: (error: BlobQueryError) => void; onProgress?: (progress: TransferProgressEvent) => void; - outputTextConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration; + outputTextConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration | BlobQueryArrowConfiguration; } // @public diff --git a/sdk/storage/storage-blob/src/Clients.ts b/sdk/storage/storage-blob/src/Clients.ts index 691574528bde..38f9731f3246 100644 --- a/sdk/storage/storage-blob/src/Clients.ts +++ b/sdk/storage/storage-blob/src/Clients.ts @@ -102,7 +102,8 @@ import { TagConditions, MatchConditions, ModificationConditions, - ModifiedAccessConditions + ModifiedAccessConditions, + BlobQueryArrowField } from "./models"; import { PageBlobGetPageRangesDiffResponse, @@ -3307,6 +3308,30 @@ export interface BlobQueryCsvTextConfiguration { hasHeaders?: boolean; } +/** + * Options to query blob with Apache Arrow format. Only valid for {@link BlockBlobQueryOptions.outputTextConfiguration}. + * + * @export + * @interface BlobQueryArrowConfiguration + */ +export interface BlobQueryArrowConfiguration { + /** + * Kind. + * + * @type {"arrow"} + * @memberof BlobQueryArrowConfiguration + */ + kind: "arrow"; + + /** + * List of {@link BlobQueryArrowField} describing the schema of the data. + * + * @type {BlobQueryArrowField[]} + * @memberof BlobQueryArrowConfiguration + */ + schema: BlobQueryArrowField[]; +} + /** * Options to configure {@link BlockBlobClient.query} operation. * @@ -3332,10 +3357,13 @@ export interface BlockBlobQueryOptions extends CommonOptions { /** * Configurations for the query output. * - * @type {BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration} + * @type {BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration| BlobQueryArrowConfiguration} * @memberof BlockBlobQueryOptions */ - outputTextConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration; + outputTextConfiguration?: + | BlobQueryJsonTextConfiguration + | BlobQueryCsvTextConfiguration + | BlobQueryArrowConfiguration; /** * Callback to receive events on the progress of query operation. * diff --git a/sdk/storage/storage-blob/src/index.ts b/sdk/storage/storage-blob/src/index.ts index 2a8bb0beab25..109e19a3d10d 100644 --- a/sdk/storage/storage-blob/src/index.ts +++ b/sdk/storage/storage-blob/src/index.ts @@ -29,7 +29,9 @@ export { BlobDownloadResponseParsed, ObjectReplicationPolicy, ObjectReplicationRule, - ObjectReplicationStatus + ObjectReplicationStatus, + BlobQueryArrowField, + BlobQueryArrowFieldType } from "./models"; export * from "./Pipeline"; export * from "./policies/AnonymousCredentialPolicy"; diff --git a/sdk/storage/storage-blob/src/models.ts b/sdk/storage/storage-blob/src/models.ts index b2b7de3fd1c0..6c48ea86425a 100644 --- a/sdk/storage/storage-blob/src/models.ts +++ b/sdk/storage/storage-blob/src/models.ts @@ -223,6 +223,9 @@ export interface ObjectReplicationRule { * This is used when retrieving the Object Replication Properties on the source blob. The policy id for the * destination blob is set in ObjectReplicationDestinationPolicyId of the respective method responses * (e.g. {@link BlobProperties.ObjectReplicationDestinationPolicyId}. + * + * @export + * @interface ObjectReplicationPolicy */ export interface ObjectReplicationPolicy { /** @@ -236,7 +239,7 @@ export interface ObjectReplicationPolicy { /** * The Rule ID(s) and respective Replication Status(s) that are under the Policy ID. * - * @type {string} + * @type {ObjectReplicationRule[]} * @memberof ObjectReplicationPolicy */ rules: ObjectReplicationRule[]; @@ -265,3 +268,54 @@ export interface BlobDownloadResponseParsed extends BlobDownloadResponseModel { */ objectReplicationDestinationPolicyId?: string; } + +/** + * The type of a {@link BlobQueryArrowField}. + */ +export type BlobQueryArrowFieldType = + | "int64" + | "bool" + | "timestamp[ms]" + | "string" + | "double" + | "decimal"; + +/** + * Describe a field in {@link BlobQueryArrowConfiguration}. + * + * @export + * @interface BlobQueryArrowField + */ +export interface BlobQueryArrowField { + /** + * The type of the field. + * + * @type {BlobQueryArrowFieldType} + * @memberof BlobQueryArrowField + */ + type: BlobQueryArrowFieldType; + + /** + * The name of the field. + * + * @type {string} + * @memberof BlobQueryArrowField + */ + name?: string; + + /** + * The precision of the field. Required if type is "decimal". + * + * @type {number} + * @memberof BlobQueryArrowField + */ + precision?: number; + + /** + * The scale of the field. Required if type is is "decimal". + * + * @type {number} + * @memberof BlobQueryArrowField + */ + scale?: number; +} diff --git a/sdk/storage/storage-blob/src/utils/utils.common.ts b/sdk/storage/storage-blob/src/utils/utils.common.ts index 86fb0a15b251..395e84ccd055 100644 --- a/sdk/storage/storage-blob/src/utils/utils.common.ts +++ b/sdk/storage/storage-blob/src/utils/utils.common.ts @@ -4,7 +4,11 @@ import { AbortSignalLike } from "@azure/abort-controller"; import { HttpHeaders, isNode, URLBuilder } from "@azure/core-http"; -import { BlobQueryCsvTextConfiguration, BlobQueryJsonTextConfiguration } from "../Clients"; +import { + BlobQueryArrowConfiguration, + BlobQueryCsvTextConfiguration, + BlobQueryJsonTextConfiguration +} from "../Clients"; import { QuerySerialization, BlobTags } from "../generated/src/models"; import { DevelopmentConnectionString, HeaderConstants, URLConstants } from "./constants"; import { @@ -650,11 +654,14 @@ export function toTags(tags?: BlobTags): Tags | undefined { * Convert BlobQueryTextConfiguration to QuerySerialization type. * * @export - * @param {(BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration)} [textConfiguration] + * @param {(BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration | BlobQueryArrowConfiguration)} [textConfiguration] * @returns {(QuerySerialization | undefined)} */ export function toQuerySerialization( - textConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration + textConfiguration?: + | BlobQueryJsonTextConfiguration + | BlobQueryCsvTextConfiguration + | BlobQueryArrowConfiguration ): QuerySerialization | undefined { if (textConfiguration === undefined) { return undefined; @@ -683,6 +690,16 @@ export function toQuerySerialization( } } }; + case "arrow": + return { + format: { + type: "arrow", + arrowConfiguration: { + schema: textConfiguration.schema + } + } + }; + default: throw Error("Invalid BlobQueryTextConfiguration."); } diff --git a/sdk/storage/storage-blob/src/utils/utils.node.ts b/sdk/storage/storage-blob/src/utils/utils.node.ts index ed67cbade40d..ddd5897dc480 100644 --- a/sdk/storage/storage-blob/src/utils/utils.node.ts +++ b/sdk/storage/storage-blob/src/utils/utils.node.ts @@ -107,6 +107,30 @@ export async function streamToBuffer2( }); } +/** + * Reads a readable stream into a buffer. + * + * @export + * @param {NodeJS.ReadableStream} stream A Node.js Readable stream + * @param {string} [encoding] Encoding of the Readable stream + * @returns {Promise} with the count of bytes read. + */ +export async function streamToBuffer3( + readableStream: NodeJS.ReadableStream, + encoding?: string +): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + readableStream.on("data", (data: Buffer | string) => { + chunks.push(data instanceof Buffer ? data : Buffer.from(data, encoding)); + }); + readableStream.on("end", () => { + resolve(Buffer.concat(chunks)); + }); + readableStream.on("error", reject); + }); +} + /** * ONLY AVAILABLE IN NODE.JS RUNTIME. * diff --git a/sdk/storage/storage-blob/test/node/blobclient.spec.ts b/sdk/storage/storage-blob/test/node/blobclient.spec.ts index 0e0ffddd1e83..68f9bfc0b7d5 100644 --- a/sdk/storage/storage-blob/test/node/blobclient.spec.ts +++ b/sdk/storage/storage-blob/test/node/blobclient.spec.ts @@ -26,6 +26,7 @@ import { } from "../utils"; import { assertClientUsesTokenCredential } from "../utils/assert"; import { readStreamToLocalFileWithLogs } from "../utils/testutils.node"; +import { streamToBuffer3 } from "../../src/utils/utils.node"; dotenv.config(); @@ -643,4 +644,27 @@ describe("BlobClient Node.js only", () => { }); assert.deepStrictEqual(await bodyToString(response), jsonContent); }); + + it("query should work with arrow output configurations", async function() { + const csvContent = "100,200,300,400\n150,250,350,450\n"; + await blockBlobClient.upload(csvContent, csvContent.length); + + const response = await blockBlobClient.query("select * from BlobStorage", { + outputTextConfiguration: { + kind: "arrow", + schema: [ + { + type: "decimal", + name: "name", + precision: 4, + scale: 2 + } + ] + } + }); + assert.equal( + (await streamToBuffer3(response.readableStreamBody!)).toString("hex"), + "ffffffff800000001000000000000a000c000600050008000a000000000103000c000000080008000000040008000000040000000100000014000000100014000800060007000c000000100010000000000001072400000014000000040000000000000008000c0004000800080000000400000002000000040000006e616d650000000000000000ffffffff700000001000000000000a000e000600050008000a000000000303001000000000000a000c000000040008000a0000003000000004000000020000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000" + ); + }); }); diff --git a/sdk/storage/storage-file-datalake/recordings/node/datalakepathclient_nodejs_only/recording_quick_query_should_work_with_arrow_output_configuration.js b/sdk/storage/storage-file-datalake/recordings/node/datalakepathclient_nodejs_only/recording_quick_query_should_work_with_arrow_output_configuration.js new file mode 100644 index 000000000000..46972661dd2e --- /dev/null +++ b/sdk/storage/storage-file-datalake/recordings/node/datalakepathclient_nodejs_only/recording_quick_query_should_work_with_arrow_output_configuration.js @@ -0,0 +1,193 @@ +let nock = require('nock'); + +module.exports.hash = "5d698c75689d843c8dc3544d7ca6d674"; + +module.exports.testInfo = {"uniqueName":{"filesystem":"filesystem160084667955801065","file":"file160084668061309353"},"newDate":{}} + +nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true}) + .put('/filesystem160084667955801065') + .query(true) + .reply(201, "", [ + 'Transfer-Encoding', + 'chunked', + 'Last-Modified', + 'Wed, 23 Sep 2020 07:38:00 GMT', + 'ETag', + '"0x8D85F93990039E1"', + 'x-ms-request-id', + 'ff775b84-501e-0005-277c-91eaf9000000', + 'x-ms-client-request-id', + 'bcc444f1-1b4b-423b-b675-ffaed21ae2e1', + 'x-ms-version', + '2020-02-10', + 'Date', + 'Wed, 23 Sep 2020 07:38:00 GMT' +]); + +nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true}) + .put('/filesystem160084667955801065/file160084668061309353') + .query(true) + .reply(201, "", [ + 'Last-Modified', + 'Wed, 23 Sep 2020 07:38:01 GMT', + 'ETag', + '"0x8D85F939996BD58"', + 'x-ms-request-id', + '51b9d5bf-801f-0008-457c-91d877000000', + 'x-ms-version', + '2020-02-10', + 'x-ms-client-request-id', + '52fb17bc-9218-4700-b9a0-51280c043c71', + 'Date', + 'Wed, 23 Sep 2020 07:38:01 GMT', + 'Content-Length', + '0' +]); + +nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true}) + .patch('/filesystem160084667955801065/file160084668061309353', "Hello World") + .query(true) + .reply(202, "", [ + 'x-ms-request-server-encrypted', + 'false', + 'x-ms-request-id', + '51b9d5c0-801f-0008-467c-91d877000000', + 'x-ms-version', + '2020-02-10', + 'x-ms-client-request-id', + 'd7ce3602-64ba-419e-ae00-f0db9a5c9dd7', + 'Date', + 'Wed, 23 Sep 2020 07:38:01 GMT', + 'Content-Length', + '0' +]); + +nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true}) + .patch('/filesystem160084667955801065/file160084668061309353') + .query(true) + .reply(200, "", [ + 'Last-Modified', + 'Wed, 23 Sep 2020 07:38:01 GMT', + 'ETag', + '"0x8D85F9399E32F9A"', + 'x-ms-request-server-encrypted', + 'false', + 'x-ms-request-id', + '51b9d5c1-801f-0008-477c-91d877000000', + 'x-ms-version', + '2020-02-10', + 'x-ms-client-request-id', + '8d500751-7371-44ac-a85a-0b047fcea14f', + 'Date', + 'Wed, 23 Sep 2020 07:38:01 GMT', + 'Content-Length', + '0' +]); + +nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true}) + .put('/filesystem160084667955801065/file1600846680613093532') + .query(true) + .reply(201, "", [ + 'Last-Modified', + 'Wed, 23 Sep 2020 07:38:02 GMT', + 'ETag', + '"0x8D85F939A0843F0"', + 'x-ms-request-id', + '51b9d5c2-801f-0008-487c-91d877000000', + 'x-ms-version', + '2020-02-10', + 'x-ms-client-request-id', + 'a7335df3-c7c7-4620-90f1-64d4b905f209', + 'Date', + 'Wed, 23 Sep 2020 07:38:01 GMT', + 'Content-Length', + '0' +]); + +nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true}) + .patch('/filesystem160084667955801065/file1600846680613093532', "100,200,300,400\n150,250,350,450\n") + .query(true) + .reply(202, "", [ + 'x-ms-request-server-encrypted', + 'false', + 'x-ms-request-id', + '51b9d5c3-801f-0008-497c-91d877000000', + 'x-ms-version', + '2020-02-10', + 'x-ms-client-request-id', + '4c2ca286-fc1b-4038-ade1-a9bd8555ac48', + 'Date', + 'Wed, 23 Sep 2020 07:38:01 GMT', + 'Content-Length', + '0' +]); + +nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true}) + .patch('/filesystem160084667955801065/file1600846680613093532') + .query(true) + .reply(200, "", [ + 'Last-Modified', + 'Wed, 23 Sep 2020 07:38:02 GMT', + 'ETag', + '"0x8D85F939A548F3E"', + 'x-ms-request-server-encrypted', + 'false', + 'x-ms-request-id', + '51b9d5c4-801f-0008-4a7c-91d877000000', + 'x-ms-version', + '2020-02-10', + 'x-ms-client-request-id', + 'c6fca687-c9a8-4dad-98d4-662808b8fa99', + 'Date', + 'Wed, 23 Sep 2020 07:38:02 GMT', + 'Content-Length', + '0' +]); + +nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true}) + .post('/filesystem160084667955801065/file1600846680613093532', "SQLselect * from BlobStoragearrowdecimalname42") + .query(true) + .reply(200, Buffer.from("4f626a0102166176726f2e736368656d61be1e5b0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e726573756c7444617461222c0a2020202022646f63223a2022486f6c647320726573756c74206461746120696e2074686520666f726d61742073706563696669656420666f72207468697320717565727920284353562c204a534f4e2c206574632e292e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a202264617461222c0a20202020202020202274797065223a20226279746573220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e6572726f72222c0a2020202022646f63223a2022416e206572726f722074686174206f63637572726564207768696c652070726f63657373696e67207468652071756572792e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a2022666174616c222c0a20202020202020202274797065223a2022626f6f6c65616e222c0a202020202020202022646f63223a2022496620747275652c2074686973206572726f722070726576656e747320667572746865722071756572792070726f63657373696e672e20204d6f726520726573756c742064617461206d61792062652072657475726e65642c20627574207468657265206973206e6f2067756172616e746565207468617420616c6c206f6620746865206f726967696e616c20646174612077696c6c2062652070726f6365737365642e202049662066616c73652c2074686973206572726f7220646f6573206e6f742070726576656e7420667572746865722071756572792070726f63657373696e672e220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a20226e616d65222c0a20202020202020202274797065223a2022737472696e67222c0a202020202020202022646f63223a2022546865206e616d65206f6620746865206572726f72220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a20226465736372697074696f6e222c0a20202020202020202274797065223a2022737472696e67222c0a202020202020202022646f63223a202241206465736372697074696f6e206f6620746865206572726f72220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a2022706f736974696f6e222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520626c6f62206f666673657420617420776869636820746865206572726f72206f63637572726564220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e70726f6772657373222c0a2020202022646f63223a2022496e666f726d6174696f6e2061626f7574207468652070726f6772657373206f6620746865207175657279222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a202262797465735363616e6e6564222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a2022546865206e756d626572206f6620627974657320746861742068617665206265656e207363616e6e6564220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a2022746f74616c4279746573222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520746f74616c206e756d626572206f6620627974657320746f206265207363616e6e656420696e2074686973207175657279220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e656e64222c0a2020202022646f63223a202253656e74206173207468652066696e616c206d657373616765206f662074686520726573706f6e73652c20696e6469636174696e67207468617420616c6c20726573756c74732068617665206265656e2073656e742e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a2022746f74616c4279746573222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520746f74616c206e756d626572206f6620627974657320746f206265207363616e6e656420696e2074686973207175657279220a2020202020207d0a202020205d0a20207d0a5d0a00796d4c0f3e9bcb4f8fa2ff91e58363b0028604008004ffffffff800000001000000000000a000c000600050008000a000000000103000c000000080008000000040008000000040000000100000014000000100014000800060007000c000000100010000000000001072400000014000000040000000000000008000c0004000800080000000400000002000000040000006e616d650000000000000000ffffffff700000001000000000000a000e000600050008000a000000000303001000000000000a000c000000040008000a0000003000000004000000020000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000796d4c0f3e9bcb4f8fa2ff91e58363b002040000796d4c0f3e9bcb4f8fa2ff91e58363b0028e01020116436c69656e744572726f726e4e756d626572206f6620636f6c756d6e7320696e2074686520726f7720646f6573206e6f74206d617463682074686520736368656d612e00796d4c0f3e9bcb4f8fa2ff91e58363b00206044040796d4c0f3e9bcb4f8fa2ff91e58363b002040640796d4c0f3e9bcb4f8fa2ff91e58363b0", "hex"), [ + 'Transfer-Encoding', + 'chunked', + 'Content-Type', + 'avro/binary', + 'Last-Modified', + 'Wed, 23 Sep 2020 07:38:02 GMT', + 'Accept-Ranges', + 'bytes', + 'ETag', + '"0x8D85F939A548F3E"', + 'x-ms-creation-time', + 'Wed, 23 Sep 2020 07:38:02 GMT', + 'x-ms-lease-state', + 'available', + 'x-ms-lease-status', + 'unlocked', + 'x-ms-blob-type', + 'BlockBlob', + 'x-ms-request-id', + 'ff775b88-501e-0005-287c-91eaf9000000', + 'x-ms-version', + '2020-02-10', + 'x-ms-client-request-id', + '27daec12-f082-427c-bfcb-8c54f3ddafea', + 'Date', + 'Wed, 23 Sep 2020 07:38:02 GMT' +]); + +nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true}) + .delete('/filesystem160084667955801065') + .query(true) + .reply(202, "", [ + 'Transfer-Encoding', + 'chunked', + 'x-ms-request-id', + '0b36984e-d01e-0006-0c7c-911999000000', + 'x-ms-client-request-id', + '878568e4-a9ce-4286-b7fd-9849bff17e6f', + 'x-ms-version', + '2020-02-10', + 'Date', + 'Wed, 23 Sep 2020 07:38:03 GMT' +]); diff --git a/sdk/storage/storage-file-datalake/review/storage-file-datalake.api.md b/sdk/storage/storage-file-datalake/review/storage-file-datalake.api.md index 402bef103bd0..997898bf1077 100644 --- a/sdk/storage/storage-file-datalake/review/storage-file-datalake.api.md +++ b/sdk/storage/storage-file-datalake/review/storage-file-datalake.api.md @@ -7,6 +7,7 @@ import { AbortSignalLike } from '@azure/abort-controller'; import { BaseRequestPolicy } from '@azure/core-http'; import { BlobLeaseClient } from '@azure/storage-blob'; +import { BlobQueryArrowField } from '@azure/storage-blob'; import * as coreHttp from '@azure/core-http'; import { deserializationPolicy } from '@azure/core-http'; import { HttpHeaders } from '@azure/core-http'; @@ -374,6 +375,12 @@ export interface FileParallelUploadOptions extends CommonOptions { umask?: string; } +// @public +export interface FileQueryArrowConfiguration { + kind: "arrow"; + schema: BlobQueryArrowField[]; +} + // @public export interface FileQueryCsvTextConfiguration { columnSeparator?: string; @@ -405,7 +412,7 @@ export interface FileQueryOptions extends CommonOptions { inputTextConfiguration?: FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration; onError?: (error: FileQueryError) => void; onProgress?: (progress: TransferProgressEvent) => void; - outputTextConfiguration?: FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration; + outputTextConfiguration?: FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration | FileQueryArrowConfiguration; } // @public (undocumented) diff --git a/sdk/storage/storage-file-datalake/src/models.ts b/sdk/storage/storage-file-datalake/src/models.ts index 2edde72320d0..d47b2799ba69 100644 --- a/sdk/storage/storage-file-datalake/src/models.ts +++ b/sdk/storage/storage-file-datalake/src/models.ts @@ -7,7 +7,8 @@ import { HttpResponse, TransferProgressEvent } from "@azure/core-http"; import { LeaseAccessConditions, ModifiedAccessConditions as ModifiedAccessConditionsModel, - UserDelegationKeyModel + UserDelegationKeyModel, + BlobQueryArrowField } from "@azure/storage-blob"; export type ModifiedAccessConditions = Omit; @@ -1240,6 +1241,30 @@ export interface FileQueryCsvTextConfiguration { hasHeaders?: boolean; } +/** + * Options to query file with Apache Arrow format. Only valid for {@link FileQueryOptions.outputTextConfiguration}. + * + * @export + * @interface FileQueryArrowConfiguration + */ +export interface FileQueryArrowConfiguration { + /** + * Kind. + * + * @type {"arrow"} + * @memberof FileQueryArrowConfiguration + */ + kind: "arrow"; + + /** + * List of {@link BlobQueryArrowField} describing the schema of the data. + * + * @type {BlobQueryArrowField[]} + * @memberof FileQueryArrowConfiguration + */ + schema: BlobQueryArrowField[]; +} + /** * File query error type. * @@ -1305,10 +1330,13 @@ export interface FileQueryOptions extends CommonOptions { /** * Configurations for the query output. * - * @type {FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration} + * @type {FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration | FileQueryArrowConfiguration} * @memberof FileQueryOptions */ - outputTextConfiguration?: FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration; + outputTextConfiguration?: + | FileQueryJsonTextConfiguration + | FileQueryCsvTextConfiguration + | FileQueryArrowConfiguration; /** * Callback to receive events on the progress of query operation. * diff --git a/sdk/storage/storage-file-datalake/test/node/pathclient.spec.ts b/sdk/storage/storage-file-datalake/test/node/pathclient.spec.ts index 7ca6fc22de69..94160870ec40 100644 --- a/sdk/storage/storage-file-datalake/test/node/pathclient.spec.ts +++ b/sdk/storage/storage-file-datalake/test/node/pathclient.spec.ts @@ -334,6 +334,28 @@ describe("DataLakePathClient Node.js only", () => { const response = await fileClient2.query("select * from BlobStorage"); assert.deepStrictEqual(await bodyToString(response), csvContent); }); + + it("quick query should work with arrow output configuration", async () => { + const csvContent = "100,200,300,400\n150,250,350,450\n"; + const fileClient2 = fileSystemClient.getFileClient(fileName + "2"); + await fileClient2.create(); + await fileClient2.append(csvContent, 0, csvContent.length); + await fileClient2.flush(csvContent.length); + + await fileClient2.query("select * from BlobStorage", { + outputTextConfiguration: { + kind: "arrow", + schema: [ + { + type: "decimal", + name: "name", + precision: 4, + scale: 2 + } + ] + } + }); + }); }); describe("DataLakePathClient setAccessControlRecursive Node.js only", () => {