diff --git a/sdk/storage/storage-blob/recordings/node/blobclient_nodejs_only/recording_query_should_work_with_arrow_output_configurations.js b/sdk/storage/storage-blob/recordings/node/blobclient_nodejs_only/recording_query_should_work_with_arrow_output_configurations.js
new file mode 100644
index 000000000000..c413bec53011
--- /dev/null
+++ b/sdk/storage/storage-blob/recordings/node/blobclient_nodejs_only/recording_query_should_work_with_arrow_output_configurations.js
@@ -0,0 +1,123 @@
+let nock = require('nock');
+
+module.exports.hash = "e29af77e299520ce36fecae9203c86a0";
+
+module.exports.testInfo = {"uniqueName":{"container":"container160084451581307559","blob":"blob160084451748308902"},"newDate":{}}
+
+nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true})
+ .put('/container160084451581307559')
+ .query(true)
+ .reply(201, "", [
+ 'Transfer-Encoding',
+ 'chunked',
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:01:57 GMT',
+ 'ETag',
+ '"0x8D85F8E8FAC3266"',
+ 'x-ms-request-id',
+ 'ff7752d8-501e-0005-2177-91eaf9000000',
+ 'x-ms-client-request-id',
+ 'c6105029-9e60-4acf-81e0-3d9eb4c8b8ba',
+ 'x-ms-version',
+ '2020-02-10',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:01:57 GMT'
+]);
+
+nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true})
+ .put('/container160084451581307559/blob160084451748308902', "Hello World")
+ .reply(201, "", [
+ 'Transfer-Encoding',
+ 'chunked',
+ 'Content-MD5',
+ 'sQqNsWTgdUEFt6mb5y4/5Q==',
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:01:57 GMT',
+ 'ETag',
+ '"0x8D85F8E8FD825FA"',
+ 'x-ms-request-id',
+ 'ff7752dc-501e-0005-2277-91eaf9000000',
+ 'x-ms-client-request-id',
+ 'ae929343-963f-4594-a532-341e47cc78a7',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-content-crc64',
+ 'YeJLfssylmU=',
+ 'x-ms-request-server-encrypted',
+ 'false',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:01:57 GMT'
+]);
+
+nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true})
+ .put('/container160084451581307559/blob160084451748308902', "100,200,300,400\n150,250,350,450\n")
+ .reply(201, "", [
+ 'Transfer-Encoding',
+ 'chunked',
+ 'Content-MD5',
+ 'v9C7YWQTetukQaGSOQcgRQ==',
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:01:57 GMT',
+ 'ETag',
+ '"0x8D85F8E9003549C"',
+ 'x-ms-request-id',
+ 'ff7752dd-501e-0005-2377-91eaf9000000',
+ 'x-ms-client-request-id',
+ '1b596356-6e7b-465c-9ee8-c15725386c19',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-content-crc64',
+ 'gema9E3+zEY=',
+ 'x-ms-request-server-encrypted',
+ 'false',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:01:57 GMT'
+]);
+
+nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true})
+ .post('/container160084451581307559/blob160084451748308902', "SQLselect * from BlobStoragearrowdecimalname42")
+ .query(true)
+ .reply(200, Buffer.from("4f626a0102166176726f2e736368656d61be1e5b0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e726573756c7444617461222c0a2020202022646f63223a2022486f6c647320726573756c74206461746120696e2074686520666f726d61742073706563696669656420666f72207468697320717565727920284353562c204a534f4e2c206574632e292e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a202264617461222c0a20202020202020202274797065223a20226279746573220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e6572726f72222c0a2020202022646f63223a2022416e206572726f722074686174206f63637572726564207768696c652070726f63657373696e67207468652071756572792e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a2022666174616c222c0a20202020202020202274797065223a2022626f6f6c65616e222c0a202020202020202022646f63223a2022496620747275652c2074686973206572726f722070726576656e747320667572746865722071756572792070726f63657373696e672e20204d6f726520726573756c742064617461206d61792062652072657475726e65642c20627574207468657265206973206e6f2067756172616e746565207468617420616c6c206f6620746865206f726967696e616c20646174612077696c6c2062652070726f6365737365642e202049662066616c73652c2074686973206572726f7220646f6573206e6f742070726576656e7420667572746865722071756572792070726f63657373696e672e220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a20226e616d65222c0a20202020202020202274797065223a2022737472696e67222c0a202020202020202022646f63223a2022546865206e616d65206f6620746865206572726f72220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a20226465736372697074696f6e222c0a20202020202020202274797065223a2022737472696e67222c0a202020202020202022646f63223a202241206465736372697074696f6e206f6620746865206572726f72220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a2022706f736974696f6e222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520626c6f62206f666673657420617420776869636820746865206572726f72206f63637572726564220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e70726f6772657373222c0a2020202022646f63223a2022496e666f726d6174696f6e2061626f7574207468652070726f6772657373206f6620746865207175657279222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a202262797465735363616e6e6564222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a2022546865206e756d626572206f6620627974657320746861742068617665206265656e207363616e6e6564220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a2022746f74616c4279746573222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520746f74616c206e756d626572206f6620627974657320746f206265207363616e6e656420696e2074686973207175657279220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e656e64222c0a2020202022646f63223a202253656e74206173207468652066696e616c206d657373616765206f662074686520726573706f6e73652c20696e6469636174696e67207468617420616c6c20726573756c74732068617665206265656e2073656e742e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a2022746f74616c4279746573222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520746f74616c206e756d626572206f6620627974657320746f206265207363616e6e656420696e2074686973207175657279220a2020202020207d0a202020205d0a20207d0a5d0a00ede8894823947843a9db686b39a1f2ae028604008004ffffffff800000001000000000000a000c000600050008000a000000000103000c000000080008000000040008000000040000000100000014000000100014000800060007000c000000100010000000000001072400000014000000040000000000000008000c0004000800080000000400000002000000040000006e616d650000000000000000ffffffff700000001000000000000a000e000600050008000a000000000303001000000000000a000c000000040008000a0000003000000004000000020000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000ede8894823947843a9db686b39a1f2ae02040000ede8894823947843a9db686b39a1f2ae028e01020116436c69656e744572726f726e4e756d626572206f6620636f6c756d6e7320696e2074686520726f7720646f6573206e6f74206d617463682074686520736368656d612e00ede8894823947843a9db686b39a1f2ae0206044040ede8894823947843a9db686b39a1f2ae02040640ede8894823947843a9db686b39a1f2ae", "hex"), [
+ 'Transfer-Encoding',
+ 'chunked',
+ 'Content-Type',
+ 'avro/binary',
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:01:57 GMT',
+ 'Accept-Ranges',
+ 'bytes',
+ 'ETag',
+ '"0x8D85F8E9003549C"',
+ 'x-ms-creation-time',
+ 'Wed, 23 Sep 2020 07:01:57 GMT',
+ 'x-ms-lease-state',
+ 'available',
+ 'x-ms-lease-status',
+ 'unlocked',
+ 'x-ms-blob-type',
+ 'BlockBlob',
+ 'x-ms-request-id',
+ 'ff7752de-501e-0005-2477-91eaf9000000',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-client-request-id',
+ '3d3df40b-b46c-4834-9cc2-92acf74c1dd1',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:01:57 GMT'
+]);
+
+nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true})
+ .delete('/container160084451581307559')
+ .query(true)
+ .reply(202, "", [
+ 'Transfer-Encoding',
+ 'chunked',
+ 'x-ms-request-id',
+ 'ff7752df-501e-0005-2577-91eaf9000000',
+ 'x-ms-client-request-id',
+ 'dd172217-6cf1-4bb8-82c1-2f17bceecefc',
+ 'x-ms-version',
+ '2020-02-10',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:01:58 GMT'
+]);
diff --git a/sdk/storage/storage-blob/review/storage-blob.api.md b/sdk/storage/storage-blob/review/storage-blob.api.md
index 1736483c5198..2fc23eec9775 100644
--- a/sdk/storage/storage-blob/review/storage-blob.api.md
+++ b/sdk/storage/storage-blob/review/storage-blob.api.md
@@ -885,6 +885,23 @@ export interface BlobProperties {
tagCount?: number;
}
+// @public
+export interface BlobQueryArrowConfiguration {
+ kind: "arrow";
+ schema: BlobQueryArrowField[];
+}
+
+// @public
+export interface BlobQueryArrowField {
+ name?: string;
+ precision?: number;
+ scale?: number;
+ type: BlobQueryArrowFieldType;
+}
+
+// @public
+export type BlobQueryArrowFieldType = "int64" | "bool" | "timestamp[ms]" | "string" | "double" | "decimal";
+
// @public
export interface BlobQueryCsvTextConfiguration {
columnSeparator?: string;
@@ -1363,7 +1380,7 @@ export interface BlockBlobQueryOptions extends CommonOptions {
inputTextConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration;
onError?: (error: BlobQueryError) => void;
onProgress?: (progress: TransferProgressEvent) => void;
- outputTextConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration;
+ outputTextConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration | BlobQueryArrowConfiguration;
}
// @public
diff --git a/sdk/storage/storage-blob/src/Clients.ts b/sdk/storage/storage-blob/src/Clients.ts
index 691574528bde..38f9731f3246 100644
--- a/sdk/storage/storage-blob/src/Clients.ts
+++ b/sdk/storage/storage-blob/src/Clients.ts
@@ -102,7 +102,8 @@ import {
TagConditions,
MatchConditions,
ModificationConditions,
- ModifiedAccessConditions
+ ModifiedAccessConditions,
+ BlobQueryArrowField
} from "./models";
import {
PageBlobGetPageRangesDiffResponse,
@@ -3307,6 +3308,30 @@ export interface BlobQueryCsvTextConfiguration {
hasHeaders?: boolean;
}
+/**
+ * Options to query blob with Apache Arrow format. Only valid for {@link BlockBlobQueryOptions.outputTextConfiguration}.
+ *
+ * @export
+ * @interface BlobQueryArrowConfiguration
+ */
+export interface BlobQueryArrowConfiguration {
+ /**
+ * Kind.
+ *
+ * @type {"arrow"}
+ * @memberof BlobQueryArrowConfiguration
+ */
+ kind: "arrow";
+
+ /**
+ * List of {@link BlobQueryArrowField} describing the schema of the data.
+ *
+ * @type {BlobQueryArrowField[]}
+ * @memberof BlobQueryArrowConfiguration
+ */
+ schema: BlobQueryArrowField[];
+}
+
/**
* Options to configure {@link BlockBlobClient.query} operation.
*
@@ -3332,10 +3357,13 @@ export interface BlockBlobQueryOptions extends CommonOptions {
/**
* Configurations for the query output.
*
- * @type {BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration}
+ * @type {BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration| BlobQueryArrowConfiguration}
* @memberof BlockBlobQueryOptions
*/
- outputTextConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration;
+ outputTextConfiguration?:
+ | BlobQueryJsonTextConfiguration
+ | BlobQueryCsvTextConfiguration
+ | BlobQueryArrowConfiguration;
/**
* Callback to receive events on the progress of query operation.
*
diff --git a/sdk/storage/storage-blob/src/index.ts b/sdk/storage/storage-blob/src/index.ts
index 2a8bb0beab25..109e19a3d10d 100644
--- a/sdk/storage/storage-blob/src/index.ts
+++ b/sdk/storage/storage-blob/src/index.ts
@@ -29,7 +29,9 @@ export {
BlobDownloadResponseParsed,
ObjectReplicationPolicy,
ObjectReplicationRule,
- ObjectReplicationStatus
+ ObjectReplicationStatus,
+ BlobQueryArrowField,
+ BlobQueryArrowFieldType
} from "./models";
export * from "./Pipeline";
export * from "./policies/AnonymousCredentialPolicy";
diff --git a/sdk/storage/storage-blob/src/models.ts b/sdk/storage/storage-blob/src/models.ts
index b2b7de3fd1c0..6c48ea86425a 100644
--- a/sdk/storage/storage-blob/src/models.ts
+++ b/sdk/storage/storage-blob/src/models.ts
@@ -223,6 +223,9 @@ export interface ObjectReplicationRule {
* This is used when retrieving the Object Replication Properties on the source blob. The policy id for the
* destination blob is set in ObjectReplicationDestinationPolicyId of the respective method responses
* (e.g. {@link BlobProperties.ObjectReplicationDestinationPolicyId}.
+ *
+ * @export
+ * @interface ObjectReplicationPolicy
*/
export interface ObjectReplicationPolicy {
/**
@@ -236,7 +239,7 @@ export interface ObjectReplicationPolicy {
/**
* The Rule ID(s) and respective Replication Status(s) that are under the Policy ID.
*
- * @type {string}
+ * @type {ObjectReplicationRule[]}
* @memberof ObjectReplicationPolicy
*/
rules: ObjectReplicationRule[];
@@ -265,3 +268,54 @@ export interface BlobDownloadResponseParsed extends BlobDownloadResponseModel {
*/
objectReplicationDestinationPolicyId?: string;
}
+
+/**
+ * The type of a {@link BlobQueryArrowField}.
+ */
+export type BlobQueryArrowFieldType =
+ | "int64"
+ | "bool"
+ | "timestamp[ms]"
+ | "string"
+ | "double"
+ | "decimal";
+
+/**
+ * Describe a field in {@link BlobQueryArrowConfiguration}.
+ *
+ * @export
+ * @interface BlobQueryArrowField
+ */
+export interface BlobQueryArrowField {
+ /**
+ * The type of the field.
+ *
+ * @type {BlobQueryArrowFieldType}
+ * @memberof BlobQueryArrowField
+ */
+ type: BlobQueryArrowFieldType;
+
+ /**
+ * The name of the field.
+ *
+ * @type {string}
+ * @memberof BlobQueryArrowField
+ */
+ name?: string;
+
+ /**
+ * The precision of the field. Required if type is "decimal".
+ *
+ * @type {number}
+ * @memberof BlobQueryArrowField
+ */
+ precision?: number;
+
+ /**
+ * The scale of the field. Required if type is is "decimal".
+ *
+ * @type {number}
+ * @memberof BlobQueryArrowField
+ */
+ scale?: number;
+}
diff --git a/sdk/storage/storage-blob/src/utils/utils.common.ts b/sdk/storage/storage-blob/src/utils/utils.common.ts
index 86fb0a15b251..395e84ccd055 100644
--- a/sdk/storage/storage-blob/src/utils/utils.common.ts
+++ b/sdk/storage/storage-blob/src/utils/utils.common.ts
@@ -4,7 +4,11 @@
import { AbortSignalLike } from "@azure/abort-controller";
import { HttpHeaders, isNode, URLBuilder } from "@azure/core-http";
-import { BlobQueryCsvTextConfiguration, BlobQueryJsonTextConfiguration } from "../Clients";
+import {
+ BlobQueryArrowConfiguration,
+ BlobQueryCsvTextConfiguration,
+ BlobQueryJsonTextConfiguration
+} from "../Clients";
import { QuerySerialization, BlobTags } from "../generated/src/models";
import { DevelopmentConnectionString, HeaderConstants, URLConstants } from "./constants";
import {
@@ -650,11 +654,14 @@ export function toTags(tags?: BlobTags): Tags | undefined {
* Convert BlobQueryTextConfiguration to QuerySerialization type.
*
* @export
- * @param {(BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration)} [textConfiguration]
+ * @param {(BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration | BlobQueryArrowConfiguration)} [textConfiguration]
* @returns {(QuerySerialization | undefined)}
*/
export function toQuerySerialization(
- textConfiguration?: BlobQueryJsonTextConfiguration | BlobQueryCsvTextConfiguration
+ textConfiguration?:
+ | BlobQueryJsonTextConfiguration
+ | BlobQueryCsvTextConfiguration
+ | BlobQueryArrowConfiguration
): QuerySerialization | undefined {
if (textConfiguration === undefined) {
return undefined;
@@ -683,6 +690,16 @@ export function toQuerySerialization(
}
}
};
+ case "arrow":
+ return {
+ format: {
+ type: "arrow",
+ arrowConfiguration: {
+ schema: textConfiguration.schema
+ }
+ }
+ };
+
default:
throw Error("Invalid BlobQueryTextConfiguration.");
}
diff --git a/sdk/storage/storage-blob/src/utils/utils.node.ts b/sdk/storage/storage-blob/src/utils/utils.node.ts
index ed67cbade40d..ddd5897dc480 100644
--- a/sdk/storage/storage-blob/src/utils/utils.node.ts
+++ b/sdk/storage/storage-blob/src/utils/utils.node.ts
@@ -107,6 +107,30 @@ export async function streamToBuffer2(
});
}
+/**
+ * Reads a readable stream into a buffer.
+ *
+ * @export
+ * @param {NodeJS.ReadableStream} stream A Node.js Readable stream
+ * @param {string} [encoding] Encoding of the Readable stream
+ * @returns {Promise} with the count of bytes read.
+ */
+export async function streamToBuffer3(
+ readableStream: NodeJS.ReadableStream,
+ encoding?: string
+): Promise {
+ return new Promise((resolve, reject) => {
+ const chunks: Buffer[] = [];
+ readableStream.on("data", (data: Buffer | string) => {
+ chunks.push(data instanceof Buffer ? data : Buffer.from(data, encoding));
+ });
+ readableStream.on("end", () => {
+ resolve(Buffer.concat(chunks));
+ });
+ readableStream.on("error", reject);
+ });
+}
+
/**
* ONLY AVAILABLE IN NODE.JS RUNTIME.
*
diff --git a/sdk/storage/storage-blob/test/node/blobclient.spec.ts b/sdk/storage/storage-blob/test/node/blobclient.spec.ts
index 0e0ffddd1e83..68f9bfc0b7d5 100644
--- a/sdk/storage/storage-blob/test/node/blobclient.spec.ts
+++ b/sdk/storage/storage-blob/test/node/blobclient.spec.ts
@@ -26,6 +26,7 @@ import {
} from "../utils";
import { assertClientUsesTokenCredential } from "../utils/assert";
import { readStreamToLocalFileWithLogs } from "../utils/testutils.node";
+import { streamToBuffer3 } from "../../src/utils/utils.node";
dotenv.config();
@@ -643,4 +644,27 @@ describe("BlobClient Node.js only", () => {
});
assert.deepStrictEqual(await bodyToString(response), jsonContent);
});
+
+ it("query should work with arrow output configurations", async function() {
+ const csvContent = "100,200,300,400\n150,250,350,450\n";
+ await blockBlobClient.upload(csvContent, csvContent.length);
+
+ const response = await blockBlobClient.query("select * from BlobStorage", {
+ outputTextConfiguration: {
+ kind: "arrow",
+ schema: [
+ {
+ type: "decimal",
+ name: "name",
+ precision: 4,
+ scale: 2
+ }
+ ]
+ }
+ });
+ assert.equal(
+ (await streamToBuffer3(response.readableStreamBody!)).toString("hex"),
+ "ffffffff800000001000000000000a000c000600050008000a000000000103000c000000080008000000040008000000040000000100000014000000100014000800060007000c000000100010000000000001072400000014000000040000000000000008000c0004000800080000000400000002000000040000006e616d650000000000000000ffffffff700000001000000000000a000e000600050008000a000000000303001000000000000a000c000000040008000a0000003000000004000000020000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000"
+ );
+ });
});
diff --git a/sdk/storage/storage-file-datalake/recordings/node/datalakepathclient_nodejs_only/recording_quick_query_should_work_with_arrow_output_configuration.js b/sdk/storage/storage-file-datalake/recordings/node/datalakepathclient_nodejs_only/recording_quick_query_should_work_with_arrow_output_configuration.js
new file mode 100644
index 000000000000..46972661dd2e
--- /dev/null
+++ b/sdk/storage/storage-file-datalake/recordings/node/datalakepathclient_nodejs_only/recording_quick_query_should_work_with_arrow_output_configuration.js
@@ -0,0 +1,193 @@
+let nock = require('nock');
+
+module.exports.hash = "5d698c75689d843c8dc3544d7ca6d674";
+
+module.exports.testInfo = {"uniqueName":{"filesystem":"filesystem160084667955801065","file":"file160084668061309353"},"newDate":{}}
+
+nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true})
+ .put('/filesystem160084667955801065')
+ .query(true)
+ .reply(201, "", [
+ 'Transfer-Encoding',
+ 'chunked',
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:38:00 GMT',
+ 'ETag',
+ '"0x8D85F93990039E1"',
+ 'x-ms-request-id',
+ 'ff775b84-501e-0005-277c-91eaf9000000',
+ 'x-ms-client-request-id',
+ 'bcc444f1-1b4b-423b-b675-ffaed21ae2e1',
+ 'x-ms-version',
+ '2020-02-10',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:38:00 GMT'
+]);
+
+nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true})
+ .put('/filesystem160084667955801065/file160084668061309353')
+ .query(true)
+ .reply(201, "", [
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:38:01 GMT',
+ 'ETag',
+ '"0x8D85F939996BD58"',
+ 'x-ms-request-id',
+ '51b9d5bf-801f-0008-457c-91d877000000',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-client-request-id',
+ '52fb17bc-9218-4700-b9a0-51280c043c71',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:38:01 GMT',
+ 'Content-Length',
+ '0'
+]);
+
+nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true})
+ .patch('/filesystem160084667955801065/file160084668061309353', "Hello World")
+ .query(true)
+ .reply(202, "", [
+ 'x-ms-request-server-encrypted',
+ 'false',
+ 'x-ms-request-id',
+ '51b9d5c0-801f-0008-467c-91d877000000',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-client-request-id',
+ 'd7ce3602-64ba-419e-ae00-f0db9a5c9dd7',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:38:01 GMT',
+ 'Content-Length',
+ '0'
+]);
+
+nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true})
+ .patch('/filesystem160084667955801065/file160084668061309353')
+ .query(true)
+ .reply(200, "", [
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:38:01 GMT',
+ 'ETag',
+ '"0x8D85F9399E32F9A"',
+ 'x-ms-request-server-encrypted',
+ 'false',
+ 'x-ms-request-id',
+ '51b9d5c1-801f-0008-477c-91d877000000',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-client-request-id',
+ '8d500751-7371-44ac-a85a-0b047fcea14f',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:38:01 GMT',
+ 'Content-Length',
+ '0'
+]);
+
+nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true})
+ .put('/filesystem160084667955801065/file1600846680613093532')
+ .query(true)
+ .reply(201, "", [
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:38:02 GMT',
+ 'ETag',
+ '"0x8D85F939A0843F0"',
+ 'x-ms-request-id',
+ '51b9d5c2-801f-0008-487c-91d877000000',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-client-request-id',
+ 'a7335df3-c7c7-4620-90f1-64d4b905f209',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:38:01 GMT',
+ 'Content-Length',
+ '0'
+]);
+
+nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true})
+ .patch('/filesystem160084667955801065/file1600846680613093532', "100,200,300,400\n150,250,350,450\n")
+ .query(true)
+ .reply(202, "", [
+ 'x-ms-request-server-encrypted',
+ 'false',
+ 'x-ms-request-id',
+ '51b9d5c3-801f-0008-497c-91d877000000',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-client-request-id',
+ '4c2ca286-fc1b-4038-ade1-a9bd8555ac48',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:38:01 GMT',
+ 'Content-Length',
+ '0'
+]);
+
+nock('https://fakestorageaccount.dfs.core.windows.net:443', {"encodedQueryParams":true})
+ .patch('/filesystem160084667955801065/file1600846680613093532')
+ .query(true)
+ .reply(200, "", [
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:38:02 GMT',
+ 'ETag',
+ '"0x8D85F939A548F3E"',
+ 'x-ms-request-server-encrypted',
+ 'false',
+ 'x-ms-request-id',
+ '51b9d5c4-801f-0008-4a7c-91d877000000',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-client-request-id',
+ 'c6fca687-c9a8-4dad-98d4-662808b8fa99',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:38:02 GMT',
+ 'Content-Length',
+ '0'
+]);
+
+nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true})
+ .post('/filesystem160084667955801065/file1600846680613093532', "SQLselect * from BlobStoragearrowdecimalname42")
+ .query(true)
+ .reply(200, Buffer.from("4f626a0102166176726f2e736368656d61be1e5b0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e726573756c7444617461222c0a2020202022646f63223a2022486f6c647320726573756c74206461746120696e2074686520666f726d61742073706563696669656420666f72207468697320717565727920284353562c204a534f4e2c206574632e292e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a202264617461222c0a20202020202020202274797065223a20226279746573220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e6572726f72222c0a2020202022646f63223a2022416e206572726f722074686174206f63637572726564207768696c652070726f63657373696e67207468652071756572792e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a2022666174616c222c0a20202020202020202274797065223a2022626f6f6c65616e222c0a202020202020202022646f63223a2022496620747275652c2074686973206572726f722070726576656e747320667572746865722071756572792070726f63657373696e672e20204d6f726520726573756c742064617461206d61792062652072657475726e65642c20627574207468657265206973206e6f2067756172616e746565207468617420616c6c206f6620746865206f726967696e616c20646174612077696c6c2062652070726f6365737365642e202049662066616c73652c2074686973206572726f7220646f6573206e6f742070726576656e7420667572746865722071756572792070726f63657373696e672e220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a20226e616d65222c0a20202020202020202274797065223a2022737472696e67222c0a202020202020202022646f63223a2022546865206e616d65206f6620746865206572726f72220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a20226465736372697074696f6e222c0a20202020202020202274797065223a2022737472696e67222c0a202020202020202022646f63223a202241206465736372697074696f6e206f6620746865206572726f72220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a2022706f736974696f6e222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520626c6f62206f666673657420617420776869636820746865206572726f72206f63637572726564220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e70726f6772657373222c0a2020202022646f63223a2022496e666f726d6174696f6e2061626f7574207468652070726f6772657373206f6620746865207175657279222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a202262797465735363616e6e6564222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a2022546865206e756d626572206f6620627974657320746861742068617665206265656e207363616e6e6564220a2020202020207d2c0a2020202020207b0a2020202020202020226e616d65223a2022746f74616c4279746573222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520746f74616c206e756d626572206f6620627974657320746f206265207363616e6e656420696e2074686973207175657279220a2020202020207d0a202020205d0a20207d2c0a20207b0a202020202274797065223a20227265636f7264222c0a20202020226e616d65223a2022636f6d2e6d6963726f736f66742e617a7572652e73746f726167652e7175657279426c6f62436f6e74656e74732e656e64222c0a2020202022646f63223a202253656e74206173207468652066696e616c206d657373616765206f662074686520726573706f6e73652c20696e6469636174696e67207468617420616c6c20726573756c74732068617665206265656e2073656e742e222c0a20202020226669656c6473223a205b0a2020202020207b0a2020202020202020226e616d65223a2022746f74616c4279746573222c0a20202020202020202274797065223a20226c6f6e67222c0a202020202020202022646f63223a202254686520746f74616c206e756d626572206f6620627974657320746f206265207363616e6e656420696e2074686973207175657279220a2020202020207d0a202020205d0a20207d0a5d0a00796d4c0f3e9bcb4f8fa2ff91e58363b0028604008004ffffffff800000001000000000000a000c000600050008000a000000000103000c000000080008000000040008000000040000000100000014000000100014000800060007000c000000100010000000000001072400000014000000040000000000000008000c0004000800080000000400000002000000040000006e616d650000000000000000ffffffff700000001000000000000a000e000600050008000a000000000303001000000000000a000c000000040008000a0000003000000004000000020000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000796d4c0f3e9bcb4f8fa2ff91e58363b002040000796d4c0f3e9bcb4f8fa2ff91e58363b0028e01020116436c69656e744572726f726e4e756d626572206f6620636f6c756d6e7320696e2074686520726f7720646f6573206e6f74206d617463682074686520736368656d612e00796d4c0f3e9bcb4f8fa2ff91e58363b00206044040796d4c0f3e9bcb4f8fa2ff91e58363b002040640796d4c0f3e9bcb4f8fa2ff91e58363b0", "hex"), [
+ 'Transfer-Encoding',
+ 'chunked',
+ 'Content-Type',
+ 'avro/binary',
+ 'Last-Modified',
+ 'Wed, 23 Sep 2020 07:38:02 GMT',
+ 'Accept-Ranges',
+ 'bytes',
+ 'ETag',
+ '"0x8D85F939A548F3E"',
+ 'x-ms-creation-time',
+ 'Wed, 23 Sep 2020 07:38:02 GMT',
+ 'x-ms-lease-state',
+ 'available',
+ 'x-ms-lease-status',
+ 'unlocked',
+ 'x-ms-blob-type',
+ 'BlockBlob',
+ 'x-ms-request-id',
+ 'ff775b88-501e-0005-287c-91eaf9000000',
+ 'x-ms-version',
+ '2020-02-10',
+ 'x-ms-client-request-id',
+ '27daec12-f082-427c-bfcb-8c54f3ddafea',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:38:02 GMT'
+]);
+
+nock('https://fakestorageaccount.blob.core.windows.net:443', {"encodedQueryParams":true})
+ .delete('/filesystem160084667955801065')
+ .query(true)
+ .reply(202, "", [
+ 'Transfer-Encoding',
+ 'chunked',
+ 'x-ms-request-id',
+ '0b36984e-d01e-0006-0c7c-911999000000',
+ 'x-ms-client-request-id',
+ '878568e4-a9ce-4286-b7fd-9849bff17e6f',
+ 'x-ms-version',
+ '2020-02-10',
+ 'Date',
+ 'Wed, 23 Sep 2020 07:38:03 GMT'
+]);
diff --git a/sdk/storage/storage-file-datalake/review/storage-file-datalake.api.md b/sdk/storage/storage-file-datalake/review/storage-file-datalake.api.md
index 402bef103bd0..997898bf1077 100644
--- a/sdk/storage/storage-file-datalake/review/storage-file-datalake.api.md
+++ b/sdk/storage/storage-file-datalake/review/storage-file-datalake.api.md
@@ -7,6 +7,7 @@
import { AbortSignalLike } from '@azure/abort-controller';
import { BaseRequestPolicy } from '@azure/core-http';
import { BlobLeaseClient } from '@azure/storage-blob';
+import { BlobQueryArrowField } from '@azure/storage-blob';
import * as coreHttp from '@azure/core-http';
import { deserializationPolicy } from '@azure/core-http';
import { HttpHeaders } from '@azure/core-http';
@@ -374,6 +375,12 @@ export interface FileParallelUploadOptions extends CommonOptions {
umask?: string;
}
+// @public
+export interface FileQueryArrowConfiguration {
+ kind: "arrow";
+ schema: BlobQueryArrowField[];
+}
+
// @public
export interface FileQueryCsvTextConfiguration {
columnSeparator?: string;
@@ -405,7 +412,7 @@ export interface FileQueryOptions extends CommonOptions {
inputTextConfiguration?: FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration;
onError?: (error: FileQueryError) => void;
onProgress?: (progress: TransferProgressEvent) => void;
- outputTextConfiguration?: FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration;
+ outputTextConfiguration?: FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration | FileQueryArrowConfiguration;
}
// @public (undocumented)
diff --git a/sdk/storage/storage-file-datalake/src/models.ts b/sdk/storage/storage-file-datalake/src/models.ts
index 2edde72320d0..d47b2799ba69 100644
--- a/sdk/storage/storage-file-datalake/src/models.ts
+++ b/sdk/storage/storage-file-datalake/src/models.ts
@@ -7,7 +7,8 @@ import { HttpResponse, TransferProgressEvent } from "@azure/core-http";
import {
LeaseAccessConditions,
ModifiedAccessConditions as ModifiedAccessConditionsModel,
- UserDelegationKeyModel
+ UserDelegationKeyModel,
+ BlobQueryArrowField
} from "@azure/storage-blob";
export type ModifiedAccessConditions = Omit;
@@ -1240,6 +1241,30 @@ export interface FileQueryCsvTextConfiguration {
hasHeaders?: boolean;
}
+/**
+ * Options to query file with Apache Arrow format. Only valid for {@link FileQueryOptions.outputTextConfiguration}.
+ *
+ * @export
+ * @interface FileQueryArrowConfiguration
+ */
+export interface FileQueryArrowConfiguration {
+ /**
+ * Kind.
+ *
+ * @type {"arrow"}
+ * @memberof FileQueryArrowConfiguration
+ */
+ kind: "arrow";
+
+ /**
+ * List of {@link BlobQueryArrowField} describing the schema of the data.
+ *
+ * @type {BlobQueryArrowField[]}
+ * @memberof FileQueryArrowConfiguration
+ */
+ schema: BlobQueryArrowField[];
+}
+
/**
* File query error type.
*
@@ -1305,10 +1330,13 @@ export interface FileQueryOptions extends CommonOptions {
/**
* Configurations for the query output.
*
- * @type {FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration}
+ * @type {FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration | FileQueryArrowConfiguration}
* @memberof FileQueryOptions
*/
- outputTextConfiguration?: FileQueryJsonTextConfiguration | FileQueryCsvTextConfiguration;
+ outputTextConfiguration?:
+ | FileQueryJsonTextConfiguration
+ | FileQueryCsvTextConfiguration
+ | FileQueryArrowConfiguration;
/**
* Callback to receive events on the progress of query operation.
*
diff --git a/sdk/storage/storage-file-datalake/test/node/pathclient.spec.ts b/sdk/storage/storage-file-datalake/test/node/pathclient.spec.ts
index 7ca6fc22de69..94160870ec40 100644
--- a/sdk/storage/storage-file-datalake/test/node/pathclient.spec.ts
+++ b/sdk/storage/storage-file-datalake/test/node/pathclient.spec.ts
@@ -334,6 +334,28 @@ describe("DataLakePathClient Node.js only", () => {
const response = await fileClient2.query("select * from BlobStorage");
assert.deepStrictEqual(await bodyToString(response), csvContent);
});
+
+ it("quick query should work with arrow output configuration", async () => {
+ const csvContent = "100,200,300,400\n150,250,350,450\n";
+ const fileClient2 = fileSystemClient.getFileClient(fileName + "2");
+ await fileClient2.create();
+ await fileClient2.append(csvContent, 0, csvContent.length);
+ await fileClient2.flush(csvContent.length);
+
+ await fileClient2.query("select * from BlobStorage", {
+ outputTextConfiguration: {
+ kind: "arrow",
+ schema: [
+ {
+ type: "decimal",
+ name: "name",
+ precision: 4,
+ scale: 2
+ }
+ ]
+ }
+ });
+ });
});
describe("DataLakePathClient setAccessControlRecursive Node.js only", () => {