From d415d577163b5e37699f362e96b5eaaddb36189c Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 14 Nov 2024 14:44:50 +0000 Subject: [PATCH 1/6] .Net 9 target & package updates --- .../Microsoft.Azure.Databricks.Client.Sample.csproj | 6 +++--- .../Microsoft.Azure.Databricks.Client.Test.csproj | 12 ++++++------ .../Microsoft.Azure.Databricks.Client.csproj | 9 +++++++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj b/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj index 23ba987..89787f9 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj @@ -2,14 +2,14 @@ Exe - net6.0;net8.0 + net6.0;net8.0;net9.0 latest false - - + + diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj index fe82e4f..edff776 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj @@ -1,7 +1,7 @@ - + - net6.0;net8.0 + net6.0;net8.0;net9.0 enable enable false @@ -9,14 +9,14 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj index b51fe13..5865d7d 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj @@ -1,7 +1,7 @@ - net6.0;net8.0 + net6.0;net8.0;net9.0 true false true @@ -33,6 +33,11 @@ - + + + + + + From 9245a824ad3b8e54e8c9f026967060f428bec292 Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 14 Nov 2024 15:36:18 +0000 Subject: [PATCH 2/6] Whitespace fixes --- .../Microsoft.Azure.Databricks.Client.Test.csproj | 2 +- .../Microsoft.Azure.Databricks.Client.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj index edff776..c359058 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj @@ -9,7 +9,7 @@ - + diff --git a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj index 5865d7d..6fa23e1 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj @@ -35,7 +35,7 @@ - + From 5144a806eac6055a5db3fa1864a822935c2b89b0 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 17 Nov 2024 20:31:27 -0800 Subject: [PATCH 3/6] fix: adding support for deserialization for StatementExecutionResultChunk.DataArray (#238) * fix: adding support for deserialization for StatementExecutionResultChunk.DataArray * fix: equality check * fix code format * adding deserialization method --- .../StatementExecutionApiClientTest.cs | 424 ++++++++++-------- .../Models/StatementExecution.cs | 40 +- 2 files changed, 267 insertions(+), 197 deletions(-) diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/StatementExecutionApiClientTest.cs b/csharp/Microsoft.Azure.Databricks.Client.Test/StatementExecutionApiClientTest.cs index ca9419f..aa7f391 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Test/StatementExecutionApiClientTest.cs +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/StatementExecutionApiClientTest.cs @@ -1,185 +1,239 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using Microsoft.Azure.Databricks.Client.Models; -using Moq; -using Moq.Contrib.HttpClient; -using System.Net; -using System.Text.Json; -using System.Text.Json.Nodes; - -namespace Microsoft.Azure.Databricks.Client.Test; - -[TestClass] -public class StatementExecutionApiClientTest : ApiClientTest -{ - private static readonly Uri StatementExecutionApiUri = new(BaseApiUri, "2.0/sql/statements"); - - [TestMethod] - public async Task TestExecute() - { - const string expectedRequest = "{\"statement\":\"string\",\"warehouse_id\":\"string\",\"parameters\":[]}"; - const string expectedResponse = @" - { - ""statement_id"": ""string"", - ""status"": { - ""state"": ""SUCCEEDED"" - }, - ""manifest"": { - ""format"": ""JSON_ARRAY"", - ""schema"": { - ""column_count"": 1, - ""columns"": [ - { - ""name"": ""string"", - ""position"": 0, - ""type_name"": ""string"", - ""type_text"": ""string"" - } - ] - } - }, - ""result"": { - ""chunk_index"": 0, - ""row_offset"": 0, - ""row_count"": 1, - ""data_array"": [ - [ ""0"" ] - ] - } - } - "; - - var expected = JsonSerializer.Deserialize(expectedResponse, Options); - - var handler = CreateMockHandler(); - handler - .SetupRequest(HttpMethod.Post, StatementExecutionApiUri) - .ReturnsResponse(HttpStatusCode.OK, JsonSerializer.Serialize(expected, Options), "application/json") - .Verifiable(); - - var hc = handler.CreateClient(); - hc.BaseAddress = BaseApiUri; - - using var client = new StatementExecutionApiClient(hc); - var statement = JsonNode.Parse(expectedRequest).Deserialize(Options); - - var actual = await client.Execute(statement); - Assert.AreEqual(expected, actual); - - handler.VerifyRequest( - HttpMethod.Post, - StatementExecutionApiUri, - GetMatcher(expectedRequest), - Times.Once() - ); - } - - [TestMethod] - public async Task TestGet() - { - string testId = "1234-567890-cited123"; - string apiUri = $"{StatementExecutionApiUri}/{testId}"; - - const string expectedResponse = @" - { - ""statement_id"": ""string"", - ""status"": { - ""state"": ""SUCCEEDED"" - }, - ""manifest"": { - ""format"": ""JSON_ARRAY"", - ""schema"": { - ""column_count"": 1, - ""columns"": [ - { - ""name"": ""string"", - ""position"": 0, - ""type_name"": ""string"", - ""type_text"": ""string"" - } - ] - } - }, - ""result"": { - ""chunk_index"": 0, - ""row_offset"": 0, - ""row_count"": 1, - ""data_array"": [ - [ ""0"" ] - ] - } - } - "; - - var expected = JsonSerializer.Deserialize(expectedResponse, Options); - - var handler = CreateMockHandler(); - handler - .SetupRequest(HttpMethod.Get, apiUri) - .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); - - var hc = handler.CreateClient(); - hc.BaseAddress = BaseApiUri; - - using var client = new StatementExecutionApiClient(hc); - var actual = await client.Get(testId); - Assert.AreEqual(expected, actual); - } - - [TestMethod] - public async Task TestCancel() - { - string testId = "1234-567890-cited123"; - string apiUri = $"{StatementExecutionApiUri}/{testId}/cancel"; - var handler = CreateMockHandler(); - handler - .SetupRequest(HttpMethod.Post, apiUri) - .ReturnsResponse(HttpStatusCode.OK) - .Verifiable(); - - var hc = handler.CreateClient(); - hc.BaseAddress = BaseApiUri; - - using var client = new StatementExecutionApiClient(hc); - await client.Cancel(testId); - handler.VerifyRequest( - HttpMethod.Post, - apiUri, - Times.Once() - ); - } - - [TestMethod] - public async Task TestGetResultChunk() - { - string testId = "1234-567890-cited123"; - int chunkIndex = 0; - string apiUri = $"{StatementExecutionApiUri}/{testId}/result/chunks/{chunkIndex}"; - - const string expectedResponse = @" - { - ""chunk_index"": 0, - ""row_offset"": 0, - ""row_count"": 1, - ""data_array"": [ - [ ""0"" ] - ] - } - "; - - var expected = JsonSerializer.Deserialize(expectedResponse, Options); - - var handler = CreateMockHandler(); - handler - .SetupRequest(HttpMethod.Get, apiUri) - .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); - - var hc = handler.CreateClient(); - hc.BaseAddress = BaseApiUri; - - using var client = new StatementExecutionApiClient(hc); - var actual = await client.GetResultChunk(testId, chunkIndex); - Assert.AreEqual(expected, actual); - } -} +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Microsoft.Azure.Databricks.Client.Models; +using Moq; +using Moq.Contrib.HttpClient; +using System.Net; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Microsoft.Azure.Databricks.Client.Test; + +[TestClass] +public class StatementExecutionApiClientTest : ApiClientTest +{ + public record Person + { + public string Id { get; set; } + public string Firstname { get; set; } + public string Lastname { get; set; } + + public static Person FromJsonArray(JsonArray array, StatementExecutionSchema schema) => new(array); + + public Person(JsonArray array) + { + Id = array[0]?.GetValue() ?? string.Empty; + Firstname = array[1]?.GetValue() ?? string.Empty; + Lastname = array[2]?.GetValue() ?? string.Empty; + } + } + + private static readonly Uri StatementExecutionApiUri = new(BaseApiUri, "2.0/sql/statements"); + + [TestMethod] + public void TestDeserialization() + { + const string result = @" + { + ""chunk_index"": 0, + ""row_offset"": 0, + ""row_count"": 2, + ""data_array"": [ + [ + ""id1"", ""Anthony"", ""Clark"" + ], + [ + ""id2"", ""Clarissa"", ""Bradley"" + ] + ] + } + "; + + var deserialized = JsonSerializer.Deserialize(result, Options); + Assert.IsNotNull(deserialized); + + var columns = new StatementExecutionSchemaColumn[] { + new() { Name = "id", Position = 0, TypeName = "string", TypeText = "string" }, + new() { Name = "firstname", Position = 1, TypeName = "string", TypeText = "string" }, + new() { Name = "lastname", Position = 2, TypeName = "string", TypeText = "string" } + }; + + var execution = new StatementExecution + { + Manifest = new StatementExecutionManifest { Schema = new StatementExecutionSchema { ColumnCount = 3, Columns = columns }, Format = StatementFormat.JSON_ARRAY }, + Result = deserialized + }; + + var leads = execution.DeserializeResults(Person.FromJsonArray); + Assert.AreEqual(2, leads.Count()); + } + + [TestMethod] + public async Task TestExecute() + { + const string expectedRequest = "{\"statement\":\"string\",\"warehouse_id\":\"string\",\"parameters\":[]}"; + const string expectedResponse = @" + { + ""statement_id"": ""string"", + ""status"": { + ""state"": ""SUCCEEDED"" + }, + ""manifest"": { + ""format"": ""JSON_ARRAY"", + ""schema"": { + ""column_count"": 1, + ""columns"": [ + { + ""name"": ""string"", + ""position"": 0, + ""type_name"": ""string"", + ""type_text"": ""string"" + } + ] + } + }, + ""result"": { + ""chunk_index"": 0, + ""row_offset"": 0, + ""row_count"": 1, + ""data_array"": [ + [ ""0"" ] + ] + } + } + "; + + var expected = JsonSerializer.Deserialize(expectedResponse, Options); + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Post, StatementExecutionApiUri) + .ReturnsResponse(HttpStatusCode.OK, JsonSerializer.Serialize(expected, Options), "application/json") + .Verifiable(); + + var hc = handler.CreateClient(); + hc.BaseAddress = BaseApiUri; + + using var client = new StatementExecutionApiClient(hc); + var statement = JsonNode.Parse(expectedRequest).Deserialize(Options); + + var actual = await client.Execute(statement); + Assert.AreEqual(expected, actual); + + handler.VerifyRequest( + HttpMethod.Post, + StatementExecutionApiUri, + GetMatcher(expectedRequest), + Times.Once() + ); + } + + [TestMethod] + public async Task TestGet() + { + string testId = "1234-567890-cited123"; + string apiUri = $"{StatementExecutionApiUri}/{testId}"; + + const string expectedResponse = @" + { + ""statement_id"": ""string"", + ""status"": { + ""state"": ""SUCCEEDED"" + }, + ""manifest"": { + ""format"": ""JSON_ARRAY"", + ""schema"": { + ""column_count"": 1, + ""columns"": [ + { + ""name"": ""string"", + ""position"": 0, + ""type_name"": ""string"", + ""type_text"": ""string"" + } + ] + } + }, + ""result"": { + ""chunk_index"": 0, + ""row_offset"": 0, + ""row_count"": 1, + ""data_array"": [ + [ ""0"" ] + ] + } + } + "; + + var expected = JsonSerializer.Deserialize(expectedResponse, Options); + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, apiUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + var hc = handler.CreateClient(); + hc.BaseAddress = BaseApiUri; + + using var client = new StatementExecutionApiClient(hc); + var actual = await client.Get(testId); + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public async Task TestCancel() + { + string testId = "1234-567890-cited123"; + string apiUri = $"{StatementExecutionApiUri}/{testId}/cancel"; + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Post, apiUri) + .ReturnsResponse(HttpStatusCode.OK) + .Verifiable(); + + var hc = handler.CreateClient(); + hc.BaseAddress = BaseApiUri; + + using var client = new StatementExecutionApiClient(hc); + await client.Cancel(testId); + handler.VerifyRequest( + HttpMethod.Post, + apiUri, + Times.Once() + ); + } + + [TestMethod] + public async Task TestGetResultChunk() + { + string testId = "1234-567890-cited123"; + int chunkIndex = 0; + string apiUri = $"{StatementExecutionApiUri}/{testId}/result/chunks/{chunkIndex}"; + + const string expectedResponse = @" + { + ""chunk_index"": 0, + ""row_offset"": 0, + ""row_count"": 1, + ""data_array"": [ + [ ""0"" ] + ] + } + "; + + var expected = JsonSerializer.Deserialize(expectedResponse, Options); + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, apiUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + var hc = handler.CreateClient(); + hc.BaseAddress = BaseApiUri; + + using var client = new StatementExecutionApiClient(hc); + var actual = await client.GetResultChunk(testId, chunkIndex); + Assert.AreEqual(expected, actual); + } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/Models/StatementExecution.cs b/csharp/Microsoft.Azure.Databricks.Client/Models/StatementExecution.cs index 0334df3..15247f5 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Models/StatementExecution.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/Models/StatementExecution.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json.Serialization; +using System.Text.Json.Nodes; namespace Microsoft.Azure.Databricks.Client.Models; @@ -285,6 +286,16 @@ public record StatementExecution /// [JsonPropertyName("result")] public StatementExecutionResultChunk Result { get; set; } + + public IEnumerable DeserializeResults(Func rowFactory) + { + if (this.Manifest.Format == StatementFormat.JSON_ARRAY) + { + return this.Result.DataArray.Select(row => rowFactory((JsonArray)row, this.Manifest.Schema)); + } + + throw new NotSupportedException($"The format {this.Manifest.Format} is not supported by this method."); + } } public record StatementExecutionStatus @@ -580,7 +591,7 @@ public record StatementExecutionResultChunk : StatementExecutionResult /// The JSON_ARRAY format is an array of arrays of values, where each non-null value is formatted as a string. Null values are encoded as JSON null. /// [JsonPropertyName("data_array")] - public string[][] DataArray { get; set; } = Array.Empty(); + public JsonArray DataArray { get; set; } = new JsonArray(); /// /// The list of external links to the result data in cloud storage. This field is not available when using INLINE disposition. @@ -590,20 +601,25 @@ public record StatementExecutionResultChunk : StatementExecutionResult public virtual bool Equals(StatementExecutionResultChunk other) { - if (other is null) - { - return false; - } - - if (DataArray.Length != other.DataArray.Length) - { - return false; - } - - return base.Equals(other) + return other is not null + && JsonArrayEquals(DataArray, other.DataArray) && ExternalLinks.SequenceEqual(other.ExternalLinks); } + private static bool JsonArrayEquals(JsonArray array1, JsonArray array2) + { + if (array1.Count != array2.Count) + return false; + + for (int i = 0; i < array1.Count; i++) + { + if (!JsonNode.DeepEquals(array1[i], array2[i])) + return false; + } + + return true; + } + public override int GetHashCode() { return base.GetHashCode(); From e07cfcb8b24cce4aca1e4ed0f2aa9f38a7742a2c Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 17 Nov 2024 21:09:16 -0800 Subject: [PATCH 4/6] Install dotnet 9 for PR build (#244) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 58ee772..983bf39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: - name: Install .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 6.x + dotnet-version: 9.x - name: Restore run: dotnet restore "$Solution_Name" From 99e18fc71970e9c7e0259892baec8af487a0efb5 Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 14 Nov 2024 14:44:50 +0000 Subject: [PATCH 5/6] .Net 9 target & package updates --- .../Microsoft.Azure.Databricks.Client.Sample.csproj | 6 +++--- .../Microsoft.Azure.Databricks.Client.Test.csproj | 12 ++++++------ .../Microsoft.Azure.Databricks.Client.csproj | 9 +++++++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj b/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj index 23ba987..89787f9 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj @@ -2,14 +2,14 @@ Exe - net6.0;net8.0 + net6.0;net8.0;net9.0 latest false - - + + diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj index fe82e4f..edff776 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj @@ -1,7 +1,7 @@ - + - net6.0;net8.0 + net6.0;net8.0;net9.0 enable enable false @@ -9,14 +9,14 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj index b51fe13..5865d7d 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj @@ -1,7 +1,7 @@ - net6.0;net8.0 + net6.0;net8.0;net9.0 true false true @@ -33,6 +33,11 @@ - + + + + + + From c91d47835a1df2ad2be8f9b1819d20cd3b18f3a2 Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 14 Nov 2024 15:36:18 +0000 Subject: [PATCH 6/6] Whitespace fixes --- .../Microsoft.Azure.Databricks.Client.Test.csproj | 2 +- .../Microsoft.Azure.Databricks.Client.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj index edff776..c359058 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj @@ -9,7 +9,7 @@ - + diff --git a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj index 5865d7d..6fa23e1 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj @@ -35,7 +35,7 @@ - +