diff --git a/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram .MachineLearning.cs b/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram .MachineLearning.cs new file mode 100644 index 0000000..aa97433 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram .MachineLearning.cs @@ -0,0 +1,20 @@ +using Microsoft.Azure.Databricks.Client.Models.UnityCatalog; +using System; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Databricks.Client.Sample; + +internal static partial class SampleProgram +{ + private static async Task TestExperimentApiClient(DatabricksClient client) + { + string run_id = "sample_run_id"; + PrintDelimiter(); + Console.WriteLine("Listing metastores"); + var data = await client.MachineLearning.Experiments.GetRun(run_id); + Console.WriteLine($"\t{data.Info.RunId}"); + PrintDelimiter(); + + } +} \ No newline at end of file diff --git a/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.UnityCatalog.cs b/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.UnityCatalog.cs index 118cdc7..a055784 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.UnityCatalog.cs +++ b/csharp/Microsoft.Azure.Databricks.Client.Sample/SampleProgram.UnityCatalog.cs @@ -73,5 +73,22 @@ private static async Task TestUnityCatalogApi(DatabricksClient client) Console.WriteLine($"Deleting created catalog {catalog.Name}..."); await client.UnityCatalog.Catalogs.Delete(catalog.Name); Console.WriteLine("Catalog deleted"); + + Console.WriteLine("Listing Regsitered Models"); + var registeredModels = await client.UnityCatalog.RegisteredModels.ListRegisteredModels(); + foreach (var model in registeredModels) + { + Console.WriteLine($"\t{model.MetastoreId}, {model.Name}"); + } + PrintDelimiter(); + + var fullName = "main.default.revenue_forecasting"; + Console.WriteLine("Listing Regsitered Models"); + var modelVersionsList = await client.UnityCatalog.ModelVersion.ListModelVersions(fullName); + foreach (var model in modelVersionsList) + { + Console.WriteLine($"\t{model.MetastoreId}, {model.CatalogName}"); + } + PrintDelimiter(); } } \ No newline at end of file diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/MachineLearning/ExperimentApiClientTest.cs b/csharp/Microsoft.Azure.Databricks.Client.Test/MachineLearning/ExperimentApiClientTest.cs new file mode 100644 index 0000000..ce7ee26 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/MachineLearning/ExperimentApiClientTest.cs @@ -0,0 +1,100 @@ + + +using Microsoft.Azure.Databricks.Client.MachineLearning; +using Moq.Contrib.HttpClient; +using System.Net; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Azure.Databricks.Client.Models.MachineLearning.Experiment; + +namespace Microsoft.Azure.Databricks.Client.Test.MachineLearning; + +[TestClass] +public class ExperimentApiClientTests : MachineLearningApiClientTest +{ + [TestMethod] + public async Task GetRunTest() + { + var run_id = "19d13cbfa8134b699f8b41fdab0cdd4c"; + var requestUri = $"{MlflowBaseUri}/runs/get?run_id={run_id}"; + var expectedResponse = @" + { + ""run"": { + ""info"": { + ""run_id"": ""string"", + ""run_uuid"": ""string"", + ""experiment_id"": ""string"", + ""user_id"": ""string"", + ""status"": ""RUNNING"", + ""start_time"": 0, + ""end_time"": 0, + ""artifact_uri"": ""string"", + ""lifecycle_stage"": ""string"" + }, + ""data"": { + ""metrics"": [ + { + ""key"": ""string"", + ""value"": 0.1, + ""timestamp"": 0, + ""step"": ""0"" + } + ], + ""params"": [ + { + ""key"": ""string"", + ""value"": ""string"" + } + ], + ""tags"": [ + { + ""key"": ""string"", + ""value"": ""string"" + } + ] + }, + ""inputs"": { + ""dataset_inputs"": [ + { + ""tags"": [ + { + ""key"": ""string"", + ""value"": ""string"" + } + ], + ""dataset"": { + ""name"": ""string"", + ""digest"": ""string"", + ""source_type"": ""string"", + ""source"": ""string"", + ""schema"": ""string"", + ""profile"": ""string"" + } + } + ] + } + } + }"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, requestUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = BaseApiUri; + + using var client = new ExperimentApiClient(mockClient); + var response = await client.GetRun(run_id); + + var expected = JsonNode.Parse(expectedResponse)?["run"].Deserialize(Options); + + Assert.IsNotNull(expected, "Expected object is null."); + Assert.IsNotNull(response, "Response object is null."); + + if (response != null && expected != null) + { + Assert.AreEqual(expected.ToString(), response.ToString()); + } + } +} \ No newline at end of file diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/MachineLearning/MachineLearningApiClientTest.cs b/csharp/Microsoft.Azure.Databricks.Client.Test/MachineLearning/MachineLearningApiClientTest.cs new file mode 100644 index 0000000..907ea69 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/MachineLearning/MachineLearningApiClientTest.cs @@ -0,0 +1,7 @@ +namespace Microsoft.Azure.Databricks.Client.Test.MachineLearning; + +[TestClass] +public class MachineLearningApiClientTest : ApiClientTest +{ + protected static readonly Uri MlflowBaseUri = new(BaseApiUri, "2.0/mlflow"); +} diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/UnityCatalog/ModelVersionApiClientTests.cs b/csharp/Microsoft.Azure.Databricks.Client.Test/UnityCatalog/ModelVersionApiClientTests.cs new file mode 100644 index 0000000..f836add --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/UnityCatalog/ModelVersionApiClientTests.cs @@ -0,0 +1,172 @@ +using Microsoft.Azure.Databricks.Client.Models.UnityCatalog; +using Microsoft.Azure.Databricks.Client.UnityCatalog; +using Moq.Contrib.HttpClient; +using System.Net; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Microsoft.Azure.Databricks.Client.Test.UnityCatalog; + +[TestClass] +public class ModelVersionApiClientTests : UnityCatalogApiClientTest +{ + [TestMethod] + public async Task ListModelVersionsTest() + { + var full_name = "main.default.revenue_forecasting"; + var requestUri = $"{BaseApiUri}models/{full_name}/versions"; + + var expectedResponse = @" + { + ""model_versions"": [ + { + ""model_name"": ""revenue_forecasting_model"", + ""catalog_name"": ""main"", + ""schema_name"": ""default"", + ""comment"": ""This model version forecasts future revenue given historical data, using classic ML techniques"", + ""source"": ""dbfs:/databricks/mlflow-tracking/1234567890/abcdef/artifacts/model"", + ""run_id"": ""abcdef"", + ""run_workspace_id"": 6051234418418567, + ""version"": 1, + ""status"": ""READY"", + ""id"": ""9876543-21zy-abcd-3210-abcdef456789"", + ""metastore_id"": ""11111111-1111-1111-1111-111111111111"", + ""created_at"": 1666369196203, + ""created_by"": ""Alice@example.com"", + ""updated_at"": 1666369196203, + ""updated_by"": ""Alice@example.com"", + ""storage_location"": ""s3://my-bucket/hello/world/models/2222-2222/versions/9876543-21zy-abcd-3210-abcdef456789"" + }, + { + ""model_name"": ""revenue_forecasting_model"", + ""catalog_name"": ""main"", + ""schema_name"": ""default"", + ""comment"": ""This model version forecasts future revenue given historical data, using deep learning"", + ""source"": ""dbfs:/databricks/mlflow-tracking/1234567890/abcdef/artifacts/model"", + ""run_id"": ""abcdef"", + ""run_workspace_id"": 6051234418418567, + ""version"": 2, + ""status"": ""READY"", + ""id"": ""01234567-89ab-cdef-0123-456789abcdef"", + ""metastore_id"": ""11111111-1111-1111-1111-111111111111"", + ""created_at"": 1666369196907, + ""created_by"": ""Alice@example.com"", + ""updated_at"": 1666369196907, + ""updated_by"": ""Alice@example.com"", + ""storage_location"": ""s3://my-bucket/hello/world/models/2222-2222/versions/01234567-89ab-cdef-0123-456789abcdef"" + } + ], + ""next_page_token"": ""some-page-token"" + }"; + var expected = JsonNode.Parse(expectedResponse)?["model_versions"].Deserialize>(Options); + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, requestUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = ApiClientTest.BaseApiUri; + + using var client = new ModelVersionApiClient(mockClient); + var response = await client.ListModelVersions(full_name); + + var responseJson = JsonSerializer.Serialize(response, Options); + CollectionAssert.AreEqual(expected?.ToList(), response?.ToList()); + } + + + [TestMethod] + public async Task GetModelVersionTest() + { + var expectedResponse = @" + { + ""model_name"": ""revenue_forecasting_model"", + ""catalog_name"": ""main"", + ""schema_name"": ""default"", + ""comment"": ""This model version forecasts future revenue given historical data, using classic ML techniques"", + ""source"": ""dbfs:/databricks/mlflow-tracking/1234567890/abcdef/artifacts/model"", + ""run_id"": ""abcdef"", + ""run_workspace_id"": 6051234418418567, + ""version"": 1, + ""status"": ""READY"", + ""id"": ""01234567-89ab-cdef-0123-456789abcdef"", + ""metastore_id"": ""11111111-1111-1111-1111-111111111111"", + ""created_at"": 1666369196203, + ""created_by"": ""Alice@example.com"", + ""updated_at"": 1666369196203, + ""updated_by"": ""Alice@example.com"", + ""storage_location"": ""s3://my-bucket/hello/world/models/2222-2222/versions/01234567-89ab-cdef-0123-456789abcdef"" + } + "; + + var full_name = "main.default.revenue_forecasting_model"; + var version = 2; + var requestUri = $"{BaseApiUri}models/{full_name}/versions/{version}"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, requestUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = ApiClientTest.BaseApiUri; + + using var client = new ModelVersionApiClient(mockClient); + var response = await client.GetModelVersion(full_name, version); + + var responseJson = JsonSerializer.Serialize(response, Options); + AssertJsonDeepEquals(expectedResponse, responseJson); + } + + [TestMethod] + public async Task GetModelVersionByAliasTest() + { + var full_name = "main.default.revenue_forecasting_model"; + var alias = "champion"; + var requestUri = $"{BaseApiUri}models/{full_name}/aliases/{alias}"; + + + var expectedResponse = @" + { + ""model_name"": ""revenue_forecasting_model"", + ""catalog_name"": ""main"", + ""schema_name"": ""default"", + ""comment"": ""This model version forecasts future revenue, given historical data"", + ""source"": ""dbfs:/databricks/mlflow-tracking/1234567890/abcdef/artifacts/model"", + ""run_id"": ""abcdef"", + ""run_workspace_id"": 6051234418418567, + ""version"": 1, + ""status"": ""PENDING_REGISTRATION"", + ""id"": ""01234567-89ab-cdef-0123-456789abcdef"", + ""metastore_id"": ""11111111-1111-1111-1111-111111111111"", + ""created_at"": 1666369196203, + ""created_by"": ""Alice@example.com"", + ""updated_at"": 1666369196203, + ""updated_by"": ""Alice@example.com"", + ""storage_location"": ""s3://my-bucket/hello/world/models/2222-2222/versions/01234567-89ab-cdef-0123-456789abcdef"", + ""aliases"": [ + { + ""alias_name"": ""champion"", + ""version_num"": 1 + } + ] + } + "; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, requestUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = ApiClientTest.BaseApiUri; + + using var client = new ModelVersionApiClient(mockClient); + + var response = await client.GetModelVersionByAlias(full_name, alias); + var responseJson = JsonSerializer.Serialize(response, Options); + AssertJsonDeepEquals(expectedResponse, responseJson); + } +} \ No newline at end of file diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/UnityCatalog/RegisteredModelsApiClientTests.cs b/csharp/Microsoft.Azure.Databricks.Client.Test/UnityCatalog/RegisteredModelsApiClientTests.cs new file mode 100644 index 0000000..c8ae285 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/UnityCatalog/RegisteredModelsApiClientTests.cs @@ -0,0 +1,159 @@ +using Microsoft.Azure.Databricks.Client.Models.UnityCatalog; +using Microsoft.Azure.Databricks.Client.UnityCatalog; +using Moq; +using Moq.Contrib.HttpClient; +using System.Net; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Microsoft.Azure.Databricks.Client.Test.UnityCatalog; + +[TestClass] +public class RegisteredModelsApiClientTests : UnityCatalogApiClientTest +{ + [TestMethod] + public async Task ListRegisteredModelsTest() + { + var requestUri = $"{BaseApiUri}models"; + var expectedResponse = @" + { + ""registered_models"": [ + { + ""name"": ""revenue_forecasting_model"", + ""catalog_name"": ""main"", + ""schema_name"": ""default"", + ""full_name"": ""main.default.revenue_forecasting_model"", + ""owner"": ""Alice@example.com"", + ""id"": ""01234567-89ab-cdef-0123-456789abcdef"", + ""metastore_id"": ""11111111-1111-1111-1111-111111111111"", + ""created_at"": 1666369196203, + ""created_by"": ""Alice@example.com"", + ""updated_at"": 1666369196203, + ""updated_by"": ""Alice@example.com"", + ""storage_location"": ""s3://my-bucket/hello/world/models/01234567-89ab-cdef-0123-456789abcdef"", + ""securable_type"": ""FUNCTION"", + ""securable_kind"": ""FUNCTION_REGISTERED_MODEL"", + ""comment"": ""This model contains model versions that forecast future revenue, given historical data"" + }, + { + ""name"": ""fraud_detection_model"", + ""catalog_name"": ""main"", + ""schema_name"": ""default"", + ""full_name"": ""main.default.fraud_detection_model"", + ""owner"": ""Alice@example.com"", + ""id"": ""9876543-21zy-abcd-3210-abcdef456789"", + ""metastore_id"": ""11111111-1111-1111-1111-111111111111"", + ""created_at"": 1666369196345, + ""created_by"": ""Alice@example.com"", + ""updated_at"": 1666369196345, + ""updated_by"": ""Alice@example.com"", + ""storage_location"": ""s3://my-bucket/hello/world/models/9876543-21zy-abcd-3210-abcdef456789"", + ""securable_type"": ""FUNCTION"", + ""securable_kind"": ""FUNCTION_REGISTERED_MODEL"", + ""comment"": ""This model contains model versions that identify fraudulent transactions"" + } + ], + ""next_page_token"": ""some-page-token"" + }"; + + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, requestUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = ApiClientTest.BaseApiUri; + + using var client = new RegisteredModelsApiClient(mockClient); + var response = await client.ListRegisteredModels(); + + var responseJson = JsonSerializer.Serialize(response, Options); + var expected = JsonNode.Parse(expectedResponse)?["registered_models"].Deserialize>(Options); + CollectionAssert.AreEqual(expected?.ToList(), response?.ToList()); + } + + + [TestMethod] + public void GetRegisteredModelTest() + { + var expectedResponse = @" + { + ""name"": ""my_model"", + ""catalog_name"": ""main"", + ""schema_name"": ""default"", + ""full_name"": ""main.default.my_model"", + ""owner"": ""Alice@example.com"", + ""id"": ""01234567-89ab-cdef-0123-456789abcdef"", + ""metastore_id"": ""11111111-1111-1111-1111-111111111111"", + ""created_at"": 1666369196203, + ""created_by"": ""Alice@example.com"", + ""updated_at"": 1666369196203, + ""updated_by"": ""Alice@example.com"", + ""storage_location"": ""s3://my-bucket/hello/world/my-model"", + ""securable_type"": ""FUNCTION"", + ""securable_kind"": ""FUNCTION_REGISTERED_MODEL"", + ""comment"": ""This is my first model"" + } + "; + + var full_name = "main.default.my_model"; + var requestUri = $"{BaseApiUri}models/{full_name}"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Get, requestUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = ApiClientTest.BaseApiUri; + + using var client = new RegisteredModelsApiClient(mockClient); + var response = client.GetRegisteredModel(full_name).Result; + + var responseJson = JsonSerializer.Serialize(response, Options); + AssertJsonDeepEquals(expectedResponse, responseJson); + } + + [TestMethod] + public async Task SetRegisteredModelAliasTest() + { + var full_name = "main.default.revenue_forecasting_model"; + var alias = "champion"; + var version_num = 2; + var requestUri = $"{BaseApiUri}models/{full_name}/aliases/{alias}"; + + + var expectedRequest = @" + { + ""version_num"": 2 + }"; + + var expectedResponse = @" + { + ""alias_name"": ""champion"", + ""version_num"": 2 + }"; + + var handler = CreateMockHandler(); + handler + .SetupRequest(HttpMethod.Put, requestUri) + .ReturnsResponse(HttpStatusCode.OK, expectedResponse, "application/json"); + + + var mockClient = handler.CreateClient(); + mockClient.BaseAddress = ApiClientTest.BaseApiUri; + + using var client = new RegisteredModelsApiClient(mockClient); + var response = await client.SetRegisteredModelAlias(full_name, alias, version_num); + var responseJson = JsonSerializer.Serialize(response, Options); + AssertJsonDeepEquals(expectedResponse, responseJson); + + handler.VerifyRequest( + HttpMethod.Put, + requestUri, + GetMatcher(expectedRequest), + Times.Once()); + } +} \ No newline at end of file diff --git a/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs index 5472401..f1ed964 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs @@ -21,6 +21,8 @@ public abstract class ApiClient : IDisposable protected string BaseUnityCatalogUri => "2.1/unity-catalog"; + protected string BaseMLFlowApiUri => "2.0/mlflow"; + protected static readonly JsonSerializerOptions Options = new() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault, diff --git a/csharp/Microsoft.Azure.Databricks.Client/DatabricksClient.cs b/csharp/Microsoft.Azure.Databricks.Client/DatabricksClient.cs index c3e080b..044f9c7 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/DatabricksClient.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/DatabricksClient.cs @@ -35,6 +35,7 @@ private DatabricksClient(HttpClient httpClient) this.Repos = new ReposApiClient(httpClient); this.Pipelines = new PipelinesApiClient(httpClient); this.UnityCatalog = new UnityCatalogClient(httpClient); + this.MachineLearning = new MachineLearningClient(httpClient); } /// @@ -211,6 +212,8 @@ public static DatabricksClient CreateClient(string baseUrl, TokenCredential cred public virtual UnityCatalogClient UnityCatalog { get; } + public virtual MachineLearningClient MachineLearning { get; } + public void Dispose() { Clusters.Dispose(); @@ -227,6 +230,7 @@ public void Dispose() Repos.Dispose(); Pipelines.Dispose(); UnityCatalog.Dispose(); + MachineLearning.Dispose(); GC.SuppressFinalize(this); } } \ No newline at end of file diff --git a/csharp/Microsoft.Azure.Databricks.Client/MachineLearning/ExperimentApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/MachineLearning/ExperimentApiClient.cs new file mode 100644 index 0000000..b14e184 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/MachineLearning/ExperimentApiClient.cs @@ -0,0 +1,23 @@ +using Microsoft.Azure.Databricks.Client.Models.MachineLearning.Experiment; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Databricks.Client.MachineLearning; + +public class ExperimentApiClient : ApiClient, IExperimentApi +{ + public ExperimentApiClient(HttpClient httpClient) : base(httpClient) + { + } + + public async Task GetRun(string run_id, CancellationToken cancellationToken = default) + { + var requestUri = $"{BaseMLFlowApiUri}/runs/get?run_id={run_id}"; + var jsonResponse = await HttpGet(HttpClient, requestUri, cancellationToken).ConfigureAwait(false); + jsonResponse.TryGetPropertyValue("run", out var run); + return run.Deserialize(Options); + } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/MachineLearning/IExperimentApi.cs b/csharp/Microsoft.Azure.Databricks.Client/MachineLearning/IExperimentApi.cs new file mode 100644 index 0000000..008240a --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/MachineLearning/IExperimentApi.cs @@ -0,0 +1,16 @@ +using Microsoft.Azure.Databricks.Client.Models.MachineLearning.Experiment; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Databricks.Client.MachineLearning; + +public interface IExperimentApi : IDisposable +{ + /// + /// Gets the metadata, metrics, params, and tags for a run. In the case where multiple metrics with the same key are logged for a run, return only the value with the latest timestamp. + /// If there are multiple values with the latest timestamp, return the maximum of these values. + /// + /// ID of the run to fetch. Must be provided. + Task GetRun(string run_id, CancellationToken cancellationToken = default); +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/MachineLearningClient.cs b/csharp/Microsoft.Azure.Databricks.Client/MachineLearningClient.cs new file mode 100644 index 0000000..c09428f --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/MachineLearningClient.cs @@ -0,0 +1,15 @@ +using Microsoft.Azure.Databricks.Client.MachineLearning; +using System; +using System.Net.Http; + +namespace Microsoft.Azure.Databricks.Client; + +public class MachineLearningClient : ApiClient, IDisposable +{ + public MachineLearningClient(HttpClient httpClient) : base(httpClient) + { + this.Experiments = new ExperimentApiClient(httpClient); + } + + public virtual IExperimentApi Experiments { get; set; } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/Models/MachineLearning/Experiment/Run.cs b/csharp/Microsoft.Azure.Databricks.Client/Models/MachineLearning/Experiment/Run.cs new file mode 100644 index 0000000..dee3e23 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/Models/MachineLearning/Experiment/Run.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Azure.Databricks.Client.Models.MachineLearning.Experiment; + +public record Run +{ + [JsonPropertyName("info")] + public Info Info { get; set; } + + [JsonPropertyName("data")] + public Data Data { get; set; } + + [JsonPropertyName("inputs")] + public Inputs Inputs { get; set; } +} + +public record Info +{ + [JsonPropertyName("run_id")] + public string RunId { get; set; } + + [JsonPropertyName("experiment_id")] + public string ExperimentId { get; set; } + + [Obsolete("This field is deprecated as of MLflow 1.0, and will be removed in a future MLflow release. Use 'mlflow.user' tag instead.")] + [JsonPropertyName("user_id")] + public string UserId { get; set; } + + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("start_time")] + public DateTimeOffset? StartTime { get; set; } + + [JsonPropertyName("end_time")] + public DateTimeOffset? EndTime { get; set; } + + [JsonPropertyName("artifact_uri")] + public string ArtifactUri { get; set; } + + [JsonPropertyName("lifecycle_stage")] + public string LifecycleStage { get; set; } +} + +public record Data +{ + [JsonPropertyName("metrics")] + public IEnumerable Metrics { get; set; } + + [JsonPropertyName("params")] + public IEnumerable Params { get; set; } + + [JsonPropertyName("tags")] + public IEnumerable Tags { get; set; } +} + +public record Metric +{ + [JsonPropertyName("key")] + public string Key { get; set; } + + [JsonPropertyName("value")] + public double Value { get; set; } + + [JsonPropertyName("timestamp")] + public DateTimeOffset? Timestamp { get; set; } + + [JsonPropertyName("step")] + public string Step { get; set; } +} + +public record Param +{ + [JsonPropertyName("key")] + public string Key { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } +} + +public record Tag +{ + [JsonPropertyName("key")] + public string Key { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } +} + +public record Inputs +{ + [JsonPropertyName("dataset_inputs")] + public IEnumerable DatasetInputs { get; set; } +} + +public record DatasetInput +{ + [JsonPropertyName("tags")] + public IEnumerable Tags { get; set; } + + [JsonPropertyName("dataset")] + public Dataset Dataset { get; set; } +} + +public record Dataset +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("digest")] + public string Digest { get; set; } + + [JsonPropertyName("source_type")] + public string SourceType { get; set; } + + [JsonPropertyName("source")] + public string Source { get; set; } + + [JsonPropertyName("schema")] + public string Schema { get; set; } + + [JsonPropertyName("profile")] + public string Profile { get; set; } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/Models/RunResultState.cs b/csharp/Microsoft.Azure.Databricks.Client/Models/RunResultState.cs index e048e15..6846b4a 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Models/RunResultState.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/Models/RunResultState.cs @@ -31,14 +31,14 @@ public enum RunResultState /// The run was canceled at user request. /// CANCELED, - + MAXIMUM_CONCURRENT_RUNS_REACHED, - + EXCLUDED, - + SUCCESS_WITH_FAILURES, - + UPSTREAM_FAILED, - + UPSTREAM_CANCELED } \ No newline at end of file diff --git a/csharp/Microsoft.Azure.Databricks.Client/Models/UnityCatalog/ModelVersions.cs b/csharp/Microsoft.Azure.Databricks.Client/Models/UnityCatalog/ModelVersions.cs new file mode 100644 index 0000000..81cfa32 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/Models/UnityCatalog/ModelVersions.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; +namespace Microsoft.Azure.Databricks.Client.Models.UnityCatalog; + +public record ModelVersion +{ + [JsonPropertyName("model_name")] + public string ModelName { get; set; } + + [JsonPropertyName("catalog_name")] + public string CatalogName { get; set; } + + [JsonPropertyName("schema_name")] + public string SchemaName { get; set; } + + [JsonPropertyName("comment")] + public string Comment { get; set; } + + [JsonPropertyName("source")] + public string Source { get; set; } + + [JsonPropertyName("run_id")] + public string RunId { get; set; } + + [JsonPropertyName("run_workspace_id")] + public long RunWorkspaceId { get; set; } + + [JsonPropertyName("version")] + public int Version { get; set; } + + [JsonPropertyName("status")] + public string Status { get; set; } + + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("metastore_id")] + public string MetastoreId { get; set; } + + [JsonPropertyName("created_at")] + public DateTimeOffset? CreatedAt { get; set; } + + [JsonPropertyName("created_by")] + public string CreatedBy { get; set; } + + [JsonPropertyName("updated_at")] + public DateTimeOffset? UpdatedAt { get; set; } + + [JsonPropertyName("updated_by")] + public string UpdatedBy { get; set; } + + [JsonPropertyName("storage_location")] + public string StorageLocation { get; set; } + + [JsonPropertyName("aliases")] + public IEnumerable Aliases { get; set; } +} + +public record Alias +{ + [JsonPropertyName("alias_name")] + public string AliasName { get; set; } + + [JsonPropertyName("version_num")] + public int VersionNum { get; set; } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/Models/UnityCatalog/RegisteredModel.cs b/csharp/Microsoft.Azure.Databricks.Client/Models/UnityCatalog/RegisteredModel.cs new file mode 100644 index 0000000..00fb748 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/Models/UnityCatalog/RegisteredModel.cs @@ -0,0 +1,61 @@ +using System; +using System.Text.Json.Serialization; + +namespace Microsoft.Azure.Databricks.Client.Models.UnityCatalog; + +public record RegisteredModel +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("catalog_name")] + public string CatalogName { get; set; } + + [JsonPropertyName("schema_name")] + public string SchemaName { get; set; } + + [JsonPropertyName("full_name")] + public string FullName { get; set; } + + [JsonPropertyName("owner")] + public string Owner { get; set; } + + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("metastore_id")] + public string MetastoreId { get; set; } + + [JsonPropertyName("created_at")] + public DateTimeOffset? CreatedAt { get; set; } + + [JsonPropertyName("created_by")] + public string CreatedBy { get; set; } + + [JsonPropertyName("updated_at")] + public DateTimeOffset? UpdatedAt { get; set; } + + [JsonPropertyName("updated_by")] + public string UpdatedBy { get; set; } + + [JsonPropertyName("storage_location")] + public string StorageLocation { get; set; } + + [JsonPropertyName("securable_type")] + public string SecurableType { get; set; } + + [JsonPropertyName("securable_kind")] + public string SecurableKind { get; set; } + + [JsonPropertyName("comment")] + public string Comment { get; set; } +} + +public record RegisteredModelAlias +{ + [JsonPropertyName("alias_name")] + public string AliasName { get; set; } + + [JsonPropertyName("version_num")] + public int VersionNum { get; set; } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/Interfaces/IModelVersionApi.cs b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/Interfaces/IModelVersionApi.cs new file mode 100644 index 0000000..453e016 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/Interfaces/IModelVersionApi.cs @@ -0,0 +1,32 @@ +using Microsoft.Azure.Databricks.Client.Models.UnityCatalog; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Databricks.Client.UnityCatalog; + +public interface IModelVersionApi : IDisposable +{ + /// + /// List model versions. You can list model versions under a particular schema, or list all model versions in the current metastore. + /// The returned models are filtered based on the privileges of the calling user. For example, the metastore admin is able to list all the model versions. A regular user needs to be the owner or have the EXECUTE privilege on the parent registered model to recieve the model versions in the response. For the latter case, the caller must also be the owner or have the USE_CATALOG privilege on the parent catalog and the USE_SCHEMA privilege on the parent schema. + /// + Task> ListModelVersions(string full_name, int max_results = 0, CancellationToken cancellationToken = default); + + /// + /// Get a model version. + /// The caller must be a metastore admin or an owner of (or have the EXECUTE privilege on) the parent registered model. For the latter case, the caller must also be the owner or have the USE_CATALOG privilege on the parent catalog and the USE_SCHEMA privilege on the parent schema. + /// + Task GetModelVersion( + string full_name, + int version, + CancellationToken cancellationToken = default + ); + + /// + /// Get a model version by alias + /// The caller must be a metastore admin or an owner of (or have the EXECUTE privilege on) the registered model. For the latter case, the caller must also be the owner or have the USE_CATALOG privilege on the parent catalog and the USE_SCHEMA privilege on the parent schema. + /// + Task GetModelVersionByAlias(string full_name, string alias, CancellationToken cancellationToken = default); +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/Interfaces/IRegisteredModelsApi.cs b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/Interfaces/IRegisteredModelsApi.cs new file mode 100644 index 0000000..f1ac16b --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/Interfaces/IRegisteredModelsApi.cs @@ -0,0 +1,37 @@ +using Microsoft.Azure.Databricks.Client.Models.UnityCatalog; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Databricks.Client.UnityCatalog; + +public interface IRegisteredModelsApi : IDisposable +{ + /// + /// List registered models. You can list registered models under a particular schema, or list all registered models in the current metastore. + /// + Task> ListRegisteredModels( + string catalog_name = null, + string schema_name = null, + int max_results = 0, + CancellationToken cancellationToken = default + ); + + /// + ///Get a registered model + /// The caller must be a metastore admin or an owner of(or have the EXECUTE privilege on) the registered model.For the latter case, the caller must also be the owner or have the USE_CATALOG privilege on the parent catalog and the USE_SCHEMA privilege on the parent schema. + /// + Task GetRegisteredModel(string full_name, CancellationToken cancellationToken = default); + + /// + /// Set an alias on the specified registered model. + /// The caller must be a metastore admin or an owner of the registered model.For the latter case, the caller must also be the owner or have the USE_CATALOG privilege on the parent catalog and the USE_SCHEMA privilege on the parent schema. + /// + Task SetRegisteredModelAlias( + string full_name, + string alias, + int version_num, + CancellationToken cancellationToken = default + ); +} \ No newline at end of file diff --git a/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/ModelVersionApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/ModelVersionApiClient.cs new file mode 100644 index 0000000..fdf1522 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/ModelVersionApiClient.cs @@ -0,0 +1,56 @@ +using Microsoft.Azure.Databricks.Client.Models.UnityCatalog; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Databricks.Client.UnityCatalog +{ + public class ModelVersionApiClient : ApiClient, IModelVersionApi + { + public ModelVersionApiClient(HttpClient httpClient) : base(httpClient) { } + + public async Task> ListModelVersions( + string full_name, + int max_results = 0, + CancellationToken cancellationToken = default) + { + var requestUriSb = new StringBuilder($"{BaseUnityCatalogUri}/models/{full_name}/versions"); + if (max_results > 0) + { + requestUriSb.Append($"?max_results={max_results}"); + } + + var requestUri = requestUriSb.ToString(); + var modelVersionsJson = await HttpGet(HttpClient, requestUri, cancellationToken).ConfigureAwait(false); + modelVersionsJson.TryGetPropertyValue("model_versions", out var modelVersions); + return modelVersions?.Deserialize>(Options) ?? Enumerable.Empty(); + } + + + + public async Task GetModelVersion( + string full_name, + int version, + CancellationToken cancellationToken = default + ) + { + var requestUriSb = new StringBuilder($"{BaseUnityCatalogUri}/models/{full_name}/versions/{version}"); + var requestUri = requestUriSb.ToString(); + return await HttpGet(HttpClient, requestUri, cancellationToken).ConfigureAwait(false); + } + + public async Task GetModelVersionByAlias( + string full_name, + string alias, + CancellationToken cancellationToken = default) + { + var requestUri = $"{BaseUnityCatalogUri}/models/{full_name}/aliases/{alias}"; + return await HttpGet(HttpClient, requestUri, cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/RegisteredModelApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/RegisteredModelApiClient.cs new file mode 100644 index 0000000..5a4e620 --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalog/RegisteredModelApiClient.cs @@ -0,0 +1,66 @@ +using Microsoft.Azure.Databricks.Client.Models.UnityCatalog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Databricks.Client.UnityCatalog; + +public class RegisteredModelsApiClient : ApiClient, IRegisteredModelsApi +{ + public RegisteredModelsApiClient(HttpClient httpClient) : base(httpClient) { } + + + public async Task> ListRegisteredModels( + string catalog_name = null, + string schema_name = null, + int max_results = 0, + CancellationToken cancellationToken = default) + { + var queryParameters = new List(); + + if (!string.IsNullOrEmpty(catalog_name)) + { + queryParameters.Add($"catalog_name={Uri.EscapeDataString(catalog_name)}"); + } + if (!string.IsNullOrEmpty(schema_name)) + { + queryParameters.Add($"schema_name={Uri.EscapeDataString(schema_name)}"); + } + if (max_results > 0) + { + queryParameters.Add($"max_results={max_results}"); + } + + var queryString = queryParameters.Count > 0 ? "?" + string.Join("&", queryParameters) : string.Empty; + var requestUri = $"{BaseUnityCatalogUri}/models{queryString}"; + + + var registeredModelsList = await HttpGet(HttpClient, requestUri, cancellationToken).ConfigureAwait(false); + + registeredModelsList.TryGetPropertyValue("registered_models", out var registeredModels); + return registeredModels?.Deserialize>(Options) ?? Enumerable.Empty(); + } + + public async Task GetRegisteredModel(string full_name, CancellationToken cancellationToken = default) + { + var requestUri = $"{BaseUnityCatalogUri}/models/{full_name}"; + return await HttpGet(HttpClient, requestUri, cancellationToken); + } + + public async Task SetRegisteredModelAlias( + string full_name, + string alias, + int version_num, + CancellationToken cancellationToken = default) + { + var requestUri = $"{BaseUnityCatalogUri}/models/{full_name}/aliases/{alias}"; + var request = new { version_num }; + return await HttpPut(HttpClient, requestUri, request, cancellationToken); + } +} diff --git a/csharp/Microsoft.Azure.Databricks.Client/UnityCatalogClient.cs b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalogClient.cs index 9a1ffb7..e78e240 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/UnityCatalogClient.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/UnityCatalogClient.cs @@ -1,4 +1,5 @@ -using Microsoft.Azure.Databricks.Client.UnityCatalog; +using Microsoft.Azure.Databricks.Client.MachineLearning; +using Microsoft.Azure.Databricks.Client.UnityCatalog; using Microsoft.Azure.Databricks.Client.UnityCatalog.Interfaces; using System; using System.Net.Http; @@ -23,6 +24,8 @@ public UnityCatalogClient(HttpClient httpClient) : base(httpClient) this.UnityCatalogPermissions = new UnityCatalogPermissionsApiClient(httpClient); this.Volumes = new VolumesApiClient(httpClient); this.Lineage = new LineageApiClient(httpClient); + this.ModelVersion = new ModelVersionApiClient(httpClient); + this.RegisteredModels = new RegisteredModelsApiClient(httpClient); } public virtual ICatalogsApi Catalogs { get; set; } @@ -52,4 +55,8 @@ public UnityCatalogClient(HttpClient httpClient) : base(httpClient) public virtual IVolumesApi Volumes { get; set; } public virtual ILineageApi Lineage { get; set; } + + public virtual IModelVersionApi ModelVersion { get; set; } + + public virtual IRegisteredModelsApi RegisteredModels { get; set; } }