Merge pull request #2484 from srsiva/master

SDK changes to support ADLS as option for default storage

src/ResourceManagement/AzureStackAdmin/AzureStackManagement/Microsoft.AzureStack.Management.nuget.proj

@@ -1,12 +1,12 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ItemGroup>
-    <!--
-    Microsoft.AzureStack.Management
-    -->
-    <SdkNuGetPackage Include="Microsoft.AzureStack.Management">
-      <PackageVersion>0.10.1-preview</PackageVersion>
-      <Folder>$(MSBuildThisFileDirectory)</Folder>
-    </SdkNuGetPackage>
-  </ItemGroup>
-</Project>
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <!--
+    Microsoft.AzureStack.Management
+    -->
+    <SdkNuGetPackage Include="Microsoft.AzureStack.Management">
+      <PackageVersion>0.10.1-preview</PackageVersion>
+      <Folder>$(MSBuildThisFileDirectory)</Folder>
+    </SdkNuGetPackage>
+  </ItemGroup>
+</Project>

src/ResourceManagement/HDInsight/HDInsight.Tests/HDInsight.Tests.csproj

@@ -22,6 +22,8 @@
     <Compile Include="ScenarioTests\ListClusterTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="ScenarioTests\ScriptActionsOnRunningCluster.cs" />
+    <Compile Include="UnitTests\ClusterCreateTests.cs" />
+    <Compile Include="UnitTests\StorageAccountTests.cs" />
     <Compile Include="UnitTests\ClusterCreateDefaultVmTypeTests.cs" />
     <Compile Include="UnitTests\MetastoreTests.cs" />
   </ItemGroup>
@@ -69,6 +71,9 @@
     <None Include="SessionRecords\HDInsight.Tests.DataLakeTests\TestCreateDataLakeClusterUsingClusterParameters.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\HDInsight.Tests.DataLakeTests\TestCreateDefaultFsDataLakeClusterUsingClusterParameters.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\HDInsight.Tests.GetCapabilitiesTests\TestGetCapabilities.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -144,4 +149,4 @@
       <Name>HDInsightManagement</Name>
     </ProjectReference>
   </ItemGroup>
-</Project>
+</Project>

src/ResourceManagement/HDInsight/HDInsight.Tests/Helpers/GetClusterSpecHelpers.cs

@@ -22,6 +22,7 @@ namespace HDInsight.Tests.Helpers
 {
     public static class GetClusterSpecHelpers
     {
+        private const string ADLDefaultStorageAccountName = "";
         private const string DefaultContainer = "";
         private const string StorageAccountName = "";
         private const string StorageAccountKey = "";
@@ -227,13 +228,12 @@ public static ClusterCreateParameters GetCustomCreateParametersPaas()
                 ClusterSizeInNodes = 3,
                 ClusterType = "Hadoop",
                 WorkerNodeSize = "Large",
-                DefaultStorageAccountName = StorageAccountName,
-                DefaultStorageAccountKey = StorageAccountKey,
+                DefaultStorageInfo = GetDefaultAzureStorageInfo(),
                 OSType = OSType.Windows,
                 UserName = HttpUser,
                 Password = HttpPassword,
-                DefaultStorageContainer = DefaultContainer,
-                Location =  "West US"
+                Location = "East US",
+                Version = "3.2"
             };
             var actions = new List<ScriptAction>();
             var action = new ScriptAction("action", new Uri("https://uri.com"), "params");
@@ -251,17 +251,47 @@ public static ClusterCreateParameters GetCustomCreateParametersIaas()
                 ClusterSizeInNodes = 3,
                 ClusterType = "Hadoop",
                 WorkerNodeSize = "Large",
-                DefaultStorageAccountName = StorageAccountName,
-                DefaultStorageAccountKey = StorageAccountKey,
+                DefaultStorageInfo = GetDefaultAzureStorageInfo(),
                 OSType = OSType.Linux,
                 UserName = HttpUser,
                 Password = HttpPassword,
-                DefaultStorageContainer = DefaultContainer,
-                Location = "West US",
+                Location = "East US",
+                SshUserName = SshUser,
+                SshPassword = SshPassword,
+                Version = "3.2"
+            };
+            return clusterparams;
+        }
+
+        public static ClusterCreateParameters GetDataLakeDefaultFsCreateParametersIaas()
+        {
+            var storageInfo = GetDefaultAzureDataLakeStoreInfo();
+            return GetDefaultFsCreateParametersIaas(storageInfo);
+        }
+
+        public static ClusterCreateParameters GetAzureBlobDefaultFsCreateParametersIaas(bool specifyDefaultContainer = true)
+        {
+            var storageInfo = GetDefaultAzureStorageInfo(specifyDefaultContainer);
+            return GetDefaultFsCreateParametersIaas(storageInfo);
+        }
+
+        private static ClusterCreateParameters GetDefaultFsCreateParametersIaas(StorageInfo defaultStorageInfo)
+        {
+            var clusterparams = new ClusterCreateParameters
+            {
+                ClusterSizeInNodes = 3,
+                ClusterType = "Hadoop",
+                WorkerNodeSize = "Large",
+                DefaultStorageInfo = defaultStorageInfo,
+                OSType = OSType.Linux,
+                UserName = HttpUser,
+                Password = HttpPassword,
+                Location = "East US",
                 SshUserName = SshUser,
                 SshPassword = SshPassword,
                 Version = "3.2"
             };
+
             return clusterparams;
         }
 
@@ -272,13 +302,11 @@ public static ClusterCreateParameters GetCustomVmSizesCreateParametersIaas()
                 ClusterSizeInNodes = 1,
                 ClusterType = "HBase",
                 WorkerNodeSize = "Large",
-                DefaultStorageAccountName = StorageAccountName,
-                DefaultStorageAccountKey = StorageAccountKey,
+                DefaultStorageInfo = GetDefaultAzureStorageInfo(),
                 OSType = OSType.Linux,
                 UserName = HttpUser,
                 Password = HttpPassword,
-                DefaultStorageContainer = DefaultContainer,
-                Location = "West US",
+                Location = "East US",
                 SshUserName = SshUser,
                 SshPassword = SshPassword,
                 Version = "3.2",
@@ -329,5 +357,47 @@ public static ClusterCreateParametersExtended AddConfigurations(ClusterCreatePar
 
             return cluster;
         }
+
+        /// <summary>
+        /// Returns appropriate AzureStorageInfo based on test-mode.
+        /// </summary>
+        /// <returns></returns>
+        private static StorageInfo GetDefaultAzureStorageInfo(bool specifyDefaultContainer=true)
+        {
+            bool recordMode = HDInsightManagementTestUtilities.IsRecordMode();
+
+            if(recordMode)
+            {
+                return (specifyDefaultContainer) 
+                    ? new AzureStorageInfo(StorageAccountName, StorageAccountKey, DefaultContainer) 
+                    : new AzureStorageInfo(StorageAccountName, StorageAccountKey);
+            }
+            else 
+            {
+                string testStorageAccountName = "tmp.blob.core.windows.net";
+                string testStorageAccountKey = "teststorageaccountkey";
+                string testContainer = "testdefaultcontainer";
+
+                return (specifyDefaultContainer) 
+                    ? new AzureStorageInfo(testStorageAccountName, testStorageAccountKey, testContainer)
+                    : new AzureStorageInfo(testStorageAccountName, testStorageAccountKey);
+            }
+        }
+
+        /// <summary>
+        /// Returns appropriate AzureDataLakeStoreInfo based on test-mode.
+        /// </summary>
+        /// <returns></returns>
+        private static StorageInfo GetDefaultAzureDataLakeStoreInfo()
+        {
+            bool recordMode = HDInsightManagementTestUtilities.IsRecordMode();
+            string ADLClusterRootPath = "/Clusters/SDK";
+
+            return recordMode
+               ? new AzureDataLakeStoreInfo(ADLDefaultStorageAccountName, ADLClusterRootPath)
+               : new AzureDataLakeStoreInfo("tmp.azuredatalakestore.net", ADLClusterRootPath);
+        }
+
+
     }
 }

src/ResourceManagement/HDInsight/HDInsight.Tests/Helpers/HDInsightManagementTestUtilities.cs

@@ -17,6 +17,7 @@
 using Microsoft.Azure.Management.Resources;
 using Microsoft.Azure.Management.Resources.Models;
 using Microsoft.Azure.Test;
+using System;
 
 namespace HDInsight.Tests.Helpers
 {
@@ -85,5 +86,12 @@ public static void WaitForClusterToMoveToRunning(string resourceGroup, string dn
 
             Xunit.Assert.True(!createError);
         }
+
+        public static bool IsRecordMode()
+        {
+            string testMode = System.Environment.GetEnvironmentVariable("AZURE_TEST_MODE");
+            bool recordMode = !string.IsNullOrEmpty(testMode) && testMode.Equals("Record", StringComparison.OrdinalIgnoreCase);
+            return recordMode;
+        }
     }
 }

src/ResourceManagement/HDInsight/HDInsight.Tests/ScenarioTests/DataLakeTests.cs

@@ -103,28 +103,85 @@ public void TestCreateDataLakeClusterUsingClusterParameters()
                 //set variables
                 const string dnsname = "hdisdk-datalake5";
 
-                var spec = GetDataLakeClusterParameters();
-                spec.Version = "3.2";
+                var clusterCreateParams = GetDataLakeClusterParameters();
 
-                var createresponse = client.Clusters.Create(resourceGroup, dnsname, spec);
+                var createresponse = client.Clusters.Create(resourceGroup, dnsname, clusterCreateParams);
+                Assert.Equal(dnsname, createresponse.Cluster.Name);
 
                 //TODO: Assert if data lake configurations are set once shefali adds GetConfigurations support
                 var getresponse = client.Clusters.Get(resourceGroup, dnsname);
                 Assert.Equal(createresponse.Cluster.Properties.CreatedDate, getresponse.Cluster.Properties.CreatedDate);
                 Assert.Equal(createresponse.Cluster.Name, getresponse.Cluster.Name);
 
-                OperationResource result = client.Clusters.Delete(resourceGroup, dnsname);
+                // Assert cluster state
+                Assert.Equal(createresponse.Cluster.Properties.ClusterState, "Error"); // due to invalid script action
+                Assert.Equal(createresponse.Cluster.Properties.ErrorInfos.Count, 1);
+
+                // delete the cluster
+                var result = client.Clusters.Delete(resourceGroup, dnsname);
+                Assert.Equal(result.StatusCode, HttpStatusCode.OK);
+                Assert.Equal(result.State, AsyncOperationState.Succeeded);
             }
         }
 
-        private ClusterCreateParameters GetDataLakeClusterParameters()
+        [Fact]
+        public void TestCreateDefaultFsDataLakeClusterUsingClusterParameters()
         {
-            var spec = GetClusterSpecHelpers.GetCustomCreateParametersPaas();
+            var handler = new RecordedDelegatingHandler { StatusCodeToReturn = HttpStatusCode.OK };
 
-            ServicePrincipal servicePrincipal = new ServicePrincipal(new Guid(ApplicationId), new Guid(AadTenantId), CertificateFileBytes,
-                CertificatePassword);
-            spec.Principal = servicePrincipal;
-            return spec;
+            using (var context = UndoContext.Current)
+            {
+                context.Start();
+
+                //get clients
+                var client = HDInsightManagementTestUtilities.GetHDInsightManagementClient(handler);
+                var resourceManagementClient = HDInsightManagementTestUtilities.GetResourceManagementClient(handler);
+
+                //create resourcegroup
+                var resourceGroup = HDInsightManagementTestUtilities.CreateResourceGroup(resourceManagementClient);
+
+                //set variables
+                const string dnsname = "hdisdk-defaultfsdatalake1";
+
+                var clusterCreateParams = GetDefaultFsDatalakeClusterParameters();
+
+                var createresponse = client.Clusters.Create(resourceGroup, dnsname, clusterCreateParams);
+                Assert.Equal(dnsname, createresponse.Cluster.Name);
+                Assert.Equal(createresponse.Cluster.Properties.ClusterState, "Running");
+
+                var getresponse = client.Clusters.Get(resourceGroup, dnsname);
+                Assert.Equal(createresponse.Cluster.Properties.CreatedDate, getresponse.Cluster.Properties.CreatedDate);
+                Assert.Equal(createresponse.Cluster.Name, getresponse.Cluster.Name);
+
+                // delete the cluster
+                var result = client.Clusters.Delete(resourceGroup, dnsname);
+                Assert.Equal(result.StatusCode, HttpStatusCode.OK);
+                Assert.Equal(result.State, AsyncOperationState.Succeeded);
+            }
+        }
+
+        /// <summary>
+        /// ClusterCreateParameters used for DataLake default FS
+        /// </summary>
+        /// <returns></returns>
+        private ClusterCreateParameters GetDefaultFsDatalakeClusterParameters()
+        {
+            var clusterCreateParams = GetClusterSpecHelpers.GetDataLakeDefaultFsCreateParametersIaas();
+            clusterCreateParams.Principal = new ServicePrincipal(new Guid(ApplicationId), new Guid(AadTenantId), 
+                                                                    CertificateFileBytes, CertificatePassword);
+            return clusterCreateParams;
+        }
+
+        /// <summary>
+        /// ClusterCreateParameters used for DataLake additional FS
+        /// </summary>
+        /// <returns></returns>
+        private ClusterCreateParameters GetDataLakeClusterParameters()
+        {
+            var clusterCreateParams = GetClusterSpecHelpers.GetCustomCreateParametersPaas();
+            clusterCreateParams.Principal = new ServicePrincipal(new Guid(ApplicationId), new Guid(AadTenantId), 
+                                                                    CertificateFileBytes, CertificatePassword);
+            return clusterCreateParams;
         }
     }
 }

src/ResourceManagement/HDInsight/HDInsight.Tests/ScenarioTests/ScriptActionsOnRunningCluster.cs

@@ -112,9 +112,9 @@ public void TestScriptActionsOnRunningCluster()
                     string failingScriptUri = "http://bing.com";
 
                     //this is set only for RECORD mode, playback this uri doesnt matter
-                    if (!string.IsNullOrEmpty(clusterCreateParams.DefaultStorageAccountName))
+                    if (HDInsightManagementTestUtilities.IsRecordMode())
                     {
-                        failingScriptUri = string.Format(FailingScriptLocationFormat, clusterCreateParams.DefaultStorageAccountName, FailingScriptLocationContainer);
+                        failingScriptUri = string.Format(FailingScriptLocationFormat, clusterCreateParams.DefaultStorageInfo.StorageAccountName, FailingScriptLocationContainer);
                     }
 
                     var executeScriptActionParams = GetExecuteScriptActionParams(true, scriptName, failingScriptUri);
@@ -234,14 +234,14 @@ private ClusterCreateParameters CreateClusterToValidateScriptActions(string reso
             client.Clusters.Create(resourceGroup, dnsName, clusterCreateParams);
 
             HDInsightManagementTestUtilities.WaitForClusterToMoveToRunning(resourceGroup, dnsName, client);
-
-            string storageAccountName =clusterCreateParams.DefaultStorageAccountName.Split('.')[0];
 
-            //upload only in record mode, storageaccount will be empty in playback mode
-            if (!string.IsNullOrEmpty(storageAccountName))
+            // upload only in record mode
+            if (HDInsightManagementTestUtilities.IsRecordMode())
             {
                 //upload failing script to the cluster  default storage account
-                var creds = new StorageCredentials(storageAccountName, clusterCreateParams.DefaultStorageAccountKey);
+                var defaultStorageAccount = clusterCreateParams.DefaultStorageInfo as AzureStorageInfo;
+                var storageAccountName = defaultStorageAccount.StorageAccountName.Split('.')[0];
+                var creds = new StorageCredentials(storageAccountName, defaultStorageAccount.StorageAccountKey);
 
                 var storageAccount = new CloudStorageAccount(creds, true);
                 var blobClient = storageAccount.CreateCloudBlobClient();