Merge branch 'master' of https://github.com/drcrook1/DataBricks_ML_De…

…vOps

Project_One/notebooks/train_model.py

@@ -1,9 +1,7 @@
 # Databricks notebook source
 # FUTURE SERVICE PRINCIPAL STUFF FOR MOUNTING
 secrets = {}
-secrets["blob_name"] = dbutils.secrets.get(scope = "data-lake", key = "blob-name")
-secrets["blob_key"] = dbutils.secrets.get(scope = "data-lake", key = "blob-key") 
-secrets["container_name"] = dbutils.secrets.get(scope = "data-lake", key = "container-name")
+secrets["datalake_fqdn"] = dbutils.secrets.get(scope = "data-lake", key = "datalake-fqdn")
 secrets["subscription_id"] = dbutils.secrets.get(scope = "data-lake", key = "subscription-id")
 secrets["resource_group"] = dbutils.secrets.get(scope = "data-lake", key = "resource-group")
 secrets["ml_workspace_name"] = dbutils.secrets.get(scope = "data-lake", key = "ml-workspace-name")
@@ -12,30 +10,13 @@
 secrets["sp_password"] = dbutils.secrets.get(scope = "data-lake", key = "sp-password")
 secrets["sp_tenant_id"] = dbutils.secrets.get(scope = "data-lake", key = "sp-tenant-id")
 secrets["sp_token_endpoint"] = dbutils.secrets.get(scope = "data-lake", key = "sp-token-endpoint")
-try:
-  secrets["created_by"] = dbutils.secrets.get(scope = "data-lake", key = "created-by")
-except Exception as e:
-  print("falling back to user set created_by")
+secrets["created_by"] = dbutils.secrets.get(scope = "data-lake", key = "created-by")
+if(secrets["created_by"] == "dev"):
+  print("dev state, replacing with specific dev alias.")
   secrets["created_by"] = "dacrook"
 
 # COMMAND ----------
 
-#
-# THIS IS FOR BLOB MOUNTING
-#
-#create the config variable to pass into mount
-#configs = {"fs.azure.account.key." + secrets["blob_name"] + ".blob.core.windows.net":"" + secrets["blob_key"] + ""}
-
-#try:
-  #mounting the external blob storage as mount point datalake for data storage.
-#  dbutils.fs.mount( source = "wasbs://" + secrets["container_name"] + "@" + secrets["blob_name"] + ".blob.core.windows.net/", 
-#                   mount_point = "/mnt/datalake/", 
- #                  extra_configs = configs)
-#except Exception as e:
-#  print("already mounted; no need to do so.")
-
-# COMMAND ----------
-
 #
 # THIS IS FOR ADLS V2 Mounting
 #
@@ -48,7 +29,7 @@
 try:
   #mounting the external blob storage as mount point datalake for data storage.
   dbutils.fs.mount(
-    source = "abfss://" + secrets["container_name"] + "@" + secrets["storage_account_name"] + ".dfs.core.windows.net", #blobcontainername@storageaccount
+    source = secrets["datalake_fqdn"], #blobcontainername@storageaccount
     mount_point = "/mnt/datalake",
     extra_configs = configs)
 except Exception as e:
@@ -137,4 +118,4 @@
 try:
   dbutils.fs.unmount("/mnt/datalake")
 except Exception as e:
-  print("already unmounted; no need to unmount again.")
+  print("already unmounted; no need to unmount again.")