Skip to content

Commit

Permalink
[ADLS]Remove Secondary Endpoint for ADLS (Azure#10144)
Browse files Browse the repository at this point in the history
  • Loading branch information
xiafu-msft authored Mar 7, 2020
1 parent 8def83e commit ae25cea
Show file tree
Hide file tree
Showing 16 changed files with 399 additions and 344 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
"blob": {"primary": "BlobEndpoint", "secondary": "BlobSecondaryEndpoint"},
"queue": {"primary": "QueueEndpoint", "secondary": "QueueSecondaryEndpoint"},
"file": {"primary": "FileEndpoint", "secondary": "FileSecondaryEndpoint"},
"dfs": {"primary": "BlobEndpoint", "secondary": "BlobSecondaryEndpoint"},
"dfs": {"primary": "BlobEndpoint", "secondary": "BlobEndpoint"},
}


Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,7 @@ def from_connection_string(
:return a DataLakeFileClient
:rtype ~azure.storage.filedatalake.DataLakeFileClient
"""
account_url, secondary, credential = parse_connection_str(conn_str, credential, 'dfs')
if 'secondary_hostname' not in kwargs:
kwargs['secondary_hostname'] = secondary
account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs')
return cls(
account_url, file_system_name=file_system_name, file_path=file_path,
credential=credential, **kwargs)
Expand All @@ -114,24 +112,27 @@ def create_file(self, content_settings=None, # type: Optional[ContentSettings]
:param ~azure.storage.filedatalake.ContentSettings content_settings:
ContentSettings object used to set path properties.
:param metadata:
Name-value pairs associated with the blob as metadata.
Name-value pairs associated with the file as metadata.
:type metadata: dict(str, str)
:keyword ~azure.storage.filedatalake.DataLakeLeaseClient or str lease:
Required if the blob has an active lease. Value can be a DataLakeLeaseClient object
:keyword lease:
Required if the file has an active lease. Value can be a DataLakeLeaseClient object
or the lease ID as a string.
:keyword str umask: Optional and only valid if Hierarchical Namespace is enabled for the account.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword str umask:
Optional and only valid if Hierarchical Namespace is enabled for the account.
When creating a file or directory and the parent folder does not have a default ACL,
the umask restricts the permissions of the file or directory to be created.
The resulting permission is given by p & ^u, where p is the permission and u is the umask.
For example, if p is 0777 and u is 0057, then the resulting permission is 0720.
The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027.
The umask must be specified in 4-digit octal notation (e.g. 0766).
:keyword str permissions: Optional and only valid if Hierarchical Namespace
is enabled for the account. Sets POSIX access permissions for the file
owner, the file owning group, and others. Each class may be granted
read, write, or execute permission. The sticky bit is also supported.
Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are
supported.
:keyword str permissions:
Optional and only valid if Hierarchical Namespace
is enabled for the account. Sets POSIX access permissions for the file
owner, the file owning group, and others. Each class may be granted
read, write, or execute permission. The sticky bit is also supported.
Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are
supported.
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
Expand Down Expand Up @@ -161,9 +162,9 @@ def delete_file(self, **kwargs):
Marks the specified file for deletion.
:keyword lease:
Required if the blob has an active lease. Value can be a LeaseClient object
Required if the file has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.blob.LeaseClient or str
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
Expand Down Expand Up @@ -375,11 +376,11 @@ def append_data(self, data, # type: Union[AnyStr, Iterable[AnyStr], IO[AnyStr]]
with the hash that was sent. This is primarily valuable for detecting
bitflips on the wire if using http instead of https as https (the default)
will already validate. Note that this MD5 hash is not stored with the
blob.
file.
:keyword lease:
Required if the blob has an active lease. Value can be a LeaseClient object
Required if the file has an active lease. Value can be a DataLakeLeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:return: dict of the response header
"""
options = self._append_data_options(
Expand Down Expand Up @@ -489,9 +490,9 @@ def read_file(self, offset=None, # type: Optional[int]
:param int stream:
User provided stream to write the downloaded data into.
:keyword lease:
If specified, download_blob only succeeds if the blob's lease is active
and matches this ID. Required if the blob has an active lease.
:type lease: ~azure.storage.blob.LeaseClient or str
If specified, download only succeeds if the file's lease is active
and matches this ID. Required if the file has an active lease.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
Expand Down Expand Up @@ -550,19 +551,21 @@ def rename_file(self, new_name, # type: str
Required if the file/directory has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
:type lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword str umask: Optional and only valid if Hierarchical Namespace is enabled for the account.
:keyword str umask:
Optional and only valid if Hierarchical Namespace is enabled for the account.
When creating a file or directory and the parent folder does not have a default ACL,
the umask restricts the permissions of the file or directory to be created.
The resulting permission is given by p & ^u, where p is the permission and u is the umask.
For example, if p is 0777 and u is 0057, then the resulting permission is 0720.
The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027.
The umask must be specified in 4-digit octal notation (e.g. 0766).
:keyword permissions: Optional and only valid if Hierarchical Namespace
is enabled for the account. Sets POSIX access permissions for the file
owner, the file owning group, and others. Each class may be granted
read, write, or execute permission. The sticky bit is also supported.
Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are
supported.
:keyword permissions:
Optional and only valid if Hierarchical Namespace
is enabled for the account. Sets POSIX access permissions for the file
owner, the file owning group, and others. Each class may be granted
read, write, or execute permission. The sticky bit is also supported.
Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are
supported.
:type permissions: str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ def acquire(self, lease_duration=-1, **kwargs):
# type: (int, Optional[int], **Any) -> None
"""Requests a new lease.
If the container does not have an active lease, the Blob service creates a
lease on the container and returns a new lease ID.
If the file/file system does not have an active lease, the DataLake service creates a
lease on the file/file system and returns a new lease ID.
:param int lease_duration:
Specifies the duration of the lease, in seconds, or negative one
Expand Down Expand Up @@ -107,9 +107,9 @@ def renew(self, **kwargs):
"""Renews the lease.
The lease can be renewed if the lease ID specified in the
lease client matches that associated with the container or blob. Note that
the lease may be renewed even if it has expired as long as the container
or blob has not been leased again since the expiration of that lease. When you
lease client matches that associated with the file system or file. Note that
the lease may be renewed even if it has expired as long as the file system
or file has not been leased again since the expiration of that lease. When you
renew a lease, the lease duration clock resets.
:keyword ~datetime.datetime if_modified_since:
Expand Down Expand Up @@ -141,8 +141,8 @@ def release(self, **kwargs):
"""Release the lease.
The lease may be released if the client lease id specified matches
that associated with the container or blob. Releasing the lease allows another client
to immediately acquire the lease for the container or blob as soon as the release is complete.
that associated with the file system or file. Releasing the lease allows another client
to immediately acquire the lease for the file system or file as soon as the release is complete.
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
Expand Down Expand Up @@ -173,7 +173,7 @@ def change(self, proposed_lease_id, **kwargs):
"""Change the lease ID of an active lease.
:param str proposed_lease_id:
Proposed lease ID, in a GUID string format. The Blob service returns 400
Proposed lease ID, in a GUID string format. The DataLake service returns 400
(Invalid request) if the proposed lease ID is not in the correct format.
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
Expand Down Expand Up @@ -201,12 +201,12 @@ def change(self, proposed_lease_id, **kwargs):

def break_lease(self, lease_break_period=None, **kwargs):
# type: (Optional[int], Any) -> int
"""Break the lease, if the container or blob has an active lease.
"""Break the lease, if the file system or file has an active lease.
Once a lease is broken, it cannot be renewed. Any authorized request can break the lease;
the request is not required to specify a matching lease ID. When a lease
is broken, the lease break period is allowed to elapse, during which time
no lease operation except break and release can be performed on the container or blob.
no lease operation except break and release can be performed on the file system or file.
When a lease is successfully broken, the response indicates the interval
in seconds until a new lease can be acquired.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from ._file_system_client import FileSystemClient
from ._data_lake_directory_client import DataLakeDirectoryClient
from ._data_lake_file_client import DataLakeFileClient
from ._models import UserDelegationKey, FileSystemPropertiesPaged
from ._models import UserDelegationKey, FileSystemPropertiesPaged, LocationMode
from ._serialize import convert_dfs_url_to_blob_url


Expand All @@ -29,8 +29,7 @@ class DataLakeServiceClient(StorageAccountHostsMixin):
can also be retrieved using the `get_client` functions.
:ivar str url:
The full endpoint URL to the datalake service endpoint. This could be either the
primary endpoint, or the secondary endpoint depending on the current `location_mode`.
The full endpoint URL to the datalake service endpoint.
:ivar str primary_endpoint:
The full primary endpoint URL.
:ivar str primary_hostname:
Expand Down Expand Up @@ -80,16 +79,18 @@ def __init__(
blob_account_url = convert_dfs_url_to_blob_url(account_url)
self._blob_account_url = blob_account_url
self._blob_service_client = BlobServiceClient(blob_account_url, credential, **kwargs)
self._blob_service_client._hosts[LocationMode.SECONDARY] = "" #pylint: disable=protected-access

_, sas_token = parse_query(parsed_url.query)
self._query_str, self._raw_credential = self._format_query_string(sas_token, credential)

super(DataLakeServiceClient, self).__init__(parsed_url, service='dfs',
credential=self._raw_credential, **kwargs)
# ADLS doesn't support secondary endpoint, make sure it's empty
self._hosts[LocationMode.SECONDARY] = ""

def _format_url(self, hostname):
"""Format the endpoint URL according to the current location
mode hostname.
"""Format the endpoint URL according to hostname
"""
formated_url = "{}://{}/{}".format(self.scheme, hostname, self._query_str)
return formated_url
Expand All @@ -114,9 +115,7 @@ def from_connection_string(
:return a DataLakeServiceClient
:rtype ~azure.storage.filedatalake.DataLakeServiceClient
"""
account_url, secondary, credential = parse_connection_str(conn_str, credential, 'dfs')
if 'secondary_hostname' not in kwargs:
kwargs['secondary_hostname'] = secondary
account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs')
return cls(account_url, credential=credential, **kwargs)

def get_user_delegation_key(self, key_start_time, # type: datetime
Expand Down Expand Up @@ -191,7 +190,8 @@ def create_file_system(self, file_system, # type: Union[FileSystemProperties, s
be raised. This method returns a client with which to interact with the newly
created file system.
:param str file_system: The name of the file system to create.
:param str file_system:
The name of the file system to create.
:param metadata:
A dict with name-value pairs to associate with the
file system as metadata. Example: `{'Category':'test'}`
Expand Down Expand Up @@ -228,10 +228,11 @@ def delete_file_system(self, file_system, # type: Union[FileSystemProperties, s
The file system to delete. This can either be the name of the file system,
or an instance of FileSystemProperties.
:type file_system: str or ~azure.storage.filedatalake.FileSystemProperties
:keyword ~azure.storage.filedatalake.DataLakeLeaseClient lease:
:keyword lease:
If specified, delete_file_system only succeeds if the
file system's lease is active and matches this ID.
Required if the file system has an active lease.
:paramtype lease: ~azure.storage.filedatalake.DataLakeLeaseClient or str
:keyword ~datetime.datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
Expand Down Expand Up @@ -290,7 +291,7 @@ def get_file_system_client(self, file_system # type: Union[FileSystemProperties
:caption: Getting the file system client to interact with a specific file system.
"""
return FileSystemClient(self.url, file_system, credential=self._raw_credential, _configuration=self._config,
_pipeline=self._pipeline, _location_mode=self._location_mode, _hosts=self._hosts,
_pipeline=self._pipeline, _hosts=self._hosts,
require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key,
key_resolver_function=self.key_resolver_function)

Expand Down Expand Up @@ -325,7 +326,7 @@ def get_directory_client(self, file_system, # type: Union[FileSystemProperties,
return DataLakeDirectoryClient(self.url, file_system, directory_name=directory,
credential=self._raw_credential,
_configuration=self._config, _pipeline=self._pipeline,
_location_mode=self._location_mode, _hosts=self._hosts,
_hosts=self._hosts,
require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
key_resolver_function=self.key_resolver_function
Expand Down Expand Up @@ -367,6 +368,6 @@ def get_file_client(self, file_system, # type: Union[FileSystemProperties, str]
return DataLakeFileClient(
self.url, file_system, file_path=file_path, credential=self._raw_credential,
_hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline,
_location_mode=self._location_mode, require_encryption=self.require_encryption,
require_encryption=self.require_encryption,
key_encryption_key=self.key_encryption_key,
key_resolver_function=self.key_resolver_function)
Loading

0 comments on commit ae25cea

Please sign in to comment.