From fc06ee72eee1785915c17eef20c6fd3463c3a653 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sat, 23 May 2026 07:29:37 -0400 Subject: [PATCH 1/4] Formatted Datalake --- .../azure/__init__.py | 2 +- .../azure/storage/__init__.py | 2 +- .../azure/storage/filedatalake/__init__.py | 108 ++-- .../_data_lake_directory_client.py | 131 ++-- .../filedatalake/_data_lake_file_client.py | 120 ++-- .../filedatalake/_data_lake_file_client.pyi | 6 +- .../_data_lake_file_client_helpers.py | 96 +-- .../storage/filedatalake/_data_lake_lease.py | 14 +- .../storage/filedatalake/_data_lake_lease.pyi | 7 +- .../filedatalake/_data_lake_service_client.py | 108 ++-- .../_data_lake_service_client_helpers.py | 4 +- .../storage/filedatalake/_deserialize.py | 174 +++-- .../azure/storage/filedatalake/_download.py | 14 +- .../filedatalake/_file_system_client.py | 189 +++--- .../_file_system_client_helpers.py | 16 +- .../filedatalake/_list_paths_helper.py | 49 +- .../azure/storage/filedatalake/_models.py | 426 +++++++------ .../storage/filedatalake/_path_client.py | 170 ++--- .../filedatalake/_path_client_helpers.py | 170 ++--- .../filedatalake/_quick_query_helper.py | 5 +- .../azure/storage/filedatalake/_serialize.py | 126 ++-- .../filedatalake/_shared/base_client.py | 8 +- .../filedatalake/_shared/base_client_async.py | 2 +- .../storage/filedatalake/_shared/constants.py | 1 - .../filedatalake/_shared/request_handlers.py | 1 - .../filedatalake/_shared/response_handlers.py | 8 +- .../storage/filedatalake/_shared/uploads.py | 1 - .../filedatalake/_shared_access_signature.py | 11 +- .../storage/filedatalake/_upload_helper.py | 102 +-- .../storage/filedatalake/aio/__init__.py | 16 +- .../aio/_data_lake_directory_client_async.py | 127 ++-- .../aio/_data_lake_file_client_async.py | 112 ++-- .../aio/_data_lake_file_client_async.pyi | 6 +- .../aio/_data_lake_lease_async.py | 14 +- .../aio/_data_lake_lease_async.pyi | 10 +- .../aio/_data_lake_service_client_async.py | 123 ++-- .../filedatalake/aio/_download_async.py | 14 +- .../aio/_file_system_client_async.py | 199 +++--- .../filedatalake/aio/_list_paths_helper.py | 51 +- .../azure/storage/filedatalake/aio/_models.py | 5 +- .../filedatalake/aio/_path_client_async.py | 183 +++--- .../filedatalake/aio/_upload_helper.py | 102 +-- .../azure-storage-file-datalake/conftest.py | 1 + .../datalake_samples_access_control.py | 25 +- .../datalake_samples_access_control_async.py | 23 +- ...talake_samples_access_control_recursive.py | 73 ++- ..._samples_access_control_recursive_async.py | 72 ++- .../samples/datalake_samples_directory.py | 20 +- .../datalake_samples_directory_async.py | 22 +- .../samples/datalake_samples_file_system.py | 18 +- .../datalake_samples_file_system_async.py | 20 +- .../datalake_samples_instantiate_client.py | 7 +- ...talake_samples_instantiate_client_async.py | 8 +- .../samples/datalake_samples_query.py | 81 ++- .../samples/datalake_samples_service.py | 34 +- .../samples/datalake_samples_service_async.py | 32 +- .../datalake_samples_upload_download.py | 24 +- .../datalake_samples_upload_download_async.py | 25 +- .../azure-storage-file-datalake/setup.py | 66 +- .../tests/perfstress_tests/_test_base.py | 28 +- .../tests/perfstress_tests/append.py | 10 +- .../tests/perfstress_tests/upload.py | 12 +- .../perfstress_tests/upload_from_file.py | 14 +- .../tests/settings/settings_fake.py | 2 +- .../tests/settings/testcase.py | 37 +- .../tests/test_cpk.py | 37 +- .../tests/test_cpk_async.py | 32 +- .../tests/test_datalake_service_client.py | 131 ++-- .../test_datalake_service_client_async.py | 129 ++-- .../tests/test_directory.py | 506 ++++++++------- .../tests/test_directory_async.py | 518 ++++++++------- .../tests/test_file.py | 600 +++++++++--------- .../tests/test_file_async.py | 481 +++++++------- .../tests/test_file_system.py | 223 ++++--- .../tests/test_file_system_async.py | 260 ++++---- .../tests/test_helpers.py | 37 +- .../tests/test_helpers_async.py | 44 +- .../tests/test_large_file.py | 39 +- .../tests/test_large_file_async.py | 34 +- .../tests/test_quick_query.py | 508 +++++++-------- 80 files changed, 3669 insertions(+), 3597 deletions(-) diff --git a/sdk/storage/azure-storage-file-datalake/azure/__init__.py b/sdk/storage/azure-storage-file-datalake/azure/__init__.py index 0d1f7edf5dc6..d55ccad1f573 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/__init__.py +++ b/sdk/storage/azure-storage-file-datalake/azure/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/__init__.py b/sdk/storage/azure-storage-file-datalake/azure/storage/__init__.py index 0d1f7edf5dc6..d55ccad1f573 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/__init__.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/__init__.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/__init__.py index 1dbc506484f6..904ff89f01f0 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/__init__.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/__init__.py @@ -47,8 +47,12 @@ UserDelegationKey, ) -from ._shared_access_signature import generate_account_sas, generate_file_system_sas, generate_directory_sas, \ - generate_file_sas +from ._shared_access_signature import ( + generate_account_sas, + generate_file_system_sas, + generate_directory_sas, + generate_file_sas, +) from ._shared.policies import ExponentialRetry, LinearRetry from ._shared.models import StorageErrorCode, Services @@ -57,54 +61,54 @@ __version__ = VERSION __all__ = [ - 'AccessControlChangeCounters', - 'AccessControlChangeFailure', - 'AccessControlChangeResult', - 'AccessControlChanges', - 'AccessPolicy', - 'AccountSasPermissions', - 'AnalyticsLogging', - 'ArrowDialect', - 'ArrowType', - 'ContentSettings', - 'CorsRule', - 'CustomerProvidedEncryptionKey', - 'DataLakeDirectoryClient', - 'DataLakeFileClient', - 'DataLakeFileQueryError', - 'DataLakeFileQueryError', - 'DataLakeLeaseClient', - 'DataLakeServiceClient', - 'DeletedPathProperties', - 'DelimitedJsonDialect', - 'DelimitedTextDialect', - 'DirectoryProperties', - 'DirectorySasPermissions', - 'EncryptionScopeOptions', - 'ExponentialRetry', - 'FileProperties', - 'FileSasPermissions', - 'FileSystemClient', - 'FileSystemProperties', - 'FileSystemPropertiesPaged', - 'FileSystemSasPermissions', - 'generate_account_sas', - 'generate_directory_sas', - 'generate_file_sas', - 'generate_file_system_sas', - 'LeaseProperties', - 'LinearRetry', - 'LocationMode', - 'Metrics', - 'PathProperties', - 'PublicAccess', - 'QuickQueryDialect', - 'ResourceTypes', - 'RetentionPolicy', - 'StaticWebsite', - 'StorageErrorCode', - 'StorageStreamDownloader', - 'UserDelegationKey', - 'VERSION', - 'Services' + "AccessControlChangeCounters", + "AccessControlChangeFailure", + "AccessControlChangeResult", + "AccessControlChanges", + "AccessPolicy", + "AccountSasPermissions", + "AnalyticsLogging", + "ArrowDialect", + "ArrowType", + "ContentSettings", + "CorsRule", + "CustomerProvidedEncryptionKey", + "DataLakeDirectoryClient", + "DataLakeFileClient", + "DataLakeFileQueryError", + "DataLakeFileQueryError", + "DataLakeLeaseClient", + "DataLakeServiceClient", + "DeletedPathProperties", + "DelimitedJsonDialect", + "DelimitedTextDialect", + "DirectoryProperties", + "DirectorySasPermissions", + "EncryptionScopeOptions", + "ExponentialRetry", + "FileProperties", + "FileSasPermissions", + "FileSystemClient", + "FileSystemProperties", + "FileSystemPropertiesPaged", + "FileSystemSasPermissions", + "generate_account_sas", + "generate_directory_sas", + "generate_file_sas", + "generate_file_system_sas", + "LeaseProperties", + "LinearRetry", + "LocationMode", + "Metrics", + "PathProperties", + "PublicAccess", + "QuickQueryDialect", + "ResourceTypes", + "RetentionPolicy", + "StaticWebsite", + "StorageErrorCode", + "StorageStreamDownloader", + "UserDelegationKey", + "VERSION", + "Services", ] diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py index 26bed950dba6..55c5b0f78822 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py @@ -6,10 +6,7 @@ # pylint: disable=docstring-keyword-should-match-keyword-only import functools -from typing import ( - Any, cast, Dict, Optional, Union, - TYPE_CHECKING -) +from typing import Any, cast, Dict, Optional, Union, TYPE_CHECKING from urllib.parse import quote, unquote from typing_extensions import Self @@ -84,22 +81,29 @@ class DataLakeDirectoryClient(PathClient): """The hostname of the primary endpoint.""" def __init__( - self, account_url: str, + self, + account_url: str, file_system_name: str, directory_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> None: - super(DataLakeDirectoryClient, self).__init__(account_url, file_system_name, path_name=directory_name, - credential=credential, **kwargs) + super(DataLakeDirectoryClient, self).__init__( + account_url, file_system_name, path_name=directory_name, credential=credential, **kwargs + ) @classmethod def from_connection_string( - cls, conn_str: str, + cls, + conn_str: str, file_system_name: str, directory_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> Self: """ Create DataLakeDirectoryClient from a Connection String. @@ -135,15 +139,18 @@ def from_connection_string( :return: A DataLakeDirectoryClient. :rtype: ~azure.storage.filedatalake.DataLakeDirectoryClient """ - account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs') + account_url, _, credential = parse_connection_str(conn_str, credential, "dfs") return cls( - account_url, file_system_name=file_system_name, directory_name=directory_name, - credential=credential, **kwargs) + account_url, + file_system_name=file_system_name, + directory_name=directory_name, + credential=credential, + **kwargs, + ) @distributed_trace def create_directory( - self, metadata: Optional[Dict[str, str]] = None, - **kwargs: Any + self, metadata: Optional[Dict[str, str]] = None, **kwargs: Any ) -> Dict[str, Union[str, "datetime"]]: """ Create a new directory. @@ -227,7 +234,7 @@ def create_directory( :dedent: 8 :caption: Create directory. """ - return self._create('directory', metadata=metadata, **kwargs) + return self._create("directory", metadata=metadata, **kwargs) @distributed_trace def delete_directory(self, **kwargs: Any) -> None: @@ -332,11 +339,11 @@ def get_directory_properties(self, **kwargs: Any) -> DirectoryProperties: :dedent: 4 :caption: Getting the properties for a file/directory. """ - upn = kwargs.pop('upn', None) + upn = kwargs.pop("upn", None) if upn: - headers = kwargs.pop('headers', {}) - headers['x-ms-upn'] = str(upn) - kwargs['headers'] = headers + headers = kwargs.pop("headers", {}) + headers["x-ms-upn"] = str(upn) + kwargs["headers"] = headers return cast(DirectoryProperties, self._get_path_properties(cls=deserialize_dir_properties, **kwargs)) @distributed_trace @@ -425,21 +432,26 @@ def rename_directory(self, new_name: str, **kwargs: Any) -> "DataLakeDirectoryCl :caption: Rename the source directory. """ new_file_system, new_path, new_dir_sas = _parse_rename_path( - new_name, self.file_system_name, self._query_str, self._raw_credential) + new_name, self.file_system_name, self._query_str, self._raw_credential + ) new_directory_client = DataLakeDirectoryClient( - f"{self.scheme}://{self.primary_hostname}", new_file_system, directory_name=new_path, - credential=self._raw_credential or new_dir_sas, _hosts=self._hosts, _configuration=self._config, - _pipeline=self._pipeline) + f"{self.scheme}://{self.primary_hostname}", + new_file_system, + directory_name=new_path, + credential=self._raw_credential or new_dir_sas, + _hosts=self._hosts, + _configuration=self._config, + _pipeline=self._pipeline, + ) new_directory_client._rename_path( # pylint: disable=protected-access - f'/{quote(unquote(self.file_system_name))}/{quote(unquote(self.path_name))}{self._query_str}', **kwargs) + f"/{quote(unquote(self.file_system_name))}/{quote(unquote(self.path_name))}{self._query_str}", **kwargs + ) return new_directory_client @distributed_trace def create_sub_directory( - self, sub_directory: Union[DirectoryProperties, str], - metadata: Optional[Dict[str, str]] = None, - **kwargs: Any + self, sub_directory: Union[DirectoryProperties, str], metadata: Optional[Dict[str, str]] = None, **kwargs: Any ) -> "DataLakeDirectoryClient": """ Create a subdirectory and return the subdirectory client to be interacted with. @@ -524,8 +536,7 @@ def create_sub_directory( @distributed_trace def delete_sub_directory( # pylint: disable=delete-operation-wrong-return-type - self, sub_directory: Union[DirectoryProperties, str], - **kwargs: Any + self, sub_directory: Union[DirectoryProperties, str], **kwargs: Any ) -> "DataLakeDirectoryClient": """ Marks the specified subdirectory for deletion. @@ -661,13 +672,14 @@ def create_file(self, file: Union[FileProperties, str], **kwargs: Any) -> DataLa @distributed_trace def get_paths( - self, *, + self, + *, recursive: bool = True, max_results: Optional[int] = None, upn: Optional[bool] = None, start_from: Optional[str] = None, timeout: Optional[int] = None, - **kwargs: Any + **kwargs: Any, ) -> ItemPaged["PathProperties"]: """Returns a generator to list the paths under specified file system and directory. The generator will lazily follow the continuation tokens returned by the service. @@ -702,15 +714,16 @@ def get_paths( url = f"{self.scheme}://{hostname}/{quote(self.file_system_name)}" client = self._build_generated_client(url) command = functools.partial( - client.file_system.list_paths, - path=self.path_name, - begin_from=start_from, - timeout=timeout, - **kwargs + client.file_system.list_paths, path=self.path_name, begin_from=start_from, timeout=timeout, **kwargs ) return ItemPaged( - command, recursive, path=self.path_name, max_results=max_results, - upn=upn, page_iterator_class=PathPropertiesPaged, **kwargs + command, + recursive, + path=self.path_name, + max_results=max_results, + upn=upn, + page_iterator_class=PathPropertiesPaged, + **kwargs, ) def get_file_client(self, file: Union[FileProperties, str]) -> DataLakeFileClient: @@ -726,18 +739,24 @@ def get_file_client(self, file: Union[FileProperties, str]) -> DataLakeFileClien :rtype: ~azure.storage.filedatalake.DataLakeFileClient """ if isinstance(file, FileProperties): - file_path = file.get('name') + file_path = file.get("name") else: - file_path = self.path_name + '/' + str(file) + file_path = self.path_name + "/" + str(file) _pipeline = Pipeline( - transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access - policies=self._pipeline._impl_policies # pylint: disable = protected-access + transport=TransportWrapper(self._pipeline._transport), # pylint: disable = protected-access + policies=self._pipeline._impl_policies, # pylint: disable = protected-access ) return DataLakeFileClient( - self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential, + self.url, + self.file_system_name, + file_path=file_path, + credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=_pipeline, + ) def get_sub_directory_client(self, sub_directory: Union[DirectoryProperties, str]) -> "DataLakeDirectoryClient": """Get a client to interact with the specified subdirectory of the current directory. @@ -752,15 +771,21 @@ def get_sub_directory_client(self, sub_directory: Union[DirectoryProperties, str :rtype: ~azure.storage.filedatalake.DataLakeDirectoryClient """ if isinstance(sub_directory, DirectoryProperties): - subdir_path = sub_directory.get('name') + subdir_path = sub_directory.get("name") else: - subdir_path = self.path_name + '/' + str(sub_directory) + subdir_path = self.path_name + "/" + str(sub_directory) _pipeline = Pipeline( - transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # pylint: disable=protected-access + transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # pylint: disable=protected-access ) return DataLakeDirectoryClient( - self.url, self.file_system_name, directory_name=subdir_path, credential=self._raw_credential, + self.url, + self.file_system_name, + directory_name=subdir_path, + credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=_pipeline, + ) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py index 7991643287f0..84e2aec68f82 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py @@ -6,10 +6,7 @@ # pylint: disable=docstring-keyword-should-match-keyword-only from datetime import datetime -from typing import ( - Any, AnyStr, cast, Dict, IO, Iterable, Optional, Union, - TYPE_CHECKING -) +from typing import Any, AnyStr, cast, Dict, IO, Iterable, Optional, Union, TYPE_CHECKING from urllib.parse import quote, unquote from typing_extensions import Self @@ -87,22 +84,29 @@ class DataLakeFileClient(PathClient): """The hostname of the primary endpoint.""" def __init__( - self, account_url: str, + self, + account_url: str, file_system_name: str, file_path: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> None: - super(DataLakeFileClient, self).__init__(account_url, file_system_name, path_name=file_path, - credential=credential, **kwargs) + super(DataLakeFileClient, self).__init__( + account_url, file_system_name, path_name=file_path, credential=credential, **kwargs + ) @classmethod def from_connection_string( - cls, conn_str: str, + cls, + conn_str: str, file_system_name: str, file_path: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> Self: """ Create DataLakeFileClient from a Connection String. @@ -137,16 +141,15 @@ def from_connection_string( :returns: A DataLakeFileClient. :rtype: ~azure.storage.filedatalake.DataLakeFileClient """ - account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs') - return cls( - account_url, file_system_name=file_system_name, file_path=file_path, - credential=credential, **kwargs) + account_url, _, credential = parse_connection_str(conn_str, credential, "dfs") + return cls(account_url, file_system_name=file_system_name, file_path=file_path, credential=credential, **kwargs) @distributed_trace def create_file( - self, content_settings: Optional["ContentSettings"] = None, + self, + content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Union[str, datetime]]: """ Create a new file. @@ -240,7 +243,7 @@ def create_file( :dedent: 4 :caption: Create file. """ - return self._create('file', content_settings=content_settings, metadata=metadata, **kwargs) + return self._create("file", content_settings=content_settings, metadata=metadata, **kwargs) @distributed_trace def delete_file(self, **kwargs: Any) -> None: @@ -343,18 +346,16 @@ def get_file_properties(self, **kwargs: Any) -> FileProperties: :dedent: 4 :caption: Getting the properties for a file. """ - upn = kwargs.pop('upn', None) + upn = kwargs.pop("upn", None) if upn: - headers = kwargs.pop('headers', {}) - headers['x-ms-upn'] = str(upn) - kwargs['headers'] = headers + headers = kwargs.pop("headers", {}) + headers["x-ms-upn"] = str(upn) + kwargs["headers"] = headers return cast(FileProperties, self._get_path_properties(cls=deserialize_file_properties, **kwargs)) @distributed_trace def set_file_expiry( - self, expiry_options: str, - expires_on: Optional[Union[datetime, int]] = None, - **kwargs: Any + self, expiry_options: str, expires_on: Optional[Union[datetime, int]] = None, **kwargs: Any ) -> None: """Sets the time a file will expire and be deleted. @@ -382,10 +383,11 @@ def set_file_expiry( @distributed_trace def upload_data( - self, data: Union[bytes, str, Iterable[AnyStr], IO[bytes]], + self, + data: Union[bytes, str, Iterable[AnyStr], IO[bytes]], length: Optional[int] = None, overwrite: Optional[bool] = False, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Any]: """ Upload data to a file. @@ -465,29 +467,20 @@ def upload_data( :rtype: Dict[str, Any] """ options = _upload_options( - data, - self.scheme, - self._config, - self._client.path, - length=length, - overwrite=overwrite, - **kwargs + data, self.scheme, self._config, self._client.path, length=length, overwrite=overwrite, **kwargs ) return upload_datalake_file(**options) @distributed_trace def append_data( - self, data: Union[bytes, Iterable[bytes], IO[bytes]], - offset: int, - length: Optional[int] = None, - **kwargs: Any + self, data: Union[bytes, Iterable[bytes], IO[bytes]], offset: int, length: Optional[int] = None, **kwargs: Any ) -> Dict[str, Any]: """Append data to the file. :param data: Content to be appended to file :type data: Union[bytes, Iterable[bytes], IO[bytes]] :param int offset: start position of the data to be appended to. - :param length: + :param length: Size of the data to append. Optional if the length of data can be determined. For Iterable and IO, if the length is not provided and cannot be determined, all data will be read into memory. :type length: int or None @@ -541,23 +534,14 @@ def append_data( :dedent: 4 :caption: Append data to the file. """ - options = _append_data_options( - data=data, - offset=offset, - scheme=self.scheme, - length=length, - **kwargs) + options = _append_data_options(data=data, offset=offset, scheme=self.scheme, length=length, **kwargs) try: return self._client.path.append_data(**options) except HttpResponseError as error: process_storage_error(error) @distributed_trace - def flush_data( - self, offset: int, - retain_uncommitted_data: Optional[bool] = False, - **kwargs: Any - ) -> Dict[str, Any]: + def flush_data(self, offset: int, retain_uncommitted_data: Optional[bool] = False, **kwargs: Any) -> Dict[str, Any]: """Commit the previous appended data. :param int offset: offset is equal to the length of the file after commit @@ -645,12 +629,7 @@ def flush_data( :dedent: 8 :caption: Commit the previous appended data. """ - options = _flush_data_options( - offset, - self.scheme, - retain_uncommitted_data=retain_uncommitted_data, - **kwargs - ) + options = _flush_data_options(offset, self.scheme, retain_uncommitted_data=retain_uncommitted_data, **kwargs) try: return self._client.path.flush_data(**options) except HttpResponseError as error: @@ -658,9 +637,7 @@ def flush_data( @distributed_trace def download_file( - self, offset: Optional[int] = None, - length: Optional[int] = None, - **kwargs: Any + self, offset: Optional[int] = None, length: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader: """Downloads a file to the StorageStreamDownloader. The readall() method must be used to read all the content, or readinto() must be used to download the file into @@ -820,16 +797,22 @@ def rename_file(self, new_name: str, **kwargs: Any) -> "DataLakeFileClient": :caption: Rename the source file. """ new_file_system, new_path, new_file_sas = _parse_rename_path( - new_name, self.file_system_name, self._query_str, self._raw_credential) + new_name, self.file_system_name, self._query_str, self._raw_credential + ) new_file_client = DataLakeFileClient( - f"{self.scheme}://{self.primary_hostname}", new_file_system, file_path=new_path, + f"{self.scheme}://{self.primary_hostname}", + new_file_system, + file_path=new_path, credential=self._raw_credential or new_file_sas, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode + _hosts=self._hosts, + _configuration=self._config, + _pipeline=self._pipeline, + _location_mode=self._location_mode, ) new_file_client._rename_path( # pylint: disable=protected-access - f'/{quote(unquote(self.file_system_name))}/{quote(unquote(self.path_name))}{self._query_str}', **kwargs) + f"/{quote(unquote(self.file_system_name))}/{quote(unquote(self.path_name))}{self._query_str}", **kwargs + ) return new_file_client @distributed_trace @@ -905,8 +888,7 @@ def query_file(self, query_expression: str, **kwargs: Any) -> DataLakeFileQueryR :caption: select/project on datalake file data by providing simple query expressions. """ query_expression = query_expression.replace("from DataLakeStorage", "from BlobStorage") - blob_quick_query_reader = self._blob_client.query_blob(query_expression, - blob_format=kwargs.pop('file_format', None), - error_cls=DataLakeFileQueryError, - **kwargs) + blob_quick_query_reader = self._blob_client.query_blob( + query_expression, blob_format=kwargs.pop("file_format", None), error_cls=DataLakeFileQueryError, **kwargs + ) return DataLakeFileQueryReader(blob_quick_query_reader) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.pyi b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.pyi index 020a89378f46..55eb2fda723a 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.pyi +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.pyi @@ -149,7 +149,7 @@ class DataLakeFileClient(PathClient): if_unmodified_since: Optional[datetime] = None, etag: Optional[str] = None, match_condition: Optional[MatchConditions] = None, - validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None, + validate_content: Optional[Union[bool, Literal["auto", "crc64", "md5"]]] = None, cpk: Optional[CustomerProvidedEncryptionKey] = None, max_concurrency: Optional[int] = None, chunk_size: Optional[int] = None, @@ -166,7 +166,7 @@ class DataLakeFileClient(PathClient): length: Optional[int] = None, *, flush: Optional[bool] = None, - validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None, + validate_content: Optional[Union[bool, Literal["auto", "crc64", "md5"]]] = None, lease_action: Optional[Literal["acquire", "auto-renew", "release", "acquire-release"]] = None, lease_duration: int = -1, lease: Optional[Union[DataLakeLeaseClient, str]] = None, @@ -207,7 +207,7 @@ class DataLakeFileClient(PathClient): cpk: Optional[CustomerProvidedEncryptionKey] = None, max_concurrency: Optional[int] = None, progress_hook: Optional[Callable[[int, int], None]] = None, - validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None, + validate_content: Optional[Union[bool, Literal["auto", "crc64", "md5"]]] = None, timeout: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader: ... diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client_helpers.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client_helpers.py index 5cf92d7bcfce..2af453a9919d 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client_helpers.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client_helpers.py @@ -6,8 +6,16 @@ from io import BytesIO from typing import ( - Any, AnyStr, AsyncGenerator, AsyncIterable, cast, - Dict, IO, Iterable, Optional, Union, + Any, + AnyStr, + AsyncGenerator, + AsyncIterable, + cast, + Dict, + IO, + Iterable, + Optional, + Union, TYPE_CHECKING, ) @@ -17,7 +25,7 @@ get_cpk_info, get_lease_action_properties, get_mod_conditions, - get_path_http_headers + get_path_http_headers, ) from ._shared.constants import DEFAULT_MAX_CONCURRENCY from ._shared.request_handlers import get_length, read_length @@ -37,10 +45,10 @@ def _append_data_options( offset: int, scheme: str, length: Optional[int] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Any]: if isinstance(data, str): - data = data.encode(kwargs.pop('encoding', 'UTF-8')) # type: ignore + data = data.encode(kwargs.pop("encoding", "UTF-8")) # type: ignore if length is None: length = get_length(data) if length is None: @@ -48,19 +56,19 @@ def _append_data_options( if isinstance(data, bytes): data = data[:length] - validate_content = parse_validation_option(kwargs.pop('validate_content', None)) + validate_content = parse_validation_option(kwargs.pop("validate_content", None)) cpk_info = get_cpk_info(scheme, kwargs) kwargs.update(get_lease_action_properties(kwargs)) options = { - 'body': data, - 'position': offset, - 'content_length': length, - 'validate_content': validate_content, - 'cpk_info': cpk_info, - 'timeout': kwargs.pop('timeout', None), - 'cls': return_response_headers + "body": data, + "position": offset, + "content_length": length, + "validate_content": validate_content, + "cpk_info": cpk_info, + "timeout": kwargs.pop("timeout", None), + "cls": return_response_headers, } options.update(kwargs) return options @@ -71,7 +79,7 @@ def _flush_data_options( scheme: str, content_settings: Optional["ContentSettings"] = None, retain_uncommitted_data: Optional[bool] = False, - **kwargs + **kwargs, ) -> Dict[str, Any]: mod_conditions = get_mod_conditions(kwargs) @@ -83,15 +91,15 @@ def _flush_data_options( kwargs.update(get_lease_action_properties(kwargs)) options = { - 'position': offset, - 'content_length': 0, - 'path_http_headers': path_http_headers, - 'retain_uncommitted_data': retain_uncommitted_data, - 'close': kwargs.pop('close', False), - 'modified_access_conditions': mod_conditions, - 'cpk_info': cpk_info, - 'timeout': kwargs.pop('timeout', None), - 'cls': return_response_headers + "position": offset, + "content_length": 0, + "path_http_headers": path_http_headers, + "retain_uncommitted_data": retain_uncommitted_data, + "close": kwargs.pop("close", False), + "modified_access_conditions": mod_conditions, + "cpk_info": cpk_info, + "timeout": kwargs.pop("timeout", None), + "cls": return_response_headers, } options.update(kwargs) return options @@ -103,9 +111,9 @@ def _upload_options( config: "StorageConfiguration", path: "PathOperations", length: Optional[int] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Any]: - encoding = kwargs.pop('encoding', 'UTF-8') + encoding = kwargs.pop("encoding", "UTF-8") if isinstance(data, str): data = data.encode(encoding) if length is None: @@ -116,35 +124,35 @@ def _upload_options( stream: Optional[Any] = None if isinstance(data, bytes): stream = BytesIO(data) - elif hasattr(data, 'read'): + elif hasattr(data, "read"): stream = data - elif hasattr(data, '__iter__'): + elif hasattr(data, "__iter__"): stream = IterStreamer(data, encoding=encoding) - elif hasattr(data, '__aiter__'): + elif hasattr(data, "__aiter__"): stream = AsyncIterStreamer(cast(AsyncGenerator, data), encoding=encoding) else: raise TypeError(f"Unsupported data type: {type(data)}") - validate_content = parse_validation_option(kwargs.pop('validate_content', None)) - content_settings = kwargs.pop('content_settings', None) - metadata = kwargs.pop('metadata', None) - max_concurrency = kwargs.pop('max_concurrency', None) + validate_content = parse_validation_option(kwargs.pop("validate_content", None)) + content_settings = kwargs.pop("content_settings", None) + metadata = kwargs.pop("metadata", None) + max_concurrency = kwargs.pop("max_concurrency", None) if max_concurrency is None: max_concurrency = DEFAULT_MAX_CONCURRENCY - kwargs['properties'] = add_metadata_headers(metadata) - kwargs['lease_access_conditions'] = get_access_conditions(kwargs.pop('lease', None)) - kwargs['modified_access_conditions'] = get_mod_conditions(kwargs) - kwargs['cpk_info'] = get_cpk_info(scheme, kwargs) + kwargs["properties"] = add_metadata_headers(metadata) + kwargs["lease_access_conditions"] = get_access_conditions(kwargs.pop("lease", None)) + kwargs["modified_access_conditions"] = get_mod_conditions(kwargs) + kwargs["cpk_info"] = get_cpk_info(scheme, kwargs) if content_settings: - kwargs['path_http_headers'] = get_path_http_headers(content_settings) - - kwargs['stream'] = stream - kwargs['length'] = length - kwargs['validate_content'] = validate_content - kwargs['max_concurrency'] = max_concurrency - kwargs['client'] = path - kwargs['file_settings'] = config + kwargs["path_http_headers"] = get_path_http_headers(content_settings) + + kwargs["stream"] = stream + kwargs["length"] = length + kwargs["validate_content"] = validate_content + kwargs["max_concurrency"] = max_concurrency + kwargs["client"] = path + kwargs["file_settings"] = config return kwargs diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_lease.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_lease.py index 01ae48891ccc..603a233ea6dc 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_lease.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_lease.py @@ -6,10 +6,7 @@ # pylint: disable=docstring-keyword-should-match-keyword-only import uuid -from typing import ( - Union, Optional, Any, - TYPE_CHECKING -) +from typing import Union, Optional, Any, TYPE_CHECKING from typing_extensions import Self from azure.core.tracing.decorator import distributed_trace @@ -49,16 +46,17 @@ class DataLakeLeaseClient: # pylint: disable=client-accepts-api-version-keyword This will be `None` if no lease has yet been acquired or modified.""" def __init__( # pylint: disable=missing-client-constructor-parameter-credential, missing-client-constructor-parameter-kwargs - self, client: Union["FileSystemClient", "DataLakeDirectoryClient", "DataLakeFileClient"], - lease_id: Optional[str] = None + self, + client: Union["FileSystemClient", "DataLakeDirectoryClient", "DataLakeFileClient"], + lease_id: Optional[str] = None, ) -> None: self.id = lease_id or str(uuid.uuid4()) self.last_modified = None self.etag = None - if hasattr(client, '_blob_client'): + if hasattr(client, "_blob_client"): _client = client._blob_client - elif hasattr(client, '_container_client'): + elif hasattr(client, "_container_client"): _client = client._container_client else: raise TypeError("Lease must use any of FileSystemClient, DataLakeDirectoryClient, or DataLakeFileClient.") diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_lease.pyi b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_lease.pyi index d9ee9df15e6a..b6ba38048762 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_lease.pyi +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_lease.pyi @@ -7,7 +7,9 @@ from datetime import datetime from typing import ( - Any, Optional, Union, + Any, + Optional, + Union, ) from types import TracebackType from typing_extensions import Self @@ -18,7 +20,6 @@ from ._file_system_client import FileSystemClient from ._data_lake_directory_client import DataLakeDirectoryClient from ._data_lake_file_client import DataLakeFileClient - class DataLakeLeaseClient: id: str etag: Optional[str] @@ -26,7 +27,7 @@ class DataLakeLeaseClient: def __init__( self, client: Union[FileSystemClient, DataLakeDirectoryClient, DataLakeFileClient], - lease_id: Optional[str] = None + lease_id: Optional[str] = None, ) -> None: ... def __enter__(self) -> Self: ... def __exit__( diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py index a8cc3d21f39f..dcd70be5e028 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py @@ -5,10 +5,7 @@ # -------------------------------------------------------------------------- # pylint: disable=docstring-keyword-should-match-keyword-only -from typing import ( - Any, cast, Dict, Optional, Union, - TYPE_CHECKING -) +from typing import Any, cast, Dict, Optional, Union, TYPE_CHECKING from typing_extensions import Self from azure.core.paging import ItemPaged @@ -27,7 +24,7 @@ FileSystemProperties, FileSystemPropertiesPaged, LocationMode, - UserDelegationKey + UserDelegationKey, ) from ._serialize import convert_dfs_url_to_blob_url, get_api_version from ._shared.base_client import parse_connection_str, parse_query, StorageAccountHostsMixin, TransportWrapper @@ -97,8 +94,11 @@ class DataLakeServiceClient(StorageAccountHostsMixin): """The hostname of the primary endpoint.""" def __init__( - self, account_url: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long + self, + account_url: str, + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"] + ] = None, # pylint: disable=line-too-long **kwargs: Any ) -> None: parsed_url = _parse_url(account_url=account_url) @@ -111,17 +111,15 @@ def __init__( _, sas_token = parse_query(parsed_url.query) self._query_str, self._raw_credential = self._format_query_string(sas_token, credential) - super(DataLakeServiceClient, self).__init__(parsed_url, service='dfs', - credential=self._raw_credential, **kwargs) + super(DataLakeServiceClient, self).__init__( + parsed_url, service="dfs", credential=self._raw_credential, **kwargs + ) # ADLS doesn't support secondary endpoint, make sure it's empty self._hosts[LocationMode.SECONDARY] = "" self._api_version = get_api_version(kwargs) self._client = AzureDataLakeStorageRESTAPI( - self.url, - version=self._api_version, - base_url=self.url, - pipeline=self._pipeline + self.url, version=self._api_version, base_url=self.url, pipeline=self._pipeline ) def __enter__(self) -> Self: @@ -154,8 +152,11 @@ def _format_url(self, hostname: str) -> str: @classmethod def from_connection_string( - cls, conn_str: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long + cls, + conn_str: str, + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"] + ] = None, # pylint: disable=line-too-long **kwargs: Any ) -> Self: """ @@ -193,12 +194,13 @@ def from_connection_string( :dedent: 8 :caption: Creating the DataLakeServiceClient from a connection string. """ - account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs') + account_url, _, credential = parse_connection_str(conn_str, credential, "dfs") return cls(account_url, credential=credential, **kwargs) @distributed_trace def get_user_delegation_key( - self, key_start_time: "datetime", + self, + key_start_time: "datetime", key_expiry_time: "datetime", *, delegated_user_tid: Optional[str] = None, @@ -241,9 +243,7 @@ def get_user_delegation_key( @distributed_trace def list_file_systems( - self, name_starts_with: Optional[str] = None, - include_metadata: bool = False, - **kwargs: Any + self, name_starts_with: Optional[str] = None, include_metadata: bool = False, **kwargs: Any ) -> ItemPaged[FileSystemProperties]: """Returns a generator to list the file systems under the specified account. @@ -285,16 +285,15 @@ def list_file_systems( :caption: Listing the file systems in the datalake service. """ item_paged = self._blob_service_client.list_containers( - name_starts_with=name_starts_with, - include_metadata=include_metadata, - **kwargs + name_starts_with=name_starts_with, include_metadata=include_metadata, **kwargs ) item_paged._page_iterator_class = FileSystemPropertiesPaged # pylint: disable=protected-access return cast(ItemPaged[FileSystemProperties], item_paged) @distributed_trace def create_file_system( - self, file_system: Union[FileSystemProperties, str], + self, + file_system: Union[FileSystemProperties, str], metadata: Optional[Dict[str, str]] = None, public_access: Optional["PublicAccess"] = None, **kwargs: Any @@ -365,7 +364,7 @@ def _rename_file_system(self, name: str, new_name: str, **kwargs: Any) -> FileSy :returns: A FileSystemClient with the specified file system renamed. :rtype: ~azure.storage.filedatalake.FileSystemClient """ - self._blob_service_client._rename_container(name, new_name, **kwargs) # pylint: disable=protected-access + self._blob_service_client._rename_container(name, new_name, **kwargs) # pylint: disable=protected-access renamed_file_system = self.get_file_system_client(new_name) return renamed_file_system @@ -392,14 +391,15 @@ def undelete_file_system(self, name: str, deleted_version: str, **kwargs: Any) - :returns: The restored solft-deleted FileSystemClient. :rtype: ~azure.storage.filedatalake.FileSystemClient """ - new_name = kwargs.pop('new_name', None) + new_name = kwargs.pop("new_name", None) file_system = self.get_file_system_client(new_name or name) - self._blob_service_client.undelete_container( - name, deleted_version, new_name=new_name, **kwargs) + self._blob_service_client.undelete_container(name, deleted_version, new_name=new_name, **kwargs) return file_system @distributed_trace - def delete_file_system(self, file_system: Union[FileSystemProperties, str], **kwargs: Any) -> FileSystemClient: # pylint: disable=delete-operation-wrong-return-type + def delete_file_system( + self, file_system: Union[FileSystemProperties, str], **kwargs: Any + ) -> FileSystemClient: # pylint: disable=delete-operation-wrong-return-type """Marks the specified file system for deletion. The file system and any files contained within it are later deleted during garbage collection. @@ -481,16 +481,20 @@ def get_file_system_client(self, file_system: Union[FileSystemProperties, str]) _pipeline = Pipeline( transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # pylint: disable=protected-access + ) + return FileSystemClient( + self.url, + file_system_name, + credential=self._raw_credential, + api_version=self.api_version, + _configuration=self._config, + _pipeline=_pipeline, + _hosts=self._hosts, ) - return FileSystemClient(self.url, file_system_name, credential=self._raw_credential, - api_version=self.api_version, - _configuration=self._config, - _pipeline=_pipeline, _hosts=self._hosts) def get_directory_client( - self, file_system: Union[FileSystemProperties, str], - directory: Union[DirectoryProperties, str] + self, file_system: Union[FileSystemProperties, str], directory: Union[DirectoryProperties, str] ) -> DataLakeDirectoryClient: """Get a client to interact with the specified directory. @@ -527,17 +531,21 @@ def get_directory_client( _pipeline = Pipeline( transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # pylint: disable=protected-access + ) + return DataLakeDirectoryClient( + self.url, + file_system_name, + directory_name=directory_name, + credential=self._raw_credential, + api_version=self.api_version, + _configuration=self._config, + _pipeline=_pipeline, + _hosts=self._hosts, ) - return DataLakeDirectoryClient(self.url, file_system_name, directory_name=directory_name, - credential=self._raw_credential, - api_version=self.api_version, - _configuration=self._config, _pipeline=_pipeline, - _hosts=self._hosts) def get_file_client( - self, file_system: Union[FileSystemProperties, str], - file_path: Union[FileProperties, str] + self, file_system: Union[FileSystemProperties, str], file_path: Union[FileProperties, str] ) -> DataLakeFileClient: """Get a client to interact with the specified file. @@ -574,12 +582,18 @@ def get_file_client( _pipeline = Pipeline( transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # pylint: disable=protected-access ) return DataLakeFileClient( - self.url, file_system_name, file_path=file_path, credential=self._raw_credential, + self.url, + file_system_name, + file_path=file_path, + credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=_pipeline, + ) @distributed_trace def set_service_properties(self, **kwargs: Any) -> None: diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client_helpers.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client_helpers.py index 600302e39c44..cc381dc581e9 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client_helpers.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client_helpers.py @@ -13,11 +13,11 @@ def _parse_url(account_url: str) -> "ParseResult": try: - if not account_url.lower().startswith('http'): + if not account_url.lower().startswith("http"): account_url = "https://" + account_url except AttributeError as exc: raise ValueError("Account URL must be a string.") from exc - parsed_url = urlparse(account_url.rstrip('/')) + parsed_url = urlparse(account_url.rstrip("/")) if not parsed_url.netloc: raise ValueError(f"Invalid URL: {account_url}") return parsed_url diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_deserialize.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_deserialize.py index c758060a65e3..4283701dcc38 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_deserialize.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_deserialize.py @@ -4,10 +4,7 @@ # license information. # -------------------------------------------------------------------------- import logging -from typing import ( - Any, cast, Collection, Dict, List, NoReturn, Tuple, - TYPE_CHECKING -) +from typing import Any, cast, Collection, Dict, List, NoReturn, Tuple, TYPE_CHECKING from xml.etree.ElementTree import Element from azure.core.pipeline.policies import ContentDecodePolicy @@ -17,7 +14,7 @@ ResourceModifiedError, ClientAuthenticationError, ResourceNotFoundError, - ResourceExistsError + ResourceExistsError, ) from ._models import ( AnalyticsLogging, @@ -28,7 +25,7 @@ Metrics, PathProperties, RetentionPolicy, - StaticWebsite + StaticWebsite, ) from ._shared.models import StorageErrorCode from ._shared.response_handlers import deserialize_metadata @@ -36,52 +33,40 @@ if TYPE_CHECKING: from azure.core.rest import HttpResponse from azure.storage.blob import BlobProperties - from ._generated.models import ( - BlobItemInternal, - Path, - PathList - ) + from ._generated.models import BlobItemInternal, Path, PathList from ._models import ContentSettings _LOGGER = logging.getLogger(__name__) -def deserialize_dir_properties( - response: "HttpResponse", - obj: Any, - headers: Dict[str, Any] -) -> DirectoryProperties: +def deserialize_dir_properties(response: "HttpResponse", obj: Any, headers: Dict[str, Any]) -> DirectoryProperties: metadata = deserialize_metadata(response, obj, headers) dir_properties = DirectoryProperties( metadata=metadata, - owner=response.headers.get('x-ms-owner'), - group=response.headers.get('x-ms-group'), - permissions=response.headers.get('x-ms-permissions'), - acl=response.headers.get('x-ms-acl'), - **headers + owner=response.headers.get("x-ms-owner"), + group=response.headers.get("x-ms-group"), + permissions=response.headers.get("x-ms-permissions"), + acl=response.headers.get("x-ms-acl"), + **headers, ) return dir_properties -def deserialize_file_properties( - response: "HttpResponse", - obj: Any, - headers: Dict[str, Any] -) -> FileProperties: +def deserialize_file_properties(response: "HttpResponse", obj: Any, headers: Dict[str, Any]) -> FileProperties: metadata = deserialize_metadata(response, obj, headers) # DataLake specific headers that are not deserialized in blob are pulled directly from the raw response header file_properties = FileProperties( metadata=metadata, - encryption_context=response.headers.get('x-ms-encryption-context'), - owner=response.headers.get('x-ms-owner'), - group=response.headers.get('x-ms-group'), - permissions=response.headers.get('x-ms-permissions'), - acl=response.headers.get('x-ms-acl'), - **headers + encryption_context=response.headers.get("x-ms-encryption-context"), + owner=response.headers.get("x-ms-owner"), + group=response.headers.get("x-ms-group"), + permissions=response.headers.get("x-ms-permissions"), + acl=response.headers.get("x-ms-acl"), + **headers, ) - if 'Content-Range' in headers: - if 'x-ms-blob-content-md5' in headers: - file_properties.content_settings.content_md5 = headers['x-ms-blob-content-md5'] + if "Content-Range" in headers: + if "x-ms-blob-content-md5" in headers: + file_properties.content_settings.content_md5 = headers["x-ms-blob-content-md5"] else: file_properties.content_settings.content_md5 = None return file_properties @@ -92,14 +77,14 @@ def deserialize_path_properties(path_list: List["Path"]) -> List[PathProperties] def return_headers_and_deserialized_path_list( # pylint: disable=name-too-long, unused-argument - _, - deserialized: "PathList", - response_headers: Dict[str, Any] + _, deserialized: "PathList", response_headers: Dict[str, Any] ) -> Tuple[Collection["Path"], Dict[str, Any]]: return deserialized.paths if deserialized.paths else {}, normalize_headers(response_headers) -def get_deleted_path_properties_from_generated_code(generated: "BlobItemInternal") -> DeletedPathProperties: # pylint: disable=name-too-long +def get_deleted_path_properties_from_generated_code( + generated: "BlobItemInternal", +) -> DeletedPathProperties: # pylint: disable=name-too-long deleted_path = DeletedPathProperties() deleted_path.name = generated.name deleted_path.deleted_time = generated.properties.deleted_time @@ -109,19 +94,27 @@ def get_deleted_path_properties_from_generated_code(generated: "BlobItemInternal def is_file_path(_, __, headers: Dict[str, Any]) -> bool: - return headers['x-ms-resource-type'] == "file" + return headers["x-ms-resource-type"] == "file" def get_datalake_service_properties(datalake_properties: Dict[str, Any]) -> Dict[str, Any]: datalake_properties["analytics_logging"] = AnalyticsLogging._from_generated( # pylint: disable=protected-access - datalake_properties["analytics_logging"]) - datalake_properties["hour_metrics"] = Metrics._from_generated(datalake_properties["hour_metrics"]) # pylint: disable=protected-access + datalake_properties["analytics_logging"] + ) + datalake_properties["hour_metrics"] = Metrics._from_generated( + datalake_properties["hour_metrics"] + ) # pylint: disable=protected-access datalake_properties["minute_metrics"] = Metrics._from_generated( # pylint: disable=protected-access - datalake_properties["minute_metrics"]) - datalake_properties["delete_retention_policy"] = RetentionPolicy._from_generated( # pylint: disable=protected-access - datalake_properties["delete_retention_policy"]) + datalake_properties["minute_metrics"] + ) + datalake_properties["delete_retention_policy"] = ( + RetentionPolicy._from_generated( # pylint: disable=protected-access + datalake_properties["delete_retention_policy"] + ) + ) datalake_properties["static_website"] = StaticWebsite._from_generated( # pylint: disable=protected-access - datalake_properties["static_website"]) + datalake_properties["static_website"] + ) return datalake_properties @@ -141,11 +134,11 @@ def from_blob_properties(blob_properties: "BlobProperties", **additional_args: A file_props.content_settings = cast("ContentSettings", blob_properties.content_settings) # Parse additional Datalake-only properties - file_props.encryption_context = additional_args.pop('encryption_context', None) - file_props.owner = additional_args.pop('owner', None) - file_props.group = additional_args.pop('group', None) - file_props.permissions = additional_args.pop('permissions', None) - file_props.acl = additional_args.pop('acl', None) + file_props.encryption_context = additional_args.pop("encryption_context", None) + file_props.owner = additional_args.pop("owner", None) + file_props.group = additional_args.pop("group", None) + file_props.permissions = additional_args.pop("permissions", None) + file_props.acl = additional_args.pop("acl", None) return file_props @@ -153,9 +146,9 @@ def from_blob_properties(blob_properties: "BlobProperties", **additional_args: A def normalize_headers(headers: Dict[str, Any]) -> Dict[str, Any]: normalized = {} for key, value in headers.items(): - if key.startswith('x-ms-'): + if key.startswith("x-ms-"): key = key[5:] - normalized[key.lower().replace('-', '_')] = value + normalized[key.lower().replace("-", "_")] = value return normalized @@ -167,7 +160,7 @@ def process_storage_error(storage_error) -> NoReturn: # type: ignore [misc] # p # If it is one of those three then it has been serialized prior by the generated layer. if isinstance(storage_error, (ResourceNotFoundError, ClientAuthenticationError, ResourceExistsError)): serialized = True - error_code = storage_error.response.headers.get('x-ms-error-code') + error_code = storage_error.response.headers.get("x-ms-error-code") error_message = storage_error.message additional_data = {} error_dict = {} @@ -175,23 +168,21 @@ def process_storage_error(storage_error) -> NoReturn: # type: ignore [misc] # p error_body = ContentDecodePolicy.deserialize_from_http_generics(storage_error.response) # If it is an XML response if isinstance(error_body, Element): - error_dict = { - child.tag.lower(): child.text - for child in error_body - } + error_dict = {child.tag.lower(): child.text for child in error_body} # If it is a JSON response elif isinstance(error_body, dict): - error_dict = error_body.get('error', {}) + error_dict = error_body.get("error", {}) elif not error_code: _LOGGER.warning( - 'Unexpected return type %s from ContentDecodePolicy.deserialize_from_http_generics.', type(error_body)) - error_dict = {'message': str(error_body)} + "Unexpected return type %s from ContentDecodePolicy.deserialize_from_http_generics.", type(error_body) + ) + error_dict = {"message": str(error_body)} # If we extracted from a Json or XML response if error_dict: - error_code = error_dict.get('code') - error_message = error_dict.get('message') - additional_data = {k: v for k, v in error_dict.items() if k not in {'code', 'message'}} + error_code = error_dict.get("code") + error_message = error_dict.get("message") + additional_data = {k: v for k, v in error_dict.items() if k not in {"code", "message"}} except DecodeError: pass @@ -202,33 +193,36 @@ def process_storage_error(storage_error) -> NoReturn: # type: ignore [misc] # p error_code = StorageErrorCode(error_code) if error_code in [StorageErrorCode.condition_not_met]: raise_error = ResourceModifiedError - if error_code in [StorageErrorCode.invalid_authentication_info, - StorageErrorCode.authentication_failed]: + if error_code in [StorageErrorCode.invalid_authentication_info, StorageErrorCode.authentication_failed]: raise_error = ClientAuthenticationError - if error_code in [StorageErrorCode.resource_not_found, - StorageErrorCode.invalid_property_name, - StorageErrorCode.invalid_source_uri, - StorageErrorCode.source_path_not_found, - StorageErrorCode.lease_name_mismatch, - StorageErrorCode.file_system_not_found, - StorageErrorCode.path_not_found, - StorageErrorCode.parent_not_found, - StorageErrorCode.invalid_destination_path, - StorageErrorCode.invalid_rename_source_path, - StorageErrorCode.lease_is_already_broken, - StorageErrorCode.invalid_source_or_destination_resource_type, - StorageErrorCode.rename_destination_parent_path_not_found]: + if error_code in [ + StorageErrorCode.resource_not_found, + StorageErrorCode.invalid_property_name, + StorageErrorCode.invalid_source_uri, + StorageErrorCode.source_path_not_found, + StorageErrorCode.lease_name_mismatch, + StorageErrorCode.file_system_not_found, + StorageErrorCode.path_not_found, + StorageErrorCode.parent_not_found, + StorageErrorCode.invalid_destination_path, + StorageErrorCode.invalid_rename_source_path, + StorageErrorCode.lease_is_already_broken, + StorageErrorCode.invalid_source_or_destination_resource_type, + StorageErrorCode.rename_destination_parent_path_not_found, + ]: raise_error = ResourceNotFoundError - if error_code in [StorageErrorCode.account_already_exists, - StorageErrorCode.account_being_created, - StorageErrorCode.resource_already_exists, - StorageErrorCode.resource_type_mismatch, - StorageErrorCode.source_path_is_being_deleted, - StorageErrorCode.path_already_exists, - StorageErrorCode.destination_path_is_being_deleted, - StorageErrorCode.file_system_already_exists, - StorageErrorCode.file_system_being_deleted, - StorageErrorCode.path_conflict]: + if error_code in [ + StorageErrorCode.account_already_exists, + StorageErrorCode.account_being_created, + StorageErrorCode.resource_already_exists, + StorageErrorCode.resource_type_mismatch, + StorageErrorCode.source_path_is_being_deleted, + StorageErrorCode.path_already_exists, + StorageErrorCode.destination_path_is_being_deleted, + StorageErrorCode.file_system_already_exists, + StorageErrorCode.file_system_being_deleted, + StorageErrorCode.path_conflict, + ]: raise_error = ResourceExistsError except ValueError: # Got an unknown error code @@ -256,6 +250,6 @@ def process_storage_error(storage_error) -> NoReturn: # type: ignore [misc] # p try: # `from None` prevents us from double printing the exception (suppresses generated layer error context) - exec("raise error from None") # pylint: disable=exec-used # nosec + exec("raise error from None") # pylint: disable=exec-used # nosec except SyntaxError as exc: raise error from exc diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_download.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_download.py index f8b6aa3fbd3c..5c20cb1731c0 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_download.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_download.py @@ -3,10 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import ( - Any, cast, IO, Iterator, - TYPE_CHECKING -) +from typing import Any, cast, IO, Iterator, TYPE_CHECKING from ._deserialize import from_blob_properties @@ -31,13 +28,12 @@ def __init__(self, downloader: Any) -> None: self.name = self._downloader.name # Parse additional Datalake-only properties - encryption_context = self._downloader._response.response.headers.get('x-ms-encryption-context') - acl = self._downloader._response.response.headers.get('x-ms-acl') + encryption_context = self._downloader._response.response.headers.get("x-ms-encryption-context") + acl = self._downloader._response.response.headers.get("x-ms-acl") self.properties = from_blob_properties( - self._downloader.properties, - encryption_context=encryption_context, - acl=acl) + self._downloader.properties, encryption_context=encryption_context, acl=acl + ) self.size = self._downloader.size def __len__(self) -> int: diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py index 2178be109500..d0849f91ea29 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py @@ -6,10 +6,7 @@ # pylint: disable=too-many-lines, docstring-keyword-should-match-keyword-only import functools -from typing import ( - Any, cast, Dict, Optional, Union, - TYPE_CHECKING -) +from typing import Any, cast, Dict, Optional, Union, TYPE_CHECKING from typing_extensions import Self from azure.core.exceptions import HttpResponseError @@ -30,7 +27,7 @@ FileProperties, FileSystemProperties, LocationMode, - PublicAccess + PublicAccess, ) from ._shared.base_client import parse_connection_str, parse_query, TransportWrapper, StorageAccountHostsMixin from ._serialize import convert_dfs_url_to_blob_url, get_api_version @@ -93,10 +90,13 @@ class FileSystemClient(StorageAccountHostsMixin): """The hostname of the primary endpoint.""" def __init__( - self, account_url: str, + self, + account_url: str, file_system_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> None: if not file_system_name: raise ValueError("Please specify a file system name.") @@ -107,24 +107,21 @@ def __init__( # TODO: add self.account_url to base_client and remove _blob_account_url self._blob_account_url = blob_account_url - datalake_hosts = kwargs.pop('_hosts', None) + datalake_hosts = kwargs.pop("_hosts", None) blob_hosts = None if datalake_hosts: blob_primary_account_url = convert_dfs_url_to_blob_url(datalake_hosts[LocationMode.PRIMARY]) blob_hosts = {LocationMode.PRIMARY: blob_primary_account_url, LocationMode.SECONDARY: ""} self._container_client = ContainerClient( - self._blob_account_url, - self.file_system_name, - credential=credential, - _hosts=blob_hosts, - **kwargs + self._blob_account_url, self.file_system_name, credential=credential, _hosts=blob_hosts, **kwargs ) _, sas_token = parse_query(parsed_url.query) self._query_str, self._raw_credential = self._format_query_string(sas_token, credential) - super(FileSystemClient, self).__init__(parsed_url, service='dfs', credential=self._raw_credential, - _hosts=datalake_hosts, **kwargs) + super(FileSystemClient, self).__init__( + parsed_url, service="dfs", credential=self._raw_credential, _hosts=datalake_hosts, **kwargs + ) # ADLS doesn't support secondary endpoint, make sure it's empty self._hosts[LocationMode.SECONDARY] = "" @@ -156,11 +153,7 @@ def close(self) -> None: def _build_generated_client(self, url: str) -> AzureDataLakeStorageRESTAPI: client = AzureDataLakeStorageRESTAPI( - url, - version=self._api_version, - base_url=url, - file_system=self.file_system_name, - pipeline=self._pipeline + url, version=self._api_version, base_url=url, file_system=self.file_system_name, pipeline=self._pipeline ) return client @@ -169,10 +162,13 @@ def _format_url(self, hostname: str) -> str: @classmethod def from_connection_string( - cls, conn_str: str, + cls, + conn_str: str, file_system_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> Self: """ Create FileSystemClient from a Connection String. @@ -210,14 +206,12 @@ def from_connection_string( :dedent: 8 :caption: Create FileSystemClient from connection string """ - account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs') + account_url, _, credential = parse_connection_str(conn_str, credential, "dfs") return cls(account_url, file_system_name=file_system_name, credential=credential, **kwargs) @distributed_trace def acquire_lease( - self, lease_duration: int = -1, - lease_id: Optional[str] = None, - **kwargs: Any + self, lease_duration: int = -1, lease_id: Optional[str] = None, **kwargs: Any ) -> DataLakeLeaseClient: """ Requests a new lease. If the file system does not have an active lease, @@ -273,9 +267,7 @@ def acquire_lease( @distributed_trace def create_file_system( - self, metadata: Optional[Dict[str, str]] = None, - public_access: Optional[PublicAccess] = None, - **kwargs: Any + self, metadata: Optional[Dict[str, str]] = None, public_access: Optional[PublicAccess] = None, **kwargs: Any ) -> Dict[str, Union[str, "datetime"]]: """Creates a new file system under the specified account. @@ -315,12 +307,12 @@ def create_file_system( :dedent: 12 :caption: Creating a file system in the datalake service. """ - encryption_scope_options = kwargs.pop('encryption_scope_options', None) + encryption_scope_options = kwargs.pop("encryption_scope_options", None) return self._container_client.create_container( metadata=metadata, public_access=public_access, container_encryption_scope=encryption_scope_options, - **kwargs + **kwargs, ) @distributed_trace @@ -360,11 +352,17 @@ def _rename_file_system(self, new_name: str, **kwargs: Any) -> "FileSystemClient :rtype: ~azure.storage.filedatalake.FileSystemClient """ self._container_client._rename_container(new_name, **kwargs) # pylint: disable=protected-access - #TODO: self._raw_credential would not work with SAS tokens + # TODO: self._raw_credential would not work with SAS tokens renamed_file_system = FileSystemClient( - f"{self.scheme}://{self.primary_hostname}", file_system_name=new_name, - credential=self._raw_credential, api_version=self.api_version, _configuration=self._config, - _pipeline=self._pipeline, _location_mode=self._location_mode, _hosts=self._hosts) + f"{self.scheme}://{self.primary_hostname}", + file_system_name=new_name, + credential=self._raw_credential, + api_version=self.api_version, + _configuration=self._config, + _pipeline=self._pipeline, + _location_mode=self._location_mode, + _hosts=self._hosts, + ) return renamed_file_system @distributed_trace @@ -441,13 +439,12 @@ def get_file_system_properties(self, **kwargs: Any) -> FileSystemProperties: :caption: Getting properties on the file system. """ container_properties = self._container_client.get_container_properties(**kwargs) - return FileSystemProperties._convert_from_container_props(container_properties) # pylint: disable=protected-access + return FileSystemProperties._convert_from_container_props( + container_properties + ) # pylint: disable=protected-access @distributed_trace - def set_file_system_metadata( - self, metadata: Dict[str, str], - **kwargs: Any - ) -> Dict[str, Union[str, "datetime"]]: + def set_file_system_metadata(self, metadata: Dict[str, str], **kwargs: Any) -> Dict[str, Union[str, "datetime"]]: """Sets one or more user-defined name-value pairs for the specified file system. Each call to this operation replaces all existing metadata attached to the file system. To remove all metadata from the file system, @@ -499,9 +496,10 @@ def set_file_system_metadata( @distributed_trace def set_file_system_access_policy( - self, signed_identifiers: Dict[str, "AccessPolicy"], + self, + signed_identifiers: Dict[str, "AccessPolicy"], public_access: Optional[Union[str, "PublicAccess"]] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Union[str, "datetime"]]: """Sets the permissions for the specified file system or stored access policies that may be used with Shared Access Signatures. The permissions @@ -540,9 +538,7 @@ def set_file_system_access_policy( :rtype: Dict[str, Union[str, ~datetime.datetime]] """ return self._container_client.set_container_access_policy( - cast(Dict[str, "BlobAccessPolicy"], signed_identifiers), - public_access=public_access, - **kwargs + cast(Dict[str, "BlobAccessPolicy"], signed_identifiers), public_access=public_access, **kwargs ) @distributed_trace @@ -565,16 +561,19 @@ def get_file_system_access_policy(self, **kwargs: Any) -> Dict[str, Any]: """ access_policy = self._container_client.get_container_access_policy(**kwargs) return { - 'public_access': PublicAccess._from_generated(access_policy['public_access']), # pylint: disable=protected-access - 'signed_identifiers': access_policy['signed_identifiers'] + "public_access": PublicAccess._from_generated( + access_policy["public_access"] + ), # pylint: disable=protected-access + "signed_identifiers": access_policy["signed_identifiers"], } @distributed_trace def get_paths( - self, path: Optional[str] = None, + self, + path: Optional[str] = None, recursive: Optional[bool] = True, max_results: Optional[int] = None, - **kwargs: Any + **kwargs: Any, ) -> ItemPaged["PathProperties"]: """Returns a generator to list the paths(could be files or directories) under the specified file system. The generator will lazily follow the continuation tokens returned by @@ -617,25 +616,18 @@ def get_paths( :dedent: 8 :caption: List the paths in the file system. """ - timeout = kwargs.pop('timeout', None) + timeout = kwargs.pop("timeout", None) begin_from = kwargs.pop("start_from", None) command = functools.partial( - self._client.file_system.list_paths, - path=path, - timeout=timeout, - begin_from=begin_from, - **kwargs + self._client.file_system.list_paths, path=path, timeout=timeout, begin_from=begin_from, **kwargs ) return ItemPaged( - command, recursive, path=path, max_results=max_results, - page_iterator_class=PathPropertiesPaged, **kwargs + command, recursive, path=path, max_results=max_results, page_iterator_class=PathPropertiesPaged, **kwargs ) @distributed_trace def create_directory( - self, directory: Union[DirectoryProperties, str], - metadata: Optional[Dict[str, str]] = None, - **kwargs: Any + self, directory: Union[DirectoryProperties, str], metadata: Optional[Dict[str, str]] = None, **kwargs: Any ) -> DataLakeDirectoryClient: """ Create directory @@ -726,8 +718,7 @@ def create_directory( @distributed_trace def delete_directory( # pylint: disable=delete-operation-wrong-return-type - self, directory: Union[DirectoryProperties, str], - **kwargs: Any + self, directory: Union[DirectoryProperties, str], **kwargs: Any ) -> DataLakeDirectoryClient: """ Marks the specified path for deletion. @@ -780,10 +771,7 @@ def delete_directory( # pylint: disable=delete-operation-wrong-return-type return directory_client @distributed_trace - def create_file( - self, file: Union[FileProperties, str], - **kwargs: Any - ) -> DataLakeFileClient: + def create_file(self, file: Union[FileProperties, str], **kwargs: Any) -> DataLakeFileClient: """ Create file @@ -881,8 +869,7 @@ def create_file( @distributed_trace def delete_file( # pylint: disable=delete-operation-wrong-return-type - self, file: Union[FileProperties, str], - **kwargs: Any + self, file: Union[FileProperties, str], **kwargs: Any ) -> DataLakeFileClient: """ Marks the specified file for deletion. @@ -935,9 +922,7 @@ def delete_file( # pylint: disable=delete-operation-wrong-return-type return file_client def _undelete_path( - self, deleted_path_name: str, - deletion_id: str, - **kwargs: Any + self, deleted_path_name: str, deletion_id: str, **kwargs: Any ) -> Union[DataLakeDirectoryClient, DataLakeFileClient]: """Restores soft-deleted path. @@ -964,14 +949,10 @@ def _undelete_path( pipeline = Pipeline( transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # pylint: disable=protected-access ) path_client = AzureDataLakeStorageRESTAPI( - url, - version=self._api_version, - filesystem=self.file_system_name, - path=deleted_path_name, - pipeline=pipeline + url, version=self._api_version, filesystem=self.file_system_name, path=deleted_path_name, pipeline=pipeline ) try: is_file = path_client.path.undelete(undelete_source=undelete_source, cls=is_file_path, **kwargs) @@ -987,7 +968,7 @@ def _get_root_directory_client(self) -> DataLakeDirectoryClient: :returns: A DataLakeDirectoryClient. :rtype: ~azure.storage.filedatalake.DataLakeDirectoryClient """ - return self.get_directory_client('/') + return self.get_directory_client("/") def get_directory_client(self, directory: Union[DirectoryProperties, str]) -> DataLakeDirectoryClient: """Get a client to interact with the specified directory. @@ -1011,18 +992,23 @@ def get_directory_client(self, directory: Union[DirectoryProperties, str]) -> Da :caption: Getting the directory client to interact with a specific directory. """ if isinstance(directory, DirectoryProperties): - directory_name = directory.get('name') + directory_name = directory.get("name") else: directory_name = str(directory) _pipeline = Pipeline( transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # pylint: disable=protected-access + ) + return DataLakeDirectoryClient( + self.url, + self.file_system_name, + directory_name=directory_name, + credential=self._raw_credential, + api_version=self.api_version, + _configuration=self._config, + _pipeline=_pipeline, + _hosts=self._hosts, ) - return DataLakeDirectoryClient(self.url, self.file_system_name, directory_name=directory_name, - credential=self._raw_credential, - api_version=self.api_version, - _configuration=self._config, _pipeline=_pipeline, - _hosts=self._hosts) def get_file_client(self, file_path: Union[FileProperties, str]) -> DataLakeFileClient: """Get a client to interact with the specified file. @@ -1051,12 +1037,18 @@ def get_file_client(self, file_path: Union[FileProperties, str]) -> DataLakeFile file_path = str(file_path) _pipeline = Pipeline( transport=TransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # pylint: disable=protected-access ) return DataLakeFileClient( - self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential, + self.url, + self.file_system_name, + file_path=file_path, + credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=_pipeline, + ) @distributed_trace def list_deleted_paths(self, **kwargs: Any) -> ItemPaged[DeletedPathProperties]: @@ -1082,14 +1074,19 @@ def list_deleted_paths(self, **kwargs: Any) -> ItemPaged[DeletedPathProperties]: :rtype: ~azure.core.paging.ItemPaged[~azure.storage.filedatalake.DeletedPathProperties] """ - path_prefix = kwargs.pop('path_prefix', None) - timeout = kwargs.pop('timeout', None) - results_per_page = kwargs.pop('results_per_page', None) + path_prefix = kwargs.pop("path_prefix", None) + timeout = kwargs.pop("timeout", None) + results_per_page = kwargs.pop("results_per_page", None) command = functools.partial( self._datalake_client_for_blob_operation.file_system.list_blob_hierarchy_segment, showonly="deleted", timeout=timeout, - **kwargs) + **kwargs, + ) return ItemPaged( - command, prefix=path_prefix, page_iterator_class=DeletedPathPropertiesPaged, - results_per_page=results_per_page, **kwargs) + command, + prefix=path_prefix, + page_iterator_class=DeletedPathPropertiesPaged, + results_per_page=results_per_page, + **kwargs, + ) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client_helpers.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client_helpers.py index e95278cf3887..702197fc99d5 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client_helpers.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client_helpers.py @@ -13,11 +13,11 @@ def _parse_url(account_url: str) -> "ParseResult": try: - if not account_url.lower().startswith('http'): + if not account_url.lower().startswith("http"): account_url = "https://" + account_url except AttributeError as exc: raise ValueError("account URL must be a string.") from exc - parsed_url = urlparse(account_url.rstrip('/')) + parsed_url = urlparse(account_url.rstrip("/")) if not parsed_url.netloc: raise ValueError(f"Invalid URL: {account_url}") return parsed_url @@ -25,16 +25,16 @@ def _parse_url(account_url: str) -> "ParseResult": def _format_url(scheme: str, hostname: str, file_system_name: Union[str, bytes], query_str: str) -> str: if isinstance(file_system_name, str): - file_system_name = file_system_name.encode('UTF-8') + file_system_name = file_system_name.encode("UTF-8") return f"{scheme}://{hostname}/{quote(file_system_name)}{query_str}" def _undelete_path_options(deleted_path_name, deletion_id, url): - quoted_path = quote(unquote(deleted_path_name.strip('/'))) - url_and_token = url.replace('.dfs.', '.blob.').split('?') + quoted_path = quote(unquote(deleted_path_name.strip("/"))) + url_and_token = url.replace(".dfs.", ".blob.").split("?") try: - url = url_and_token[0] + '/' + quoted_path + url_and_token[1] + url = url_and_token[0] + "/" + quoted_path + url_and_token[1] except IndexError: - url = url_and_token[0] + '/' + quoted_path - undelete_source = quoted_path + f'?deletionid={deletion_id}' if deletion_id else None + url = url_and_token[0] + "/" + quoted_path + undelete_source = quoted_path + f"?deletionid={deletion_id}" if deletion_id else None return quoted_path, url, undelete_source diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_list_paths_helper.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_list_paths_helper.py index d2c0ba471f42..56ff803994eb 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_list_paths_helper.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_list_paths_helper.py @@ -4,10 +4,7 @@ # license information. # -------------------------------------------------------------------------- -from typing import ( - Any, Callable, cast, Dict, - List, Optional, Tuple, Union -) +from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Union from azure.core.paging import PageIterator from azure.core.exceptions import HttpResponseError @@ -15,13 +12,9 @@ from ._deserialize import ( get_deleted_path_properties_from_generated_code, process_storage_error, - return_headers_and_deserialized_path_list -) -from ._generated.models import ( - BlobItemInternal, - BlobPrefix as GenBlobPrefix, - Path + return_headers_and_deserialized_path_list, ) +from ._generated.models import BlobItemInternal, BlobPrefix as GenBlobPrefix, Path from ._models import DeletedPathProperties, PathProperties from ._shared.models import DictMixin from ._shared.response_handlers import return_context_and_deserialized @@ -43,11 +36,11 @@ class DirectoryPrefix(DictMixin): options include "primary" and "secondary".""" def __init__(self, **kwargs: Any) -> None: - self.name = kwargs.get('prefix') # type: ignore [assignment] - self.results_per_page = kwargs.get('results_per_page') # type: ignore [assignment] - self.file_system = kwargs.get('container') # type: ignore [assignment] - self.delimiter = kwargs.get('delimiter') # type: ignore [assignment] - self.location_mode = kwargs.get('location_mode') # type: ignore [assignment] + self.name = kwargs.get("prefix") # type: ignore [assignment] + self.results_per_page = kwargs.get("results_per_page") # type: ignore [assignment] + self.file_system = kwargs.get("container") # type: ignore [assignment] + self.delimiter = kwargs.get("delimiter") # type: ignore [assignment] + self.location_mode = kwargs.get("location_mode") # type: ignore [assignment] class DeletedPathPropertiesPaged(PageIterator): @@ -74,18 +67,17 @@ class DeletedPathPropertiesPaged(PageIterator): options include "primary" and "secondary".""" def __init__( - self, command: Callable, + self, + command: Callable, container: Optional[str] = None, prefix: Optional[str] = None, results_per_page: Optional[int] = None, continuation_token: Optional[str] = None, delimiter: Optional[str] = None, - location_mode: Optional[str] = None + location_mode: Optional[str] = None, ) -> None: super(DeletedPathPropertiesPaged, self).__init__( - get_next=self._get_next_cb, - extract_data=self._extract_data_cb, - continuation_token=continuation_token or "" + get_next=self._get_next_cb, extract_data=self._extract_data_cb, continuation_token=continuation_token or "" ) self._command = command self.service_endpoint = None @@ -104,7 +96,7 @@ def _get_next_cb(self, continuation_token): marker=continuation_token or None, max_results=self.results_per_page, cls=return_context_and_deserialized, - use_location=self.location_mode + use_location=self.location_mode, ) except HttpResponseError as error: process_storage_error(error) @@ -134,7 +126,7 @@ def _build_item( container=self.container, prefix=item.name, results_per_page=self.results_per_page, - location_mode=self.location_mode + location_mode=self.location_mode, ) return item @@ -157,17 +149,16 @@ class PathPropertiesPaged(PageIterator): """The path list to build the items for the current page.""" def __init__( - self, command: Callable, + self, + command: Callable, recursive: bool, path: Optional[str] = None, max_results: Optional[int] = None, continuation_token: Optional[str] = None, - upn: Optional[str] = None + upn: Optional[str] = None, ) -> None: super(PathPropertiesPaged, self).__init__( - get_next=self._get_next_cb, - extract_data=self._extract_data_cb, - continuation_token=continuation_token or "" + get_next=self._get_next_cb, extract_data=self._extract_data_cb, continuation_token=continuation_token or "" ) self._command = command self.recursive = recursive @@ -185,7 +176,7 @@ def _get_next_cb(self, continuation_token): path=self.path, max_results=self.results_per_page, upn=self.upn, - cls=return_headers_and_deserialized_path_list + cls=return_headers_and_deserialized_path_list, ) except HttpResponseError as error: process_storage_error(error) @@ -194,7 +185,7 @@ def _extract_data_cb(self, get_next_return): self.path_list, self._response = cast(Tuple[List[Path], Dict[str, Any]], get_next_return) self.current_page = [self._build_item(item) for item in self.path_list] - return self._response['continuation'] or None, self.current_page + return self._response["continuation"] or None, self.current_page @staticmethod def _build_item(item: Union[Path, PathProperties]) -> PathProperties: diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py index 0a5660a8be26..3bb7949c17b5 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py @@ -6,10 +6,7 @@ # pylint: disable=too-few-public-methods, too-many-instance-attributes, super-init-not-called, too-many-lines from enum import Enum -from typing import ( - Any, Dict, List, Optional, Union, - TYPE_CHECKING -) +from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING from typing_extensions import Self from azure.core import CaseInsensitiveEnumMeta @@ -29,7 +26,7 @@ Logging as GenLogging, Metrics as GenMetrics, RetentionPolicy as GenRetentionPolicy, - StaticWebsite as GenStaticWebsite + StaticWebsite as GenStaticWebsite, ) from azure.storage.blob._models import ContainerPropertiesPaged @@ -91,7 +88,7 @@ class Metrics(GenMetrics): policy will be disabled by default. """ - version: str = '1.0' + version: str = "1.0" """The version of Storage Analytics to configure.""" enabled: bool = False """Indicates whether metrics are enabled for the Datalake service.""" @@ -101,10 +98,10 @@ class Metrics(GenMetrics): """Determines how long the associated data should persist.""" def __init__(self, **kwargs: Any) -> None: - self.version = kwargs.get('version', '1.0') - self.enabled = kwargs.get('enabled', False) - self.include_apis = kwargs.get('include_apis') - self.retention_policy = kwargs.get('retention_policy') or RetentionPolicy() + self.version = kwargs.get("version", "1.0") + self.enabled = kwargs.get("enabled", False) + self.include_apis = kwargs.get("include_apis") + self.retention_policy = kwargs.get("retention_policy") or RetentionPolicy() @classmethod def _from_generated(cls, generated): @@ -114,7 +111,9 @@ def _from_generated(cls, generated): version=generated.version, enabled=generated.enabled, include_apis=generated.include_apis, - retention_policy=RetentionPolicy._from_generated(generated.retention_policy) # pylint: disable=protected-access + retention_policy=RetentionPolicy._from_generated( + generated.retention_policy + ), # pylint: disable=protected-access ) @@ -164,11 +163,11 @@ class CorsRule(GenCorsRule): """The number of seconds that the client/browser should cache a pre-flight response.""" def __init__(self, allowed_origins: List[str], allowed_methods: List[str], **kwargs: Any) -> None: - self.allowed_origins = ','.join(allowed_origins) - self.allowed_methods = ','.join(allowed_methods) - self.allowed_headers = ','.join(kwargs.get('allowed_headers', [])) - self.exposed_headers = ','.join(kwargs.get('exposed_headers', [])) - self.max_age_in_seconds = kwargs.get('max_age_in_seconds', 0) + self.allowed_origins = ",".join(allowed_origins) + self.allowed_methods = ",".join(allowed_methods) + self.allowed_headers = ",".join(kwargs.get("allowed_headers", [])) + self.exposed_headers = ",".join(kwargs.get("exposed_headers", [])) + self.max_age_in_seconds = kwargs.get("max_age_in_seconds", 0) @staticmethod def _to_generated(rules: Optional[List["CorsRule"]]) -> Optional[List[GenCorsRule]]: @@ -224,16 +223,14 @@ class AccountSasPermissions(BlobAccountSasPermissions): """ def __init__( - self, read: bool = False, + self, + read: bool = False, write: bool = False, delete: bool = False, list: bool = False, # pylint: disable=redefined-builtin - create: bool = False + create: bool = False, ) -> None: - super(AccountSasPermissions, self).__init__( - read=read, create=create, write=write, list=list, - delete=delete - ) + super(AccountSasPermissions, self).__init__(read=read, create=create, write=write, list=list, delete=delete) class FileSystemSasPermissions: @@ -296,7 +293,8 @@ class FileSystemSasPermissions: """Allows the user to set permissions and POSIX ACLs on files and directories.""" def __init__( - self, read: bool = False, + self, + read: bool = False, write: bool = False, delete: bool = False, list: bool = False, # pylint: disable=redefined-builtin @@ -306,24 +304,26 @@ def __init__( self.write = write self.delete = delete self.list = list - self.add = kwargs.pop('add', None) - self.create = kwargs.pop('create', None) - self.tags = kwargs.pop('tags', None) - self.move = kwargs.pop('move', None) - self.execute = kwargs.pop('execute', None) - self.manage_ownership = kwargs.pop('manage_ownership', None) - self.manage_access_control = kwargs.pop('manage_access_control', None) - self._str = (('r' if self.read else '') + - ('a' if self.add else '') + - ('c' if self.create else '') + - ('w' if self.write else '') + - ('d' if self.delete else '') + - ('l' if self.list else '') + - ('t' if self.tags else '') + - ('m' if self.move else '') + - ('e' if self.execute else '') + - ('o' if self.manage_ownership else '') + - ('p' if self.manage_access_control else '')) + self.add = kwargs.pop("add", None) + self.create = kwargs.pop("create", None) + self.tags = kwargs.pop("tags", None) + self.move = kwargs.pop("move", None) + self.execute = kwargs.pop("execute", None) + self.manage_ownership = kwargs.pop("manage_ownership", None) + self.manage_access_control = kwargs.pop("manage_access_control", None) + self._str = ( + ("r" if self.read else "") + + ("a" if self.add else "") + + ("c" if self.create else "") + + ("w" if self.write else "") + + ("d" if self.delete else "") + + ("l" if self.list else "") + + ("t" if self.tags else "") + + ("m" if self.move else "") + + ("e" if self.execute else "") + + ("o" if self.manage_ownership else "") + + ("p" if self.manage_access_control else "") + ) def __str__(self): return self._str @@ -341,22 +341,31 @@ def from_string(cls, permission: str) -> Self: :return: A FileSystemSasPermissions object :rtype: ~azure.storage.filedatalake.FileSystemSasPermissions """ - p_read = 'r' in permission - p_add = 'a' in permission - p_create = 'c' in permission - p_write = 'w' in permission - p_delete = 'd' in permission - p_list = 'l' in permission - p_tags = 't' in permission - p_move = 'm' in permission - p_execute = 'e' in permission - p_manage_ownership = 'o' in permission - p_manage_access_control = 'p' in permission - - parsed = cls(read=p_read, write=p_write, delete=p_delete, list=p_list, - tags=p_tags, add=p_add, create=p_create, move=p_move, - execute=p_execute, manage_ownership=p_manage_ownership, - manage_access_control=p_manage_access_control) + p_read = "r" in permission + p_add = "a" in permission + p_create = "c" in permission + p_write = "w" in permission + p_delete = "d" in permission + p_list = "l" in permission + p_tags = "t" in permission + p_move = "m" in permission + p_execute = "e" in permission + p_manage_ownership = "o" in permission + p_manage_access_control = "p" in permission + + parsed = cls( + read=p_read, + write=p_write, + delete=p_delete, + list=p_list, + tags=p_tags, + add=p_add, + create=p_create, + move=p_move, + execute=p_execute, + manage_ownership=p_manage_ownership, + manage_access_control=p_manage_access_control, + ) return parsed @@ -420,34 +429,32 @@ class DirectorySasPermissions: """Allows the user to set permissions and POSIX ACLs on files and directories.""" def __init__( - self, read: bool = False, - create: bool = False, - write: bool = False, - delete: bool = False, - **kwargs: Any + self, read: bool = False, create: bool = False, write: bool = False, delete: bool = False, **kwargs: Any ) -> None: self.read = read self.create = create self.write = write self.delete = delete - self.add = kwargs.pop('add', None) - self.list = kwargs.pop('list', None) - self.tags = kwargs.pop('tags', None) - self.move = kwargs.pop('move', None) - self.execute = kwargs.pop('execute', None) - self.manage_ownership = kwargs.pop('manage_ownership', None) - self.manage_access_control = kwargs.pop('manage_access_control', None) - self._str = (('r' if self.read else '') + - ('a' if self.add else '') + - ('c' if self.create else '') + - ('w' if self.write else '') + - ('d' if self.delete else '') + - ('l' if self.list else '') + - ('t' if self.tags else '') + - ('m' if self.move else '') + - ('e' if self.execute else '') + - ('o' if self.manage_ownership else '') + - ('p' if self.manage_access_control else '')) + self.add = kwargs.pop("add", None) + self.list = kwargs.pop("list", None) + self.tags = kwargs.pop("tags", None) + self.move = kwargs.pop("move", None) + self.execute = kwargs.pop("execute", None) + self.manage_ownership = kwargs.pop("manage_ownership", None) + self.manage_access_control = kwargs.pop("manage_access_control", None) + self._str = ( + ("r" if self.read else "") + + ("a" if self.add else "") + + ("c" if self.create else "") + + ("w" if self.write else "") + + ("d" if self.delete else "") + + ("l" if self.list else "") + + ("t" if self.tags else "") + + ("m" if self.move else "") + + ("e" if self.execute else "") + + ("o" if self.manage_ownership else "") + + ("p" if self.manage_access_control else "") + ) def __str__(self): return self._str @@ -465,21 +472,31 @@ def from_string(cls, permission: str) -> Self: :return: A DirectorySasPermissions object :rtype: ~azure.storage.filedatalake.DirectorySasPermissions """ - p_read = 'r' in permission - p_add = 'a' in permission - p_create = 'c' in permission - p_write = 'w' in permission - p_delete = 'd' in permission - p_list = 'l' in permission - p_tags = 't' in permission - p_move = 'm' in permission - p_execute = 'e' in permission - p_manage_ownership = 'o' in permission - p_manage_access_control = 'p' in permission - - parsed = cls(read=p_read, create=p_create, write=p_write, delete=p_delete, add=p_add, list=p_list, - tags=p_tags, move=p_move, execute=p_execute, manage_ownership=p_manage_ownership, - manage_access_control=p_manage_access_control) + p_read = "r" in permission + p_add = "a" in permission + p_create = "c" in permission + p_write = "w" in permission + p_delete = "d" in permission + p_list = "l" in permission + p_tags = "t" in permission + p_move = "m" in permission + p_execute = "e" in permission + p_manage_ownership = "o" in permission + p_manage_access_control = "p" in permission + + parsed = cls( + read=p_read, + create=p_create, + write=p_write, + delete=p_delete, + add=p_add, + list=p_list, + tags=p_tags, + move=p_move, + execute=p_execute, + manage_ownership=p_manage_ownership, + manage_access_control=p_manage_access_control, + ) return parsed @@ -539,32 +556,30 @@ class FileSasPermissions: """Allows the user to set permissions and POSIX ACLs on files and directories.""" def __init__( - self, read: bool = False, - create: bool = False, - write: bool = False, - delete: bool = False, - **kwargs: Any + self, read: bool = False, create: bool = False, write: bool = False, delete: bool = False, **kwargs: Any ) -> None: self.read = read self.create = create self.write = write self.delete = delete - self.add = kwargs.pop('add', None) - self.tags = kwargs.pop('tags', None) - self.move = kwargs.pop('move', None) - self.execute = kwargs.pop('execute', None) - self.manage_ownership = kwargs.pop('manage_ownership', None) - self.manage_access_control = kwargs.pop('manage_access_control', None) - self._str = (('r' if self.read else '') + - ('a' if self.add else '') + - ('c' if self.create else '') + - ('w' if self.write else '') + - ('d' if self.delete else '') + - ('t' if self.tags else '') + - ('m' if self.move else '') + - ('e' if self.execute else '') + - ('o' if self.manage_ownership else '') + - ('p' if self.manage_access_control else '')) + self.add = kwargs.pop("add", None) + self.tags = kwargs.pop("tags", None) + self.move = kwargs.pop("move", None) + self.execute = kwargs.pop("execute", None) + self.manage_ownership = kwargs.pop("manage_ownership", None) + self.manage_access_control = kwargs.pop("manage_access_control", None) + self._str = ( + ("r" if self.read else "") + + ("a" if self.add else "") + + ("c" if self.create else "") + + ("w" if self.write else "") + + ("d" if self.delete else "") + + ("t" if self.tags else "") + + ("m" if self.move else "") + + ("e" if self.execute else "") + + ("o" if self.manage_ownership else "") + + ("p" if self.manage_access_control else "") + ) def __str__(self): return self._str @@ -582,20 +597,29 @@ def from_string(cls, permission: str) -> Self: :return: A FileSasPermissions object :rtype: ~azure.storage.filedatalake.FileSasPermissions """ - p_read = 'r' in permission - p_add = 'a' in permission - p_create = 'c' in permission - p_write = 'w' in permission - p_delete = 'd' in permission - p_tags = 't' in permission - p_move = 'm' in permission - p_execute = 'e' in permission - p_manage_ownership = 'o' in permission - p_manage_access_control = 'p' in permission - - parsed = cls(read=p_read, create=p_create, write=p_write, delete=p_delete, add=p_add, - tags=p_tags, move=p_move, execute=p_execute, manage_ownership=p_manage_ownership, - manage_access_control=p_manage_access_control) + p_read = "r" in permission + p_add = "a" in permission + p_create = "c" in permission + p_write = "w" in permission + p_delete = "d" in permission + p_tags = "t" in permission + p_move = "m" in permission + p_execute = "e" in permission + p_manage_ownership = "o" in permission + p_manage_access_control = "p" in permission + + parsed = cls( + read=p_read, + create=p_create, + write=p_write, + delete=p_delete, + add=p_add, + tags=p_tags, + move=p_move, + execute=p_execute, + manage_ownership=p_manage_ownership, + manage_access_control=p_manage_access_control, + ) return parsed @@ -643,12 +667,13 @@ class AccessPolicy(BlobAccessPolicy): """ def __init__( - self, permission: Optional[Union[FileSystemSasPermissions, str]] = None, + self, + permission: Optional[Union[FileSystemSasPermissions, str]] = None, expiry: Optional[Union["datetime", str]] = None, **kwargs: Any ) -> None: super(AccessPolicy, self).__init__( - permission=permission, expiry=expiry, start=kwargs.pop('start', None) # type: ignore [arg-type] + permission=permission, expiry=expiry, start=kwargs.pop("start", None) # type: ignore [arg-type] ) @@ -740,11 +765,11 @@ def __init__(self, **kwargs: Any) -> None: self.metadata = None # type: ignore [assignment] self.deleted = None self.deleted_version = None - default_encryption_scope = kwargs.get('x-ms-default-encryption-scope') + default_encryption_scope = kwargs.get("x-ms-default-encryption-scope") if default_encryption_scope: self.encryption_scope = EncryptionScopeOptions( default_encryption_scope=default_encryption_scope, - prevent_encryption_scope_override=kwargs.get('x-ms-deny-encryption-scope-override', False) + prevent_encryption_scope_override=kwargs.get("x-ms-deny-encryption-scope-override", False), ) @classmethod @@ -757,7 +782,8 @@ def _from_generated(cls, generated): props.etag = generated.properties.etag props.lease = LeaseProperties._from_generated(generated) # pylint: disable=protected-access props.public_access = PublicAccess._from_generated( # pylint: disable=protected-access - generated.properties.public_access) + generated.properties.public_access + ) props.has_immutability_policy = generated.properties.has_immutability_policy props.has_legal_hold = generated.properties.has_legal_hold props.metadata = generated.metadata @@ -768,7 +794,8 @@ def _from_generated(cls, generated): def _convert_from_container_props(cls, container_properties): container_properties.__class__ = cls container_properties.public_access = PublicAccess._from_generated( # pylint: disable=protected-access - container_properties.public_access) + container_properties.public_access + ) container_properties.lease.__class__ = LeaseProperties return container_properties @@ -829,22 +856,22 @@ class DirectoryProperties(DictMixin): """The POSIX ACL permissions of the file or directory.""" def __init__(self, **kwargs: Any) -> None: - self.name = kwargs.get('name') # type: ignore [assignment] - self.etag = kwargs.get('ETag') # type: ignore [assignment] + self.name = kwargs.get("name") # type: ignore [assignment] + self.etag = kwargs.get("ETag") # type: ignore [assignment] self.deleted = False - self.metadata = kwargs.get('metadata') # type: ignore [assignment] + self.metadata = kwargs.get("metadata") # type: ignore [assignment] self.lease = LeaseProperties(**kwargs) - self.last_modified = kwargs.get('Last-Modified') # type: ignore [assignment] - self.creation_time = kwargs.get('x-ms-creation-time') # type: ignore [assignment] + self.last_modified = kwargs.get("Last-Modified") # type: ignore [assignment] + self.creation_time = kwargs.get("x-ms-creation-time") # type: ignore [assignment] self.deleted_time = None self.remaining_retention_days = None - self.encryption_scope = kwargs.get('x-ms-encryption-scope') + self.encryption_scope = kwargs.get("x-ms-encryption-scope") # This is being passed directly not coming from headers - self.owner = kwargs.get('owner', None) - self.group = kwargs.get('group', None) - self.permissions = kwargs.get('permissions', None) - self.acl = kwargs.get('acl', None) + self.owner = kwargs.get("owner", None) + self.group = kwargs.get("group", None) + self.permissions = kwargs.get("permissions", None) + self.acl = kwargs.get("acl", None) class FileProperties(DictMixin): @@ -892,26 +919,26 @@ class FileProperties(DictMixin): """The POSIX ACL permissions of the file or directory.""" def __init__(self, **kwargs: Any) -> None: - self.name = kwargs.get('name') # type: ignore [assignment] - self.etag = kwargs.get('ETag') # type: ignore [assignment] + self.name = kwargs.get("name") # type: ignore [assignment] + self.etag = kwargs.get("ETag") # type: ignore [assignment] self.deleted = False - self.metadata = kwargs.get('metadata') # type: ignore [assignment] + self.metadata = kwargs.get("metadata") # type: ignore [assignment] self.lease = LeaseProperties(**kwargs) - self.last_modified = kwargs.get('Last-Modified') # type: ignore [assignment] - self.creation_time = kwargs.get('x-ms-creation-time') # type: ignore [assignment] - self.size = kwargs.get('Content-Length') # type: ignore [assignment] + self.last_modified = kwargs.get("Last-Modified") # type: ignore [assignment] + self.creation_time = kwargs.get("x-ms-creation-time") # type: ignore [assignment] + self.size = kwargs.get("Content-Length") # type: ignore [assignment] self.deleted_time = None self.expiry_time = kwargs.get("x-ms-expiry-time") self.remaining_retention_days = None self.content_settings = ContentSettings(**kwargs) - self.encryption_scope = kwargs.get('x-ms-encryption-scope') + self.encryption_scope = kwargs.get("x-ms-encryption-scope") # This is being passed directly not coming from headers - self.encryption_context = kwargs.get('encryption_context') - self.owner = kwargs.get('owner', None) - self.group = kwargs.get('group', None) - self.permissions = kwargs.get('permissions', None) - self.acl = kwargs.get('acl', None) + self.encryption_context = kwargs.get("encryption_context") + self.owner = kwargs.get("owner", None) + self.group = kwargs.get("group", None) + self.permissions = kwargs.get("permissions", None) + self.acl = kwargs.get("acl", None) class PathProperties(DictMixin): @@ -955,18 +982,18 @@ class PathProperties(DictMixin): """Specifies the encryption context to set on the file.""" def __init__(self, **kwargs: Any) -> None: - self.name = kwargs.pop('name', None) # type: ignore [assignment] - self.owner = kwargs.get('owner', None) # type: ignore [assignment] - self.group = kwargs.get('group', None) # type: ignore [assignment] - self.permissions = kwargs.get('permissions', None) # type: ignore [assignment] - self.last_modified = kwargs.get('last_modified', None) # type: ignore [assignment] - self.is_directory = kwargs.get('is_directory', False) # type: ignore [assignment] - self.etag = kwargs.get('etag', None) # type: ignore [assignment] - self.content_length = kwargs.get('content_length', None) # type: ignore [assignment] - self.creation_time = kwargs.get('creation_time', None) # type: ignore [assignment] - self.expiry_time = kwargs.get('expiry_time', None) - self.encryption_scope = kwargs.get('x-ms-encryption-scope', None) - self.encryption_context = kwargs.get('x-ms-encryption-context', None) + self.name = kwargs.pop("name", None) # type: ignore [assignment] + self.owner = kwargs.get("owner", None) # type: ignore [assignment] + self.group = kwargs.get("group", None) # type: ignore [assignment] + self.permissions = kwargs.get("permissions", None) # type: ignore [assignment] + self.last_modified = kwargs.get("last_modified", None) # type: ignore [assignment] + self.is_directory = kwargs.get("is_directory", False) # type: ignore [assignment] + self.etag = kwargs.get("etag", None) # type: ignore [assignment] + self.content_length = kwargs.get("content_length", None) # type: ignore [assignment] + self.creation_time = kwargs.get("creation_time", None) # type: ignore [assignment] + self.expiry_time = kwargs.get("expiry_time", None) + self.encryption_scope = kwargs.get("x-ms-encryption-scope", None) + self.encryption_context = kwargs.get("x-ms-encryption-context", None) @classmethod def _from_generated(cls, generated): @@ -977,7 +1004,7 @@ def _from_generated(cls, generated): path_prop.permissions = generated.permissions path_prop.last_modified = _rfc_1123_to_datetime(generated.last_modified) path_prop.is_directory = bool(generated.is_directory) - path_prop.etag = generated.additional_properties.get('etag') + path_prop.etag = generated.additional_properties.get("etag") path_prop.content_length = generated.content_length path_prop.creation_time = _filetime_to_datetime(generated.creation_time) path_prop.expiry_time = _filetime_to_datetime(generated.expiry_time) @@ -999,9 +1026,10 @@ class ResourceTypes(BlobResourceTypes): """ def __init__( - self, service: bool = False, + self, + service: bool = False, file_system: bool = False, - object: bool = False # pylint: disable=redefined-builtin + object: bool = False, # pylint: disable=redefined-builtin ) -> None: super(ResourceTypes, self).__init__(service=service, container=file_system, object=object) @@ -1034,14 +1062,14 @@ class PublicAccess(str, Enum, metaclass=CaseInsensitiveEnumMeta): Specifies whether data in the file system may be accessed publicly and the level of access. """ - FILE = 'blob' + FILE = "blob" """ Specifies public read access for files. file data within this file system can be read via anonymous request, but file system data is not available. Clients cannot enumerate files within the container via anonymous request. """ - FILESYSTEM = 'container' + FILESYSTEM = "container" """ Specifies full public read access for file system and file data. Clients can enumerate files within the file system via anonymous request, but cannot enumerate file systems @@ -1065,8 +1093,8 @@ class LocationMode: must use PRIMARY. """ - PRIMARY = 'primary' #: Requests should be sent to the primary location. - SECONDARY = 'secondary' #: Requests should be sent to the secondary location, if possible. + PRIMARY = "primary" #: Requests should be sent to the primary location. + SECONDARY = "secondary" #: Requests should be sent to the secondary location, if possible. class DelimitedJsonDialect(BlobDelimitedJSON): @@ -1131,9 +1159,9 @@ class CustomerProvidedEncryptionKey(BlobCustomerProvidedEncryptionKey): class QuickQueryDialect(str, Enum, metaclass=CaseInsensitiveEnumMeta): """Specifies the quick query input/output dialect.""" - DELIMITEDTEXT = 'DelimitedTextDialect' - DELIMITEDJSON = 'DelimitedJsonDialect' - PARQUET = 'ParquetDialect' + DELIMITEDTEXT = "DelimitedTextDialect" + DELIMITEDJSON = "DelimitedJsonDialect" + PARQUET = "ParquetDialect" class ArrowType(str, Enum, metaclass=CaseInsensitiveEnumMeta): @@ -1143,7 +1171,7 @@ class ArrowType(str, Enum, metaclass=CaseInsensitiveEnumMeta): TIMESTAMP_MS = "timestamp[ms]" STRING = "string" DOUBLE = "double" - DECIMAL = 'decimal' + DECIMAL = "decimal" class DataLakeFileQueryError: @@ -1161,10 +1189,11 @@ class DataLakeFileQueryError: """The blob offset at which the error occurred.""" def __init__( - self, error: Optional[str] = None, + self, + error: Optional[str] = None, is_fatal: bool = False, description: Optional[str] = None, - position: Optional[int] = None + position: Optional[int] = None, ) -> None: self.error = error self.is_fatal = is_fatal @@ -1235,10 +1264,11 @@ class AccessControlChanges(DictMixin): """An opaque continuation token that may be used to resume the operations in case of failures.""" def __init__( - self, batch_counters: AccessControlChangeCounters, + self, + batch_counters: AccessControlChangeCounters, aggregate_counters: AccessControlChangeCounters, batch_failures: List[AccessControlChangeFailure], - continuation: Optional[str] + continuation: Optional[str], ) -> None: self.batch_counters = batch_counters self.aggregate_counters = aggregate_counters @@ -1261,7 +1291,7 @@ class DeletedPathProperties(DictMixin): """The filesystem associated with the deleted path.""" def __init__(self, **kwargs: Any) -> None: - self.name = kwargs.get('name') # type: ignore [assignment] + self.name = kwargs.get("name") # type: ignore [assignment] self.deleted_time = None self.remaining_retention_days = None self.deletion_id = None @@ -1284,11 +1314,11 @@ class AnalyticsLogging(GenLogging): policy will be disabled by default.""" def __init__(self, **kwargs: Any) -> None: - self.version = kwargs.get('version', '1.0') - self.delete = kwargs.get('delete', False) - self.read = kwargs.get('read', False) - self.write = kwargs.get('write', False) - self.retention_policy = kwargs.get('retention_policy') or RetentionPolicy() + self.version = kwargs.get("version", "1.0") + self.delete = kwargs.get("delete", False) + self.read = kwargs.get("read", False) + self.write = kwargs.get("write", False) + self.retention_policy = kwargs.get("retention_policy") or RetentionPolicy() @classmethod def _from_generated(cls, generated): @@ -1299,7 +1329,9 @@ def _from_generated(cls, generated): delete=generated.delete, read=generated.read, write=generated.write, - retention_policy=RetentionPolicy._from_generated(generated.retention_policy) # pylint: disable=protected-access + retention_policy=RetentionPolicy._from_generated( + generated.retention_policy + ), # pylint: disable=protected-access ) @@ -1327,11 +1359,11 @@ class StaticWebsite(GenStaticWebsite): """Absolute path of the default index page.""" def __init__(self, **kwargs: Any) -> None: - self.enabled = kwargs.get('enabled', False) + self.enabled = kwargs.get("enabled", False) if self.enabled: - self.index_document = kwargs.get('index_document') - self.error_document404_path = kwargs.get('error_document404_path') - self.default_index_document_path = kwargs.get('default_index_document_path') + self.index_document = kwargs.get("index_document") + self.error_document404_path = kwargs.get("error_document404_path") + self.default_index_document_path = kwargs.get("default_index_document_path") else: self.index_document = None self.error_document404_path = None @@ -1345,5 +1377,5 @@ def _from_generated(cls, generated): enabled=generated.enabled, index_document=generated.index_document, error_document404_path=generated.error_document404_path, - default_index_document_path=generated.default_index_document_path + default_index_document_path=generated.default_index_document_path, ) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py index 54e1ecf13492..537510ed00a0 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py @@ -6,10 +6,7 @@ # pylint: disable=docstring-keyword-should-match-keyword-only from datetime import datetime -from typing import ( - Any, Callable, cast, Dict, Optional, Union, - TYPE_CHECKING -) +from typing import Any, Callable, cast, Dict, Optional, Union, TYPE_CHECKING from typing_extensions import Self from azure.core.exceptions import AzureError, HttpResponseError @@ -35,7 +32,7 @@ _parse_url, _rename_path_options, _set_access_control_options, - _set_access_control_recursive_options + _set_access_control_recursive_options, ) from ._shared.base_client import StorageAccountHostsMixin, parse_query from ._serialize import ( @@ -80,19 +77,23 @@ class PathClient(StorageAccountHostsMixin): authentication. Only has an effect when credential is of type TokenCredential. The value could be https://storage.azure.com/ (default) or https://.blob.core.windows.net. """ + def __init__( - self, account_url: str, + self, + account_url: str, file_system_name: str, path_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"]] = None, # pylint: disable=line-too-long + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential"] + ] = None, # pylint: disable=line-too-long **kwargs: Any ) -> None: # remove the preceding/trailing delimiter from the path components - file_system_name = file_system_name.strip('/') + file_system_name = file_system_name.strip("/") # the name of root directory is / - if path_name != '/': - path_name = path_name.strip('/') + if path_name != "/": + path_name = path_name.strip("/") if not (file_system_name and path_name): raise ValueError("Please specify a file system name and file path.") @@ -101,14 +102,11 @@ def __init__( blob_account_url = convert_dfs_url_to_blob_url(account_url) self._blob_account_url = blob_account_url - datalake_hosts = kwargs.pop('_hosts', None) + datalake_hosts = kwargs.pop("_hosts", None) blob_hosts = None if datalake_hosts: blob_primary_account_url = convert_dfs_url_to_blob_url(datalake_hosts[LocationMode.PRIMARY]) - blob_hosts = { - LocationMode.PRIMARY: blob_primary_account_url, - LocationMode.SECONDARY: "" - } + blob_hosts = {LocationMode.PRIMARY: blob_primary_account_url, LocationMode.SECONDARY: ""} self._blob_client = BlobClient( account_url=blob_account_url, container_name=file_system_name, @@ -125,11 +123,7 @@ def __init__( self._query_str, self._raw_credential = self._format_query_string(sas_token, credential) super(PathClient, self).__init__( - parsed_url, - service='dfs', - credential=self._raw_credential, - _hosts=datalake_hosts, - **kwargs + parsed_url, service="dfs", credential=self._raw_credential, _hosts=datalake_hosts, **kwargs ) # ADLS doesn't support secondary endpoint, make sure it's empty @@ -167,7 +161,7 @@ def _build_generated_client(self, url: str) -> AzureDataLakeStorageRESTAPI: base_url=url, file_system=self.file_system_name, path=self.path_name, - pipeline=self._pipeline + pipeline=self._pipeline, ) return client @@ -175,7 +169,8 @@ def _format_url(self, hostname: str) -> str: return _format_url(self.scheme, hostname, self.file_system_name, self.path_name, self._query_str) def _create( - self, resource_type: str, + self, + resource_type: str, content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, **kwargs: Any @@ -269,8 +264,8 @@ def _create( :return: A dictionary of response headers. :rtype: Dict[str, Union[str, ~datetime.datetime]] """ - lease_id = kwargs.get('lease_id', None) - lease_duration = kwargs.get('lease_duration', None) + lease_id = kwargs.get("lease_id", None) + lease_duration = kwargs.get("lease_duration", None) if lease_id and not lease_duration: raise ValueError("Please specify a lease_id and a lease_duration.") if lease_duration and not lease_id: @@ -318,19 +313,19 @@ def _delete(self, **kwargs: Any) -> Dict[str, Any]: # Perform paginated delete only if using OAuth, deleting a directory, and api version is 2023-08-03 or later # The pagination is only for ACL checks, the final request remains the atomic delete operation paginated = None - if (compare_api_versions(self.api_version, '2023-08-03') >= 0 and - hasattr(self.credential, 'get_token') and - kwargs.get('recursive')): # Directory delete will always specify recursive + if ( + compare_api_versions(self.api_version, "2023-08-03") >= 0 + and hasattr(self.credential, "get_token") + and kwargs.get("recursive") + ): # Directory delete will always specify recursive paginated = True options = _delete_path_options(paginated, **kwargs) try: response_headers = self._client.path.delete(**options) # Loop until continuation token is None for paginated delete - while response_headers['continuation']: - response_headers = self._client.path.delete( - continuation=response_headers['continuation'], - **options) + while response_headers["continuation"]: + response_headers = self._client.path.delete(continuation=response_headers["continuation"], **options) return response_headers except HttpResponseError as error: @@ -338,7 +333,8 @@ def _delete(self, **kwargs: Any) -> Dict[str, Any]: @distributed_trace def set_access_control( - self, owner: Optional[str] = None, + self, + owner: Optional[str] = None, group: Optional[str] = None, permissions: Optional[str] = None, acl: Optional[str] = None, @@ -504,11 +500,10 @@ def set_access_control_recursive(self, acl: str, **kwargs: Any) -> AccessControl if not acl: raise ValueError("The Access Control List must be set for this operation") - progress_hook = kwargs.pop('progress_hook', None) - max_batches = kwargs.pop('max_batches', None) - options = _set_access_control_recursive_options(mode='set', acl=acl, **kwargs) - return self._set_access_control_internal(options=options, progress_hook=progress_hook, - max_batches=max_batches) + progress_hook = kwargs.pop("progress_hook", None) + max_batches = kwargs.pop("max_batches", None) + options = _set_access_control_recursive_options(mode="set", acl=acl, **kwargs) + return self._set_access_control_internal(options=options, progress_hook=progress_hook, max_batches=max_batches) @distributed_trace def update_access_control_recursive(self, acl: str, **kwargs: Any) -> AccessControlChangeResult: @@ -557,11 +552,10 @@ def update_access_control_recursive(self, acl: str, **kwargs: Any) -> AccessCont if not acl: raise ValueError("The Access Control List must be set for this operation") - progress_hook = kwargs.pop('progress_hook', None) - max_batches = kwargs.pop('max_batches', None) - options = _set_access_control_recursive_options(mode='modify', acl=acl, **kwargs) - return self._set_access_control_internal(options=options, progress_hook=progress_hook, - max_batches=max_batches) + progress_hook = kwargs.pop("progress_hook", None) + max_batches = kwargs.pop("max_batches", None) + options = _set_access_control_recursive_options(mode="modify", acl=acl, **kwargs) + return self._set_access_control_internal(options=options, progress_hook=progress_hook, max_batches=max_batches) @distributed_trace def remove_access_control_recursive(self, acl: str, **kwargs: Any) -> AccessControlChangeResult: @@ -609,19 +603,19 @@ def remove_access_control_recursive(self, acl: str, **kwargs: Any) -> AccessCont if not acl: raise ValueError("The Access Control List must be set for this operation") - progress_hook = kwargs.pop('progress_hook', None) - max_batches = kwargs.pop('max_batches', None) - options = _set_access_control_recursive_options(mode='remove', acl=acl, **kwargs) - return self._set_access_control_internal(options=options, progress_hook=progress_hook, - max_batches=max_batches) + progress_hook = kwargs.pop("progress_hook", None) + max_batches = kwargs.pop("max_batches", None) + options = _set_access_control_recursive_options(mode="remove", acl=acl, **kwargs) + return self._set_access_control_internal(options=options, progress_hook=progress_hook, max_batches=max_batches) def _set_access_control_internal( - self, options: Dict[str, Any], + self, + options: Dict[str, Any], progress_hook: Optional[Callable[[AccessControlChanges], None]], - max_batches: Optional[int] = None + max_batches: Optional[int] = None, ) -> AccessControlChangeResult: try: - continue_on_failure = options.get('force_flag') + continue_on_failure = options.get("force_flag") total_directories_successful = 0 total_files_success = 0 total_failure_count = 0 @@ -637,44 +631,56 @@ def _set_access_control_internal( total_files_success += resp.files_successful total_failure_count += resp.failure_count batch_count += 1 - current_continuation_token = headers['continuation'] + current_continuation_token = headers["continuation"] if current_continuation_token is not None: last_continuation_token = current_continuation_token if progress_hook is not None: - progress_hook(AccessControlChanges( - batch_counters=AccessControlChangeCounters( - directories_successful=resp.directories_successful, - files_successful=resp.files_successful, - failure_count=resp.failure_count, - ), - aggregate_counters=AccessControlChangeCounters( - directories_successful=total_directories_successful, - files_successful=total_files_success, - failure_count=total_failure_count, - ), - batch_failures=[AccessControlChangeFailure( - name=failure.name, - is_directory=failure.type == 'DIRECTORY', - error_message=failure.error_message) for failure in resp.failed_entries], - continuation=last_continuation_token - )) + progress_hook( + AccessControlChanges( + batch_counters=AccessControlChangeCounters( + directories_successful=resp.directories_successful, + files_successful=resp.files_successful, + failure_count=resp.failure_count, + ), + aggregate_counters=AccessControlChangeCounters( + directories_successful=total_directories_successful, + files_successful=total_files_success, + failure_count=total_failure_count, + ), + batch_failures=[ + AccessControlChangeFailure( + name=failure.name, + is_directory=failure.type == "DIRECTORY", + error_message=failure.error_message, + ) + for failure in resp.failed_entries + ], + continuation=last_continuation_token, + ) + ) # update the continuation token, if there are more operations that cannot be completed in a single call - max_batches_satisfied = (max_batches is not None and batch_count == max_batches) + max_batches_satisfied = max_batches is not None and batch_count == max_batches continue_operation = bool(current_continuation_token) and not max_batches_satisfied - options['continuation'] = current_continuation_token + options["continuation"] = current_continuation_token # currently the service stops on any failure, so we should send back the last continuation token # for the user to retry the failed updates # otherwise we should just return what the service gave us - return AccessControlChangeResult(counters=AccessControlChangeCounters( - directories_successful=total_directories_successful, - files_successful=total_files_success, - failure_count=total_failure_count), - continuation=last_continuation_token - if total_failure_count > 0 and not continue_on_failure else current_continuation_token) + return AccessControlChangeResult( + counters=AccessControlChangeCounters( + directories_successful=total_directories_successful, + files_successful=total_files_success, + failure_count=total_failure_count, + ), + continuation=( + last_continuation_token + if total_failure_count > 0 and not continue_on_failure + else current_continuation_token + ), + ) except HttpResponseError as error: error.continuation_token = last_continuation_token process_storage_error(error) @@ -805,11 +811,11 @@ def _get_path_properties(self, **kwargs: Any) -> Union[DirectoryProperties, File :dedent: 8 :caption: Getting the properties for a file/directory. """ - upn = kwargs.pop('upn', None) + upn = kwargs.pop("upn", None) if upn: - headers = kwargs.pop('headers', {}) - headers['x-ms-upn'] = str(upn) - kwargs['headers'] = headers + headers = kwargs.pop("headers", {}) + headers["x-ms-upn"] = str(upn) + kwargs["headers"] = headers path_properties = self._blob_client.get_blob_properties(**kwargs) return cast(Union[DirectoryProperties, FileProperties], path_properties) @@ -916,9 +922,7 @@ def set_http_headers(self, content_settings: Optional["ContentSettings"] = None, @distributed_trace def acquire_lease( - self, lease_duration: int = -1, - lease_id: Optional[str] = None, - **kwargs: Any + self, lease_duration: int = -1, lease_id: Optional[str] = None, **kwargs: Any ) -> DataLakeLeaseClient: """ Requests a new lease. If the file or directory does not have an active lease, diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client_helpers.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client_helpers.py index 0ef3bad262f9..b10949582b5a 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client_helpers.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client_helpers.py @@ -5,10 +5,7 @@ # -------------------------------------------------------------------------- import re -from typing import ( - Any, Dict, Optional, Tuple, Union, - TYPE_CHECKING -) +from typing import Any, Dict, Optional, Tuple, Union, TYPE_CHECKING from urllib.parse import quote, urlparse from ._serialize import ( @@ -19,7 +16,7 @@ get_lease_id, get_mod_conditions, get_path_http_headers, - get_source_mod_conditions + get_source_mod_conditions, ) from ._shared.response_handlers import return_headers_and_deserialized, return_response_headers @@ -33,11 +30,11 @@ def _parse_url(account_url: str) -> "ParseResult": try: - if not account_url.lower().startswith('http'): + if not account_url.lower().startswith("http"): account_url = "https://" + account_url except AttributeError as exc: raise ValueError("Account URL must be a string.") from exc - parsed_url = urlparse(account_url.rstrip('/')) + parsed_url = urlparse(account_url.rstrip("/")) if not parsed_url.netloc: raise ValueError(f"Invalid URL: {account_url}") return parsed_url @@ -45,7 +42,7 @@ def _parse_url(account_url: str) -> "ParseResult": def _format_url(scheme: str, hostname: str, file_system_name: Union[str, bytes], path_name: str, query_str: str) -> str: if isinstance(file_system_name, str): - file_system_name = file_system_name.encode('UTF-8') + file_system_name = file_system_name.encode("UTF-8") return f"{scheme}://{hostname}/{quote(file_system_name)}/{quote(path_name, safe='~/')}{query_str}" @@ -54,9 +51,9 @@ def _create_path_options( scheme: str, content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Any]: - access_conditions = get_access_conditions(kwargs.pop('lease', None)) + access_conditions = get_access_conditions(kwargs.pop("lease", None)) mod_conditions = get_mod_conditions(kwargs) path_http_headers = None @@ -65,49 +62,49 @@ def _create_path_options( cpk_info = get_cpk_info(scheme, kwargs) - expires_on = kwargs.pop('expires_on', None) + expires_on = kwargs.pop("expires_on", None) if expires_on: try: expires_on = convert_datetime_to_rfc1123(expires_on) - kwargs['expiry_options'] = 'Absolute' + kwargs["expiry_options"] = "Absolute" except AttributeError: expires_on = str(expires_on) - kwargs['expiry_options'] = 'RelativeToNow' + kwargs["expiry_options"] = "RelativeToNow" options = { - 'resource': resource_type, - 'properties': add_metadata_headers(metadata), - 'permissions': kwargs.pop('permissions', None), - 'umask': kwargs.pop('umask', None), - 'owner': kwargs.pop('owner', None), - 'group': kwargs.pop('group', None), - 'acl': kwargs.pop('acl', None), - 'proposed_lease_id': kwargs.pop('lease_id', None), - 'lease_duration': kwargs.pop('lease_duration', None), - 'expiry_options': kwargs.pop('expiry_options', None), - 'expires_on': expires_on, - 'path_http_headers': path_http_headers, - 'lease_access_conditions': access_conditions, - 'modified_access_conditions': mod_conditions, - 'cpk_info': cpk_info, - 'timeout': kwargs.pop('timeout', None), - 'encryption_context': kwargs.pop('encryption_context', None), - 'cls': return_response_headers + "resource": resource_type, + "properties": add_metadata_headers(metadata), + "permissions": kwargs.pop("permissions", None), + "umask": kwargs.pop("umask", None), + "owner": kwargs.pop("owner", None), + "group": kwargs.pop("group", None), + "acl": kwargs.pop("acl", None), + "proposed_lease_id": kwargs.pop("lease_id", None), + "lease_duration": kwargs.pop("lease_duration", None), + "expiry_options": kwargs.pop("expiry_options", None), + "expires_on": expires_on, + "path_http_headers": path_http_headers, + "lease_access_conditions": access_conditions, + "modified_access_conditions": mod_conditions, + "cpk_info": cpk_info, + "timeout": kwargs.pop("timeout", None), + "encryption_context": kwargs.pop("encryption_context", None), + "cls": return_response_headers, } options.update(kwargs) return options def _delete_path_options(paginated: Optional[bool], **kwargs) -> Dict[str, Any]: - access_conditions = get_access_conditions(kwargs.pop('lease', None)) + access_conditions = get_access_conditions(kwargs.pop("lease", None)) mod_conditions = get_mod_conditions(kwargs) options = { - 'paginated': paginated, - 'lease_access_conditions': access_conditions, - 'modified_access_conditions': mod_conditions, - 'cls': return_response_headers, - 'timeout': kwargs.pop('timeout', None) + "paginated": paginated, + "lease_access_conditions": access_conditions, + "modified_access_conditions": mod_conditions, + "cls": return_response_headers, + "timeout": kwargs.pop("timeout", None), } options.update(kwargs) return options @@ -118,36 +115,36 @@ def _set_access_control_options( group: Optional[str] = None, permissions: Optional[str] = None, acl: Optional[str] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Any]: - access_conditions = get_access_conditions(kwargs.pop('lease', None)) + access_conditions = get_access_conditions(kwargs.pop("lease", None)) mod_conditions = get_mod_conditions(kwargs) options = { - 'owner': owner, - 'group': group, - 'permissions': permissions, - 'acl': acl, - 'lease_access_conditions': access_conditions, - 'modified_access_conditions': mod_conditions, - 'timeout': kwargs.pop('timeout', None), - 'cls': return_response_headers + "owner": owner, + "group": group, + "permissions": permissions, + "acl": acl, + "lease_access_conditions": access_conditions, + "modified_access_conditions": mod_conditions, + "timeout": kwargs.pop("timeout", None), + "cls": return_response_headers, } options.update(kwargs) return options def _get_access_control_options(upn: Optional[bool] = None, **kwargs: Any) -> Dict[str, Any]: - access_conditions = get_access_conditions(kwargs.pop('lease', None)) + access_conditions = get_access_conditions(kwargs.pop("lease", None)) mod_conditions = get_mod_conditions(kwargs) options = { - 'action': 'getAccessControl', - 'upn': upn if upn else False, - 'lease_access_conditions': access_conditions, - 'modified_access_conditions': mod_conditions, - 'timeout': kwargs.pop('timeout', None), - 'cls': return_response_headers + "action": "getAccessControl", + "upn": upn if upn else False, + "lease_access_conditions": access_conditions, + "modified_access_conditions": mod_conditions, + "timeout": kwargs.pop("timeout", None), + "cls": return_response_headers, } options.update(kwargs) return options @@ -155,13 +152,13 @@ def _get_access_control_options(upn: Optional[bool] = None, **kwargs: Any) -> Di def _set_access_control_recursive_options(mode: str, acl: str, **kwargs: Any) -> Dict[str, Any]: options = { - 'mode': mode, - 'force_flag': kwargs.pop('continue_on_failure', None), - 'timeout': kwargs.pop('timeout', None), - 'continuation': kwargs.pop('continuation_token', None), - 'max_records': kwargs.pop('batch_size', None), - 'acl': acl, - 'cls': return_headers_and_deserialized + "mode": mode, + "force_flag": kwargs.pop("continue_on_failure", None), + "timeout": kwargs.pop("timeout", None), + "continuation": kwargs.pop("continuation_token", None), + "max_records": kwargs.pop("batch_size", None), + "acl": acl, + "cls": return_headers_and_deserialized, } options.update(kwargs) return options @@ -171,13 +168,13 @@ def _rename_path_options( rename_source: str, content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Any]: - if metadata or kwargs.pop('permissions', None) or kwargs.pop('umask', None): + if metadata or kwargs.pop("permissions", None) or kwargs.pop("umask", None): raise ValueError("metadata, permissions, umask is not supported for this operation") - access_conditions = get_access_conditions(kwargs.pop('lease', None)) - source_lease_id = get_lease_id(kwargs.pop('source_lease', None)) + access_conditions = get_access_conditions(kwargs.pop("lease", None)) + source_lease_id = get_lease_id(kwargs.pop("source_lease", None)) mod_conditions = get_mod_conditions(kwargs) source_mod_conditions = get_source_mod_conditions(kwargs) @@ -186,15 +183,15 @@ def _rename_path_options( path_http_headers = get_path_http_headers(content_settings) options = { - 'rename_source': rename_source, - 'path_http_headers': path_http_headers, - 'lease_access_conditions': access_conditions, - 'source_lease_id': source_lease_id, - 'modified_access_conditions': mod_conditions, - 'source_modified_access_conditions': source_mod_conditions, - 'timeout': kwargs.pop('timeout', None), - 'mode': 'legacy', - 'cls': return_response_headers + "rename_source": rename_source, + "path_http_headers": path_http_headers, + "lease_access_conditions": access_conditions, + "source_lease_id": source_lease_id, + "modified_access_conditions": mod_conditions, + "source_modified_access_conditions": source_mod_conditions, + "timeout": kwargs.pop("timeout", None), + "mode": "legacy", + "cls": return_response_headers, } options.update(kwargs) return options @@ -204,27 +201,36 @@ def _parse_rename_path( new_name: str, file_system_name: str, query_str: str, - raw_credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "TokenCredential", "AsyncTokenCredential"]] # pylint: disable=line-too-long + raw_credential: Optional[ + Union[ + str, + Dict[str, str], + "AzureNamedKeyCredential", + "AzureSasCredential", + "TokenCredential", + "AsyncTokenCredential", + ] + ], # pylint: disable=line-too-long ) -> Tuple[str, str, Optional[str]]: - new_name = new_name.strip('/') - new_file_system = new_name.split('/')[0] - new_path = new_name[len(new_file_system):].strip('/') + new_name = new_name.strip("/") + new_file_system = new_name.split("/")[0] + new_path = new_name[len(new_file_system) :].strip("/") new_sas = None - sas_split = new_path.split('?') + sas_split = new_path.split("?") # If there is a ?, there could be a SAS token if len(sas_split) > 0: # Check last element for SAS by looking for sv= and sig= potential_sas = sas_split[-1] - if re.search(r'sv=\d{4}-\d{2}-\d{2}', potential_sas) and 'sig=' in potential_sas: + if re.search(r"sv=\d{4}-\d{2}-\d{2}", potential_sas) and "sig=" in potential_sas: new_sas = potential_sas # Remove SAS from new path - new_path = new_path[:-(len(new_sas) + 1)] + new_path = new_path[: -(len(new_sas) + 1)] if not new_sas: if not raw_credential and new_file_system != file_system_name: raise ValueError("please provide the sas token for the new file") if not raw_credential and new_file_system == file_system_name: - new_sas = query_str.strip('?') + new_sas = query_str.strip("?") return new_file_system, new_path, new_sas diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_quick_query_helper.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_quick_query_helper.py index c0db2674084d..d621f70909a0 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_quick_query_helper.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_quick_query_helper.py @@ -4,10 +4,7 @@ # license information. # -------------------------------------------------------------------------- -from typing import ( - Any, Dict, IO, Iterable, Union, - TYPE_CHECKING -) +from typing import Any, Dict, IO, Iterable, Union, TYPE_CHECKING if TYPE_CHECKING: from azure.storage.blob import BlobQueryReader diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_serialize.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_serialize.py index 4eafa8e8cbf7..87c2f41416a7 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_serialize.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_serialize.py @@ -4,10 +4,7 @@ # license information. # -------------------------------------------------------------------------- -from typing import ( - Any, cast, Dict, Literal, Optional, Union, - TYPE_CHECKING -) +from typing import Any, cast, Dict, Literal, Optional, Union, TYPE_CHECKING from azure.storage.blob._serialize import _get_match_headers from ._generated.models import ( @@ -29,43 +26,43 @@ EncryptionAlgorithmType = Literal["AES256"] _SUPPORTED_API_VERSIONS = [ - '2019-02-02', - '2019-07-07', - '2019-10-10', - '2019-12-12', - '2020-02-10', - '2020-04-08', - '2020-06-12', - '2020-08-04', - '2020-10-02', - '2020-12-06', - '2021-02-12', - '2021-04-10', - '2021-06-08', - '2021-08-06', - '2021-12-02', - '2022-11-02', - '2023-01-03', - '2023-05-03', - '2023-08-03', - '2023-11-03', - '2024-05-04', - '2024-08-04', - '2024-11-04', - '2025-01-05', - '2025-05-05', - '2025-07-05', - '2025-11-05', - '2026-02-06', - '2026-04-06', - '2026-06-06', + "2019-02-02", + "2019-07-07", + "2019-10-10", + "2019-12-12", + "2020-02-10", + "2020-04-08", + "2020-06-12", + "2020-08-04", + "2020-10-02", + "2020-12-06", + "2021-02-12", + "2021-04-10", + "2021-06-08", + "2021-08-06", + "2021-12-02", + "2022-11-02", + "2023-01-03", + "2023-05-03", + "2023-08-03", + "2023-11-03", + "2024-05-04", + "2024-08-04", + "2024-11-04", + "2025-01-05", + "2025-05-05", + "2025-07-05", + "2025-11-05", + "2026-02-06", + "2026-04-06", + "2026-06-06", ] # This list must be in chronological order! def get_api_version(kwargs: Dict[str, Any]) -> str: - api_version = kwargs.get('api_version', None) + api_version = kwargs.get("api_version", None) if api_version and api_version not in _SUPPORTED_API_VERSIONS: - versions = '\n'.join(_SUPPORTED_API_VERSIONS) + versions = "\n".join(_SUPPORTED_API_VERSIONS) raise ValueError(f"Unsupported API version '{api_version}'. Please select from:\n{versions}") return api_version or _SUPPORTED_API_VERSIONS[-1] @@ -81,13 +78,12 @@ def compare_api_versions(version1: str, version2: str) -> int: def convert_dfs_url_to_blob_url(dfs_account_url: str) -> str: - return dfs_account_url.replace('.dfs.', '.blob.', 1) + return dfs_account_url.replace(".dfs.", ".blob.", 1) def convert_datetime_to_rfc1123(date: "datetime") -> str: weekday = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][date.weekday()] - month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", - "Oct", "Nov", "Dec"][date.month - 1] + month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][date.month - 1] return f"{weekday}, {date.day:02} {month} {date.year:04} {date.hour:02}:{date.minute:02}:{date.second:02} GMT" @@ -97,33 +93,33 @@ def add_metadata_headers(metadata: Optional[Dict[str, str]] = None) -> Optional[ headers = [] if metadata: for key, value in metadata.items(): - headers.append(key + '=') + headers.append(key + "=") headers.append(encode_base64(value)) - headers.append(',') + headers.append(",") if headers: del headers[-1] - return ''.join(headers) + return "".join(headers) def get_mod_conditions(kwargs: Dict[str, Any]) -> ModifiedAccessConditions: - if_match, if_none_match = _get_match_headers(kwargs, 'match_condition', 'etag') + if_match, if_none_match = _get_match_headers(kwargs, "match_condition", "etag") return ModifiedAccessConditions( - if_modified_since=kwargs.pop('if_modified_since', None), - if_unmodified_since=kwargs.pop('if_unmodified_since', None), - if_match=if_match or kwargs.pop('if_match', None), - if_none_match=if_none_match or kwargs.pop('if_none_match', None) + if_modified_since=kwargs.pop("if_modified_since", None), + if_unmodified_since=kwargs.pop("if_unmodified_since", None), + if_match=if_match or kwargs.pop("if_match", None), + if_none_match=if_none_match or kwargs.pop("if_none_match", None), ) def get_source_mod_conditions(kwargs: Dict[str, Any]) -> SourceModifiedAccessConditions: - if_match, if_none_match = _get_match_headers(kwargs, 'source_match_condition', 'source_etag') + if_match, if_none_match = _get_match_headers(kwargs, "source_match_condition", "source_etag") return SourceModifiedAccessConditions( - source_if_modified_since=kwargs.pop('source_if_modified_since', None), - source_if_unmodified_since=kwargs.pop('source_if_unmodified_since', None), - source_if_match=if_match or kwargs.pop('source_if_match', None), - source_if_none_match=if_none_match or kwargs.pop('source_if_none_match', None) + source_if_modified_since=kwargs.pop("source_if_modified_since", None), + source_if_unmodified_since=kwargs.pop("source_if_unmodified_since", None), + source_if_match=if_match or kwargs.pop("source_if_match", None), + source_if_none_match=if_none_match or kwargs.pop("source_if_none_match", None), ) @@ -134,13 +130,13 @@ def get_path_http_headers(content_settings: "ContentSettings") -> PathHTTPHeader content_md5=bytearray(content_settings.content_md5) if content_settings.content_md5 else None, content_encoding=content_settings.content_encoding, content_language=content_settings.content_language, - content_disposition=content_settings.content_disposition + content_disposition=content_settings.content_disposition, ) return path_headers def get_access_conditions( - lease: Optional[Union["BlobLeaseClient", "BlobLeaseClientAsync", str]] + lease: Optional[Union["BlobLeaseClient", "BlobLeaseClientAsync", str]], ) -> Optional[LeaseAccessConditions]: if not lease: return None @@ -162,9 +158,9 @@ def get_lease_id(lease: Optional[Union["BlobLeaseClient", "BlobLeaseClientAsync" def get_lease_action_properties(kwargs: Dict[str, Any]) -> Dict[str, Any]: - lease_action = kwargs.pop('lease_action', None) - lease_duration = kwargs.pop('lease_duration', None) - lease = kwargs.pop('lease', None) + lease_action = kwargs.pop("lease_action", None) + lease_duration = kwargs.pop("lease_duration", None) + lease = kwargs.pop("lease", None) if hasattr(lease, "id"): lease_id = lease.id else: @@ -174,7 +170,7 @@ def get_lease_action_properties(kwargs: Dict[str, Any]) -> Dict[str, Any]: access_conditions = None # Acquiring a new lease - if lease_action in ['acquire', 'acquire-release']: + if lease_action in ["acquire", "acquire-release"]: # Use provided lease id as the new lease id proposed_lease_id = lease_id # Assign a default lease duration if not provided @@ -184,22 +180,22 @@ def get_lease_action_properties(kwargs: Dict[str, Any]) -> Dict[str, Any]: access_conditions = LeaseAccessConditions(lease_id=lease_id) if lease_id else None return { - 'lease_action': lease_action, - 'lease_duration': lease_duration, - 'proposed_lease_id': proposed_lease_id, - 'lease_access_conditions': access_conditions + "lease_action": lease_action, + "lease_duration": lease_duration, + "proposed_lease_id": proposed_lease_id, + "lease_access_conditions": access_conditions, } def get_cpk_info(scheme: str, kwargs: Dict[str, Any]) -> Optional[CpkInfo]: - cpk: Optional[CustomerProvidedEncryptionKey] = kwargs.pop('cpk', None) + cpk: Optional[CustomerProvidedEncryptionKey] = kwargs.pop("cpk", None) if cpk: - if scheme.lower() != 'https': + if scheme.lower() != "https": raise ValueError("Customer provided encryption key must be used over HTTPS.") return CpkInfo( encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, - encryption_algorithm=cast(EncryptionAlgorithmType, cpk.algorithm) + encryption_algorithm=cast(EncryptionAlgorithmType, cpk.algorithm), ) return None diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py index 57095ca402aa..08e54267f1c7 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py @@ -86,7 +86,7 @@ def _construct_endpoints(netloc: str, account_part: str) -> Tuple[str, str, str] :return: The account name, primary hostname, and secondary hostname. :rtype: Tuple[str, str, str] """ - domain_suffix = netloc[len(account_part):] + domain_suffix = netloc[len(account_part) :] secondary_idx = account_part.find(_SECONDARY_SUFFIX) # Case where customer provides secondary URL @@ -184,7 +184,7 @@ def url(self) -> str: :return: The full endpoint URL to this entity, including SAS token if used. :rtype: str """ - return self._format_url(self._hosts[self._location_mode]) # type: ignore + return self._format_url(self._hosts[self._location_mode]) # type: ignore @property def primary_endpoint(self) -> str: @@ -217,7 +217,7 @@ def secondary_endpoint(self) -> str: """ if not self._hosts[LocationMode.SECONDARY]: raise ValueError("No secondary host configured.") - return self._format_url(self._hosts[LocationMode.SECONDARY]) # type: ignore + return self._format_url(self._hosts[LocationMode.SECONDARY]) # type: ignore @property def secondary_hostname(self) -> Optional[str]: @@ -455,7 +455,7 @@ def parse_connection_str( if any(len(tup) != 2 for tup in conn_settings_list): raise ValueError("Connection string is either blank or malformed.") conn_settings = dict((key.upper(), val) for key, val in conn_settings_list) - if conn_settings.get('USEDEVELOPMENTSTORAGE') == 'true': + if conn_settings.get("USEDEVELOPMENTSTORAGE") == "true": return _get_development_storage_endpoint(service), None, DEVSTORE_ACCOUNT_KEY endpoints = _SERVICE_PARAMS[service] primary = None diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client_async.py index 2e023b1cc8d9..7169ac25464c 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client_async.py @@ -213,7 +213,7 @@ def parse_connection_str( if any(len(tup) != 2 for tup in conn_settings_list): raise ValueError("Connection string is either blank or malformed.") conn_settings = dict((key.upper(), val) for key, val in conn_settings_list) - if conn_settings.get('USEDEVELOPMENTSTORAGE') == 'true': + if conn_settings.get("USEDEVELOPMENTSTORAGE") == "true": return _get_development_storage_endpoint(service), None, DEVSTORE_ACCOUNT_KEY endpoints = _SERVICE_PARAMS[service] primary = None diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/constants.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/constants.py index e2cad9cacbbc..a6f4cdb34dfd 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/constants.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/constants.py @@ -6,7 +6,6 @@ from .._serialize import _SUPPORTED_API_VERSIONS - X_MS_VERSION = _SUPPORTED_API_VERSIONS[-1] # Connection defaults diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/request_handlers.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/request_handlers.py index b23f65859690..699635565b18 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/request_handlers.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/request_handlers.py @@ -12,7 +12,6 @@ import isodate - _LOGGER = logging.getLogger(__name__) _REQUEST_DELIMITER_PREFIX = "batch_" diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/response_handlers.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/response_handlers.py index 1ed0d1073331..ef7150cfe070 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/response_handlers.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/response_handlers.py @@ -21,7 +21,6 @@ from .models import get_enum_value, StorageErrorCode, UserDelegationKey from .parser import _to_utc_datetime - SV_DOCS_URL = "https://learn.microsoft.com/rest/api/storageservices/versioning-for-the-azure-storage-services" _LOGGER = logging.getLogger(__name__) @@ -176,8 +175,11 @@ def process_storage_error(storage_error) -> NoReturn: # type: ignore [misc] # p error_message += f"\n{name}:{info}" if additional_data.get("headername") == "x-ms-version" and error_code == StorageErrorCode.INVALID_HEADER_VALUE: - error_message = ("The provided service version is not enabled on this storage account." + - f"Please see {SV_DOCS_URL} for additional information.\n" + error_message) + error_message = ( + "The provided service version is not enabled on this storage account." + + f"Please see {SV_DOCS_URL} for additional information.\n" + + error_message + ) # No need to create an instance if it has already been serialized by the generated layer if serialized: diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads.py index 7a5fb3f3dc91..88d52eab5d12 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads.py @@ -16,7 +16,6 @@ from .request_handlers import get_length from .response_handlers import return_response_headers - _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 _ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM = "{0} should be a seekable file-like/io.IOBase type stream object." diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared_access_signature.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared_access_signature.py index 506721dd69ab..55d3451c9d76 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared_access_signature.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared_access_signature.py @@ -5,10 +5,7 @@ # -------------------------------------------------------------------------- # pylint: disable=docstring-keyword-should-match-keyword-only -from typing import ( - Any, Callable, cast, Dict, Optional, Union, - TYPE_CHECKING -) +from typing import Any, Callable, cast, Dict, Optional, Union, TYPE_CHECKING from urllib.parse import parse_qs from azure.storage.blob import generate_account_sas as generate_blob_account_sas @@ -16,7 +13,6 @@ from ._shared.models import Services from ._shared.shared_access_signature import QueryStringConstants - if TYPE_CHECKING: from azure.storage.blob import BlobSasPermissions, ContainerSasPermissions from azure.storage.blob._shared.models import Services as BlobServices @@ -27,7 +23,7 @@ FileSasPermissions, FileSystemSasPermissions, ResourceTypes, - UserDelegationKey + UserDelegationKey, ) @@ -481,7 +477,7 @@ def generate_file_sas( :rtype: str """ if directory_name: - path = directory_name.rstrip('/') + "/" + file_name + path = directory_name.rstrip("/") + "/" + file_name else: path = file_name return generate_blob_sas( @@ -499,6 +495,7 @@ def generate_file_sas( **kwargs ) + def _is_credential_sastoken(credential: Any) -> bool: if not credential or not isinstance(credential, str): return False diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_upload_helper.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_upload_helper.py index d9215a46cc5d..f45f9c68f36d 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_upload_helper.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_upload_helper.py @@ -4,20 +4,13 @@ # license information. # -------------------------------------------------------------------------- -from typing import ( - Any, cast, Dict, IO, Optional, - TYPE_CHECKING -) +from typing import Any, cast, Dict, IO, Optional, TYPE_CHECKING from azure.core.exceptions import HttpResponseError from ._deserialize import process_storage_error from ._shared.response_handlers import return_response_headers -from ._shared.uploads import ( - DataLakeFileChunkUploader, - upload_data_chunks, - upload_substream_blocks -) +from ._shared.uploads import DataLakeFileChunkUploader, upload_data_chunks, upload_substream_blocks if TYPE_CHECKING: from ._generated.operations import PathOperations @@ -25,12 +18,14 @@ def _any_conditions(modified_access_conditions=None, **kwargs): # pylint: disable=unused-argument - return any([ - modified_access_conditions.if_modified_since, - modified_access_conditions.if_unmodified_since, - modified_access_conditions.if_none_match, - modified_access_conditions.if_match - ]) + return any( + [ + modified_access_conditions.if_modified_since, + modified_access_conditions.if_unmodified_since, + modified_access_conditions.if_none_match, + modified_access_conditions.if_match, + ] + ) def upload_datalake_file( @@ -46,46 +41,54 @@ def upload_datalake_file( try: if length == 0: return {} - properties = kwargs.pop('properties', None) - umask = kwargs.pop('umask', None) - permissions = kwargs.pop('permissions', None) - path_http_headers = kwargs.pop('path_http_headers', None) - modified_access_conditions = kwargs.pop('modified_access_conditions', None) - chunk_size = kwargs.pop('chunk_size', 100 * 1024 * 1024) - encryption_context = kwargs.pop('encryption_context', None) - progress_hook = kwargs.pop('progress_hook', None) + properties = kwargs.pop("properties", None) + umask = kwargs.pop("umask", None) + permissions = kwargs.pop("permissions", None) + path_http_headers = kwargs.pop("path_http_headers", None) + modified_access_conditions = kwargs.pop("modified_access_conditions", None) + chunk_size = kwargs.pop("chunk_size", 100 * 1024 * 1024) + encryption_context = kwargs.pop("encryption_context", None) + progress_hook = kwargs.pop("progress_hook", None) if not overwrite: # if customers didn't specify access conditions, they cannot flush data to existing file if not _any_conditions(modified_access_conditions): - modified_access_conditions.if_none_match = '*' + modified_access_conditions.if_none_match = "*" if properties or umask or permissions: raise ValueError("metadata, umask and permissions can be set only when overwrite is enabled") if overwrite: - response = cast(Dict[str, Any], client.create( - resource='file', - path_http_headers=path_http_headers, - properties=properties, - modified_access_conditions=modified_access_conditions, - umask=umask, - permissions=permissions, - encryption_context=encryption_context, - cls=return_response_headers, - **kwargs - )) + response = cast( + Dict[str, Any], + client.create( + resource="file", + path_http_headers=path_http_headers, + properties=properties, + modified_access_conditions=modified_access_conditions, + umask=umask, + permissions=permissions, + encryption_context=encryption_context, + cls=return_response_headers, + **kwargs + ), + ) # this modified_access_conditions will be applied to flush_data to make sure # no other flush between create and the current flush - modified_access_conditions.if_match = response['etag'] + modified_access_conditions.if_match = response["etag"] modified_access_conditions.if_none_match = None modified_access_conditions.if_modified_since = None modified_access_conditions.if_unmodified_since = None - use_original_upload_path = file_settings.use_byte_buffer or \ - validate_content or chunk_size < file_settings.min_large_chunk_upload_threshold or \ - hasattr(stream, 'seekable') and not stream.seekable() or \ - not hasattr(stream, 'seek') or not hasattr(stream, 'tell') + use_original_upload_path = ( + file_settings.use_byte_buffer + or validate_content + or chunk_size < file_settings.min_large_chunk_upload_threshold + or hasattr(stream, "seekable") + and not stream.seekable() + or not hasattr(stream, "seek") + or not hasattr(stream, "tell") + ) if use_original_upload_path: upload_data_chunks( @@ -112,13 +115,16 @@ def upload_datalake_file( **kwargs ) - return cast(Dict[str, Any], client.flush_data( - position=length, - path_http_headers=path_http_headers, - modified_access_conditions=modified_access_conditions, - close=True, - cls=return_response_headers, - **kwargs - )) + return cast( + Dict[str, Any], + client.flush_data( + position=length, + path_http_headers=path_http_headers, + modified_access_conditions=modified_access_conditions, + close=True, + cls=return_response_headers, + **kwargs + ), + ) except HttpResponseError as error: process_storage_error(error) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/__init__.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/__init__.py index c24dde8d3478..13741264bcd5 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/__init__.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/__init__.py @@ -13,12 +13,12 @@ from ._data_lake_lease_async import DataLakeLeaseClient __all__ = [ - 'DataLakeServiceClient', - 'FileSystemClient', - 'DataLakeDirectoryClient', - 'DataLakeFileClient', - 'DataLakeLeaseClient', - 'ExponentialRetry', - 'LinearRetry', - 'StorageStreamDownloader' + "DataLakeServiceClient", + "FileSystemClient", + "DataLakeDirectoryClient", + "DataLakeFileClient", + "DataLakeLeaseClient", + "ExponentialRetry", + "LinearRetry", + "StorageStreamDownloader", ] diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py index c0cd414369b5..c56fa9dd37f7 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py @@ -6,10 +6,7 @@ # pylint: disable=docstring-keyword-should-match-keyword-only import functools -from typing import ( - Any, cast, Dict, Optional, Union, - TYPE_CHECKING -) +from typing import Any, cast, Dict, Optional, Union, TYPE_CHECKING from typing_extensions import Self try: @@ -89,22 +86,29 @@ class DataLakeDirectoryClient(PathClient): """The hostname of the primary endpoint.""" def __init__( - self, account_url: str, + self, + account_url: str, file_system_name: str, directory_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> None: - super(DataLakeDirectoryClient, self).__init__(account_url, file_system_name, path_name=directory_name, - credential=credential, **kwargs) + super(DataLakeDirectoryClient, self).__init__( + account_url, file_system_name, path_name=directory_name, credential=credential, **kwargs + ) @classmethod def from_connection_string( - cls, conn_str: str, + cls, + conn_str: str, file_system_name: str, directory_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> Self: """ Create DataLakeDirectoryClient from a Connection String. @@ -140,15 +144,18 @@ def from_connection_string( :return: A DataLakeDirectoryClient. :rtype: ~azure.storage.filedatalake.aio.DataLakeDirectoryClient """ - account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs') + account_url, _, credential = parse_connection_str(conn_str, credential, "dfs") return cls( - account_url, file_system_name=file_system_name, directory_name=directory_name, - credential=credential, **kwargs) + account_url, + file_system_name=file_system_name, + directory_name=directory_name, + credential=credential, + **kwargs, + ) @distributed_trace_async async def create_directory( - self, metadata: Optional[Dict[str, str]] = None, - **kwargs + self, metadata: Optional[Dict[str, str]] = None, **kwargs ) -> Dict[str, Union[str, "datetime"]]: """ Create a new directory. @@ -232,7 +239,7 @@ async def create_directory( :dedent: 8 :caption: Create directory. """ - return await self._create('directory', metadata=metadata, **kwargs) + return await self._create("directory", metadata=metadata, **kwargs) @distributed_trace_async async def exists(self, **kwargs: Any) -> bool: @@ -353,11 +360,11 @@ async def get_directory_properties(self, **kwargs: Any) -> DirectoryProperties: :dedent: 4 :caption: Getting the properties for a file/directory. """ - upn = kwargs.pop('upn', None) + upn = kwargs.pop("upn", None) if upn: - headers = kwargs.pop('headers', {}) - headers['x-ms-upn'] = str(upn) - kwargs['headers'] = headers + headers = kwargs.pop("headers", {}) + headers["x-ms-upn"] = str(upn) + kwargs["headers"] = headers props = await self._get_path_properties(cls=deserialize_dir_properties, **kwargs) return cast(DirectoryProperties, props) @@ -431,21 +438,26 @@ async def rename_directory(self, new_name: str, **kwargs: Any) -> "DataLakeDirec :caption: Rename the source directory. """ new_file_system, new_path, new_dir_sas = _parse_rename_path( - new_name, self.file_system_name, self._query_str, self._raw_credential) + new_name, self.file_system_name, self._query_str, self._raw_credential + ) new_directory_client = DataLakeDirectoryClient( - f"{self.scheme}://{self.primary_hostname}", new_file_system, directory_name=new_path, + f"{self.scheme}://{self.primary_hostname}", + new_file_system, + directory_name=new_path, credential=self._raw_credential or new_dir_sas, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=self._pipeline, + ) await new_directory_client._rename_path( # pylint: disable=protected-access - f'/{quote(unquote(self.file_system_name))}/{quote(unquote(self.path_name))}{self._query_str}', **kwargs) + f"/{quote(unquote(self.file_system_name))}/{quote(unquote(self.path_name))}{self._query_str}", **kwargs + ) return new_directory_client @distributed_trace_async async def create_sub_directory( - self, sub_directory: Union[DirectoryProperties, str], - metadata: Optional[Dict[str, str]] = None, - **kwargs: Any + self, sub_directory: Union[DirectoryProperties, str], metadata: Optional[Dict[str, str]] = None, **kwargs: Any ) -> "DataLakeDirectoryClient": """ Create a subdirectory and return the subdirectory client to be interacted with. @@ -530,8 +542,7 @@ async def create_sub_directory( @distributed_trace_async async def delete_sub_directory( - self, sub_directory: Union[DirectoryProperties, str], - **kwargs: Any + self, sub_directory: Union[DirectoryProperties, str], **kwargs: Any ) -> "DataLakeDirectoryClient": """ Marks the specified subdirectory for deletion. @@ -667,13 +678,14 @@ async def create_file(self, file: Union[FileProperties, str], **kwargs: Any) -> @distributed_trace def get_paths( - self, *, + self, + *, recursive: bool = True, max_results: Optional[int] = None, upn: Optional[bool] = None, start_from: Optional[str] = None, timeout: Optional[int] = None, - **kwargs: Any + **kwargs: Any, ) -> AsyncItemPaged["PathProperties"]: """Returns an async generator to list the paths under specified file system and directory. The generator will lazily follow the continuation tokens returned by the service. @@ -708,15 +720,16 @@ def get_paths( url = f"{self.scheme}://{hostname}/{quote(self.file_system_name)}" client = self._build_generated_client(url) command = functools.partial( - client.file_system.list_paths, - path=self.path_name, - begin_from=start_from, - timeout=timeout, - **kwargs + client.file_system.list_paths, path=self.path_name, begin_from=start_from, timeout=timeout, **kwargs ) return AsyncItemPaged( - command, recursive, path=self.path_name, max_results=max_results, - upn=upn, page_iterator_class=PathPropertiesPaged, **kwargs + command, + recursive, + path=self.path_name, + max_results=max_results, + upn=upn, + page_iterator_class=PathPropertiesPaged, + **kwargs, ) def get_file_client(self, file: Union[FileProperties, str]) -> DataLakeFileClient: @@ -732,18 +745,24 @@ def get_file_client(self, file: Union[FileProperties, str]) -> DataLakeFileClien :rtype: ~azure.storage.filedatalake.aio.DataLakeFileClient """ if isinstance(file, FileProperties): - file_path = file.get('name') + file_path = file.get("name") else: - file_path = self.path_name + '/' + str(file) + file_path = self.path_name + "/" + str(file) _pipeline = AsyncPipeline( transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # type: ignore [arg-type] # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # type: ignore [arg-type] # pylint: disable=protected-access ) return DataLakeFileClient( - self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential, + self.url, + self.file_system_name, + file_path=file_path, + credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=_pipeline, + ) def get_sub_directory_client(self, sub_directory: Union[DirectoryProperties, str]) -> "DataLakeDirectoryClient": """Get a client to interact with the specified subdirectory of the current directory. @@ -758,15 +777,21 @@ def get_sub_directory_client(self, sub_directory: Union[DirectoryProperties, str :rtype: ~azure.storage.filedatalake.aio.DataLakeDirectoryClient """ if isinstance(sub_directory, DirectoryProperties): - subdir_path = sub_directory.get('name') + subdir_path = sub_directory.get("name") else: - subdir_path = self.path_name + '/' + str(sub_directory) + subdir_path = self.path_name + "/" + str(sub_directory) _pipeline = AsyncPipeline( - transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # type: ignore [arg-type] # pylint: disable=protected-access + transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # type: ignore [arg-type] # pylint: disable=protected-access ) return DataLakeDirectoryClient( - self.url, self.file_system_name, directory_name=subdir_path, credential=self._raw_credential, + self.url, + self.file_system_name, + directory_name=subdir_path, + credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=_pipeline, + ) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py index f13042518f98..d268d444e7b7 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py @@ -6,10 +6,7 @@ # pylint: disable=docstring-keyword-should-match-keyword-only from datetime import datetime -from typing import ( - Any, AnyStr, AsyncIterable, cast, Dict, IO, Iterable, Optional, Union, - TYPE_CHECKING -) +from typing import Any, AnyStr, AsyncIterable, cast, Dict, IO, Iterable, Optional, Union, TYPE_CHECKING from urllib.parse import quote, unquote from typing_extensions import Self @@ -87,22 +84,29 @@ class DataLakeFileClient(PathClient): """The hostname of the primary endpoint.""" def __init__( - self, account_url: str, + self, + account_url: str, file_system_name: str, file_path: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> None: - super(DataLakeFileClient, self).__init__(account_url, file_system_name, path_name=file_path, - credential=credential, **kwargs) + super(DataLakeFileClient, self).__init__( + account_url, file_system_name, path_name=file_path, credential=credential, **kwargs + ) @classmethod def from_connection_string( - cls, conn_str: str, + cls, + conn_str: str, file_system_name: str, file_path: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> Self: """ Create DataLakeFileClient from a Connection String. @@ -137,16 +141,15 @@ def from_connection_string( :returns: A DataLakeFileClient. :rtype: ~azure.storage.filedatalake.aio.DataLakeFileClient """ - account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs') - return cls( - account_url, file_system_name=file_system_name, file_path=file_path, - credential=credential, **kwargs) + account_url, _, credential = parse_connection_str(conn_str, credential, "dfs") + return cls(account_url, file_system_name=file_system_name, file_path=file_path, credential=credential, **kwargs) @distributed_trace_async async def create_file( - self, content_settings: Optional["ContentSettings"] = None, + self, + content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Union[str, datetime]]: """ Create a new file. @@ -240,7 +243,7 @@ async def create_file( :dedent: 4 :caption: Create file. """ - return await self._create('file', content_settings=content_settings, metadata=metadata, **kwargs) + return await self._create("file", content_settings=content_settings, metadata=metadata, **kwargs) @distributed_trace_async async def exists(self, **kwargs: Any) -> bool: @@ -359,19 +362,17 @@ async def get_file_properties(self, **kwargs: Any) -> FileProperties: :dedent: 4 :caption: Getting the properties for a file. """ - upn = kwargs.pop('upn', None) + upn = kwargs.pop("upn", None) if upn: - headers = kwargs.pop('headers', {}) - headers['x-ms-upn'] = str(upn) - kwargs['headers'] = headers + headers = kwargs.pop("headers", {}) + headers["x-ms-upn"] = str(upn) + kwargs["headers"] = headers props = await self._get_path_properties(cls=deserialize_file_properties, **kwargs) return cast(FileProperties, props) @distributed_trace_async async def set_file_expiry( - self, expiry_options: str, - expires_on: Optional[Union[datetime, int]] = None, - **kwargs: Any + self, expiry_options: str, expires_on: Optional[Union[datetime, int]] = None, **kwargs: Any ) -> None: """Sets the time a file will expire and be deleted. @@ -398,10 +399,11 @@ async def set_file_expiry( @distributed_trace_async async def upload_data( - self, data: Union[bytes, str, Iterable[AnyStr], AsyncIterable[AnyStr], IO[bytes]], + self, + data: Union[bytes, str, Iterable[AnyStr], AsyncIterable[AnyStr], IO[bytes]], length: Optional[int] = None, overwrite: Optional[bool] = False, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Any]: """ Upload data to a file. @@ -481,29 +483,24 @@ async def upload_data( :rtype: Dict[str, Any] """ options = _upload_options( - data, - self.scheme, - self._config, - self._client.path, - length=length, - overwrite=overwrite, - **kwargs + data, self.scheme, self._config, self._client.path, length=length, overwrite=overwrite, **kwargs ) return await upload_datalake_file(**options) @distributed_trace_async async def append_data( - self, data: Union[bytes, Iterable[bytes], AsyncIterable[bytes], IO[bytes]], + self, + data: Union[bytes, Iterable[bytes], AsyncIterable[bytes], IO[bytes]], offset: int, length: Optional[int] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Any]: """Append data to the file. :param data: Content to be appended to file :type data: Union[bytes, Iterable[bytes], AsyncIterable[bytes], IO[bytes]] :param int offset: start position of the data to be appended to. - :param length: + :param length: Size of the data to append. Optional if the length of data can be determined. For Iterable and IO, if the length is not provided and cannot be determined, all data will be read into memory. :type length: int or None @@ -557,12 +554,7 @@ async def append_data( :dedent: 4 :caption: Append data to the file. """ - options = _append_data_options( - data=data, - offset=offset, - scheme=self.scheme, - length=length, - **kwargs) + options = _append_data_options(data=data, offset=offset, scheme=self.scheme, length=length, **kwargs) try: return await self._client.path.append_data(**options) except HttpResponseError as error: @@ -570,9 +562,7 @@ async def append_data( @distributed_trace_async async def flush_data( - self, offset: int, - retain_uncommitted_data: Optional[bool] = False, - **kwargs: Any + self, offset: int, retain_uncommitted_data: Optional[bool] = False, **kwargs: Any ) -> Dict[str, Any]: """Commit the previous appended data. @@ -661,12 +651,7 @@ async def flush_data( :dedent: 12 :caption: Commit the previous appended data. """ - options = _flush_data_options( - offset, - self.scheme, - retain_uncommitted_data=retain_uncommitted_data, - **kwargs - ) + options = _flush_data_options(offset, self.scheme, retain_uncommitted_data=retain_uncommitted_data, **kwargs) try: return await self._client.path.flush_data(**options) except HttpResponseError as error: @@ -674,9 +659,7 @@ async def flush_data( @distributed_trace_async async def download_file( - self, offset: Optional[int] = None, - length: Optional[int] = None, - **kwargs: Any + self, offset: Optional[int] = None, length: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader: """Downloads a file to the StorageStreamDownloader. The readall() method must be used to read all the content, or readinto() must be used to download the file into @@ -820,13 +803,20 @@ async def rename_file(self, new_name: str, **kwargs: Any) -> "DataLakeFileClient :caption: Rename the source file. """ new_file_system, new_path, new_file_sas = _parse_rename_path( - new_name, self.file_system_name, self._query_str, self._raw_credential) + new_name, self.file_system_name, self._query_str, self._raw_credential + ) new_file_client = DataLakeFileClient( - f"{self.scheme}://{self.primary_hostname}", new_file_system, file_path=new_path, + f"{self.scheme}://{self.primary_hostname}", + new_file_system, + file_path=new_path, credential=self._raw_credential or new_file_sas, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=self._pipeline, + _location_mode=self._location_mode, + ) await new_file_client._rename_path( # pylint: disable=protected-access - f'/{quote(unquote(self.file_system_name))}/{quote(unquote(self.path_name))}{self._query_str}', **kwargs) + f"/{quote(unquote(self.file_system_name))}/{quote(unquote(self.path_name))}{self._query_str}", **kwargs + ) return new_file_client diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.pyi b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.pyi index ff21c42b73e6..d532ccfbc1ba 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.pyi +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.pyi @@ -148,7 +148,7 @@ class DataLakeFileClient(PathClient): if_unmodified_since: Optional[datetime] = None, etag: Optional[str] = None, match_condition: Optional[MatchConditions] = None, - validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None, + validate_content: Optional[Union[bool, Literal["auto", "crc64", "md5"]]] = None, cpk: Optional[CustomerProvidedEncryptionKey] = None, max_concurrency: Optional[int] = None, chunk_size: Optional[int] = None, @@ -165,7 +165,7 @@ class DataLakeFileClient(PathClient): length: Optional[int] = None, *, flush: Optional[bool] = None, - validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None, + validate_content: Optional[Union[bool, Literal["auto", "crc64", "md5"]]] = None, lease_action: Optional[Literal["acquire", "auto-renew", "release", "acquire-release"]] = None, lease_duration: int = -1, lease: Optional[Union[DataLakeLeaseClient, str]] = None, @@ -206,7 +206,7 @@ class DataLakeFileClient(PathClient): cpk: Optional[CustomerProvidedEncryptionKey] = None, max_concurrency: Optional[int] = None, progress_hook: Optional[Callable[[int, Optional[int]], Awaitable[None]]] = None, - validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None, + validate_content: Optional[Union[bool, Literal["auto", "crc64", "md5"]]] = None, timeout: Optional[int] = None, **kwargs: Any ) -> StorageStreamDownloader: ... diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_lease_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_lease_async.py index 5010f5290625..49661cb409ae 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_lease_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_lease_async.py @@ -6,10 +6,7 @@ # pylint: disable=docstring-keyword-should-match-keyword-only import uuid -from typing import ( - Union, Optional, Any, - TYPE_CHECKING -) +from typing import Union, Optional, Any, TYPE_CHECKING from typing_extensions import Self from azure.core.tracing.decorator_async import distributed_trace_async @@ -49,16 +46,17 @@ class DataLakeLeaseClient: # pylint: disable=client-accepts-api-version-keyword This will be `None` if no lease has yet been acquired or modified.""" def __init__( # pylint: disable=missing-client-constructor-parameter-credential, missing-client-constructor-parameter-kwargs - self, client: Union["FileSystemClient", "DataLakeDirectoryClient", "DataLakeFileClient"], - lease_id: Optional[str] = None + self, + client: Union["FileSystemClient", "DataLakeDirectoryClient", "DataLakeFileClient"], + lease_id: Optional[str] = None, ) -> None: self.id = lease_id or str(uuid.uuid4()) self.last_modified = None self.etag = None - if hasattr(client, '_blob_client'): + if hasattr(client, "_blob_client"): _client = client._blob_client - elif hasattr(client, '_container_client'): + elif hasattr(client, "_container_client"): _client = client._container_client else: raise TypeError("Lease must use any of FileSystemClient, DataLakeDirectoryClient, or DataLakeFileClient.") diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_lease_async.pyi b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_lease_async.pyi index cfb42dcae656..8acac55a0257 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_lease_async.pyi +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_lease_async.pyi @@ -7,7 +7,9 @@ from datetime import datetime from typing import ( - Any, Optional, Union, + Any, + Optional, + Union, ) from types import TracebackType from typing_extensions import Self @@ -18,14 +20,14 @@ from ._file_system_client_async import FileSystemClient from ._data_lake_directory_client_async import DataLakeDirectoryClient from ._data_lake_file_client_async import DataLakeFileClient - class DataLakeLeaseClient: id: str etag: Optional[str] last_modified: Optional[datetime] def __init__( - self, client: Union[FileSystemClient, DataLakeDirectoryClient, DataLakeFileClient], - lease_id: Optional[str] = None + self, + client: Union[FileSystemClient, DataLakeDirectoryClient, DataLakeFileClient], + lease_id: Optional[str] = None, ) -> None: ... async def __aenter__(self) -> Self: ... async def __aexit__( diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_service_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_service_client_async.py index 673782455886..1ac5d5a5e60b 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_service_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_service_client_async.py @@ -5,10 +5,7 @@ # -------------------------------------------------------------------------- # pylint: disable=docstring-keyword-should-match-keyword-only -from typing import ( - Any, cast, Dict, Optional, Union, - TYPE_CHECKING -) +from typing import Any, cast, Dict, Optional, Union, TYPE_CHECKING from typing_extensions import Self from azure.core.async_paging import AsyncItemPaged @@ -19,13 +16,7 @@ from .._data_lake_service_client_helpers import _format_url, _parse_url from .._deserialize import get_datalake_service_properties from .._generated.aio import AzureDataLakeStorageRESTAPI -from .._models import ( - DirectoryProperties, - FileProperties, - FileSystemProperties, - LocationMode, - UserDelegationKey -) +from .._models import DirectoryProperties, FileProperties, FileSystemProperties, LocationMode, UserDelegationKey from .._serialize import convert_dfs_url_to_blob_url, get_api_version from .._shared.base_client import parse_query, StorageAccountHostsMixin from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper, parse_connection_str @@ -100,11 +91,14 @@ class DataLakeServiceClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMi """The hostname of the primary endpoint.""" def __init__( - self, account_url: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long + self, + account_url: str, + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"] + ] = None, # pylint: disable=line-too-long **kwargs: Any ) -> None: - kwargs['retry_policy'] = kwargs.get('retry_policy') or ExponentialRetry(**kwargs) + kwargs["retry_policy"] = kwargs.get("retry_policy") or ExponentialRetry(**kwargs) parsed_url = _parse_url(account_url=account_url) blob_account_url = convert_dfs_url_to_blob_url(account_url) @@ -116,19 +110,17 @@ def __init__( _, sas_token = parse_query(parsed_url.query) self._query_str, self._raw_credential = self._format_query_string(sas_token, credential) - super(DataLakeServiceClient, self).__init__(parsed_url, service='dfs', - credential=self._raw_credential, **kwargs) + super(DataLakeServiceClient, self).__init__( + parsed_url, service="dfs", credential=self._raw_credential, **kwargs + ) # ADLS doesn't support secondary endpoint, make sure it's empty self._hosts[LocationMode.SECONDARY] = "" self._api_version = get_api_version(kwargs) self._client = AzureDataLakeStorageRESTAPI( - self.url, - version=self._api_version, - base_url=self.url, - pipeline=self._pipeline + self.url, version=self._api_version, base_url=self.url, pipeline=self._pipeline ) - self._loop = kwargs.get('loop', None) + self._loop = kwargs.get("loop", None) async def __aenter__(self) -> Self: await self._client.__aenter__() @@ -140,7 +132,7 @@ async def __aexit__(self, *args: Any) -> None: await self._client.__aexit__(*args) async def close(self) -> None: # type: ignore - """ This method is to close the sockets opened by the client. + """This method is to close the sockets opened by the client. It need not be used when using with a context manager. :return: None @@ -160,8 +152,11 @@ def _format_url(self, hostname: str) -> str: @classmethod def from_connection_string( - cls, conn_str: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long + cls, + conn_str: str, + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"] + ] = None, # pylint: disable=line-too-long **kwargs: Any ) -> Self: """ @@ -199,12 +194,13 @@ def from_connection_string( :dedent: 8 :caption: Creating the DataLakeServiceClient from a connection string. """ - account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs') + account_url, _, credential = parse_connection_str(conn_str, credential, "dfs") return cls(account_url, credential=credential, **kwargs) @distributed_trace_async async def get_user_delegation_key( - self, key_start_time: "datetime", + self, + key_start_time: "datetime", key_expiry_time: "datetime", *, delegated_user_tid: Optional[str] = None, @@ -247,9 +243,7 @@ async def get_user_delegation_key( @distributed_trace def list_file_systems( - self, name_starts_with: Optional[str] = None, - include_metadata: bool = False, - **kwargs: Any + self, name_starts_with: Optional[str] = None, include_metadata: bool = False, **kwargs: Any ) -> AsyncItemPaged[FileSystemProperties]: """Returns a generator to list the file systems under the specified account. @@ -290,17 +284,19 @@ def list_file_systems( :dedent: 8 :caption: Listing the file systems in the datalake service. """ - item_paged = cast(AsyncItemPaged[FileSystemProperties], self._blob_service_client.list_containers( - name_starts_with=name_starts_with, - include_metadata=include_metadata, - **kwargs - )) + item_paged = cast( + AsyncItemPaged[FileSystemProperties], + self._blob_service_client.list_containers( + name_starts_with=name_starts_with, include_metadata=include_metadata, **kwargs + ), + ) item_paged._page_iterator_class = FileSystemPropertiesPaged # pylint: disable=protected-access return item_paged @distributed_trace_async async def create_file_system( - self, file_system: Union[FileSystemProperties, str], + self, + file_system: Union[FileSystemProperties, str], metadata: Optional[Dict[str, str]] = None, public_access: Optional["PublicAccess"] = None, **kwargs: Any @@ -371,7 +367,7 @@ async def _rename_file_system(self, name: str, new_name: str, **kwargs: Any) -> :returns: FileSystemClient with the newly specified name. :rtype: ~azure.storage.filedatalake.FileSystemClient """ - await self._blob_service_client._rename_container(name, new_name, **kwargs) # pylint: disable=protected-access + await self._blob_service_client._rename_container(name, new_name, **kwargs) # pylint: disable=protected-access renamed_file_system = self.get_file_system_client(new_name) return renamed_file_system @@ -398,15 +394,14 @@ async def undelete_file_system(self, name: str, deleted_version: str, **kwargs: :returns: The FileSystemClient of the restored soft-deleted filesystem. :rtype: ~azure.storage.filedatalake.FileSystemClient """ - new_name = kwargs.pop('new_name', None) + new_name = kwargs.pop("new_name", None) await self._blob_service_client.undelete_container(name, deleted_version, new_name=new_name, **kwargs) file_system = self.get_file_system_client(new_name or name) return file_system @distributed_trace_async async def delete_file_system( - self, file_system: Union[FileSystemProperties, str], - **kwargs: Any + self, file_system: Union[FileSystemProperties, str], **kwargs: Any ) -> FileSystemClient: """Marks the specified file system for deletion. @@ -489,16 +484,20 @@ def get_file_system_client(self, file_system: Union[FileSystemProperties, str]) _pipeline = AsyncPipeline( transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # type: ignore [arg-type] # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # type: ignore [arg-type] # pylint: disable=protected-access + ) + return FileSystemClient( + self.url, + file_system_name, + credential=self._raw_credential, + api_version=self.api_version, + _configuration=self._config, + _pipeline=_pipeline, + _hosts=self._hosts, ) - return FileSystemClient(self.url, file_system_name, credential=self._raw_credential, - api_version=self.api_version, - _configuration=self._config, - _pipeline=_pipeline, _hosts=self._hosts) def get_directory_client( - self, file_system: Union[FileSystemProperties,str], - directory: Union[DirectoryProperties, str] + self, file_system: Union[FileSystemProperties, str], directory: Union[DirectoryProperties, str] ) -> DataLakeDirectoryClient: """Get a client to interact with the specified directory. @@ -535,17 +534,21 @@ def get_directory_client( _pipeline = AsyncPipeline( transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # type: ignore [arg-type] # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # type: ignore [arg-type] # pylint: disable=protected-access + ) + return DataLakeDirectoryClient( + self.url, + file_system_name, + directory_name=directory_name, + credential=self._raw_credential, + api_version=self.api_version, + _configuration=self._config, + _pipeline=_pipeline, + _hosts=self._hosts, ) - return DataLakeDirectoryClient(self.url, file_system_name, directory_name=directory_name, - credential=self._raw_credential, - api_version=self.api_version, - _configuration=self._config, _pipeline=_pipeline, - _hosts=self._hosts) def get_file_client( - self, file_system: Union[FileSystemProperties, str], - file_path: Union[FileProperties, str] + self, file_system: Union[FileSystemProperties, str], file_path: Union[FileProperties, str] ) -> DataLakeFileClient: """Get a client to interact with the specified file. @@ -582,12 +585,18 @@ def get_file_client( _pipeline = AsyncPipeline( transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # type: ignore [arg-type] # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # type: ignore [arg-type] # pylint: disable=protected-access ) return DataLakeFileClient( - self.url, file_system_name, file_path=file_path, credential=self._raw_credential, + self.url, + file_system_name, + file_path=file_path, + credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=_pipeline, + ) @distributed_trace_async async def set_service_properties(self, **kwargs: Any) -> None: diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_download_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_download_async.py index e85314cb06f8..d5ba877bfec4 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_download_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_download_async.py @@ -3,10 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import ( - Any, AsyncIterator, cast, IO, - TYPE_CHECKING -) +from typing import Any, AsyncIterator, cast, IO, TYPE_CHECKING from .._deserialize import from_blob_properties @@ -31,13 +28,12 @@ def __init__(self, downloader: Any) -> None: self.name = self._downloader.name # Parse additional Datalake-only properties - encryption_context = self._downloader._response.response.headers.get('x-ms-encryption-context') - acl = self._downloader._response.response.headers.get('x-ms-acl') + encryption_context = self._downloader._response.response.headers.get("x-ms-encryption-context") + acl = self._downloader._response.response.headers.get("x-ms-acl") self.properties = from_blob_properties( - self._downloader.properties, - encryption_context=encryption_context, - acl=acl) + self._downloader.properties, encryption_context=encryption_context, acl=acl + ) self.size = self._downloader.size def __len__(self) -> int: diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py index 64355b81c310..12a88a63d04d 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py @@ -6,10 +6,7 @@ # pylint: disable=too-many-lines, docstring-keyword-should-match-keyword-only import functools -from typing import ( - Any, cast, Dict, Optional, Union, - TYPE_CHECKING -) +from typing import Any, cast, Dict, Optional, Union, TYPE_CHECKING from typing_extensions import Self from azure.core.async_paging import AsyncItemPaged @@ -27,7 +24,7 @@ FileProperties, FileSystemProperties, LocationMode, - PublicAccess + PublicAccess, ) from .._serialize import convert_dfs_url_to_blob_url, get_api_version from .._shared.base_client import parse_query, StorageAccountHostsMixin @@ -97,12 +94,15 @@ class FileSystemClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin): """The hostname of the primary endpoint.""" def __init__( - self, account_url: str, + self, + account_url: str, file_system_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> None: - kwargs['retry_policy'] = kwargs.get('retry_policy') or ExponentialRetry(**kwargs) + kwargs["retry_policy"] = kwargs.get("retry_policy") or ExponentialRetry(**kwargs) if not file_system_name: raise ValueError("Please specify a file system name.") @@ -113,24 +113,21 @@ def __init__( # TODO: add self.account_url to base_client and remove _blob_account_url self._blob_account_url = blob_account_url - datalake_hosts = kwargs.pop('_hosts', None) + datalake_hosts = kwargs.pop("_hosts", None) blob_hosts = None if datalake_hosts: blob_primary_account_url = convert_dfs_url_to_blob_url(datalake_hosts[LocationMode.PRIMARY]) blob_hosts = {LocationMode.PRIMARY: blob_primary_account_url, LocationMode.SECONDARY: ""} self._container_client = ContainerClient( - self._blob_account_url, - self.file_system_name, - credential=credential, - _hosts=blob_hosts, - **kwargs + self._blob_account_url, self.file_system_name, credential=credential, _hosts=blob_hosts, **kwargs ) _, sas_token = parse_query(parsed_url.query) self._query_str, self._raw_credential = self._format_query_string(sas_token, credential) - super(FileSystemClient, self).__init__(parsed_url, service='dfs', credential=self._raw_credential, - _hosts=datalake_hosts, **kwargs) + super(FileSystemClient, self).__init__( + parsed_url, service="dfs", credential=self._raw_credential, _hosts=datalake_hosts, **kwargs + ) # ADLS doesn't support secondary endpoint, make sure it's empty self._hosts[LocationMode.SECONDARY] = "" @@ -138,7 +135,7 @@ def __init__( self._api_version = get_api_version(kwargs) self._client = self._build_generated_client(self.url) self._datalake_client_for_blob_operation = self._build_generated_client(self._container_client.url) - self._loop = kwargs.get('loop', None) + self._loop = kwargs.get("loop", None) async def __aenter__(self) -> Self: await self._client.__aenter__() @@ -164,11 +161,7 @@ async def close(self) -> None: # type: ignore def _build_generated_client(self, url: str) -> AzureDataLakeStorageRESTAPI: client = AzureDataLakeStorageRESTAPI( - url, - version=self._api_version, - base_url=url, - file_system=self.file_system_name, - pipeline=self._pipeline + url, version=self._api_version, base_url=url, file_system=self.file_system_name, pipeline=self._pipeline ) return client @@ -177,10 +170,13 @@ def _format_url(self, hostname: str) -> str: @classmethod def from_connection_string( - cls, conn_str: str, + cls, + conn_str: str, file_system_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long - **kwargs: Any + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"] + ] = None, # pylint: disable=line-too-long + **kwargs: Any, ) -> Self: """ Create FileSystemClient from a Connection String. @@ -209,14 +205,12 @@ def from_connection_string( :returns: A FileSystemClient. :rtype: ~azure.storage.filedatalake.FileSystemClient """ - account_url, _, credential = parse_connection_str(conn_str, credential, 'dfs') + account_url, _, credential = parse_connection_str(conn_str, credential, "dfs") return cls(account_url, file_system_name=file_system_name, credential=credential, **kwargs) @distributed_trace_async async def acquire_lease( - self, lease_duration: int = -1, - lease_id: Optional[str] = None, - **kwargs: Any + self, lease_duration: int = -1, lease_id: Optional[str] = None, **kwargs: Any ) -> DataLakeLeaseClient: """ Requests a new lease. If the file system does not have an active lease, @@ -272,9 +266,7 @@ async def acquire_lease( @distributed_trace_async async def create_file_system( - self, metadata: Optional[Dict[str, str]] = None, - public_access: Optional[PublicAccess] = None, - **kwargs: Any + self, metadata: Optional[Dict[str, str]] = None, public_access: Optional[PublicAccess] = None, **kwargs: Any ) -> Dict[str, Union[str, "datetime"]]: """Creates a new file system under the specified account. @@ -314,12 +306,12 @@ async def create_file_system( :dedent: 16 :caption: Creating a file system in the datalake service. """ - encryption_scope_options = kwargs.pop('encryption_scope_options', None) + encryption_scope_options = kwargs.pop("encryption_scope_options", None) return await self._container_client.create_container( metadata=metadata, public_access=public_access, container_encryption_scope=encryption_scope_options, - **kwargs + **kwargs, ) @distributed_trace_async @@ -362,9 +354,15 @@ async def _rename_file_system(self, new_name: str, **kwargs: Any) -> "FileSystem await self._container_client._rename_container(new_name, **kwargs) # pylint: disable=protected-access # TODO: self._raw_credential would not work with SAS tokens renamed_file_system = FileSystemClient( - f"{self.scheme}://{self.primary_hostname}", file_system_name=new_name, - credential=self._raw_credential, api_version=self.api_version, _configuration=self._config, - _pipeline=self._pipeline, _location_mode=self._location_mode, _hosts=self._hosts) + f"{self.scheme}://{self.primary_hostname}", + file_system_name=new_name, + credential=self._raw_credential, + api_version=self.api_version, + _configuration=self._config, + _pipeline=self._pipeline, + _location_mode=self._location_mode, + _hosts=self._hosts, + ) return renamed_file_system @distributed_trace_async @@ -443,12 +441,13 @@ async def get_file_system_properties(self, **kwargs: Any) -> FileSystemPropertie :caption: Getting properties on the file system. """ container_properties = await self._container_client.get_container_properties(**kwargs) - return FileSystemProperties._convert_from_container_props(container_properties) # pylint: disable=protected-access + return FileSystemProperties._convert_from_container_props( + container_properties + ) # pylint: disable=protected-access @distributed_trace_async async def set_file_system_metadata( - self, metadata: Dict[str, str], - **kwargs: Any + self, metadata: Dict[str, str], **kwargs: Any ) -> Dict[str, Union[str, "datetime"]]: """Sets one or more user-defined name-value pairs for the specified file system. Each call to this operation replaces all existing metadata @@ -502,9 +501,10 @@ async def set_file_system_metadata( @distributed_trace_async async def set_file_system_access_policy( - self, signed_identifiers: Dict[str, "AccessPolicy"], + self, + signed_identifiers: Dict[str, "AccessPolicy"], public_access: Optional[Union[str, "PublicAccess"]] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, Union[str, "datetime"]]: """Sets the permissions for the specified file system or stored access policies that may be used with Shared Access Signatures. The permissions @@ -543,9 +543,7 @@ async def set_file_system_access_policy( :rtype: Dict[str, Union[str, ~datetime.datetime]] """ return await self._container_client.set_container_access_policy( - cast(Dict[str, "BlobAccessPolicy"], signed_identifiers), - public_access=public_access, - **kwargs + cast(Dict[str, "BlobAccessPolicy"], signed_identifiers), public_access=public_access, **kwargs ) @distributed_trace_async @@ -568,16 +566,19 @@ async def get_file_system_access_policy(self, **kwargs: Any) -> Dict[str, Any]: """ access_policy = await self._container_client.get_container_access_policy(**kwargs) return { - 'public_access': PublicAccess._from_generated(access_policy['public_access']), # pylint: disable=protected-access - 'signed_identifiers': access_policy['signed_identifiers'] + "public_access": PublicAccess._from_generated( + access_policy["public_access"] + ), # pylint: disable=protected-access + "signed_identifiers": access_policy["signed_identifiers"], } @distributed_trace def get_paths( - self, path: Optional[str] = None, + self, + path: Optional[str] = None, recursive: Optional[bool] = True, max_results: Optional[int] = None, - **kwargs: Any + **kwargs: Any, ) -> AsyncItemPaged["PathProperties"]: """Returns a generator to list the paths(could be files or directories) under the specified file system. The generator will lazily follow the continuation tokens returned by @@ -621,25 +622,18 @@ def get_paths( :dedent: 12 :caption: List the blobs in the file system. """ - timeout = kwargs.pop('timeout', None) + timeout = kwargs.pop("timeout", None) begin_from = kwargs.pop("start_from", None) command = functools.partial( - self._client.file_system.list_paths, - path=path, - timeout=timeout, - begin_from=begin_from, - **kwargs + self._client.file_system.list_paths, path=path, timeout=timeout, begin_from=begin_from, **kwargs ) return AsyncItemPaged( - command, recursive, path=path, max_results=max_results, - page_iterator_class=PathPropertiesPaged, **kwargs + command, recursive, path=path, max_results=max_results, page_iterator_class=PathPropertiesPaged, **kwargs ) @distributed_trace_async async def create_directory( - self, directory: Union[DirectoryProperties, str], - metadata: Optional[Dict[str, str]] = None, - **kwargs: Any + self, directory: Union[DirectoryProperties, str], metadata: Optional[Dict[str, str]] = None, **kwargs: Any ) -> DataLakeDirectoryClient: """ Create directory @@ -730,8 +724,7 @@ async def create_directory( @distributed_trace_async async def delete_directory( - self, directory: Union[DirectoryProperties, str], - **kwargs: Any + self, directory: Union[DirectoryProperties, str], **kwargs: Any ) -> DataLakeDirectoryClient: """ Marks the specified path for deletion. @@ -784,10 +777,7 @@ async def delete_directory( return directory_client @distributed_trace_async - async def create_file( - self, file: Union[FileProperties, str], - **kwargs: Any - ) -> DataLakeFileClient: + async def create_file(self, file: Union[FileProperties, str], **kwargs: Any) -> DataLakeFileClient: """ Create file @@ -884,10 +874,7 @@ async def create_file( return file_client @distributed_trace_async - async def delete_file( - self, file: Union[FileProperties, str], - **kwargs: Any - ) -> DataLakeFileClient: + async def delete_file(self, file: Union[FileProperties, str], **kwargs: Any) -> DataLakeFileClient: """ Marks the specified file for deletion. @@ -938,9 +925,7 @@ async def delete_file( @distributed_trace_async async def _undelete_path( - self, deleted_path_name: str, - deletion_id: str, - **kwargs: Any + self, deleted_path_name: str, deletion_id: str, **kwargs: Any ) -> Union[DataLakeDirectoryClient, DataLakeFileClient]: """Restores soft-deleted path. @@ -968,18 +953,15 @@ async def _undelete_path( pipeline = AsyncPipeline( transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # type: ignore [arg-type] # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # type: ignore [arg-type] # pylint: disable=protected-access ) path_client = AzureDataLakeStorageRESTAPI( - url, - version=self._api_version, - filesystem=self.file_system_name, - path=deleted_path_name, - pipeline=pipeline + url, version=self._api_version, filesystem=self.file_system_name, path=deleted_path_name, pipeline=pipeline ) try: - is_file = cast(bool, await path_client.path.undelete( - undelete_source=undelete_source, cls=is_file_path, **kwargs)) + is_file = cast( + bool, await path_client.path.undelete(undelete_source=undelete_source, cls=is_file_path, **kwargs) + ) if is_file: return self.get_file_client(deleted_path_name) return self.get_directory_client(deleted_path_name) @@ -992,7 +974,7 @@ def _get_root_directory_client(self) -> DataLakeDirectoryClient: :returns: A DataLakeDirectoryClient. :rtype: ~azure.storage.filedatalake.aio.DataLakeDirectoryClient """ - return self.get_directory_client('/') + return self.get_directory_client("/") def get_directory_client(self, directory: Union[DirectoryProperties, str]) -> DataLakeDirectoryClient: """Get a client to interact with the specified directory. @@ -1016,19 +998,24 @@ def get_directory_client(self, directory: Union[DirectoryProperties, str]) -> Da :caption: Getting the directory client to interact with a specific directory. """ if isinstance(directory, DirectoryProperties): - directory_name = directory.get('name') + directory_name = directory.get("name") else: directory_name = str(directory) _pipeline = AsyncPipeline( transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # type: ignore [arg-type] # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # type: ignore [arg-type] # pylint: disable=protected-access + ) + return DataLakeDirectoryClient( + self.url, + self.file_system_name, + directory_name=directory_name, + credential=self._raw_credential, + api_version=self.api_version, + _configuration=self._config, + _pipeline=_pipeline, + _hosts=self._hosts, + loop=self._loop, ) - return DataLakeDirectoryClient(self.url, self.file_system_name, directory_name=directory_name, - credential=self._raw_credential, - api_version=self.api_version, - _configuration=self._config, _pipeline=_pipeline, - _hosts=self._hosts, - loop=self._loop) def get_file_client(self, file_path: Union[FileProperties, str]) -> DataLakeFileClient: """Get a client to interact with the specified file. @@ -1057,12 +1044,19 @@ def get_file_client(self, file_path: Union[FileProperties, str]) -> DataLakeFile file_path = str(file_path) _pipeline = AsyncPipeline( transport=AsyncTransportWrapper(self._pipeline._transport), # pylint: disable=protected-access - policies=self._pipeline._impl_policies # type: ignore [arg-type] # pylint: disable=protected-access + policies=self._pipeline._impl_policies, # type: ignore [arg-type] # pylint: disable=protected-access ) return DataLakeFileClient( - self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential, + self.url, + self.file_system_name, + file_path=file_path, + credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline, loop=self._loop) + _hosts=self._hosts, + _configuration=self._config, + _pipeline=_pipeline, + loop=self._loop, + ) @distributed_trace def list_deleted_paths(self, **kwargs: Any) -> AsyncItemPaged[DeletedPathProperties]: @@ -1088,14 +1082,19 @@ def list_deleted_paths(self, **kwargs: Any) -> AsyncItemPaged[DeletedPathPropert :rtype: ~azure.core.async_paging.AsyncItemPaged[~azure.storage.filedatalake.DeletedPathProperties] """ - path_prefix = kwargs.pop('path_prefix', None) - timeout = kwargs.pop('timeout', None) - results_per_page = kwargs.pop('results_per_page', None) + path_prefix = kwargs.pop("path_prefix", None) + timeout = kwargs.pop("timeout", None) + results_per_page = kwargs.pop("results_per_page", None) command = functools.partial( self._datalake_client_for_blob_operation.file_system.list_blob_hierarchy_segment, showonly="deleted", timeout=timeout, - **kwargs) + **kwargs, + ) return AsyncItemPaged( - command, prefix=path_prefix, page_iterator_class=DeletedPathPropertiesPaged, - results_per_page=results_per_page, **kwargs) + command, + prefix=path_prefix, + page_iterator_class=DeletedPathPropertiesPaged, + results_per_page=results_per_page, + **kwargs, + ) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_list_paths_helper.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_list_paths_helper.py index d65effc98f32..6e1245bacc65 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_list_paths_helper.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_list_paths_helper.py @@ -4,10 +4,7 @@ # license information. # -------------------------------------------------------------------------- -from typing import ( - Any, Callable, cast, Dict, - List, Optional, Tuple, Union -) +from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Union from azure.core.exceptions import HttpResponseError from azure.core.async_paging import AsyncPageIterator @@ -15,13 +12,9 @@ from .._deserialize import ( get_deleted_path_properties_from_generated_code, process_storage_error, - return_headers_and_deserialized_path_list -) -from .._generated.models import ( - BlobItemInternal, - BlobPrefix as GenBlobPrefix, - Path + return_headers_and_deserialized_path_list, ) +from .._generated.models import BlobItemInternal, BlobPrefix as GenBlobPrefix, Path from .._models import DeletedPathProperties, PathProperties from .._shared.models import DictMixin from .._shared.response_handlers import return_context_and_deserialized @@ -43,11 +36,11 @@ class DirectoryPrefix(DictMixin): options include "primary" and "secondary".""" def __init__(self, **kwargs: Any) -> None: - self.name = kwargs.get('prefix') # type: ignore [assignment] - self.results_per_page = kwargs.get('results_per_page') # type: ignore [assignment] - self.file_system = kwargs.get('container') # type: ignore [assignment] - self.delimiter = kwargs.get('delimiter') # type: ignore [assignment] - self.location_mode = kwargs.get('location_mode') # type: ignore [assignment] + self.name = kwargs.get("prefix") # type: ignore [assignment] + self.results_per_page = kwargs.get("results_per_page") # type: ignore [assignment] + self.file_system = kwargs.get("container") # type: ignore [assignment] + self.delimiter = kwargs.get("delimiter") # type: ignore [assignment] + self.location_mode = kwargs.get("location_mode") # type: ignore [assignment] class DeletedPathPropertiesPaged(AsyncPageIterator): @@ -74,18 +67,17 @@ class DeletedPathPropertiesPaged(AsyncPageIterator): options include "primary" and "secondary".""" def __init__( - self, command: Callable, + self, + command: Callable, container: Optional[str] = None, prefix: Optional[str] = None, results_per_page: Optional[int] = None, continuation_token: Optional[str] = None, delimiter: Optional[str] = None, - location_mode: Optional[str] = None + location_mode: Optional[str] = None, ) -> None: super(DeletedPathPropertiesPaged, self).__init__( - get_next=self._get_next_cb, - extract_data=self._extract_data_cb, - continuation_token=continuation_token or "" + get_next=self._get_next_cb, extract_data=self._extract_data_cb, continuation_token=continuation_token or "" ) self._command = command self.service_endpoint = None @@ -104,7 +96,7 @@ async def _get_next_cb(self, continuation_token): marker=continuation_token or None, max_results=self.results_per_page, cls=return_context_and_deserialized, - use_location=self.location_mode + use_location=self.location_mode, ) except HttpResponseError as error: process_storage_error(error) @@ -116,7 +108,7 @@ async def _extract_data_cb(self, get_next_return): self.marker = self._response.marker self.results_per_page = self._response.max_results self.container = self._response.container_name - self.current_page = self._response.segment.blob_prefixes + self._response.segment.blob_items + self.current_page = self._response.segment.blob_prefixes + self._response.segment.blob_items self.current_page = [self._build_item(item) for item in self.current_page] self.delimiter = self._response.delimiter @@ -132,7 +124,7 @@ def _build_item(self, item): container=self.container, prefix=item.name, results_per_page=self.results_per_page, - location_mode=self.location_mode + location_mode=self.location_mode, ) return item @@ -155,17 +147,16 @@ class PathPropertiesPaged(AsyncPageIterator): """The path list to build the items for the current page.""" def __init__( - self, command: Callable, + self, + command: Callable, recursive: bool, path: Optional[str] = None, max_results: Optional[int] = None, continuation_token: Optional[str] = None, - upn: Optional[str] = None + upn: Optional[str] = None, ) -> None: super(PathPropertiesPaged, self).__init__( - get_next=self._get_next_cb, - extract_data=self._extract_data_cb, - continuation_token=continuation_token or "" + get_next=self._get_next_cb, extract_data=self._extract_data_cb, continuation_token=continuation_token or "" ) self._command = command self.recursive = recursive @@ -183,7 +174,7 @@ async def _get_next_cb(self, continuation_token): path=self.path, max_results=self.results_per_page, upn=self.upn, - cls=return_headers_and_deserialized_path_list + cls=return_headers_and_deserialized_path_list, ) except HttpResponseError as error: process_storage_error(error) @@ -192,7 +183,7 @@ async def _extract_data_cb(self, get_next_return): self.path_list, self._response = cast(Tuple[List[Path], Dict[str, Any]], get_next_return) self.current_page = [self._build_item(item) for item in self.path_list] - return self._response['continuation'] or None, self.current_page + return self._response["continuation"] or None, self.current_page @staticmethod def _build_item(item: Union[Path, PathProperties]) -> PathProperties: diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_models.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_models.py index 347c94b3733c..08203973984f 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_models.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_models.py @@ -39,10 +39,7 @@ class FileSystemPropertiesPaged(ContainerPropertiesPaged): """The current page of listed results.""" def __init__(self, *args: Any, **kwargs: Any) -> None: - super(FileSystemPropertiesPaged, self).__init__( - *args, - **kwargs - ) + super(FileSystemPropertiesPaged, self).__init__(*args, **kwargs) @staticmethod def _build_item(item): diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_path_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_path_client_async.py index 210be85bbb2c..0c9dc7a42e36 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_path_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_path_client_async.py @@ -6,10 +6,7 @@ # pylint: disable=docstring-keyword-should-match-keyword-only from datetime import datetime -from typing import ( - Any, Awaitable, Callable, cast, Dict, Optional, Union, - TYPE_CHECKING -) +from typing import Any, Awaitable, Callable, cast, Dict, Optional, Union, TYPE_CHECKING from typing_extensions import Self from azure.core.exceptions import AzureError, HttpResponseError @@ -34,7 +31,7 @@ _parse_url, _rename_path_options, _set_access_control_options, - _set_access_control_recursive_options + _set_access_control_recursive_options, ) from .._serialize import compare_api_versions, convert_dfs_url_to_blob_url, get_api_version from .._shared.base_client import parse_query, StorageAccountHostsMixin @@ -79,21 +76,25 @@ class PathClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin): # ty authentication. Only has an effect when credential is of type AsyncTokenCredential. The value could be https://storage.azure.com/ (default) or https://.blob.core.windows.net. """ + def __init__( - self, account_url: str, + self, + account_url: str, file_system_name: str, path_name: str, - credential: Optional[Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"]] = None, # pylint: disable=line-too-long + credential: Optional[ + Union[str, Dict[str, str], "AzureNamedKeyCredential", "AzureSasCredential", "AsyncTokenCredential"] + ] = None, # pylint: disable=line-too-long **kwargs: Any ) -> None: - kwargs['retry_policy'] = kwargs.get('retry_policy') or ExponentialRetry(**kwargs) + kwargs["retry_policy"] = kwargs.get("retry_policy") or ExponentialRetry(**kwargs) # remove the preceding/trailing delimiter from the path components - file_system_name = file_system_name.strip('/') + file_system_name = file_system_name.strip("/") # the name of root directory is / - if path_name != '/': - path_name = path_name.strip('/') + if path_name != "/": + path_name = path_name.strip("/") if not (file_system_name and path_name): raise ValueError("Please specify a file system name and file path.") @@ -102,14 +103,11 @@ def __init__( blob_account_url = convert_dfs_url_to_blob_url(account_url) self._blob_account_url = blob_account_url - datalake_hosts = kwargs.pop('_hosts', None) + datalake_hosts = kwargs.pop("_hosts", None) blob_hosts = None if datalake_hosts: blob_primary_account_url = convert_dfs_url_to_blob_url(datalake_hosts[LocationMode.PRIMARY]) - blob_hosts = { - LocationMode.PRIMARY: blob_primary_account_url, - LocationMode.SECONDARY: "" - } + blob_hosts = {LocationMode.PRIMARY: blob_primary_account_url, LocationMode.SECONDARY: ""} self._blob_client = BlobClient( account_url=blob_account_url, container_name=file_system_name, @@ -126,11 +124,7 @@ def __init__( self._query_str, self._raw_credential = self._format_query_string(sas_token, credential) super(PathClient, self).__init__( - parsed_url, - service='dfs', - credential=self._raw_credential, - _hosts=datalake_hosts, - **kwargs + parsed_url, service="dfs", credential=self._raw_credential, _hosts=datalake_hosts, **kwargs ) # ADLS doesn't support secondary endpoint, make sure it's empty @@ -138,7 +132,7 @@ def __init__( self._api_version = get_api_version(kwargs) self._client = self._build_generated_client(self.url) self._datalake_client_for_blob_operation = self._build_generated_client(self._blob_client.url) - self._loop = kwargs.get('loop', None) + self._loop = kwargs.get("loop", None) async def __aenter__(self) -> Self: await self._client.__aenter__() @@ -170,7 +164,7 @@ def _build_generated_client(self, url: str) -> AzureDataLakeStorageRESTAPI: base_url=url, file_system=self.file_system_name, path=self.path_name, - pipeline=self._pipeline + pipeline=self._pipeline, ) return client @@ -178,7 +172,8 @@ def _format_url(self, hostname: str) -> str: return _format_url(self.scheme, hostname, self.file_system_name, self.path_name, self._query_str) async def _create( - self, resource_type: str, + self, + resource_type: str, content_settings: Optional["ContentSettings"] = None, metadata: Optional[Dict[str, str]] = None, **kwargs: Any @@ -272,8 +267,8 @@ async def _create( :return: A dictionary of response headers. :rtype: Dict[str, Union[str, ~datetime.datetime]] """ - lease_id = kwargs.get('lease_id', None) - lease_duration = kwargs.get('lease_duration', None) + lease_id = kwargs.get("lease_id", None) + lease_duration = kwargs.get("lease_duration", None) if lease_id and not lease_duration: raise ValueError("Please specify a lease_id and a lease_duration.") if lease_duration and not lease_id: @@ -321,19 +316,21 @@ async def _delete(self, **kwargs: Any) -> Dict[str, Any]: # Perform paginated delete only if using OAuth, deleting a directory, and api version is 2023-08-03 or later # The pagination is only for ACL checks, the final request remains the atomic delete operation paginated = None - if (compare_api_versions(self.api_version, '2023-08-03') >= 0 and - hasattr(self.credential, 'get_token') and - kwargs.get('recursive')): # Directory delete will always specify recursive + if ( + compare_api_versions(self.api_version, "2023-08-03") >= 0 + and hasattr(self.credential, "get_token") + and kwargs.get("recursive") + ): # Directory delete will always specify recursive paginated = True options = _delete_path_options(paginated, **kwargs) try: response_headers = await self._client.path.delete(**options) # Loop until continuation token is None for paginated delete - while response_headers['continuation']: + while response_headers["continuation"]: response_headers = await self._client.path.delete( - continuation=response_headers['continuation'], - **options) + continuation=response_headers["continuation"], **options + ) return response_headers except HttpResponseError as error: @@ -341,7 +338,8 @@ async def _delete(self, **kwargs: Any) -> Dict[str, Any]: @distributed_trace_async async def set_access_control( - self, owner: Optional[str] = None, + self, + owner: Optional[str] = None, group: Optional[str] = None, permissions: Optional[str] = None, acl: Optional[str] = None, @@ -509,11 +507,12 @@ async def set_access_control_recursive(self, acl: str, **kwargs: Any) -> AccessC if not acl: raise ValueError("The Access Control List must be set for this operation") - progress_hook = kwargs.pop('progress_hook', None) - max_batches = kwargs.pop('max_batches', None) - options = _set_access_control_recursive_options(mode='set', acl=acl, **kwargs) - return await self._set_access_control_internal(options=options, progress_hook=progress_hook, - max_batches=max_batches) + progress_hook = kwargs.pop("progress_hook", None) + max_batches = kwargs.pop("max_batches", None) + options = _set_access_control_recursive_options(mode="set", acl=acl, **kwargs) + return await self._set_access_control_internal( + options=options, progress_hook=progress_hook, max_batches=max_batches + ) @distributed_trace_async async def update_access_control_recursive(self, acl: str, **kwargs: Any) -> AccessControlChangeResult: @@ -563,11 +562,12 @@ async def update_access_control_recursive(self, acl: str, **kwargs: Any) -> Acce if not acl: raise ValueError("The Access Control List must be set for this operation") - progress_hook = kwargs.pop('progress_hook', None) - max_batches = kwargs.pop('max_batches', None) - options = _set_access_control_recursive_options(mode='modify', acl=acl, **kwargs) - return await self._set_access_control_internal(options=options, progress_hook=progress_hook, - max_batches=max_batches) + progress_hook = kwargs.pop("progress_hook", None) + max_batches = kwargs.pop("max_batches", None) + options = _set_access_control_recursive_options(mode="modify", acl=acl, **kwargs) + return await self._set_access_control_internal( + options=options, progress_hook=progress_hook, max_batches=max_batches + ) @distributed_trace_async async def remove_access_control_recursive(self, acl: str, **kwargs: Any) -> AccessControlChangeResult: @@ -615,19 +615,21 @@ async def remove_access_control_recursive(self, acl: str, **kwargs: Any) -> Acce if not acl: raise ValueError("The Access Control List must be set for this operation") - progress_hook = kwargs.pop('progress_hook', None) - max_batches = kwargs.pop('max_batches', None) - options = _set_access_control_recursive_options(mode='remove', acl=acl, **kwargs) - return await self._set_access_control_internal(options=options, progress_hook=progress_hook, - max_batches=max_batches) + progress_hook = kwargs.pop("progress_hook", None) + max_batches = kwargs.pop("max_batches", None) + options = _set_access_control_recursive_options(mode="remove", acl=acl, **kwargs) + return await self._set_access_control_internal( + options=options, progress_hook=progress_hook, max_batches=max_batches + ) async def _set_access_control_internal( - self, options: Dict[str, Any], + self, + options: Dict[str, Any], progress_hook: Optional[Callable[[AccessControlChanges], Awaitable[Any]]], - max_batches: Optional[int] = None + max_batches: Optional[int] = None, ) -> AccessControlChangeResult: try: - continue_on_failure = options.get('force_flag') + continue_on_failure = options.get("force_flag") total_directories_successful = 0 total_files_success = 0 total_failure_count = 0 @@ -643,44 +645,56 @@ async def _set_access_control_internal( total_files_success += resp.files_successful total_failure_count += resp.failure_count batch_count += 1 - current_continuation_token = headers['continuation'] + current_continuation_token = headers["continuation"] if current_continuation_token is not None: last_continuation_token = current_continuation_token if progress_hook is not None: - await progress_hook(AccessControlChanges( - batch_counters=AccessControlChangeCounters( - directories_successful=resp.directories_successful, - files_successful=resp.files_successful, - failure_count=resp.failure_count, - ), - aggregate_counters=AccessControlChangeCounters( - directories_successful=total_directories_successful, - files_successful=total_files_success, - failure_count=total_failure_count, - ), - batch_failures=[AccessControlChangeFailure( - name=failure.name, - is_directory=failure.type == 'DIRECTORY', - error_message=failure.error_message) for failure in resp.failed_entries], - continuation=last_continuation_token - )) + await progress_hook( + AccessControlChanges( + batch_counters=AccessControlChangeCounters( + directories_successful=resp.directories_successful, + files_successful=resp.files_successful, + failure_count=resp.failure_count, + ), + aggregate_counters=AccessControlChangeCounters( + directories_successful=total_directories_successful, + files_successful=total_files_success, + failure_count=total_failure_count, + ), + batch_failures=[ + AccessControlChangeFailure( + name=failure.name, + is_directory=failure.type == "DIRECTORY", + error_message=failure.error_message, + ) + for failure in resp.failed_entries + ], + continuation=last_continuation_token, + ) + ) # update the continuation token, if there are more operations that cannot be completed in a single call - max_batches_satisfied = (max_batches is not None and batch_count == max_batches) + max_batches_satisfied = max_batches is not None and batch_count == max_batches continue_operation = bool(current_continuation_token) and not max_batches_satisfied - options['continuation'] = current_continuation_token + options["continuation"] = current_continuation_token # currently the service stops on any failure, so we should send back the last continuation token # for the user to retry the failed updates # otherwise we should just return what the service gave us - return AccessControlChangeResult(counters=AccessControlChangeCounters( - directories_successful=total_directories_successful, - files_successful=total_files_success, - failure_count=total_failure_count), - continuation=last_continuation_token - if total_failure_count > 0 and not continue_on_failure else current_continuation_token) + return AccessControlChangeResult( + counters=AccessControlChangeCounters( + directories_successful=total_directories_successful, + files_successful=total_files_success, + failure_count=total_failure_count, + ), + continuation=( + last_continuation_token + if total_failure_count > 0 and not continue_on_failure + else current_continuation_token + ), + ) except HttpResponseError as error: error.continuation_token = last_continuation_token process_storage_error(error) @@ -800,11 +814,11 @@ async def _get_path_properties(self, **kwargs: Any) -> Union[DirectoryProperties and system properties for the file or directory. :rtype: DirectoryProperties or FileProperties """ - upn = kwargs.pop('upn', None) + upn = kwargs.pop("upn", None) if upn: - headers = kwargs.pop('headers', {}) - headers['x-ms-upn'] = str(upn) - kwargs['headers'] = headers + headers = kwargs.pop("headers", {}) + headers["x-ms-upn"] = str(upn) + kwargs["headers"] = headers path_properties = await self._blob_client.get_blob_properties(**kwargs) return cast(Union[DirectoryProperties, FileProperties], path_properties) @@ -871,8 +885,7 @@ async def set_metadata(self, metadata: Dict[str, str], **kwargs: Any) -> Dict[st @distributed_trace_async async def set_http_headers( - self, content_settings: Optional["ContentSettings"] = None, - **kwargs: Any + self, content_settings: Optional["ContentSettings"] = None, **kwargs: Any ) -> Dict[str, Any]: """Sets system properties on the file or directory. @@ -914,9 +927,7 @@ async def set_http_headers( @distributed_trace_async async def acquire_lease( - self, lease_duration: int = -1, - lease_id: Optional[str] = None, - **kwargs: Any + self, lease_duration: int = -1, lease_id: Optional[str] = None, **kwargs: Any ) -> DataLakeLeaseClient: """ Requests a new lease. If the file or directory does not have an active lease, diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_upload_helper.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_upload_helper.py index ebef79ee1514..8c50e1f41519 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_upload_helper.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_upload_helper.py @@ -4,20 +4,13 @@ # license information. # -------------------------------------------------------------------------- -from typing import ( - Any, cast, Dict, IO, Optional, - TYPE_CHECKING -) +from typing import Any, cast, Dict, IO, Optional, TYPE_CHECKING from azure.core.exceptions import HttpResponseError from .._deserialize import process_storage_error from .._shared.response_handlers import return_response_headers -from .._shared.uploads_async import ( - DataLakeFileChunkUploader, - upload_data_chunks, - upload_substream_blocks -) +from .._shared.uploads_async import DataLakeFileChunkUploader, upload_data_chunks, upload_substream_blocks if TYPE_CHECKING: from .._generated.aio.operations import PathOperations @@ -25,12 +18,14 @@ def _any_conditions(modified_access_conditions=None, **kwargs): # pylint: disable=unused-argument - return any([ - modified_access_conditions.if_modified_since, - modified_access_conditions.if_unmodified_since, - modified_access_conditions.if_none_match, - modified_access_conditions.if_match - ]) + return any( + [ + modified_access_conditions.if_modified_since, + modified_access_conditions.if_unmodified_since, + modified_access_conditions.if_none_match, + modified_access_conditions.if_match, + ] + ) async def upload_datalake_file( @@ -46,46 +41,54 @@ async def upload_datalake_file( try: if length == 0: return {} - properties = kwargs.pop('properties', None) - umask = kwargs.pop('umask', None) - permissions = kwargs.pop('permissions', None) - path_http_headers = kwargs.pop('path_http_headers', None) - modified_access_conditions = kwargs.pop('modified_access_conditions', None) - chunk_size = kwargs.pop('chunk_size', 100 * 1024 * 1024) - encryption_context = kwargs.pop('encryption_context', None) - progress_hook = kwargs.pop('progress_hook', None) + properties = kwargs.pop("properties", None) + umask = kwargs.pop("umask", None) + permissions = kwargs.pop("permissions", None) + path_http_headers = kwargs.pop("path_http_headers", None) + modified_access_conditions = kwargs.pop("modified_access_conditions", None) + chunk_size = kwargs.pop("chunk_size", 100 * 1024 * 1024) + encryption_context = kwargs.pop("encryption_context", None) + progress_hook = kwargs.pop("progress_hook", None) if not overwrite: # if customers didn't specify access conditions, they cannot flush data to existing file if not _any_conditions(modified_access_conditions): - modified_access_conditions.if_none_match = '*' + modified_access_conditions.if_none_match = "*" if properties or umask or permissions: raise ValueError("metadata, umask and permissions can be set only when overwrite is enabled") if overwrite: - response = cast(Dict[str, Any], await client.create( - resource='file', - path_http_headers=path_http_headers, - properties=properties, - modified_access_conditions=modified_access_conditions, - umask=umask, - permissions=permissions, - encryption_context=encryption_context, - cls=return_response_headers, - **kwargs - )) + response = cast( + Dict[str, Any], + await client.create( + resource="file", + path_http_headers=path_http_headers, + properties=properties, + modified_access_conditions=modified_access_conditions, + umask=umask, + permissions=permissions, + encryption_context=encryption_context, + cls=return_response_headers, + **kwargs + ), + ) # this modified_access_conditions will be applied to flush_data to make sure # no other flush between create and the current flush - modified_access_conditions.if_match = response['etag'] + modified_access_conditions.if_match = response["etag"] modified_access_conditions.if_none_match = None modified_access_conditions.if_modified_since = None modified_access_conditions.if_unmodified_since = None - use_original_upload_path = file_settings.use_byte_buffer or \ - validate_content or chunk_size < file_settings.min_large_chunk_upload_threshold or \ - hasattr(stream, 'seekable') and not stream.seekable() or \ - not hasattr(stream, 'seek') or not hasattr(stream, 'tell') + use_original_upload_path = ( + file_settings.use_byte_buffer + or validate_content + or chunk_size < file_settings.min_large_chunk_upload_threshold + or hasattr(stream, "seekable") + and not stream.seekable() + or not hasattr(stream, "seek") + or not hasattr(stream, "tell") + ) if use_original_upload_path: await upload_data_chunks( @@ -112,13 +115,16 @@ async def upload_datalake_file( **kwargs ) - return cast(Dict[str, Any], await client.flush_data( - position=length, - path_http_headers=path_http_headers, - modified_access_conditions=modified_access_conditions, - close=True, - cls=return_response_headers, - **kwargs - )) + return cast( + Dict[str, Any], + await client.flush_data( + position=length, + path_http_headers=path_http_headers, + modified_access_conditions=modified_access_conditions, + close=True, + cls=return_response_headers, + **kwargs + ), + ) except HttpResponseError as error: process_storage_error(error) diff --git a/sdk/storage/azure-storage-file-datalake/conftest.py b/sdk/storage/azure-storage-file-datalake/conftest.py index b830bd31a95b..ffcb0c5bce44 100644 --- a/sdk/storage/azure-storage-file-datalake/conftest.py +++ b/sdk/storage/azure-storage-file-datalake/conftest.py @@ -23,6 +23,7 @@ if platform.python_implementation() == "PyPy": collect_ignore_glob.append("tests/*_async.py") + @pytest.fixture(scope="session", autouse=True) def add_sanitizers(test_proxy): subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID", "00000000-0000-0000-0000-000000000000") diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control.py index 0b0c9960f398..21c95e475630 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control.py @@ -37,15 +37,15 @@ def access_control_sample(filesystem_client): # get and display the permissions of the parent directory acl_props = directory_client.get_access_control() - print("Permissions of directory '{}' are {}.".format(dir_name, acl_props['permissions'])) + print("Permissions of directory '{}' are {}.".format(dir_name, acl_props["permissions"])) # set the permissions of the parent directory - new_dir_permissions = 'rwx------' + new_dir_permissions = "rwx------" directory_client.set_access_control(permissions=new_dir_permissions) # get and display the permissions of the parent directory again acl_props = directory_client.get_access_control() - print("New permissions of directory '{}' are {}.".format(dir_name, acl_props['permissions'])) + print("New permissions of directory '{}' are {}.".format(dir_name, acl_props["permissions"])) # iterate through every file and set their permissions to match the directory for file in filesystem_client.get_paths(dir_name): @@ -54,7 +54,7 @@ def access_control_sample(filesystem_client): # get the access control properties of the file acl_props = file_client.get_access_control() - if acl_props['permissions'] != new_dir_permissions: + if acl_props["permissions"] != new_dir_permissions: file_client.set_access_control(permissions=new_dir_permissions) print("Set the permissions of file '{}' to {}.".format(file.name, new_dir_permissions)) else: @@ -64,11 +64,13 @@ def access_control_sample(filesystem_client): def create_child_files(directory_client, num_child_files): import concurrent.futures import itertools + # Use a thread pool because it is too slow otherwise with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + def create_file(): # generate a random name - file_name = str(uuid.uuid4()).replace('-', '') + file_name = str(uuid.uuid4()).replace("-", "") directory_client.get_file_client(file_name).create_file() futures = {executor.submit(create_file) for _ in itertools.repeat(None, num_child_files)} @@ -77,14 +79,13 @@ def create_file(): def run(): - account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") - account_key = os.getenv('DATALAKE_STORAGE_ACCOUNT_KEY', "") + account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") + account_key = os.getenv("DATALAKE_STORAGE_ACCOUNT_KEY", "") # set up the service client with the credentials from the environment variables - service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format( - "https", - account_name - ), credential=account_key) + service_client = DataLakeServiceClient( + account_url="{}://{}.dfs.core.windows.net".format("https", account_name), credential=account_key + ) # generate a random name for testing purpose fs_name = "testfs{}".format(random.randint(1, 1000)) @@ -101,5 +102,5 @@ def run(): filesystem_client.delete_file_system() -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_async.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_async.py index 3f37c0c6741a..c79880f6b7d4 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_async.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_async.py @@ -38,15 +38,15 @@ async def access_control_sample(filesystem_client): # get and display the permissions of the parent directory acl_props = await directory_client.get_access_control() - print("Permissions of directory '{}' are {}.".format(dir_name, acl_props['permissions'])) + print("Permissions of directory '{}' are {}.".format(dir_name, acl_props["permissions"])) # set the permissions of the parent directory - new_dir_permissions = 'rwx------' + new_dir_permissions = "rwx------" await directory_client.set_access_control(permissions=new_dir_permissions) # get and display the permissions of the parent directory again acl_props = await directory_client.get_access_control() - print("New permissions of directory '{}' are {}.".format(dir_name, acl_props['permissions'])) + print("New permissions of directory '{}' are {}.".format(dir_name, acl_props["permissions"])) # iterate through every file and set their permissions to match the directory async for file in filesystem_client.get_paths(dir_name): @@ -55,7 +55,7 @@ async def access_control_sample(filesystem_client): # get the access control properties of the file acl_props = await file_client.get_access_control() - if acl_props['permissions'] != new_dir_permissions: + if acl_props["permissions"] != new_dir_permissions: await file_client.set_access_control(permissions=new_dir_permissions) print("Set the permissions of file '{}' to {}.".format(file.name, new_dir_permissions)) else: @@ -67,7 +67,7 @@ async def create_child_files(directory_client, num_child_files): async def create_file(): # generate a random name - file_name = str(uuid.uuid4()).replace('-', '') + file_name = str(uuid.uuid4()).replace("-", "") file_client = directory_client.get_file_client(file_name) await file_client.create_file() @@ -77,13 +77,12 @@ async def create_file(): async def main(): - account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") - account_key = os.getenv('DATALAKE_STORAGE_ACCOUNT_KEY', "") + account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") + account_key = os.getenv("DATALAKE_STORAGE_ACCOUNT_KEY", "") # set up the service client with the credentials from the environment variables - service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format( - "https", - account_name - ), credential=account_key) + service_client = DataLakeServiceClient( + account_url="{}://{}.dfs.core.windows.net".format("https", account_name), credential=account_key + ) async with service_client: # generate a random name for testing purpose @@ -101,5 +100,5 @@ async def main(): await filesystem_client.delete_file_system() -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive.py index 3b393947e17f..658755851ade 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive.py @@ -39,59 +39,75 @@ def recursive_access_control_sample(filesystem_client): # get and display the permissions of the parent directory acl_props = directory_client.get_access_control() - print("Permissions of directory '{}' are {}.".format(dir_name, acl_props['permissions'])) + print("Permissions of directory '{}' are {}.".format(dir_name, acl_props["permissions"])) # set the permissions of the entire directory tree recursively # update/remove acl operations are performed the same way - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" failed_entries = [] # the progress callback is invoked each time a batch is completed def progress_callback(acl_changes): - print(("In this batch: {} directories and {} files were processed successfully, {} failures were counted. " + - "In total, {} directories and {} files were processed successfully, {} failures were counted.") - .format(acl_changes.batch_counters.directories_successful, acl_changes.batch_counters.files_successful, - acl_changes.batch_counters.failure_count, acl_changes.aggregate_counters.directories_successful, - acl_changes.aggregate_counters.files_successful, acl_changes.aggregate_counters.failure_count)) + print( + ( + "In this batch: {} directories and {} files were processed successfully, {} failures were counted. " + + "In total, {} directories and {} files were processed successfully, {} failures were counted." + ).format( + acl_changes.batch_counters.directories_successful, + acl_changes.batch_counters.files_successful, + acl_changes.batch_counters.failure_count, + acl_changes.aggregate_counters.directories_successful, + acl_changes.aggregate_counters.files_successful, + acl_changes.aggregate_counters.failure_count, + ) + ) # keep track of failed entries if there are any failed_entries.append(acl_changes.batch_failures) # illustrate the operation by using a small batch_size try: - acl_change_result = directory_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=5) + acl_change_result = directory_client.set_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=5 + ) except AzureError as error: # if the error has continuation_token, you can restart the operation using that continuation_token if error.continuation_token: - acl_change_result = \ - directory_client.set_access_control_recursive(acl=acl, - continuation_token=error.continuation_token, - progress_hook=progress_callback, - batch_size=5) - - print("Summary: {} directories and {} files were updated successfully, {} failures were counted." - .format(acl_change_result.counters.directories_successful, acl_change_result.counters.files_successful, - acl_change_result.counters.failure_count)) + acl_change_result = directory_client.set_access_control_recursive( + acl=acl, continuation_token=error.continuation_token, progress_hook=progress_callback, batch_size=5 + ) + + print( + "Summary: {} directories and {} files were updated successfully, {} failures were counted.".format( + acl_change_result.counters.directories_successful, + acl_change_result.counters.files_successful, + acl_change_result.counters.failure_count, + ) + ) # if an error was encountered, a continuation token would be returned if the operation can be resumed if acl_change_result.continuation is not None: - print("The operation can be resumed by passing the continuation token {} again into the access control method." - .format(acl_change_result.continuation)) + print( + "The operation can be resumed by passing the continuation token {} again into the access control method.".format( + acl_change_result.continuation + ) + ) # get and display the permissions of the parent directory again acl_props = directory_client.get_access_control() - print("New permissions of directory '{}' and its children are {}.".format(dir_name, acl_props['permissions'])) + print("New permissions of directory '{}' and its children are {}.".format(dir_name, acl_props["permissions"])) def create_child_files(directory_client, num_child_files): import concurrent.futures import itertools + # Use a thread pool because it is too slow otherwise with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + def create_file(): # generate a random name - file_name = str(uuid.uuid4()).replace('-', '') + file_name = str(uuid.uuid4()).replace("-", "") directory_client.get_file_client(file_name).create_file() futures = {executor.submit(create_file) for _ in itertools.repeat(None, num_child_files)} @@ -100,14 +116,13 @@ def create_file(): def run(): - account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") - account_key = os.getenv('DATALAKE_STORAGE_ACCOUNT_KEY', "") + account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") + account_key = os.getenv("DATALAKE_STORAGE_ACCOUNT_KEY", "") # set up the service client with the credentials from the environment variables - service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format( - "https", - account_name - ), credential=account_key) + service_client = DataLakeServiceClient( + account_url="{}://{}.dfs.core.windows.net".format("https", account_name), credential=account_key + ) # generate a random name for testing purpose fs_name = "testfs{}".format(random.randint(1, 1000)) @@ -124,5 +139,5 @@ def run(): filesystem_client.delete_file_system() -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive_async.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive_async.py index 581e7b8a9f24..587e1888667b 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive_async.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive_async.py @@ -41,50 +41,63 @@ async def recursive_access_control_sample(filesystem_client): # get and display the permissions of the parent directory acl_props = await directory_client.get_access_control() - print("Permissions of directory '{}' are {}.".format(dir_name, acl_props['permissions'])) + print("Permissions of directory '{}' are {}.".format(dir_name, acl_props["permissions"])) # set the permissions of the entire directory tree recursively # update/remove acl operations are performed the same way - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" failed_entries = [] # the progress callback is invoked each time a batch is completed async def progress_callback(acl_changes): - print(("In this batch: {} directories and {} files were processed successfully, {} failures were counted. " + - "In total, {} directories and {} files were processed successfully, {} failures were counted.") - .format(acl_changes.batch_counters.directories_successful, acl_changes.batch_counters.files_successful, - acl_changes.batch_counters.failure_count, acl_changes.aggregate_counters.directories_successful, - acl_changes.aggregate_counters.files_successful, acl_changes.aggregate_counters.failure_count)) + print( + ( + "In this batch: {} directories and {} files were processed successfully, {} failures were counted. " + + "In total, {} directories and {} files were processed successfully, {} failures were counted." + ).format( + acl_changes.batch_counters.directories_successful, + acl_changes.batch_counters.files_successful, + acl_changes.batch_counters.failure_count, + acl_changes.aggregate_counters.directories_successful, + acl_changes.aggregate_counters.files_successful, + acl_changes.aggregate_counters.failure_count, + ) + ) # keep track of failed entries if there are any failed_entries.append(acl_changes.batch_failures) # illustrate the operation by using a small batch_size try: - acl_change_result = await directory_client.set_access_control_recursive(acl=acl, - progress_hook=progress_callback, - batch_size=5) + acl_change_result = await directory_client.set_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=5 + ) except AzureError as error: # if the error has continuation_token, you can restart the operation using that continuation_token if error.continuation_token: - acl_change_result = \ - await directory_client.set_access_control_recursive(acl=acl, - continuation_token=error.continuation_token, - progress_hook=progress_callback, - batch_size=5) - - print("Summary: {} directories and {} files were updated successfully, {} failures were counted." - .format(acl_change_result.counters.directories_successful, acl_change_result.counters.files_successful, - acl_change_result.counters.failure_count)) + acl_change_result = await directory_client.set_access_control_recursive( + acl=acl, continuation_token=error.continuation_token, progress_hook=progress_callback, batch_size=5 + ) + + print( + "Summary: {} directories and {} files were updated successfully, {} failures were counted.".format( + acl_change_result.counters.directories_successful, + acl_change_result.counters.files_successful, + acl_change_result.counters.failure_count, + ) + ) # if an error was encountered, a continuation token would be returned if the operation can be resumed if acl_change_result.continuation is not None: - print("The operation can be resumed by passing the continuation token {} again into the access control method." - .format(acl_change_result.continuation)) + print( + "The operation can be resumed by passing the continuation token {} again into the access control method.".format( + acl_change_result.continuation + ) + ) # get and display the permissions of the parent directory again acl_props = await directory_client.get_access_control() - print("New permissions of directory '{}' and its children are {}.".format(dir_name, acl_props['permissions'])) + print("New permissions of directory '{}' and its children are {}.".format(dir_name, acl_props["permissions"])) async def create_child_files(directory_client, num_child_files): @@ -92,7 +105,7 @@ async def create_child_files(directory_client, num_child_files): async def create_file(): # generate a random name - file_name = str(uuid.uuid4()).replace('-', '') + file_name = str(uuid.uuid4()).replace("-", "") file_client = directory_client.get_file_client(file_name) await file_client.create_file() @@ -102,14 +115,13 @@ async def create_file(): async def main(): - account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") - account_key = os.getenv('DATALAKE_STORAGE_ACCOUNT_KEY', "") + account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") + account_key = os.getenv("DATALAKE_STORAGE_ACCOUNT_KEY", "") # set up the service client with the credentials from the environment variables - service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format( - "https", - account_name - ), credential=account_key) + service_client = DataLakeServiceClient( + account_url="{}://{}.dfs.core.windows.net".format("https", account_name), credential=account_key + ) async with service_client: # generate a random name for testing purpose @@ -127,5 +139,5 @@ async def main(): await filesystem_client.delete_file_system() -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_directory.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_directory.py index a8b25790ba03..a0812163a847 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_directory.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_directory.py @@ -52,8 +52,7 @@ def directory_sample(filesystem_client): # [START rename_directory] new_dir_name = "testdir2" print("Renaming the directory named '{}' to '{}'.".format(dir_name, new_dir_name)) - new_directory = directory_client\ - .rename_directory(new_name=directory_client.file_system_name + '/' + new_dir_name) + new_directory = directory_client.rename_directory(new_name=directory_client.file_system_name + "/" + new_dir_name) # [END rename_directory] # display the properties of the new directory to make sure it was renamed successfully @@ -72,11 +71,13 @@ def directory_sample(filesystem_client): def create_child_files(directory_client, num_child_files): import concurrent.futures import itertools + # Use a thread pool because it is too slow otherwise with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + def create_file(): # generate a random name - file_name = str(uuid.uuid4()).replace('-', '') + file_name = str(uuid.uuid4()).replace("-", "") directory_client.get_file_client(file_name).create_file() futures = {executor.submit(create_file) for _ in itertools.repeat(None, num_child_files)} @@ -85,14 +86,13 @@ def create_file(): def run(): - account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") - account_key = os.getenv('DATALAKE_STORAGE_ACCOUNT_KEY', "") + account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") + account_key = os.getenv("DATALAKE_STORAGE_ACCOUNT_KEY", "") # set up the service client with the credentials from the environment variables - service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format( - "https", - account_name - ), credential=account_key) + service_client = DataLakeServiceClient( + account_url="{}://{}.dfs.core.windows.net".format("https", account_name), credential=account_key + ) # generate a random name for testing purpose fs_name = "dicretorytestfs{}".format(random.randint(1, 1000)) @@ -109,5 +109,5 @@ def run(): filesystem_client.delete_file_system() -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_directory_async.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_directory_async.py index fd7fd1e1dca5..9caeff14b1e1 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_directory_async.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_directory_async.py @@ -53,8 +53,9 @@ async def directory_sample(filesystem_client): # [START rename_directory] new_dir_name = "testdir2async" print("Renaming the directory named '{}' to '{}'.".format(dir_name, new_dir_name)) - new_directory = await directory_client\ - .rename_directory(new_name=directory_client.file_system_name + '/' + new_dir_name) + new_directory = await directory_client.rename_directory( + new_name=directory_client.file_system_name + "/" + new_dir_name + ) # [END rename_directory] # display the properties of the new directory to make sure it was renamed successfully @@ -72,11 +73,12 @@ async def directory_sample(filesystem_client): async def create_child_files(directory_client, num_child_files): import itertools + # Use a thread pool because it is too slow otherwise async def create_file(): # generate a random name - file_name = str(uuid.uuid4()).replace('-', '') + file_name = str(uuid.uuid4()).replace("-", "") file_client = directory_client.get_file_client(file_name) await file_client.create_file() @@ -86,14 +88,13 @@ async def create_file(): async def main(): - account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") - account_key = os.getenv('DATALAKE_STORAGE_ACCOUNT_KEY', "") + account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") + account_key = os.getenv("DATALAKE_STORAGE_ACCOUNT_KEY", "") # set up the service client with the credentials from the environment variables - service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format( - "https", - account_name - ), credential=account_key) + service_client = DataLakeServiceClient( + account_url="{}://{}.dfs.core.windows.net".format("https", account_name), credential=account_key + ) async with service_client: # generate a random name for testing purpose @@ -110,5 +111,6 @@ async def main(): # clean up the demo filesystem await filesystem_client.delete_file_system() -if __name__ == '__main__': + +if __name__ == "__main__": asyncio.run(main()) diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_file_system.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_file_system.py index d322da0540c5..60a0af1a53ec 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_file_system.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_file_system.py @@ -27,15 +27,16 @@ class FileSystemSamples(object): - connection_string = os.environ['DATALAKE_STORAGE_CONNECTION_STRING'] + connection_string = os.environ["DATALAKE_STORAGE_CONNECTION_STRING"] - #--Begin File System Samples----------------------------------------------------------------- + # --Begin File System Samples----------------------------------------------------------------- def file_system_sample(self): # [START create_file_system_client_from_service] # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) # Instantiate a FileSystemClient @@ -61,6 +62,7 @@ def acquire_lease_on_file_system(self): # Instantiate a DataLakeServiceClient using a connection string # [START create_data_lake_service_client_from_conn_str] from azure.storage.filedatalake import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) # [END create_data_lake_service_client_from_conn_str] @@ -85,6 +87,7 @@ def set_metadata_on_file_system(self): # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) # Instantiate a FileSystemClient @@ -96,7 +99,7 @@ def set_metadata_on_file_system(self): # [START set_file_system_metadata] # Create key, value pairs for metadata - metadata = {'type': 'test'} + metadata = {"type": "test"} # Set metadata on the file system file_system_client.set_file_system_metadata(metadata=metadata) @@ -113,6 +116,7 @@ def list_paths_in_file_system(self): # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) # Instantiate a FileSystemClient @@ -132,7 +136,7 @@ def list_paths_in_file_system(self): # [START get_paths_in_file_system] path_list = file_system_client.get_paths() for path in path_list: - print(path.name + '\n') + print(path.name + "\n") # [END get_paths_in_file_system] # Delete file system @@ -142,6 +146,7 @@ def get_file_client_from_file_system(self): # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) # Instantiate a FileSystemClient @@ -165,6 +170,7 @@ def get_directory_client_from_file_system(self): # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) # Instantiate a FileSystemClient @@ -187,6 +193,7 @@ def get_directory_client_from_file_system(self): def create_file_from_file_system(self): # [START create_file_system_client_from_connection_string] from azure.storage.filedatalake import FileSystemClient + file_system_client = FileSystemClient.from_connection_string(self.connection_string, "filesystem") # [END create_file_system_client_from_connection_string] @@ -210,7 +217,8 @@ def create_file_from_file_system(self): file_system_client.delete_file_system() -if __name__ == '__main__': + +if __name__ == "__main__": sample = FileSystemSamples() sample.file_system_sample() sample.acquire_lease_on_file_system() diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_file_system_async.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_file_system_async.py index 3eb60d6a9862..782b5c863129 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_file_system_async.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_file_system_async.py @@ -16,6 +16,7 @@ Set the environment variables with your own values before running the sample: 1) DATALAKE_STORAGE_CONNECTION_STRING - the connection string to your storage account """ + import asyncio import os @@ -27,15 +28,16 @@ class FileSystemSamplesAsync(object): - connection_string = os.environ['DATALAKE_STORAGE_CONNECTION_STRING'] + connection_string = os.environ["DATALAKE_STORAGE_CONNECTION_STRING"] - #--Begin File System Samples----------------------------------------------------------------- + # --Begin File System Samples----------------------------------------------------------------- async def file_system_sample(self): # [START create_file_system_client_from_service] # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake.aio import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) async with datalake_service_client: @@ -62,6 +64,7 @@ async def acquire_lease_on_file_system(self): # Instantiate a DataLakeServiceClient using a connection string # [START create_data_lake_service_client_from_conn_str] from azure.storage.filedatalake.aio import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) # [END create_data_lake_service_client_from_conn_str] @@ -87,6 +90,7 @@ async def set_metadata_on_file_system(self): # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake.aio import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) async with datalake_service_client: @@ -99,7 +103,7 @@ async def set_metadata_on_file_system(self): # [START set_file_system_metadata] # Create key, value pairs for metadata - metadata = {'type': 'test'} + metadata = {"type": "test"} # Set metadata on the file system await file_system_client.set_file_system_metadata(metadata=metadata) @@ -116,6 +120,7 @@ async def list_paths_in_file_system(self): # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake.aio import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) async with datalake_service_client: @@ -137,7 +142,7 @@ async def list_paths_in_file_system(self): # [START get_paths_in_file_system] path_list = file_system_client.get_paths() async for path in path_list: - print(path.name + '\n') + print(path.name + "\n") # [END get_paths_in_file_system] # Delete file system @@ -147,6 +152,7 @@ async def get_file_client_from_file_system(self): # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake.aio import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) async with datalake_service_client: @@ -171,6 +177,7 @@ async def get_directory_client_from_file_system(self): # Instantiate a DataLakeServiceClient using a connection string from azure.storage.filedatalake.aio import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) async with datalake_service_client: @@ -194,6 +201,7 @@ async def get_directory_client_from_file_system(self): async def create_file_from_file_system(self): # [START create_file_system_client_from_connection_string] from azure.storage.filedatalake.aio import FileSystemClient + file_system_client = FileSystemClient.from_connection_string(self.connection_string, "filesystemforcreateasync") # [END create_file_system_client_from_connection_string] @@ -218,6 +226,7 @@ async def create_file_from_file_system(self): await file_system_client.delete_file_system() + async def main(): sample = FileSystemSamplesAsync() await sample.file_system_sample() @@ -227,5 +236,6 @@ async def main(): await sample.get_file_client_from_file_system() await sample.create_file_from_file_system() -if __name__ == '__main__': + +if __name__ == "__main__": asyncio.run(main()) diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_instantiate_client.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_instantiate_client.py index 800a38666e75..694b60455268 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_instantiate_client.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_instantiate_client.py @@ -17,12 +17,14 @@ """ import os -connection_string = os.environ['DATALAKE_STORAGE_CONNECTION_STRING'] + +connection_string = os.environ["DATALAKE_STORAGE_CONNECTION_STRING"] def instantiate_directory_client_from_conn_str(): # [START instantiate_directory_client_from_conn_str] from azure.storage.filedatalake import DataLakeDirectoryClient + DataLakeDirectoryClient.from_connection_string(connection_string, "myfilesystem", "mydirectory") # [END instantiate_directory_client_from_conn_str] @@ -30,10 +32,11 @@ def instantiate_directory_client_from_conn_str(): def instantiate_file_client_from_conn_str(): # [START instantiate_file_client_from_conn_str] from azure.storage.filedatalake import DataLakeFileClient + DataLakeFileClient.from_connection_string(connection_string, "myfilesystem", "mydirectory", "myfile") # [END instantiate_file_client_from_conn_str] -if __name__ == '__main__': +if __name__ == "__main__": instantiate_directory_client_from_conn_str() instantiate_file_client_from_conn_str() diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_instantiate_client_async.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_instantiate_client_async.py index 70c065b4e3f3..4f3eb6928510 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_instantiate_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_instantiate_client_async.py @@ -15,14 +15,17 @@ 1) DATALAKE_STORAGE_CONNECTION_STRING - the connection string to your storage account connection str could be obtained from portal.azure.com your storage account. """ + import asyncio import os -connection_string = os.environ['DATALAKE_STORAGE_CONNECTION_STRING'] + +connection_string = os.environ["DATALAKE_STORAGE_CONNECTION_STRING"] async def instantiate_directory_client_from_conn_str(): # [START instantiate_directory_client_from_conn_str] from azure.storage.filedatalake.aio import DataLakeDirectoryClient + DataLakeDirectoryClient.from_connection_string(connection_string, "myfilesystem", "mydirectory") # [END instantiate_directory_client_from_conn_str] @@ -30,6 +33,7 @@ async def instantiate_directory_client_from_conn_str(): async def instantiate_file_client_from_conn_str(): # [START instantiate_file_client_from_conn_str] from azure.storage.filedatalake.aio import DataLakeFileClient + DataLakeFileClient.from_connection_string(connection_string, "myfilesystem", "mydirectory", "myfile") # [END instantiate_file_client_from_conn_str] @@ -39,5 +43,5 @@ async def main(): await instantiate_file_client_from_conn_str() -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_query.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_query.py index 0597e18fd4fa..8dd1eed4112c 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_query.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_query.py @@ -14,48 +14,51 @@ Set the environment variables with your own values before running the sample. 1) DATALAKE_STORAGE_CONNECTION_STRING - the connection string to your storage account """ + import os import sys from azure.core.exceptions import ResourceExistsError from azure.storage.filedatalake import DataLakeServiceClient, DelimitedJsonDialect, DelimitedTextDialect -CSV_DATA = b'Service,Package,Version,RepoPath,MissingDocs\r\nApp Configuration,' \ - b'azure-data-appconfiguration,1,appconfiguration,FALSE\r\nEvent Hubs' \ - b'\r\nEvent Hubs - Azure Storage CheckpointStore,' \ - b'azure-messaging-eventhubs-checkpointstore-blob,1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity,' \ - b'1.1.0-beta.1,identity,FALSE\r\nKey Vault - Certificates,azure-security-keyvault-certificates,' \ - b'4.0.0,keyvault,FALSE\r\nKey Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault,' \ - b'FALSE\r\nKey Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n' \ - b'Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\nStorage - Blobs Batch,' \ - b'azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\nStorage - Blobs Cryptography,' \ - b'azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\nStorage - File Shares,' \ - b'azure-storage-file-share,12.2.0,storage,FALSE\r\nStorage - Queues,' \ - b'azure-storage-queue,12.3.0,storage,FALSE\r\nText Analytics,' \ - b'azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\nTracing,' \ - b'azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\nService,Package,Version,RepoPath,' \ - b'MissingDocs\r\nApp Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n' \ - b'Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n' \ - b'Event Hubs - Azure Storage CheckpointStore,azure-messaging-eventhubs-checkpointstore-blob,' \ - b'1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity,1.1.0-beta.1,identity,FALSE\r\n' \ - b'Key Vault - Certificates,azure-security-keyvault-certificates,4.0.0,keyvault,FALSE\r\n' \ - b'Key Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault,FALSE\r\n' \ - b'Key Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n' \ - b'Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\n' \ - b'Storage - Blobs Batch,azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\n' \ - b'Storage - Blobs Cryptography,azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\n' \ - b'Storage - File Shares,azure-storage-file-share,12.2.0,storage,FALSE\r\n' \ - b'Storage - Queues,azure-storage-queue,12.3.0,storage,FALSE\r\n' \ - b'Text Analytics,azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\n' \ - b'Tracing,azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\n' \ - b'Service,Package,Version,RepoPath,MissingDocs\r\n' \ - b'App Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n' \ - b'Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n' +CSV_DATA = ( + b"Service,Package,Version,RepoPath,MissingDocs\r\nApp Configuration," + b"azure-data-appconfiguration,1,appconfiguration,FALSE\r\nEvent Hubs" + b"\r\nEvent Hubs - Azure Storage CheckpointStore," + b"azure-messaging-eventhubs-checkpointstore-blob,1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity," + b"1.1.0-beta.1,identity,FALSE\r\nKey Vault - Certificates,azure-security-keyvault-certificates," + b"4.0.0,keyvault,FALSE\r\nKey Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault," + b"FALSE\r\nKey Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n" + b"Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\nStorage - Blobs Batch," + b"azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\nStorage - Blobs Cryptography," + b"azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\nStorage - File Shares," + b"azure-storage-file-share,12.2.0,storage,FALSE\r\nStorage - Queues," + b"azure-storage-queue,12.3.0,storage,FALSE\r\nText Analytics," + b"azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\nTracing," + b"azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\nService,Package,Version,RepoPath," + b"MissingDocs\r\nApp Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n" + b"Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n" + b"Event Hubs - Azure Storage CheckpointStore,azure-messaging-eventhubs-checkpointstore-blob," + b"1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity,1.1.0-beta.1,identity,FALSE\r\n" + b"Key Vault - Certificates,azure-security-keyvault-certificates,4.0.0,keyvault,FALSE\r\n" + b"Key Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault,FALSE\r\n" + b"Key Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n" + b"Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\n" + b"Storage - Blobs Batch,azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\n" + b"Storage - Blobs Cryptography,azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\n" + b"Storage - File Shares,azure-storage-file-share,12.2.0,storage,FALSE\r\n" + b"Storage - Queues,azure-storage-queue,12.3.0,storage,FALSE\r\n" + b"Text Analytics,azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\n" + b"Tracing,azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\n" + b"Service,Package,Version,RepoPath,MissingDocs\r\n" + b"App Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n" + b"Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n" +) def main(): try: - CONNECTION_STRING = os.environ['DATALAKE_STORAGE_CONNECTION_STRING'] + CONNECTION_STRING = os.environ["DATALAKE_STORAGE_CONNECTION_STRING"] except KeyError: print("DATALAKE_STORAGE_CONNECTION_STRING must be set.") @@ -70,6 +73,7 @@ def main(): pass # [START query] errors = [] + def on_error(error): errors.append(error) @@ -80,18 +84,11 @@ def on_error(error): # select the second column of the csv file query_expression = "SELECT _2 from DataLakeStorage" input_format = DelimitedTextDialect( - delimiter=',', - quotechar='"', - lineterminator='\n', - escapechar="", - has_header=False + delimiter=",", quotechar='"', lineterminator="\n", escapechar="", has_header=False ) - output_format = DelimitedJsonDialect(delimiter='\n') + output_format = DelimitedJsonDialect(delimiter="\n") reader = file_client.query_file( - query_expression, - on_error=on_error, - file_format=input_format, - output_format=output_format + query_expression, on_error=on_error, file_format=input_format, output_format=output_format ) content = reader.readall() # [END query] diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_service.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_service.py index 54afa6cd6514..f4e684707de8 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_service.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_service.py @@ -27,33 +27,37 @@ class DataLakeServiceSamples(object): - connection_string = os.environ['DATALAKE_STORAGE_CONNECTION_STRING'] - account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") + connection_string = os.environ["DATALAKE_STORAGE_CONNECTION_STRING"] + account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") - - #--Begin DataLake Service Samples----------------------------------------------------------------- + # --Begin DataLake Service Samples----------------------------------------------------------------- def data_lake_service_sample(self): # Instantiate a DataLakeServiceClient using a connection string # [START create_datalake_service_client] from azure.storage.filedatalake import DataLakeServiceClient + datalake_service_client = DataLakeServiceClient.from_connection_string(self.connection_string) # [END create_datalake_service_client] # Instantiate a DataLakeServiceClient Azure Identity credentials. # [START create_datalake_service_client_oauth] from azure.identity import DefaultAzureCredential + token_credential = DefaultAzureCredential() - datalake_service_client = DataLakeServiceClient("https://{}.dfs.core.windows.net".format(self.account_name), - credential=token_credential) + datalake_service_client = DataLakeServiceClient( + "https://{}.dfs.core.windows.net".format(self.account_name), credential=token_credential + ) # [END create_datalake_service_client_oauth] # get user delegation key # [START get_user_delegation_key] from datetime import datetime, timedelta - user_delegation_key = datalake_service_client.get_user_delegation_key(datetime.utcnow(), - datetime.utcnow() + timedelta(hours=1)) + + user_delegation_key = datalake_service_client.get_user_delegation_key( + datetime.utcnow(), datetime.utcnow() + timedelta(hours=1) + ) # [END get_user_delegation_key] # Create file systems @@ -72,19 +76,19 @@ def data_lake_service_sample(self): # Get Clients from DataLakeServiceClient file_system_client = datalake_service_client.get_file_system_client(file_system_client.file_system_name) # [START get_directory_client_from_service_client] - directory_client = datalake_service_client.get_directory_client(file_system_client.file_system_name, - "mydirectory") + directory_client = datalake_service_client.get_directory_client( + file_system_client.file_system_name, "mydirectory" + ) # [END get_directory_client_from_service_client] # [START get_file_client_from_service_client] file_client = datalake_service_client.get_file_client(file_system_client.file_system_name, "myfile") # [END get_file_client_from_service_client] # Create file and set properties - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} from azure.storage.filedatalake import ContentSettings - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") file_client.create_file(content_settings=content_settings) file_client.set_metadata(metadata=metadata) file_props = file_client.get_file_properties() @@ -102,6 +106,6 @@ def data_lake_service_sample(self): file_system_client.delete_file_system() -if __name__ == '__main__': +if __name__ == "__main__": sample = DataLakeServiceSamples() sample.data_lake_service_sample() diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_service_async.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_service_async.py index 978a2ae08e5e..55335dae6119 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_service_async.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_service_async.py @@ -28,11 +28,11 @@ from datetime import datetime, timedelta from azure.storage.filedatalake.aio import DataLakeServiceClient +connection_string = os.environ["DATALAKE_STORAGE_CONNECTION_STRING"] +account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") -connection_string = os.environ['DATALAKE_STORAGE_CONNECTION_STRING'] -account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") +# --Begin DataLake Service Samples----------------------------------------------------------------- -#--Begin DataLake Service Samples----------------------------------------------------------------- async def main(): @@ -45,16 +45,19 @@ async def main(): # Instantiate a DataLakeServiceClient Azure Identity credentials. # [START create_datalake_service_client_oauth] from azure.identity.aio import DefaultAzureCredential + token_credential = DefaultAzureCredential() - datalake_service_client = DataLakeServiceClient("https://{}.dfs.core.windows.net".format(account_name), - credential=token_credential) + datalake_service_client = DataLakeServiceClient( + "https://{}.dfs.core.windows.net".format(account_name), credential=token_credential + ) # [END create_datalake_service_client_oauth] async with datalake_service_client: # get user delegation key # [START get_user_delegation_key] - user_delegation_key = await datalake_service_client.get_user_delegation_key(datetime.utcnow(), - datetime.utcnow() + timedelta(hours=1)) + user_delegation_key = await datalake_service_client.get_user_delegation_key( + datetime.utcnow(), datetime.utcnow() + timedelta(hours=1) + ) # [END get_user_delegation_key] # Create file systems @@ -73,20 +76,19 @@ async def main(): # Get Clients from DataLakeServiceClient file_system_client = datalake_service_client.get_file_system_client(file_system_client.file_system_name) # [START get_directory_client_from_service_client] - directory_client = datalake_service_client.get_directory_client(file_system_client.file_system_name, - "mydirectory") + directory_client = datalake_service_client.get_directory_client( + file_system_client.file_system_name, "mydirectory" + ) # [END get_directory_client_from_service_client] # [START get_file_client_from_service_client] file_client = datalake_service_client.get_file_client(file_system_client.file_system_name, "myfile") # [END get_file_client_from_service_client] # Create file and set properties - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} from azure.storage.filedatalake import ContentSettings - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline' - ) + + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") await file_client.create_file(content_settings=content_settings) await file_client.set_metadata(metadata=metadata) file_props = await file_client.get_file_properties() @@ -106,5 +108,5 @@ async def main(): await token_credential.close() -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_upload_download.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_upload_download.py index 8d3d46ea3a75..bbefab305e58 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_upload_download.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_upload_download.py @@ -30,9 +30,11 @@ from azure.storage.filedatalake import ( DataLakeServiceClient, ) + current_dir = os.path.dirname(os.path.abspath(__file__)) SOURCE_FILE = os.path.join(current_dir, "SampleSource.txt") + def upload_download_sample(filesystem_client): # create a file before writing content to it file_name = "testfile" @@ -43,7 +45,7 @@ def upload_download_sample(filesystem_client): # [END create_file] # prepare the file content with 4KB of random data - file_content = get_random_bytes(4*1024) + file_content = get_random_bytes(4 * 1024) # append data to the file # the data remain uncommitted until flush is performed @@ -78,11 +80,11 @@ def upload_download_sample(filesystem_client): # Rename the file # [START rename_file] - new_client = file_client.rename_file(file_client.file_system_name + '/' + 'newname') + new_client = file_client.rename_file(file_client.file_system_name + "/" + "newname") # [END rename_file] # download the renamed file in to local file - with open(SOURCE_FILE, 'wb') as stream: + with open(SOURCE_FILE, "wb") as stream: download = new_client.download_file() download.readinto(stream) @@ -90,24 +92,24 @@ def upload_download_sample(filesystem_client): new_client.delete_file() # [END delete_file] + # help method to provide random bytes to serve as file content def get_random_bytes(size): rand = random.Random() result = bytearray(size) for i in range(size): - result[i] = int(rand.random()*255) # random() is consistent between python 2 and 3 + result[i] = int(rand.random() * 255) # random() is consistent between python 2 and 3 return bytes(result) def run(): - account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") - account_key = os.getenv('DATALAKE_STORAGE_ACCOUNT_KEY', "") + account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") + account_key = os.getenv("DATALAKE_STORAGE_ACCOUNT_KEY", "") # set up the service client with the credentials from the environment variables - service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format( - "https", - account_name - ), credential=account_key) + service_client = DataLakeServiceClient( + account_url="{}://{}.dfs.core.windows.net".format("https", account_name), credential=account_key + ) # generate a random name for testing purpose fs_name = "testfs{}download".format(random.randint(1, 1000)) @@ -124,5 +126,5 @@ def run(): filesystem_client.delete_file_system() -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_upload_download_async.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_upload_download_async.py index 016df1d6aff2..2866b4f4890c 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_upload_download_async.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_upload_download_async.py @@ -23,6 +23,7 @@ 1) DATALAKE_STORAGE_ACCOUNT_NAME - the storage account name 2) DATALAKE_STORAGE_ACCOUNT_KEY - the storage account key """ + import asyncio import os import random @@ -30,9 +31,11 @@ from azure.storage.filedatalake.aio import ( DataLakeServiceClient, ) + current_dir = os.path.dirname(os.path.abspath(__file__)) SOURCE_FILE = os.path.join(current_dir, "SampleSource.txt") + async def upload_download_sample(filesystem_client): # create a file before writing content to it file_name = "testfile" @@ -43,7 +46,7 @@ async def upload_download_sample(filesystem_client): # [END create_file] # prepare the file content with 4KB of random data - file_content = get_random_bytes(4*1024) + file_content = get_random_bytes(4 * 1024) # append data to the file # the data remain uncommitted until flush is performed @@ -78,11 +81,11 @@ async def upload_download_sample(filesystem_client): # Rename the file # [START rename_file] - new_client = await file_client.rename_file(file_client.file_system_name + '/' + 'newname') + new_client = await file_client.rename_file(file_client.file_system_name + "/" + "newname") # [END rename_file] # download the renamed file in to local file - with open(SOURCE_FILE, 'wb') as stream: + with open(SOURCE_FILE, "wb") as stream: download = await new_client.download_file() await download.readinto(stream) @@ -90,24 +93,24 @@ async def upload_download_sample(filesystem_client): await new_client.delete_file() # [END delete_file] + # help method to provide random bytes to serve as file content def get_random_bytes(size): rand = random.Random() result = bytearray(size) for i in range(size): - result[i] = int(rand.random()*255) # random() is consistent between python 2 and 3 + result[i] = int(rand.random() * 255) # random() is consistent between python 2 and 3 return bytes(result) async def main(): - account_name = os.getenv('DATALAKE_STORAGE_ACCOUNT_NAME', "") - account_key = os.getenv('DATALAKE_STORAGE_ACCOUNT_KEY', "") + account_name = os.getenv("DATALAKE_STORAGE_ACCOUNT_NAME", "") + account_key = os.getenv("DATALAKE_STORAGE_ACCOUNT_KEY", "") # set up the service client with the credentials from the environment variables - service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format( - "https", - account_name - ), credential=account_key) + service_client = DataLakeServiceClient( + account_url="{}://{}.dfs.core.windows.net".format("https", account_name), credential=account_key + ) async with service_client: # generate a random name for testing purpose @@ -125,5 +128,5 @@ async def main(): await filesystem_client.delete_file_system() -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/sdk/storage/azure-storage-file-datalake/setup.py b/sdk/storage/azure-storage-file-datalake/setup.py index 69a3c66a2b59..8e643a43e72f 100644 --- a/sdk/storage/azure-storage-file-datalake/setup.py +++ b/sdk/storage/azure-storage-file-datalake/setup.py @@ -12,14 +12,13 @@ from setuptools import setup, find_packages - # Change the PACKAGE_NAME only to change folder and different name PACKAGE_NAME = "azure-storage-file-datalake" NAMESPACE_NAME = "azure.storage.filedatalake" PACKAGE_PPRINT_NAME = "Azure File DataLake Storage" # a-b-c => a/b/c -package_folder_path = NAMESPACE_NAME.replace('.', '/') +package_folder_path = NAMESPACE_NAME.replace(".", "/") # azure-storage v0.36.0 and prior are not compatible with this package try: @@ -28,8 +27,8 @@ try: ver = azure.storage.__version__ raise Exception( - f'This package is incompatible with azure-storage=={ver}. ' + - ' Uninstall it with "pip uninstall azure-storage".' + f"This package is incompatible with azure-storage=={ver}. " + + ' Uninstall it with "pip uninstall azure-storage".' ) except AttributeError: pass @@ -37,51 +36,52 @@ pass # Version extraction inspired from 'requests' -with open(os.path.join(package_folder_path, '_version.py'), 'r') as fd: - version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', - fd.read(), re.MULTILINE).group(1) +with open(os.path.join(package_folder_path, "_version.py"), "r") as fd: + version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) if not version: - raise RuntimeError('Cannot find version information') + raise RuntimeError("Cannot find version information") setup( name=PACKAGE_NAME, version=version, include_package_data=True, - description=f'Microsoft {PACKAGE_PPRINT_NAME} Client Library for Python', - long_description=open('README.md', 'r').read(), - long_description_content_type='text/markdown', - license='MIT License', - author='Microsoft Corporation', - author_email='ascl@microsoft.com', - url='https://github.com/Azure/azure-sdk-for-python', + description=f"Microsoft {PACKAGE_PPRINT_NAME} Client Library for Python", + long_description=open("README.md", "r").read(), + long_description_content_type="text/markdown", + license="MIT License", + author="Microsoft Corporation", + author_email="ascl@microsoft.com", + url="https://github.com/Azure/azure-sdk-for-python", keywords="azure, azure sdk", classifiers=[ - 'Development Status :: 4 - Beta', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "License :: OSI Approved :: MIT License", ], zip_safe=False, - packages=find_packages(exclude=[ - # Exclude packages that will be covered by PEP420 or nspkg - 'azure', - 'azure.storage', - 'tests', - ]), + packages=find_packages( + exclude=[ + # Exclude packages that will be covered by PEP420 or nspkg + "azure", + "azure.storage", + "tests", + ] + ), python_requires=">=3.9", install_requires=[ "azure-core>=1.37.0", "azure-storage-blob>=12.31.0b1", "typing-extensions>=4.6.0", - "isodate>=0.6.1" + "isodate>=0.6.1", ], extras_require={ "aio": [ diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py index a4097fcf59b0..873590701bff 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py +++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py @@ -21,7 +21,8 @@ def __init__(self, arguments): if not _ServiceTest.service_client or self.args.no_client_share: _ServiceTest.service_client = SyncDataLakeServiceClient.from_connection_string(conn_str=connection_string) _ServiceTest.async_service_client = AsyncDataLakeServiceClient.from_connection_string( - conn_str=connection_string) + conn_str=connection_string + ) self.service_client = _ServiceTest.service_client self.async_service_client = _ServiceTest.async_service_client @@ -33,26 +34,21 @@ async def close(self): def add_arguments(parser): super(_ServiceTest, _ServiceTest).add_arguments(parser) parser.add_argument( - '-c', - '--max-concurrency', - nargs='?', + "-c", + "--max-concurrency", + nargs="?", type=int, - help='Maximum number of concurrent threads used for data transfer. Defaults to 1', - default=1 + help="Maximum number of concurrent threads used for data transfer. Defaults to 1", + default=1, ) parser.add_argument( - '-s', - '--size', - nargs='?', - type=int, - help='Size of data to transfer. Default is 10240.', - default=10240 + "-s", "--size", nargs="?", type=int, help="Size of data to transfer. Default is 10240.", default=10240 ) parser.add_argument( - '--no-client-share', - action='store_true', - help='Create one ServiceClient per test instance. Default is to share a single ServiceClient.', - default=False + "--no-client-share", + action="store_true", + help="Create one ServiceClient per test instance. Default is to share a single ServiceClient.", + default=False, ) diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py index 45414cd020b3..5de31941d992 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py +++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py @@ -24,14 +24,8 @@ async def setup(self): def run_sync(self): self.upload_stream.reset() - self.file_client.append_data( - self.upload_stream, - length=self.args.size, - offset=0) + self.file_client.append_data(self.upload_stream, length=self.args.size, offset=0) async def run_async(self): self.upload_stream_async.reset() - await self.async_file_client.append_data( - self.upload_stream_async, - length=self.args.size, - offset=0) + await self.async_file_client.append_data(self.upload_stream_async, length=self.args.size, offset=0) diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py index f2383deaa7b0..8212fd54d7ef 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py +++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py @@ -17,15 +17,11 @@ def __init__(self, arguments): def run_sync(self): self.upload_stream.reset() self.file_client.upload_data( - self.upload_stream, - length=self.args.size, - overwrite=True, - max_concurrency=self.args.max_concurrency) + self.upload_stream, length=self.args.size, overwrite=True, max_concurrency=self.args.max_concurrency + ) async def run_async(self): self.upload_stream_async.reset() await self.async_file_client.upload_data( - self.upload_stream_async, - length=self.args.size, - overwrite=True, - max_concurrency=self.args.max_concurrency) + self.upload_stream_async, length=self.args.size, overwrite=True, max_concurrency=self.args.max_concurrency + ) diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py index 9e552bb7db8e..39d588914af9 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py +++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py @@ -26,15 +26,9 @@ async def global_cleanup(self): await super().global_cleanup() def run_sync(self): - with open(UploadFromFileTest.temp_file, 'rb') as fp: - self.file_client.upload_data( - fp, - overwrite=True, - max_concurrency=self.args.max_concurrency) + with open(UploadFromFileTest.temp_file, "rb") as fp: + self.file_client.upload_data(fp, overwrite=True, max_concurrency=self.args.max_concurrency) async def run_async(self): - with open(UploadFromFileTest.temp_file, 'rb') as fp: - await self.async_file_client.upload_data( - fp, - overwrite=True, - max_concurrency=self.args.max_concurrency) + with open(UploadFromFileTest.temp_file, "rb") as fp: + await self.async_file_client.upload_data(fp, overwrite=True, max_concurrency=self.args.max_concurrency) diff --git a/sdk/storage/azure-storage-file-datalake/tests/settings/settings_fake.py b/sdk/storage/azure-storage-file-datalake/tests/settings/settings_fake.py index d3f83754d0db..a270d6de817f 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/settings/settings_fake.py +++ b/sdk/storage/azure-storage-file-datalake/tests/settings/settings_fake.py @@ -10,7 +10,7 @@ STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_NAME = "fakename" STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_KEY = "fakekey" -ACCOUNT_URL_SUFFIX = 'core.windows.net' +ACCOUNT_URL_SUFFIX = "core.windows.net" RUN_IN_LIVE = "False" SKIP_LIVE_RECORDING = "True" diff --git a/sdk/storage/azure-storage-file-datalake/tests/settings/testcase.py b/sdk/storage/azure-storage-file-datalake/tests/settings/testcase.py index b8cbb7efde74..27c3a57aa467 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/settings/testcase.py +++ b/sdk/storage/azure-storage-file-datalake/tests/settings/testcase.py @@ -11,7 +11,6 @@ from devtools_testutils import EnvironmentVariableLoader, EnvironmentVariableOptions from devtools_testutils.fake_credentials import STORAGE_ACCOUNT_FAKE_KEY - try: # Running locally - use configuration in settings_real.py from .settings_real import * @@ -20,26 +19,30 @@ from .settings_fake import * -LOGGING_FORMAT = '%(asctime)s %(name)-20s %(levelname)-5s %(message)s' -LOGGING_FORMAT = '%(asctime)s %(name)-20s %(levelname)-5s %(message)s' -os.environ['DATALAKE_STORAGE_ACCOUNT_NAME'] = (os.environ.get('DATALAKE_STORAGE_ACCOUNT_NAME', None) or - DATALAKE_STORAGE_ACCOUNT_NAME) -os.environ['DATALAKE_STORAGE_ACCOUNT_KEY'] = (os.environ.get('DATALAKE_STORAGE_ACCOUNT_KEY', None) or - DATALAKE_STORAGE_ACCOUNT_KEY) +LOGGING_FORMAT = "%(asctime)s %(name)-20s %(levelname)-5s %(message)s" +LOGGING_FORMAT = "%(asctime)s %(name)-20s %(levelname)-5s %(message)s" +os.environ["DATALAKE_STORAGE_ACCOUNT_NAME"] = ( + os.environ.get("DATALAKE_STORAGE_ACCOUNT_NAME", None) or DATALAKE_STORAGE_ACCOUNT_NAME +) +os.environ["DATALAKE_STORAGE_ACCOUNT_KEY"] = ( + os.environ.get("DATALAKE_STORAGE_ACCOUNT_KEY", None) or DATALAKE_STORAGE_ACCOUNT_KEY +) -os.environ['STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_NAME'] = os.environ.get( - 'STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_NAME', None) or STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_NAME -os.environ['STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_KEY'] = os.environ.get( - 'STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_KEY', None) or STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_KEY +os.environ["STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_NAME"] = ( + os.environ.get("STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_NAME", None) or STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_NAME +) +os.environ["STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_KEY"] = ( + os.environ.get("STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_KEY", None) or STORAGE_DATA_LAKE_SOFT_DELETE_ACCOUNT_KEY +) -os.environ['AZURE_TEST_RUN_LIVE'] = os.environ.get('AZURE_TEST_RUN_LIVE', None) or RUN_IN_LIVE -os.environ['AZURE_SKIP_LIVE_RECORDING'] = os.environ.get( - 'AZURE_SKIP_LIVE_RECORDING', None) or SKIP_LIVE_RECORDING -os.environ['PROTOCOL'] = PROTOCOL -os.environ['ACCOUNT_URL_SUFFIX'] = ACCOUNT_URL_SUFFIX +os.environ["AZURE_TEST_RUN_LIVE"] = os.environ.get("AZURE_TEST_RUN_LIVE", None) or RUN_IN_LIVE +os.environ["AZURE_SKIP_LIVE_RECORDING"] = os.environ.get("AZURE_SKIP_LIVE_RECORDING", None) or SKIP_LIVE_RECORDING +os.environ["PROTOCOL"] = PROTOCOL +os.environ["ACCOUNT_URL_SUFFIX"] = ACCOUNT_URL_SUFFIX DataLakePreparer = functools.partial( - EnvironmentVariableLoader, "storage", + EnvironmentVariableLoader, + "storage", datalake_storage_account_name="storagename", datalake_storage_account_key=STORAGE_ACCOUNT_FAKE_KEY, storage_data_lake_soft_delete_account_name="storagesoftdelname", diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_cpk.py b/sdk/storage/azure-storage-file-datalake/tests/test_cpk.py index 045c4e7ebe23..51c76dea15cd 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_cpk.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_cpk.py @@ -12,21 +12,20 @@ from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError from azure.storage.filedatalake import CustomerProvidedEncryptionKey, DataLakeServiceClient - # ------------------------------------------------------------------------------ -TEST_DIRECTORY_PREFIX = 'directory' -TEST_FILE_PREFIX = 'file' +TEST_DIRECTORY_PREFIX = "directory" +TEST_FILE_PREFIX = "file" TEST_ENCRYPTION_KEY = CustomerProvidedEncryptionKey( - key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", - key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=") + key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=" +) # ------------------------------------------------------------------------------ class TestDatalakeCpk(StorageRecordedTestCase): def _setup(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=account_key.secret) - self.file_system_name = self.get_resource_name('utfilesystem') + self.file_system_name = self.get_resource_name("utfilesystem") if self.is_live: file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -65,12 +64,13 @@ def _create_file(self, directory_name=None, file_name=None, cpk=None): self._create_directory(directory_name, cpk) if not file_name: file_name = self._get_file_reference() - file_client = self.dsc.get_file_client(self.file_system_name, directory_name + '/' + file_name) + file_client = self.dsc.get_file_client(self.file_system_name, directory_name + "/" + file_name) try: file_client.create_file(cpk=cpk) except ResourceExistsError: pass return file_client + # --------------------------------------------------------------------------- @DataLakePreparer() @@ -83,12 +83,12 @@ def test_create_directory_cpk(self, **kwargs): self._setup(datalake_storage_account_name, datalake_storage_account_key) # Act - directory_client = self.dsc.get_directory_client(self.file_system_name, 'cpkdirectory') + directory_client = self.dsc.get_directory_client(self.file_system_name, "cpkdirectory") response = directory_client.create_directory(cpk=TEST_ENCRYPTION_KEY) # Assert assert response is not None - assert response['request_server_encrypted'] + assert response["request_server_encrypted"] # assert TEST_ENCRYPTION_KEY.key_hash == response['encryption_key_sha256'] @DataLakePreparer() @@ -102,7 +102,7 @@ def test_create_sub_directory_cpk(self, **kwargs): directory_client = self._create_directory(cpk=TEST_ENCRYPTION_KEY) # Act - sub_directory_client = directory_client.create_sub_directory('cpksubdirectory', cpk=TEST_ENCRYPTION_KEY) + sub_directory_client = directory_client.create_sub_directory("cpksubdirectory", cpk=TEST_ENCRYPTION_KEY) props = sub_directory_client.get_directory_properties(cpk=TEST_ENCRYPTION_KEY) # Assert @@ -117,14 +117,14 @@ def test_create_file_cpk(self, **kwargs): # Arrange self._setup(datalake_storage_account_name, datalake_storage_account_key) directory_client = self._create_directory(cpk=TEST_ENCRYPTION_KEY) - file_client = directory_client.get_file_client('cpkfile') + file_client = directory_client.get_file_client("cpkfile") # Act response = file_client.create_file(cpk=TEST_ENCRYPTION_KEY) # Assert assert response is not None - assert response['request_server_encrypted'] + assert response["request_server_encrypted"] # assert TEST_ENCRYPTION_KEY.key_hash == response['encryption_key_sha256'] @DataLakePreparer() @@ -210,7 +210,7 @@ def test_file_upload_data_file_cpk(self, **kwargs): # Assert assert response is not None - assert response['request_server_encrypted'] + assert response["request_server_encrypted"] # assert TEST_ENCRYPTION_KEY.key_hash == response['encryption_key_sha256'] @DataLakePreparer() @@ -231,7 +231,7 @@ def test_file_append_flush_data_cpk(self, **kwargs): # Assert assert response is not None - assert response['request_server_encrypted'] + assert response["request_server_encrypted"] # assert TEST_ENCRYPTION_KEY.key_hash == response['encryption_key_sha256'] @DataLakePreparer() @@ -266,7 +266,7 @@ def test_set_metadata_cpk(self, **kwargs): self._setup(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() file_client = self._create_file(directory_name=directory_name, cpk=TEST_ENCRYPTION_KEY) - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} # Act file_client.set_metadata(metadata, cpk=TEST_ENCRYPTION_KEY) @@ -294,10 +294,7 @@ def on_error(error): errors.append(error) # Act - reader = file_client.query_file( - "SELECT * from DataLakeStorage", - on_error=on_error, - cpk=TEST_ENCRYPTION_KEY) + reader = file_client.query_file("SELECT * from DataLakeStorage", on_error=on_error, cpk=TEST_ENCRYPTION_KEY) reader.readall() # Assert diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_cpk_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_cpk_async.py index ebabf6451105..0039cba978ea 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_cpk_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_cpk_async.py @@ -12,21 +12,20 @@ from azure.storage.filedatalake import CustomerProvidedEncryptionKey from azure.storage.filedatalake.aio import DataLakeServiceClient - # ------------------------------------------------------------------------------ -TEST_DIRECTORY_PREFIX = 'directory' -TEST_FILE_PREFIX = 'file' +TEST_DIRECTORY_PREFIX = "directory" +TEST_FILE_PREFIX = "file" TEST_ENCRYPTION_KEY = CustomerProvidedEncryptionKey( - key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", - key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=") + key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=" +) # ------------------------------------------------------------------------------ class TestDatalakeCpkAsync(AsyncStorageRecordedTestCase): async def _setup(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=account_key.secret) - self.file_system_name = self.get_resource_name('utfilesystem') + self.file_system_name = self.get_resource_name("utfilesystem") if self.is_live: file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -66,12 +65,13 @@ async def _create_file(self, directory_name=None, file_name=None, cpk=None): await self._create_directory(directory_name, cpk) if not file_name: file_name = self._get_file_reference() - file_client = self.dsc.get_file_client(self.file_system_name, directory_name + '/' + file_name) + file_client = self.dsc.get_file_client(self.file_system_name, directory_name + "/" + file_name) try: await file_client.create_file(cpk=cpk) except ResourceExistsError: pass return file_client + # --------------------------------------------------------------------------- @DataLakePreparer() @@ -84,12 +84,12 @@ async def test_create_directory_cpk(self, **kwargs): await self._setup(datalake_storage_account_name, datalake_storage_account_key) # Act - directory_client = self.dsc.get_directory_client(self.file_system_name, 'cpkdirectory') + directory_client = self.dsc.get_directory_client(self.file_system_name, "cpkdirectory") response = await directory_client.create_directory(cpk=TEST_ENCRYPTION_KEY) # Assert assert response is not None - assert response['request_server_encrypted'] + assert response["request_server_encrypted"] # assert TEST_ENCRYPTION_KEY.key_hash == response['encryption_key_sha256'] @DataLakePreparer() @@ -103,7 +103,7 @@ async def test_create_sub_directory_cpk(self, **kwargs): directory_client = await self._create_directory(cpk=TEST_ENCRYPTION_KEY) # Act - sub_directory_client = await directory_client.create_sub_directory('cpksubdirectory', cpk=TEST_ENCRYPTION_KEY) + sub_directory_client = await directory_client.create_sub_directory("cpksubdirectory", cpk=TEST_ENCRYPTION_KEY) props = await sub_directory_client.get_directory_properties(cpk=TEST_ENCRYPTION_KEY) # Assert @@ -118,14 +118,14 @@ async def test_create_file_cpk(self, **kwargs): # Arrange await self._setup(datalake_storage_account_name, datalake_storage_account_key) directory_client = await self._create_directory(cpk=TEST_ENCRYPTION_KEY) - file_client = directory_client.get_file_client('cpkfile') + file_client = directory_client.get_file_client("cpkfile") # Act response = await file_client.create_file(cpk=TEST_ENCRYPTION_KEY) # Assert assert response is not None - assert response['request_server_encrypted'] + assert response["request_server_encrypted"] # assert TEST_ENCRYPTION_KEY.key_hash == response['encryption_key_sha256'] @DataLakePreparer() @@ -211,7 +211,7 @@ async def test_file_upload_data_file_cpk(self, **kwargs): # Assert assert response is not None - assert response['request_server_encrypted'] + assert response["request_server_encrypted"] # assert TEST_ENCRYPTION_KEY.key_hash == response['encryption_key_sha256'] @DataLakePreparer() @@ -232,7 +232,7 @@ async def test_file_append_flush_data_cpk(self, **kwargs): # Assert assert response is not None - assert response['request_server_encrypted'] + assert response["request_server_encrypted"] # assert TEST_ENCRYPTION_KEY.key_hash == response['encryption_key_sha256'] @DataLakePreparer() @@ -268,7 +268,7 @@ async def test_set_metadata_cpk(self, **kwargs): await self._setup(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() file_client = await self._create_file(directory_name=directory_name, cpk=TEST_ENCRYPTION_KEY) - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} # Act await file_client.set_metadata(metadata, cpk=TEST_ENCRYPTION_KEY) diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client.py b/sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client.py index 653312d37131..513b59d3653a 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client.py @@ -28,25 +28,24 @@ ) from azure.storage.filedatalake._shared.parser import DEVSTORE_ACCOUNT_KEY, DEVSTORE_ACCOUNT_NAME - # ------------------------------------------------------------------------------ -TEST_FILE_SYSTEM_PREFIX = 'filesystem' +TEST_FILE_SYSTEM_PREFIX = "filesystem" # ------------------------------------------------------------------------------ class TestDatalakeService(StorageRecordedTestCase): # --Helpers----------------------------------------------------------------- def _setup(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, account_key.secret) self.config = self.dsc._config def _assert_properties_default(self, prop): assert prop is not None - self._assert_logging_equal(prop['analytics_logging'], AnalyticsLogging()) - self._assert_metrics_equal(prop['hour_metrics'], Metrics()) - self._assert_metrics_equal(prop['minute_metrics'], Metrics()) - self._assert_cors_equal(prop['cors'], []) + self._assert_logging_equal(prop["analytics_logging"], AnalyticsLogging()) + self._assert_metrics_equal(prop["hour_metrics"], Metrics()) + self._assert_metrics_equal(prop["minute_metrics"], Metrics()) + self._assert_cors_equal(prop["cors"], []) def _assert_logging_equal(self, log1, log2): if log1 is None or log2 is None: @@ -134,14 +133,14 @@ def test_datalake_service_properties(self, **kwargs): hour_metrics=Metrics(), minute_metrics=Metrics(), cors=[], - target_version='2014-02-14' + target_version="2014-02-14", ) # Assert assert resp is None props = self.dsc.get_service_properties() self._assert_properties_default(props) - assert '2014-02-14' == props['target_version'] + assert "2014-02-14" == props["target_version"] @DataLakePreparer() @recorded_by_proxy @@ -161,11 +160,11 @@ def test_set_default_service_version(self, **kwargs): self._setup(datalake_storage_account_name, datalake_storage_account_key) # Act - self.dsc.set_service_properties(target_version='2014-02-14') + self.dsc.set_service_properties(target_version="2014-02-14") # Assert received_props = self.dsc.get_service_properties() - assert received_props['target_version'] == '2014-02-14' + assert received_props["target_version"] == "2014-02-14" @DataLakePreparer() @recorded_by_proxy @@ -181,7 +180,7 @@ def test_set_delete_retention_policy(self, **kwargs): # Assert received_props = self.dsc.get_service_properties() - self._assert_delete_retention_policy_equal(received_props['delete_retention_policy'], delete_retention_policy) + self._assert_delete_retention_policy_equal(received_props["delete_retention_policy"], delete_retention_policy) @DataLakePreparer() @recorded_by_proxy @@ -195,7 +194,7 @@ def test_set_delete_retention_policy_edge_cases(self, **kwargs): # Assert received_props = self.dsc.get_service_properties() - self._assert_delete_retention_policy_equal(received_props['delete_retention_policy'], delete_retention_policy) + self._assert_delete_retention_policy_equal(received_props["delete_retention_policy"], delete_retention_policy) # Should work with maximum settings delete_retention_policy = RetentionPolicy(enabled=True, days=365) @@ -203,7 +202,7 @@ def test_set_delete_retention_policy_edge_cases(self, **kwargs): # Assert received_props = self.dsc.get_service_properties() - self._assert_delete_retention_policy_equal(received_props['delete_retention_policy'], delete_retention_policy) + self._assert_delete_retention_policy_equal(received_props["delete_retention_policy"], delete_retention_policy) # Should not work with 0 days delete_retention_policy = RetentionPolicy(enabled=True, days=0) @@ -214,7 +213,8 @@ def test_set_delete_retention_policy_edge_cases(self, **kwargs): # Assert received_props = self.dsc.get_service_properties() self._assert_delete_retention_policy_not_equal( - received_props['delete_retention_policy'], delete_retention_policy) + received_props["delete_retention_policy"], delete_retention_policy + ) # Should not work with 366 days delete_retention_policy = RetentionPolicy(enabled=True, days=366) @@ -225,7 +225,8 @@ def test_set_delete_retention_policy_edge_cases(self, **kwargs): # Assert received_props = self.dsc.get_service_properties() self._assert_delete_retention_policy_not_equal( - received_props['delete_retention_policy'], delete_retention_policy) + received_props["delete_retention_policy"], delete_retention_policy + ) @DataLakePreparer() @recorded_by_proxy @@ -235,16 +236,15 @@ def test_set_static_website_properties(self, **kwargs): self._setup(datalake_storage_account_name, datalake_storage_account_key) static_website = StaticWebsite( - enabled=True, - index_document="index.html", - error_document404_path="errors/error/404error.html") + enabled=True, index_document="index.html", error_document404_path="errors/error/404error.html" + ) # Act self.dsc.set_service_properties(static_website=static_website) # Assert received_props = self.dsc.get_service_properties() - self._assert_static_website_equal(received_props['static_website'], static_website) + self._assert_static_website_equal(received_props["static_website"], static_website) @DataLakePreparer() @recorded_by_proxy @@ -253,15 +253,16 @@ def test_disabled_static_website_properties(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - static_website = StaticWebsite(enabled=False, index_document="index.html", - error_document404_path="errors/error/404error.html") + static_website = StaticWebsite( + enabled=False, index_document="index.html", error_document404_path="errors/error/404error.html" + ) # Act self.dsc.set_service_properties(static_website=static_website) # Assert received_props = self.dsc.get_service_properties() - self._assert_static_website_equal(received_props['static_website'], StaticWebsite(enabled=False)) + self._assert_static_website_equal(received_props["static_website"], StaticWebsite(enabled=False)) @DataLakePreparer() @recorded_by_proxy @@ -270,10 +271,10 @@ def test_set_static_website_props_dont_impact_other_props(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - cors_rule1 = CorsRule(['www.xyz.com'], ['GET']) + cors_rule1 = CorsRule(["www.xyz.com"], ["GET"]) - allowed_origins = ['www.xyz.com', "www.ab.com", "www.bc.com"] - allowed_methods = ['GET', 'PUT'] + allowed_origins = ["www.xyz.com", "www.ab.com", "www.bc.com"] + allowed_methods = ["GET", "PUT"] max_age_in_seconds = 500 exposed_headers = ["x-ms-meta-data*", "x-ms-meta-source*", "x-ms-meta-abc", "x-ms-meta-bcd"] allowed_headers = ["x-ms-meta-data*", "x-ms-meta-target*", "x-ms-meta-xyz", "x-ms-meta-foo"] @@ -282,7 +283,8 @@ def test_set_static_website_props_dont_impact_other_props(self, **kwargs): allowed_methods, max_age_in_seconds=max_age_in_seconds, exposed_headers=exposed_headers, - allowed_headers=allowed_headers) + allowed_headers=allowed_headers, + ) cors = [cors_rule1, cors_rule2] @@ -291,19 +293,20 @@ def test_set_static_website_props_dont_impact_other_props(self, **kwargs): # Assert cors is updated received_props = self.dsc.get_service_properties() - self._assert_cors_equal(received_props['cors'], cors) + self._assert_cors_equal(received_props["cors"], cors) # Arrange to set static website properties - static_website = StaticWebsite(enabled=True, index_document="index.html", - error_document404_path="errors/error/404error.html") + static_website = StaticWebsite( + enabled=True, index_document="index.html", error_document404_path="errors/error/404error.html" + ) # Act to set static website self.dsc.set_service_properties(static_website=static_website) # Assert static website was updated was cors was unchanged received_props = self.dsc.get_service_properties() - self._assert_static_website_equal(received_props['static_website'], static_website) - self._assert_cors_equal(received_props['cors'], cors) + self._assert_static_website_equal(received_props["static_website"], static_website) + self._assert_cors_equal(received_props["cors"], cors) @DataLakePreparer() @recorded_by_proxy @@ -312,15 +315,16 @@ def test_set_logging(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - logging = AnalyticsLogging(read=True, write=True, delete=True, - retention_policy=RetentionPolicy(enabled=True, days=5)) + logging = AnalyticsLogging( + read=True, write=True, delete=True, retention_policy=RetentionPolicy(enabled=True, days=5) + ) # Act self.dsc.set_service_properties(analytics_logging=logging) # Assert received_props = self.dsc.get_service_properties() - self._assert_logging_equal(received_props['analytics_logging'], logging) + self._assert_logging_equal(received_props["analytics_logging"], logging) @DataLakePreparer() @recorded_by_proxy @@ -329,15 +333,14 @@ def test_set_hour_metrics(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - hour_metrics = Metrics( - include_apis=False, enabled=True, retention_policy=RetentionPolicy(enabled=True, days=5)) + hour_metrics = Metrics(include_apis=False, enabled=True, retention_policy=RetentionPolicy(enabled=True, days=5)) # Act self.dsc.set_service_properties(hour_metrics=hour_metrics) # Assert received_props = self.dsc.get_service_properties() - self._assert_metrics_equal(received_props['hour_metrics'], hour_metrics) + self._assert_metrics_equal(received_props["hour_metrics"], hour_metrics) @DataLakePreparer() @recorded_by_proxy @@ -346,15 +349,16 @@ def test_set_minute_metrics(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - minute_metrics = Metrics(enabled=True, include_apis=True, - retention_policy=RetentionPolicy(enabled=True, days=5)) + minute_metrics = Metrics( + enabled=True, include_apis=True, retention_policy=RetentionPolicy(enabled=True, days=5) + ) # Act self.dsc.set_service_properties(minute_metrics=minute_metrics) # Assert received_props = self.dsc.get_service_properties() - self._assert_metrics_equal(received_props['minute_metrics'], minute_metrics) + self._assert_metrics_equal(received_props["minute_metrics"], minute_metrics) @DataLakePreparer() @recorded_by_proxy @@ -363,10 +367,10 @@ def test_set_cors(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - cors_rule1 = CorsRule(['www.xyz.com'], ['GET']) + cors_rule1 = CorsRule(["www.xyz.com"], ["GET"]) - allowed_origins = ['www.xyz.com', "www.ab.com", "www.bc.com"] - allowed_methods = ['GET', 'PUT'] + allowed_origins = ["www.xyz.com", "www.ab.com", "www.bc.com"] + allowed_methods = ["GET", "PUT"] max_age_in_seconds = 500 exposed_headers = ["x-ms-meta-data*", "x-ms-meta-source*", "x-ms-meta-abc", "x-ms-meta-bcd"] allowed_headers = ["x-ms-meta-data*", "x-ms-meta-target*", "x-ms-meta-xyz", "x-ms-meta-foo"] @@ -375,7 +379,8 @@ def test_set_cors(self, **kwargs): allowed_methods, max_age_in_seconds=max_age_in_seconds, exposed_headers=exposed_headers, - allowed_headers=allowed_headers) + allowed_headers=allowed_headers, + ) cors = [cors_rule1, cors_rule2] @@ -384,29 +389,29 @@ def test_set_cors(self, **kwargs): # Assert received_props = self.dsc.get_service_properties() - self._assert_cors_equal(received_props['cors'], cors) + self._assert_cors_equal(received_props["cors"], cors) @DataLakePreparer() def test_connectionstring_without_secondary(self): test_conn_str = "DefaultEndpointsProtocol=https;AccountName=foo;AccountKey=bar" client = DataLakeServiceClient.from_connection_string(test_conn_str) - assert client.url == 'https://foo.dfs.core.windows.net/' - assert client.primary_hostname == 'foo.dfs.core.windows.net' + assert client.url == "https://foo.dfs.core.windows.net/" + assert client.primary_hostname == "foo.dfs.core.windows.net" assert not client.secondary_hostname client = FileSystemClient.from_connection_string(test_conn_str, "fsname") - assert client.url == 'https://foo.dfs.core.windows.net/fsname' - assert client.primary_hostname == 'foo.dfs.core.windows.net' + assert client.url == "https://foo.dfs.core.windows.net/fsname" + assert client.primary_hostname == "foo.dfs.core.windows.net" assert not client.secondary_hostname client = DataLakeFileClient.from_connection_string(test_conn_str, "fsname", "fpath") - assert client.url == 'https://foo.dfs.core.windows.net/fsname/fpath' - assert client.primary_hostname == 'foo.dfs.core.windows.net' + assert client.url == "https://foo.dfs.core.windows.net/fsname/fpath" + assert client.primary_hostname == "foo.dfs.core.windows.net" assert not client.secondary_hostname client = DataLakeDirectoryClient.from_connection_string(test_conn_str, "fsname", "dname") - assert client.url == 'https://foo.dfs.core.windows.net/fsname/dname' - assert client.primary_hostname == 'foo.dfs.core.windows.net' + assert client.url == "https://foo.dfs.core.windows.net/fsname/dname" + assert client.primary_hostname == "foo.dfs.core.windows.net" assert not client.secondary_hostname @DataLakePreparer() @@ -447,9 +452,9 @@ def test_datalake_clients_properly_close(self): account_key = NamedTuple("StorageAccountKey", [("secret", str)])("adlskey") self._setup(account_name, account_key) - file_system_client = self.dsc.get_file_system_client(file_system='testfs') - dir_client = self.dsc.get_directory_client(file_system='testfs', directory='testdir') - file_client = dir_client.get_file_client(file='testfile') + file_system_client = self.dsc.get_file_system_client(file_system="testfs") + dir_client = self.dsc.get_directory_client(file_system="testfs", directory="testdir") + file_client = dir_client.get_file_client(file="testfile") # Mocks self.dsc._blob_service_client.__exit__ = MagicMock() @@ -489,19 +494,19 @@ def test_storage_account_audience_service_client(self, **kwargs): # Arrange self._setup(datalake_storage_account_name, datalake_storage_account_key) - self.dsc.create_file_system('testfs1') + self.dsc.create_file_system("testfs1") # Act token_credential = self.get_credential(DataLakeServiceClient) dsc = DataLakeServiceClient( self.account_url(datalake_storage_account_name, "blob"), credential=token_credential, - audience=f'https://{datalake_storage_account_name}.blob.core.windows.net/' + audience=f"https://{datalake_storage_account_name}.blob.core.windows.net/", ) # Assert response1 = dsc.list_file_systems() - response2 = dsc.create_file_system('testfs11') + response2 = dsc.create_file_system("testfs11") assert response1 is not None assert response2 is not None @@ -513,16 +518,16 @@ def test_bad_audience_service_client(self, **kwargs): # Arrange self._setup(datalake_storage_account_name, datalake_storage_account_key) - self.dsc.create_file_system('testfs2') + self.dsc.create_file_system("testfs2") # Act token_credential = self.get_credential(DataLakeServiceClient) dsc = DataLakeServiceClient( self.account_url(datalake_storage_account_name, "blob"), credential=token_credential, - audience='https://badaudience.blob.core.windows.net/' + audience="https://badaudience.blob.core.windows.net/", ) # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge dsc.list_file_systems() - dsc.create_file_system('testfs22') + dsc.create_file_system("testfs22") diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client_async.py index 4e7942402a87..71dcfb37b59b 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_datalake_service_client_async.py @@ -30,28 +30,27 @@ FileSystemClient, ) - if sys.version_info >= (3, 8): from unittest.mock import AsyncMock # ------------------------------------------------------------------------------ -TEST_FILE_SYSTEM_PREFIX = 'filesystem' +TEST_FILE_SYSTEM_PREFIX = "filesystem" # ------------------------------------------------------------------------------ class TestDatalakeServiceAsync(AsyncStorageRecordedTestCase): def _setup(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=account_key.secret, logging_enable=True) self.config = self.dsc._config # --Helpers----------------------------------------------------------------- def _assert_properties_default(self, prop): assert prop is not None - self._assert_logging_equal(prop['analytics_logging'], AnalyticsLogging()) - self._assert_metrics_equal(prop['hour_metrics'], Metrics()) - self._assert_metrics_equal(prop['minute_metrics'], Metrics()) - self._assert_cors_equal(prop['cors'], []) + self._assert_logging_equal(prop["analytics_logging"], AnalyticsLogging()) + self._assert_metrics_equal(prop["hour_metrics"], Metrics()) + self._assert_metrics_equal(prop["minute_metrics"], Metrics()) + self._assert_cors_equal(prop["cors"], []) def _assert_logging_equal(self, log1, log2): if log1 is None or log2 is None: @@ -139,13 +138,13 @@ async def test_datalake_service_properties(self, **kwargs): hour_metrics=Metrics(), minute_metrics=Metrics(), cors=[], - target_version='2014-02-14' + target_version="2014-02-14", ) # Assert props = await self.dsc.get_service_properties() self._assert_properties_default(props) - assert '2014-02-14' == props['target_version'] + assert "2014-02-14" == props["target_version"] @DataLakePreparer() @recorded_by_proxy_async @@ -165,11 +164,11 @@ async def test_set_default_service_version(self, **kwargs): self._setup(datalake_storage_account_name, datalake_storage_account_key) # Act - await self.dsc.set_service_properties(target_version='2014-02-14') + await self.dsc.set_service_properties(target_version="2014-02-14") # Assert received_props = await self.dsc.get_service_properties() - assert received_props['target_version'] == '2014-02-14' + assert received_props["target_version"] == "2014-02-14" @DataLakePreparer() @recorded_by_proxy_async @@ -185,7 +184,7 @@ async def test_set_delete_retention_policy(self, **kwargs): # Assert received_props = await self.dsc.get_service_properties() - self._assert_delete_retention_policy_equal(received_props['delete_retention_policy'], delete_retention_policy) + self._assert_delete_retention_policy_equal(received_props["delete_retention_policy"], delete_retention_policy) @DataLakePreparer() @recorded_by_proxy_async @@ -199,7 +198,7 @@ async def test_set_delete_retention_policy_edge_cases(self, **kwargs): # Assert received_props = await self.dsc.get_service_properties() - self._assert_delete_retention_policy_equal(received_props['delete_retention_policy'], delete_retention_policy) + self._assert_delete_retention_policy_equal(received_props["delete_retention_policy"], delete_retention_policy) # Should work with maximum settings delete_retention_policy = RetentionPolicy(enabled=True, days=365) @@ -207,7 +206,7 @@ async def test_set_delete_retention_policy_edge_cases(self, **kwargs): # Assert received_props = await self.dsc.get_service_properties() - self._assert_delete_retention_policy_equal(received_props['delete_retention_policy'], delete_retention_policy) + self._assert_delete_retention_policy_equal(received_props["delete_retention_policy"], delete_retention_policy) # Should not work with 0 days delete_retention_policy = RetentionPolicy(enabled=True, days=0) @@ -218,8 +217,7 @@ async def test_set_delete_retention_policy_edge_cases(self, **kwargs): # Assert received_props = await self.dsc.get_service_properties() self._assert_delete_retention_policy_not_equal( - received_props['delete_retention_policy'], - delete_retention_policy + received_props["delete_retention_policy"], delete_retention_policy ) # Should not work with 366 days @@ -231,8 +229,7 @@ async def test_set_delete_retention_policy_edge_cases(self, **kwargs): # Assert received_props = await self.dsc.get_service_properties() self._assert_delete_retention_policy_not_equal( - received_props['delete_retention_policy'], - delete_retention_policy + received_props["delete_retention_policy"], delete_retention_policy ) @DataLakePreparer() @@ -243,16 +240,15 @@ async def test_set_static_website_properties(self, **kwargs): self._setup(datalake_storage_account_name, datalake_storage_account_key) static_website = StaticWebsite( - enabled=True, - index_document="index.html", - error_document404_path="errors/error/404error.html") + enabled=True, index_document="index.html", error_document404_path="errors/error/404error.html" + ) # Act await self.dsc.set_service_properties(static_website=static_website) # Assert received_props = await self.dsc.get_service_properties() - self._assert_static_website_equal(received_props['static_website'], static_website) + self._assert_static_website_equal(received_props["static_website"], static_website) @DataLakePreparer() @recorded_by_proxy_async @@ -261,15 +257,16 @@ async def test_disabled_static_website_properties(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - static_website = StaticWebsite(enabled=False, index_document="index.html", - error_document404_path="errors/error/404error.html") + static_website = StaticWebsite( + enabled=False, index_document="index.html", error_document404_path="errors/error/404error.html" + ) # Act await self.dsc.set_service_properties(static_website=static_website) # Assert received_props = await self.dsc.get_service_properties() - self._assert_static_website_equal(received_props['static_website'], StaticWebsite(enabled=False)) + self._assert_static_website_equal(received_props["static_website"], StaticWebsite(enabled=False)) @DataLakePreparer() @recorded_by_proxy_async @@ -278,10 +275,10 @@ async def test_set_static_website_props_dont_impact_other_props(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - cors_rule1 = CorsRule(['www.xyz.com'], ['GET']) + cors_rule1 = CorsRule(["www.xyz.com"], ["GET"]) - allowed_origins = ['www.xyz.com', "www.ab.com", "www.bc.com"] - allowed_methods = ['GET', 'PUT'] + allowed_origins = ["www.xyz.com", "www.ab.com", "www.bc.com"] + allowed_methods = ["GET", "PUT"] max_age_in_seconds = 500 exposed_headers = ["x-ms-meta-data*", "x-ms-meta-source*", "x-ms-meta-abc", "x-ms-meta-bcd"] allowed_headers = ["x-ms-meta-data*", "x-ms-meta-target*", "x-ms-meta-xyz", "x-ms-meta-foo"] @@ -290,7 +287,8 @@ async def test_set_static_website_props_dont_impact_other_props(self, **kwargs): allowed_methods, max_age_in_seconds=max_age_in_seconds, exposed_headers=exposed_headers, - allowed_headers=allowed_headers) + allowed_headers=allowed_headers, + ) cors = [cors_rule1, cors_rule2] @@ -299,19 +297,20 @@ async def test_set_static_website_props_dont_impact_other_props(self, **kwargs): # Assert cors is updated received_props = await self.dsc.get_service_properties() - self._assert_cors_equal(received_props['cors'], cors) + self._assert_cors_equal(received_props["cors"], cors) # Arrange to set static website properties - static_website = StaticWebsite(enabled=True, index_document="index.html", - error_document404_path="errors/error/404error.html") + static_website = StaticWebsite( + enabled=True, index_document="index.html", error_document404_path="errors/error/404error.html" + ) # Act to set static website await self.dsc.set_service_properties(static_website=static_website) # Assert static website was updated was cors was unchanged received_props = await self.dsc.get_service_properties() - self._assert_static_website_equal(received_props['static_website'], static_website) - self._assert_cors_equal(received_props['cors'], cors) + self._assert_static_website_equal(received_props["static_website"], static_website) + self._assert_cors_equal(received_props["cors"], cors) @DataLakePreparer() @recorded_by_proxy_async @@ -321,10 +320,7 @@ async def test_set_logging(self, **kwargs): self._setup(datalake_storage_account_name, datalake_storage_account_key) logging = AnalyticsLogging( - read=True, - write=True, - delete=True, - retention_policy=RetentionPolicy(enabled=True, days=5) + read=True, write=True, delete=True, retention_policy=RetentionPolicy(enabled=True, days=5) ) # Act @@ -332,7 +328,7 @@ async def test_set_logging(self, **kwargs): # Assert received_props = await self.dsc.get_service_properties() - self._assert_logging_equal(received_props['analytics_logging'], logging) + self._assert_logging_equal(received_props["analytics_logging"], logging) @DataLakePreparer() @recorded_by_proxy_async @@ -341,15 +337,14 @@ async def test_set_hour_metrics(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - hour_metrics = Metrics( - include_apis=False, enabled=True, retention_policy=RetentionPolicy(enabled=True, days=5)) + hour_metrics = Metrics(include_apis=False, enabled=True, retention_policy=RetentionPolicy(enabled=True, days=5)) # Act await self.dsc.set_service_properties(hour_metrics=hour_metrics) # Assert received_props = await self.dsc.get_service_properties() - self._assert_metrics_equal(received_props['hour_metrics'], hour_metrics) + self._assert_metrics_equal(received_props["hour_metrics"], hour_metrics) @DataLakePreparer() @recorded_by_proxy_async @@ -359,14 +354,15 @@ async def test_set_minute_metrics(self, **kwargs): self._setup(datalake_storage_account_name, datalake_storage_account_key) minute_metrics = Metrics( - enabled=True, include_apis=True, retention_policy=RetentionPolicy(enabled=True, days=5)) + enabled=True, include_apis=True, retention_policy=RetentionPolicy(enabled=True, days=5) + ) # Act await self.dsc.set_service_properties(minute_metrics=minute_metrics) # Assert received_props = await self.dsc.get_service_properties() - self._assert_metrics_equal(received_props['minute_metrics'], minute_metrics) + self._assert_metrics_equal(received_props["minute_metrics"], minute_metrics) @DataLakePreparer() @recorded_by_proxy_async @@ -375,10 +371,10 @@ async def test_set_cors(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - cors_rule1 = CorsRule(['www.xyz.com'], ['GET']) + cors_rule1 = CorsRule(["www.xyz.com"], ["GET"]) - allowed_origins = ['www.xyz.com', "www.ab.com", "www.bc.com"] - allowed_methods = ['GET', 'PUT'] + allowed_origins = ["www.xyz.com", "www.ab.com", "www.bc.com"] + allowed_methods = ["GET", "PUT"] max_age_in_seconds = 500 exposed_headers = ["x-ms-meta-data*", "x-ms-meta-source*", "x-ms-meta-abc", "x-ms-meta-bcd"] allowed_headers = ["x-ms-meta-data*", "x-ms-meta-target*", "x-ms-meta-xyz", "x-ms-meta-foo"] @@ -387,7 +383,8 @@ async def test_set_cors(self, **kwargs): allowed_methods, max_age_in_seconds=max_age_in_seconds, exposed_headers=exposed_headers, - allowed_headers=allowed_headers) + allowed_headers=allowed_headers, + ) cors = [cors_rule1, cors_rule2] @@ -396,29 +393,29 @@ async def test_set_cors(self, **kwargs): # Assert received_props = await self.dsc.get_service_properties() - self._assert_cors_equal(received_props['cors'], cors) + self._assert_cors_equal(received_props["cors"], cors) @DataLakePreparer() async def test_connectionstring_without_secondary(self): test_conn_str = "DefaultEndpointsProtocol=https;AccountName=foo;AccountKey=bar" client = DataLakeServiceClient.from_connection_string(test_conn_str) - assert client.url == 'https://foo.dfs.core.windows.net/' - assert client.primary_hostname == 'foo.dfs.core.windows.net' + assert client.url == "https://foo.dfs.core.windows.net/" + assert client.primary_hostname == "foo.dfs.core.windows.net" assert not client.secondary_hostname client = FileSystemClient.from_connection_string(test_conn_str, "fsname") - assert client.url == 'https://foo.dfs.core.windows.net/fsname' - assert client.primary_hostname == 'foo.dfs.core.windows.net' + assert client.url == "https://foo.dfs.core.windows.net/fsname" + assert client.primary_hostname == "foo.dfs.core.windows.net" assert not client.secondary_hostname client = DataLakeFileClient.from_connection_string(test_conn_str, "fsname", "fpath") - assert client.url == 'https://foo.dfs.core.windows.net/fsname/fpath' - assert client.primary_hostname == 'foo.dfs.core.windows.net' + assert client.url == "https://foo.dfs.core.windows.net/fsname/fpath" + assert client.primary_hostname == "foo.dfs.core.windows.net" assert not client.secondary_hostname client = DataLakeDirectoryClient.from_connection_string(test_conn_str, "fsname", "dname") - assert client.url == 'https://foo.dfs.core.windows.net/fsname/dname' - assert client.primary_hostname == 'foo.dfs.core.windows.net' + assert client.url == "https://foo.dfs.core.windows.net/fsname/dname" + assert client.primary_hostname == "foo.dfs.core.windows.net" assert not client.secondary_hostname @DataLakePreparer() @@ -460,9 +457,9 @@ async def test_datalake_clients_properly_close(self): account_key = NamedTuple("StorageAccountKey", [("secret", str)])("adlskey") self._setup(account_name, account_key) - file_system_client = self.dsc.get_file_system_client(file_system='testfs') - dir_client = self.dsc.get_directory_client(file_system='testfs', directory='testdir') - file_client = dir_client.get_file_client(file='testfile') + file_system_client = self.dsc.get_file_system_client(file_system="testfs") + dir_client = self.dsc.get_directory_client(file_system="testfs", directory="testdir") + file_client = dir_client.get_file_client(file="testfile") # Mocks self.dsc._blob_service_client.__aexit__ = AsyncMock() @@ -501,7 +498,7 @@ async def test_storage_account_audience_service_client(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - file_system_name = self.get_resource_name('filesystem') + file_system_name = self.get_resource_name("filesystem") await self.dsc.create_file_system(file_system_name) # Act @@ -509,12 +506,12 @@ async def test_storage_account_audience_service_client(self, **kwargs): dsc = DataLakeServiceClient( self.account_url(datalake_storage_account_name, "blob"), credential=token_credential, - audience=f'https://{datalake_storage_account_name}.blob.core.windows.net/' + audience=f"https://{datalake_storage_account_name}.blob.core.windows.net/", ) # Assert response1 = dsc.list_file_systems() - response2 = dsc.create_file_system(file_system_name + '1') + response2 = dsc.create_file_system(file_system_name + "1") assert response1 is not None assert response2 is not None @@ -525,7 +522,7 @@ async def test_bad_audience_service_client(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setup(datalake_storage_account_name, datalake_storage_account_key) - file_system_name = self.get_resource_name('filesystem') + file_system_name = self.get_resource_name("filesystem") await self.dsc.create_file_system(file_system_name) # Act @@ -533,9 +530,9 @@ async def test_bad_audience_service_client(self, **kwargs): dsc = DataLakeServiceClient( self.account_url(datalake_storage_account_name, "blob"), credential=token_credential, - audience='https://badaudience.blob.core.windows.net/' + audience="https://badaudience.blob.core.windows.net/", ) # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge dsc.list_file_systems() - await dsc.create_file_system(file_system_name + '1') + await dsc.create_file_system(file_system_name + "1") diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_directory.py b/sdk/storage/azure-storage-file-datalake/tests/test_directory.py index 052ce7fddea3..ef6fb8e182d2 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_directory.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_directory.py @@ -34,12 +34,14 @@ ) from azure.storage.filedatalake._models import AccessControlChangeCounters, AccessControlChangeResult - # ------------------------------------------------------------------------------ -TEST_DIRECTORY_PREFIX = 'directory' -REMOVE_ACL = "mask," + "default:user,default:group," + \ - "user:ec3595d6-2c17-4696-8caa-7e139758d24a,group:ec3595d6-2c17-4696-8caa-7e139758d24a," + \ - "default:user:ec3595d6-2c17-4696-8caa-7e139758d24a,default:group:ec3595d6-2c17-4696-8caa-7e139758d24a" +TEST_DIRECTORY_PREFIX = "directory" +REMOVE_ACL = ( + "mask," + + "default:user,default:group," + + "user:ec3595d6-2c17-4696-8caa-7e139758d24a,group:ec3595d6-2c17-4696-8caa-7e139758d24a," + + "default:user:ec3595d6-2c17-4696-8caa-7e139758d24a,default:group:ec3595d6-2c17-4696-8caa-7e139758d24a" +) # ------------------------------------------------------------------------------ @@ -47,11 +49,11 @@ class TestDirectory(StorageRecordedTestCase): def _setUp(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=account_key.secret, logging_enable=True) self.config = self.dsc._config - self.file_system_name = self.get_resource_name('filesystem') + self.file_system_name = self.get_resource_name("filesystem") if not self.is_playback(): file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -83,9 +85,9 @@ def _create_directory_and_get_directory_client(self, directory_name=None): def _create_sub_directory_and_files(self, directory_client, num_of_dirs, num_of_files_per_dir): # the name suffix matter since we need to avoid creating the same directories/files in record mode for i in range(0, num_of_dirs): - sub_dir = directory_client.create_sub_directory(self.get_resource_name('subdir' + str(i))) + sub_dir = directory_client.create_sub_directory(self.get_resource_name("subdir" + str(i))) for j in range(0, num_of_files_per_dir): - sub_dir.create_file(self.get_resource_name('subfile' + str(j))) + sub_dir.create_file(self.get_resource_name("subfile" + str(j))) # --Helpers----------------------------------------------------------------- @@ -98,9 +100,7 @@ def test_create_directory(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") # Act directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) created = directory_client.create_directory(content_settings=content_settings) @@ -115,8 +115,8 @@ def test_create_directory_owner_group_acl(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' - test_string_acl = 'user::rwx,group::r-x,other::rwx' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" + test_string_acl = "user::rwx,group::r-x,other::rwx" # Arrange directory_name = self._get_directory_reference() @@ -127,9 +127,9 @@ def test_create_directory_owner_group_acl(self, **kwargs): # Assert acl_properties = directory_client.get_access_control() assert acl_properties is not None - assert acl_properties['owner'] == test_string - assert acl_properties['group'] == test_string - assert acl_properties['acl'] == test_string_acl + assert acl_properties["owner"] == test_string + assert acl_properties["group"] == test_string + assert acl_properties["acl"] == test_string_acl @DataLakePreparer() @recorded_by_proxy @@ -138,7 +138,7 @@ def test_create_directory_proposed_lease_id(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" test_duration = 15 # Arrange directory_name = self._get_directory_reference() @@ -148,9 +148,9 @@ def test_create_directory_proposed_lease_id(self, **kwargs): # Assert properties = directory_client.get_directory_properties() assert properties is not None - assert properties.lease['status'] == 'locked' - assert properties.lease['state'] == 'leased' - assert properties.lease['duration'] == 'fixed' + assert properties.lease["status"] == "locked" + assert properties.lease["state"] == "leased" + assert properties.lease["duration"] == "fixed" @DataLakePreparer() @recorded_by_proxy @@ -159,21 +159,21 @@ def test_create_sub_directory_proposed_lease_id(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" test_duration = 15 # Arrange directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) - directory_client = directory_client.create_sub_directory(sub_directory='sub1', - lease_id=test_string, - lease_duration=test_duration) + directory_client = directory_client.create_sub_directory( + sub_directory="sub1", lease_id=test_string, lease_duration=test_duration + ) # Assert properties = directory_client.get_directory_properties() assert properties is not None - assert properties.lease['status'] == 'locked' - assert properties.lease['state'] == 'leased' - assert properties.lease['duration'] == 'fixed' + assert properties.lease["status"] == "locked" + assert properties.lease["state"] == "leased" + assert properties.lease["duration"] == "fixed" @DataLakePreparer() @recorded_by_proxy @@ -202,8 +202,9 @@ def test_using_oauth_token_credential_to_create_directory(self, **kwargs): # generate a token with directory level create permission directory_name = self._get_directory_reference() token_credential = self.get_credential(DataLakeServiceClient) - directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, - credential=token_credential) + directory_client = DataLakeDirectoryClient( + self.dsc.url, self.file_system_name, directory_name, credential=token_credential + ) response = directory_client.create_directory() assert response is not None @@ -242,7 +243,7 @@ def test_create_directory_with_permission(self, **kwargs): # Assert assert created - assert prop['permissions'] == 'rwxr--r--' + assert prop["permissions"] == "rwxr--r--" @DataLakePreparer() @recorded_by_proxy @@ -253,9 +254,7 @@ def test_create_directory_with_content_settings(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") # Act directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) created = directory_client.create_directory(content_settings=content_settings) @@ -272,7 +271,7 @@ def test_create_directory_with_metadata(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} # Act directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) created = directory_client.create_directory(metadata=metadata) @@ -291,7 +290,7 @@ def test_delete_directory(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) @@ -312,7 +311,7 @@ def test_delete_directory_with_if_modified_since(self, **kwargs): prop = directory_client.get_directory_properties() with pytest.raises(ResourceModifiedError): - directory_client.delete_directory(if_modified_since=prop['last_modified']) + directory_client.delete_directory(if_modified_since=prop["last_modified"]) @DataLakePreparer() @pytest.mark.live_test_only @@ -324,7 +323,7 @@ def test_delete_directory_paginated(self, **kwargs): # Set this to object id (not client id) of an AAD app that does not have permission # to storage account through RBAC. # Also make sure oauth settings (TENANT_ID, CLIENT_ID, CLIENT_SECRET) are pointing to this AAD app - object_id = '68bff720-253b-428c-b124-603700654ea9' + object_id = "68bff720-253b-428c-b124-603700654ea9" # Arrange self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -333,12 +332,12 @@ def test_delete_directory_paginated(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - for i in range (0, 5020): + for i in range(0, 5020): file_client = directory_client.get_file_client(f"file{i}") file_client.create_file() root_directory = self.dsc.get_directory_client(self.file_system_name, "/") - acl = root_directory.get_access_control()['acl'] + acl = root_directory.get_access_control()["acl"] # Add permission for AAD app on root directory new_acl = acl + "," + f"user:{object_id}:rwx" @@ -346,10 +345,7 @@ def test_delete_directory_paginated(self, **kwargs): token_credential = self.get_credential(DataLakeServiceClient) directory_client_oauth = DataLakeDirectoryClient( - self.dsc.url, - self.file_system_name, - directory_name, - credential=token_credential + self.dsc.url, self.file_system_name, directory_name, credential=token_credential ) # Act @@ -366,19 +362,20 @@ def test_create_sub_directory_and_delete_sub_directory(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} # Create a directory first, to prepare for creating sub directory directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) # Create sub directory from the current directory - sub_directory_name = 'subdir' + sub_directory_name = "subdir" sub_directory_created = directory_client.create_sub_directory(sub_directory_name) # to make sure the sub directory was indeed created by get sub_directory properties from sub directory client - sub_directory_client = self.dsc.get_directory_client(self.file_system_name, - directory_name + '/' + sub_directory_name) + sub_directory_client = self.dsc.get_directory_client( + self.file_system_name, directory_name + "/" + sub_directory_name + ) sub_properties = sub_directory_client.get_directory_properties() # Assert @@ -398,11 +395,11 @@ def test_set_access_control(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) - response = directory_client.set_access_control(permissions='0777') + response = directory_client.set_access_control(permissions="0777") # Assert assert response is not None @@ -414,18 +411,18 @@ def test_set_access_control_with_acl(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" directory_client.set_access_control(acl=acl) access_control = directory_client.get_access_control() # Assert assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -438,8 +435,9 @@ def test_set_access_control_if_none_modified(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) resp = directory_client.create_directory() - response = directory_client.set_access_control(permissions='0777', etag=resp['etag'], - match_condition=MatchConditions.IfNotModified) + response = directory_client.set_access_control( + permissions="0777", etag=resp["etag"], match_condition=MatchConditions.IfNotModified + ) # Assert assert response is not None @@ -451,9 +449,9 @@ def test_get_access_control(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) - directory_client.create_directory(metadata=metadata, permissions='0777') + directory_client.create_directory(metadata=metadata, permissions="0777") # Act response = directory_client.get_access_control() @@ -469,13 +467,13 @@ def test_get_access_control_with_match_conditions(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) - resp = directory_client.create_directory(permissions='0777', umask='0000') + resp = directory_client.create_directory(permissions="0777", umask="0000") # Act - response = directory_client.get_access_control(etag=resp['etag'], match_condition=MatchConditions.IfNotModified) + response = directory_client.get_access_control(etag=resp["etag"], match_condition=MatchConditions.IfNotModified) # Assert assert response is not None - assert response['permissions'] == 'rwxrwxrwx' + assert response["permissions"] == "rwxrwxrwx" @DataLakePreparer() @recorded_by_proxy @@ -491,7 +489,7 @@ def test_set_access_control_recursive(self, **kwargs): num_file_per_sub_dir = 5 self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" summary = directory_client.set_access_control_recursive(acl=acl) # Assert @@ -502,7 +500,7 @@ def test_set_access_control_recursive(self, **kwargs): assert summary.continuation is None access_control = directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -524,11 +522,13 @@ def callback(response): response_list.append(response) if len(response_list) == 2: raise ServiceRequestError("network problem") - acl = 'user::rwx,group::r-x,other::rwx' + + acl = "user::rwx,group::r-x,other::rwx" with pytest.raises(AzureError) as acl_error: - directory_client.set_access_control_recursive(acl=acl, batch_size=2, max_batches=2, - raw_response_hook=callback, retry_total=0) + directory_client.set_access_control_recursive( + acl=acl, batch_size=2, max_batches=2, raw_response_hook=callback, retry_total=0 + ) assert acl_error.value.continuation_token is not None assert acl_error.value.message == "network problem" assert acl_error.typename == "ServiceRequestError" @@ -547,7 +547,7 @@ def test_set_access_control_recursive_in_batches(self, **kwargs): num_file_per_sub_dir = 5 self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" summary = directory_client.set_access_control_recursive(acl=acl, batch_size=2) # Assert @@ -558,7 +558,7 @@ def test_set_access_control_recursive_in_batches(self, **kwargs): assert summary.continuation is None access_control = directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -574,7 +574,7 @@ def test_set_access_control_recursive_in_batches_with_progress_callback(self, ** num_file_per_sub_dir = 5 self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) last_response = AccessControlChangeResult(None, "") @@ -585,8 +585,7 @@ def progress_callback(resp): last_response.counters = resp.aggregate_counters - summary = directory_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2) + summary = directory_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, batch_size=2) # Assert assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included @@ -601,15 +600,15 @@ def progress_callback(resp): assert summary.counters.failure_count == last_response.counters.failure_count access_control = directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @pytest.mark.live_test_only @DataLakePreparer() def test_set_access_control_recursive_with_failures(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - url = self.account_url(datalake_storage_account_name, 'dfs') - variables = kwargs.pop('variables', {}) + url = self.account_url(datalake_storage_account_name, "dfs") + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -621,20 +620,24 @@ def test_set_access_control_recursive_with_failures(self, **kwargs): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(owner=test_guid) - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir1').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir2').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir1/file1').create_file( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir2/file2').create_file( - owner=test_guid, permissions='0777') - directory_client.get_file_client('file3').create_file() + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir1").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir2").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir1/file1").create_file( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir2/file2").create_file( + owner=test_guid, permissions="0777" + ) + directory_client.get_file_client("file3").create_file() # User delegation SAS with provided owner permissions token_credential = self.get_credential(DataLakeServiceClient) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) owner_dsc = DataLakeServiceClient(url, credential=token_credential) user_delegation_key = owner_dsc.get_user_delegation_key(start_time, expiry_time) sas_token = self.generate_sas( @@ -643,9 +646,9 @@ def test_set_access_control_recursive_with_failures(self, **kwargs): self.file_system_name, directory_name, user_delegation_key, - permission='racwdlmeop', + permission="racwdlmeop", expiry=expiry_time, - agent_object_id=test_guid + agent_object_id=test_guid, ) if self.is_live: @@ -653,7 +656,7 @@ def test_set_access_control_recursive_with_failures(self, **kwargs): owner_dir_client = DataLakeDirectoryClient(url, self.file_system_name, directory_name, sas_token) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) failed_entries = [] @@ -663,8 +666,7 @@ def progress_callback(resp): running_tally.failure_count += resp.batch_counters.failure_count failed_entries.append(resp.batch_failures) - summary = owner_dir_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2) + summary = owner_dir_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, batch_size=2) # Assert assert summary.counters.failure_count == 1 @@ -680,8 +682,8 @@ def progress_callback(resp): def test_set_access_control_recursive_stop_on_failures(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - url = self.account_url(datalake_storage_account_name, 'dfs') - variables = kwargs.pop('variables', {}) + url = self.account_url(datalake_storage_account_name, "dfs") + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -693,20 +695,24 @@ def test_set_access_control_recursive_stop_on_failures(self, **kwargs): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(owner=test_guid) - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir1').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir2').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir1/file1').create_file( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir2/file2').create_file( - owner=test_guid, permissions='0777') - directory_client.get_file_client('file3').create_file() + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir1").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir2").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir1/file1").create_file( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir2/file2").create_file( + owner=test_guid, permissions="0777" + ) + directory_client.get_file_client("file3").create_file() # User delegation SAS with provided owner permissions token_credential = self.get_credential(DataLakeServiceClient) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) owner_dsc = DataLakeServiceClient(url, credential=token_credential) user_delegation_key = owner_dsc.get_user_delegation_key(start_time, expiry_time) sas_token = self.generate_sas( @@ -715,9 +721,9 @@ def test_set_access_control_recursive_stop_on_failures(self, **kwargs): self.file_system_name, directory_name, user_delegation_key, - permission='racwdlmeop', + permission="racwdlmeop", expiry=expiry_time, - agent_object_id=test_guid + agent_object_id=test_guid, ) if self.is_live: @@ -725,7 +731,7 @@ def test_set_access_control_recursive_stop_on_failures(self, **kwargs): owner_dir_client = DataLakeDirectoryClient(url, self.file_system_name, directory_name, sas_token) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) failed_entries = [] @@ -736,8 +742,7 @@ def progress_callback(resp): if resp.batch_failures: failed_entries.append(resp.batch_failures) - summary = owner_dir_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2) + summary = owner_dir_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, batch_size=2) # Assert assert summary.counters.failure_count == 1 @@ -753,8 +758,8 @@ def progress_callback(resp): def test_set_access_control_recursive_continue_on_failures(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - url = self.account_url(datalake_storage_account_name, 'dfs') - variables = kwargs.pop('variables', {}) + url = self.account_url(datalake_storage_account_name, "dfs") + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -766,22 +771,26 @@ def test_set_access_control_recursive_continue_on_failures(self, **kwargs): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(owner=test_guid) - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir1').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir2').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir1/file1').create_file( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir2/file2').create_file( - owner=test_guid, permissions='0777') - - directory_client.get_file_client('file3').create_file() - self.dsc.get_directory_client(self.file_system_name, directory_name + '/dir3').create_directory() + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir1").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir2").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir1/file1").create_file( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir2/file2").create_file( + owner=test_guid, permissions="0777" + ) + + directory_client.get_file_client("file3").create_file() + self.dsc.get_directory_client(self.file_system_name, directory_name + "/dir3").create_directory() # User delegation SAS with provided owner permissions token_credential = self.get_credential(DataLakeServiceClient) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) owner_dsc = DataLakeServiceClient(url, credential=token_credential) user_delegation_key = owner_dsc.get_user_delegation_key(start_time, expiry_time) sas_token = self.generate_sas( @@ -790,9 +799,9 @@ def test_set_access_control_recursive_continue_on_failures(self, **kwargs): self.file_system_name, directory_name, user_delegation_key, - permission='racwdlmeop', + permission="racwdlmeop", expiry=expiry_time, - agent_object_id=test_guid + agent_object_id=test_guid, ) if self.is_live: @@ -800,7 +809,7 @@ def test_set_access_control_recursive_continue_on_failures(self, **kwargs): owner_dir_client = DataLakeDirectoryClient(url, self.file_system_name, directory_name, sas_token) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) failed_entries = [] @@ -811,8 +820,9 @@ def progress_callback(resp): if resp.batch_failures: failed_entries.append(resp.batch_failures) - summary = owner_dir_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2, continue_on_failure=True) + summary = owner_dir_client.set_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=2, continue_on_failure=True + ) # Assert assert summary.counters.failure_count == 2 @@ -837,7 +847,7 @@ def test_set_access_control_recursive_in_batches_with_explicit_iteration(self, * num_file_per_sub_dir = 5 self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) result = AccessControlChangeResult(None, "") iteration_count = 0 @@ -846,7 +856,8 @@ def test_set_access_control_recursive_in_batches_with_explicit_iteration(self, * while result.continuation is not None: result = directory_client.set_access_control_recursive( - acl=acl, batch_size=batch_size, max_batches=max_batches, continuation=result.continuation) + acl=acl, batch_size=batch_size, max_batches=max_batches, continuation=result.continuation + ) running_tally.directories_successful += result.counters.directories_successful running_tally.files_successful += result.counters.files_successful @@ -859,7 +870,7 @@ def test_set_access_control_recursive_in_batches_with_explicit_iteration(self, * assert running_tally.failure_count == 0 access_control = directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -875,16 +886,16 @@ def test_update_access_control_recursive(self, **kwargs): num_file_per_sub_dir = 5 self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" summary = directory_client.update_access_control_recursive(acl=acl) # Assert - assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included + assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included assert summary.counters.files_successful == num_sub_dirs * num_file_per_sub_dir assert summary.counters.failure_count == 0 access_control = directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -900,16 +911,16 @@ def test_update_access_control_recursive_in_batches(self, **kwargs): num_file_per_sub_dir = 5 self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" summary = directory_client.update_access_control_recursive(acl=acl, batch_size=2) # Assert - assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included + assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included assert summary.counters.files_successful == num_sub_dirs * num_file_per_sub_dir assert summary.counters.failure_count == 0 access_control = directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -925,7 +936,7 @@ def test_update_access_control_recursive_in_batches_with_progress_callback(self, num_file_per_sub_dir = 5 self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) last_response = AccessControlChangeResult(None, "") @@ -936,8 +947,9 @@ def progress_callback(resp): last_response.counters = resp.aggregate_counters - summary = directory_client.update_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2) + summary = directory_client.update_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=2 + ) # Assert assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included @@ -952,15 +964,15 @@ def progress_callback(resp): assert summary.counters.failure_count == last_response.counters.failure_count access_control = directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @pytest.mark.live_test_only @DataLakePreparer() def test_update_access_control_recursive_with_failures(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - url = self.account_url(datalake_storage_account_name, 'dfs') - variables = kwargs.pop('variables', {}) + url = self.account_url(datalake_storage_account_name, "dfs") + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -972,20 +984,24 @@ def test_update_access_control_recursive_with_failures(self, **kwargs): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(owner=test_guid) - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir1').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir2').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir1/file1').create_file( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir2/file2').create_file( - owner=test_guid, permissions='0777') - directory_client.get_file_client('file3').create_file() + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir1").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir2").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir1/file1").create_file( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir2/file2").create_file( + owner=test_guid, permissions="0777" + ) + directory_client.get_file_client("file3").create_file() # User delegation SAS with provided owner permissions token_credential = self.get_credential(DataLakeServiceClient) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) owner_dsc = DataLakeServiceClient(url, credential=token_credential) user_delegation_key = owner_dsc.get_user_delegation_key(start_time, expiry_time) sas_token = self.generate_sas( @@ -994,9 +1010,9 @@ def test_update_access_control_recursive_with_failures(self, **kwargs): self.file_system_name, directory_name, user_delegation_key, - permission='racwdlmeop', + permission="racwdlmeop", expiry=expiry_time, - agent_object_id=test_guid + agent_object_id=test_guid, ) if self.is_live: @@ -1004,7 +1020,7 @@ def test_update_access_control_recursive_with_failures(self, **kwargs): owner_dir_client = DataLakeDirectoryClient(url, self.file_system_name, directory_name, sas_token) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) failed_entries = [] @@ -1015,8 +1031,9 @@ def progress_callback(resp): if resp.batch_failures: failed_entries.append(resp.batch_failures) - summary = owner_dir_client.update_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2) + summary = owner_dir_client.update_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=2 + ) # Assert assert summary.counters.failure_count == 1 @@ -1093,8 +1110,9 @@ def progress_callback(resp): last_response.counters = resp.aggregate_counters - summary = directory_client.remove_access_control_recursive(acl=REMOVE_ACL, progress_hook=progress_callback, - batch_size=2) + summary = directory_client.remove_access_control_recursive( + acl=REMOVE_ACL, progress_hook=progress_callback, batch_size=2 + ) # Assert assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included @@ -1112,8 +1130,8 @@ def progress_callback(resp): def test_remove_access_control_recursive_with_failures(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - url = self.account_url(datalake_storage_account_name, 'dfs') - variables = kwargs.pop('variables', {}) + url = self.account_url(datalake_storage_account_name, "dfs") + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -1125,20 +1143,24 @@ def test_remove_access_control_recursive_with_failures(self, **kwargs): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(owner=test_guid) - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir1').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir2').create_directory( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir1/file1').create_file( - owner=test_guid, permissions='0777') - self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir2/file2').create_file( - owner=test_guid, permissions='0777') - directory_client.get_file_client('file3').create_file() + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir1").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir2").create_directory( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir1/file1").create_file( + owner=test_guid, permissions="0777" + ) + self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir2/file2").create_file( + owner=test_guid, permissions="0777" + ) + directory_client.get_file_client("file3").create_file() # User delegation SAS with provided owner permissions token_credential = self.get_credential(DataLakeServiceClient) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) owner_dsc = DataLakeServiceClient(url, credential=token_credential) user_delegation_key = owner_dsc.get_user_delegation_key(start_time, expiry_time) sas_token = self.generate_sas( @@ -1147,9 +1169,9 @@ def test_remove_access_control_recursive_with_failures(self, **kwargs): self.file_system_name, directory_name, user_delegation_key, - permission='racwdlmeop', + permission="racwdlmeop", expiry=expiry_time, - agent_object_id=test_guid + agent_object_id=test_guid, ) if self.is_live: @@ -1168,7 +1190,8 @@ def progress_callback(resp): failed_entries.append(resp.batch_failures) summary = owner_dir_client.remove_access_control_recursive( - acl=REMOVE_ACL, progress_hook=progress_callback, batch_size=2) + acl=REMOVE_ACL, progress_hook=progress_callback, batch_size=2 + ) # Assert assert summary.counters.failure_count == 1 @@ -1186,9 +1209,7 @@ def test_rename_from(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() @@ -1197,12 +1218,13 @@ def test_rename_from(self, **kwargs): new_directory_client = self.dsc.get_directory_client(self.file_system_name, new_name) - new_directory_client._rename_path('/' + self.file_system_name + '/' + directory_name, - content_settings=content_settings) + new_directory_client._rename_path( + "/" + self.file_system_name + "/" + directory_name, content_settings=content_settings + ) properties = new_directory_client.get_directory_properties() assert properties is not None - assert properties.get('content_settings') is None + assert properties.get("content_settings") is None @DataLakePreparer() @recorded_by_proxy @@ -1218,7 +1240,7 @@ def test_rename_from_a_shorter_directory_to_longer_directory(self, **kwargs): new_directory_client = self._create_directory_and_get_directory_client(directory_name=new_name) new_directory_client = new_directory_client.create_sub_directory("newsub") - new_directory_client._rename_path('/' + self.file_system_name + '/' + directory_name) + new_directory_client._rename_path("/" + self.file_system_name + "/" + directory_name) properties = new_directory_client.get_directory_properties() assert properties is not None @@ -1243,7 +1265,7 @@ def test_rename_from_a_directory_in_another_file_system(self, **kwargs): new_directory_client = new_directory_client.create_sub_directory("newsub") # rename dir1 under filesystem1 to dir2 under filesystem2 - new_directory_client._rename_path('/' + old_file_system_name + '/' + old_dir_name) + new_directory_client._rename_path("/" + old_file_system_name + "/" + old_dir_name) properties = new_directory_client.get_directory_properties() assert properties is not None @@ -1272,7 +1294,7 @@ def test_rename_from_an_unencoded_directory_in_another_file_system(self, **kwarg new_file_system_client.create_directory(new_name) # rename dir1 under filesystem1 to dir2 under filesystem2 - new_directory_client = old_dir_client.rename_directory('/' + new_file_system_name + '/' + new_name) + new_directory_client = old_dir_client.rename_directory("/" + new_file_system_name + "/" + new_name) properties = new_directory_client.get_directory_properties() file_properties = new_directory_client.get_file_client(file_name).get_file_properties() @@ -1300,7 +1322,7 @@ def test_rename_to_an_existing_directory_in_another_file_system(self, **kwargs): source_directory_client = source_directory_client.create_sub_directory("subdir") # rename dir2 under filesystem2 to dir1 under filesystem1 - res = source_directory_client.rename_directory('/' + destination_file_system_name + '/' + destination_dir_name) + res = source_directory_client.rename_directory("/" + destination_file_system_name + "/" + destination_dir_name) # the source directory has been renamed to destination directory, so it cannot be found with pytest.raises(HttpResponseError): @@ -1329,11 +1351,13 @@ def test_rename_with_none_existing_destination_condition_and_source_unmodified_c # rename dir2 under filesystem2 to a non existing directory under filesystem1, # when dir1 does not exist and dir2 wasn't modified - etag = source_directory_client.get_directory_properties()['etag'] - res = source_directory_client.rename_directory('/' + destination_file_system_name + '/' + non_existing_dir_name, - match_condition=MatchConditions.IfMissing, - source_etag=etag, - source_match_condition=MatchConditions.IfNotModified) + etag = source_directory_client.get_directory_properties()["etag"] + res = source_directory_client.rename_directory( + "/" + destination_file_system_name + "/" + non_existing_dir_name, + match_condition=MatchConditions.IfMissing, + source_etag=etag, + source_match_condition=MatchConditions.IfNotModified, + ) # the source directory has been renamed to destination directory, so it cannot be found with pytest.raises(HttpResponseError): @@ -1360,8 +1384,7 @@ def test_rename_to_an_non_existing_directory_in_another_file_system(self, **kwar source_directory_client = source_directory_client.create_sub_directory("subdir") # rename dir2 under filesystem2 to dir1 under filesystem1 - res = source_directory_client.rename_directory('/' + destination_file_system_name + '/' + non_existing_dir_name) - + res = source_directory_client.rename_directory("/" + destination_file_system_name + "/" + non_existing_dir_name) # the source directory has been renamed to destination directory, so it cannot be found with pytest.raises(HttpResponseError): @@ -1380,7 +1403,7 @@ def test_rename_directory_to_non_empty_directory(self, **kwargs): dir1.create_sub_directory("subdir") dir2 = self._create_directory_and_get_directory_client("dir2") - dir2.rename_directory(dir1.file_system_name + '/' + dir1.path_name) + dir2.rename_directory(dir1.file_system_name + "/" + dir1.path_name) with pytest.raises(HttpResponseError): dir2.get_directory_properties() @@ -1405,7 +1428,7 @@ def test_rename_dir_with_file_system_sas(self, **kwargs): # read the created file which is under root directory dir_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, "olddir", credential=token) dir_client.create_directory() - new_client = dir_client.rename_directory(dir_client.file_system_name + '/' + 'newdir') + new_client = dir_client.rename_directory(dir_client.file_system_name + "/" + "newdir") new_client.get_directory_properties() assert new_client.path_name == "newdir" @@ -1418,11 +1441,11 @@ def test_rename_directory_special_chars(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) - dir_client = self._create_directory_and_get_directory_client('olddir') - new_client = dir_client.rename_directory(dir_client.file_system_name + '/' + '?!@#$%^&*.?test') + dir_client = self._create_directory_and_get_directory_client("olddir") + new_client = dir_client.rename_directory(dir_client.file_system_name + "/" + "?!@#$%^&*.?test") new_props = new_client.get_directory_properties() - assert new_props.name == '?!@#$%^&*.?test' + assert new_props.name == "?!@#$%^&*.?test" @DataLakePreparer() @recorded_by_proxy @@ -1433,7 +1456,7 @@ def test_get_properties(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory(metadata=metadata) @@ -1441,7 +1464,7 @@ def test_get_properties(self, **kwargs): # Assert assert properties assert properties.metadata is not None - assert properties.metadata['hello'] == metadata['hello'] + assert properties.metadata["hello"] == metadata["hello"] @DataLakePreparer() @recorded_by_proxy @@ -1450,11 +1473,11 @@ def test_directory_encryption_scope_from_file_system(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=datalake_storage_account_key.secret, logging_enable=True) self.config = self.dsc._config - self.file_system_name = self.get_resource_name('filesystem') - dir_name = 'testdir' + self.file_system_name = self.get_resource_name("filesystem") + dir_name = "testdir" file_system = self.dsc.get_file_system_client(self.file_system_name) encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") @@ -1466,8 +1489,8 @@ def test_directory_encryption_scope_from_file_system(self, **kwargs): # Assert assert props - assert props['encryption_scope'] is not None - assert props['encryption_scope'] == encryption_scope.default_encryption_scope + assert props["encryption_scope"] is not None + assert props["encryption_scope"] == encryption_scope.default_encryption_scope @pytest.mark.live_test_only @DataLakePreparer() @@ -1491,8 +1514,9 @@ def test_using_directory_sas_to_read(self, **kwargs): expiry=datetime.utcnow() + timedelta(hours=1), ) - directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, - credential=token) + directory_client = DataLakeDirectoryClient( + self.dsc.url, self.file_system_name, directory_name, credential=token + ) access_control = directory_client.get_access_control() assert access_control is not None @@ -1516,8 +1540,9 @@ def test_using_directory_sas_to_create(self, **kwargs): permission=DirectorySasPermissions(create=True), expiry=datetime.utcnow() + timedelta(hours=1), ) - directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, - credential=token) + directory_client = DataLakeDirectoryClient( + self.dsc.url, self.file_system_name, directory_name, credential=token + ) response = directory_client.create_directory() assert response is not None @@ -1543,8 +1568,9 @@ def test_using_directory_sas_to_create_file(self, **kwargs): expiry=datetime.utcnow() + timedelta(hours=1), ) - directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, - credential=token) + directory_client = DataLakeDirectoryClient( + self.dsc.url, self.file_system_name, directory_name, credential=token + ) directory_client.create_sub_directory("subdir") with pytest.raises(HttpResponseError): @@ -1565,14 +1591,16 @@ def test_storage_account_audience_dir_client(self, **kwargs): # Act token_credential = self.get_credential(DataLakeServiceClient) directory_client = DataLakeDirectoryClient( - self.dsc.url, self.file_system_name, directory_name, + self.dsc.url, + self.file_system_name, + directory_name, credential=token_credential, - audience=f'https://{datalake_storage_account_name}.blob.core.windows.net/' + audience=f"https://{datalake_storage_account_name}.blob.core.windows.net/", ) # Assert response1 = directory_client.exists() - response2 = directory_client.create_sub_directory('testsubdir') + response2 = directory_client.create_sub_directory("testsubdir") assert response1 is not None assert response2 is not None @@ -1591,13 +1619,16 @@ def test_bad_audience_dir_client(self, **kwargs): # Act token_credential = self.get_credential(DataLakeServiceClient) directory_client = DataLakeDirectoryClient( - self.dsc.url, self.file_system_name, directory_name, - credential=token_credential, audience='https://badaudience.blob.core.windows.net/' + self.dsc.url, + self.file_system_name, + directory_name, + credential=token_credential, + audience="https://badaudience.blob.core.windows.net/", ) # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge directory_client.exists() - directory_client.create_sub_directory('testsubdir') + directory_client.create_sub_directory("testsubdir") @DataLakePreparer() @recorded_by_proxy @@ -1607,35 +1638,36 @@ def test_directory_get_paths(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() - directory_client1 = self.dsc.get_directory_client(self.file_system_name, directory_name + '1') - directory_client1.get_file_client('file0').create_file() - directory_client1.get_file_client('file1').create_file() - directory_client2 = self.dsc.get_directory_client(self.file_system_name, directory_name + '2') - directory_client2.get_file_client('file2').create_file() + directory_client1 = self.dsc.get_directory_client(self.file_system_name, directory_name + "1") + directory_client1.get_file_client("file0").create_file() + directory_client1.get_file_client("file1").create_file() + directory_client2 = self.dsc.get_directory_client(self.file_system_name, directory_name + "2") + directory_client2.get_file_client("file2").create_file() path_response = list(directory_client1.get_paths()) assert len(path_response) == 2 - assert path_response[0]['name'] == directory_name + '1/file0' - assert path_response[1]['name'] == directory_name + '1/file1' + assert path_response[0]["name"] == directory_name + "1/file0" + assert path_response[1]["name"] == directory_name + "1/file1" subdir_client1 = directory_client2.create_sub_directory("subdir1") - subdir_client1.get_file_client('file3').create_file() - subdir_client1.get_file_client('file4').create_file() - subdir_client1.get_file_client('file5').create_file() + subdir_client1.get_file_client("file3").create_file() + subdir_client1.get_file_client("file4").create_file() + subdir_client1.get_file_client("file5").create_file() subdir_client2 = directory_client2.create_sub_directory("subdir2") - subdir_client2.get_file_client('file6').create_file() - subdir_client2.get_file_client('file7').create_file() - subdir_client2.get_file_client('file8').create_file() + subdir_client2.get_file_client("file6").create_file() + subdir_client2.get_file_client("file7").create_file() + subdir_client2.get_file_client("file8").create_file() path_response = list(directory_client2.get_paths(recursive=True, start_from="subdir1/file4", max_results=2)) assert len(path_response) == 6 - assert path_response[0]['name'] == directory_name + "2/subdir1/file4" - assert path_response[1]['name'] == directory_name + "2/subdir1/file5" - assert path_response[2]['name'] == directory_name + "2/subdir2" - assert path_response[3]['name'] == directory_name + "2/subdir2/file6" - assert path_response[4]['name'] == directory_name + "2/subdir2/file7" - assert path_response[5]['name'] == directory_name + "2/subdir2/file8" + assert path_response[0]["name"] == directory_name + "2/subdir1/file4" + assert path_response[1]["name"] == directory_name + "2/subdir1/file5" + assert path_response[2]["name"] == directory_name + "2/subdir2" + assert path_response[3]["name"] == directory_name + "2/subdir2/file6" + assert path_response[4]["name"] == directory_name + "2/subdir2/file7" + assert path_response[5]["name"] == directory_name + "2/subdir2/file8" + # ------------------------------------------------------------------------------ -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_directory_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_directory_async.py index a6bc9d696201..3ebeb9f914be 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_directory_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_directory_async.py @@ -37,12 +37,14 @@ from azure.storage.filedatalake._serialize import _SUPPORTED_API_VERSIONS from azure.storage.filedatalake.aio import DataLakeDirectoryClient, DataLakeServiceClient - # ------------------------------------------------------------------------------ -TEST_DIRECTORY_PREFIX = 'directory' -REMOVE_ACL = "mask," + "default:user,default:group," + \ - "user:ec3595d6-2c17-4696-8caa-7e139758d24a,group:ec3595d6-2c17-4696-8caa-7e139758d24a," + \ - "default:user:ec3595d6-2c17-4696-8caa-7e139758d24a,default:group:ec3595d6-2c17-4696-8caa-7e139758d24a" +TEST_DIRECTORY_PREFIX = "directory" +REMOVE_ACL = ( + "mask," + + "default:user,default:group," + + "user:ec3595d6-2c17-4696-8caa-7e139758d24a,group:ec3595d6-2c17-4696-8caa-7e139758d24a," + + "default:user:ec3595d6-2c17-4696-8caa-7e139758d24a,default:group:ec3595d6-2c17-4696-8caa-7e139758d24a" +) # ------------------------------------------------------------------------------ @@ -50,11 +52,11 @@ class TestDirectoryAsync(AsyncStorageRecordedTestCase): async def _setUp(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=account_key.secret) self.config = self.dsc._config - self.file_system_name = self.get_resource_name('filesystem') + self.file_system_name = self.get_resource_name("filesystem") if not self.is_playback(): file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -86,9 +88,9 @@ async def _create_directory_and_get_directory_client(self, directory_name=None): async def _create_sub_directory_and_files(self, directory_client, num_of_dirs, num_of_files_per_dir): # the name suffix matter since we need to avoid creating the same directories/files in record mode for i in range(0, num_of_dirs): - sub_dir = await directory_client.create_sub_directory(self.get_resource_name('subdir' + str(i))) + sub_dir = await directory_client.create_sub_directory(self.get_resource_name("subdir" + str(i))) for j in range(0, num_of_files_per_dir): - await sub_dir.create_file(self.get_resource_name('subfile' + str(j))) + await sub_dir.create_file(self.get_resource_name("subfile" + str(j))) # --Helpers----------------------------------------------------------------- @@ -101,9 +103,7 @@ async def test_create_directory(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") # Act directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) created = await directory_client.create_directory(content_settings=content_settings) @@ -118,8 +118,8 @@ async def test_create_directory_owner_group_acl_async(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' - test_string_acl = 'user::rwx,group::r-x,other::rwx' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" + test_string_acl = "user::rwx,group::r-x,other::rwx" # Arrange directory_name = self._get_directory_reference() @@ -130,9 +130,9 @@ async def test_create_directory_owner_group_acl_async(self, **kwargs): # Assert acl_properties = await directory_client.get_access_control() assert acl_properties is not None - assert acl_properties['owner'] == test_string - assert acl_properties['group'] == test_string - assert acl_properties['acl'] == test_string_acl + assert acl_properties["owner"] == test_string + assert acl_properties["group"] == test_string + assert acl_properties["acl"] == test_string_acl @DataLakePreparer() @recorded_by_proxy_async @@ -141,7 +141,7 @@ async def test_create_directory_proposed_lease_id_async(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" test_duration = 15 # Arrange directory_name = self._get_directory_reference() @@ -151,9 +151,9 @@ async def test_create_directory_proposed_lease_id_async(self, **kwargs): # Assert properties = await directory_client.get_directory_properties() assert properties is not None - assert properties.lease['status'] == 'locked' - assert properties.lease['state'] == 'leased' - assert properties.lease['duration'] == 'fixed' + assert properties.lease["status"] == "locked" + assert properties.lease["state"] == "leased" + assert properties.lease["duration"] == "fixed" @DataLakePreparer() @recorded_by_proxy_async @@ -162,21 +162,21 @@ async def test_create_sub_directory_proposed_lease_id_async(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" test_duration = 15 # Arrange directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) - directory_client = await directory_client.create_sub_directory(sub_directory='sub1', - lease_id=test_string, - lease_duration=test_duration) + directory_client = await directory_client.create_sub_directory( + sub_directory="sub1", lease_id=test_string, lease_duration=test_duration + ) # Assert properties = await directory_client.get_directory_properties() assert properties is not None - assert properties.lease['status'] == 'locked' - assert properties.lease['state'] == 'leased' - assert properties.lease['duration'] == 'fixed' + assert properties.lease["status"] == "locked" + assert properties.lease["state"] == "leased" + assert properties.lease["duration"] == "fixed" @DataLakePreparer() @recorded_by_proxy_async @@ -206,8 +206,9 @@ async def test_using_oauth_token_credential_to_create_directory(self, **kwargs): directory_name = self._get_directory_reference() token_credential = self.get_credential(DataLakeServiceClient, is_async=True) - directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, - credential=token_credential) + directory_client = DataLakeDirectoryClient( + self.dsc.url, self.file_system_name, directory_name, credential=token_credential + ) response = await directory_client.create_directory() assert response is not None @@ -246,7 +247,7 @@ async def test_create_directory_with_permission(self, **kwargs): # Assert assert created - assert prop['permissions'] == 'rwxr--r--' + assert prop["permissions"] == "rwxr--r--" @DataLakePreparer() @recorded_by_proxy_async @@ -257,9 +258,7 @@ async def test_create_directory_with_content_settings(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") # Act directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) created = await directory_client.create_directory(content_settings=content_settings) @@ -276,7 +275,7 @@ async def test_create_directory_with_metadata(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} # Act directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) created = await directory_client.create_directory(metadata=metadata) @@ -295,7 +294,7 @@ async def test_delete_directory(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory(metadata=metadata) @@ -316,7 +315,7 @@ async def test_delete_directory_with_if_modified_since(self, **kwargs): prop = await directory_client.get_directory_properties() with pytest.raises(ResourceModifiedError): - await directory_client.delete_directory(if_modified_since=prop['last_modified']) + await directory_client.delete_directory(if_modified_since=prop["last_modified"]) @DataLakePreparer() @pytest.mark.live_test_only @@ -328,7 +327,7 @@ async def test_delete_directory_paginated(self, **kwargs): # Set this to object id (not client id) of an AAD app that does not have permission # to storage account through RBAC. # Also make sure oauth settings (TENANT_ID, CLIENT_ID, CLIENT_SECRET) are pointing to this AAD app - object_id = '68bff720-253b-428c-b124-603700654ea9' + object_id = "68bff720-253b-428c-b124-603700654ea9" # Arrange await self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -337,12 +336,12 @@ async def test_delete_directory_paginated(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - for i in range (0, 5020): + for i in range(0, 5020): file_client = directory_client.get_file_client(f"file{i}") await file_client.create_file() root_directory = self.dsc.get_directory_client(self.file_system_name, "/") - acl = (await root_directory.get_access_control())['acl'] + acl = (await root_directory.get_access_control())["acl"] # Add permission for AAD app on root directory new_acl = acl + "," + f"user:{object_id}:rwx" @@ -350,10 +349,7 @@ async def test_delete_directory_paginated(self, **kwargs): token_credential = self.get_credential(DataLakeServiceClient, is_async=True) directory_client_oauth = DataLakeDirectoryClient( - self.dsc.url, - self.file_system_name, - directory_name, - credential=token_credential + self.dsc.url, self.file_system_name, directory_name, credential=token_credential ) # Act @@ -370,19 +366,20 @@ async def test_create_sub_directory_and_delete_sub_directory(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} # Create a directory first, to prepare for creating sub directory directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory(metadata=metadata) # Create sub directory from the current directory - sub_directory_name = 'subdir' + sub_directory_name = "subdir" sub_directory_created = await directory_client.create_sub_directory(sub_directory_name) # to make sure the sub directory was indeed created by get sub_directory properties from sub directory client - sub_directory_client = self.dsc.get_directory_client(self.file_system_name, - directory_name + '/' + sub_directory_name) + sub_directory_client = self.dsc.get_directory_client( + self.file_system_name, directory_name + "/" + sub_directory_name + ) sub_properties = await sub_directory_client.get_directory_properties() # Assert @@ -402,11 +399,11 @@ async def test_set_access_control(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory(metadata=metadata) - response = await directory_client.set_access_control(permissions='0777') + response = await directory_client.set_access_control(permissions="0777") # Assert assert response is not None @@ -418,18 +415,18 @@ async def test_set_access_control_with_acl(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory(metadata=metadata) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" await directory_client.set_access_control(acl=acl) access_control = await directory_client.get_access_control() # Assert assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy_async @@ -442,8 +439,9 @@ async def test_set_access_control_if_none_modified(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) resp = await directory_client.create_directory() - response = await directory_client.set_access_control(permissions='0777', etag=resp['etag'], - match_condition=MatchConditions.IfNotModified) + response = await directory_client.set_access_control( + permissions="0777", etag=resp["etag"], match_condition=MatchConditions.IfNotModified + ) # Assert assert response is not None @@ -455,9 +453,9 @@ async def test_get_access_control(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) - await directory_client.create_directory(metadata=metadata, permissions='0777') + await directory_client.create_directory(metadata=metadata, permissions="0777") # Act response = await directory_client.get_access_control() @@ -473,14 +471,15 @@ async def test_get_access_control_with_match_conditions(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) - resp = await directory_client.create_directory(permissions='0777', umask='0000') + resp = await directory_client.create_directory(permissions="0777", umask="0000") # Act - response = await directory_client.get_access_control(etag=resp['etag'], - match_condition=MatchConditions.IfNotModified) + response = await directory_client.get_access_control( + etag=resp["etag"], match_condition=MatchConditions.IfNotModified + ) # Assert assert response is not None - assert response['permissions'] == 'rwxrwxrwx' + assert response["permissions"] == "rwxrwxrwx" @DataLakePreparer() @recorded_by_proxy_async @@ -496,7 +495,7 @@ async def test_set_access_control_recursive(self, **kwargs): num_file_per_sub_dir = 5 await self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" summary = await directory_client.set_access_control_recursive(acl=acl) # Assert @@ -506,7 +505,7 @@ async def test_set_access_control_recursive(self, **kwargs): assert summary.continuation is None access_control = await directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy_async @@ -528,11 +527,13 @@ def callback(response): response_list.append(response) if len(response_list) == 2: raise ServiceRequestError("network problem") - acl = 'user::rwx,group::r-x,other::rwx' + + acl = "user::rwx,group::r-x,other::rwx" with pytest.raises(AzureError) as acl_error: - await directory_client.set_access_control_recursive(acl=acl, batch_size=2, max_batches=2, - raw_response_hook=callback, retry_total=0) + await directory_client.set_access_control_recursive( + acl=acl, batch_size=2, max_batches=2, raw_response_hook=callback, retry_total=0 + ) assert acl_error.value.continuation_token is not None assert acl_error.value.message == "network problem" assert acl_error.typename == "ServiceRequestError" @@ -551,7 +552,7 @@ async def test_set_access_control_recursive_in_batches(self, **kwargs): num_file_per_sub_dir = 5 await self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" summary = await directory_client.set_access_control_recursive(acl=acl, batch_size=2) # Assert @@ -561,7 +562,7 @@ async def test_set_access_control_recursive_in_batches(self, **kwargs): assert summary.continuation is None access_control = await directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy_async @@ -577,7 +578,7 @@ async def test_set_access_control_recursive_in_batches_with_progress_callback(se num_file_per_sub_dir = 5 await self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) last_response = AccessControlChangeResult(None, "") @@ -588,8 +589,9 @@ async def progress_callback(resp): last_response.counters = resp.aggregate_counters - summary = await directory_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2) + summary = await directory_client.set_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=2 + ) # Assert assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included @@ -604,15 +606,15 @@ async def progress_callback(resp): assert summary.counters.failure_count == last_response.counters.failure_count access_control = await directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @pytest.mark.live_test_only @DataLakePreparer() async def test_set_access_control_recursive_with_failures(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - url = self.account_url(datalake_storage_account_name, 'dfs') - variables = kwargs.pop('variables', {}) + url = self.account_url(datalake_storage_account_name, "dfs") + variables = kwargs.pop("variables", {}) await self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -624,20 +626,24 @@ async def test_set_access_control_recursive_with_failures(self, **kwargs): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory(owner=test_guid) - await self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir1').create_directory( - owner=test_guid, permissions='0777') - await self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir2').create_directory( - owner=test_guid, permissions='0777') - await self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir1/file1').create_file( - owner=test_guid, permissions='0777') - await self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir2/file2').create_file( - owner=test_guid, permissions='0777') - await directory_client.get_file_client('file3').create_file() + await self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir1").create_directory( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir2").create_directory( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir1/file1").create_file( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir2/file2").create_file( + owner=test_guid, permissions="0777" + ) + await directory_client.get_file_client("file3").create_file() # User delegation SAS with provided owner permissions token_credential = self.get_credential(DataLakeServiceClient, is_async=True) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) owner_dsc = DataLakeServiceClient(url, credential=token_credential) user_delegation_key = await owner_dsc.get_user_delegation_key(start_time, expiry_time) sas_token = self.generate_sas( @@ -646,9 +652,9 @@ async def test_set_access_control_recursive_with_failures(self, **kwargs): self.file_system_name, directory_name, user_delegation_key, - permission='racwdlmeop', + permission="racwdlmeop", expiry=expiry_time, - agent_object_id=test_guid + agent_object_id=test_guid, ) if self.is_live: @@ -656,7 +662,7 @@ async def test_set_access_control_recursive_with_failures(self, **kwargs): owner_dir_client = DataLakeDirectoryClient(url, self.file_system_name, directory_name, sas_token) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) failed_entries = [] @@ -666,8 +672,9 @@ async def progress_callback(resp): running_tally.failure_count += resp.batch_counters.failure_count failed_entries.append(resp.batch_failures) - summary = await owner_dir_client.set_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2) + summary = await owner_dir_client.set_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=2 + ) # Assert assert summary.counters.failure_count == 1 @@ -692,7 +699,7 @@ async def test_set_access_control_recursive_in_batches_with_explicit_iteration(s num_file_per_sub_dir = 5 await self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) result = AccessControlChangeResult(None, "") iteration_count = 0 @@ -700,9 +707,9 @@ async def test_set_access_control_recursive_in_batches_with_explicit_iteration(s batch_size = 2 while result.continuation is not None: - result = await directory_client.set_access_control_recursive(acl=acl, batch_size=batch_size, - max_batches=max_batches, - continuation=result.continuation) + result = await directory_client.set_access_control_recursive( + acl=acl, batch_size=batch_size, max_batches=max_batches, continuation=result.continuation + ) running_tally.directories_successful += result.counters.directories_successful running_tally.files_successful += result.counters.files_successful @@ -715,7 +722,7 @@ async def test_set_access_control_recursive_in_batches_with_explicit_iteration(s assert running_tally.failure_count == 0 access_control = await directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy_async @@ -731,7 +738,7 @@ async def test_update_access_control_recursive(self, **kwargs): num_file_per_sub_dir = 5 await self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" summary = await directory_client.update_access_control_recursive(acl=acl) # Assert @@ -740,7 +747,7 @@ async def test_update_access_control_recursive(self, **kwargs): assert summary.counters.failure_count == 0 access_control = await directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy_async @@ -756,7 +763,7 @@ async def test_update_access_control_recursive_in_batches(self, **kwargs): num_file_per_sub_dir = 5 await self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" summary = await directory_client.update_access_control_recursive(acl=acl, batch_size=2) # Assert @@ -765,7 +772,7 @@ async def test_update_access_control_recursive_in_batches(self, **kwargs): assert summary.counters.failure_count == 0 access_control = await directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy_async @@ -781,7 +788,7 @@ async def test_update_access_control_recursive_in_batches_with_progress_callback num_file_per_sub_dir = 5 await self._create_sub_directory_and_files(directory_client, num_sub_dirs, num_file_per_sub_dir) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) last_response = AccessControlChangeResult(None, "") @@ -792,8 +799,9 @@ async def progress_callback(resp): last_response.counters = resp.aggregate_counters - summary = await directory_client.update_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2) + summary = await directory_client.update_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=2 + ) # Assert assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included @@ -804,15 +812,15 @@ async def progress_callback(resp): assert summary.counters.failure_count == running_tally.failure_count access_control = await directory_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @pytest.mark.live_test_only @DataLakePreparer() async def test_update_access_control_recursive_with_failures(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - url = self.account_url(datalake_storage_account_name, 'dfs') - variables = kwargs.pop('variables', {}) + url = self.account_url(datalake_storage_account_name, "dfs") + variables = kwargs.pop("variables", {}) await self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -824,20 +832,24 @@ async def test_update_access_control_recursive_with_failures(self, **kwargs): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory(owner=test_guid) - await self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir1').create_directory( - owner=test_guid, permissions='0777') - await self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir2').create_directory( - owner=test_guid, permissions='0777') - await self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir1/file1').create_file( - owner=test_guid, permissions='0777') - await self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir2/file2').create_file( - owner=test_guid, permissions='0777') - await directory_client.get_file_client('file3').create_file() + await self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir1").create_directory( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir2").create_directory( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir1/file1").create_file( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir2/file2").create_file( + owner=test_guid, permissions="0777" + ) + await directory_client.get_file_client("file3").create_file() # User delegation SAS with provided owner permissions token_credential = self.get_credential(DataLakeServiceClient, is_async=True) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) owner_dsc = DataLakeServiceClient(url, credential=token_credential) user_delegation_key = await owner_dsc.get_user_delegation_key(start_time, expiry_time) sas_token = self.generate_sas( @@ -846,9 +858,9 @@ async def test_update_access_control_recursive_with_failures(self, **kwargs): self.file_system_name, directory_name, user_delegation_key, - permission='racwdlmeop', + permission="racwdlmeop", expiry=expiry_time, - agent_object_id=test_guid + agent_object_id=test_guid, ) if self.is_live: @@ -856,7 +868,7 @@ async def test_update_access_control_recursive_with_failures(self, **kwargs): owner_dir_client = DataLakeDirectoryClient(url, self.file_system_name, directory_name, sas_token) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) failed_entries = [] @@ -867,8 +879,9 @@ async def progress_callback(resp): if resp.batch_failures: failed_entries.append(resp.batch_failures) - summary = await owner_dir_client.update_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2) + summary = await owner_dir_client.update_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=2 + ) # Assert assert summary.counters.failure_count == 1 @@ -884,8 +897,8 @@ async def progress_callback(resp): async def test_update_access_control_recursive_continue_on_failures(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - url = self.account_url(datalake_storage_account_name, 'dfs') - variables = kwargs.pop('variables', {}) + url = self.account_url(datalake_storage_account_name, "dfs") + variables = kwargs.pop("variables", {}) await self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -897,22 +910,26 @@ async def test_update_access_control_recursive_continue_on_failures(self, **kwar directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory(owner=test_guid) - await self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir1').create_directory( - owner=test_guid, permissions='0777') - await self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir2').create_directory( - owner=test_guid, permissions='0777') - await self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir1/file1').create_file( - owner=test_guid, permissions='0777') - await self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir2/file2').create_file( - owner=test_guid, permissions='0777') - - await directory_client.get_file_client('file3').create_file() - await self.dsc.get_directory_client(self.file_system_name, directory_name + '/dir3').create_directory() + await self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir1").create_directory( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir2").create_directory( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir1/file1").create_file( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir2/file2").create_file( + owner=test_guid, permissions="0777" + ) + + await directory_client.get_file_client("file3").create_file() + await self.dsc.get_directory_client(self.file_system_name, directory_name + "/dir3").create_directory() # User delegation SAS with provided owner permissions token_credential = self.get_credential(DataLakeServiceClient, is_async=True) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) owner_dsc = DataLakeServiceClient(url, credential=token_credential) user_delegation_key = await owner_dsc.get_user_delegation_key(start_time, expiry_time) sas_token = self.generate_sas( @@ -921,9 +938,9 @@ async def test_update_access_control_recursive_continue_on_failures(self, **kwar self.file_system_name, directory_name, user_delegation_key, - permission='racwdlmeop', + permission="racwdlmeop", expiry=expiry_time, - agent_object_id=test_guid + agent_object_id=test_guid, ) if self.is_live: @@ -931,7 +948,7 @@ async def test_update_access_control_recursive_continue_on_failures(self, **kwar owner_dir_client = DataLakeDirectoryClient(url, self.file_system_name, directory_name, sas_token) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" running_tally = AccessControlChangeCounters(0, 0, 0) failed_entries = [] @@ -942,8 +959,9 @@ async def progress_callback(resp): if resp.batch_failures: failed_entries.append(resp.batch_failures) - summary = await owner_dir_client.update_access_control_recursive(acl=acl, progress_hook=progress_callback, - batch_size=2, continue_on_failure=True) + summary = await owner_dir_client.update_access_control_recursive( + acl=acl, progress_hook=progress_callback, batch_size=2, continue_on_failure=True + ) # Assert assert summary.counters.failure_count == 2 @@ -1020,9 +1038,9 @@ async def progress_callback(resp): last_response.counters = resp.aggregate_counters - summary = await directory_client.remove_access_control_recursive(acl=REMOVE_ACL, - progress_hook=progress_callback, - batch_size=2) + summary = await directory_client.remove_access_control_recursive( + acl=REMOVE_ACL, progress_hook=progress_callback, batch_size=2 + ) # Assert assert summary.counters.directories_successful == num_sub_dirs + 1 # +1 as the dir itself was also included @@ -1037,8 +1055,8 @@ async def progress_callback(resp): async def test_remove_access_control_recursive_with_failures(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - url = self.account_url(datalake_storage_account_name, 'dfs') - variables = kwargs.pop('variables', {}) + url = self.account_url(datalake_storage_account_name, "dfs") + variables = kwargs.pop("variables", {}) await self._setUp(datalake_storage_account_name, datalake_storage_account_key) @@ -1050,20 +1068,24 @@ async def test_remove_access_control_recursive_with_failures(self, **kwargs): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory(owner=test_guid) - await self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir1').create_directory( - owner=test_guid, permissions='0777') - await self.dsc.get_directory_client(self.file_system_name, directory_name + '/subdir2').create_directory( - owner=test_guid, permissions='0777') - await self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir1/file1').create_file( - owner=test_guid, permissions='0777') - await self.dsc.get_file_client(self.file_system_name, directory_name + '/subdir2/file2').create_file( - owner=test_guid, permissions='0777') - await directory_client.get_file_client('file3').create_file() + await self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir1").create_directory( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_directory_client(self.file_system_name, directory_name + "/subdir2").create_directory( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir1/file1").create_file( + owner=test_guid, permissions="0777" + ) + await self.dsc.get_file_client(self.file_system_name, directory_name + "/subdir2/file2").create_file( + owner=test_guid, permissions="0777" + ) + await directory_client.get_file_client("file3").create_file() # User delegation SAS with provided owner permissions token_credential = self.get_credential(DataLakeServiceClient, is_async=True) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) owner_dsc = DataLakeServiceClient(url, credential=token_credential) user_delegation_key = await owner_dsc.get_user_delegation_key(start_time, expiry_time) sas_token = self.generate_sas( @@ -1072,9 +1094,9 @@ async def test_remove_access_control_recursive_with_failures(self, **kwargs): self.file_system_name, directory_name, user_delegation_key, - permission='racwdlmeop', + permission="racwdlmeop", expiry=expiry_time, - agent_object_id=test_guid + agent_object_id=test_guid, ) if self.is_live: @@ -1093,9 +1115,7 @@ async def progress_callback(resp): failed_entries.append(resp.batch_failures) summary = await owner_dir_client.remove_access_control_recursive( - acl=REMOVE_ACL, - progress_hook=progress_callback, - batch_size=2 + acl=REMOVE_ACL, progress_hook=progress_callback, batch_size=2 ) # Assert @@ -1114,9 +1134,7 @@ async def test_rename_from(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() @@ -1125,12 +1143,13 @@ async def test_rename_from(self, **kwargs): new_directory_client = self.dsc.get_directory_client(self.file_system_name, new_name) - await new_directory_client._rename_path('/' + self.file_system_name + '/' + directory_name, - content_settings=content_settings) + await new_directory_client._rename_path( + "/" + self.file_system_name + "/" + directory_name, content_settings=content_settings + ) properties = await new_directory_client.get_directory_properties() assert properties is not None - assert properties.get('content_settings') is None + assert properties.get("content_settings") is None @DataLakePreparer() @recorded_by_proxy_async @@ -1146,7 +1165,7 @@ async def test_rename_from_a_shorter_directory_to_longer_directory(self, **kwarg new_directory_client = await self._create_directory_and_get_directory_client(directory_name=new_name) new_directory_client = await new_directory_client.create_sub_directory("newsub") - await new_directory_client._rename_path('/' + self.file_system_name + '/' + directory_name) + await new_directory_client._rename_path("/" + self.file_system_name + "/" + directory_name) properties = await new_directory_client.get_directory_properties() assert properties is not None @@ -1175,7 +1194,7 @@ async def test_rename_from_a_directory_in_another_file_system(self, **kwargs): new_directory_client = await new_directory_client.create_sub_directory("newsub") # rename dir1 under filesystem1 to dir2 under filesystem2 - await new_directory_client._rename_path('/' + old_file_system_name + '/' + old_dir_name) + await new_directory_client._rename_path("/" + old_file_system_name + "/" + old_dir_name) properties = await new_directory_client.get_directory_properties() assert properties is not None @@ -1192,7 +1211,7 @@ async def test_rename_from_an_unencoded_directory_in_another_file_system(self, * old_dir_name = "old dir" old_client = self.dsc.get_file_system_client(old_file_system_name) await old_client.create_file_system() - old_dir_client =await old_client.create_directory(old_dir_name) + old_dir_client = await old_client.create_directory(old_dir_name) file_name = "oldfile" await old_dir_client.create_file(file_name) @@ -1205,7 +1224,7 @@ async def test_rename_from_an_unencoded_directory_in_another_file_system(self, * await new_file_system_client.create_directory(new_name) # rename dir1 under filesystem1 to dir2 under filesystem2 - new_directory_client = await old_dir_client.rename_directory('/' + new_file_system_name + '/' + new_name) + new_directory_client = await old_dir_client.rename_directory("/" + new_file_system_name + "/" + new_name) properties = await new_directory_client.get_directory_properties() file_properties = await new_directory_client.get_file_client(file_name).get_file_properties() @@ -1236,7 +1255,8 @@ async def test_rename_to_an_existing_directory_in_another_file_system(self, **kw # rename dir2 under filesystem2 to dir1 under filesystem1 res = await source_directory_client.rename_directory( - '/' + destination_file_system_name + '/' + destination_dir_name) + "/" + destination_file_system_name + "/" + destination_dir_name + ) # the source directory has been renamed to destination directory, so it cannot be found with pytest.raises(HttpResponseError): @@ -1266,12 +1286,13 @@ async def test_rename_with_none_existing_destination_condition_and_source_unmodi # rename dir2 under filesystem2 to a non existing directory under filesystem1, # when dir1 does not exist and dir2 wasn't modified properties = await source_directory_client.get_directory_properties() - etag = properties['etag'] + etag = properties["etag"] res = await source_directory_client.rename_directory( - '/' + destination_file_system_name + '/' + non_existing_dir_name, + "/" + destination_file_system_name + "/" + non_existing_dir_name, match_condition=MatchConditions.IfMissing, source_etag=etag, - source_match_condition=MatchConditions.IfNotModified) + source_match_condition=MatchConditions.IfNotModified, + ) # the source directory has been renamed to destination directory, so it cannot be found with pytest.raises(HttpResponseError): @@ -1299,7 +1320,8 @@ async def test_rename_to_an_non_existing_directory_in_another_file_system(self, # rename dir2 under filesystem2 to dir1 under filesystem1 res = await source_directory_client.rename_directory( - '/' + destination_file_system_name + '/' + non_existing_dir_name) + "/" + destination_file_system_name + "/" + non_existing_dir_name + ) # the source directory has been renamed to destination directory, so it cannot be found with pytest.raises(HttpResponseError): @@ -1319,7 +1341,7 @@ async def test_rename_directory_to_non_empty_directory(self, **kwargs): await dir1.create_sub_directory("subdir") dir2 = await self._create_directory_and_get_directory_client("dir2") - await dir2.rename_directory(dir1.file_system_name + '/' + dir1.path_name) + await dir2.rename_directory(dir1.file_system_name + "/" + dir1.path_name) with pytest.raises(HttpResponseError): await dir2.get_directory_properties() @@ -1344,7 +1366,7 @@ async def test_rename_dir_with_file_system_sas(self, **kwargs): # read the created file which is under root directory dir_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, "olddir", credential=token) await dir_client.create_directory() - new_client = await dir_client.rename_directory(dir_client.file_system_name + '/' + 'newdir') + new_client = await dir_client.rename_directory(dir_client.file_system_name + "/" + "newdir") properties = await new_client.get_directory_properties() assert properties.name == "newdir" @@ -1356,28 +1378,28 @@ async def test_rename_dir_with_file_sas(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - token = generate_directory_sas(self.dsc.account_name, - self.file_system_name, - "olddir", - datalake_storage_account_key.secret, - permission=DirectorySasPermissions(read=True, create=True, write=True, - delete=True, move=True), - expiry=datetime.utcnow() + timedelta(hours=1), - ) - - new_token = generate_directory_sas(self.dsc.account_name, - self.file_system_name, - "newdir", - datalake_storage_account_key.secret, - permission=DirectorySasPermissions(read=True, create=True, write=True, - delete=True), - expiry=datetime.utcnow() + timedelta(hours=1), - ) + token = generate_directory_sas( + self.dsc.account_name, + self.file_system_name, + "olddir", + datalake_storage_account_key.secret, + permission=DirectorySasPermissions(read=True, create=True, write=True, delete=True, move=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) + + new_token = generate_directory_sas( + self.dsc.account_name, + self.file_system_name, + "newdir", + datalake_storage_account_key.secret, + permission=DirectorySasPermissions(read=True, create=True, write=True, delete=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) # read the created file which is under root directory dir_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, "olddir", credential=token) await dir_client.create_directory() - new_client = await dir_client.rename_directory(dir_client.file_system_name+'/'+'newdir'+'?'+new_token) + new_client = await dir_client.rename_directory(dir_client.file_system_name + "/" + "newdir" + "?" + new_token) properties = await new_client.get_directory_properties() assert properties.name == "newdir" @@ -1390,11 +1412,11 @@ async def test_rename_directory_special_chars(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - dir_client = await self._create_directory_and_get_directory_client('olddir') - new_client = await dir_client.rename_directory(dir_client.file_system_name + '/' + '?!@#$%^&*.?test') + dir_client = await self._create_directory_and_get_directory_client("olddir") + new_client = await dir_client.rename_directory(dir_client.file_system_name + "/" + "?!@#$%^&*.?test") new_props = await new_client.get_directory_properties() - assert new_props.name == '?!@#$%^&*.?test' + assert new_props.name == "?!@#$%^&*.?test" @DataLakePreparer() @recorded_by_proxy_async @@ -1405,7 +1427,7 @@ async def test_get_properties(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_name = self._get_directory_reference() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory(metadata=metadata) @@ -1413,7 +1435,7 @@ async def test_get_properties(self, **kwargs): # Assert assert properties assert properties.metadata is not None - assert properties.metadata['hello'] == metadata['hello'] + assert properties.metadata["hello"] == metadata["hello"] @DataLakePreparer() @recorded_by_proxy_async @@ -1422,11 +1444,11 @@ async def test_directory_encryption_scope_from_file_system_async(self, **kwargs) datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=datalake_storage_account_key.secret, logging_enable=True) self.config = self.dsc._config - self.file_system_name = self.get_resource_name('filesystem') - dir_name = 'testdir' + self.file_system_name = self.get_resource_name("filesystem") + dir_name = "testdir" file_system = self.dsc.get_file_system_client(self.file_system_name) encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") @@ -1438,8 +1460,8 @@ async def test_directory_encryption_scope_from_file_system_async(self, **kwargs) # Assert assert props - assert props['encryption_scope'] is not None - assert props['encryption_scope'] == encryption_scope.default_encryption_scope + assert props["encryption_scope"] is not None + assert props["encryption_scope"] == encryption_scope.default_encryption_scope @pytest.mark.live_test_only @DataLakePreparer() @@ -1463,8 +1485,9 @@ async def test_using_directory_sas_to_read(self, **kwargs): expiry=datetime.utcnow() + timedelta(hours=1), ) - directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, - credential=token) + directory_client = DataLakeDirectoryClient( + self.dsc.url, self.file_system_name, directory_name, credential=token + ) access_control = await directory_client.get_access_control() assert access_control is not None @@ -1488,8 +1511,9 @@ async def test_using_directory_sas_to_create(self, **kwargs): permission=DirectorySasPermissions(create=True), expiry=datetime.utcnow() + timedelta(hours=1), ) - directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, - credential=token) + directory_client = DataLakeDirectoryClient( + self.dsc.url, self.file_system_name, directory_name, credential=token + ) response = await directory_client.create_directory() assert response is not None @@ -1501,20 +1525,22 @@ async def test_using_directory_sas_to_create_file(self, **kwargs): newest_api_version = _SUPPORTED_API_VERSIONS[-1] - service_client = DataLakeServiceClient("https://abc.dfs.core.windows.net", credential='fake') + service_client = DataLakeServiceClient("https://abc.dfs.core.windows.net", credential="fake") filesys_client = service_client.get_file_system_client("filesys") - dir_client = DataLakeDirectoryClient("https://abc.dfs.core.windows.net", "filesys", "dir", credential='fake') + dir_client = DataLakeDirectoryClient("https://abc.dfs.core.windows.net", "filesys", "dir", credential="fake") file_client = dir_client.get_file_client("file") assert service_client.api_version == newest_api_version assert filesys_client.api_version == newest_api_version assert dir_client.api_version == newest_api_version assert file_client.api_version == newest_api_version - service_client2 = DataLakeServiceClient("https://abc.dfs.core.windows.net", credential='fake', - api_version="2019-02-02") + service_client2 = DataLakeServiceClient( + "https://abc.dfs.core.windows.net", credential="fake", api_version="2019-02-02" + ) filesys_client2 = service_client2.get_file_system_client("filesys") - dir_client2 = DataLakeDirectoryClient("https://abc.dfs.core.windows.net", "filesys", "dir", credential='fake', - api_version="2019-02-02") + dir_client2 = DataLakeDirectoryClient( + "https://abc.dfs.core.windows.net", "filesys", "dir", credential="fake", api_version="2019-02-02" + ) file_client2 = dir_client2.get_file_client("file") assert service_client2.api_version == "2019-02-02" assert filesys_client2.api_version == "2019-02-02" @@ -1537,14 +1563,16 @@ async def test_storage_account_audience_dir_client(self, **kwargs): # Act token_credential = self.get_credential(DataLakeServiceClient, is_async=True) directory_client = DataLakeDirectoryClient( - self.dsc.url, self.file_system_name, directory_name, + self.dsc.url, + self.file_system_name, + directory_name, credential=token_credential, - audience=f'https://{datalake_storage_account_name}.blob.core.windows.net/' + audience=f"https://{datalake_storage_account_name}.blob.core.windows.net/", ) # Assert response1 = directory_client.exists() - response2 = directory_client.create_sub_directory('testsubdir') + response2 = directory_client.create_sub_directory("testsubdir") assert response1 is not None assert response2 is not None @@ -1564,13 +1592,16 @@ async def test_bad_audience_dir_client(self, **kwargs): # Act token_credential = self.get_credential(DataLakeServiceClient, is_async=True) directory_client = DataLakeDirectoryClient( - self.dsc.url, self.file_system_name, directory_name, - credential=token_credential, audience='https://badaudience.blob.core.windows.net/' + self.dsc.url, + self.file_system_name, + directory_name, + credential=token_credential, + audience="https://badaudience.blob.core.windows.net/", ) # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge await directory_client.exists() - await directory_client.create_sub_directory('testsubdir') + await directory_client.create_sub_directory("testsubdir") @DataLakePreparer() @recorded_by_proxy_async @@ -1580,41 +1611,42 @@ async def test_directory_get_paths(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) directory_name = self._get_directory_reference() - directory_client1 = self.dsc.get_directory_client(self.file_system_name, directory_name + '1') - await directory_client1.get_file_client('file0').create_file() - await directory_client1.get_file_client('file1').create_file() - directory_client2 = self.dsc.get_directory_client(self.file_system_name, directory_name + '2') - directory_client2.get_file_client('file2').create_file() + directory_client1 = self.dsc.get_directory_client(self.file_system_name, directory_name + "1") + await directory_client1.get_file_client("file0").create_file() + await directory_client1.get_file_client("file1").create_file() + directory_client2 = self.dsc.get_directory_client(self.file_system_name, directory_name + "2") + directory_client2.get_file_client("file2").create_file() path_response = [] async for path in directory_client1.get_paths(): path_response.append(path) assert len(path_response) == 2 - assert path_response[0]['name'] == directory_name + '1/file0' - assert path_response[1]['name'] == directory_name + '1/file1' + assert path_response[0]["name"] == directory_name + "1/file0" + assert path_response[1]["name"] == directory_name + "1/file1" subdir_client1 = await directory_client2.create_sub_directory("subdir1") - await subdir_client1.get_file_client('file3').create_file() - await subdir_client1.get_file_client('file4').create_file() - await subdir_client1.get_file_client('file5').create_file() + await subdir_client1.get_file_client("file3").create_file() + await subdir_client1.get_file_client("file4").create_file() + await subdir_client1.get_file_client("file5").create_file() subdir_client2 = await directory_client2.create_sub_directory("subdir2") - await subdir_client2.get_file_client('file6').create_file() - await subdir_client2.get_file_client('file7').create_file() - await subdir_client2.get_file_client('file8').create_file() + await subdir_client2.get_file_client("file6").create_file() + await subdir_client2.get_file_client("file7").create_file() + await subdir_client2.get_file_client("file8").create_file() path_response = [] async for path in directory_client2.get_paths(recursive=True, start_from="subdir1/file4", max_results=2): path_response.append(path) assert len(path_response) == 6 - assert path_response[0]['name'] == directory_name + "2/subdir1/file4" - assert path_response[1]['name'] == directory_name + "2/subdir1/file5" - assert path_response[2]['name'] == directory_name + "2/subdir2" - assert path_response[3]['name'] == directory_name + "2/subdir2/file6" - assert path_response[4]['name'] == directory_name + "2/subdir2/file7" - assert path_response[5]['name'] == directory_name + "2/subdir2/file8" + assert path_response[0]["name"] == directory_name + "2/subdir1/file4" + assert path_response[1]["name"] == directory_name + "2/subdir1/file5" + assert path_response[2]["name"] == directory_name + "2/subdir2" + assert path_response[3]["name"] == directory_name + "2/subdir2/file6" + assert path_response[4]["name"] == directory_name + "2/subdir2/file7" + assert path_response[5]["name"] == directory_name + "2/subdir2/file8" + # ------------------------------------------------------------------------------ -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_file.py b/sdk/storage/azure-storage-file-datalake/tests/test_file.py index 8aad8143c6aa..d52b6fdce6f6 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_file.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_file.py @@ -42,20 +42,19 @@ ResourceTypes, ) - # ------------------------------------------------------------------------------ -TEST_DIRECTORY_PREFIX = 'directory' -TEST_FILE_PREFIX = 'file' -TEST_FILE_SYSTEM_PREFIX = 'filesystem' +TEST_DIRECTORY_PREFIX = "directory" +TEST_FILE_PREFIX = "file" +TEST_FILE_SYSTEM_PREFIX = "filesystem" # ------------------------------------------------------------------------------ class TestFile(StorageRecordedTestCase): def _setUp(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=account_key.secret, logging_enable=True) self.config = self.dsc._config - self.file_system_name = self.get_resource_name('filesystem') + self.file_system_name = self.get_resource_name("filesystem") if not self.is_playback(): file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -91,7 +90,7 @@ def _create_file_and_return_client(self, directory="", file=None): self._create_directory_and_return_client(directory) if not file: file = self._get_file_reference() - file_client = self.dsc.get_file_client(self.file_system_name, directory + '/' + file) + file_client = self.dsc.get_file_client(self.file_system_name, directory + "/" + file) file_client.create_file() return file_client @@ -120,7 +119,7 @@ def test_create_file(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") response = file_client.create_file() # Assert @@ -133,8 +132,8 @@ def test_create_file_owner_group_acl(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' - test_string_acl = 'user::rwx,group::r-x,other::rwx' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" + test_string_acl = "user::rwx,group::r-x,other::rwx" # Arrange directory_name = self._get_directory_reference() @@ -142,15 +141,15 @@ def test_create_file_owner_group_acl(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file(owner=test_string, group=test_string, acl=test_string_acl) # Assert acl_properties = file_client.get_access_control() assert acl_properties is not None - assert acl_properties['owner'] == test_string - assert acl_properties['group'] == test_string - assert acl_properties['acl'] == test_string_acl + assert acl_properties["owner"] == test_string + assert acl_properties["group"] == test_string + assert acl_properties["acl"] == test_string_acl @DataLakePreparer() @recorded_by_proxy @@ -159,7 +158,7 @@ def test_create_file_proposed_lease_id(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" test_duration = 15 # Arrange directory_name = self._get_directory_reference() @@ -168,15 +167,15 @@ def test_create_file_proposed_lease_id(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file(lease_id=test_string, lease_duration=test_duration) # Assert properties = file_client.get_file_properties() assert properties is not None - assert properties.lease['status'] == 'locked' - assert properties.lease['state'] == 'leased' - assert properties.lease['duration'] == 'fixed' + assert properties.lease["status"] == "locked" + assert properties.lease["state"] == "leased" + assert properties.lease["duration"] == "fixed" @DataLakePreparer() @recorded_by_proxy @@ -193,14 +192,14 @@ def test_create_file_relative_expiry(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file(expires_on=test_expiry_time) # Assert file_properties = file_client.get_file_properties() - expiry_time = file_properties['expiry_time'] + expiry_time = file_properties["expiry_time"] expiry_time = expiry_time.replace(tzinfo=None) # Strip timezone info to be able to compare - creation_time = file_properties['creation_time'] + creation_time = file_properties["creation_time"] creation_time = creation_time.replace(tzinfo=None) # Strip timezone info to be able to compare assert file_properties is not None assert self._is_almost_equal(expiry_time, creation_time + timedelta(days=1), timedelta(seconds=60)) is True @@ -220,12 +219,12 @@ def test_create_file_absolute_expiry(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file(expires_on=test_expiry_time) # Assert file_properties = file_client.get_file_properties() - expiry_time = file_properties['expiry_time'] + expiry_time = file_properties["expiry_time"] expiry_time = expiry_time.replace(tzinfo=None) # Strip timezone info to be able to compare assert file_properties is not None assert self._is_almost_equal(expiry_time, test_expiry_time, timedelta(seconds=1)) is True @@ -240,10 +239,13 @@ def test_create_file_extra_backslashes(self, **kwargs): # Arrange file_client = self._create_file_and_return_client() - new_file_client = DataLakeFileClient(self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name + '/', - '/' + file_client.path_name, - credential=datalake_storage_account_key.secret, logging_enable=True) + new_file_client = DataLakeFileClient( + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name + "/", + "/" + file_client.path_name, + credential=datalake_storage_account_key.secret, + logging_enable=True, + ) response = new_file_client.create_file() # Assert @@ -262,8 +264,8 @@ def test_file_exists(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client1 = directory_client.get_file_client('filename') - file_client2 = directory_client.get_file_client('nonexistentfile') + file_client1 = directory_client.get_file_client("filename") + file_client2 = directory_client.get_file_client("nonexistentfile") file_client1.create_file() assert file_client1.exists() @@ -281,8 +283,7 @@ def test_create_file_using_oauth_token_credential(self, **kwargs): token_credential = self.get_credential(DataLakeServiceClient) # Create a directory to put the file under that - file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, file_name, - credential=token_credential) + file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, file_name, credential=token_credential) response = file_client.create_file() @@ -316,17 +317,17 @@ def test_create_file_with_lease_id(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") # Act file_client.create_file() - lease = file_client.acquire_lease(lease_id='00000000-1111-2222-3333-444444444444') + lease = file_client.acquire_lease(lease_id="00000000-1111-2222-3333-444444444444") create_resp = file_client.create_file(lease=lease) # Assert file_properties = file_client.get_file_properties() assert file_properties is not None - assert file_properties.etag == create_resp.get('etag') - assert file_properties.last_modified == create_resp.get('last_modified') + assert file_properties.etag == create_resp.get("etag") + assert file_properties.last_modified == create_resp.get("last_modified") @DataLakePreparer() @recorded_by_proxy @@ -357,11 +358,11 @@ def test_append_data(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file() # Act - response = file_client.append_data(b'abc', 0, 3) + response = file_client.append_data(b"abc", 0, 3) assert response is not None @DataLakePreparer() @@ -377,39 +378,39 @@ def test_append_data_lease_action(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file() - data = b'Hello world' - lease_id = '670d43d1-ecde-4ae9-9c37-d22d340e7719' + data = b"Hello world" + lease_id = "670d43d1-ecde-4ae9-9c37-d22d340e7719" # Act / Assert # ---Acquire--- - file_client.append_data(data, 0, len(data), lease_action='acquire', lease_duration=30, lease=lease_id) + file_client.append_data(data, 0, len(data), lease_action="acquire", lease_duration=30, lease=lease_id) lease = file_client.get_file_properties().lease - assert lease.state == 'leased' - assert lease.duration == 'fixed' + assert lease.state == "leased" + assert lease.duration == "fixed" # ---Renew--- - file_client.append_data(data, 0, len(data), lease_action='auto-renew', lease=lease_id) + file_client.append_data(data, 0, len(data), lease_action="auto-renew", lease=lease_id) lease = file_client.get_file_properties().lease - assert lease.state == 'leased' - assert lease.duration == 'fixed' + assert lease.state == "leased" + assert lease.duration == "fixed" # ---Release--- - file_client.append_data(data, 0, len(data), flush=True, lease_action='release', lease=lease_id) + file_client.append_data(data, 0, len(data), flush=True, lease_action="release", lease=lease_id) lease = file_client.get_file_properties().lease - assert lease.state == 'available' + assert lease.state == "available" assert not lease.duration # ---Acquire and release--- - file_client.append_data(data, 0, len(data), flush=True, lease_action='acquire-release', lease=lease_id) + file_client.append_data(data, 0, len(data), flush=True, lease_action="acquire-release", lease=lease_id) lease = file_client.get_file_properties().lease - assert lease.state == 'available' + assert lease.state == "available" assert not lease.duration @DataLakePreparer() @@ -425,7 +426,7 @@ def test_append_empty_data(self, **kwargs): file_client.flush_data(0) file_props = file_client.get_file_properties() - assert file_props['size'] == 0 + assert file_props["size"] == 0 @DataLakePreparer() @recorded_by_proxy @@ -440,17 +441,17 @@ def test_flush_data(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file() # Act - file_client.append_data(b'abc', 0, 3) + file_client.append_data(b"abc", 0, 3) response = file_client.flush_data(3) # Assert prop = file_client.get_file_properties() assert response is not None - assert prop['size'] == 3 + assert prop["size"] == 3 @DataLakePreparer() @recorded_by_proxy @@ -465,43 +466,43 @@ def test_flush_data_lease_action(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file() - data = b'Hello world' - lease_id = 'c8107e94-ab42-42ac-92d6-6458764982af' + data = b"Hello world" + lease_id = "c8107e94-ab42-42ac-92d6-6458764982af" # Act / Assert # ---Acquire--- file_client.append_data(data, 0, len(data)) - file_client.flush_data(len(data), lease_action='acquire', lease_duration=30, lease=lease_id) + file_client.flush_data(len(data), lease_action="acquire", lease_duration=30, lease=lease_id) lease = file_client.get_file_properties().lease - assert lease.state == 'leased' - assert lease.duration == 'fixed' + assert lease.state == "leased" + assert lease.duration == "fixed" # ---Renew--- file_client.append_data(data, 0, len(data), lease=lease_id) - file_client.flush_data(len(data), lease_action='auto-renew', lease=lease_id) + file_client.flush_data(len(data), lease_action="auto-renew", lease=lease_id) lease = file_client.get_file_properties().lease - assert lease.state == 'leased' - assert lease.duration == 'fixed' + assert lease.state == "leased" + assert lease.duration == "fixed" # ---Release--- file_client.append_data(data, 0, len(data), lease=lease_id) - file_client.flush_data(len(data), lease_action='release', lease=lease_id) + file_client.flush_data(len(data), lease_action="release", lease=lease_id) lease = file_client.get_file_properties().lease - assert lease.state == 'available' + assert lease.state == "available" assert not lease.duration # ---Acquire and release--- file_client.append_data(data, 0, len(data)) - file_client.flush_data(len(data), lease_action='acquire-release', lease=lease_id) + file_client.flush_data(len(data), lease_action="acquire-release", lease=lease_id) lease = file_client.get_file_properties().lease - assert lease.state == 'available' + assert lease.state == "available" assert not lease.duration @DataLakePreparer() @@ -517,16 +518,16 @@ def test_flush_data_with_bool(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file() # Act - response = file_client.append_data(b'abc', 0, 3, flush=True) + response = file_client.append_data(b"abc", 0, 3, flush=True) # Assert prop = file_client.get_file_properties() assert response is not None - assert prop['size'] == 3 + assert prop["size"] == 3 @DataLakePreparer() @recorded_by_proxy @@ -541,19 +542,19 @@ def test_flush_data_with_match_condition(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") resp = file_client.create_file() # Act - file_client.append_data(b'abc', 0, 3) + file_client.append_data(b"abc", 0, 3) # flush is successful because it isn't touched - response = file_client.flush_data(3, etag=resp['etag'], match_condition=MatchConditions.IfNotModified) + response = file_client.flush_data(3, etag=resp["etag"], match_condition=MatchConditions.IfNotModified) - file_client.append_data(b'abc', 3, 3) + file_client.append_data(b"abc", 3, 3) with pytest.raises(ResourceModifiedError): # flush is unsuccessful because extra data were appended. - file_client.flush_data(6, etag=resp['etag'], match_condition=MatchConditions.IfNotModified) + file_client.flush_data(6, etag=resp["etag"], match_condition=MatchConditions.IfNotModified) @DataLakePreparer() @recorded_by_proxy @@ -569,8 +570,8 @@ def test_upload_data_to_none_existing_file(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') - data = self.get_random_bytes(200*1024) + file_client = directory_client.get_file_client("filename") + data = self.get_random_bytes(200 * 1024) file_client.upload_data(data, overwrite=True, max_concurrency=3) downloaded_data = file_client.download_file().readall() @@ -589,16 +590,16 @@ def test_upload_data_in_substreams(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") # Get 16MB data - data = self.get_random_bytes(16*1024*1024) + data = self.get_random_bytes(16 * 1024 * 1024) # Ensure chunk size is greater than threshold (8MB > 4MB) - for optimized upload - file_client.upload_data(data, chunk_size=8*1024*1024, overwrite=True, max_concurrency=3) + file_client.upload_data(data, chunk_size=8 * 1024 * 1024, overwrite=True, max_concurrency=3) downloaded_data = file_client.download_file().readall() assert data == downloaded_data # Run on single thread - file_client.upload_data(data, chunk_size=8*1024*1024, overwrite=True) + file_client.upload_data(data, chunk_size=8 * 1024 * 1024, overwrite=True) downloaded_data = file_client.download_file().readall() assert data == downloaded_data @@ -616,7 +617,7 @@ def test_upload_data_to_existing_file(self, **kwargs): directory_client.create_directory() # create an existing file - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file() file_client.append_data(b"abc", 0) file_client.flush_data(3) @@ -644,20 +645,15 @@ def test_upload_data_to_existing_file_with_content_settings(self, **kwargs): directory_client.create_directory() # create an existing file - file_client = directory_client.get_file_client('filename') - etag = file_client.create_file()['etag'] + file_client = directory_client.get_file_client("filename") + etag = file_client.create_file()["etag"] # to override the existing file data = self.get_random_bytes(100) - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") file_client.upload_data( - data, - content_settings=content_settings, - etag=etag, - match_condition=MatchConditions.IfNotModified + data, content_settings=content_settings, etag=etag, match_condition=MatchConditions.IfNotModified ) downloaded_data = file_client.download_file().readall() @@ -680,8 +676,8 @@ def test_upload_data_to_existing_file_with_permission_and_umask(self, **kwargs): directory_client.create_directory() # create an existing file - file_client = directory_client.get_file_client('filename') - etag = file_client.create_file()['etag'] + file_client = directory_client.get_file_client("filename") + etag = file_client.create_file()["etag"] # to override the existing file data = self.get_random_bytes(100) @@ -689,10 +685,10 @@ def test_upload_data_to_existing_file_with_permission_and_umask(self, **kwargs): file_client.upload_data( data, overwrite=True, - permissions='0777', + permissions="0777", umask="0000", etag=etag, - match_condition=MatchConditions.IfNotModified + match_condition=MatchConditions.IfNotModified, ) downloaded_data = file_client.download_file().readall() @@ -700,7 +696,7 @@ def test_upload_data_to_existing_file_with_permission_and_umask(self, **kwargs): # Assert assert data == downloaded_data - assert prop['permissions'] == 'rwxrwxrwx' + assert prop["permissions"] == "rwxrwxrwx" @DataLakePreparer() @recorded_by_proxy @@ -715,7 +711,7 @@ def test_upload_data_with_none_max_concurrency(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") data = self.get_random_bytes(100) # max_concurrency=None should not raise TypeError file_client.upload_data(data, overwrite=True, max_concurrency=None) @@ -778,27 +774,30 @@ def test_read_file_with_user_delegation_key(self, **kwargs): # Get user delegation key token_credential = self.get_credential(DataLakeServiceClient) service_client = DataLakeServiceClient( - self.account_url(datalake_storage_account_name, 'dfs'), - credential=token_credential, - logging_enable=True + self.account_url(datalake_storage_account_name, "dfs"), credential=token_credential, logging_enable=True + ) + user_delegation_key = service_client.get_user_delegation_key( + datetime.utcnow(), datetime.utcnow() + timedelta(hours=1) + ) + + sas_token = generate_file_sas( + file_client.account_name, + file_client.file_system_name, + None, + file_client.path_name, + user_delegation_key, + permission=FileSasPermissions(read=True, create=True, write=True, delete=True), + expiry=datetime.utcnow() + timedelta(hours=1), ) - user_delegation_key = service_client.get_user_delegation_key(datetime.utcnow(), - datetime.utcnow() + timedelta(hours=1)) - - sas_token = generate_file_sas(file_client.account_name, - file_client.file_system_name, - None, - file_client.path_name, - user_delegation_key, - permission=FileSasPermissions(read=True, create=True, write=True, delete=True), - expiry=datetime.utcnow() + timedelta(hours=1), - ) # download the data and make sure it is the same as uploaded data - new_file_client = DataLakeFileClient(self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name, - file_client.path_name, - credential=sas_token, logging_enable=True) + new_file_client = DataLakeFileClient( + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name, + file_client.path_name, + credential=sas_token, + logging_enable=True, + ) downloaded_data = new_file_client.download_file().readall() assert data == downloaded_data @@ -821,33 +820,35 @@ def test_set_acl_with_user_delegation_key(self, **kwargs): # Get user delegation key token_credential = self.get_credential(DataLakeServiceClient) service_client = DataLakeServiceClient( - self.account_url(datalake_storage_account_name, 'dfs'), - credential=token_credential + self.account_url(datalake_storage_account_name, "dfs"), credential=token_credential + ) + user_delegation_key = service_client.get_user_delegation_key( + datetime.utcnow(), datetime.utcnow() + timedelta(hours=1) + ) + + sas_token = generate_file_sas( + file_client.account_name, + file_client.file_system_name, + None, + file_client.path_name, + user_delegation_key, + permission=FileSasPermissions(execute=True, manage_access_control=True, manage_ownership=True), + expiry=datetime.utcnow() + timedelta(hours=1), ) - user_delegation_key = service_client.get_user_delegation_key(datetime.utcnow(), - datetime.utcnow() + timedelta(hours=1)) - - sas_token = generate_file_sas(file_client.account_name, - file_client.file_system_name, - None, - file_client.path_name, - user_delegation_key, - permission=FileSasPermissions(execute=True, manage_access_control=True, - manage_ownership=True), - expiry=datetime.utcnow() + timedelta(hours=1), - ) # download the data and make sure it is the same as uploaded data - new_file_client = DataLakeFileClient(self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name, - file_client.path_name, - credential=sas_token) - acl = 'user::rwx,group::r-x,other::rwx' + new_file_client = DataLakeFileClient( + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name, + file_client.path_name, + credential=sas_token, + ) + acl = "user::rwx,group::r-x,other::rwx" owner = "dc140949-53b7-44af-b1e9-cd994951fb86" new_file_client.set_access_control(acl=acl, owner=owner) access_control = new_file_client.get_access_control() - assert acl == access_control['acl'] - assert owner == access_control['owner'] + assert acl == access_control["acl"] + assert owner == access_control["owner"] @pytest.mark.live_test_only @DataLakePreparer() @@ -864,36 +865,38 @@ def test_preauthorize_user_with_user_delegation_key(self, **kwargs): # Upload data to file file_client.append_data(data, 0, len(data)) file_client.flush_data(len(data)) - file_client.set_access_control(owner="68390a19-a643-458b-b726-408abf67b4fc", permissions='0777') + file_client.set_access_control(owner="68390a19-a643-458b-b726-408abf67b4fc", permissions="0777") acl = file_client.get_access_control() # Get user delegation key token_credential = self.get_credential(DataLakeServiceClient) service_client = DataLakeServiceClient( - self.account_url(datalake_storage_account_name, 'dfs'), - credential=token_credential + self.account_url(datalake_storage_account_name, "dfs"), credential=token_credential + ) + user_delegation_key = service_client.get_user_delegation_key( + datetime.utcnow(), datetime.utcnow() + timedelta(hours=1) + ) + + sas_token = generate_file_sas( + file_client.account_name, + file_client.file_system_name, + None, + file_client.path_name, + user_delegation_key, + permission=FileSasPermissions(read=True, write=True, manage_access_control=True, manage_ownership=True), + expiry=datetime.utcnow() + timedelta(hours=1), + preauthorized_agent_object_id="68390a19-a643-458b-b726-408abf67b4fc", ) - user_delegation_key = service_client.get_user_delegation_key(datetime.utcnow(), - datetime.utcnow() + timedelta(hours=1)) - - sas_token = generate_file_sas(file_client.account_name, - file_client.file_system_name, - None, - file_client.path_name, - user_delegation_key, - permission=FileSasPermissions(read=True, write=True, manage_access_control=True, - manage_ownership=True), - expiry=datetime.utcnow() + timedelta(hours=1), - preauthorized_agent_object_id="68390a19-a643-458b-b726-408abf67b4fc" - ) # download the data and make sure it is the same as uploaded data - new_file_client = DataLakeFileClient(self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name, - file_client.path_name, - credential=sas_token) + new_file_client = DataLakeFileClient( + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name, + file_client.path_name, + credential=sas_token, + ) - acl = new_file_client.set_access_control(permissions='0777') + acl = new_file_client.set_access_control(permissions="0777") assert acl is not None @DataLakePreparer() @@ -941,7 +944,6 @@ def test_read_file_to_text(self, **kwargs): # Assert assert data == downloaded_data - @pytest.mark.live_test_only @DataLakePreparer() def test_account_sas(self, **kwargs): @@ -985,10 +987,7 @@ def test_account_sas_raises_if_sas_already_in_uri(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) with pytest.raises(ValueError): DataLakeFileClient( - self.dsc.url + "?sig=foo", - self.file_system_name, - "foo", - credential=AzureSasCredential("?foo=bar") + self.dsc.url + "?sig=foo", self.file_system_name, "foo", credential=AzureSasCredential("?foo=bar") ) @pytest.mark.live_test_only @@ -1015,8 +1014,9 @@ def test_file_sas_only_applies_to_file_level(self, **kwargs): ) # read the created file which is under root directory - file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, directory_name+'/'+file_name, - credential=token) + file_client = DataLakeFileClient( + self.dsc.url, self.file_system_name, directory_name + "/" + file_name, credential=token + ) properties = file_client.get_file_properties() # make sure we can read the file properties @@ -1032,8 +1032,9 @@ def test_file_sas_only_applies_to_file_level(self, **kwargs): file_system_client.get_file_system_properties() # the token is for file level, so users are not supposed to have access to directory level operations - directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, - credential=token) + directory_client = DataLakeDirectoryClient( + self.dsc.url, self.file_system_name, directory_name, credential=token + ) with pytest.raises(ClientAuthenticationError): directory_client.get_directory_properties() @@ -1064,11 +1065,7 @@ def test_delete_file_oauth(self, **kwargs): file_name = self._get_file_reference() token_credential = self.get_credential(DataLakeServiceClient) - file_client = DataLakeFileClient( - self.dsc.url, - self.file_system_name, - file_name, - credential=token_credential) + file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, file_name, credential=token_credential) file_client.create_file() # Act @@ -1088,7 +1085,7 @@ def test_delete_file_with_if_unmodified_since(self, **kwargs): file_client = self._create_file_and_return_client() prop = file_client.get_file_properties() - file_client.delete_file(if_unmodified_since=prop['last_modified']) + file_client.delete_file(if_unmodified_since=prop["last_modified"]) # Make sure the file was deleted with pytest.raises(ResourceNotFoundError): @@ -1103,7 +1100,7 @@ def test_set_access_control(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = self._create_file_and_return_client() - response = file_client.set_access_control(permissions='0777') + response = file_client.set_access_control(permissions="0777") # Assert assert response is not None @@ -1118,7 +1115,7 @@ def test_set_access_control_with_match_conditions(self, **kwargs): file_client = self._create_file_and_return_client() with pytest.raises(ResourceModifiedError): - file_client.set_access_control(permissions='0777', match_condition=MatchConditions.IfMissing) + file_client.set_access_control(permissions="0777", match_condition=MatchConditions.IfMissing) @DataLakePreparer() @recorded_by_proxy @@ -1128,7 +1125,7 @@ def test_get_access_control(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = self._create_file_and_return_client() - file_client.set_access_control(permissions='0777') + file_client.set_access_control(permissions="0777") # Act response = file_client.get_access_control() @@ -1144,12 +1141,12 @@ def test_get_access_control_with_if_modified_since(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = self._create_file_and_return_client() - file_client.set_access_control(permissions='0777') + file_client.set_access_control(permissions="0777") prop = file_client.get_file_properties() # Act - response = file_client.get_access_control(if_modified_since=prop['last_modified']-timedelta(minutes=15)) + response = file_client.get_access_control(if_modified_since=prop["last_modified"] - timedelta(minutes=15)) # Assert assert response is not None @@ -1161,7 +1158,7 @@ def test_set_access_control_recursive(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" file_client = self._create_file_and_return_client() summary = file_client.set_access_control_recursive(acl=acl) @@ -1172,7 +1169,7 @@ def test_set_access_control_recursive(self, **kwargs): assert summary.counters.failure_count == 0 access_control = file_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -1181,7 +1178,7 @@ def test_update_access_control_recursive(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" file_client = self._create_file_and_return_client() summary = file_client.update_access_control_recursive(acl=acl) @@ -1192,7 +1189,7 @@ def test_update_access_control_recursive(self, **kwargs): assert summary.counters.failure_count == 0 access_control = file_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -1201,9 +1198,12 @@ def test_remove_access_control_recursive(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - acl = "mask," + "default:user,default:group," + \ - "user:ec3595d6-2c17-4696-8caa-7e139758d24a,group:ec3595d6-2c17-4696-8caa-7e139758d24a," + \ - "default:user:ec3595d6-2c17-4696-8caa-7e139758d24a,default:group:ec3595d6-2c17-4696-8caa-7e139758d24a" + acl = ( + "mask," + + "default:user,default:group," + + "user:ec3595d6-2c17-4696-8caa-7e139758d24a,group:ec3595d6-2c17-4696-8caa-7e139758d24a," + + "default:user:ec3595d6-2c17-4696-8caa-7e139758d24a,default:group:ec3595d6-2c17-4696-8caa-7e139758d24a" + ) file_client = self._create_file_and_return_client() summary = file_client.remove_access_control_recursive(acl=acl) @@ -1222,10 +1222,8 @@ def test_get_properties(self, **kwargs): # Arrange directory_client = self._create_directory_and_return_client() - metadata = {'hello': 'world', 'number': '42'} - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + metadata = {"hello": "world", "number": "42"} + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") file_client = directory_client.create_file("newfile", metadata=metadata, content_settings=content_settings) file_client.append_data(b"abc", 0, 3) file_client.flush_data(3) @@ -1234,7 +1232,7 @@ def test_get_properties(self, **kwargs): # Assert assert properties assert properties.size == 3 - assert properties.metadata['hello'] == metadata['hello'] + assert properties.metadata["hello"] == metadata["hello"] assert properties.content_settings.content_language == content_settings.content_language @DataLakePreparer() @@ -1242,17 +1240,15 @@ def test_get_properties(self, **kwargs): def test_set_expiry(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - variables = kwargs.pop('variables', {}) + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_client = self._create_directory_and_return_client() - metadata = {'hello': 'world', 'number': '42'} - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + metadata = {"hello": "world", "number": "42"} + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) file_client = directory_client.create_file("newfile", metadata=metadata, content_settings=content_settings) # Act / Assert @@ -1279,7 +1275,7 @@ def test_rename_file_with_non_used_name(self, **kwargs): data_bytes = b"abc" file_client.append_data(data_bytes, 0, 3) file_client.flush_data(3) - new_client = file_client.rename_file(file_client.file_system_name+'/'+'newname') + new_client = file_client.rename_file(file_client.file_system_name + "/" + "newname") data = new_client.download_file().readall() assert data == data_bytes @@ -1292,10 +1288,10 @@ def test_file_encryption_scope_from_file_system(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=datalake_storage_account_key.secret, logging_enable=True) - self.file_system_name = self.get_resource_name('filesystem') - file_name = 'testfile' + self.file_system_name = self.get_resource_name("filesystem") + file_name = "testfile" encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -1306,8 +1302,8 @@ def test_file_encryption_scope_from_file_system(self, **kwargs): # Assert assert props - assert props['encryption_scope'] is not None - assert props['encryption_scope'] == encryption_scope.default_encryption_scope + assert props["encryption_scope"] is not None + assert props["encryption_scope"] == encryption_scope.default_encryption_scope @pytest.mark.live_test_only @DataLakePreparer() @@ -1331,7 +1327,7 @@ def test_rename_file_with_file_system_sas(self, **kwargs): data_bytes = b"abc" file_client.append_data(data_bytes, 0, 3) file_client.flush_data(3) - new_client = file_client.rename_file(file_client.file_system_name+'/'+'newname') + new_client = file_client.rename_file(file_client.file_system_name + "/" + "newname") data = new_client.download_file().readall() assert data == data_bytes @@ -1345,23 +1341,25 @@ def test_rename_file_with_file_sas(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # SAS URL is calculated from storage key, so this test runs live only - token = generate_file_sas(self.dsc.account_name, - self.file_system_name, - None, - "oldfile", - datalake_storage_account_key.secret, - permission=FileSasPermissions(read=True, create=True, write=True, delete=True), - expiry=datetime.utcnow() + timedelta(hours=1), - ) - - new_token = generate_file_sas(self.dsc.account_name, - self.file_system_name, - None, - "newname", - datalake_storage_account_key.secret, - permission=FileSasPermissions(read=True, create=True, write=True, delete=True), - expiry=datetime.utcnow() + timedelta(hours=1), - ) + token = generate_file_sas( + self.dsc.account_name, + self.file_system_name, + None, + "oldfile", + datalake_storage_account_key.secret, + permission=FileSasPermissions(read=True, create=True, write=True, delete=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) + + new_token = generate_file_sas( + self.dsc.account_name, + self.file_system_name, + None, + "newname", + datalake_storage_account_key.secret, + permission=FileSasPermissions(read=True, create=True, write=True, delete=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) # read the created file which is under root directory file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, "oldfile", credential=token) @@ -1369,7 +1367,7 @@ def test_rename_file_with_file_sas(self, **kwargs): data_bytes = b"abc" file_client.append_data(data_bytes, 0, 3) file_client.flush_data(3) - new_client = file_client.rename_file(file_client.file_system_name+'/'+'newname'+'?'+new_token) + new_client = file_client.rename_file(file_client.file_system_name + "/" + "newname" + "?" + new_token) data = new_client.download_file().readall() assert data == data_bytes @@ -1396,7 +1394,7 @@ def test_rename_file_with_account_sas(self, **kwargs): data_bytes = b"abc" file_client.append_data(data_bytes, 0, 3) file_client.flush_data(3) - new_client = file_client.rename_file(file_client.file_system_name+'/'+'newname') + new_client = file_client.rename_file(file_client.file_system_name + "/" + "newname") data = new_client.download_file().readall() assert data == data_bytes @@ -1420,7 +1418,7 @@ def test_rename_file_to_existing_file(self, **kwargs): data_bytes = b"abc" file_client.append_data(data_bytes, 0, 3) file_client.flush_data(3) - new_client = file_client.rename_file(file_client.file_system_name+'/'+existing_file_client.path_name) + new_client = file_client.rename_file(file_client.file_system_name + "/" + existing_file_client.path_name) new_url = file_client.url data = new_client.download_file().readall() @@ -1452,7 +1450,7 @@ def test_rename_file_will_not_change_existing_directory(self, **kwargs): f4.append_data(b"file4", 0, 5) f4.flush_data(5) - new_client = f3.rename_file(f1.file_system_name+'/'+f1.path_name) + new_client = f3.rename_file(f1.file_system_name + "/" + f1.path_name) assert new_client.download_file().readall() == b"file3" @@ -1489,7 +1487,7 @@ def test_rename_file_different_filesystem_with_sas(self, **kwargs): file_client.append_data(b"abc", 0, 3, flush=True) # Create another filesystem to rename to - new_file_system = self.dsc.get_file_system_client(self.file_system_name + '2') + new_file_system = self.dsc.get_file_system_client(self.file_system_name + "2") new_file_system.create_file_system() # Get different SAS to new file system @@ -1503,11 +1501,11 @@ def test_rename_file_different_filesystem_with_sas(self, **kwargs): ) # ? in new name to test parsing - new_name = new_file_system.file_system_name + '/' + 'new?file' + '?' + new_sas + new_name = new_file_system.file_system_name + "/" + "new?file" + "?" + new_sas new_client = file_client.rename_file(new_name) new_props = new_client.get_file_properties() - assert new_props.name == 'new?file' + assert new_props.name == "new?file" new_file_system.delete_file_system() @@ -1522,10 +1520,10 @@ def test_rename_file_special_chars(self, **kwargs): file_client = self._create_file_and_return_client(file="oldfile") file_client.append_data(b"abc", 0, 3, flush=True) - new_client = file_client.rename_file(file_client.file_system_name + '/' + '?!@#$%^&*.?test') + new_client = file_client.rename_file(file_client.file_system_name + "/" + "?!@#$%^&*.?test") new_props = new_client.get_file_properties() - assert new_props.name == '?!@#$%^&*.?test' + assert new_props.name == "?!@#$%^&*.?test" @DataLakePreparer() @recorded_by_proxy @@ -1538,7 +1536,7 @@ def test_read_file_read(self, **kwargs): self.dsc._config.max_chunk_get_size = 1024 file_client = self._create_file_and_return_client() - data = b'12345' * 205 * 5 # 5125 bytes + data = b"12345" * 205 * 5 # 5125 bytes file_client.append_data(data, 0, len(data), flush=True) stream = file_client.download_file() @@ -1564,10 +1562,10 @@ def test_create_and_read_file_encryption_context(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=datalake_storage_account_key.secret) - self.file_system_name = self.get_resource_name('filesystem') - file_name = 'testfile' + self.file_system_name = self.get_resource_name("filesystem") + file_name = "testfile" file_system = self.dsc.get_file_system_client(self.file_system_name) try: file_system.create_file_system() @@ -1576,23 +1574,23 @@ def test_create_and_read_file_encryption_context(self, **kwargs): file_client = file_system.get_file_client(file_name) # Act - file_client.create_file(encryption_context='encryptionContext') + file_client.create_file(encryption_context="encryptionContext") properties = file_client.get_file_properties() read_response = file_client.download_file() path_response = list(file_system.get_paths()) assert properties - assert properties['encryption_context'] is not None - assert properties['encryption_context'] == 'encryptionContext' + assert properties["encryption_context"] is not None + assert properties["encryption_context"] == "encryptionContext" assert read_response.properties - assert read_response.properties['encryption_context'] is not None - assert read_response.properties['encryption_context'] == 'encryptionContext' + assert read_response.properties["encryption_context"] is not None + assert read_response.properties["encryption_context"] == "encryptionContext" - assert path_response[0]['encryption_context'] - assert path_response[0]['encryption_context'] is not None - assert path_response[0]['encryption_context'] == 'encryptionContext' + assert path_response[0]["encryption_context"] + assert path_response[0]["encryption_context"] is not None + assert path_response[0]["encryption_context"] == "encryptionContext" @DataLakePreparer() @recorded_by_proxy @@ -1601,11 +1599,11 @@ def test_upload_file_encryption_context(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=datalake_storage_account_key.secret) - self.file_system_name = self.get_resource_name('filesystem') + self.file_system_name = self.get_resource_name("filesystem") data = self.get_random_bytes(200 * 1024) - file_name = 'testfile' + file_name = "testfile" file_system = self.dsc.get_file_system_client(self.file_system_name) try: file_system.create_file_system() @@ -1614,7 +1612,7 @@ def test_upload_file_encryption_context(self, **kwargs): file_client = file_system.get_file_client(file_name) # Act - file_client.upload_data(data, overwrite=True, encryption_context='encryptionContext') + file_client.upload_data(data, overwrite=True, encryption_context="encryptionContext") downloaded_data = file_client.download_file().readall() properties = file_client.get_file_properties() @@ -1622,8 +1620,8 @@ def test_upload_file_encryption_context(self, **kwargs): # Assert assert data == downloaded_data assert properties - assert properties['encryption_context'] is not None - assert properties['encryption_context'] == 'encryptionContext' + assert properties["encryption_context"] is not None + assert properties["encryption_context"] == "encryptionContext" @DataLakePreparer() @recorded_by_proxy @@ -1636,21 +1634,21 @@ def test_dir_and_file_properties_owner_group_acl_permissions(self, **kwargs): directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client1 = directory_client.get_file_client('filename') + file_client1 = directory_client.get_file_client("filename") file_client1.create_file() directory_properties = directory_client.get_directory_properties() file_properties = file_client1.get_file_properties(upn=True) # Assert - assert directory_properties['owner'] is not None - assert directory_properties['group'] is not None - assert directory_properties['permissions'] is not None - assert directory_properties['acl'] is not None - assert file_properties['owner'] is not None - assert file_properties['group'] is not None - assert file_properties['permissions'] is not None - assert file_properties['acl'] is not None + assert directory_properties["owner"] is not None + assert directory_properties["group"] is not None + assert directory_properties["permissions"] is not None + assert directory_properties["acl"] is not None + assert file_properties["owner"] is not None + assert file_properties["group"] is not None + assert file_properties["permissions"] is not None + assert file_properties["acl"] is not None @DataLakePreparer() @recorded_by_proxy @@ -1665,15 +1663,15 @@ def test_storage_account_audience_file_client(self, **kwargs): # Act token_credential = self.get_credential(DataLakeServiceClient) fc = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name + '/', - '/' + file_client.path_name, + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name + "/", + "/" + file_client.path_name, credential=token_credential, - audience=f'https://{datalake_storage_account_name}.blob.core.windows.net/' + audience=f"https://{datalake_storage_account_name}.blob.core.windows.net/", ) # Assert - data = b'Hello world' + data = b"Hello world" response1 = fc.get_file_properties() response2 = fc.upload_data(data, overwrite=True) assert response1 is not None @@ -1692,15 +1690,15 @@ def test_bad_audience_file_client(self, **kwargs): # Act token_credential = self.get_credential(DataLakeServiceClient) fc = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name + '/', - '/' + file_client.path_name, + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name + "/", + "/" + file_client.path_name, credential=token_credential, - audience='https://badaudience.blob.core.windows.net/' + audience="https://badaudience.blob.core.windows.net/", ) # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge - data = b'Hello world' + data = b"Hello world" fc.get_file_properties() fc.upload_data(data, overwrite=True) @@ -1711,12 +1709,12 @@ def test_mock_transport_no_content_validation(self, **kwargs): transport = MockStorageTransport() file_client = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), + self.account_url(datalake_storage_account_name, "dfs"), "filesystem/", "dir/file.txt", credential=datalake_storage_account_key.secret, transport=transport, - retry_total=0 + retry_total=0, ) data = file_client.download_file() @@ -1742,12 +1740,12 @@ def test_mock_transport_with_content_validation(self, **kwargs): transport = MockStorageTransport() file_client = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), + self.account_url(datalake_storage_account_name, "dfs"), "filesystem/", "dir/file.txt", credential=datalake_storage_account_key.secret, transport=transport, - retry_total=0 + retry_total=0, ) data = b"Hello World!" @@ -1766,19 +1764,14 @@ def test_progress_hook_upload_data(self, **kwargs): # Arrange self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = self._create_file_and_return_client( - directory=self._get_directory_reference(), - file=self._get_file_reference() + directory=self._get_directory_reference(), file=self._get_file_reference() ) data = self.get_random_bytes(8 * 1024) progress = ProgressTracker(len(data), 1024) # Act file_client.upload_data( - data, - overwrite=True, - progress_hook=progress.assert_progress, - max_concurrency=1, - chunk_size=1024 + data, overwrite=True, progress_hook=progress.assert_progress, max_concurrency=1, chunk_size=1024 ) # Assert @@ -1793,17 +1786,16 @@ def test_download_file_decompress(self, **kwargs): # Arrange self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = self._create_file_and_return_client() - compressed_data = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH' - b'\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00') + compressed_data = ( + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH" + b"\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00" + ) decompressed_data = b"hello from gzip" - content_settings = ContentSettings(content_encoding='gzip') + content_settings = ContentSettings(content_encoding="gzip") # Act / Assert file_client.upload_data( - data=compressed_data, - length=len(compressed_data), - overwrite=True, - content_settings=content_settings + data=compressed_data, length=len(compressed_data), overwrite=True, content_settings=content_settings ) result = file_client.download_file(decompress=True).readall() @@ -1822,7 +1814,7 @@ def test_download_file_no_decompress_chunks(self, **kwargs): file_name = self._get_file_reference() file_client = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), + self.account_url(datalake_storage_account_name, "dfs"), self.file_system_name, file_name, credential=datalake_storage_account_key.secret, @@ -1831,16 +1823,15 @@ def test_download_file_no_decompress_chunks(self, **kwargs): ) file_client.create_file() - compressed_data = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH' - b'\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00') - content_settings = ContentSettings(content_encoding='gzip') + compressed_data = ( + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH" + b"\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00" + ) + content_settings = ContentSettings(content_encoding="gzip") # Act / Assert file_client.upload_data( - data=compressed_data, - length=len(compressed_data), - overwrite=True, - content_settings=content_settings + data=compressed_data, length=len(compressed_data), overwrite=True, content_settings=content_settings ) result = file_client.download_file(decompress=False).readall() @@ -1853,7 +1844,7 @@ def test_datalake_dynamic_user_delegation_sas(self, **kwargs): token_credential = self.get_credential(DataLakeServiceClient) dsc = DataLakeServiceClient(self.account_url(datalake_storage_account_name, "dfs"), credential=token_credential) - fs_name, file_name = self.get_resource_name('filesystem'), self.get_resource_name('file') + fs_name, file_name = self.get_resource_name("filesystem"), self.get_resource_name("file") fs = dsc.create_file_system(fs_name) file = fs.create_file(file_name) file.upload_data(b"abc", overwrite=True) @@ -1884,7 +1875,7 @@ def test_datalake_dynamic_user_delegation_sas(self, **kwargs): permission=FileSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), request_headers=request_headers, - request_query_params=request_query_params + request_query_params=request_query_params, ) def callback(request): @@ -1894,10 +1885,10 @@ def callback(request): request.http_request.url = request.http_request.url + "&" + extra identity_file = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), + self.account_url(datalake_storage_account_name, "dfs"), file.file_system_name, file.path_name, - credential=sas_token + credential=sas_token, ) props = identity_file.get_file_properties(raw_request_hook=callback) assert props is not None @@ -1912,7 +1903,7 @@ def test_data_lake_tags(self, **kwargs): directory_name = self._get_directory_reference() self._create_directory_and_return_client(directory_name) file_name = self._get_file_reference() - file_client = self.dsc.get_file_client(self.file_system_name, directory_name + '/' + file_name) + file_client = self.dsc.get_file_client(self.file_system_name, directory_name + "/" + file_name) first_resp = file_client.create_file() early = file_client.get_file_properties().last_modified @@ -1927,14 +1918,14 @@ def test_data_lake_tags(self, **kwargs): with pytest.raises(ResourceModifiedError): file_client.get_tags(if_modified_since=early) with pytest.raises(ResourceModifiedError): - file_client.set_tags(first_tags, etag=first_resp['etag'], match_condition=MatchConditions.IfModified) + file_client.set_tags(first_tags, etag=first_resp["etag"], match_condition=MatchConditions.IfModified) file_client.set_tags(first_tags, if_unmodified_since=early) tags = file_client.get_tags(if_unmodified_since=early) assert tags == first_tags - file_client.set_tags(second_tags, etag=first_resp['etag'], match_condition=MatchConditions.IfNotModified) - tags = file_client.get_tags(etag=first_resp['etag'], match_condition=MatchConditions.IfNotModified) + file_client.set_tags(second_tags, etag=first_resp["etag"], match_condition=MatchConditions.IfNotModified) + tags = file_client.get_tags(etag=first_resp["etag"], match_condition=MatchConditions.IfNotModified) assert tags == second_tags data = b"abc123" @@ -1945,16 +1936,17 @@ def test_data_lake_tags(self, **kwargs): with pytest.raises(ResourceModifiedError): file_client.get_tags(if_unmodified_since=early) with pytest.raises(ResourceModifiedError): - file_client.set_tags(first_tags, etag=first_resp['etag'], match_condition=MatchConditions.IfNotModified) + file_client.set_tags(first_tags, etag=first_resp["etag"], match_condition=MatchConditions.IfNotModified) file_client.set_tags(first_tags, if_modified_since=early) tags = file_client.get_tags(if_modified_since=early) assert tags == first_tags - file_client.set_tags(second_tags, etag=first_resp['etag'], match_condition=MatchConditions.IfModified) - tags = file_client.get_tags(etag=first_resp['etag'], match_condition=MatchConditions.IfModified) + file_client.set_tags(second_tags, etag=first_resp["etag"], match_condition=MatchConditions.IfModified) + tags = file_client.get_tags(etag=first_resp["etag"], match_condition=MatchConditions.IfModified) assert tags == second_tags + # ------------------------------------------------------------------------------ -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_file_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_file_async.py index 1a0d494fb508..86aed283a6f6 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_file_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_file_async.py @@ -45,20 +45,19 @@ FileSystemClient, ) - # ------------------------------------------------------------------------------ -TEST_DIRECTORY_PREFIX = 'directory' -TEST_FILE_PREFIX = 'file' -TEST_FILE_SYSTEM_PREFIX = 'filesystem' +TEST_DIRECTORY_PREFIX = "directory" +TEST_FILE_PREFIX = "file" +TEST_FILE_SYSTEM_PREFIX = "filesystem" # ------------------------------------------------------------------------------ class TestFileAsync(AsyncStorageRecordedTestCase): async def _setUp(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=account_key.secret) self.config = self.dsc._config - self.file_system_name = self.get_resource_name('filesystem') + self.file_system_name = self.get_resource_name("filesystem") if not self.is_playback(): file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -95,7 +94,7 @@ async def _create_file_and_return_client(self, directory="", file=None): await self._create_directory_and_return_client(directory) if not file: file = self._get_file_reference() - file_client = self.dsc.get_file_client(self.file_system_name, directory + '/' + file) + file_client = self.dsc.get_file_client(self.file_system_name, directory + "/" + file) await file_client.create_file() return file_client @@ -124,7 +123,7 @@ async def test_create_file(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") response = await file_client.create_file() # Assert @@ -137,8 +136,8 @@ async def test_create_file_owner_group_acl(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' - test_string_acl = 'user::rwx,group::r-x,other::rwx' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" + test_string_acl = "user::rwx,group::r-x,other::rwx" # Arrange directory_name = self._get_directory_reference() @@ -146,15 +145,15 @@ async def test_create_file_owner_group_acl(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file(owner=test_string, group=test_string, acl=test_string_acl) # Assert acl_properties = await file_client.get_access_control() assert acl_properties is not None - assert acl_properties['owner'] == test_string - assert acl_properties['group'] == test_string - assert acl_properties['acl'] == test_string_acl + assert acl_properties["owner"] == test_string + assert acl_properties["group"] == test_string + assert acl_properties["acl"] == test_string_acl @DataLakePreparer() @recorded_by_proxy_async @@ -163,7 +162,7 @@ async def test_create_file_proposed_lease_id(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - test_string = '4cf4e284-f6a8-4540-b53e-c3469af032dc' + test_string = "4cf4e284-f6a8-4540-b53e-c3469af032dc" test_duration = 15 # Arrange directory_name = self._get_directory_reference() @@ -172,15 +171,15 @@ async def test_create_file_proposed_lease_id(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file(lease_id=test_string, lease_duration=test_duration) # Assert properties = await file_client.get_file_properties() assert properties is not None - assert properties.lease['status'] == 'locked' - assert properties.lease['state'] == 'leased' - assert properties.lease['duration'] == 'fixed' + assert properties.lease["status"] == "locked" + assert properties.lease["state"] == "leased" + assert properties.lease["duration"] == "fixed" @DataLakePreparer() @recorded_by_proxy_async @@ -197,14 +196,14 @@ async def test_create_file_relative_expiry(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file(expires_on=test_expiry_time) # Assert file_properties = await file_client.get_file_properties() - expiry_time = file_properties['expiry_time'] + expiry_time = file_properties["expiry_time"] expiry_time = expiry_time.replace(tzinfo=None) # Strip timezone info to be able to compare - creation_time = file_properties['creation_time'] + creation_time = file_properties["creation_time"] creation_time = creation_time.replace(tzinfo=None) # Strip timezone info to be able to compare assert file_properties is not None assert self._is_almost_equal(expiry_time, creation_time + timedelta(days=1), timedelta(seconds=60)) is True @@ -224,12 +223,12 @@ async def test_create_file_absolute_expiry(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file(expires_on=test_expiry_time) # Assert file_properties = await file_client.get_file_properties() - expiry_time = file_properties['expiry_time'] + expiry_time = file_properties["expiry_time"] expiry_time = expiry_time.replace(tzinfo=None) # Strip timezone info to be able to compare assert file_properties is not None assert self._is_almost_equal(expiry_time, test_expiry_time, timedelta(seconds=1)) is True @@ -244,10 +243,13 @@ async def test_create_file_extra_backslashes(self, **kwargs): # Arrange file_client = await self._create_file_and_return_client() - new_file_client = DataLakeFileClient(self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name + '/', - '/' + file_client.path_name, - credential=datalake_storage_account_key.secret, logging_enable=True) + new_file_client = DataLakeFileClient( + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name + "/", + "/" + file_client.path_name, + credential=datalake_storage_account_key.secret, + logging_enable=True, + ) response = await new_file_client.create_file() # Assert @@ -266,8 +268,8 @@ async def test_file_exists(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client1 = directory_client.get_file_client('filename') - file_client2 = directory_client.get_file_client('nonexistentfile') + file_client1 = directory_client.get_file_client("filename") + file_client2 = directory_client.get_file_client("nonexistentfile") await file_client1.create_file() assert await file_client1.exists() @@ -285,8 +287,7 @@ async def test_create_file_using_oauth_token_credential(self, **kwargs): token_credential = self.get_credential(DataLakeServiceClient, is_async=True) # Create a directory to put the file under that - file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, file_name, - credential=token_credential) + file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, file_name, credential=token_credential) response = file_client.create_file() @@ -320,17 +321,17 @@ async def test_create_file_with_lease_id(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") # Act await file_client.create_file() - lease = await file_client.acquire_lease(lease_id='00000000-1111-2222-3333-444444444444') + lease = await file_client.acquire_lease(lease_id="00000000-1111-2222-3333-444444444444") create_resp = await file_client.create_file(lease=lease) # Assert file_properties = await file_client.get_file_properties() assert file_properties is not None - assert file_properties.etag == create_resp.get('etag') - assert file_properties.last_modified == create_resp.get('last_modified') + assert file_properties.etag == create_resp.get("etag") + assert file_properties.last_modified == create_resp.get("last_modified") @DataLakePreparer() @recorded_by_proxy_async @@ -361,11 +362,11 @@ async def test_append_data(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file() # Act - response = await file_client.append_data(b'abc', 0, 3) + response = await file_client.append_data(b"abc", 0, 3) assert response is not None @@ -382,39 +383,39 @@ async def test_append_data_lease_action(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file() - data = b'Hello world' - lease_id = '670d43d1-ecde-4ae9-9c37-d22d340e7719' + data = b"Hello world" + lease_id = "670d43d1-ecde-4ae9-9c37-d22d340e7719" # Act / Assert # ---Acquire--- - await file_client.append_data(data, 0, len(data), lease_action='acquire', lease_duration=30, lease=lease_id) + await file_client.append_data(data, 0, len(data), lease_action="acquire", lease_duration=30, lease=lease_id) lease = (await file_client.get_file_properties()).lease - assert lease.state == 'leased' - assert lease.duration == 'fixed' + assert lease.state == "leased" + assert lease.duration == "fixed" # ---Renew--- - await file_client.append_data(data, 0, len(data), lease_action='auto-renew', lease=lease_id) + await file_client.append_data(data, 0, len(data), lease_action="auto-renew", lease=lease_id) lease = (await file_client.get_file_properties()).lease - assert lease.state == 'leased' - assert lease.duration == 'fixed' + assert lease.state == "leased" + assert lease.duration == "fixed" # ---Release--- - await file_client.append_data(data, 0, len(data), flush=True, lease_action='release', lease=lease_id) + await file_client.append_data(data, 0, len(data), flush=True, lease_action="release", lease=lease_id) lease = (await file_client.get_file_properties()).lease - assert lease.state == 'available' + assert lease.state == "available" assert not lease.duration # ---Acquire and release--- - await file_client.append_data(data, 0, len(data), flush=True, lease_action='acquire-release', lease=lease_id) + await file_client.append_data(data, 0, len(data), flush=True, lease_action="acquire-release", lease=lease_id) lease = (await file_client.get_file_properties()).lease - assert lease.state == 'available' + assert lease.state == "available" assert not lease.duration @DataLakePreparer() @@ -430,7 +431,7 @@ async def test_append_empty_data(self, **kwargs): await file_client.flush_data(0) file_props = await file_client.get_file_properties() - assert file_props['size'] == 0 + assert file_props["size"] == 0 @DataLakePreparer() @recorded_by_proxy_async @@ -445,17 +446,17 @@ async def test_flush_data(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file() # Act - await file_client.append_data(b'abc', 0, 3) + await file_client.append_data(b"abc", 0, 3) response = await file_client.flush_data(3) # Assert prop = await file_client.get_file_properties() assert response is not None - assert prop['size'] == 3 + assert prop["size"] == 3 @DataLakePreparer() @recorded_by_proxy_async @@ -470,43 +471,43 @@ async def test_flush_data_lease_action(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file() - data = b'Hello world' - lease_id = 'c8107e94-ab42-42ac-92d6-6458764982af' + data = b"Hello world" + lease_id = "c8107e94-ab42-42ac-92d6-6458764982af" # Act / Assert # ---Acquire--- await file_client.append_data(data, 0, len(data)) - await file_client.flush_data(len(data), lease_action='acquire', lease_duration=30, lease=lease_id) + await file_client.flush_data(len(data), lease_action="acquire", lease_duration=30, lease=lease_id) lease = (await file_client.get_file_properties()).lease - assert lease.state == 'leased' - assert lease.duration == 'fixed' + assert lease.state == "leased" + assert lease.duration == "fixed" # ---Renew--- await file_client.append_data(data, 0, len(data), lease=lease_id) - await file_client.flush_data(len(data), lease_action='auto-renew', lease=lease_id) + await file_client.flush_data(len(data), lease_action="auto-renew", lease=lease_id) lease = (await file_client.get_file_properties()).lease - assert lease.state == 'leased' - assert lease.duration == 'fixed' + assert lease.state == "leased" + assert lease.duration == "fixed" # ---Release--- await file_client.append_data(data, 0, len(data), lease=lease_id) - await file_client.flush_data(len(data), lease_action='release', lease=lease_id) + await file_client.flush_data(len(data), lease_action="release", lease=lease_id) lease = (await file_client.get_file_properties()).lease - assert lease.state == 'available' + assert lease.state == "available" assert not lease.duration # ---Acquire and release--- await file_client.append_data(data, 0, len(data)) - await file_client.flush_data(len(data), lease_action='acquire-release', lease=lease_id) + await file_client.flush_data(len(data), lease_action="acquire-release", lease=lease_id) lease = (await file_client.get_file_properties()).lease - assert lease.state == 'available' + assert lease.state == "available" assert not lease.duration @DataLakePreparer() @@ -522,16 +523,16 @@ async def test_flush_data_with_bool(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file() # Act - response = await file_client.append_data(b'abc', 0, 3, flush=True) + response = await file_client.append_data(b"abc", 0, 3, flush=True) # Assert prop = await file_client.get_file_properties() assert response is not None - assert prop['size'] == 3 + assert prop["size"] == 3 @DataLakePreparer() @recorded_by_proxy_async @@ -546,19 +547,19 @@ async def test_flush_data_with_match_condition(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") resp = await file_client.create_file() # Act - await file_client.append_data(b'abc', 0, 3) + await file_client.append_data(b"abc", 0, 3) # flush is successful because it isn't touched - response = await file_client.flush_data(3, etag=resp['etag'], match_condition=MatchConditions.IfNotModified) + response = await file_client.flush_data(3, etag=resp["etag"], match_condition=MatchConditions.IfNotModified) - await file_client.append_data(b'abc', 3, 3) + await file_client.append_data(b"abc", 3, 3) with pytest.raises(ResourceModifiedError): # flush is unsuccessful because extra data were appended. - await file_client.flush_data(6, etag=resp['etag'], match_condition=MatchConditions.IfNotModified) + await file_client.flush_data(6, etag=resp["etag"], match_condition=MatchConditions.IfNotModified) @pytest.mark.live_test_only @DataLakePreparer() @@ -574,7 +575,7 @@ async def test_upload_data_in_substreams(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") # Get 16MB data raw_data = self.get_random_bytes(16 * 1024 * 1024) # Ensure chunk size is greater than threshold (8MB > 4MB) - for optimized upload @@ -603,8 +604,8 @@ async def test_upload_data(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') - data = self.get_random_bytes(400*1024) + file_client = directory_client.get_file_client("filename") + data = self.get_random_bytes(400 * 1024) await file_client.upload_data(data, overwrite=True, max_concurrency=5) downloaded_data = await (await file_client.download_file()).readall() @@ -624,7 +625,7 @@ async def test_upload_data_to_existing_file(self, **kwargs): await directory_client.create_directory() # create an existing file - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file() await file_client.append_data(b"abc", 0) await file_client.flush_data(3) @@ -652,18 +653,17 @@ async def test_upload_data_to_existing_file_with_content_settings(self, **kwargs await directory_client.create_directory() # create an existing file - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") resp = await file_client.create_file() - etag = resp['etag'] + etag = resp["etag"] # to override the existing file data = self.get_random_bytes(100) - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") await file_client.upload_data( - data, content_settings=content_settings, etag=etag, match_condition=MatchConditions.IfNotModified) + data, content_settings=content_settings, etag=etag, match_condition=MatchConditions.IfNotModified + ) downloaded_data = await (await file_client.download_file()).readall() properties = await file_client.get_file_properties() @@ -686,9 +686,9 @@ async def test_upload_data_to_existing_file_with_permissions_and_umask(self, **k await directory_client.create_directory() # create an existing file - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") resp = await file_client.create_file() - etag = resp['etag'] + etag = resp["etag"] # to override the existing file data = self.get_random_bytes(100) @@ -696,17 +696,17 @@ async def test_upload_data_to_existing_file_with_permissions_and_umask(self, **k await file_client.upload_data( data, overwrite=True, - permissions='0777', + permissions="0777", umask="0000", etag=etag, - match_condition=MatchConditions.IfNotModified + match_condition=MatchConditions.IfNotModified, ) downloaded_data = await (await file_client.download_file()).readall() prop = await file_client.get_access_control() assert data == downloaded_data - assert prop['permissions'] == 'rwxrwxrwx' + assert prop["permissions"] == "rwxrwxrwx" @DataLakePreparer() @recorded_by_proxy_async @@ -721,7 +721,7 @@ async def test_upload_data_from_async_generator(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - data = b'Hello Async World!' + data = b"Hello Async World!" async def data_generator(): for _ in range(3): @@ -729,12 +729,12 @@ async def data_generator(): await asyncio.sleep(0.1) # Act - file_client = directory_client.get_file_client('filename') - await file_client.upload_data(data_generator(), length=len(data*3), overwrite=True) + file_client = directory_client.get_file_client("filename") + await file_client.upload_data(data_generator(), length=len(data * 3), overwrite=True) # Assert result = await (await file_client.download_file()).readall() - assert result == data*3 + assert result == data * 3 @DataLakePreparer() @recorded_by_proxy_async @@ -749,7 +749,7 @@ async def test_upload_data_with_none_max_concurrency(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") data = self.get_random_bytes(100) # max_concurrency=None should not raise TypeError await file_client.upload_data(data, overwrite=True, max_concurrency=None) @@ -812,24 +812,29 @@ async def test_read_file_with_user_delegation_key(self, **kwargs): # Get user delegation key token_credential = self.get_credential(DataLakeServiceClient, is_async=True) service_client = DataLakeServiceClient( - self.account_url(datalake_storage_account_name, 'dfs'), credential=token_credential) - user_delegation_key = await service_client.get_user_delegation_key(datetime.utcnow(), - datetime.utcnow() + timedelta(hours=1)) - - sas_token = generate_file_sas(file_client.account_name, - file_client.file_system_name, - None, - file_client.path_name, - user_delegation_key, - permission=FileSasPermissions(read=True, create=True, write=True, delete=True), - expiry=datetime.utcnow() + timedelta(hours=1), - ) + self.account_url(datalake_storage_account_name, "dfs"), credential=token_credential + ) + user_delegation_key = await service_client.get_user_delegation_key( + datetime.utcnow(), datetime.utcnow() + timedelta(hours=1) + ) + + sas_token = generate_file_sas( + file_client.account_name, + file_client.file_system_name, + None, + file_client.path_name, + user_delegation_key, + permission=FileSasPermissions(read=True, create=True, write=True, delete=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) # download the data and make sure it is the same as uploaded data - new_file_client = DataLakeFileClient(self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name, - file_client.path_name, - credential=sas_token) + new_file_client = DataLakeFileClient( + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name, + file_client.path_name, + credential=sas_token, + ) downloaded_data = await (await new_file_client.download_file()).readall() assert data == downloaded_data @@ -922,10 +927,7 @@ async def test_account_sas_raises_if_sas_already_in_uri(self, **kwargs): with pytest.raises(ValueError): DataLakeFileClient( - self.dsc.url + "?sig=foo", - self.file_system_name, - "foo", - credential=AzureSasCredential("?foo=bar") + self.dsc.url + "?sig=foo", self.file_system_name, "foo", credential=AzureSasCredential("?foo=bar") ) @pytest.mark.live_test_only @@ -953,8 +955,9 @@ async def test_file_sas_only_applies_to_file_level(self, **kwargs): ) # read the created file which is under root directory - file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, directory_name + '/' + file_name, - credential=token) + file_client = DataLakeFileClient( + self.dsc.url, self.file_system_name, directory_name + "/" + file_name, credential=token + ) properties = await file_client.get_file_properties() # make sure we can read the file properties @@ -970,8 +973,9 @@ async def test_file_sas_only_applies_to_file_level(self, **kwargs): await file_system_client.get_file_system_properties() # the token is for file level, so users are not supposed to have access to directory level operations - directory_client = DataLakeDirectoryClient(self.dsc.url, self.file_system_name, directory_name, - credential=token) + directory_client = DataLakeDirectoryClient( + self.dsc.url, self.file_system_name, directory_name, credential=token + ) with pytest.raises(ClientAuthenticationError): await directory_client.get_directory_properties() @@ -1002,11 +1006,7 @@ async def test_delete_file_oauth(self, **kwargs): file_name = self._get_file_reference() token_credential = self.get_credential(DataLakeServiceClient, is_async=True) - file_client = DataLakeFileClient( - self.dsc.url, - self.file_system_name, - file_name, - credential=token_credential) + file_client = DataLakeFileClient(self.dsc.url, self.file_system_name, file_name, credential=token_credential) await file_client.create_file() # Act @@ -1026,7 +1026,7 @@ async def test_delete_file_with_if_unmodified_since(self, **kwargs): file_client = await self._create_file_and_return_client() prop = await file_client.get_file_properties() - await file_client.delete_file(if_unmodified_since=prop['last_modified']) + await file_client.delete_file(if_unmodified_since=prop["last_modified"]) # Make sure the file was deleted with pytest.raises(ResourceNotFoundError): @@ -1041,7 +1041,7 @@ async def test_set_access_control(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = await self._create_file_and_return_client() - response = await file_client.set_access_control(permissions='0777') + response = await file_client.set_access_control(permissions="0777") # Assert assert response is not None @@ -1056,7 +1056,7 @@ async def test_set_access_control_with_match_conditions(self, **kwargs): file_client = await self._create_file_and_return_client() with pytest.raises(ResourceModifiedError): - await file_client.set_access_control(permissions='0777', match_condition=MatchConditions.IfMissing) + await file_client.set_access_control(permissions="0777", match_condition=MatchConditions.IfMissing) @DataLakePreparer() @recorded_by_proxy_async @@ -1066,7 +1066,7 @@ async def test_get_access_control(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = await self._create_file_and_return_client() - await file_client.set_access_control(permissions='0777') + await file_client.set_access_control(permissions="0777") # Act response = await file_client.get_access_control() @@ -1082,12 +1082,12 @@ async def test_get_access_control_with_if_modified_since(self, **kwargs): await self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = await self._create_file_and_return_client() - await file_client.set_access_control(permissions='0777') + await file_client.set_access_control(permissions="0777") prop = await file_client.get_file_properties() # Act - response = await file_client.get_access_control(if_modified_since=prop['last_modified'] - timedelta(minutes=15)) + response = await file_client.get_access_control(if_modified_since=prop["last_modified"] - timedelta(minutes=15)) # Assert assert response is not None @@ -1102,12 +1102,11 @@ async def test_get_properties(self, **kwargs): # Arrange directory_client = await self._create_directory_and_return_client() - metadata = {'hello': 'world', 'number': '42'} - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') - file_client = await directory_client.create_file("newfile", metadata=metadata, - content_settings=content_settings) + metadata = {"hello": "world", "number": "42"} + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") + file_client = await directory_client.create_file( + "newfile", metadata=metadata, content_settings=content_settings + ) await file_client.append_data(b"abc", 0, 3) await file_client.flush_data(3) properties = await file_client.get_file_properties() @@ -1115,7 +1114,7 @@ async def test_get_properties(self, **kwargs): # Assert assert properties assert properties.size == 3 - assert properties.metadata['hello'] == metadata['hello'] + assert properties.metadata["hello"] == metadata["hello"] assert properties.content_settings.content_language == content_settings.content_language @DataLakePreparer() @@ -1125,7 +1124,7 @@ async def test_set_access_control_recursive(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" file_client = await self._create_file_and_return_client() summary = await file_client.set_access_control_recursive(acl=acl) @@ -1136,7 +1135,7 @@ async def test_set_access_control_recursive(self, **kwargs): assert summary.counters.failure_count == 0 access_control = await file_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy_async @@ -1145,7 +1144,7 @@ async def test_update_access_control_recursive(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" file_client = await self._create_file_and_return_client() summary = await file_client.update_access_control_recursive(acl=acl) @@ -1156,7 +1155,7 @@ async def test_update_access_control_recursive(self, **kwargs): assert summary.counters.failure_count == 0 access_control = await file_client.get_access_control() assert access_control is not None - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy_async @@ -1165,9 +1164,12 @@ async def test_remove_access_control_recursive(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - acl = "mask," + "default:user,default:group," + \ - "user:ec3595d6-2c17-4696-8caa-7e139758d24a,group:ec3595d6-2c17-4696-8caa-7e139758d24a," + \ - "default:user:ec3595d6-2c17-4696-8caa-7e139758d24a,default:group:ec3595d6-2c17-4696-8caa-7e139758d24a" + acl = ( + "mask," + + "default:user,default:group," + + "user:ec3595d6-2c17-4696-8caa-7e139758d24a,group:ec3595d6-2c17-4696-8caa-7e139758d24a," + + "default:user:ec3595d6-2c17-4696-8caa-7e139758d24a,default:group:ec3595d6-2c17-4696-8caa-7e139758d24a" + ) file_client = await self._create_file_and_return_client() summary = await file_client.remove_access_control_recursive(acl=acl) @@ -1181,19 +1183,18 @@ async def test_remove_access_control_recursive(self, **kwargs): async def test_set_expiry(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - variables = kwargs.pop('variables', {}) + variables = kwargs.pop("variables", {}) await self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange directory_client = await self._create_directory_and_return_client() - metadata = {'hello': 'world', 'number': '42'} - content_settings = ContentSettings( - content_language='spanish', - content_disposition='inline') - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + metadata = {"hello": "world", "number": "42"} + content_settings = ContentSettings(content_language="spanish", content_disposition="inline") + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) file_client = await directory_client.create_file( - "newfile", metadata=metadata, content_settings=content_settings) + "newfile", metadata=metadata, content_settings=content_settings + ) # Act / Assert await file_client.set_file_expiry("Absolute", expires_on=expiry_time) @@ -1219,7 +1220,7 @@ async def test_rename_file_with_non_used_name(self, **kwargs): data_bytes = b"abc" await file_client.append_data(data_bytes, 0, 3) await file_client.flush_data(3) - new_client = await file_client.rename_file(file_client.file_system_name + '/' + 'newname') + new_client = await file_client.rename_file(file_client.file_system_name + "/" + "newname") data = await (await new_client.download_file()).readall() assert data == data_bytes @@ -1243,7 +1244,7 @@ async def test_rename_file_to_existing_file(self, **kwargs): data_bytes = b"abc" await file_client.append_data(data_bytes, 0, 3) await file_client.flush_data(3) - new_client = await file_client.rename_file(file_client.file_system_name + '/' + existing_file_client.path_name) + new_client = await file_client.rename_file(file_client.file_system_name + "/" + existing_file_client.path_name) new_url = file_client.url data = await (await new_client.download_file()).readall() @@ -1257,10 +1258,10 @@ async def test_file_encryption_scope_from_file_system_async(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=datalake_storage_account_key.secret, logging_enable=True) - self.file_system_name = self.get_resource_name('filesystem') - file_name = 'testfile' + self.file_system_name = self.get_resource_name("filesystem") + file_name = "testfile" encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -1271,8 +1272,8 @@ async def test_file_encryption_scope_from_file_system_async(self, **kwargs): # Assert assert props - assert props['encryption_scope'] is not None - assert props['encryption_scope'] == encryption_scope.default_encryption_scope + assert props["encryption_scope"] is not None + assert props["encryption_scope"] == encryption_scope.default_encryption_scope @pytest.mark.live_test_only @DataLakePreparer() @@ -1308,7 +1309,7 @@ async def test_rename_file_with_file_sas(self, **kwargs): data_bytes = b"abc" await file_client.append_data(data_bytes, 0, 3) await file_client.flush_data(3) - new_client = await file_client.rename_file(file_client.file_system_name+'/'+'newname'+'?'+new_token) + new_client = await file_client.rename_file(file_client.file_system_name + "/" + "newname" + "?" + new_token) data = await (await new_client.download_file()).readall() assert data == data_bytes @@ -1339,7 +1340,7 @@ async def test_rename_file_will_not_change_existing_directory(self, **kwargs): await f4.append_data(b"file4", 0, 5) await f4.flush_data(5) - new_client = await f3.rename_file(f1.file_system_name + '/' + f1.path_name) + new_client = await f3.rename_file(f1.file_system_name + "/" + f1.path_name) assert await (await new_client.download_file()).readall() == b"file3" @@ -1376,7 +1377,7 @@ async def test_rename_file_different_filesystem_with_sas(self, **kwargs): await file_client.append_data(b"abc", 0, 3, flush=True) # Create another filesystem to rename to - new_file_system = self.dsc.get_file_system_client(self.file_system_name + '2') + new_file_system = self.dsc.get_file_system_client(self.file_system_name + "2") await new_file_system.create_file_system() # Get different SAS to new file system @@ -1390,11 +1391,11 @@ async def test_rename_file_different_filesystem_with_sas(self, **kwargs): ) # ? in new name to test parsing - new_name = new_file_system.file_system_name + '/' + 'new?file' + '?' + new_sas + new_name = new_file_system.file_system_name + "/" + "new?file" + "?" + new_sas new_client = await file_client.rename_file(new_name) new_props = await new_client.get_file_properties() - assert new_props.name == 'new?file' + assert new_props.name == "new?file" await new_file_system.delete_file_system() @@ -1409,10 +1410,10 @@ async def test_rename_file_special_chars(self, **kwargs): file_client = await self._create_file_and_return_client(file="oldfile") await file_client.append_data(b"abc", 0, 3, flush=True) - new_client = await file_client.rename_file(file_client.file_system_name + '/' + '?!@#$%^&*.?test') + new_client = await file_client.rename_file(file_client.file_system_name + "/" + "?!@#$%^&*.?test") new_props = await new_client.get_file_properties() - assert new_props.name == '?!@#$%^&*.?test' + assert new_props.name == "?!@#$%^&*.?test" @DataLakePreparer() @recorded_by_proxy_async @@ -1425,7 +1426,7 @@ async def test_read_file_read(self, **kwargs): self.dsc._config.max_chunk_get_size = 1024 file_client = await self._create_file_and_return_client() - data = b'12345' * 205 * 5 # 5125 bytes + data = b"12345" * 205 * 5 # 5125 bytes await file_client.append_data(data, 0, len(data), flush=True) stream = await file_client.download_file() @@ -1452,10 +1453,10 @@ async def test_create_and_read_file_encryption_context(self, **kwargs): # Arrange await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=datalake_storage_account_key.secret) - self.file_system_name = self.get_resource_name('filesystem') - file_name = 'testfile' + self.file_system_name = self.get_resource_name("filesystem") + file_name = "testfile" file_system = self.dsc.get_file_system_client(self.file_system_name) try: await file_system.create_file_system() @@ -1464,7 +1465,7 @@ async def test_create_and_read_file_encryption_context(self, **kwargs): file_client = file_system.get_file_client(file_name) # Act - await file_client.create_file(encryption_context='encryptionContext') + await file_client.create_file(encryption_context="encryptionContext") properties = await file_client.get_file_properties() read_response = await file_client.download_file() @@ -1473,16 +1474,16 @@ async def test_create_and_read_file_encryption_context(self, **kwargs): path_response.append(path) assert properties - assert properties['encryption_context'] is not None - assert properties['encryption_context'] == 'encryptionContext' + assert properties["encryption_context"] is not None + assert properties["encryption_context"] == "encryptionContext" assert read_response.properties - assert read_response.properties['encryption_context'] is not None - assert read_response.properties['encryption_context'] == 'encryptionContext' + assert read_response.properties["encryption_context"] is not None + assert read_response.properties["encryption_context"] == "encryptionContext" - assert path_response[0]['encryption_context'] - assert path_response[0]['encryption_context'] is not None - assert path_response[0]['encryption_context'] == 'encryptionContext' + assert path_response[0]["encryption_context"] + assert path_response[0]["encryption_context"] is not None + assert path_response[0]["encryption_context"] == "encryptionContext" @DataLakePreparer() @recorded_by_proxy_async @@ -1492,11 +1493,11 @@ async def test_upload_file_encryption_context(self, **kwargs): # Arrange await self._setUp(datalake_storage_account_name, datalake_storage_account_key) - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=datalake_storage_account_key.secret) - self.file_system_name = self.get_resource_name('filesystem') + self.file_system_name = self.get_resource_name("filesystem") data = self.get_random_bytes(200 * 1024) - file_name = 'testfile' + file_name = "testfile" file_system = self.dsc.get_file_system_client(self.file_system_name) try: await file_system.create_file_system() @@ -1505,7 +1506,7 @@ async def test_upload_file_encryption_context(self, **kwargs): file_client = file_system.get_file_client(file_name) # Act - await file_client.upload_data(data, overwrite=True, encryption_context='encryptionContext') + await file_client.upload_data(data, overwrite=True, encryption_context="encryptionContext") downloaded_data = await (await file_client.download_file()).readall() properties = await file_client.get_file_properties() @@ -1513,8 +1514,8 @@ async def test_upload_file_encryption_context(self, **kwargs): # Assert assert data == downloaded_data assert properties - assert properties['encryption_context'] is not None - assert properties['encryption_context'] == 'encryptionContext' + assert properties["encryption_context"] is not None + assert properties["encryption_context"] == "encryptionContext" @DataLakePreparer() @recorded_by_proxy_async @@ -1527,21 +1528,21 @@ async def test_dir_and_file_properties_owner_group_acl_permissions(self, **kwarg directory_name = self._get_directory_reference() directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client1 = directory_client.get_file_client('filename') + file_client1 = directory_client.get_file_client("filename") await file_client1.create_file() directory_properties = await directory_client.get_directory_properties() file_properties = await file_client1.get_file_properties(upn=True) # Assert - assert directory_properties['owner'] is not None - assert directory_properties['group'] is not None - assert directory_properties['permissions'] is not None - assert directory_properties['acl'] is not None - assert file_properties['owner'] is not None - assert file_properties['group'] is not None - assert file_properties['permissions'] is not None - assert file_properties['acl'] is not None + assert directory_properties["owner"] is not None + assert directory_properties["group"] is not None + assert directory_properties["permissions"] is not None + assert directory_properties["acl"] is not None + assert file_properties["owner"] is not None + assert file_properties["group"] is not None + assert file_properties["permissions"] is not None + assert file_properties["acl"] is not None @DataLakePreparer() @recorded_by_proxy_async @@ -1556,15 +1557,15 @@ async def test_storage_account_audience_file_client(self, **kwargs): # Act token_credential = self.get_credential(DataLakeServiceClient, is_async=True) fc = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name + '/', - '/' + file_client.path_name, + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name + "/", + "/" + file_client.path_name, credential=token_credential, - audience=f'https://{datalake_storage_account_name}.blob.core.windows.net/' + audience=f"https://{datalake_storage_account_name}.blob.core.windows.net/", ) # Assert - data = b'Hello world' + data = b"Hello world" response1 = await fc.get_file_properties() response2 = await fc.upload_data(data, overwrite=True) assert response1 is not None @@ -1583,15 +1584,15 @@ async def test_bad_audience_file_client(self, **kwargs): # Act token_credential = self.get_credential(DataLakeServiceClient, is_async=True) fc = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), - file_client.file_system_name + '/', - '/' + file_client.path_name, + self.account_url(datalake_storage_account_name, "dfs"), + file_client.file_system_name + "/", + "/" + file_client.path_name, credential=token_credential, - audience='https://badaudience.blob.core.windows.net/' + audience="https://badaudience.blob.core.windows.net/", ) # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge - data = b'Hello world' + data = b"Hello world" await fc.get_file_properties() await fc.upload_data(data, overwrite=True) @@ -1602,12 +1603,12 @@ async def test_mock_transport_no_content_validation(self, **kwargs): transport = MockStorageTransport() file_client = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), + self.account_url(datalake_storage_account_name, "dfs"), "filesystem/", "dir/file.txt", credential=datalake_storage_account_key.secret, transport=transport, - retry_total=0 + retry_total=0, ) data = await file_client.download_file() @@ -1636,12 +1637,12 @@ async def test_mock_transport_with_content_validation(self, **kwargs): transport = MockStorageTransport() file_client = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), + self.account_url(datalake_storage_account_name, "dfs"), "filesystem/", "dir/file.txt", credential=datalake_storage_account_key.secret, transport=transport, - retry_total=0 + retry_total=0, ) data = b"Hello Async World!" @@ -1661,19 +1662,14 @@ async def test_progress_hook_upload_data(self, **kwargs): # Arrange await self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = await self._create_file_and_return_client( - directory=self._get_directory_reference(), - file=self._get_file_reference() + directory=self._get_directory_reference(), file=self._get_file_reference() ) data = self.get_random_bytes(8 * 1024) progress = ProgressTracker(len(data), 1024) # Act await file_client.upload_data( - data, - overwrite=True, - progress_hook=progress.assert_progress, - max_concurrency=1, - chunk_size=1024 + data, overwrite=True, progress_hook=progress.assert_progress, max_concurrency=1, chunk_size=1024 ) # Assert @@ -1688,17 +1684,16 @@ async def test_download_file_decompress(self, **kwargs): # Arrange await self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_client = await self._create_file_and_return_client() - compressed_data = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH' - b'\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00') + compressed_data = ( + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH" + b"\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00" + ) decompressed_data = b"hello from gzip" - content_settings = ContentSettings(content_encoding='gzip') + content_settings = ContentSettings(content_encoding="gzip") # Act / Assert await file_client.upload_data( - data=compressed_data, - length=len(compressed_data), - overwrite=True, - content_settings=content_settings + data=compressed_data, length=len(compressed_data), overwrite=True, content_settings=content_settings ) file_data = await file_client.download_file(decompress=True) @@ -1719,7 +1714,7 @@ async def test_download_file_no_decompress_chunks(self, **kwargs): file_name = self._get_file_reference() file_client = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), + self.account_url(datalake_storage_account_name, "dfs"), self.file_system_name, file_name, credential=datalake_storage_account_key.secret, @@ -1728,16 +1723,15 @@ async def test_download_file_no_decompress_chunks(self, **kwargs): ) await file_client.create_file() - compressed_data = (b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH' - b'\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00') - content_settings = ContentSettings(content_encoding='gzip') + compressed_data = ( + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcaH\xcd\xc9\xc9WH+\xca\xcfUH" + b"\xaf\xca,\x00\x00\x00\x00\xff\xff\x03\x00d\xaa\x8e\xb5\x0f\x00\x00\x00" + ) + content_settings = ContentSettings(content_encoding="gzip") # Act / Assert await file_client.upload_data( - data=compressed_data, - length=len(compressed_data), - overwrite=True, - content_settings=content_settings + data=compressed_data, length=len(compressed_data), overwrite=True, content_settings=content_settings ) result = await (await file_client.download_file(decompress=False)).readall() @@ -1749,9 +1743,8 @@ async def test_datalake_dynamic_user_delegation_sas(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") token_credential = self.get_credential(DataLakeServiceClient, is_async=True) - dsc = DataLakeServiceClient( - self.account_url(datalake_storage_account_name, "dfs"), credential=token_credential) - fs_name, file_name = self.get_resource_name('filesystem'), self.get_resource_name('file') + dsc = DataLakeServiceClient(self.account_url(datalake_storage_account_name, "dfs"), credential=token_credential) + fs_name, file_name = self.get_resource_name("filesystem"), self.get_resource_name("file") fs = await dsc.create_file_system(fs_name) file = await fs.create_file(file_name) await file.upload_data(b"abc", overwrite=True) @@ -1782,7 +1775,7 @@ async def test_datalake_dynamic_user_delegation_sas(self, **kwargs): permission=FileSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1), request_headers=request_headers, - request_query_params=request_query_params + request_query_params=request_query_params, ) def callback(request): @@ -1792,10 +1785,10 @@ def callback(request): request.http_request.url = request.http_request.url + "&" + extra identity_file = DataLakeFileClient( - self.account_url(datalake_storage_account_name, 'dfs'), + self.account_url(datalake_storage_account_name, "dfs"), file.file_system_name, file.path_name, - credential=sas_token + credential=sas_token, ) props = identity_file.get_file_properties(raw_request_hook=callback) assert props is not None @@ -1810,7 +1803,7 @@ async def test_data_lake_tags(self, **kwargs): directory_name = self._get_directory_reference() await self._create_directory_and_return_client(directory_name) file_name = self._get_file_reference() - file_client = self.dsc.get_file_client(self.file_system_name, directory_name + '/' + file_name) + file_client = self.dsc.get_file_client(self.file_system_name, directory_name + "/" + file_name) first_resp = await file_client.create_file() early = (await file_client.get_file_properties()).last_modified @@ -1825,14 +1818,14 @@ async def test_data_lake_tags(self, **kwargs): with pytest.raises(ResourceModifiedError): await file_client.get_tags(if_modified_since=early) with pytest.raises(ResourceModifiedError): - await file_client.set_tags(first_tags, etag=first_resp['etag'], match_condition=MatchConditions.IfModified) + await file_client.set_tags(first_tags, etag=first_resp["etag"], match_condition=MatchConditions.IfModified) await file_client.set_tags(first_tags, if_unmodified_since=early) tags = await file_client.get_tags(if_unmodified_since=early) assert tags == first_tags - await file_client.set_tags(second_tags, etag=first_resp['etag'], match_condition=MatchConditions.IfNotModified) - tags = await file_client.get_tags(etag=first_resp['etag'], match_condition=MatchConditions.IfNotModified) + await file_client.set_tags(second_tags, etag=first_resp["etag"], match_condition=MatchConditions.IfNotModified) + tags = await file_client.get_tags(etag=first_resp["etag"], match_condition=MatchConditions.IfNotModified) assert tags == second_tags data = b"abc123" @@ -1844,16 +1837,18 @@ async def test_data_lake_tags(self, **kwargs): await file_client.get_tags(if_unmodified_since=early) with pytest.raises(ResourceModifiedError): await file_client.set_tags( - first_tags, etag=first_resp['etag'], match_condition=MatchConditions.IfNotModified) + first_tags, etag=first_resp["etag"], match_condition=MatchConditions.IfNotModified + ) await file_client.set_tags(first_tags, if_modified_since=early) tags = await file_client.get_tags(if_modified_since=early) assert tags == first_tags - await file_client.set_tags(second_tags, etag=first_resp['etag'], match_condition=MatchConditions.IfModified) - tags = await file_client.get_tags(etag=first_resp['etag'], match_condition=MatchConditions.IfModified) + await file_client.set_tags(second_tags, etag=first_resp["etag"], match_condition=MatchConditions.IfModified) + tags = await file_client.get_tags(etag=first_resp["etag"], match_condition=MatchConditions.IfModified) assert tags == second_tags + # ------------------------------------------------------------------------------ -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_file_system.py b/sdk/storage/azure-storage-file-datalake/tests/test_file_system.py index 31f344ca02bd..e0917bb10e24 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_file_system.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_file_system.py @@ -34,15 +34,14 @@ ) from azure.storage.filedatalake._models import FileSasPermissions - # ------------------------------------------------------------------------------ -TEST_FILE_SYSTEM_PREFIX = 'filesystem' +TEST_FILE_SYSTEM_PREFIX = "filesystem" # ------------------------------------------------------------------------------ class TestFileSystem(StorageRecordedTestCase): def _setUp(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, account_key.secret) self.config = self.dsc._config self.test_file_systems = [] @@ -107,7 +106,7 @@ def test_create_file_system_extra_backslash(self, **kwargs): file_system_name = self._get_file_system_reference() # Act - file_system_client = self.dsc.get_file_system_client(file_system_name + '/') + file_system_client = self.dsc.get_file_system_client(file_system_name + "/") created = file_system_client.create_file_system() # Assert @@ -131,8 +130,8 @@ def test_create_file_system_encryption_scope(self, **kwargs): # Assert assert props - assert props['encryption_scope'] is not None - assert props['encryption_scope'].default_encryption_scope == encryption_scope.default_encryption_scope + assert props["encryption_scope"] is not None + assert props["encryption_scope"].default_encryption_scope == encryption_scope.default_encryption_scope @DataLakePreparer() @recorded_by_proxy @@ -142,7 +141,7 @@ def test_create_file_system_encryption_scope_account_sas(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") token = self.generate_sas( generate_account_sas, self.dsc.account_name, @@ -150,7 +149,8 @@ def test_create_file_system_encryption_scope_account_sas(self, **kwargs): ResourceTypes(service=True, file_system=True, object=True), permission=AccountSasPermissions(write=True, read=True, create=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=5), - encryption_scope="hnstestscope1") + encryption_scope="hnstestscope1", + ) file_system_name = self._get_file_system_reference() encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") @@ -159,10 +159,10 @@ def test_create_file_system_encryption_scope_account_sas(self, **kwargs): file_system_client.create_file_system(encryption_scope_options=encryption_scope) fsc_sas = FileSystemClient(url, file_system_name, token) - fsc_sas.create_file('file1') - fsc_sas.create_directory('dir1') - dir_props = fsc_sas.get_directory_client('dir1').get_directory_properties() - file_props = fsc_sas.get_file_client('file1').get_file_properties() + fsc_sas.create_file("file1") + fsc_sas.create_directory("dir1") + dir_props = fsc_sas.get_directory_client("dir1").get_directory_properties() + file_props = fsc_sas.get_file_client("file1").get_file_properties() # Assert assert dir_props @@ -180,7 +180,7 @@ def test_create_file_system_encryption_scope_file_system_sas(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() token = self.generate_sas( generate_file_system_sas, @@ -189,7 +189,8 @@ def test_create_file_system_encryption_scope_file_system_sas(self, **kwargs): self.dsc.credential.account_key, permission=FileSystemSasPermissions(write=True, read=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=5), - encryption_scope="hnstestscope1") + encryption_scope="hnstestscope1", + ) encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") # Act @@ -197,10 +198,10 @@ def test_create_file_system_encryption_scope_file_system_sas(self, **kwargs): file_system_client.create_file_system(encryption_scope_options=encryption_scope) fsc_sas = FileSystemClient(url, file_system_name, token) - fsc_sas.create_file('file1') - fsc_sas.create_directory('dir1') - dir_props = fsc_sas.get_directory_client('dir1').get_directory_properties() - file_props = fsc_sas.get_file_client('file1').get_file_properties() + fsc_sas.create_file("file1") + fsc_sas.create_directory("dir1") + dir_props = fsc_sas.get_directory_client("dir1").get_directory_properties() + file_props = fsc_sas.get_file_client("file1").get_file_properties() # Assert assert dir_props @@ -218,24 +219,25 @@ def test_create_file_system_encryption_scope_directory_sas(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() token = self.generate_sas( generate_directory_sas, self.dsc.account_name, file_system_name, - 'dir1', + "dir1", self.dsc.credential.account_key, permission=FileSasPermissions(write=True, read=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=5), - encryption_scope="hnstestscope1") + encryption_scope="hnstestscope1", + ) encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") # Act file_system_client = self.dsc.get_file_system_client(file_system_name) file_system_client.create_file_system(encryption_scope_options=encryption_scope) - dir_client = DataLakeDirectoryClient(url, file_system_name, 'dir1', credential=token) + dir_client = DataLakeDirectoryClient(url, file_system_name, "dir1", credential=token) dir_client.create_directory() dir_props = dir_client.get_directory_properties() @@ -252,26 +254,27 @@ def test_create_file_system_encryption_scope_file_sas(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() token = self.generate_sas( generate_file_sas, self.dsc.account_name, file_system_name, - 'dir1', - 'file1', + "dir1", + "file1", self.dsc.credential.account_key, permission=FileSasPermissions(write=True, read=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=5), - encryption_scope="hnstestscope1") + encryption_scope="hnstestscope1", + ) encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") # Act file_system_client = self.dsc.get_file_system_client(file_system_name) file_system_client.create_file_system(encryption_scope_options=encryption_scope) - file_system_client.create_directory('dir1') + file_system_client.create_directory("dir1") - file_client = DataLakeFileClient(url, file_system_name, 'dir1/file1', token) + file_client = DataLakeFileClient(url, file_system_name, "dir1/file1", token) file_client.create_file() file_props = file_client.get_file_properties() @@ -306,7 +309,7 @@ def test_create_file_system_with_metadata(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} file_system_name = self._get_file_system_reference() # Act @@ -324,31 +327,31 @@ def test_create_file_system_with_metadata(self, **kwargs): def test_set_file_system_acl(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - variables = kwargs.pop('variables', {}) + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Act file_system = self._create_file_system() - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - access_policy = AccessPolicy(permission=FileSystemSasPermissions(read=True), - expiry=expiry_time, - start=start_time) - signed_identifier1 = {'testid': access_policy} + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + access_policy = AccessPolicy( + permission=FileSystemSasPermissions(read=True), expiry=expiry_time, start=start_time + ) + signed_identifier1 = {"testid": access_policy} response = file_system.set_file_system_access_policy(signed_identifier1, public_access=PublicAccess.FileSystem) - assert response.get('etag') is not None - assert response.get('last_modified') is not None + assert response.get("etag") is not None + assert response.get("last_modified") is not None acl1 = file_system.get_file_system_access_policy() - assert acl1['public_access'] is not None - assert len(acl1['signed_identifiers']) == 1 + assert acl1["public_access"] is not None + assert len(acl1["signed_identifiers"]) == 1 # If set signed identifier without specifying the access policy then it will be default to None - signed_identifier2 = {'testid': access_policy, 'test2': access_policy} + signed_identifier2 = {"testid": access_policy, "test2": access_policy} file_system.set_file_system_access_policy(signed_identifier2) acl2 = file_system.get_file_system_access_policy() - assert acl2['public_access'] is None - assert len(acl2['signed_identifiers']) == 2 + assert acl2["public_access"] is None + assert len(acl2["signed_identifiers"]) == 2 return variables @@ -382,8 +385,8 @@ def test_list_file_systems_encryption_scope(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - file_system_name1 = self._get_file_system_reference(prefix='es') - file_system_name2 = self._get_file_system_reference(prefix='es2') + file_system_name1 = self._get_file_system_reference(prefix="es") + file_system_name2 = self._get_file_system_reference(prefix="es2") encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") self.dsc.create_file_system(file_system_name1, encryption_scope_options=encryption_scope) self.dsc.create_file_system(file_system_name2, encryption_scope_options=encryption_scope) @@ -391,7 +394,7 @@ def test_list_file_systems_encryption_scope(self, **kwargs): # Act file_systems = [] for filesystem in self.dsc.list_file_systems(): - if filesystem['name'] in [file_system_name1, file_system_name2]: + if filesystem["name"] in [file_system_name1, file_system_name2]: file_systems.append(filesystem) # Assert @@ -420,7 +423,7 @@ def test_list_file_systems_account_sas(self, **kwargs): ) # Act - dsc = DataLakeServiceClient(self.account_url(datalake_storage_account_name, 'dfs'), credential=sas_token) + dsc = DataLakeServiceClient(self.account_url(datalake_storage_account_name, "dfs"), credential=sas_token) file_systems = list(dsc.list_file_systems()) # Assert @@ -507,7 +510,7 @@ def test_rename_file_system_with_source_lease(self, **kwargs): old_name = self._get_file_system_reference(prefix="old") new_name = self._get_file_system_reference(prefix="new") filesystem = self.dsc.create_file_system(old_name) - filesystem_lease_id = filesystem.acquire_lease(lease_id='00000000-1111-2222-3333-444444444444') + filesystem_lease_id = filesystem.acquire_lease(lease_id="00000000-1111-2222-3333-444444444444") with pytest.raises(HttpResponseError): self.dsc._rename_file_system(name=old_name, new_name=new_name) with pytest.raises(HttpResponseError): @@ -621,13 +624,13 @@ def test_list_file_systems_with_include_metadata(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} resp = file_system.set_file_system_metadata(metadata) # Act - file_systems = list(self.dsc.list_file_systems( - name_starts_with=file_system.file_system_name, - include_metadata=True)) + file_systems = list( + self.dsc.list_file_systems(name_starts_with=file_system.file_system_name, include_metadata=True) + ) # Assert assert file_systems is not None @@ -648,10 +651,11 @@ def test_list_file_systems_by_page(self, **kwargs): self._create_file_system(file_system_prefix="filesystem{}".format(i)) # Act - file_systems = list(next(self.dsc.list_file_systems( - results_per_page=3, - name_starts_with="file", - include_metadata=True).by_page())) + file_systems = list( + next( + self.dsc.list_file_systems(results_per_page=3, name_starts_with="file", include_metadata=True).by_page() + ) + ) # Assert assert file_systems is not None @@ -669,13 +673,13 @@ def test_list_file_systems_with_public_access(self, **kwargs): file_system_name = self._get_file_system_reference() file_system = self.dsc.get_file_system_client(file_system_name) file_system.create_file_system(public_access="blob") - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} resp = file_system.set_file_system_metadata(metadata) # Act - file_systems = list(self.dsc.list_file_systems( - name_starts_with=file_system.file_system_name, - include_metadata=True)) + file_systems = list( + self.dsc.list_file_systems(name_starts_with=file_system.file_system_name, include_metadata=True) + ) # Assert assert file_systems is not None @@ -693,7 +697,7 @@ def test_get_file_system_properties(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} file_system = self._create_file_system() file_system.set_file_system_metadata(metadata) @@ -743,14 +747,14 @@ def test_list_paths(self, **kwargs): def test_list_paths_create_expiry(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - variables = kwargs.pop('variables', {}) + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() - file_client = file_system.create_file('file1') + file_client = file_system.create_file("file1") - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(days=1)) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(days=1)) file_client.set_file_expiry("Absolute", expires_on=expiry_time) # Act @@ -774,7 +778,7 @@ def test_list_paths_no_expiry(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = self._create_file_system() - file_system.create_file('file1') + file_system.create_file("file1") # Act paths = list(file_system.get_paths(upn=True)) @@ -815,14 +819,15 @@ def test_get_deleted_paths(self, **kwargs): assert len(dir3_paths) == 2 assert dir3_paths[0].deletion_id is not None assert dir3_paths[1].deletion_id is not None - assert dir3_paths[0].name == 'dir3/file_in_dir3' - assert dir3_paths[1].name == 'dir3/subdir/file_in_subdir' + assert dir3_paths[0].name == "dir3/file_in_dir3" + assert dir3_paths[1].name == "dir3/subdir/file_in_subdir" paths_generator1 = file_system.list_deleted_paths(results_per_page=2).by_page() paths1 = list(next(paths_generator1)) paths_generator2 = file_system.list_deleted_paths(results_per_page=4).by_page( - continuation_token=paths_generator1.continuation_token) + continuation_token=paths_generator1.continuation_token + ) paths2 = list(next(paths_generator2)) # Assert @@ -860,8 +865,9 @@ def test_list_paths_with_max_per_page(self, **kwargs): generator1 = file_system.get_paths(max_results=2, upn=True).by_page() paths1 = list(next(generator1)) - generator2 = file_system.get_paths(max_results=4, upn=True)\ - .by_page(continuation_token=generator1.continuation_token) + generator2 = file_system.get_paths(max_results=4, upn=True).by_page( + continuation_token=generator1.continuation_token + ) paths2 = list(next(generator2)) assert len(paths1) == 2 @@ -886,7 +892,7 @@ def test_list_paths_under_specific_path(self, **kwargs): # create a file under the current directory file_client = subdir.create_file("file") - file_client.append_data(b"abced", 0, 5) # cspell:disable-line + file_client.append_data(b"abced", 0, 5) # cspell:disable-line file_client.flush_data(5) generator1 = file_system.get_paths(path="dir10/subdir", max_results=2, upn=True).by_page() @@ -980,11 +986,11 @@ def test_path_properties_encryption_scope(self, **kwargs): # Act file_system_client = self.dsc.get_file_system_client(file_system_name) file_system_client.create_file_system(encryption_scope_options=encryption_scope) - file_system_client.create_directory('dir1') - file_system_client.create_file('dir1/file1') + file_system_client.create_directory("dir1") + file_system_client.create_file("dir1/file1") - dir_props = file_system_client.get_directory_client('dir1').get_directory_properties() - file_props = file_system_client.get_file_client('dir1/file1').get_file_properties() + dir_props = file_system_client.get_directory_client("dir1").get_directory_properties() + file_props = file_system_client.get_file_client("dir1/file1").get_file_properties() paths = list(file_system_client.get_paths(recursive=False, upn=True)) # Assert @@ -1037,11 +1043,11 @@ def test_get_root_directory_client(self, **kwargs): file_system = self._create_file_system() directory_client = file_system._get_root_directory_client() - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" directory_client.set_access_control(acl=acl) access_control = directory_client.get_access_control() - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -1051,7 +1057,7 @@ def test_file_system_sessions_closes_properly(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - file_system_client = self._create_file_system("fenrhxsbfvsdvdsvdsadb") # cspell:disable-line + file_system_client = self._create_file_system("fenrhxsbfvsdvdsvdsadb") # cspell:disable-line with file_system_client as fs_client: with fs_client.get_file_client("file1.txt") as f_client: f_client.create_file() @@ -1072,12 +1078,12 @@ def test_undelete_dir_with_version_id(self, **kwargs): file_system_client = self._create_file_system("fs2") if file_system_client is None: file_system_client = self.dsc.get_file_system_client(self._get_file_system_reference(prefix="fs2")) - dir_path = 'dir10' + dir_path = "dir10" dir_client = file_system_client.create_directory(dir_path) resp = dir_client.delete_directory() with pytest.raises(HttpResponseError): file_system_client.get_file_client(dir_path).get_file_properties() - restored_dir_client = file_system_client._undelete_path(dir_path, resp['deletion_id']) + restored_dir_client = file_system_client._undelete_path(dir_path, resp["deletion_id"]) resp = restored_dir_client.get_directory_properties() assert resp is not None @@ -1091,12 +1097,12 @@ def test_undelete_file_with_version_id(self, **kwargs): file_system_client = self._create_file_system("fs3") if file_system_client is None: file_system_client = self.dsc.get_file_system_client(self._get_file_system_reference(prefix="fs3")) - file_path = 'dir10/file' + file_path = "dir10/file" dir_client = file_system_client.create_file(file_path) resp = dir_client.delete_file() with pytest.raises(HttpResponseError): file_system_client.get_file_client(file_path).get_file_properties() - restored_file_client = file_system_client._undelete_path(file_path, resp['deletion_id']) + restored_file_client = file_system_client._undelete_path(file_path, resp["deletion_id"]) resp = restored_file_client.get_file_properties() assert resp is not None @@ -1108,23 +1114,24 @@ def test_storage_account_audience_service_client(self, **kwargs): # Arrange self._setUp(datalake_storage_account_name, datalake_storage_account_key) - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() file_system_client = self.dsc.get_file_system_client(file_system_name) file_system_client.create_file_system() - file_system_client.create_directory('testdir1') + file_system_client.create_directory("testdir1") # Act token_credential = self.get_credential(DataLakeServiceClient) fsc = FileSystemClient( - url, file_system_name, + url, + file_system_name, credential=token_credential, - audience=f'https://{datalake_storage_account_name}.blob.core.windows.net/' + audience=f"https://{datalake_storage_account_name}.blob.core.windows.net/", ) # Assert response1 = fsc.exists() - response2 = fsc.create_directory('testdir11') + response2 = fsc.create_directory("testdir11") assert response1 is not None assert response2 is not None @@ -1136,23 +1143,21 @@ def test_bad_audience_service_client(self, **kwargs): # Arrange self._setUp(datalake_storage_account_name, datalake_storage_account_key) - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() file_system_client = self.dsc.get_file_system_client(file_system_name) file_system_client.create_file_system() - file_system_client.create_directory('testdir2') + file_system_client.create_directory("testdir2") # Act token_credential = self.get_credential(DataLakeServiceClient) fsc = FileSystemClient( - url, file_system_name, - credential=token_credential, - audience='https://badaudience.blob.core.windows.net/' + url, file_system_name, credential=token_credential, audience="https://badaudience.blob.core.windows.net/" ) # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge fsc.exists() - fsc.create_directory('testdir22') + fsc.create_directory("testdir22") @DataLakePreparer() @recorded_by_proxy @@ -1161,20 +1166,17 @@ def test_get_and_set_access_control_oauth(self, **kwargs): # Arrange token_credential = self.get_credential(DataLakeServiceClient) - dsc = DataLakeServiceClient( - self.account_url(datalake_storage_account_name, 'dfs'), - token_credential - ) + dsc = DataLakeServiceClient(self.account_url(datalake_storage_account_name, "dfs"), token_credential) file_system = dsc.create_file_system(self.get_resource_name(TEST_FILE_SYSTEM_PREFIX)) directory_client = file_system._get_root_directory_client() # Act - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" directory_client.set_access_control(acl=acl) access_control = directory_client.get_access_control() # Assert - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy @@ -1187,8 +1189,8 @@ def test_get_user_delegation_sas(self, **kwargs): self.account_url(datalake_storage_account_name, "dfs"), credential=token_credential, ) - start = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) user_delegation_key_1 = service.get_user_delegation_key(key_start_time=start, key_expiry_time=expiry) user_delegation_key_2 = service.get_user_delegation_key(key_start_time=start, key_expiry_time=expiry) @@ -1236,9 +1238,7 @@ def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: disabl user_delegation_oid = decoded.get("oid") delegated_user_tid = decoded.get("tid") user_delegation_key = dsc.get_user_delegation_key( - key_start_time=start, - key_expiry_time=expiry, - delegated_user_tid=delegated_user_tid + key_start_time=start, key_expiry_time=expiry, delegated_user_tid=delegated_user_tid ) assert user_delegation_key is not None @@ -1258,9 +1258,7 @@ def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: disabl assert "skdutid=" + delegated_user_tid in file_system_token file_system_client = FileSystemClient( - f"{account_url}?{file_system_token}", - file_system_name=file_system_name, - credential=token_credential + f"{account_url}?{file_system_token}", file_system_name=file_system_name, credential=token_credential ) paths = list(file_system_client.get_paths()) assert len(paths) == 2 @@ -1275,7 +1273,7 @@ def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: disabl user_delegation_key, permission=FileSasPermissions(write=True, read=True, delete=True), expiry=expiry, - user_delegation_oid=user_delegation_oid + user_delegation_oid=user_delegation_oid, ) assert "sduoid=" + user_delegation_oid in directory_token @@ -1285,7 +1283,7 @@ def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: disabl f"{account_url}?{directory_token}", file_system_name=file_system_name, directory_name=directory_name, - credential=token_credential + credential=token_credential, ) props = directory_client.get_directory_properties() assert props is not None @@ -1299,7 +1297,7 @@ def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: disabl credential=user_delegation_key, permission=FileSasPermissions(write=True, read=True, delete=True), expiry=expiry, - user_delegation_oid=user_delegation_oid + user_delegation_oid=user_delegation_oid, ) assert "sduoid=" + user_delegation_oid in file_token @@ -1309,11 +1307,12 @@ def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: disabl f"{account_url}?{file_token}", file_system_name=file_system_name, file_path=f"{directory_name}/{file_name}", - credential=token_credential + credential=token_credential, ) content = file_client.download_file().readall() assert content == data + # ------------------------------------------------------------------------------ -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_file_system_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_file_system_async.py index 2ba8e97baa2e..9ea1d0ffdcf8 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_file_system_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_file_system_async.py @@ -38,15 +38,14 @@ FileSystemClient, ) - # ------------------------------------------------------------------------------ -TEST_FILE_SYSTEM_PREFIX = 'filesystem' +TEST_FILE_SYSTEM_PREFIX = "filesystem" # ------------------------------------------------------------------------------ class TestFileSystemAsync(AsyncStorageRecordedTestCase): def _setUp(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=account_key.secret, logging_enable=True) self.config = self.dsc._config self.test_file_systems = [] @@ -79,7 +78,6 @@ def _is_almost_equal(self, first, second, delta): return True return False - # --Test cases for file system --------------------------------------------- @DataLakePreparer() @@ -110,7 +108,7 @@ async def test_create_file_system_async_extra_backslash(self, **kwargs): file_system_name = self._get_file_system_reference() # Act - file_system_client = self.dsc.get_file_system_client(file_system_name + '/') + file_system_client = self.dsc.get_file_system_client(file_system_name + "/") created = await file_system_client.create_file_system() # Assert @@ -134,8 +132,8 @@ async def test_create_file_system_encryption_scope(self, **kwargs): # Assert assert props - assert props['encryption_scope'] is not None - assert props['encryption_scope'].default_encryption_scope == encryption_scope.default_encryption_scope + assert props["encryption_scope"] is not None + assert props["encryption_scope"].default_encryption_scope == encryption_scope.default_encryption_scope @DataLakePreparer() @recorded_by_proxy_async @@ -145,7 +143,7 @@ async def test_create_file_system_encryption_scope_account_sas(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") token = self.generate_sas( generate_account_sas, self.dsc.account_name, @@ -153,7 +151,8 @@ async def test_create_file_system_encryption_scope_account_sas(self, **kwargs): ResourceTypes(service=True, file_system=True, object=True), permission=AccountSasPermissions(write=True, read=True, create=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=5), - encryption_scope="hnstestscope1") + encryption_scope="hnstestscope1", + ) file_system_name = self._get_file_system_reference() encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") @@ -162,10 +161,10 @@ async def test_create_file_system_encryption_scope_account_sas(self, **kwargs): await file_system_client.create_file_system(encryption_scope_options=encryption_scope) fsc_sas = FileSystemClient(url, file_system_name, token) - await fsc_sas.create_file('file1') - await fsc_sas.create_directory('dir1') - dir_props = await fsc_sas.get_directory_client('dir1').get_directory_properties() - file_props = await fsc_sas.get_file_client('file1').get_file_properties() + await fsc_sas.create_file("file1") + await fsc_sas.create_directory("dir1") + dir_props = await fsc_sas.get_directory_client("dir1").get_directory_properties() + file_props = await fsc_sas.get_file_client("file1").get_file_properties() # Assert assert dir_props @@ -183,7 +182,7 @@ async def test_create_file_system_encryption_scope_file_system_sas(self, **kwarg self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() token = self.generate_sas( generate_file_system_sas, @@ -192,7 +191,8 @@ async def test_create_file_system_encryption_scope_file_system_sas(self, **kwarg self.dsc.credential.account_key, permission=FileSystemSasPermissions(write=True, read=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=5), - encryption_scope="hnstestscope1") + encryption_scope="hnstestscope1", + ) encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") # Act @@ -200,10 +200,10 @@ async def test_create_file_system_encryption_scope_file_system_sas(self, **kwarg await file_system_client.create_file_system(encryption_scope_options=encryption_scope) fsc_sas = FileSystemClient(url, file_system_name, token) - await fsc_sas.create_file('file1') - await fsc_sas.create_directory('dir1') - dir_props = await fsc_sas.get_directory_client('dir1').get_directory_properties() - file_props = await fsc_sas.get_file_client('file1').get_file_properties() + await fsc_sas.create_file("file1") + await fsc_sas.create_directory("dir1") + dir_props = await fsc_sas.get_directory_client("dir1").get_directory_properties() + file_props = await fsc_sas.get_file_client("file1").get_file_properties() # Assert assert dir_props @@ -221,24 +221,25 @@ async def test_create_file_system_encryption_scope_directory_sas(self, **kwargs) self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() token = self.generate_sas( generate_directory_sas, self.dsc.account_name, file_system_name, - 'dir1', + "dir1", self.dsc.credential.account_key, permission=FileSasPermissions(write=True, read=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=5), - encryption_scope="hnstestscope1") + encryption_scope="hnstestscope1", + ) encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") # Act file_system_client = self.dsc.get_file_system_client(file_system_name) await file_system_client.create_file_system(encryption_scope_options=encryption_scope) - dir_client = DataLakeDirectoryClient(url, file_system_name, 'dir1', credential=token) + dir_client = DataLakeDirectoryClient(url, file_system_name, "dir1", credential=token) await dir_client.create_directory() dir_props = await dir_client.get_directory_properties() @@ -255,26 +256,27 @@ async def test_create_file_system_encryption_scope_file_sas(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() token = self.generate_sas( generate_file_sas, self.dsc.account_name, file_system_name, - 'dir1', - 'file1', + "dir1", + "file1", self.dsc.credential.account_key, permission=FileSasPermissions(write=True, read=True, delete=True), expiry=datetime.utcnow() + timedelta(hours=5), - encryption_scope="hnstestscope1") + encryption_scope="hnstestscope1", + ) encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") # Act file_system_client = self.dsc.get_file_system_client(file_system_name) await file_system_client.create_file_system(encryption_scope_options=encryption_scope) - await file_system_client.create_directory('dir1') + await file_system_client.create_directory("dir1") - file_client = DataLakeFileClient(url, file_system_name, 'dir1/file1', token) + file_client = DataLakeFileClient(url, file_system_name, "dir1/file1", token) await file_client.create_file() file_props = await file_client.get_file_properties() @@ -309,7 +311,7 @@ async def test_create_file_system_with_metadata_async(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} file_system_name = self._get_file_system_reference() # Act @@ -353,8 +355,8 @@ async def test_list_file_systems_encryption_scope(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - file_system_name1 = self._get_file_system_reference(prefix='es') - file_system_name2 = self._get_file_system_reference(prefix='es2') + file_system_name1 = self._get_file_system_reference(prefix="es") + file_system_name2 = self._get_file_system_reference(prefix="es2") encryption_scope = EncryptionScopeOptions(default_encryption_scope="hnstestscope1") await self.dsc.create_file_system(file_system_name1, encryption_scope_options=encryption_scope) await self.dsc.create_file_system(file_system_name2, encryption_scope_options=encryption_scope) @@ -362,7 +364,7 @@ async def test_list_file_systems_encryption_scope(self, **kwargs): # Act file_systems = [] async for filesystem in self.dsc.list_file_systems(): - if filesystem['name'] in [file_system_name1, file_system_name2]: + if filesystem["name"] in [file_system_name1, file_system_name2]: file_systems.append(filesystem) # Assert @@ -391,7 +393,7 @@ async def test_list_file_systems_account_sas(self, **kwargs): ) # Act - dsc = DataLakeServiceClient(self.account_url(datalake_storage_account_name, 'dfs'), credential=sas_token) + dsc = DataLakeServiceClient(self.account_url(datalake_storage_account_name, "dfs"), credential=sas_token) file_systems = [] async for filesystem in dsc.list_file_systems(): file_systems.append(filesystem) @@ -580,13 +582,14 @@ async def test_list_file_systems_with_include_metadata_async(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = await self._create_file_system() - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} await file_system.set_file_system_metadata(metadata) # Act file_systems = [] - async for fs in self.dsc.list_file_systems(name_starts_with=file_system.file_system_name, - include_metadata=True): + async for fs in self.dsc.list_file_systems( + name_starts_with=file_system.file_system_name, include_metadata=True + ): file_systems.append(fs) # Assert @@ -602,33 +605,34 @@ async def test_list_file_systems_with_include_metadata_async(self, **kwargs): async def test_set_file_system_acl(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - variables = kwargs.pop('variables', {}) + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Act file_system = await self._create_file_system() - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) - start_time = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - access_policy = AccessPolicy(permission=FileSystemSasPermissions(read=True), - expiry=expiry_time, - start=start_time) - signed_identifier1 = {'testid': access_policy} + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) + start_time = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + access_policy = AccessPolicy( + permission=FileSystemSasPermissions(read=True), expiry=expiry_time, start=start_time + ) + signed_identifier1 = {"testid": access_policy} response = await file_system.set_file_system_access_policy( - signed_identifier1, public_access=PublicAccess.FileSystem) + signed_identifier1, public_access=PublicAccess.FileSystem + ) - assert response.get('etag') is not None - assert response.get('last_modified') is not None + assert response.get("etag") is not None + assert response.get("last_modified") is not None acl1 = await file_system.get_file_system_access_policy() - assert acl1['public_access'] is not None - assert len(acl1['signed_identifiers']) == 1 + assert acl1["public_access"] is not None + assert len(acl1["signed_identifiers"]) == 1 # If set signed identifier without specifying the access policy then it will be default to None - signed_identifier2 = {'testid': access_policy, 'test2': access_policy} + signed_identifier2 = {"testid": access_policy, "test2": access_policy} await file_system.set_file_system_access_policy(signed_identifier2) acl2 = await file_system.get_file_system_access_policy() - assert acl2['public_access'] is None - assert len(acl2['signed_identifiers']) == 2 + assert acl2["public_access"] is None + assert len(acl2["signed_identifiers"]) == 2 return variables @@ -645,10 +649,11 @@ async def test_list_file_systems_by_page(self, **kwargs): # Act file_systems = [] - async for fs in await self.dsc.list_file_systems( - results_per_page=3, - name_starts_with="file", - include_metadata=True).by_page().__anext__(): + async for fs in ( + await self.dsc.list_file_systems(results_per_page=3, name_starts_with="file", include_metadata=True) + .by_page() + .__anext__() + ): file_systems.append(fs) # Assert @@ -667,13 +672,14 @@ async def test_list_file_systems_with_public_access_async(self, **kwargs): file_system_name = self._get_file_system_reference() file_system = self.dsc.get_file_system_client(file_system_name) await file_system.create_file_system(public_access="blob") - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} await file_system.set_file_system_metadata(metadata) # Act file_systems = [] - async for fs in self.dsc.list_file_systems(name_starts_with=file_system.file_system_name, - include_metadata=True): + async for fs in self.dsc.list_file_systems( + name_starts_with=file_system.file_system_name, include_metadata=True + ): file_systems.append(fs) # Assert @@ -692,7 +698,7 @@ async def test_get_file_system_properties(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - metadata = {'hello': 'world', 'number': '42'} + metadata = {"hello": "world", "number": "42"} file_system = await self._create_file_system() await file_system.set_file_system_metadata(metadata) @@ -714,8 +720,7 @@ async def test_service_client_session_closes_after_filesystem_creation(self, **k self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange dsc2 = DataLakeServiceClient(self.dsc.url, credential=datalake_storage_account_key.secret) - async with DataLakeServiceClient( - self.dsc.url, credential=datalake_storage_account_key.secret) as ds_client: + async with DataLakeServiceClient(self.dsc.url, credential=datalake_storage_account_key.secret) as ds_client: fs1 = await ds_client.create_file_system(self._get_file_system_reference(prefix="fs1")) await fs1.delete_file_system() await dsc2.create_file_system(self._get_file_system_reference(prefix="fs2")) @@ -744,14 +749,14 @@ async def test_list_paths(self, **kwargs): async def test_list_paths_create_expiry(self, **kwargs): datalake_storage_account_name = kwargs.pop("datalake_storage_account_name") datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") - variables = kwargs.pop('variables', {}) + variables = kwargs.pop("variables", {}) self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = await self._create_file_system() - file_client = await file_system.create_file('file1') + file_client = await file_system.create_file("file1") - expiry_time = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(days=1)) + expiry_time = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(days=1)) await file_client.set_file_expiry("Absolute", expires_on=expiry_time) # Act @@ -777,7 +782,7 @@ async def test_list_paths_no_expiry(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange file_system = await self._create_file_system() - await file_system.create_file('file1') + await file_system.create_file("file1") # Act paths = [] @@ -843,8 +848,9 @@ async def test_list_paths_with_max_per_page_async(self, **kwargs): async for path in await generator1.__anext__(): paths1.append(path) - generator2 = file_system.get_paths(max_results=4, upn=True) \ - .by_page(continuation_token=generator1.continuation_token) + generator2 = file_system.get_paths(max_results=4, upn=True).by_page( + continuation_token=generator1.continuation_token + ) paths2 = [] async for path in await generator2.__anext__(): paths2.append(path) @@ -871,7 +877,7 @@ async def test_list_paths_under_specific_path_async(self, **kwargs): # create a file under the current directory file_client = await subdir.create_file("file") - await file_client.append_data(b"abced", 0, 5) # cspell:disable-line + await file_client.append_data(b"abced", 0, 5) # cspell:disable-line await file_client.flush_data(5) generator1 = file_system.get_paths(path="dir10/subdir", max_results=2, upn=True).by_page() @@ -1000,16 +1006,17 @@ async def test_get_deleted_paths(self, **kwargs): assert len(dir3_paths) == 2 assert dir3_paths[0].deletion_id is not None assert dir3_paths[1].deletion_id is not None - assert dir3_paths[0].name == 'dir3/file_in_dir3' - assert dir3_paths[1].name == 'dir3/subdir/file_in_subdir' + assert dir3_paths[0].name == "dir3/file_in_dir3" + assert dir3_paths[1].name == "dir3/subdir/file_in_subdir" paths_generator1 = file_system.list_deleted_paths(results_per_page=2).by_page() paths1 = [] async for path in await paths_generator1.__anext__(): paths1.append(path) - paths_generator2 = file_system.list_deleted_paths(results_per_page=4) \ - .by_page(continuation_token=paths_generator1.continuation_token) + paths_generator2 = file_system.list_deleted_paths(results_per_page=4).by_page( + continuation_token=paths_generator1.continuation_token + ) paths2 = [] async for path in await paths_generator2.__anext__(): paths2.append(path) @@ -1032,11 +1039,11 @@ async def test_path_properties_encryption_scope(self, **kwargs): # Act file_system_client = self.dsc.get_file_system_client(file_system_name) await file_system_client.create_file_system(encryption_scope_options=encryption_scope) - await file_system_client.create_directory('dir1') - await file_system_client.create_file('dir1/file1') + await file_system_client.create_directory("dir1") + await file_system_client.create_file("dir1/file1") - dir_props = await file_system_client.get_directory_client('dir1').get_directory_properties() - file_props = await file_system_client.get_file_client('dir1/file1').get_file_properties() + dir_props = await file_system_client.get_directory_client("dir1").get_directory_properties() + file_props = await file_system_client.get_file_client("dir1/file1").get_file_properties() paths = [] async for path in file_system_client.get_paths(recursive=True, upn=True): paths.append(path) @@ -1094,11 +1101,11 @@ async def test_get_root_directory_client(self, **kwargs): file_system = await self._create_file_system() directory_client = file_system._get_root_directory_client() - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" await directory_client.set_access_control(acl=acl) access_control = await directory_client.get_access_control() - assert acl == access_control['acl'] + assert acl == access_control["acl"] @pytest.mark.live_test_only @DataLakePreparer() @@ -1108,21 +1115,19 @@ async def test_get_access_control_using_delegation_sas_async(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") token_credential = self.get_credential(DataLakeServiceClient, is_async=True) dsc = DataLakeServiceClient(url, token_credential, logging_enable=True) file_system_name = self._get_file_system_reference() - directory_client_name = '/' + directory_client_name = "/" (await dsc.create_file_system(file_system_name)).get_directory_client(directory_client_name) directory_client = self.dsc.get_directory_client(file_system_name, directory_client_name) random_guid = uuid.uuid4() - await directory_client.set_access_control(owner=random_guid, - permissions='0777') + await directory_client.set_access_control(owner=random_guid, permissions="0777") acl = await directory_client.get_access_control() - delegation_key = await dsc.get_user_delegation_key(datetime.utcnow(), - datetime.utcnow() + timedelta(hours=1)) + delegation_key = await dsc.get_user_delegation_key(datetime.utcnow(), datetime.utcnow() + timedelta(hours=1)) token = self.generate_sas( generate_file_system_sas, @@ -1130,12 +1135,14 @@ async def test_get_access_control_using_delegation_sas_async(self, **kwargs): file_system_name, delegation_key, permission=FileSystemSasPermissions( - read=True, execute=True, manage_access_control=True, manage_ownership=True), + read=True, execute=True, manage_access_control=True, manage_ownership=True + ), expiry=datetime.utcnow() + timedelta(hours=1), - agent_object_id=random_guid + agent_object_id=random_guid, + ) + sas_directory_client = DataLakeDirectoryClient( + self.dsc.url, file_system_name, directory_client_name, credential=token, logging_enable=True ) - sas_directory_client = DataLakeDirectoryClient(self.dsc.url, file_system_name, directory_client_name, - credential=token, logging_enable=True) access_control = await sas_directory_client.get_access_control() assert access_control is not None @@ -1147,18 +1154,17 @@ async def test_list_paths_using_file_sys_delegation_sas_async(self, **kwargs): datalake_storage_account_key = kwargs.pop("datalake_storage_account_key") self._setUp(datalake_storage_account_name, datalake_storage_account_key) - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") token_credential = self.get_credential(DataLakeServiceClient, is_async=True) dsc = DataLakeServiceClient(url, token_credential) file_system_name = self._get_file_system_reference() - directory_client_name = '/' + directory_client_name = "/" directory_client = (await dsc.create_file_system(file_system_name)).get_directory_client(directory_client_name) random_guid = uuid.uuid4() - await directory_client.set_access_control(owner=random_guid, permissions='0777') + await directory_client.set_access_control(owner=random_guid, permissions="0777") - delegation_key = await dsc.get_user_delegation_key(datetime.utcnow(), - datetime.utcnow() + timedelta(hours=1)) + delegation_key = await dsc.get_user_delegation_key(datetime.utcnow(), datetime.utcnow() + timedelta(hours=1)) token = self.generate_sas( generate_file_system_sas, @@ -1167,10 +1173,9 @@ async def test_list_paths_using_file_sys_delegation_sas_async(self, **kwargs): delegation_key, permission=DirectorySasPermissions(list=True), expiry=datetime.utcnow() + timedelta(hours=1), - agent_object_id=random_guid + agent_object_id=random_guid, ) - sas_directory_client = FileSystemClient(self.dsc.url, file_system_name, - credential=token) + sas_directory_client = FileSystemClient(self.dsc.url, file_system_name, credential=token) paths = [] async for path in sas_directory_client.get_paths(): paths.append(path) @@ -1203,12 +1208,12 @@ async def test_undelete_dir_with_version_id(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_system_client = await self._create_file_system("fs2") - dir_path = 'dir10' + dir_path = "dir10" dir_client = await file_system_client.create_directory(dir_path) resp = await dir_client.delete_directory() with pytest.raises(HttpResponseError): await file_system_client.get_file_client(dir_path).get_file_properties() - restored_dir_client = await file_system_client._undelete_path(dir_path, resp['deletion_id']) + restored_dir_client = await file_system_client._undelete_path(dir_path, resp["deletion_id"]) resp = await restored_dir_client.get_directory_properties() assert resp is not None @@ -1220,12 +1225,12 @@ async def test_undelete_file_with_version_id(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) file_system_client = await self._create_file_system("fs2") - file_path = 'dir10/fileŇ' + file_path = "dir10/fileŇ" dir_client = await file_system_client.create_file(file_path) resp = await dir_client.delete_file() with pytest.raises(HttpResponseError): await file_system_client.get_file_client(file_path).get_file_properties() - restored_file_client = await file_system_client._undelete_path(file_path, resp['deletion_id']) + restored_file_client = await file_system_client._undelete_path(file_path, resp["deletion_id"]) resp = await restored_file_client.get_file_properties() assert resp is not None @@ -1237,23 +1242,24 @@ async def test_storage_account_audience_service_client(self, **kwargs): # Arrange self._setUp(datalake_storage_account_name, datalake_storage_account_key) - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() file_system_client = self.dsc.get_file_system_client(file_system_name) await file_system_client.create_file_system() - await file_system_client.create_directory('testdir1') + await file_system_client.create_directory("testdir1") # Act token_credential = self.get_credential(DataLakeServiceClient, is_async=True) fsc = FileSystemClient( - url, file_system_name, + url, + file_system_name, credential=token_credential, - audience=f'https://{datalake_storage_account_name}.blob.core.windows.net/' + audience=f"https://{datalake_storage_account_name}.blob.core.windows.net/", ) # Assert response1 = await fsc.exists() - response2 = await fsc.create_directory('testdir11') + response2 = await fsc.create_directory("testdir11") assert response1 is not None assert response2 is not None @@ -1265,23 +1271,21 @@ async def test_bad_audience_service_client(self, **kwargs): # Arrange self._setUp(datalake_storage_account_name, datalake_storage_account_key) - url = self.account_url(datalake_storage_account_name, 'dfs') + url = self.account_url(datalake_storage_account_name, "dfs") file_system_name = self._get_file_system_reference() file_system_client = self.dsc.get_file_system_client(file_system_name) await file_system_client.create_file_system() - await file_system_client.create_directory('testdir2') + await file_system_client.create_directory("testdir2") # Act token_credential = self.get_credential(DataLakeServiceClient, is_async=True) fsc = FileSystemClient( - url, file_system_name, - credential=token_credential, - audience='https://badaudience.blob.core.windows.net/' + url, file_system_name, credential=token_credential, audience="https://badaudience.blob.core.windows.net/" ) # Will not raise ClientAuthenticationError despite bad audience due to Bearer Challenge await fsc.exists() - await fsc.create_directory('testdir22') + await fsc.create_directory("testdir22") @DataLakePreparer() @recorded_by_proxy_async @@ -1290,20 +1294,17 @@ async def test_get_and_set_access_control_oauth(self, **kwargs): # Arrange token_credential = self.get_credential(DataLakeServiceClient, is_async=True) - dsc = DataLakeServiceClient( - self.account_url(datalake_storage_account_name, 'dfs'), - token_credential - ) + dsc = DataLakeServiceClient(self.account_url(datalake_storage_account_name, "dfs"), token_credential) file_system = await dsc.create_file_system(self.get_resource_name(TEST_FILE_SYSTEM_PREFIX)) directory_client = file_system._get_root_directory_client() # Act - acl = 'user::rwx,group::r-x,other::rwx' + acl = "user::rwx,group::r-x,other::rwx" await directory_client.set_access_control(acl=acl) access_control = await directory_client.get_access_control() # Assert - assert acl == access_control['acl'] + assert acl == access_control["acl"] @DataLakePreparer() @recorded_by_proxy_async @@ -1316,8 +1317,8 @@ async def test_get_user_delegation_sas(self, **kwargs): self.account_url(datalake_storage_account_name, "dfs"), credential=token_credential, ) - start = self.get_datetime_variable(variables, 'start_time', datetime.utcnow()) - expiry = self.get_datetime_variable(variables, 'expiry_time', datetime.utcnow() + timedelta(hours=1)) + start = self.get_datetime_variable(variables, "start_time", datetime.utcnow()) + expiry = self.get_datetime_variable(variables, "expiry_time", datetime.utcnow() + timedelta(hours=1)) user_delegation_key_1 = await service.get_user_delegation_key(key_start_time=start, key_expiry_time=expiry) user_delegation_key_2 = await service.get_user_delegation_key(key_start_time=start, key_expiry_time=expiry) @@ -1365,9 +1366,7 @@ async def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: user_delegation_oid = decoded.get("oid") delegated_user_tid = decoded.get("tid") user_delegation_key = await dsc.get_user_delegation_key( - key_start_time=start, - key_expiry_time=expiry, - delegated_user_tid=delegated_user_tid + key_start_time=start, key_expiry_time=expiry, delegated_user_tid=delegated_user_tid ) assert user_delegation_key is not None @@ -1387,9 +1386,7 @@ async def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: assert "skdutid=" + delegated_user_tid in file_system_token file_system_client = FileSystemClient( - f"{account_url}?{file_system_token}", - file_system_name=file_system_name, - credential=token_credential + f"{account_url}?{file_system_token}", file_system_name=file_system_name, credential=token_credential ) paths = [] @@ -1408,7 +1405,7 @@ async def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: user_delegation_key, permission=FileSasPermissions(write=True, read=True, delete=True), expiry=expiry, - user_delegation_oid=user_delegation_oid + user_delegation_oid=user_delegation_oid, ) assert "sduoid=" + user_delegation_oid in directory_token @@ -1418,7 +1415,7 @@ async def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: f"{account_url}?{directory_token}", file_system_name=file_system_name, directory_name=directory_name, - credential=token_credential + credential=token_credential, ) props = await directory_client.get_directory_properties() assert props is not None @@ -1432,7 +1429,7 @@ async def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: credential=user_delegation_key, permission=FileSasPermissions(write=True, read=True, delete=True), expiry=expiry, - user_delegation_oid=user_delegation_oid + user_delegation_oid=user_delegation_oid, ) assert "sduoid=" + user_delegation_oid in file_token @@ -1442,11 +1439,12 @@ async def test_datalake_cross_tenant_delegation_sas(self, **kwargs): # pylint: f"{account_url}?{file_token}", file_system_name=file_system_name, file_path=f"{directory_name}/{file_name}", - credential=token_credential + credential=token_credential, ) content = await (await file_client.download_file()).readall() assert content == data + # ------------------------------------------------------------------------------ -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py b/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py index 6549cdbca590..9d15ecda0eac 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py @@ -35,11 +35,7 @@ def assert_complete(self): class MockHttpClientResponse(Response): def __init__( - self, url: str, - body_bytes: bytes, - headers: Dict[str, Any], - status: int = 200, - reason: str = "OK" + self, url: str, body_bytes: bytes, headers: Dict[str, Any], status: int = 200, reason: str = "OK" ) -> None: super(MockHttpClientResponse).__init__() self._url = url @@ -59,8 +55,9 @@ class MockStorageTransport(HttpTransport): This transport returns legacy http response objects from azure core and is intended only to test our backwards compatibility support. """ + def send(self, request: HttpRequest, **kwargs: Any) -> RequestsTransportResponse: - if request.method == 'GET': + if request.method == "GET": # download_file headers = { "Content-Type": "application/octet-stream", @@ -77,9 +74,9 @@ def send(self, request: HttpRequest, **kwargs: Any) -> RequestsTransportResponse request.url, b"Hello World!", headers, - ) + ), ) - elif request.method == 'HEAD': + elif request.method == "HEAD": # get_file_properties rest_response = RequestsTransportResponse( request=request, @@ -90,9 +87,9 @@ def send(self, request: HttpRequest, **kwargs: Any) -> RequestsTransportResponse "Content-Type": "application/octet-stream", "Content-Length": "1024", }, - ) + ), ) - elif request.method == 'PUT': + elif request.method == "PUT": # upload_data rest_response = RequestsTransportResponse( request=request, @@ -103,10 +100,10 @@ def send(self, request: HttpRequest, **kwargs: Any) -> RequestsTransportResponse "Content-Length": "0", }, 201, - "Created" - ) + "Created", + ), ) - elif request.method == 'PATCH': + elif request.method == "PATCH": # upload_data_chunks parsed = urlparse(request.url) if "action=flush" in parsed.query: @@ -119,8 +116,8 @@ def send(self, request: HttpRequest, **kwargs: Any) -> RequestsTransportResponse "Content-Length": "0", }, 200, - "OK" - ) + "OK", + ), ) else: rest_response = RequestsTransportResponse( @@ -132,10 +129,10 @@ def send(self, request: HttpRequest, **kwargs: Any) -> RequestsTransportResponse "Content-Length": "0", }, 202, - "Accepted" - ) + "Accepted", + ), ) - elif request.method == 'DELETE': + elif request.method == "DELETE": # delete_file rest_response = RequestsTransportResponse( request=request, @@ -146,8 +143,8 @@ def send(self, request: HttpRequest, **kwargs: Any) -> RequestsTransportResponse "Content-Length": "0", }, 202, - "Accepted" - ) + "Accepted", + ), ) else: raise ValueError("The request is not accepted as part of MockStorageTransport.") diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py index ab4e99ea7652..95196774c350 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py @@ -12,7 +12,10 @@ from aiohttp.client_proto import ResponseHandler from aiohttp.streams import StreamReader -from azure.core.pipeline.transport import AioHttpTransportResponse, AsyncHttpTransport # pylint: disable=no-name-in-module +from azure.core.pipeline.transport import ( + AioHttpTransportResponse, + AsyncHttpTransport, +) # pylint: disable=no-name-in-module from azure.core.rest import HttpRequest @@ -56,11 +59,7 @@ async def read(self, size: int = -1) -> bytes: class MockAioHttpClientResponse(ClientResponse): def __init__( - self, url: str, - body_bytes: bytes, - headers: Dict[str, Any], - status: int = 200, - reason: str = "OK" + self, url: str, body_bytes: bytes, headers: Dict[str, Any], status: int = 200, reason: str = "OK" ) -> None: super(MockAioHttpClientResponse).__init__() self._url = url @@ -78,11 +77,12 @@ def __init__( class MockStorageTransport(AsyncHttpTransport): """ - This transport returns legacy http response objects from azure core and is + This transport returns legacy http response objects from azure core and is intended only to test our backwards compatibility support. """ + async def send(self, request: HttpRequest, **kwargs: Any) -> AioHttpTransportResponse: - if request.method == 'GET': + if request.method == "GET": # download_file headers = { "Content-Type": "application/octet-stream", @@ -100,9 +100,9 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AioHttpTransportRes b"Hello Async World!", headers, ), - decompress=False + decompress=False, ) - elif request.method == 'HEAD': + elif request.method == "HEAD": # get_file_properties rest_response = AioHttpTransportResponse( request=request, @@ -114,9 +114,9 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AioHttpTransportRes "Content-Length": "1024", }, ), - decompress=False + decompress=False, ) - elif request.method == 'PUT': + elif request.method == "PUT": # upload_data rest_response = AioHttpTransportResponse( request=request, @@ -127,11 +127,11 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AioHttpTransportRes "Content-Length": "0", }, 201, - "Created" + "Created", ), - decompress=False + decompress=False, ) - elif request.method == 'PATCH': + elif request.method == "PATCH": # upload_data_chunks parsed = urlparse(request.url) if "action=flush" in parsed.query: @@ -144,9 +144,9 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AioHttpTransportRes "Content-Length": "0", }, 200, - "OK" + "OK", ), - decompress=False + decompress=False, ) else: rest_response = AioHttpTransportResponse( @@ -158,11 +158,11 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AioHttpTransportRes "Content-Length": "0", }, 202, - "Accepted" + "Accepted", ), - decompress=False + decompress=False, ) - elif request.method == 'DELETE': + elif request.method == "DELETE": # delete_file rest_response = AioHttpTransportResponse( request=request, @@ -173,9 +173,9 @@ async def send(self, request: HttpRequest, **kwargs: Any) -> AioHttpTransportRes "Content-Length": "0", }, 202, - "Accepted" + "Accepted", ), - decompress=False + decompress=False, ) else: raise ValueError("The request is not accepted as part of MockStorageTransport.") diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_large_file.py b/sdk/storage/azure-storage-file-datalake/tests/test_large_file.py index 92ad41d49281..2930ca22e15b 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_large_file.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_large_file.py @@ -19,30 +19,28 @@ from azure.storage.blob._shared.base_client import _format_shared_key_credential from azure.storage.filedatalake import DataLakeServiceClient - # ------------------------------------------------------------------------------ -TEST_DIRECTORY_PREFIX = 'directory' -TEST_FILE_PREFIX = 'file' -FILE_PATH = 'file_output.temp.dat' +TEST_DIRECTORY_PREFIX = "directory" +TEST_FILE_PREFIX = "file" +FILE_PATH = "file_output.temp.dat" LARGEST_BLOCK_SIZE = 4000 * 1024 * 1024 # ------------------------------------------------------------------------------ class TestLargeFile(StorageRecordedTestCase): def _setUp(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.payload_dropping_policy = PayloadDroppingPolicy() - credential_policy = _format_shared_key_credential(account_name, - account_key.secret) + credential_policy = _format_shared_key_credential(account_name, account_key.secret) self.dsc = DataLakeServiceClient( url, credential=account_key.secret, logging_enable=True, - _additional_pipeline_policies=[self.payload_dropping_policy, credential_policy] + _additional_pipeline_policies=[self.payload_dropping_policy, credential_policy], ) self.config = self.dsc._config - self.file_system_name = self.get_resource_name('filesystem') + self.file_system_name = self.get_resource_name("filesystem") if not self.is_playback(): file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -64,7 +62,7 @@ def test_append_large_stream_without_network(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file() data = LargeStream(LARGEST_BLOCK_SIZE) @@ -77,8 +75,7 @@ def test_append_large_stream_without_network(self, **kwargs): assert self.payload_dropping_policy.append_sizes[0] == LARGEST_BLOCK_SIZE @pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="Test failing on Pypy3 Linux, skip to investigate" + platform.python_implementation() == "PyPy", reason="Test failing on Pypy3 Linux, skip to investigate" ) @pytest.mark.live_test_only @DataLakePreparer() @@ -94,10 +91,10 @@ def test_upload_large_stream_without_network(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") file_client.create_file() - length = 2*LARGEST_BLOCK_SIZE + length = 2 * LARGEST_BLOCK_SIZE data = LargeStream(length) # Act @@ -110,7 +107,7 @@ def test_upload_large_stream_without_network(self, **kwargs): class LargeStream: - def __init__(self, length, initial_buffer_length=1024*1024): + def __init__(self, length, initial_buffer_length=1024 * 1024): self._base_data = urandom(initial_buffer_length) self._base_data_length = initial_buffer_length self._position = 0 @@ -144,9 +141,10 @@ def __init__(self): def send(self, request): # type: (PipelineRequest) -> PipelineResponse if _is_append_request(request): if request.http_request.body: - position = self.append_counter*len(self.dummy_body) + position = self.append_counter * len(self.dummy_body) request.http_request.url = re.sub( - r'position=\d+', "position=" + str(position), request.http_request.url) + r"position=\d+", "position=" + str(position), request.http_request.url + ) self.append_sizes.append(_get_body_length(request)) replacement = self.dummy_body request.http_request.body = replacement @@ -154,8 +152,7 @@ def send(self, request): # type: (PipelineRequest) -> PipelineResponse self.append_counter = self.append_counter + 1 if _is_flush_request(request): position = self.append_counter * len(self.dummy_body) - request.http_request.url = re.sub( - r'position=\d+', "position=" + str(position), request.http_request.url) + request.http_request.url = re.sub(r"position=\d+", "position=" + str(position), request.http_request.url) return self.next.send(request) @@ -173,7 +170,7 @@ def _get_body_length(request): body = request.http_request.body length = 0 if hasattr(body, "read"): - chunk = body.read(10*1024*1024) + chunk = body.read(10 * 1024 * 1024) while chunk: length = length + len(chunk) chunk = body.read(10 * 1024 * 1024) @@ -183,5 +180,5 @@ def _get_body_length(request): # ------------------------------------------------------------------------------ -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_large_file_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_large_file_async.py index e630e4beec1d..af3e25f64790 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_large_file_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_large_file_async.py @@ -20,29 +20,28 @@ from azure.storage.blob._shared.base_client import _format_shared_key_credential from azure.storage.filedatalake.aio import DataLakeServiceClient - # ------------------------------------------------------------------------------ -TEST_DIRECTORY_PREFIX = 'directory' -TEST_FILE_PREFIX = 'file' -FILE_PATH = 'file_output.temp.dat' +TEST_DIRECTORY_PREFIX = "directory" +TEST_FILE_PREFIX = "file" +FILE_PATH = "file_output.temp.dat" LARGEST_BLOCK_SIZE = 4000 * 1024 * 1024 # ------------------------------------------------------------------------------ class TestLargeFileAsync(AsyncStorageRecordedTestCase): async def _setUp(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.payload_dropping_policy = PayloadDroppingPolicy() credential_policy = _format_shared_key_credential(account_name, account_key.secret) self.dsc = DataLakeServiceClient( url, credential=account_key.secret, - _additional_pipeline_policies=[self.payload_dropping_policy, credential_policy] + _additional_pipeline_policies=[self.payload_dropping_policy, credential_policy], ) self.config = self.dsc._config - self.file_system_name = self.get_resource_name('filesystem') + self.file_system_name = self.get_resource_name("filesystem") if not self.is_playback(): file_system = self.dsc.get_file_system_client(self.file_system_name) @@ -71,7 +70,7 @@ async def test_append_large_stream_without_network(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file() data = LargeStream(LARGEST_BLOCK_SIZE) @@ -84,8 +83,7 @@ async def test_append_large_stream_without_network(self, **kwargs): assert self.payload_dropping_policy.append_sizes[0] == LARGEST_BLOCK_SIZE @pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="Test failing on Pypy3 Linux, skip to investigate" + platform.python_implementation() == "PyPy", reason="Test failing on Pypy3 Linux, skip to investigate" ) @pytest.mark.live_test_only @DataLakePreparer() @@ -100,10 +98,10 @@ async def test_upload_large_stream_without_network(self, **kwargs): directory_client = self.dsc.get_directory_client(self.file_system_name, directory_name) await directory_client.create_directory() - file_client = directory_client.get_file_client('filename') + file_client = directory_client.get_file_client("filename") await file_client.create_file() - length = 2*LARGEST_BLOCK_SIZE + length = 2 * LARGEST_BLOCK_SIZE data = LargeStream(length) # Act @@ -154,9 +152,10 @@ def __init__(self): def on_request(self, request): # type: (PipelineRequest) -> Union[None, Awaitable[None]] if _is_append_request(request): if request.http_request.body: - position = self.append_counter*len(self.dummy_body) + position = self.append_counter * len(self.dummy_body) request.http_request.url = re.sub( - r'position=\d+', "position=" + str(position), request.http_request.url) + r"position=\d+", "position=" + str(position), request.http_request.url + ) self.append_sizes.append(_get_body_length(request)) replacement = self.dummy_body request.http_request.body = replacement @@ -164,8 +163,7 @@ def on_request(self, request): # type: (PipelineRequest) -> Union[None, Awaitab self.append_counter = self.append_counter + 1 elif _is_flush_request(request): position = self.append_counter * len(self.dummy_body) - request.http_request.url = re.sub( - r'position=\d+', "position=" + str(position), request.http_request.url) + request.http_request.url = re.sub(r"position=\d+", "position=" + str(position), request.http_request.url) def _is_append_request(request): @@ -182,7 +180,7 @@ def _get_body_length(request): body = request.http_request.body length = 0 if hasattr(body, "read"): - chunk = body.read(10*1024*1024) + chunk = body.read(10 * 1024 * 1024) while chunk: length = length + len(chunk) chunk = body.read(10 * 1024 * 1024) @@ -192,5 +190,5 @@ def _get_body_length(request): # ------------------------------------------------------------------------------ -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_quick_query.py b/sdk/storage/azure-storage-file-datalake/tests/test_quick_query.py index 87e98e931783..ddf3610275c2 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_quick_query.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_quick_query.py @@ -21,74 +21,75 @@ QuickQueryDialect, ) from azure.storage.filedatalake import DataLakeServiceClient + # ------------------------------------------------------------------------------ CSV_DATA = ( - b'Service,Package,Version,RepoPath,MissingDocs\r\nApp Configuration,' - b'azure-data-appconfiguration,1,appconfiguration,FALSE\r\nEvent Hubs' - b'\r\nEvent Hubs - Azure Storage CheckpointStore,' - b'azure-messaging-eventhubs-checkpointstore-blob,1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity,' - b'1.1.0-beta.1,identity,FALSE\r\nKey Vault - Certificates,azure-security-keyvault-certificates,' - b'4.0.0,keyvault,FALSE\r\nKey Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault,' - b'FALSE\r\nKey Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n' - b'Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\nStorage - Blobs Batch,' - b'azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\nStorage - Blobs Cryptography,' - b'azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\nStorage - File Shares,' - b'azure-storage-file-share,12.2.0,storage,FALSE\r\nStorage - Queues,' - b'azure-storage-queue,12.3.0,storage,FALSE\r\nText Analytics,' - b'azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\nTracing,' - b'azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\nService,Package,Version,RepoPath,' - b'MissingDocs\r\nApp Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n' - b'Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n' - b'Event Hubs - Azure Storage CheckpointStore,azure-messaging-eventhubs-checkpointstore-blob,' - b'1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity,1.1.0-beta.1,identity,FALSE\r\n' - b'Key Vault - Certificates,azure-security-keyvault-certificates,4.0.0,keyvault,FALSE\r\n' - b'Key Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault,FALSE\r\n' - b'Key Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n' - b'Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\n' - b'Storage - Blobs Batch,azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\n' - b'Storage - Blobs Cryptography,azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\n' - b'Storage - File Shares,azure-storage-file-share,12.2.0,storage,FALSE\r\n' - b'Storage - Queues,azure-storage-queue,12.3.0,storage,FALSE\r\n' - b'Text Analytics,azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\n' - b'Tracing,azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\n' - b'Service,Package,Version,RepoPath,MissingDocs\r\n' - b'App Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n' - b'Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n' + b"Service,Package,Version,RepoPath,MissingDocs\r\nApp Configuration," + b"azure-data-appconfiguration,1,appconfiguration,FALSE\r\nEvent Hubs" + b"\r\nEvent Hubs - Azure Storage CheckpointStore," + b"azure-messaging-eventhubs-checkpointstore-blob,1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity," + b"1.1.0-beta.1,identity,FALSE\r\nKey Vault - Certificates,azure-security-keyvault-certificates," + b"4.0.0,keyvault,FALSE\r\nKey Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault," + b"FALSE\r\nKey Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n" + b"Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\nStorage - Blobs Batch," + b"azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\nStorage - Blobs Cryptography," + b"azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\nStorage - File Shares," + b"azure-storage-file-share,12.2.0,storage,FALSE\r\nStorage - Queues," + b"azure-storage-queue,12.3.0,storage,FALSE\r\nText Analytics," + b"azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\nTracing," + b"azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\nService,Package,Version,RepoPath," + b"MissingDocs\r\nApp Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n" + b"Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n" + b"Event Hubs - Azure Storage CheckpointStore,azure-messaging-eventhubs-checkpointstore-blob," + b"1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity,1.1.0-beta.1,identity,FALSE\r\n" + b"Key Vault - Certificates,azure-security-keyvault-certificates,4.0.0,keyvault,FALSE\r\n" + b"Key Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault,FALSE\r\n" + b"Key Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n" + b"Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\n" + b"Storage - Blobs Batch,azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\n" + b"Storage - Blobs Cryptography,azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\n" + b"Storage - File Shares,azure-storage-file-share,12.2.0,storage,FALSE\r\n" + b"Storage - Queues,azure-storage-queue,12.3.0,storage,FALSE\r\n" + b"Text Analytics,azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\n" + b"Tracing,azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\n" + b"Service,Package,Version,RepoPath,MissingDocs\r\n" + b"App Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n" + b"Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n" ) DATALAKE_CSV_DATA = ( - b'DataLakeStorage,Package,Version,RepoPath,MissingDocs\r\nApp Configuration,' - b'azure-data-appconfiguration,1,appconfiguration,FALSE\r\nEvent Hubs' - b'\r\nEvent Hubs - Azure Storage CheckpointStore,' - b'azure-messaging-eventhubs-checkpointstore-blob,1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity,' - b'1.1.0-beta.1,identity,FALSE\r\nKey Vault - Certificates,azure-security-keyvault-certificates,' - b'4.0.0,keyvault,FALSE\r\nKey Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault,' - b'FALSE\r\nKey Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n' - b'Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\nStorage - Blobs Batch,' - b'azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\nStorage - Blobs Cryptography,' - b'azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\nStorage - File Shares,' - b'azure-storage-file-share,12.2.0,storage,FALSE\r\nStorage - Queues,' - b'azure-storage-queue,12.3.0,storage,FALSE\r\nText Analytics,' - b'azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\nTracing,' - b'azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\nService,Package,Version,RepoPath,' - b'MissingDocs\r\nApp Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n' - b'Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n' - b'Event Hubs - Azure Storage CheckpointStore,azure-messaging-eventhubs-checkpointstore-blob,' - b'1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity,1.1.0-beta.1,identity,FALSE\r\n' - b'Key Vault - Certificates,azure-security-keyvault-certificates,4.0.0,keyvault,FALSE\r\n' - b'Key Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault,FALSE\r\n' - b'Key Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n' - b'Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\n' - b'Storage - Blobs Batch,azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\n' - b'Storage - Blobs Cryptography,azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\n' - b'Storage - File Shares,azure-storage-file-share,12.2.0,storage,FALSE\r\n' - b'Storage - Queues,azure-storage-queue,12.3.0,storage,FALSE\r\n' - b'Text Analytics,azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\n' - b'Tracing,azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\n' - b'Service,Package,Version,RepoPath,MissingDocs\r\n' - b'App Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n' - b'Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n' + b"DataLakeStorage,Package,Version,RepoPath,MissingDocs\r\nApp Configuration," + b"azure-data-appconfiguration,1,appconfiguration,FALSE\r\nEvent Hubs" + b"\r\nEvent Hubs - Azure Storage CheckpointStore," + b"azure-messaging-eventhubs-checkpointstore-blob,1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity," + b"1.1.0-beta.1,identity,FALSE\r\nKey Vault - Certificates,azure-security-keyvault-certificates," + b"4.0.0,keyvault,FALSE\r\nKey Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault," + b"FALSE\r\nKey Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n" + b"Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\nStorage - Blobs Batch," + b"azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\nStorage - Blobs Cryptography," + b"azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\nStorage - File Shares," + b"azure-storage-file-share,12.2.0,storage,FALSE\r\nStorage - Queues," + b"azure-storage-queue,12.3.0,storage,FALSE\r\nText Analytics," + b"azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\nTracing," + b"azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\nService,Package,Version,RepoPath," + b"MissingDocs\r\nApp Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n" + b"Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n" + b"Event Hubs - Azure Storage CheckpointStore,azure-messaging-eventhubs-checkpointstore-blob," + b"1.0.1,eventhubs,FALSE\r\nIdentity,azure-identity,1.1.0-beta.1,identity,FALSE\r\n" + b"Key Vault - Certificates,azure-security-keyvault-certificates,4.0.0,keyvault,FALSE\r\n" + b"Key Vault - Keys,azure-security-keyvault-keys,4.2.0-beta.1,keyvault,FALSE\r\n" + b"Key Vault - Secrets,azure-security-keyvault-secrets,4.1.0,keyvault,FALSE\r\n" + b"Storage - Blobs,azure-storage-blob,12.4.0,storage,FALSE\r\n" + b"Storage - Blobs Batch,azure-storage-blob-batch,12.4.0-beta.1,storage,FALSE\r\n" + b"Storage - Blobs Cryptography,azure-storage-blob-cryptography,12.4.0,storage,FALSE\r\n" + b"Storage - File Shares,azure-storage-file-share,12.2.0,storage,FALSE\r\n" + b"Storage - Queues,azure-storage-queue,12.3.0,storage,FALSE\r\n" + b"Text Analytics,azure-ai-textanalytics,1.0.0-beta.2,textanalytics,FALSE\r\n" + b"Tracing,azure-core-tracing-opentelemetry,1.0.0-beta.2,core,FALSE\r\n" + b"Service,Package,Version,RepoPath,MissingDocs\r\n" + b"App Configuration,azure-data-appconfiguration,1.0.1,appconfiguration,FALSE\r\n" + b"Event Hubs,azure-messaging-eventhubs,5.0.1,eventhubs,FALSE\r\n" ) CONVERTED_CSV_DATA = ( b"Service;Package;Version;RepoPath;MissingDocs.App Configuration;azure-data-appconfiguration;" @@ -124,10 +125,10 @@ class TestStorageQuickQuery(StorageRecordedTestCase): def _setUp(self, account_name, account_key): - url = self.account_url(account_name, 'dfs') + url = self.account_url(account_name, "dfs") self.dsc = DataLakeServiceClient(url, credential=account_key.secret, logging_enable=True) self.config = self.dsc._config - self.filesystem_name = self.get_resource_name('utqqcontainer') # cspell:disable-line + self.filesystem_name = self.get_resource_name("utqqcontainer") # cspell:disable-line if not self.is_playback(): try: @@ -166,7 +167,7 @@ def on_error(error): assert len(errors) == 0 assert len(reader) == len(CSV_DATA) assert len(reader) == reader._blob_query_reader._bytes_processed - assert data == CSV_DATA.replace(b'\r\n', b'\n') + assert data == CSV_DATA.replace(b"\r\n", b"\n") @DataLakePreparer() @recorded_by_proxy @@ -187,8 +188,9 @@ def on_error(error): errors.append(error) input_format = DelimitedTextDialect(has_header=True) - reader = file_client.query_file("SELECT DataLakeStorage from DataLakeStorage", on_error=on_error, - file_format=input_format) + reader = file_client.query_file( + "SELECT DataLakeStorage from DataLakeStorage", on_error=on_error, file_format=input_format + ) reader.readall() assert len(errors) == 0 @@ -213,14 +215,14 @@ def test_quick_query_iter_records(self, **kwargs): # Assert first line has header data = next(read_records) - assert data == b'Service,Package,Version,RepoPath,MissingDocs' + assert data == b"Service,Package,Version,RepoPath,MissingDocs" for record in read_records: data += record assert len(reader) == len(CSV_DATA) assert len(reader) == reader._blob_query_reader._bytes_processed - assert data == CSV_DATA.replace(b'\r\n', b'') + assert data == CSV_DATA.replace(b"\r\n", b"") @DataLakePreparer() @recorded_by_proxy @@ -240,13 +242,13 @@ def test_quick_query_readall_with_encoding(self, **kwargs): def on_error(error): errors.append(error) - reader = file_client.query_file("SELECT * from BlobStorage", on_error=on_error, encoding='utf-8') + reader = file_client.query_file("SELECT * from BlobStorage", on_error=on_error, encoding="utf-8") data = reader.readall() assert len(errors) == 0 assert len(reader) == len(CSV_DATA) assert len(reader) == reader._blob_query_reader._bytes_processed - assert data == CSV_DATA.replace(b'\r\n', b'\n').decode('utf-8') + assert data == CSV_DATA.replace(b"\r\n", b"\n").decode("utf-8") @DataLakePreparer() @recorded_by_proxy @@ -261,12 +263,12 @@ def test_quick_query_iter_records_with_encoding(self, **kwargs): file_client = self.dsc.get_file_client(self.filesystem_name, file_name) file_client.upload_data(CSV_DATA, overwrite=True) - reader = file_client.query_file("SELECT * from BlobStorage", encoding='utf-8') + reader = file_client.query_file("SELECT * from BlobStorage", encoding="utf-8") data = "".join(record for record in reader.records()) assert len(reader) == len(CSV_DATA) assert len(reader) == reader._blob_query_reader._bytes_processed - assert data == CSV_DATA.replace(b'\r\n', b'').decode('utf-8') + assert data == CSV_DATA.replace(b"\r\n", b"").decode("utf-8") @DataLakePreparer() @recorded_by_proxy @@ -284,22 +286,20 @@ def test_quick_query_iter_output_records_excluding_headers(self, **kwargs): input_format = DelimitedTextDialect(has_header=True) output_format = DelimitedTextDialect(has_header=False) reader = file_client.query_file( - "SELECT * from BlobStorage", - file_format=input_format, - output_format=output_format + "SELECT * from BlobStorage", file_format=input_format, output_format=output_format ) read_records = reader.records() # Assert first line does not include header data = next(read_records) - assert data == b'App Configuration,azure-data-appconfiguration,1,appconfiguration,FALSE' + assert data == b"App Configuration,azure-data-appconfiguration,1,appconfiguration,FALSE" for record in read_records: data += record assert len(reader) == len(CSV_DATA) assert len(reader) == reader._blob_query_reader._bytes_processed - assert data == CSV_DATA.replace(b'\r\n', b'')[44:] + assert data == CSV_DATA.replace(b"\r\n", b"")[44:] @DataLakePreparer() @recorded_by_proxy @@ -320,14 +320,14 @@ def test_quick_query_iter_output_records_including_headers(self, **kwargs): # Assert first line does not include header data = next(read_records) - assert data == b'Service,Package,Version,RepoPath,MissingDocs' + assert data == b"Service,Package,Version,RepoPath,MissingDocs" for record in read_records: data += record assert len(reader) == len(CSV_DATA) assert len(reader) == reader._blob_query_reader._bytes_processed - assert data == CSV_DATA.replace(b'\r\n', b'') + assert data == CSV_DATA.replace(b"\r\n", b"") @DataLakePreparer() @recorded_by_proxy @@ -343,7 +343,7 @@ def test_quick_query_iter_records_with_progress(self, **kwargs): file_client.upload_data(CSV_DATA, overwrite=True) reader = file_client.query_file("SELECT * from BlobStorage") - data = b'' + data = b"" progress = 0 for record in reader.records(): if record: @@ -351,7 +351,7 @@ def test_quick_query_iter_records_with_progress(self, **kwargs): progress += len(record) + 2 assert len(reader) == len(CSV_DATA) assert len(reader) == reader._blob_query_reader._bytes_processed - assert data == CSV_DATA.replace(b'\r\n', b'') + assert data == CSV_DATA.replace(b"\r\n", b"") assert progress == len(reader) @DataLakePreparer() @@ -373,23 +373,12 @@ def on_error(error): errors.append(error) input_format = DelimitedTextDialect( - delimiter=',', - quotechar='"', - lineterminator='\n', - escapechar='', - has_header=False - ) - output_format = DelimitedTextDialect( - delimiter=';', - quotechar="'", - lineterminator='.', - escapechar='\\' + delimiter=",", quotechar='"', lineterminator="\n", escapechar="", has_header=False ) + output_format = DelimitedTextDialect(delimiter=";", quotechar="'", lineterminator=".", escapechar="\\") resp = file_client.query_file( - "SELECT * from BlobStorage", - on_error=on_error, - file_format=input_format, - output_format=output_format) + "SELECT * from BlobStorage", on_error=on_error, file_format=input_format, output_format=output_format + ) query_result = resp.readall() assert len(errors) == 0 @@ -410,23 +399,13 @@ def test_quick_query_iter_records_with_serialization_setting(self, **kwargs): file_client.upload_data(CSV_DATA, overwrite=True) input_format = DelimitedTextDialect( - delimiter=',', - quotechar='"', - lineterminator='\n', - escapechar='', - has_header=False - ) - output_format = DelimitedTextDialect( - delimiter=';', - quotechar="'", - lineterminator='%', - escapechar='\\' + delimiter=",", quotechar='"', lineterminator="\n", escapechar="", has_header=False ) + output_format = DelimitedTextDialect(delimiter=";", quotechar="'", lineterminator="%", escapechar="\\") reader = file_client.query_file( - "SELECT * from BlobStorage", - file_format=input_format, - output_format=output_format) + "SELECT * from BlobStorage", file_format=input_format, output_format=output_format + ) data = [] for record in reader.records(): if record: @@ -444,16 +423,16 @@ def test_quick_query_readall_with_fatal_error_handler(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data1 = b'{name: owner}' - data2 = b'{name2: owner2}' + data1 = b"{name: owner}" + data2 = b"{name2: owner2}" data3 = ( - b'{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:' - b'{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3,' - b'shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:' - b'{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z,' - b'data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}' + b"{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:" + b"{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3," + b"shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:" + b"{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z," + b"data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}" ) - data = data1 + b'\n' + data2 + b'\n' + data3 + data = data1 + b"\n" + data2 + b"\n" + data3 # upload the json file file_name = self._get_file_reference() @@ -466,22 +445,15 @@ def on_error(error): errors.append(error) input_format = DelimitedJsonDialect() - output_format = DelimitedTextDialect( - delimiter=';', - quotechar="'", - lineterminator='.', - escapechar='\\' - ) + output_format = DelimitedTextDialect(delimiter=";", quotechar="'", lineterminator=".", escapechar="\\") resp = file_client.query_file( - "SELECT * from BlobStorage", - on_error=on_error, - file_format=input_format, - output_format=output_format) + "SELECT * from BlobStorage", on_error=on_error, file_format=input_format, output_format=output_format + ) query_result = resp.readall() assert len(errors) == 1 assert len(resp) == len(data) - assert query_result == b'' + assert query_result == b"" @DataLakePreparer() @recorded_by_proxy @@ -491,16 +463,16 @@ def test_quick_query_iter_records_with_fatal_error_handler(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data1 = b'{name: owner}' - data2 = b'{name2: owner2}' + data1 = b"{name: owner}" + data2 = b"{name2: owner2}" data3 = ( - b'{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:' - b'{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3,' - b'shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:' - b'{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z,' - b'data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}' + b"{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:" + b"{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3," + b"shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:" + b"{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z," + b"data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}" ) - data = data1 + b'\n' + data2 + b'\n' + data3 + data = data1 + b"\n" + data2 + b"\n" + data3 # upload the json file file_name = self._get_file_reference() @@ -513,24 +485,17 @@ def on_error(error): errors.append(error) input_format = DelimitedJsonDialect() - output_format = DelimitedTextDialect( - delimiter=';', - quotechar="'", - lineterminator='.', - escapechar='\\' - ) + output_format = DelimitedTextDialect(delimiter=";", quotechar="'", lineterminator=".", escapechar="\\") resp = file_client.query_file( - "SELECT * from BlobStorage", - on_error=on_error, - file_format=input_format, - output_format=output_format) + "SELECT * from BlobStorage", on_error=on_error, file_format=input_format, output_format=output_format + ) records = [] for record in resp.records(): records.append(record) assert len(errors) == 1 assert len(resp) == len(data) - assert records == [b''] + assert records == [b""] @DataLakePreparer() @recorded_by_proxy @@ -540,16 +505,16 @@ def test_quick_query_readall_with_fatal_error_handler_raise(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data1 = b'{name: owner}' - data2 = b'{name2: owner2}' + data1 = b"{name: owner}" + data2 = b"{name2: owner2}" data3 = ( - b'{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:' - b'{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3,' - b'shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:' - b'{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z,' - b'data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}' + b"{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:" + b"{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3," + b"shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:" + b"{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z," + b"data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}" ) - data = data1 + b'\n' + data2 + b'\n' + data3 + data = data1 + b"\n" + data2 + b"\n" + data3 # upload the json file file_name = self._get_file_reference() @@ -560,17 +525,10 @@ def on_error(error): raise ValueError(error.description) input_format = DelimitedJsonDialect() - output_format = DelimitedTextDialect( - delimiter=';', - quotechar="'", - lineterminator='.', - escapechar='\\' - ) + output_format = DelimitedTextDialect(delimiter=";", quotechar="'", lineterminator=".", escapechar="\\") resp = file_client.query_file( - "SELECT * from BlobStorage", - on_error=on_error, - file_format=input_format, - output_format=output_format) + "SELECT * from BlobStorage", on_error=on_error, file_format=input_format, output_format=output_format + ) with pytest.raises(ValueError): resp.readall() @@ -582,16 +540,16 @@ def test_quick_query_iter_records_with_fatal_error_handler_raise(self, **kwargs) self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data1 = b'{name: owner}' - data2 = b'{name2: owner2}' + data1 = b"{name: owner}" + data2 = b"{name2: owner2}" data3 = ( - b'{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:' - b'{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3,' - b'shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:' - b'{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z,' - b'data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}' + b"{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:" + b"{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3," + b"shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:" + b"{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z," + b"data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}" ) - data = data1 + b'\n' + data2 + b'\n' + data3 + data = data1 + b"\n" + data2 + b"\n" + data3 # upload the json file file_name = self._get_file_reference() @@ -602,17 +560,9 @@ def on_error(error): raise ValueError(error.description) input_format = DelimitedJsonDialect() - output_format = DelimitedTextDialect( - delimiter=';', - quotechar="'", - lineterminator='.', - escapechar='\\' - ) + output_format = DelimitedTextDialect(delimiter=";", quotechar="'", lineterminator=".", escapechar="\\") resp = file_client.query_file( - "SELECT * from BlobStorage", - on_error=on_error, - file_format=input_format, - output_format=output_format + "SELECT * from BlobStorage", on_error=on_error, file_format=input_format, output_format=output_format ) with pytest.raises(ValueError): @@ -627,9 +577,9 @@ def test_quick_query_readall_with_fatal_error_ignore(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data1 = b'{name: owner}' - data2 = b'{name2: owner2}' - data = data1 + b'\n' + data2 + b'\n' + data1 + data1 = b"{name: owner}" + data2 = b"{name2: owner2}" + data = data1 + b"\n" + data2 + b"\n" + data1 # upload the json file file_name = self._get_file_reference() @@ -637,16 +587,10 @@ def test_quick_query_readall_with_fatal_error_ignore(self, **kwargs): file_client.upload_data(data, overwrite=True) input_format = DelimitedJsonDialect() - output_format = DelimitedTextDialect( - delimiter=';', - quotechar="'", - lineterminator='.', - escapechar='\\' - ) + output_format = DelimitedTextDialect(delimiter=";", quotechar="'", lineterminator=".", escapechar="\\") resp = file_client.query_file( - "SELECT * from BlobStorage", - file_format=input_format, - output_format=output_format) + "SELECT * from BlobStorage", file_format=input_format, output_format=output_format + ) query_result = resp.readall() @DataLakePreparer() @@ -657,16 +601,16 @@ def test_quick_query_iter_records_with_fatal_error_ignore(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data1 = b'{name: owner}' - data2 = b'{name2: owner2}' + data1 = b"{name: owner}" + data2 = b"{name2: owner2}" data3 = ( - b'{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:' - b'{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3,' - b'shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:' - b'{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z,' - b'data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}' + b"{version:0,begin:1601-01-01T00:00:00.000Z,intervalSecs:3600,status:Finalized,config:" + b"{version:0,configVersionEtag:0x8d75ef460eb1a12,numShards:1,recordsFormat:avro,formatSchemaVersion:3," + b"shardDistFnVersion:1},chunkFilePaths:[$blobchangefeed/log/00/1601/01/01/0000/],storageDiagnostics:" + b"{version:0,lastModifiedTime:2019-11-01T17:53:18.861Z," + b"data:{aid:d305317d-a006-0042-00dd-902bbb06fc56}}}" ) - data = data1 + b'\n' + data2 + b'\n' + data3 + data = data1 + b"\n" + data2 + b"\n" + data3 # upload the json file file_name = self._get_file_reference() @@ -674,16 +618,10 @@ def test_quick_query_iter_records_with_fatal_error_ignore(self, **kwargs): file_client.upload_data(data, overwrite=True) input_format = DelimitedJsonDialect() - output_format = DelimitedTextDialect( - delimiter=';', - quotechar="'", - lineterminator='.', - escapechar='\\' - ) + output_format = DelimitedTextDialect(delimiter=";", quotechar="'", lineterminator=".", escapechar="\\") resp = file_client.query_file( - "SELECT * from BlobStorage", - file_format=input_format, - output_format=output_format) + "SELECT * from BlobStorage", file_format=input_format, output_format=output_format + ) for record in resp.records(): print(record) @@ -702,27 +640,22 @@ def test_quick_query_readall_with_nonfatal_error_handler(self, **kwargs): file_client.upload_data(CSV_DATA, overwrite=True) errors = [] + def on_error(error): errors.append(error) input_format = DelimitedTextDialect( - delimiter=',', - quotechar='"', - lineterminator='\n', - escapechar='', - has_header=True + delimiter=",", quotechar='"', lineterminator="\n", escapechar="", has_header=True ) output_format = DelimitedTextDialect( - delimiter=';', + delimiter=";", quotechar="'", - lineterminator='.', - escapechar='\\', + lineterminator=".", + escapechar="\\", ) resp = file_client.query_file( - "SELECT RepoPath from BlobStorage", - file_format=input_format, - output_format=output_format, - on_error=on_error) + "SELECT RepoPath from BlobStorage", file_format=input_format, output_format=output_format, on_error=on_error + ) query_result = resp.readall() # the error is because that line only has one column @@ -744,27 +677,22 @@ def test_quick_query_iter_records_with_nonfatal_error_handler(self, **kwargs): file_client.upload_data(CSV_DATA, overwrite=True) errors = [] + def on_error(error): errors.append(error) input_format = DelimitedTextDialect( - delimiter=',', - quotechar='"', - lineterminator='\n', - escapechar='', - has_header=True + delimiter=",", quotechar='"', lineterminator="\n", escapechar="", has_header=True ) output_format = DelimitedTextDialect( - delimiter=';', + delimiter=";", quotechar="'", - lineterminator='%', - escapechar='\\', + lineterminator="%", + escapechar="\\", ) resp = file_client.query_file( - "SELECT RepoPath from BlobStorage", - file_format=input_format, - output_format=output_format, - on_error=on_error) + "SELECT RepoPath from BlobStorage", file_format=input_format, output_format=output_format, on_error=on_error + ) data = list(resp.records()) # the error is because that line only has one column @@ -786,22 +714,17 @@ def test_quick_query_readall_with_nonfatal_error_ignore(self, **kwargs): file_client.upload_data(CSV_DATA, overwrite=True) input_format = DelimitedTextDialect( - delimiter=',', - quotechar='"', - lineterminator='\n', - escapechar='', - has_header=True + delimiter=",", quotechar='"', lineterminator="\n", escapechar="", has_header=True ) output_format = DelimitedTextDialect( - delimiter=';', + delimiter=";", quotechar="'", - lineterminator='.', - escapechar='\\', + lineterminator=".", + escapechar="\\", ) resp = file_client.query_file( - "SELECT RepoPath from BlobStorage", - file_format=input_format, - output_format=output_format) + "SELECT RepoPath from BlobStorage", file_format=input_format, output_format=output_format + ) query_result = resp.readall() assert len(resp) == len(CSV_DATA) assert len(query_result) > 0 @@ -820,22 +743,17 @@ def test_quick_query_iter_records_with_nonfatal_error_ignore(self, **kwargs): file_client.upload_data(CSV_DATA, overwrite=True) input_format = DelimitedTextDialect( - delimiter=',', - quotechar='"', - lineterminator='\n', - escapechar='', - has_header=True + delimiter=",", quotechar='"', lineterminator="\n", escapechar="", has_header=True ) output_format = DelimitedTextDialect( - delimiter=';', + delimiter=";", quotechar="'", - lineterminator='$', - escapechar='\\', + lineterminator="$", + escapechar="\\", ) resp = file_client.query_file( - "SELECT RepoPath from BlobStorage", - file_format=input_format, - output_format=output_format) + "SELECT RepoPath from BlobStorage", file_format=input_format, output_format=output_format + ) data = list(resp.records()) assert len(resp) == len(CSV_DATA) assert len(data) == 32 @@ -848,9 +766,9 @@ def test_quick_query_readall_with_json_serialization_setting(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data1 = b'{\"name\": \"owner\", \"id\": 1}' - data2 = b'{\"name2\": \"owner2\"}' - data = data1 + b'\n' + data2 + b'\n' + data1 + data1 = b'{"name": "owner", "id": 1}' + data2 = b'{"name2": "owner2"}' + data = data1 + b"\n" + data2 + b"\n" + data1 # upload the json file file_name = self._get_file_reference() @@ -858,17 +776,16 @@ def test_quick_query_readall_with_json_serialization_setting(self, **kwargs): file_client.upload_data(data, overwrite=True) errors = [] + def on_error(error): errors.append(error) - input_format = DelimitedJsonDialect(delimiter='\n') - output_format = DelimitedJsonDialect(delimiter=';') + input_format = DelimitedJsonDialect(delimiter="\n") + output_format = DelimitedJsonDialect(delimiter=";") resp = file_client.query_file( - "SELECT name from BlobStorage", - on_error=on_error, - file_format=input_format, - output_format=output_format) + "SELECT name from BlobStorage", on_error=on_error, file_format=input_format, output_format=output_format + ) query_result = resp.readall() assert len(errors) == 0 @@ -883,9 +800,9 @@ def test_quick_query_iter_records_with_json_serialization_setting(self, **kwargs self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data1 = b'{\"name\": \"owner\", \"id\": 1}' - data2 = b'{\"name2\": \"owner2\"}' - data = data1 + b'\n' + data2 + b'\n' + data1 + data1 = b'{"name": "owner", "id": 1}' + data2 = b'{"name2": "owner2"}' + data = data1 + b"\n" + data2 + b"\n" + data1 # upload the json file file_name = self._get_file_reference() @@ -893,22 +810,21 @@ def test_quick_query_iter_records_with_json_serialization_setting(self, **kwargs file_client.upload_data(data, overwrite=True) errors = [] + def on_error(error): errors.append(error) - input_format = DelimitedJsonDialect(delimiter='\n') - output_format = DelimitedJsonDialect(delimiter=';') + input_format = DelimitedJsonDialect(delimiter="\n") + output_format = DelimitedJsonDialect(delimiter=";") resp = file_client.query_file( - "SELECT name from BlobStorage", - on_error=on_error, - file_format=input_format, - output_format=output_format) + "SELECT name from BlobStorage", on_error=on_error, file_format=input_format, output_format=output_format + ) listdata = list(resp.records()) assert len(errors) == 0 assert len(resp) == len(data) - assert listdata == [b'{"name":"owner"}', b'{}', b'{"name":"owner"}', b''] + assert listdata == [b'{"name":"owner"}', b"{}", b'{"name":"owner"}', b""] @DataLakePreparer() @recorded_by_proxy @@ -918,8 +834,8 @@ def test_quick_query_with_only_input_json_serialization_setting(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data1 = b'{\"name\": \"owner\", \"id\": 1}' - data2 = b'{\"name2\": \"owner2\"}' + data1 = b'{"name": "owner", "id": 1}' + data2 = b'{"name2": "owner2"}' data = data1 + data2 + data1 # upload the json file @@ -928,17 +844,16 @@ def test_quick_query_with_only_input_json_serialization_setting(self, **kwargs): file_client.upload_data(data, overwrite=True) errors = [] + def on_error(error): errors.append(error) - input_format = DelimitedJsonDialect(delimiter='\n') + input_format = DelimitedJsonDialect(delimiter="\n") output_format = None resp = file_client.query_file( - "SELECT name from BlobStorage", - on_error=on_error, - file_format=input_format, - output_format=output_format) + "SELECT name from BlobStorage", on_error=on_error, file_format=input_format, output_format=output_format + ) query_result = resp.readall() assert len(errors) == 0 @@ -953,7 +868,7 @@ def test_quick_query_output_in_arrow_format(self, **kwargs): self._setUp(datalake_storage_account_name, datalake_storage_account_key) # Arrange - data = b'100,200,300,400\n300,400,500,600\n' + data = b"100,200,300,400\n300,400,500,600\n" # upload the json file file_name = self._get_file_reference() @@ -961,15 +876,15 @@ def test_quick_query_output_in_arrow_format(self, **kwargs): file_client.upload_data(data, overwrite=True) errors = [] + def on_error(error): errors.append(error) output_format = [ArrowDialect(ArrowType.DECIMAL, name="abc", precision=4, scale=2)] resp = file_client.query_file( - "SELECT _2 from BlobStorage WHERE _1 > 250", - on_error=on_error, - output_format=output_format) + "SELECT _2 from BlobStorage WHERE _1 > 250", on_error=on_error, output_format=output_format + ) query_result = base64.b64encode(resp.readall()) # expected_result = b'/////3gAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAEAAAAUAAAAEAAUAAgABgAHAAwAAAAQABAAAAAAAAEHEAAAABwAAAAEAAAAAAAAAAMAAABhYmMACAAMAAQACAAIAAAABAAAAAIAAAD/////cAAAABAAAAAAAAoADgAGAAUACAAKAAAAAAMEABAAAAAAAAoADAAAAAQACAAKAAAAMAAAAAQAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAD/////AAAAAP////+IAAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMEABgAAAAQAAAAAAAAAAAACgAYAAwABAAIAAoAAAA8AAAAEAAAAAEAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAABAAAAAQAAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAAA=' # pylint: disable=line-too-long @@ -989,6 +904,7 @@ def test_quick_query_input_in_arrow_format(self, **kwargs): file_client = self.dsc.get_file_client(self.filesystem_name, file_name) errors = [] + def on_error(error): errors.append(error) @@ -996,9 +912,8 @@ def on_error(error): with pytest.raises(ValueError): file_client.query_file( - "SELECT _2 from BlobStorage WHERE _1 > 250", - on_error=on_error, - file_format=input_format) + "SELECT _2 from BlobStorage WHERE _1 > 250", on_error=on_error, file_format=input_format + ) @DataLakePreparer() @recorded_by_proxy @@ -1012,7 +927,7 @@ def test_quick_query_input_in_parquet_format(self, **kwargs): file_client = self.dsc.get_file_client(self.filesystem_name, file_name) expression = "select * from blobstorage where id < 1;" - expected_data = b"0,mdifjt55.ea3,mdifjt55.ea3\n" # cspell:disable-line + expected_data = b"0,mdifjt55.ea3,mdifjt55.ea3\n" # cspell:disable-line parquet_path = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "./resources/parquet.parquet")) with open(parquet_path, "rb") as parquet_data: @@ -1041,7 +956,8 @@ def test_quick_query_output_in_parquet_format(self, **kwargs): with pytest.raises(ValueError): file_client.query_file( - expression, file_format=QuickQueryDialect.Parquet, - output_format=QuickQueryDialect.Parquet) + expression, file_format=QuickQueryDialect.Parquet, output_format=QuickQueryDialect.Parquet + ) + # ------------------------------------------------------------------------------ From 33d7af2154a33dd0d1a08ce60ff4cb9aa5962df3 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sat, 23 May 2026 19:32:22 -0400 Subject: [PATCH 2/4] pylint stuffffff --- .../storage/filedatalake/_data_lake_service_client.py | 4 ++-- .../azure/storage/filedatalake/_deserialize.py | 8 ++++---- .../azure/storage/filedatalake/_file_system_client.py | 8 ++++---- .../azure/storage/filedatalake/_models.py | 8 ++++---- .../storage/filedatalake/aio/_file_system_client_async.py | 8 ++++---- .../samples/datalake_samples_access_control_recursive.py | 5 ++--- .../datalake_samples_access_control_recursive_async.py | 5 ++--- .../tests/test_helpers_async.py | 4 ++-- 8 files changed, 24 insertions(+), 26 deletions(-) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py index dcd70be5e028..a56d723230ea 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py @@ -397,9 +397,9 @@ def undelete_file_system(self, name: str, deleted_version: str, **kwargs: Any) - return file_system @distributed_trace - def delete_file_system( + def delete_file_system( # pylint: disable=delete-operation-wrong-return-type self, file_system: Union[FileSystemProperties, str], **kwargs: Any - ) -> FileSystemClient: # pylint: disable=delete-operation-wrong-return-type + ) -> FileSystemClient: """Marks the specified file system for deletion. The file system and any files contained within it are later deleted during garbage collection. diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_deserialize.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_deserialize.py index 4283701dcc38..078efd8329dc 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_deserialize.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_deserialize.py @@ -82,9 +82,9 @@ def return_headers_and_deserialized_path_list( # pylint: disable=name-too-long, return deserialized.paths if deserialized.paths else {}, normalize_headers(response_headers) -def get_deleted_path_properties_from_generated_code( +def get_deleted_path_properties_from_generated_code( # pylint: disable=name-too-long generated: "BlobItemInternal", -) -> DeletedPathProperties: # pylint: disable=name-too-long +) -> DeletedPathProperties: deleted_path = DeletedPathProperties() deleted_path.name = generated.name deleted_path.deleted_time = generated.properties.deleted_time @@ -101,9 +101,9 @@ def get_datalake_service_properties(datalake_properties: Dict[str, Any]) -> Dict datalake_properties["analytics_logging"] = AnalyticsLogging._from_generated( # pylint: disable=protected-access datalake_properties["analytics_logging"] ) - datalake_properties["hour_metrics"] = Metrics._from_generated( + datalake_properties["hour_metrics"] = Metrics._from_generated( # pylint: disable=protected-access datalake_properties["hour_metrics"] - ) # pylint: disable=protected-access + ) datalake_properties["minute_metrics"] = Metrics._from_generated( # pylint: disable=protected-access datalake_properties["minute_metrics"] ) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py index d0849f91ea29..0c0d3bc2d8f6 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py @@ -439,9 +439,9 @@ def get_file_system_properties(self, **kwargs: Any) -> FileSystemProperties: :caption: Getting properties on the file system. """ container_properties = self._container_client.get_container_properties(**kwargs) - return FileSystemProperties._convert_from_container_props( + return FileSystemProperties._convert_from_container_props( # pylint: disable=protected-access container_properties - ) # pylint: disable=protected-access + ) @distributed_trace def set_file_system_metadata(self, metadata: Dict[str, str], **kwargs: Any) -> Dict[str, Union[str, "datetime"]]: @@ -561,9 +561,9 @@ def get_file_system_access_policy(self, **kwargs: Any) -> Dict[str, Any]: """ access_policy = self._container_client.get_container_access_policy(**kwargs) return { - "public_access": PublicAccess._from_generated( + "public_access": PublicAccess._from_generated( # pylint: disable=protected-access access_policy["public_access"] - ), # pylint: disable=protected-access + ), "signed_identifiers": access_policy["signed_identifiers"], } diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py index 3bb7949c17b5..97445299773a 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_models.py @@ -111,9 +111,9 @@ def _from_generated(cls, generated): version=generated.version, enabled=generated.enabled, include_apis=generated.include_apis, - retention_policy=RetentionPolicy._from_generated( + retention_policy=RetentionPolicy._from_generated( # pylint: disable=protected-access generated.retention_policy - ), # pylint: disable=protected-access + ), ) @@ -1329,9 +1329,9 @@ def _from_generated(cls, generated): delete=generated.delete, read=generated.read, write=generated.write, - retention_policy=RetentionPolicy._from_generated( + retention_policy=RetentionPolicy._from_generated( # pylint: disable=protected-access generated.retention_policy - ), # pylint: disable=protected-access + ), ) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py index 12a88a63d04d..89ba452cc877 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py @@ -441,9 +441,9 @@ async def get_file_system_properties(self, **kwargs: Any) -> FileSystemPropertie :caption: Getting properties on the file system. """ container_properties = await self._container_client.get_container_properties(**kwargs) - return FileSystemProperties._convert_from_container_props( + return FileSystemProperties._convert_from_container_props( # pylint: disable=protected-access container_properties - ) # pylint: disable=protected-access + ) @distributed_trace_async async def set_file_system_metadata( @@ -566,9 +566,9 @@ async def get_file_system_access_policy(self, **kwargs: Any) -> Dict[str, Any]: """ access_policy = await self._container_client.get_container_access_policy(**kwargs) return { - "public_access": PublicAccess._from_generated( + "public_access": PublicAccess._from_generated( # pylint: disable=protected-access access_policy["public_access"] - ), # pylint: disable=protected-access + ), "signed_identifiers": access_policy["signed_identifiers"], } diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive.py index 658755851ade..6f3f91d18885 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive.py @@ -88,9 +88,8 @@ def progress_callback(acl_changes): # if an error was encountered, a continuation token would be returned if the operation can be resumed if acl_change_result.continuation is not None: print( - "The operation can be resumed by passing the continuation token {} again into the access control method.".format( - acl_change_result.continuation - ) + "The operation can be resumed by passing the continuation token " + "{} again into the access control method.".format(acl_change_result.continuation) ) # get and display the permissions of the parent directory again diff --git a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive_async.py b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive_async.py index 587e1888667b..ff24c72855b1 100644 --- a/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive_async.py +++ b/sdk/storage/azure-storage-file-datalake/samples/datalake_samples_access_control_recursive_async.py @@ -90,9 +90,8 @@ async def progress_callback(acl_changes): # if an error was encountered, a continuation token would be returned if the operation can be resumed if acl_change_result.continuation is not None: print( - "The operation can be resumed by passing the continuation token {} again into the access control method.".format( - acl_change_result.continuation - ) + "The operation can be resumed by passing the continuation token " + "{} again into the access control method.".format(acl_change_result.continuation) ) # get and display the permissions of the parent directory again diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py index 95196774c350..039332df3509 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py @@ -12,10 +12,10 @@ from aiohttp.client_proto import ResponseHandler from aiohttp.streams import StreamReader -from azure.core.pipeline.transport import ( +from azure.core.pipeline.transport import ( # pylint: disable=no-name-in-module AioHttpTransportResponse, AsyncHttpTransport, -) # pylint: disable=no-name-in-module +) from azure.core.rest import HttpRequest From 56c5c5496d3329a303d5674438d8a0a4ed6a6836 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sat, 23 May 2026 19:35:20 -0400 Subject: [PATCH 3/4] Copilot feedback --- .../azure/storage/filedatalake/__init__.py | 1 - .../azure-storage-file-datalake/tests/settings/testcase.py | 1 - sdk/storage/azure-storage-file-datalake/tests/test_helpers.py | 2 +- .../azure-storage-file-datalake/tests/test_helpers_async.py | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/__init__.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/__init__.py index 904ff89f01f0..3bfde882096f 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/__init__.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/__init__.py @@ -76,7 +76,6 @@ "DataLakeDirectoryClient", "DataLakeFileClient", "DataLakeFileQueryError", - "DataLakeFileQueryError", "DataLakeLeaseClient", "DataLakeServiceClient", "DeletedPathProperties", diff --git a/sdk/storage/azure-storage-file-datalake/tests/settings/testcase.py b/sdk/storage/azure-storage-file-datalake/tests/settings/testcase.py index 27c3a57aa467..e249c439fea4 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/settings/testcase.py +++ b/sdk/storage/azure-storage-file-datalake/tests/settings/testcase.py @@ -19,7 +19,6 @@ from .settings_fake import * -LOGGING_FORMAT = "%(asctime)s %(name)-20s %(levelname)-5s %(message)s" LOGGING_FORMAT = "%(asctime)s %(name)-20s %(levelname)-5s %(message)s" os.environ["DATALAKE_STORAGE_ACCOUNT_NAME"] = ( os.environ.get("DATALAKE_STORAGE_ACCOUNT_NAME", None) or DATALAKE_STORAGE_ACCOUNT_NAME diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py b/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py index 9d15ecda0eac..319b178fb370 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py @@ -37,7 +37,7 @@ class MockHttpClientResponse(Response): def __init__( self, url: str, body_bytes: bytes, headers: Dict[str, Any], status: int = 200, reason: str = "OK" ) -> None: - super(MockHttpClientResponse).__init__() + super(MockHttpClientResponse, self).__init__() self._url = url self._body = body_bytes self._content = body_bytes diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py index 039332df3509..c172e4698237 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py @@ -61,7 +61,7 @@ class MockAioHttpClientResponse(ClientResponse): def __init__( self, url: str, body_bytes: bytes, headers: Dict[str, Any], status: int = 200, reason: str = "OK" ) -> None: - super(MockAioHttpClientResponse).__init__() + super(MockAioHttpClientResponse, self).__init__() self._url = url self._body = body_bytes self._headers = headers From da744b9ab11174b94106dad9ad0c6b7c915e480b Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 24 May 2026 02:29:36 -0400 Subject: [PATCH 4/4] boooo --- sdk/storage/azure-storage-blob/dev_requirements.txt | 1 - sdk/storage/azure-storage-file-datalake/tests/test_helpers.py | 2 +- .../azure-storage-file-datalake/tests/test_helpers_async.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/storage/azure-storage-blob/dev_requirements.txt b/sdk/storage/azure-storage-blob/dev_requirements.txt index 9657377026f0..b8770ca0db3a 100644 --- a/sdk/storage/azure-storage-blob/dev_requirements.txt +++ b/sdk/storage/azure-storage-blob/dev_requirements.txt @@ -4,4 +4,3 @@ ../azure-storage-extensions azure-mgmt-storage==20.1.0 aiohttp>=3.13.5 - diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py b/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py index 319b178fb370..9d15ecda0eac 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_helpers.py @@ -37,7 +37,7 @@ class MockHttpClientResponse(Response): def __init__( self, url: str, body_bytes: bytes, headers: Dict[str, Any], status: int = 200, reason: str = "OK" ) -> None: - super(MockHttpClientResponse, self).__init__() + super(MockHttpClientResponse).__init__() self._url = url self._body = body_bytes self._content = body_bytes diff --git a/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py b/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py index c172e4698237..039332df3509 100644 --- a/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py +++ b/sdk/storage/azure-storage-file-datalake/tests/test_helpers_async.py @@ -61,7 +61,7 @@ class MockAioHttpClientResponse(ClientResponse): def __init__( self, url: str, body_bytes: bytes, headers: Dict[str, Any], status: int = 200, reason: str = "OK" ) -> None: - super(MockAioHttpClientResponse, self).__init__() + super(MockAioHttpClientResponse).__init__() self._url = url self._body = body_bytes self._headers = headers