From d88576835472f2aac6144bc9be64647e067f8567 Mon Sep 17 00:00:00 2001 From: Azure CLI Bot Date: Fri, 26 Nov 2021 09:29:20 +0800 Subject: [PATCH] [AutoRelease] t2-datafactory-2021-11-20-87712 (#21840) * CodeGen from PR 16633 in Azure/azure-rest-api-specs [DataFactory] Add optional property disableChunking in FTP and SFTP read settings (#16633) * [DataFactory] Add optional property disableChunking in FTP and SFTP read settings * Add "disableChunking" in Synapse stable version * version,CHANGELOG Co-authored-by: SDKAuto Co-authored-by: PythonSdkPipelines --- .../azure-mgmt-datafactory/CHANGELOG.md | 18 + .../azure-mgmt-datafactory/_meta.json | 2 +- .../azure/mgmt/datafactory/_version.py | 2 +- .../azure/mgmt/datafactory/models/__init__.py | 21 +- .../azure/mgmt/datafactory/models/_models.py | 324 ++++++--------- .../mgmt/datafactory/models/_models_py3.py | 371 +++++++----------- 6 files changed, 274 insertions(+), 464 deletions(-) diff --git a/sdk/datafactory/azure-mgmt-datafactory/CHANGELOG.md b/sdk/datafactory/azure-mgmt-datafactory/CHANGELOG.md index a74d2de696cd..c717483a7d7c 100644 --- a/sdk/datafactory/azure-mgmt-datafactory/CHANGELOG.md +++ b/sdk/datafactory/azure-mgmt-datafactory/CHANGELOG.md @@ -1,5 +1,23 @@ # Release History +## 2.1.0 (2021-11-20) + +**Features** + + - Model PowerQuerySink has a new parameter flowlet + - Model DatasetCompression has a new parameter level + - Model SftpReadSettings has a new parameter disable_chunking + - Model DataFlowSink has a new parameter flowlet + - Model PowerQuerySource has a new parameter flowlet + - Model Transformation has a new parameter linked_service + - Model Transformation has a new parameter dataset + - Model Transformation has a new parameter flowlet + - Model DataFlowDebugPackage has a new parameter data_flows + - Model FtpReadSettings has a new parameter disable_chunking + - Model MappingDataFlow has a new parameter script_lines + - Model DataFlowReference has a new parameter parameters + - Model DataFlowSource has a new parameter flowlet + ## 2.0.0 (2021-10-09) **Features** diff --git a/sdk/datafactory/azure-mgmt-datafactory/_meta.json b/sdk/datafactory/azure-mgmt-datafactory/_meta.json index 9d25a2b50221..cdb17c9e0768 100644 --- a/sdk/datafactory/azure-mgmt-datafactory/_meta.json +++ b/sdk/datafactory/azure-mgmt-datafactory/_meta.json @@ -4,7 +4,7 @@ "@autorest/python@5.8.4", "@autorest/modelerfour@4.19.2" ], - "commit": "2ce915398bfadd5333820487595a9623187dcb59", + "commit": "9e8a591da83285d863866ecd2f6fe87a72758a7d", "repository_url": "https://github.com/Azure/azure-rest-api-specs", "autorest_command": "autorest specification/datafactory/resource-manager/readme.md --multiapi --python --python-mode=update --python-sdks-folder=/home/vsts/work/1/s/azure-sdk-for-python/sdk --track2 --use=@autorest/python@5.8.4 --use=@autorest/modelerfour@4.19.2 --version=3.4.5", "readme": "specification/datafactory/resource-manager/readme.md" diff --git a/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/_version.py b/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/_version.py index 48944bf3938a..83f24ab50946 100644 --- a/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/_version.py +++ b/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/_version.py @@ -6,4 +6,4 @@ # Changes may cause incorrect behavior and will be lost if the code is regenerated. # -------------------------------------------------------------------------- -VERSION = "2.0.0" +VERSION = "2.1.0" diff --git a/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/__init__.py b/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/__init__.py index d05e600bf0b6..d9c701f986d4 100644 --- a/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/__init__.py +++ b/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/__init__.py @@ -196,22 +196,16 @@ from ._models_py3 import DatabricksSparkJarActivity from ._models_py3 import DatabricksSparkPythonActivity from ._models_py3 import Dataset - from ._models_py3 import DatasetBZip2Compression from ._models_py3 import DatasetCompression from ._models_py3 import DatasetDataElement from ._models_py3 import DatasetDebugResource - from ._models_py3 import DatasetDeflateCompression from ._models_py3 import DatasetFolder - from ._models_py3 import DatasetGZipCompression from ._models_py3 import DatasetListResponse from ._models_py3 import DatasetLocation from ._models_py3 import DatasetReference from ._models_py3 import DatasetResource from ._models_py3 import DatasetSchemaDataElement from ._models_py3 import DatasetStorageFormat - from ._models_py3 import DatasetTarCompression - from ._models_py3 import DatasetTarGZipCompression - from ._models_py3 import DatasetZipDeflateCompression from ._models_py3 import Db2LinkedService from ._models_py3 import Db2Source from ._models_py3 import Db2TableDataset @@ -279,6 +273,7 @@ from ._models_py3 import FileSystemSink from ._models_py3 import FileSystemSource from ._models_py3 import FilterActivity + from ._models_py3 import Flowlet from ._models_py3 import ForEachActivity from ._models_py3 import FormatReadSettings from ._models_py3 import FormatWriteSettings @@ -891,22 +886,16 @@ from ._models import DatabricksSparkJarActivity # type: ignore from ._models import DatabricksSparkPythonActivity # type: ignore from ._models import Dataset # type: ignore - from ._models import DatasetBZip2Compression # type: ignore from ._models import DatasetCompression # type: ignore from ._models import DatasetDataElement # type: ignore from ._models import DatasetDebugResource # type: ignore - from ._models import DatasetDeflateCompression # type: ignore from ._models import DatasetFolder # type: ignore - from ._models import DatasetGZipCompression # type: ignore from ._models import DatasetListResponse # type: ignore from ._models import DatasetLocation # type: ignore from ._models import DatasetReference # type: ignore from ._models import DatasetResource # type: ignore from ._models import DatasetSchemaDataElement # type: ignore from ._models import DatasetStorageFormat # type: ignore - from ._models import DatasetTarCompression # type: ignore - from ._models import DatasetTarGZipCompression # type: ignore - from ._models import DatasetZipDeflateCompression # type: ignore from ._models import Db2LinkedService # type: ignore from ._models import Db2Source # type: ignore from ._models import Db2TableDataset # type: ignore @@ -974,6 +963,7 @@ from ._models import FileSystemSink # type: ignore from ._models import FileSystemSource # type: ignore from ._models import FilterActivity # type: ignore + from ._models import Flowlet # type: ignore from ._models import ForEachActivity # type: ignore from ._models import FormatReadSettings # type: ignore from ._models import FormatWriteSettings # type: ignore @@ -1685,22 +1675,16 @@ 'DatabricksSparkJarActivity', 'DatabricksSparkPythonActivity', 'Dataset', - 'DatasetBZip2Compression', 'DatasetCompression', 'DatasetDataElement', 'DatasetDebugResource', - 'DatasetDeflateCompression', 'DatasetFolder', - 'DatasetGZipCompression', 'DatasetListResponse', 'DatasetLocation', 'DatasetReference', 'DatasetResource', 'DatasetSchemaDataElement', 'DatasetStorageFormat', - 'DatasetTarCompression', - 'DatasetTarGZipCompression', - 'DatasetZipDeflateCompression', 'Db2LinkedService', 'Db2Source', 'Db2TableDataset', @@ -1768,6 +1752,7 @@ 'FileSystemSink', 'FileSystemSource', 'FilterActivity', + 'Flowlet', 'ForEachActivity', 'FormatReadSettings', 'FormatWriteSettings', diff --git a/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/_models.py b/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/_models.py index 5f4f0c460837..d5a79892a6bf 100644 --- a/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/_models.py +++ b/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/_models.py @@ -11051,7 +11051,7 @@ class DataFlow(msrest.serialization.Model): """Azure Data Factory nested object which contains a flow with data movements and transformations. You probably want to use the sub-classes and not this class directly. Known - sub-classes are: MappingDataFlow, WranglingDataFlow. + sub-classes are: Flowlet, MappingDataFlow, WranglingDataFlow. All required parameters must be populated in order to send to Azure. @@ -11078,7 +11078,7 @@ class DataFlow(msrest.serialization.Model): } _subtype_map = { - 'type': {'MappingDataFlow': 'MappingDataFlow', 'WranglingDataFlow': 'WranglingDataFlow'} + 'type': {'Flowlet': 'Flowlet', 'MappingDataFlow': 'MappingDataFlow', 'WranglingDataFlow': 'WranglingDataFlow'} } def __init__( @@ -11190,6 +11190,8 @@ class DataFlowDebugPackage(msrest.serialization.Model): :type session_id: str :param data_flow: Data flow instance. :type data_flow: ~azure.mgmt.datafactory.models.DataFlowDebugResource + :param data_flows: List of Data flows. + :type data_flows: list[~azure.mgmt.datafactory.models.DataFlowDebugResource] :param datasets: List of datasets. :type datasets: list[~azure.mgmt.datafactory.models.DatasetDebugResource] :param linked_services: List of linked services. @@ -11204,6 +11206,7 @@ class DataFlowDebugPackage(msrest.serialization.Model): 'additional_properties': {'key': '', 'type': '{object}'}, 'session_id': {'key': 'sessionId', 'type': 'str'}, 'data_flow': {'key': 'dataFlow', 'type': 'DataFlowDebugResource'}, + 'data_flows': {'key': 'dataFlows', 'type': '[DataFlowDebugResource]'}, 'datasets': {'key': 'datasets', 'type': '[DatasetDebugResource]'}, 'linked_services': {'key': 'linkedServices', 'type': '[LinkedServiceDebugResource]'}, 'staging': {'key': 'staging', 'type': 'DataFlowStagingInfo'}, @@ -11218,6 +11221,7 @@ def __init__( self.additional_properties = kwargs.get('additional_properties', None) self.session_id = kwargs.get('session_id', None) self.data_flow = kwargs.get('data_flow', None) + self.data_flows = kwargs.get('data_flows', None) self.datasets = kwargs.get('datasets', None) self.linked_services = kwargs.get('linked_services', None) self.staging = kwargs.get('staging', None) @@ -11418,6 +11422,8 @@ class DataFlowReference(msrest.serialization.Model): :type reference_name: str :param dataset_parameters: Reference data flow parameters from dataset. :type dataset_parameters: any + :param parameters: Data flow parameters. + :type parameters: dict[str, any] """ _validation = { @@ -11430,6 +11436,7 @@ class DataFlowReference(msrest.serialization.Model): 'type': {'key': 'type', 'type': 'str'}, 'reference_name': {'key': 'referenceName', 'type': 'str'}, 'dataset_parameters': {'key': 'datasetParameters', 'type': 'object'}, + 'parameters': {'key': 'parameters', 'type': '{object}'}, } type = "DataFlowReference" @@ -11442,6 +11449,7 @@ def __init__( self.additional_properties = kwargs.get('additional_properties', None) self.reference_name = kwargs['reference_name'] self.dataset_parameters = kwargs.get('dataset_parameters', None) + self.parameters = kwargs.get('parameters', None) class DataFlowResource(SubResource): @@ -11496,6 +11504,12 @@ class Transformation(msrest.serialization.Model): :type name: str :param description: Transformation description. :type description: str + :param dataset: Dataset reference. + :type dataset: ~azure.mgmt.datafactory.models.DatasetReference + :param linked_service: Linked service reference. + :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference """ _validation = { @@ -11505,6 +11519,9 @@ class Transformation(msrest.serialization.Model): _attribute_map = { 'name': {'key': 'name', 'type': 'str'}, 'description': {'key': 'description', 'type': 'str'}, + 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, + 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, } def __init__( @@ -11514,6 +11531,9 @@ def __init__( super(Transformation, self).__init__(**kwargs) self.name = kwargs['name'] self.description = kwargs.get('description', None) + self.dataset = kwargs.get('dataset', None) + self.linked_service = kwargs.get('linked_service', None) + self.flowlet = kwargs.get('flowlet', None) class DataFlowSink(Transformation): @@ -11529,6 +11549,8 @@ class DataFlowSink(Transformation): :type dataset: ~azure.mgmt.datafactory.models.DatasetReference :param linked_service: Linked service reference. :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference :param schema_linked_service: Schema linked service reference. :type schema_linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference """ @@ -11542,6 +11564,7 @@ class DataFlowSink(Transformation): 'description': {'key': 'description', 'type': 'str'}, 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, 'schema_linked_service': {'key': 'schemaLinkedService', 'type': 'LinkedServiceReference'}, } @@ -11550,8 +11573,6 @@ def __init__( **kwargs ): super(DataFlowSink, self).__init__(**kwargs) - self.dataset = kwargs.get('dataset', None) - self.linked_service = kwargs.get('linked_service', None) self.schema_linked_service = kwargs.get('schema_linked_service', None) @@ -11568,6 +11589,8 @@ class DataFlowSource(Transformation): :type dataset: ~azure.mgmt.datafactory.models.DatasetReference :param linked_service: Linked service reference. :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference :param schema_linked_service: Schema linked service reference. :type schema_linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference """ @@ -11581,6 +11604,7 @@ class DataFlowSource(Transformation): 'description': {'key': 'description', 'type': 'str'}, 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, 'schema_linked_service': {'key': 'schemaLinkedService', 'type': 'LinkedServiceReference'}, } @@ -11589,8 +11613,6 @@ def __init__( **kwargs ): super(DataFlowSource, self).__init__(**kwargs) - self.dataset = kwargs.get('dataset', None) - self.linked_service = kwargs.get('linked_service', None) self.schema_linked_service = kwargs.get('schema_linked_service', None) @@ -11733,17 +11755,17 @@ def __init__( class DatasetCompression(msrest.serialization.Model): """The compression method used on a dataset. - You probably want to use the sub-classes and not this class directly. Known - sub-classes are: DatasetBZip2Compression, DatasetDeflateCompression, DatasetGZipCompression, DatasetTarCompression, DatasetTarGZipCompression, DatasetZipDeflateCompression. - All required parameters must be populated in order to send to Azure. :param additional_properties: Unmatched properties from the message are deserialized to this collection. :type additional_properties: dict[str, any] :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. + string). :type type: any + :param level: The dataset compression level. Type: string (or Expression with resultType + string). + :type level: any """ _validation = { @@ -11753,10 +11775,7 @@ class DatasetCompression(msrest.serialization.Model): _attribute_map = { 'additional_properties': {'key': '', 'type': '{object}'}, 'type': {'key': 'type', 'type': 'object'}, - } - - _subtype_map = { - 'type': {'BZip2': 'DatasetBZip2Compression', 'Deflate': 'DatasetDeflateCompression', 'GZip': 'DatasetGZipCompression', 'Tar': 'DatasetTarCompression', 'TarGZip': 'DatasetTarGZipCompression', 'ZipDeflate': 'DatasetZipDeflateCompression'} + 'level': {'key': 'level', 'type': 'object'}, } def __init__( @@ -11765,37 +11784,8 @@ def __init__( ): super(DatasetCompression, self).__init__(**kwargs) self.additional_properties = kwargs.get('additional_properties', None) - self.type = 'DatasetCompression' # type: str - - -class DatasetBZip2Compression(DatasetCompression): - """The BZip2 compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - } - - def __init__( - self, - **kwargs - ): - super(DatasetBZip2Compression, self).__init__(**kwargs) - self.type = 'BZip2' # type: str + self.type = kwargs['type'] + self.level = kwargs.get('level', None) class DatasetDataElement(msrest.serialization.Model): @@ -11849,40 +11839,6 @@ def __init__( self.properties = kwargs['properties'] -class DatasetDeflateCompression(DatasetCompression): - """The Deflate compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - :param level: The Deflate compression level. - :type level: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - 'level': {'key': 'level', 'type': 'object'}, - } - - def __init__( - self, - **kwargs - ): - super(DatasetDeflateCompression, self).__init__(**kwargs) - self.type = 'Deflate' # type: str - self.level = kwargs.get('level', None) - - class DatasetFolder(msrest.serialization.Model): """The folder that this Dataset is in. If not specified, Dataset will appear at the root level. @@ -11902,40 +11858,6 @@ def __init__( self.name = kwargs.get('name', None) -class DatasetGZipCompression(DatasetCompression): - """The GZip compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - :param level: The GZip compression level. - :type level: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - 'level': {'key': 'level', 'type': 'object'}, - } - - def __init__( - self, - **kwargs - ): - super(DatasetGZipCompression, self).__init__(**kwargs) - self.type = 'GZip' # type: str - self.level = kwargs.get('level', None) - - class DatasetListResponse(msrest.serialization.Model): """A list of dataset resources. @@ -12073,104 +11995,6 @@ def __init__( self.type = kwargs.get('type', None) -class DatasetTarCompression(DatasetCompression): - """The Tar archive method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - } - - def __init__( - self, - **kwargs - ): - super(DatasetTarCompression, self).__init__(**kwargs) - self.type = 'Tar' # type: str - - -class DatasetTarGZipCompression(DatasetCompression): - """The TarGZip compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - :param level: The TarGZip compression level. - :type level: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - 'level': {'key': 'level', 'type': 'object'}, - } - - def __init__( - self, - **kwargs - ): - super(DatasetTarGZipCompression, self).__init__(**kwargs) - self.type = 'TarGZip' # type: str - self.level = kwargs.get('level', None) - - -class DatasetZipDeflateCompression(DatasetCompression): - """The ZipDeflate compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - :param level: The ZipDeflate compression level. - :type level: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - 'level': {'key': 'level', 'type': 'object'}, - } - - def __init__( - self, - **kwargs - ): - super(DatasetZipDeflateCompression, self).__init__(**kwargs) - self.type = 'ZipDeflate' # type: str - self.level = kwargs.get('level', None) - - class Db2LinkedService(LinkedService): """Linked service for DB2 data source. @@ -16009,6 +15833,61 @@ def __init__( self.condition = kwargs['condition'] +class Flowlet(DataFlow): + """Data flow flowlet. + + All required parameters must be populated in order to send to Azure. + + :param type: Required. Type of data flow.Constant filled by server. + :type type: str + :param description: The description of the data flow. + :type description: str + :param annotations: List of tags that can be used for describing the data flow. + :type annotations: list[any] + :param folder: The folder that this data flow is in. If not specified, Data flow will appear at + the root level. + :type folder: ~azure.mgmt.datafactory.models.DataFlowFolder + :param sources: List of sources in Flowlet. + :type sources: list[~azure.mgmt.datafactory.models.DataFlowSource] + :param sinks: List of sinks in Flowlet. + :type sinks: list[~azure.mgmt.datafactory.models.DataFlowSink] + :param transformations: List of transformations in Flowlet. + :type transformations: list[~azure.mgmt.datafactory.models.Transformation] + :param script: Flowlet script. + :type script: str + :param script_lines: Flowlet script lines. + :type script_lines: list[str] + """ + + _validation = { + 'type': {'required': True}, + } + + _attribute_map = { + 'type': {'key': 'type', 'type': 'str'}, + 'description': {'key': 'description', 'type': 'str'}, + 'annotations': {'key': 'annotations', 'type': '[object]'}, + 'folder': {'key': 'folder', 'type': 'DataFlowFolder'}, + 'sources': {'key': 'typeProperties.sources', 'type': '[DataFlowSource]'}, + 'sinks': {'key': 'typeProperties.sinks', 'type': '[DataFlowSink]'}, + 'transformations': {'key': 'typeProperties.transformations', 'type': '[Transformation]'}, + 'script': {'key': 'typeProperties.script', 'type': 'str'}, + 'script_lines': {'key': 'typeProperties.scriptLines', 'type': '[str]'}, + } + + def __init__( + self, + **kwargs + ): + super(Flowlet, self).__init__(**kwargs) + self.type = 'Flowlet' # type: str + self.sources = kwargs.get('sources', None) + self.sinks = kwargs.get('sinks', None) + self.transformations = kwargs.get('transformations', None) + self.script = kwargs.get('script', None) + self.script_lines = kwargs.get('script_lines', None) + + class ForEachActivity(ControlActivity): """This activity is used for iterating over a collection and execute given activities. @@ -16110,6 +15989,9 @@ class FtpReadSettings(StoreReadSettings): :type file_list_path: any :param use_binary_transfer: Specify whether to use binary transfer mode for FTP stores. :type use_binary_transfer: bool + :param disable_chunking: If true, disable parallel reading within each file. Default is false. + Type: boolean (or Expression with resultType boolean). + :type disable_chunking: any """ _validation = { @@ -16129,6 +16011,7 @@ class FtpReadSettings(StoreReadSettings): 'delete_files_after_completion': {'key': 'deleteFilesAfterCompletion', 'type': 'object'}, 'file_list_path': {'key': 'fileListPath', 'type': 'object'}, 'use_binary_transfer': {'key': 'useBinaryTransfer', 'type': 'bool'}, + 'disable_chunking': {'key': 'disableChunking', 'type': 'object'}, } def __init__( @@ -16145,6 +16028,7 @@ def __init__( self.delete_files_after_completion = kwargs.get('delete_files_after_completion', None) self.file_list_path = kwargs.get('file_list_path', None) self.use_binary_transfer = kwargs.get('use_binary_transfer', None) + self.disable_chunking = kwargs.get('disable_chunking', None) class FtpServerLinkedService(LinkedService): @@ -22502,6 +22386,8 @@ class MappingDataFlow(DataFlow): :type transformations: list[~azure.mgmt.datafactory.models.Transformation] :param script: DataFlow script. :type script: str + :param script_lines: Data flow script lines. + :type script_lines: list[str] """ _validation = { @@ -22517,6 +22403,7 @@ class MappingDataFlow(DataFlow): 'sinks': {'key': 'typeProperties.sinks', 'type': '[DataFlowSink]'}, 'transformations': {'key': 'typeProperties.transformations', 'type': '[Transformation]'}, 'script': {'key': 'typeProperties.script', 'type': 'str'}, + 'script_lines': {'key': 'typeProperties.scriptLines', 'type': '[str]'}, } def __init__( @@ -22529,6 +22416,7 @@ def __init__( self.sinks = kwargs.get('sinks', None) self.transformations = kwargs.get('transformations', None) self.script = kwargs.get('script', None) + self.script_lines = kwargs.get('script_lines', None) class MariaDBLinkedService(LinkedService): @@ -27140,7 +27028,8 @@ class PipelineRun(msrest.serialization.Model): :vartype run_end: ~datetime.datetime :ivar duration_in_ms: The duration of a pipeline run. :vartype duration_in_ms: int - :ivar status: The status of a pipeline run. + :ivar status: The status of a pipeline run. Possible values: Queued, InProgress, Succeeded, + Failed, Canceling, Cancelled. :vartype status: str :ivar message: The message from a pipeline run. :vartype message: str @@ -27510,6 +27399,8 @@ class PowerQuerySink(DataFlowSink): :type dataset: ~azure.mgmt.datafactory.models.DatasetReference :param linked_service: Linked service reference. :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference :param schema_linked_service: Schema linked service reference. :type schema_linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference :param script: sink script. @@ -27525,6 +27416,7 @@ class PowerQuerySink(DataFlowSink): 'description': {'key': 'description', 'type': 'str'}, 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, 'schema_linked_service': {'key': 'schemaLinkedService', 'type': 'LinkedServiceReference'}, 'script': {'key': 'script', 'type': 'str'}, } @@ -27573,6 +27465,8 @@ class PowerQuerySource(DataFlowSource): :type dataset: ~azure.mgmt.datafactory.models.DatasetReference :param linked_service: Linked service reference. :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference :param schema_linked_service: Schema linked service reference. :type schema_linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference :param script: source script. @@ -27588,6 +27482,7 @@ class PowerQuerySource(DataFlowSource): 'description': {'key': 'description', 'type': 'str'}, 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, 'schema_linked_service': {'key': 'schemaLinkedService', 'type': 'LinkedServiceReference'}, 'script': {'key': 'script', 'type': 'str'}, } @@ -32378,6 +32273,9 @@ class SftpReadSettings(StoreReadSettings): :param modified_datetime_end: The end of file's modified datetime. Type: string (or Expression with resultType string). :type modified_datetime_end: any + :param disable_chunking: If true, disable parallel reading within each file. Default is false. + Type: boolean (or Expression with resultType boolean). + :type disable_chunking: any """ _validation = { @@ -32398,6 +32296,7 @@ class SftpReadSettings(StoreReadSettings): 'delete_files_after_completion': {'key': 'deleteFilesAfterCompletion', 'type': 'object'}, 'modified_datetime_start': {'key': 'modifiedDatetimeStart', 'type': 'object'}, 'modified_datetime_end': {'key': 'modifiedDatetimeEnd', 'type': 'object'}, + 'disable_chunking': {'key': 'disableChunking', 'type': 'object'}, } def __init__( @@ -32415,6 +32314,7 @@ def __init__( self.delete_files_after_completion = kwargs.get('delete_files_after_completion', None) self.modified_datetime_start = kwargs.get('modified_datetime_start', None) self.modified_datetime_end = kwargs.get('modified_datetime_end', None) + self.disable_chunking = kwargs.get('disable_chunking', None) class SftpServerLinkedService(LinkedService): diff --git a/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/_models_py3.py b/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/_models_py3.py index 963145597ea9..cee8a8323b7b 100644 --- a/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/_models_py3.py +++ b/sdk/datafactory/azure-mgmt-datafactory/azure/mgmt/datafactory/models/_models_py3.py @@ -12696,7 +12696,7 @@ class DataFlow(msrest.serialization.Model): """Azure Data Factory nested object which contains a flow with data movements and transformations. You probably want to use the sub-classes and not this class directly. Known - sub-classes are: MappingDataFlow, WranglingDataFlow. + sub-classes are: Flowlet, MappingDataFlow, WranglingDataFlow. All required parameters must be populated in order to send to Azure. @@ -12723,7 +12723,7 @@ class DataFlow(msrest.serialization.Model): } _subtype_map = { - 'type': {'MappingDataFlow': 'MappingDataFlow', 'WranglingDataFlow': 'WranglingDataFlow'} + 'type': {'Flowlet': 'Flowlet', 'MappingDataFlow': 'MappingDataFlow', 'WranglingDataFlow': 'WranglingDataFlow'} } def __init__( @@ -12851,6 +12851,8 @@ class DataFlowDebugPackage(msrest.serialization.Model): :type session_id: str :param data_flow: Data flow instance. :type data_flow: ~azure.mgmt.datafactory.models.DataFlowDebugResource + :param data_flows: List of Data flows. + :type data_flows: list[~azure.mgmt.datafactory.models.DataFlowDebugResource] :param datasets: List of datasets. :type datasets: list[~azure.mgmt.datafactory.models.DatasetDebugResource] :param linked_services: List of linked services. @@ -12865,6 +12867,7 @@ class DataFlowDebugPackage(msrest.serialization.Model): 'additional_properties': {'key': '', 'type': '{object}'}, 'session_id': {'key': 'sessionId', 'type': 'str'}, 'data_flow': {'key': 'dataFlow', 'type': 'DataFlowDebugResource'}, + 'data_flows': {'key': 'dataFlows', 'type': '[DataFlowDebugResource]'}, 'datasets': {'key': 'datasets', 'type': '[DatasetDebugResource]'}, 'linked_services': {'key': 'linkedServices', 'type': '[LinkedServiceDebugResource]'}, 'staging': {'key': 'staging', 'type': 'DataFlowStagingInfo'}, @@ -12877,6 +12880,7 @@ def __init__( additional_properties: Optional[Dict[str, Any]] = None, session_id: Optional[str] = None, data_flow: Optional["DataFlowDebugResource"] = None, + data_flows: Optional[List["DataFlowDebugResource"]] = None, datasets: Optional[List["DatasetDebugResource"]] = None, linked_services: Optional[List["LinkedServiceDebugResource"]] = None, staging: Optional["DataFlowStagingInfo"] = None, @@ -12887,6 +12891,7 @@ def __init__( self.additional_properties = additional_properties self.session_id = session_id self.data_flow = data_flow + self.data_flows = data_flows self.datasets = datasets self.linked_services = linked_services self.staging = staging @@ -13112,6 +13117,8 @@ class DataFlowReference(msrest.serialization.Model): :type reference_name: str :param dataset_parameters: Reference data flow parameters from dataset. :type dataset_parameters: any + :param parameters: Data flow parameters. + :type parameters: dict[str, any] """ _validation = { @@ -13124,6 +13131,7 @@ class DataFlowReference(msrest.serialization.Model): 'type': {'key': 'type', 'type': 'str'}, 'reference_name': {'key': 'referenceName', 'type': 'str'}, 'dataset_parameters': {'key': 'datasetParameters', 'type': 'object'}, + 'parameters': {'key': 'parameters', 'type': '{object}'}, } type = "DataFlowReference" @@ -13134,12 +13142,14 @@ def __init__( reference_name: str, additional_properties: Optional[Dict[str, Any]] = None, dataset_parameters: Optional[Any] = None, + parameters: Optional[Dict[str, Any]] = None, **kwargs ): super(DataFlowReference, self).__init__(**kwargs) self.additional_properties = additional_properties self.reference_name = reference_name self.dataset_parameters = dataset_parameters + self.parameters = parameters class DataFlowResource(SubResource): @@ -13196,6 +13206,12 @@ class Transformation(msrest.serialization.Model): :type name: str :param description: Transformation description. :type description: str + :param dataset: Dataset reference. + :type dataset: ~azure.mgmt.datafactory.models.DatasetReference + :param linked_service: Linked service reference. + :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference """ _validation = { @@ -13205,6 +13221,9 @@ class Transformation(msrest.serialization.Model): _attribute_map = { 'name': {'key': 'name', 'type': 'str'}, 'description': {'key': 'description', 'type': 'str'}, + 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, + 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, } def __init__( @@ -13212,11 +13231,17 @@ def __init__( *, name: str, description: Optional[str] = None, + dataset: Optional["DatasetReference"] = None, + linked_service: Optional["LinkedServiceReference"] = None, + flowlet: Optional["DataFlowReference"] = None, **kwargs ): super(Transformation, self).__init__(**kwargs) self.name = name self.description = description + self.dataset = dataset + self.linked_service = linked_service + self.flowlet = flowlet class DataFlowSink(Transformation): @@ -13232,6 +13257,8 @@ class DataFlowSink(Transformation): :type dataset: ~azure.mgmt.datafactory.models.DatasetReference :param linked_service: Linked service reference. :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference :param schema_linked_service: Schema linked service reference. :type schema_linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference """ @@ -13245,6 +13272,7 @@ class DataFlowSink(Transformation): 'description': {'key': 'description', 'type': 'str'}, 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, 'schema_linked_service': {'key': 'schemaLinkedService', 'type': 'LinkedServiceReference'}, } @@ -13255,12 +13283,11 @@ def __init__( description: Optional[str] = None, dataset: Optional["DatasetReference"] = None, linked_service: Optional["LinkedServiceReference"] = None, + flowlet: Optional["DataFlowReference"] = None, schema_linked_service: Optional["LinkedServiceReference"] = None, **kwargs ): - super(DataFlowSink, self).__init__(name=name, description=description, **kwargs) - self.dataset = dataset - self.linked_service = linked_service + super(DataFlowSink, self).__init__(name=name, description=description, dataset=dataset, linked_service=linked_service, flowlet=flowlet, **kwargs) self.schema_linked_service = schema_linked_service @@ -13277,6 +13304,8 @@ class DataFlowSource(Transformation): :type dataset: ~azure.mgmt.datafactory.models.DatasetReference :param linked_service: Linked service reference. :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference :param schema_linked_service: Schema linked service reference. :type schema_linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference """ @@ -13290,6 +13319,7 @@ class DataFlowSource(Transformation): 'description': {'key': 'description', 'type': 'str'}, 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, 'schema_linked_service': {'key': 'schemaLinkedService', 'type': 'LinkedServiceReference'}, } @@ -13300,12 +13330,11 @@ def __init__( description: Optional[str] = None, dataset: Optional["DatasetReference"] = None, linked_service: Optional["LinkedServiceReference"] = None, + flowlet: Optional["DataFlowReference"] = None, schema_linked_service: Optional["LinkedServiceReference"] = None, **kwargs ): - super(DataFlowSource, self).__init__(name=name, description=description, **kwargs) - self.dataset = dataset - self.linked_service = linked_service + super(DataFlowSource, self).__init__(name=name, description=description, dataset=dataset, linked_service=linked_service, flowlet=flowlet, **kwargs) self.schema_linked_service = schema_linked_service @@ -13470,17 +13499,17 @@ def __init__( class DatasetCompression(msrest.serialization.Model): """The compression method used on a dataset. - You probably want to use the sub-classes and not this class directly. Known - sub-classes are: DatasetBZip2Compression, DatasetDeflateCompression, DatasetGZipCompression, DatasetTarCompression, DatasetTarGZipCompression, DatasetZipDeflateCompression. - All required parameters must be populated in order to send to Azure. :param additional_properties: Unmatched properties from the message are deserialized to this collection. :type additional_properties: dict[str, any] :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. + string). :type type: any + :param level: The dataset compression level. Type: string (or Expression with resultType + string). + :type level: any """ _validation = { @@ -13490,53 +13519,21 @@ class DatasetCompression(msrest.serialization.Model): _attribute_map = { 'additional_properties': {'key': '', 'type': '{object}'}, 'type': {'key': 'type', 'type': 'object'}, - } - - _subtype_map = { - 'type': {'BZip2': 'DatasetBZip2Compression', 'Deflate': 'DatasetDeflateCompression', 'GZip': 'DatasetGZipCompression', 'Tar': 'DatasetTarCompression', 'TarGZip': 'DatasetTarGZipCompression', 'ZipDeflate': 'DatasetZipDeflateCompression'} + 'level': {'key': 'level', 'type': 'object'}, } def __init__( self, *, + type: Any, additional_properties: Optional[Dict[str, Any]] = None, + level: Optional[Any] = None, **kwargs ): super(DatasetCompression, self).__init__(**kwargs) self.additional_properties = additional_properties - self.type = 'DatasetCompression' # type: str - - -class DatasetBZip2Compression(DatasetCompression): - """The BZip2 compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - } - - def __init__( - self, - *, - additional_properties: Optional[Dict[str, Any]] = None, - **kwargs - ): - super(DatasetBZip2Compression, self).__init__(additional_properties=additional_properties, **kwargs) - self.type = 'BZip2' # type: str + self.type = type + self.level = level class DatasetDataElement(msrest.serialization.Model): @@ -13596,43 +13593,6 @@ def __init__( self.properties = properties -class DatasetDeflateCompression(DatasetCompression): - """The Deflate compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - :param level: The Deflate compression level. - :type level: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - 'level': {'key': 'level', 'type': 'object'}, - } - - def __init__( - self, - *, - additional_properties: Optional[Dict[str, Any]] = None, - level: Optional[Any] = None, - **kwargs - ): - super(DatasetDeflateCompression, self).__init__(additional_properties=additional_properties, **kwargs) - self.type = 'Deflate' # type: str - self.level = level - - class DatasetFolder(msrest.serialization.Model): """The folder that this Dataset is in. If not specified, Dataset will appear at the root level. @@ -13654,43 +13614,6 @@ def __init__( self.name = name -class DatasetGZipCompression(DatasetCompression): - """The GZip compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - :param level: The GZip compression level. - :type level: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - 'level': {'key': 'level', 'type': 'object'}, - } - - def __init__( - self, - *, - additional_properties: Optional[Dict[str, Any]] = None, - level: Optional[Any] = None, - **kwargs - ): - super(DatasetGZipCompression, self).__init__(additional_properties=additional_properties, **kwargs) - self.type = 'GZip' # type: str - self.level = level - - class DatasetListResponse(msrest.serialization.Model): """A list of dataset resources. @@ -13840,112 +13763,6 @@ def __init__( self.type = type -class DatasetTarCompression(DatasetCompression): - """The Tar archive method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - } - - def __init__( - self, - *, - additional_properties: Optional[Dict[str, Any]] = None, - **kwargs - ): - super(DatasetTarCompression, self).__init__(additional_properties=additional_properties, **kwargs) - self.type = 'Tar' # type: str - - -class DatasetTarGZipCompression(DatasetCompression): - """The TarGZip compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - :param level: The TarGZip compression level. - :type level: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - 'level': {'key': 'level', 'type': 'object'}, - } - - def __init__( - self, - *, - additional_properties: Optional[Dict[str, Any]] = None, - level: Optional[Any] = None, - **kwargs - ): - super(DatasetTarGZipCompression, self).__init__(additional_properties=additional_properties, **kwargs) - self.type = 'TarGZip' # type: str - self.level = level - - -class DatasetZipDeflateCompression(DatasetCompression): - """The ZipDeflate compression method used on a dataset. - - All required parameters must be populated in order to send to Azure. - - :param additional_properties: Unmatched properties from the message are deserialized to this - collection. - :type additional_properties: dict[str, any] - :param type: Required. Type of dataset compression. Type: string (or Expression with resultType - string).Constant filled by server. - :type type: any - :param level: The ZipDeflate compression level. - :type level: any - """ - - _validation = { - 'type': {'required': True}, - } - - _attribute_map = { - 'additional_properties': {'key': '', 'type': '{object}'}, - 'type': {'key': 'type', 'type': 'object'}, - 'level': {'key': 'level', 'type': 'object'}, - } - - def __init__( - self, - *, - additional_properties: Optional[Dict[str, Any]] = None, - level: Optional[Any] = None, - **kwargs - ): - super(DatasetZipDeflateCompression, self).__init__(additional_properties=additional_properties, **kwargs) - self.type = 'ZipDeflate' # type: str - self.level = level - - class Db2LinkedService(LinkedService): """Linked service for DB2 data source. @@ -18359,6 +18176,70 @@ def __init__( self.condition = condition +class Flowlet(DataFlow): + """Data flow flowlet. + + All required parameters must be populated in order to send to Azure. + + :param type: Required. Type of data flow.Constant filled by server. + :type type: str + :param description: The description of the data flow. + :type description: str + :param annotations: List of tags that can be used for describing the data flow. + :type annotations: list[any] + :param folder: The folder that this data flow is in. If not specified, Data flow will appear at + the root level. + :type folder: ~azure.mgmt.datafactory.models.DataFlowFolder + :param sources: List of sources in Flowlet. + :type sources: list[~azure.mgmt.datafactory.models.DataFlowSource] + :param sinks: List of sinks in Flowlet. + :type sinks: list[~azure.mgmt.datafactory.models.DataFlowSink] + :param transformations: List of transformations in Flowlet. + :type transformations: list[~azure.mgmt.datafactory.models.Transformation] + :param script: Flowlet script. + :type script: str + :param script_lines: Flowlet script lines. + :type script_lines: list[str] + """ + + _validation = { + 'type': {'required': True}, + } + + _attribute_map = { + 'type': {'key': 'type', 'type': 'str'}, + 'description': {'key': 'description', 'type': 'str'}, + 'annotations': {'key': 'annotations', 'type': '[object]'}, + 'folder': {'key': 'folder', 'type': 'DataFlowFolder'}, + 'sources': {'key': 'typeProperties.sources', 'type': '[DataFlowSource]'}, + 'sinks': {'key': 'typeProperties.sinks', 'type': '[DataFlowSink]'}, + 'transformations': {'key': 'typeProperties.transformations', 'type': '[Transformation]'}, + 'script': {'key': 'typeProperties.script', 'type': 'str'}, + 'script_lines': {'key': 'typeProperties.scriptLines', 'type': '[str]'}, + } + + def __init__( + self, + *, + description: Optional[str] = None, + annotations: Optional[List[Any]] = None, + folder: Optional["DataFlowFolder"] = None, + sources: Optional[List["DataFlowSource"]] = None, + sinks: Optional[List["DataFlowSink"]] = None, + transformations: Optional[List["Transformation"]] = None, + script: Optional[str] = None, + script_lines: Optional[List[str]] = None, + **kwargs + ): + super(Flowlet, self).__init__(description=description, annotations=annotations, folder=folder, **kwargs) + self.type = 'Flowlet' # type: str + self.sources = sources + self.sinks = sinks + self.transformations = transformations + self.script = script + self.script_lines = script_lines + + class ForEachActivity(ControlActivity): """This activity is used for iterating over a collection and execute given activities. @@ -18470,6 +18351,9 @@ class FtpReadSettings(StoreReadSettings): :type file_list_path: any :param use_binary_transfer: Specify whether to use binary transfer mode for FTP stores. :type use_binary_transfer: bool + :param disable_chunking: If true, disable parallel reading within each file. Default is false. + Type: boolean (or Expression with resultType boolean). + :type disable_chunking: any """ _validation = { @@ -18489,6 +18373,7 @@ class FtpReadSettings(StoreReadSettings): 'delete_files_after_completion': {'key': 'deleteFilesAfterCompletion', 'type': 'object'}, 'file_list_path': {'key': 'fileListPath', 'type': 'object'}, 'use_binary_transfer': {'key': 'useBinaryTransfer', 'type': 'bool'}, + 'disable_chunking': {'key': 'disableChunking', 'type': 'object'}, } def __init__( @@ -18505,6 +18390,7 @@ def __init__( delete_files_after_completion: Optional[Any] = None, file_list_path: Optional[Any] = None, use_binary_transfer: Optional[bool] = None, + disable_chunking: Optional[Any] = None, **kwargs ): super(FtpReadSettings, self).__init__(additional_properties=additional_properties, max_concurrent_connections=max_concurrent_connections, disable_metrics_collection=disable_metrics_collection, **kwargs) @@ -18517,6 +18403,7 @@ def __init__( self.delete_files_after_completion = delete_files_after_completion self.file_list_path = file_list_path self.use_binary_transfer = use_binary_transfer + self.disable_chunking = disable_chunking class FtpServerLinkedService(LinkedService): @@ -25763,6 +25650,8 @@ class MappingDataFlow(DataFlow): :type transformations: list[~azure.mgmt.datafactory.models.Transformation] :param script: DataFlow script. :type script: str + :param script_lines: Data flow script lines. + :type script_lines: list[str] """ _validation = { @@ -25778,6 +25667,7 @@ class MappingDataFlow(DataFlow): 'sinks': {'key': 'typeProperties.sinks', 'type': '[DataFlowSink]'}, 'transformations': {'key': 'typeProperties.transformations', 'type': '[Transformation]'}, 'script': {'key': 'typeProperties.script', 'type': 'str'}, + 'script_lines': {'key': 'typeProperties.scriptLines', 'type': '[str]'}, } def __init__( @@ -25790,6 +25680,7 @@ def __init__( sinks: Optional[List["DataFlowSink"]] = None, transformations: Optional[List["Transformation"]] = None, script: Optional[str] = None, + script_lines: Optional[List[str]] = None, **kwargs ): super(MappingDataFlow, self).__init__(description=description, annotations=annotations, folder=folder, **kwargs) @@ -25798,6 +25689,7 @@ def __init__( self.sinks = sinks self.transformations = transformations self.script = script + self.script_lines = script_lines class MariaDBLinkedService(LinkedService): @@ -31124,7 +31016,8 @@ class PipelineRun(msrest.serialization.Model): :vartype run_end: ~datetime.datetime :ivar duration_in_ms: The duration of a pipeline run. :vartype duration_in_ms: int - :ivar status: The status of a pipeline run. + :ivar status: The status of a pipeline run. Possible values: Queued, InProgress, Succeeded, + Failed, Canceling, Cancelled. :vartype status: str :ivar message: The message from a pipeline run. :vartype message: str @@ -31535,6 +31428,8 @@ class PowerQuerySink(DataFlowSink): :type dataset: ~azure.mgmt.datafactory.models.DatasetReference :param linked_service: Linked service reference. :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference :param schema_linked_service: Schema linked service reference. :type schema_linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference :param script: sink script. @@ -31550,6 +31445,7 @@ class PowerQuerySink(DataFlowSink): 'description': {'key': 'description', 'type': 'str'}, 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, 'schema_linked_service': {'key': 'schemaLinkedService', 'type': 'LinkedServiceReference'}, 'script': {'key': 'script', 'type': 'str'}, } @@ -31561,11 +31457,12 @@ def __init__( description: Optional[str] = None, dataset: Optional["DatasetReference"] = None, linked_service: Optional["LinkedServiceReference"] = None, + flowlet: Optional["DataFlowReference"] = None, schema_linked_service: Optional["LinkedServiceReference"] = None, script: Optional[str] = None, **kwargs ): - super(PowerQuerySink, self).__init__(name=name, description=description, dataset=dataset, linked_service=linked_service, schema_linked_service=schema_linked_service, **kwargs) + super(PowerQuerySink, self).__init__(name=name, description=description, dataset=dataset, linked_service=linked_service, flowlet=flowlet, schema_linked_service=schema_linked_service, **kwargs) self.script = script @@ -31608,6 +31505,8 @@ class PowerQuerySource(DataFlowSource): :type dataset: ~azure.mgmt.datafactory.models.DatasetReference :param linked_service: Linked service reference. :type linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference + :param flowlet: Flowlet Reference. + :type flowlet: ~azure.mgmt.datafactory.models.DataFlowReference :param schema_linked_service: Schema linked service reference. :type schema_linked_service: ~azure.mgmt.datafactory.models.LinkedServiceReference :param script: source script. @@ -31623,6 +31522,7 @@ class PowerQuerySource(DataFlowSource): 'description': {'key': 'description', 'type': 'str'}, 'dataset': {'key': 'dataset', 'type': 'DatasetReference'}, 'linked_service': {'key': 'linkedService', 'type': 'LinkedServiceReference'}, + 'flowlet': {'key': 'flowlet', 'type': 'DataFlowReference'}, 'schema_linked_service': {'key': 'schemaLinkedService', 'type': 'LinkedServiceReference'}, 'script': {'key': 'script', 'type': 'str'}, } @@ -31634,11 +31534,12 @@ def __init__( description: Optional[str] = None, dataset: Optional["DatasetReference"] = None, linked_service: Optional["LinkedServiceReference"] = None, + flowlet: Optional["DataFlowReference"] = None, schema_linked_service: Optional["LinkedServiceReference"] = None, script: Optional[str] = None, **kwargs ): - super(PowerQuerySource, self).__init__(name=name, description=description, dataset=dataset, linked_service=linked_service, schema_linked_service=schema_linked_service, **kwargs) + super(PowerQuerySource, self).__init__(name=name, description=description, dataset=dataset, linked_service=linked_service, flowlet=flowlet, schema_linked_service=schema_linked_service, **kwargs) self.script = script @@ -37107,6 +37008,9 @@ class SftpReadSettings(StoreReadSettings): :param modified_datetime_end: The end of file's modified datetime. Type: string (or Expression with resultType string). :type modified_datetime_end: any + :param disable_chunking: If true, disable parallel reading within each file. Default is false. + Type: boolean (or Expression with resultType boolean). + :type disable_chunking: any """ _validation = { @@ -37127,6 +37031,7 @@ class SftpReadSettings(StoreReadSettings): 'delete_files_after_completion': {'key': 'deleteFilesAfterCompletion', 'type': 'object'}, 'modified_datetime_start': {'key': 'modifiedDatetimeStart', 'type': 'object'}, 'modified_datetime_end': {'key': 'modifiedDatetimeEnd', 'type': 'object'}, + 'disable_chunking': {'key': 'disableChunking', 'type': 'object'}, } def __init__( @@ -37144,6 +37049,7 @@ def __init__( delete_files_after_completion: Optional[Any] = None, modified_datetime_start: Optional[Any] = None, modified_datetime_end: Optional[Any] = None, + disable_chunking: Optional[Any] = None, **kwargs ): super(SftpReadSettings, self).__init__(additional_properties=additional_properties, max_concurrent_connections=max_concurrent_connections, disable_metrics_collection=disable_metrics_collection, **kwargs) @@ -37157,6 +37063,7 @@ def __init__( self.delete_files_after_completion = delete_files_after_completion self.modified_datetime_start = modified_datetime_start self.modified_datetime_end = modified_datetime_end + self.disable_chunking = disable_chunking class SftpServerLinkedService(LinkedService):