Merge branch 'main' of https://github.com/Azure/azure-sdk-for-python …

…into ga_azure_core_rest

* 'main' of https://github.com/Azure/azure-sdk-for-python:
  [rest] fix str check in content kwarg to be six.string_types check for 2.7 (Azure#21550)
  raise IncompleteReadError if only receive partial response (Azure#20888)
  Sync eng/common directory with azure-sdk-tools for PR 2053 (Azure#21558)
  Monitor metadata bg (Azure#21513)
  [QnA][CLU] Release prep (Azure#21518)
  Increment package version after release of azure-ai-textanalytics (Azure#21547)
  [SchemaRegistry] fix logging_enable bug (Azure#21488)
  add ReplaceLatestEntryTitle parameter for updating changelog (Azure#21485)

eng/common/TestResources/New-TestResources.ps1

@@ -334,7 +334,7 @@ try {
     $serviceName = if (Split-Path $ServiceDirectory) {
         Split-Path -Leaf $ServiceDirectory
     } else {
-        $ServiceDirectory
+        $ServiceDirectory.Trim('/')
     }
 
     $ResourceGroupName = if ($ResourceGroupName) {
@@ -600,7 +600,7 @@ try {
             $outputFile = "$($templateFile.originalFilePath).env"
 
             $environmentText = $deploymentOutputs | ConvertTo-Json;
-            $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText)
+            $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText)
             $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser)
 
             Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force

eng/scripts/Language-Settings.ps1

@@ -429,14 +429,15 @@ function Find-python-Artifacts-For-Apireview($artifactDir, $artifactName)
   return $packages
 }
 
-function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseDate)
+function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseDate, $ReplaceLatestEntryTitle)
 {
   if($null -eq $ReleaseDate)
   {
     $ReleaseDate = Get-Date -Format "yyyy-MM-dd"
   }
   pip install -r "$EngDir/versioning/requirements.txt" -q -I
-  python "$EngDir/versioning/version_set.py" --package-name $PackageName --new-version $Version --service $ServiceDirectory --release-date $ReleaseDate
+  python "$EngDir/versioning/version_set.py" --package-name $PackageName --new-version $Version `
+  --service $ServiceDirectory --release-date $ReleaseDate --replace-latest-entry-title $ReplaceLatestEntryTitle
 }
 
 function GetExistingPackageVersions ($PackageName, $GroupId=$null)

eng/versioning/version_set.py

@@ -11,6 +11,7 @@
     parser.add_argument('--new-version', required=True, help='new package version')
     parser.add_argument('--service', required=True, help='name of the service for which to set the dev build id (e.g. keyvault)')
     parser.add_argument('--release-date', help='date in the format "yyyy-MM-dd"')
+    parser.add_argument('--replace-latest-entry-title', help='indicate if to replace the latest changelog entry"')
     parser.add_argument(
         dest="glob_string",
         nargs="?",
@@ -38,4 +39,4 @@
 
     set_version_py(target_package[0], new_version)
     set_dev_classifier(target_package[0], new_version)
-    update_change_log(target_package[0], new_version, args.service, args.package_name, False, True, args.release_date)
+    update_change_log(target_package[0], new_version, args.service, args.package_name, False, args.replace_latest_entry_title, args.release_date)

sdk/cognitivelanguage/azure-ai-language-conversations/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Release History
 
-## 1.0.0b1 (unreleased)
+## 1.0.0b1 (2021-11-03)
 
 ### Features Added
 * Initial release

sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_conversation_app_language_parm_async.py

@@ -30,7 +30,7 @@ async def sample_analyze_conversation_app_language_parm_async():
     import os
     from azure.core.credentials import AzureKeyCredential
 
-    from azure.ai.language.conversations import ConversationAnalysisClient
+    from azure.ai.language.conversations.aio import ConversationAnalysisClient
     from azure.ai.language.conversations.models import ConversationAnalysisOptions
 
     # get secrets
@@ -48,7 +48,7 @@ async def sample_analyze_conversation_app_language_parm_async():
     # analyze quey
     client = ConversationAnalysisClient(conv_endpoint, AzureKeyCredential(conv_key))
     async with client:
-        result = client.analyze_conversations(
+        result = await client.analyze_conversations(
             input,
             project_name=conv_project,
             deployment_name='production'

sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_conversation_response_async.py

@@ -30,7 +30,7 @@ async def sample_analyze_orchestration_app_conversation_response_async():
     import os
     from azure.core.credentials import AzureKeyCredential
 
-    from azure.ai.language.conversations import ConversationAnalysisClient
+    from azure.ai.language.conversations.aio import ConversationAnalysisClient
     from azure.ai.language.conversations.models import ConversationAnalysisOptions
 
     # get secrets
@@ -47,7 +47,7 @@ async def sample_analyze_orchestration_app_conversation_response_async():
     # analyze query
     client = ConversationAnalysisClient(conv_endpoint, AzureKeyCredential(conv_key))
     async with client:
-        result = client.analyze_conversations(
+        result = await client.analyze_conversations(
             input,
             project_name=orchestration_project,
             deployment_name='production',

sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_luis_response_async.py

@@ -30,7 +30,7 @@ async def sample_analyze_orchestration_app_luis_response_async():
     import os
     from azure.core.credentials import AzureKeyCredential
 
-    from azure.ai.language.conversations import ConversationAnalysisClient
+    from azure.ai.language.conversations.aio import ConversationAnalysisClient
     from azure.ai.language.conversations.models import ConversationAnalysisOptions
 
     # get secrets
@@ -47,7 +47,7 @@ async def sample_analyze_orchestration_app_luis_response_async():
     # analyze query
     client = ConversationAnalysisClient(conv_endpoint, AzureKeyCredential(conv_key))
     async with client:
-        result = client.analyze_conversations(
+        result = await client.analyze_conversations(
             input,
             project_name=orchestration_project,
             deployment_name='production',

sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_qna_response_async.py

@@ -30,7 +30,7 @@ async def sample_analyze_orchestration_app_qna_response_async():
     import os
     from azure.core.credentials import AzureKeyCredential
 
-    from azure.ai.language.conversations import ConversationAnalysisClient
+    from azure.ai.language.conversations.aio import ConversationAnalysisClient
     from azure.ai.language.conversations.models import ConversationAnalysisOptions
 
     # get secrets
@@ -47,7 +47,7 @@ async def sample_analyze_orchestration_app_qna_response_async():
     # analyze query
     client = ConversationAnalysisClient(conv_endpoint, AzureKeyCredential(conv_key))
     async with client:
-        result = client.analyze_conversations(
+        result = await client.analyze_conversations(
             input,
             project_name=orchestration_project,
             deployment_name='production',

sdk/cognitivelanguage/azure-ai-language-conversations/setup.py

@@ -76,11 +76,15 @@
     packages=find_packages(exclude=[
         'tests',
         # Exclude packages that will be covered by PEP420 or nspkg
+        # This means any folder structure that only consists of a __init__.py.
+        # For example, for storage, this would mean adding 'azure.storage'
+        # in addition to the default 'azure' that is seen here.
+        'azure',
         'azure.ai',
         'azure.ai.language',
     ]),
     install_requires=[
-        "azure-core<2.0.0,>=1.19.0",
+        "azure-core<2.0.0,>=1.19.1",
         "msrest>=0.6.21",
         'azure-common~=1.1',
         'six>=1.11.0',

sdk/cognitivelanguage/azure-ai-language-questionanswering/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Release History
 
-## 1.0.0 (Unreleased)
+## 1.0.0 (2021-11-03)
 
 * We are now targeting service version `2021-10-01`
 

sdk/cognitivelanguage/azure-ai-language-questionanswering/setup.py

@@ -67,7 +67,7 @@
         'azure.ai.language',
     ]),
     install_requires=[
-        'azure-core<2.0.0,>=1.19.0',
+        'azure-core<2.0.0,>=1.19.1',
         'msrest>=0.6.21',
     ],
     extras_require={

sdk/core/azure-core/CHANGELOG.md

@@ -10,16 +10,19 @@ requests and sends them through our pipelines.
 - GA errors `StreamConsumedError`, `StreamClosedError`, and `ResponseNotReadError` to `azure.core.exceptions`. These errors
 are thrown if you mishandle streamed responses from the `azure.core.rest` module
 - add kwargs to the methods for `iter_raw` and `iter_bytes`  #21529
+- Added new error type `IncompleteReadError` which is raised if peer closes the connection before we have received the complete message body.
 
 ### Breaking Changes
 
 - SansIOHTTPPolicy.on_exception returns None instead of bool.
 
 ### Bugs Fixed
 
+- The `Content-Length` header in a http response is strictly checked against the actual number of bytes in the body,
+  rather than silently truncating data in case the underlying tcp connection is closed prematurely. 
+  (thanks to @jochen-ott-by for the contribution)   #20412
 - UnboundLocalError when SansIOHTTPPolicy handles an exception    #15222
-
-### Other Changes
+- Add default content type header of `text/plain` and content length header for users who pass unicode strings to the `content` kwarg of `HttpRequest` in 2.7  #21550
 
 ## 1.19.1 (2021-11-01)
 

sdk/core/azure-core/azure/core/exceptions.py

@@ -338,6 +338,10 @@ class DecodeError(HttpResponseError):
     """Error raised during response deserialization."""
 
 
+class IncompleteReadError(DecodeError):
+    """Error raised if peer closes the connection before we have received the complete message body."""
+
+
 class ResourceExistsError(HttpResponseError):
     """An error response with status code 4xx.
     This will not be raised directly by the Azure core pipeline."""

sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py

@@ -36,7 +36,7 @@
 from multidict import CIMultiDict
 
 from azure.core.configuration import ConnectionConfiguration
-from azure.core.exceptions import ServiceRequestError, ServiceResponseError
+from azure.core.exceptions import ServiceRequestError, ServiceResponseError, IncompleteReadError
 from azure.core.pipeline import Pipeline
 
 from ._base import HttpRequest
@@ -300,6 +300,12 @@ async def __anext__(self):
         except _ResponseStopIteration:
             internal_response.close()
             raise StopAsyncIteration()
+        except aiohttp.client_exceptions.ClientPayloadError as err:
+            # This is the case that server closes connection before we finish the reading. aiohttp library
+            # raises ClientPayloadError.
+            _LOGGER.warning("Incomplete download: %s", err)
+            internal_response.close()
+            raise IncompleteReadError(err, error=err)
         except Exception as err:
             _LOGGER.warning("Unable to stream download: %s", err)
             internal_response.close()
@@ -384,7 +390,12 @@ def text(self, encoding: Optional[str] = None) -> str:
 
     async def load_body(self) -> None:
         """Load in memory the body, so it could be accessible from sync methods."""
-        self._content = await self.internal_response.read()
+        try:
+            self._content = await self.internal_response.read()
+        except aiohttp.client_exceptions.ClientPayloadError as err:
+            # This is the case that server closes connection before we finish the reading. aiohttp library
+            # raises ClientPayloadError.
+            raise IncompleteReadError(err, error=err)
 
     def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]:
         """Generator for streaming response body data.

sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py

@@ -28,23 +28,25 @@
 import functools
 import logging
 from typing import (
-    Any, Union, Optional, AsyncIterator as AsyncIteratorType, TYPE_CHECKING, overload
+    Any, Optional, AsyncIterator as AsyncIteratorType, TYPE_CHECKING, overload
 )
 import urllib3 # type: ignore
 
 import requests
 
 from azure.core.exceptions import (
     ServiceRequestError,
-    ServiceResponseError
+    ServiceResponseError,
+    IncompleteReadError,
+    HttpResponseError,
 )
 from azure.core.pipeline import Pipeline
 from ._base import HttpRequest
 from ._base_async import (
     AsyncHttpResponse,
     _ResponseStopIteration,
     _iterate_response_content)
-from ._requests_basic import RequestsTransportResponse, _read_raw_stream
+from ._requests_basic import RequestsTransportResponse, _read_raw_stream, AzureErrorUnion
 from ._base_requests_async import RequestsAsyncTransportBase
 from .._tools import is_rest as _is_rest
 from .._tools_async import handle_no_stream_rest_response as _handle_no_stream_rest_response
@@ -134,7 +136,7 @@ async def send(self, request, **kwargs):  # pylint:disable=invalid-overridden-me
         self.open()
         loop = kwargs.get("loop", _get_running_loop())
         response = None
-        error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]]
+        error = None    # type: Optional[AzureErrorUnion]
         data_to_send = await self._retrieve_request_data(request)
         try:
             response = await loop.run_in_executor(
@@ -151,6 +153,7 @@ async def send(self, request, **kwargs):  # pylint:disable=invalid-overridden-me
                     cert=kwargs.pop('connection_cert', self.connection_config.cert),
                     allow_redirects=False,
                     **kwargs))
+            response.raw.enforce_content_length = True
 
         except urllib3.exceptions.NewConnectionError as err:
             error = ServiceRequestError(err, error=err)
@@ -161,6 +164,14 @@ async def send(self, request, **kwargs):  # pylint:disable=invalid-overridden-me
                 error = ServiceResponseError(err, error=err)
             else:
                 error = ServiceRequestError(err, error=err)
+        except requests.exceptions.ChunkedEncodingError as err:
+            msg = err.__str__()
+            if 'IncompleteRead' in msg:
+                _LOGGER.warning("Incomplete download: %s", err)
+                error = IncompleteReadError(err, error=err)
+            else:
+                _LOGGER.warning("Unable to stream download: %s", err)
+                error = HttpResponseError(err, error=err)
         except requests.RequestException as err:
             error = ServiceRequestError(err, error=err)
 
@@ -223,6 +234,15 @@ async def __anext__(self):
             raise StopAsyncIteration()
         except requests.exceptions.StreamConsumedError:
             raise
+        except requests.exceptions.ChunkedEncodingError as err:
+            msg = err.__str__()
+            if 'IncompleteRead' in msg:
+                _LOGGER.warning("Incomplete download: %s", err)
+                internal_response.close()
+                raise IncompleteReadError(err, error=err)
+            _LOGGER.warning("Unable to stream download: %s", err)
+            internal_response.close()
+            raise HttpResponseError(err, error=err)
         except Exception as err:
             _LOGGER.warning("Unable to stream download: %s", err)
             internal_response.close()

sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py

@@ -36,7 +36,9 @@
 from azure.core.configuration import ConnectionConfiguration
 from azure.core.exceptions import (
     ServiceRequestError,
-    ServiceResponseError
+    ServiceResponseError,
+    IncompleteReadError,
+    HttpResponseError,
 )
 from . import HttpRequest # pylint: disable=unused-import
 
@@ -51,6 +53,13 @@
 if TYPE_CHECKING:
     from ...rest import HttpRequest as RestHttpRequest, HttpResponse as RestHttpResponse
 
+AzureErrorUnion = Union[
+    ServiceRequestError,
+    ServiceResponseError,
+    IncompleteReadError,
+    HttpResponseError,
+]
+
 PipelineType = TypeVar("PipelineType")
 
 _LOGGER = logging.getLogger(__name__)
@@ -79,6 +88,7 @@ def _read_raw_stream(response, chunk_size=1):
     # https://github.com/psf/requests/blob/master/requests/models.py#L774
     response._content_consumed = True  # pylint: disable=protected-access
 
+
 class _RequestsTransportResponseBase(_HttpResponseBase):
     """Base class for accessing response data.
 
@@ -164,6 +174,15 @@ def __next__(self):
             raise StopIteration()
         except requests.exceptions.StreamConsumedError:
             raise
+        except requests.exceptions.ChunkedEncodingError as err:
+            msg = err.__str__()
+            if 'IncompleteRead' in msg:
+                _LOGGER.warning("Incomplete download: %s", err)
+                internal_response.close()
+                raise IncompleteReadError(err, error=err)
+            _LOGGER.warning("Unable to stream download: %s", err)
+            internal_response.close()
+            raise HttpResponseError(err, error=err)
         except Exception as err:
             _LOGGER.warning("Unable to stream download: %s", err)
             internal_response.close()
@@ -289,7 +308,7 @@ def send(self, request, **kwargs): # type: ignore
         """
         self.open()
         response = None
-        error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]]
+        error = None    # type: Optional[AzureErrorUnion]
 
         try:
             connection_timeout = kwargs.pop('connection_timeout', self.connection_config.timeout)
@@ -313,6 +332,7 @@ def send(self, request, **kwargs): # type: ignore
                 cert=kwargs.pop('connection_cert', self.connection_config.cert),
                 allow_redirects=False,
                 **kwargs)
+            response.raw.enforce_content_length = True
 
         except (urllib3.exceptions.NewConnectionError, urllib3.exceptions.ConnectTimeoutError) as err:
             error = ServiceRequestError(err, error=err)
@@ -323,6 +343,14 @@ def send(self, request, **kwargs): # type: ignore
                 error = ServiceResponseError(err, error=err)
             else:
                 error = ServiceRequestError(err, error=err)
+        except requests.exceptions.ChunkedEncodingError as err:
+            msg = err.__str__()
+            if 'IncompleteRead' in msg:
+                _LOGGER.warning("Incomplete download: %s", err)
+                error = IncompleteReadError(err, error=err)
+            else:
+                _LOGGER.warning("Unable to stream download: %s", err)
+                error = HttpResponseError(err, error=err)
         except requests.RequestException as err:
             error = ServiceRequestError(err, error=err)