From abe23890b9fa667817fe93045e5fcb912455266a Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 17 Feb 2021 22:36:31 -0800 Subject: [PATCH 01/54] Add cloud event to core --- sdk/core/azure-core/CHANGELOG.md | 5 +- sdk/core/azure-core/azure/core/_version.py | 2 +- sdk/core/azure-core/azure/core/messaging.py | 127 ++++++++++++++++++ .../tests/test_messaging_from_dict.py | 76 +++++++++++ 4 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 sdk/core/azure-core/azure/core/messaging.py create mode 100644 sdk/core/azure-core/tests/test_messaging_from_dict.py diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 90e3b12f5520..3641c7733f78 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -1,7 +1,10 @@ # Release History -## 1.11.1 (Unreleased) +## 1.12.0 (Unreleased) +### Features + +- Added `azure.core.messaging.CloudEvent` model that follows the cloud event spec. ## 1.11.0 (2021-02-08) diff --git a/sdk/core/azure-core/azure/core/_version.py b/sdk/core/azure-core/azure/core/_version.py index 14d127f747d9..7643b787eff9 100644 --- a/sdk/core/azure-core/azure/core/_version.py +++ b/sdk/core/azure-core/azure/core/_version.py @@ -9,4 +9,4 @@ # regenerated. # -------------------------------------------------------------------------- -VERSION = "1.11.1" +VERSION = "1.12.0" diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py new file mode 100644 index 000000000000..599e7a9ec471 --- /dev/null +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -0,0 +1,127 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import uuid +import base64 +from datetime import tzinfo, timedelta, datetime + +try: + from datetime import timezone + TZ_UTC = timezone.utc # type: ignore +except ImportError: + class UTC(tzinfo): + """Time Zone info for handling UTC in python2""" + + def utcoffset(self, dt): + """UTF offset for UTC is 0.""" + return timedelta(0) + + def tzname(self, dt): + """Timestamp representation.""" + return "Z" + + def dst(self, dt): + """No daylight saving for UTC.""" + return timedelta(hours=1) + + TZ_UTC = UTC() # type: ignore + +try: + from typing import TYPE_CHECKING +except ImportError: + TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any, Dict + +__all__ = ["CloudEvent"] + + +class CloudEvent(object): #pylint:disable=too-many-instance-attributes + """Properties of the CloudEvent 1.0 Schema. + All required parameters must be populated in order to send to Azure. + If data is of binary type, data_base64 can be used alternatively. Note that data and data_base64 + cannot be present at the same time. + :param source: Required. Identifies the context in which an event happened. The combination of id and source must + be unique for each distinct event. If publishing to a domain topic, source must be the domain name. + :type source: str + :param type: Required. Type of event related to the originating occurrence. + :type type: str + :keyword data: Optional. Event data specific to the event type. If data is of bytes type, it will be sent + as data_base64 in the outgoing request. + :type data: object + :keyword time: Optional. The time (in UTC) the event was generated, in RFC3339 format. + :type time: ~datetime.datetime + :keyword dataschema: Optional. Identifies the schema that data adheres to. + :type dataschema: str + :keyword datacontenttype: Optional. Content type of data value. + :type datacontenttype: str + :keyword subject: Optional. This describes the subject of the event in the context of the event producer + (identified by source). + :type subject: str + :keyword specversion: Optional. The version of the CloudEvent spec. Defaults to "1.0" + :type specversion: str + :keyword id: Optional. An identifier for the event. The combination of id and source must be + unique for each distinct event. If not provided, a random UUID will be generated and used. + :type id: Optional[str] + :ivar source: Identifies the context in which an event happened. The combination of id and source must + be unique for each distinct event. If publishing to a domain topic, source must be the domain name. + :vartype source: str + :ivar data: Event data specific to the event type. + :vartype data: object + :ivar type: Type of event related to the originating occurrence. + :vartype type: str + :ivar time: The time (in UTC) the event was generated, in RFC3339 format. + :vartype time: ~datetime.datetime + :ivar dataschema: Identifies the schema that data adheres to. + :vartype dataschema: str + :ivar datacontenttype: Content type of data value. + :vartype datacontenttype: str + :ivar subject: This describes the subject of the event in the context of the event producer + (identified by source). + :vartype subject: str + :ivar specversion: Optional. The version of the CloudEvent spec. Defaults to "1.0" + :vartype specversion: str + :ivar id: An identifier for the event. The combination of id and source must be + unique for each distinct event. If not provided, a random UUID will be generated and used. + :vartype id: Optional[str] + """ + def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin + # type: (str, str, Any) -> None + self.source = source + self.type = type + self.specversion = kwargs.pop("specversion", "1.0") + self.id = kwargs.pop("id", str(uuid.uuid4())) + self.time = kwargs.pop("time", datetime.now(TZ_UTC).isoformat()) + self.datacontenttype = kwargs.pop("datacontenttype", None) + self.dataschema = kwargs.pop("dataschema", None) + self.subject = kwargs.pop("subject", None) + self.extensions = {} + self.extensions.update(dict(kwargs.pop('extensions', {}))) + self.data = kwargs.pop("data", None) + + @classmethod + def from_dict(cls, event, **kwargs): + # type: (Dict, Any) -> CloudEvent + """ + Returns the deserialized CloudEvent object when a dict is provided. + :param event: The dict representation of the event which needs to be deserialized. + :type event: dict + :rtype: CloudEvent + """ + return cls( + id=event.pop("id", None), + source=event.pop("source", None), + type=event.pop("type", None), + specversion=event.pop("specversion", None), + data=event.pop("data", None) or base64.b64decode(event.pop("data_base64", None)), + time=event.pop("time", None), + dataschema=event.pop("dataschema", None), + datacontenttype=event.pop("datacontenttype", None), + subject=event.pop("subject", None), + extensions=event, + **kwargs + ) diff --git a/sdk/core/azure-core/tests/test_messaging_from_dict.py b/sdk/core/azure-core/tests/test_messaging_from_dict.py new file mode 100644 index 000000000000..c868213073e6 --- /dev/null +++ b/sdk/core/azure-core/tests/test_messaging_from_dict.py @@ -0,0 +1,76 @@ +import logging +import sys +import os +import pytest +import json + +from azure.core.messaging import CloudEvent + +# Cloud Event tests +def test_cloud_storage_dict(): + cloud_storage_dict = { + "id":"a0517898-9fa4-4e70-b4a3-afda1dd68672", + "source":"/subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Storage/storageAccounts/{storage-account}", + "data":{ + "api":"PutBlockList", + "client_request_id":"6d79dbfb-0e37-4fc4-981f-442c9ca65760", + "request_id":"831e1650-001e-001b-66ab-eeb76e000000", + "e_tag":"0x8D4BCC2E4835CD0", + "content_type":"application/octet-stream", + "content_length":524288, + "blob_type":"BlockBlob", + "url":"https://oc2d2817345i60006.blob.core.windows.net/oc2d2817345i200097container/oc2d2817345i20002296blob", + "sequencer":"00000000000004420000000000028963", + "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} + }, + "type":"Microsoft.Storage.BlobCreated", + "time":"2020-08-07T01:11:49.765846Z", + "specversion":"1.0" + } + + event = CloudEvent.from_dict(cloud_storage_dict) + assert event.data == { + "api":"PutBlockList", + "client_request_id":"6d79dbfb-0e37-4fc4-981f-442c9ca65760", + "request_id":"831e1650-001e-001b-66ab-eeb76e000000", + "e_tag":"0x8D4BCC2E4835CD0", + "content_type":"application/octet-stream", + "content_length":524288, + "blob_type":"BlockBlob", + "url":"https://oc2d2817345i60006.blob.core.windows.net/oc2d2817345i200097container/oc2d2817345i20002296blob", + "sequencer":"00000000000004420000000000028963", + "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} + } + assert event.specversion == "1.0" + assert event.__class__ == CloudEvent + + +def test_cloud_custom_dict_with_extensions(): + cloud_custom_dict_with_extensions = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":{"team": "event grid squad"}, + "type":"Azure.Sdk.Sample", + "time":"2020-08-07T02:06:08.11969Z", + "specversion":"1.0", + "ext1": "example", + "ext2": "example2" + } + event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) + assert event.data == {"team": "event grid squad"} + assert event.__class__ == CloudEvent + assert event.extensions == {"ext1": "example", "ext2": "example2"} + +def test_cloud_custom_dict_base64(): + cloud_custom_dict_base64 = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data_base64":'Y2xvdWRldmVudA==', + "type":"Azure.Sdk.Sample", + "time":"2020-08-07T02:06:08.11969Z", + "specversion":"1.0" + } + event = CloudEvent.from_dict(cloud_custom_dict_base64) + assert event.data == b'cloudevent' + assert event.specversion == "1.0" + assert event.__class__ == CloudEvent From 9ed935df608ae79e9b95d26b5b64236934f05c7a Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 17 Feb 2021 23:30:34 -0800 Subject: [PATCH 02/54] extensions --- sdk/core/azure-core/azure/core/messaging.py | 6 +++++- ...saging_from_dict.py => test_messaging_cloud_event.py} | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) rename sdk/core/azure-core/tests/{test_messaging_from_dict.py => test_messaging_cloud_event.py} (91%) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 599e7a9ec471..891101dafeb8 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -100,7 +100,11 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - self.extensions.update(dict(kwargs.pop('extensions', {}))) + extensions = dict(kwargs.pop('extensions', {})) + for key in extensions.keys(): + if not key.islower() or not key.isalnum(): + raise ValueError("Extensions must be lower case and alphanumeric.") + self.extensions.update(extensions) self.data = kwargs.pop("data", None) @classmethod diff --git a/sdk/core/azure-core/tests/test_messaging_from_dict.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py similarity index 91% rename from sdk/core/azure-core/tests/test_messaging_from_dict.py rename to sdk/core/azure-core/tests/test_messaging_cloud_event.py index c868213073e6..b6641f101086 100644 --- a/sdk/core/azure-core/tests/test_messaging_from_dict.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -74,3 +74,12 @@ def test_cloud_custom_dict_base64(): assert event.data == b'cloudevent' assert event.specversion == "1.0" assert event.__class__ == CloudEvent + +def test_extensions_upper_case_value_error(): + with pytest.raises(ValueError): + event = CloudEvent( + source='sample', + type='type', + data='data', + extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} + ) From d11e02f83e539ff5aa1e9ae6c4060985e0e625eb Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 10:14:52 -0800 Subject: [PATCH 03/54] raise on both --- sdk/core/azure-core/azure/core/messaging.py | 12 ++++++++---- .../azure-core/tests/test_messaging_cloud_event.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 891101dafeb8..74d9995de594 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -100,11 +100,11 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - extensions = dict(kwargs.pop('extensions', {})) - for key in extensions.keys(): + _extensions = dict(kwargs.pop('extensions', {})) + for key in _extensions.keys(): if not key.islower() or not key.isalnum(): raise ValueError("Extensions must be lower case and alphanumeric.") - self.extensions.update(extensions) + self.extensions.update(_extensions) self.data = kwargs.pop("data", None) @classmethod @@ -116,12 +116,16 @@ def from_dict(cls, event, **kwargs): :type event: dict :rtype: CloudEvent """ + data = event.pop("data", None) + data_base64 = event.pop("data_base64", None) + if data and data_base64: + raise ValueError("Invalid input. Only one of data and data_base64 must be present.") return cls( id=event.pop("id", None), source=event.pop("source", None), type=event.pop("type", None), specversion=event.pop("specversion", None), - data=event.pop("data", None) or base64.b64decode(event.pop("data_base64", None)), + data=data or base64.b64decode(data_base64), time=event.pop("time", None), dataschema=event.pop("dataschema", None), datacontenttype=event.pop("datacontenttype", None), diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index b6641f101086..bb94799bce26 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -83,3 +83,13 @@ def test_extensions_upper_case_value_error(): data='data', extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} ) + +def test_data_and_base64_both_exist_raises(): + with pytest.raises(ValueError): + CloudEvent.from_dict( + {"source":'sample', + "type":'type', + "data":'data', + "data_base64":'Y2kQ==' + } + ) From ec3474c9aed0f6b2120e56bb37659a985e98a9d8 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 10:15:44 -0800 Subject: [PATCH 04/54] minor --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 74d9995de594..b9b7cc3880b1 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -5,7 +5,7 @@ # license information. # -------------------------------------------------------------------------- import uuid -import base64 +from base64 import b64decode from datetime import tzinfo, timedelta, datetime try: @@ -125,7 +125,7 @@ def from_dict(cls, event, **kwargs): source=event.pop("source", None), type=event.pop("type", None), specversion=event.pop("specversion", None), - data=data or base64.b64decode(data_base64), + data=data or b64decode(data_base64), time=event.pop("time", None), dataschema=event.pop("dataschema", None), datacontenttype=event.pop("datacontenttype", None), From e658ce76a7d52e20c08b2e4362222f572f63e95d Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 12:12:24 -0800 Subject: [PATCH 05/54] more changes --- sdk/core/azure-core/azure/core/messaging.py | 31 ++++++++--------- .../tests/test_messaging_cloud_event.py | 33 +++++++++++++++++++ 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index b9b7cc3880b1..304dae01810d 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -12,22 +12,8 @@ from datetime import timezone TZ_UTC = timezone.utc # type: ignore except ImportError: - class UTC(tzinfo): - """Time Zone info for handling UTC in python2""" - - def utcoffset(self, dt): - """UTF offset for UTC is 0.""" - return timedelta(0) - - def tzname(self, dt): - """Timestamp representation.""" - return "Z" - - def dst(self, dt): - """No daylight saving for UTC.""" - return timedelta(hours=1) - - TZ_UTC = UTC() # type: ignore + from azure.core.pipeline.policies._utils import _FixedOffset + TZ_UTC = _FixedOffset(0) # type: ignore try: from typing import TYPE_CHECKING @@ -102,11 +88,22 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.extensions = {} _extensions = dict(kwargs.pop('extensions', {})) for key in _extensions.keys(): - if not key.islower() or not key.isalnum(): + if not key.islower() or not key.isalnum() or len(key) > 20: raise ValueError("Extensions must be lower case and alphanumeric.") self.extensions.update(_extensions) self.data = kwargs.pop("data", None) + def __repr__(self): + return ( + "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( + self.source, + self.type, + self.specversion, + self.id, + self.time + )[:1024] + ) + @classmethod def from_dict(cls, event, **kwargs): # type: (Dict, Any) -> CloudEvent diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index bb94799bce26..948d58c376b5 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -3,10 +3,24 @@ import os import pytest import json +from datetime import datetime from azure.core.messaging import CloudEvent # Cloud Event tests +def test_cloud_event_constructor(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data='cloudevent' + ) + + assert event.specversion == '1.0' + assert event.time.endswith('+00:00') + assert event.id is not None + assert event.source == 'Azure.Core.Sample' + assert event.data == 'cloudevent' + def test_cloud_storage_dict(): cloud_storage_dict = { "id":"a0517898-9fa4-4e70-b4a3-afda1dd68672", @@ -84,6 +98,16 @@ def test_extensions_upper_case_value_error(): extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} ) + +def test_extensions_name_too_long_value_error(): + with pytest.raises(ValueError): + event = CloudEvent( + source='sample', + type='type', + data='data', + extensions={"lowercase123": "accepted", "thisislowerandtoolongforaname": "not allowed"} + ) + def test_data_and_base64_both_exist_raises(): with pytest.raises(ValueError): CloudEvent.from_dict( @@ -93,3 +117,12 @@ def test_data_and_base64_both_exist_raises(): "data_base64":'Y2kQ==' } ) + +def test_cloud_event_repr(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data='cloudevent' + ) + + assert repr(event).startswith("CloudEvent(source=Azure.Core.Sample, type=SampleType, specversion=1.0,") From 50de6e65f72431e0d803ea0d7cb17910375101e9 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 13:49:20 -0800 Subject: [PATCH 06/54] Update sdk/core/azure-core/azure/core/messaging.py --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 304dae01810d..bcbf758e87bb 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -6,7 +6,7 @@ # -------------------------------------------------------------------------- import uuid from base64 import b64decode -from datetime import tzinfo, timedelta, datetime +from datetime import datetime try: from datetime import timezone From 9f3624ba49fbbc728e18318b79a3afa590272886 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 15:55:04 -0800 Subject: [PATCH 07/54] comments --- sdk/core/azure-core/azure/core/_utils.py | 31 +++++++++++++ sdk/core/azure-core/azure/core/messaging.py | 44 ++++++++++--------- .../azure/core/pipeline/policies/_utils.py | 24 +--------- .../tests/test_messaging_cloud_event.py | 20 +-------- 4 files changed, 57 insertions(+), 62 deletions(-) create mode 100644 sdk/core/azure-core/azure/core/_utils.py diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py new file mode 100644 index 000000000000..a8380c485de1 --- /dev/null +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -0,0 +1,31 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import datetime + + +class _FixedOffset(datetime.tzinfo): + """Fixed offset in minutes east from UTC. + + Copy/pasted from Python doc + + :param int offset: offset in minutes + """ + + def __init__(self, offset): + self.__offset = datetime.timedelta(minutes=offset) + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return str(self.__offset.total_seconds()/3600) + + def __repr__(self): + return "".format(self.tzname(None)) + + def dst(self, dt): + return datetime.timedelta(0) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index bcbf758e87bb..11f748e8033b 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -12,7 +12,7 @@ from datetime import timezone TZ_UTC = timezone.utc # type: ignore except ImportError: - from azure.core.pipeline.policies._utils import _FixedOffset + from azure.core._utils import _FixedOffset TZ_UTC = _FixedOffset(0) # type: ignore try: @@ -40,7 +40,7 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes as data_base64 in the outgoing request. :type data: object :keyword time: Optional. The time (in UTC) the event was generated, in RFC3339 format. - :type time: ~datetime.datetime + :type time: str :keyword dataschema: Optional. Identifies the schema that data adheres to. :type dataschema: str :keyword datacontenttype: Optional. Content type of data value. @@ -53,6 +53,10 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :keyword id: Optional. An identifier for the event. The combination of id and source must be unique for each distinct event. If not provided, a random UUID will be generated and used. :type id: Optional[str] + :keyword extensions: Optional. A CloudEvent MAY include any number of additional context attributes + with distinct names represented as key - value pairs. Each extension must be alphanumeric, lower cased + and must not exceed the length of 20 characters. + :type extensions: Optional[dict] :ivar source: Identifies the context in which an event happened. The combination of id and source must be unique for each distinct event. If publishing to a domain topic, source must be the domain name. :vartype source: str @@ -61,7 +65,7 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :ivar type: Type of event related to the originating occurrence. :vartype type: str :ivar time: The time (in UTC) the event was generated, in RFC3339 format. - :vartype time: ~datetime.datetime + :vartype time: str :ivar dataschema: Identifies the schema that data adheres to. :vartype dataschema: str :ivar datacontenttype: Content type of data value. @@ -73,7 +77,11 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :vartype specversion: str :ivar id: An identifier for the event. The combination of id and source must be unique for each distinct event. If not provided, a random UUID will be generated and used. - :vartype id: Optional[str] + :vartype id: str + :ivar extensions: A CloudEvent MAY include any number of additional context attributes + with distinct names represented as key - value pairs. Each extension must be alphanumeric, lower cased + and must not exceed the length of 20 characters. + :vartype extensions: dict """ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, Any) -> None @@ -86,11 +94,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - _extensions = dict(kwargs.pop('extensions', {})) - for key in _extensions.keys(): - if not key.islower() or not key.isalnum() or len(key) > 20: - raise ValueError("Extensions must be lower case and alphanumeric.") - self.extensions.update(_extensions) + self.extensions.update(dict(kwargs.pop('extensions', {}))) self.data = kwargs.pop("data", None) def __repr__(self): @@ -118,15 +122,15 @@ def from_dict(cls, event, **kwargs): if data and data_base64: raise ValueError("Invalid input. Only one of data and data_base64 must be present.") return cls( - id=event.pop("id", None), - source=event.pop("source", None), - type=event.pop("type", None), - specversion=event.pop("specversion", None), - data=data or b64decode(data_base64), - time=event.pop("time", None), - dataschema=event.pop("dataschema", None), - datacontenttype=event.pop("datacontenttype", None), - subject=event.pop("subject", None), - extensions=event, - **kwargs + id=event.pop("id", None), + source=event.pop("source", None), + type=event.pop("type", None), + specversion=event.pop("specversion", None), + data=data or b64decode(data_base64), + time=event.pop("time", None), + dataschema=event.pop("dataschema", None), + datacontenttype=event.pop("datacontenttype", None), + subject=event.pop("subject", None), + extensions=event, + **kwargs ) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py index 173f19869804..76ee690c1d8f 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py @@ -26,29 +26,7 @@ import datetime import email.utils from requests.structures import CaseInsensitiveDict - -class _FixedOffset(datetime.tzinfo): - """Fixed offset in minutes east from UTC. - - Copy/pasted from Python doc - - :param int offset: offset in minutes - """ - - def __init__(self, offset): - self.__offset = datetime.timedelta(minutes=offset) - - def utcoffset(self, dt): - return self.__offset - - def tzname(self, dt): - return str(self.__offset.total_seconds()/3600) - - def __repr__(self): - return "".format(self.tzname(None)) - - def dst(self, dt): - return datetime.timedelta(0) +from ..._utils import _FixedOffset def _parse_http_date(text): """Parse a HTTP date format into datetime.""" diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 948d58c376b5..a8a5fb72fbfd 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -56,6 +56,7 @@ def test_cloud_storage_dict(): "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} } assert event.specversion == "1.0" + assert event.time == "2020-08-07T01:11:49.765846Z" assert event.__class__ == CloudEvent @@ -89,25 +90,6 @@ def test_cloud_custom_dict_base64(): assert event.specversion == "1.0" assert event.__class__ == CloudEvent -def test_extensions_upper_case_value_error(): - with pytest.raises(ValueError): - event = CloudEvent( - source='sample', - type='type', - data='data', - extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} - ) - - -def test_extensions_name_too_long_value_error(): - with pytest.raises(ValueError): - event = CloudEvent( - source='sample', - type='type', - data='data', - extensions={"lowercase123": "accepted", "thisislowerandtoolongforaname": "not allowed"} - ) - def test_data_and_base64_both_exist_raises(): with pytest.raises(ValueError): CloudEvent.from_dict( From 04acd47b38699a23236392e79afc7791d049cdda Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 19 Feb 2021 12:06:20 -0800 Subject: [PATCH 08/54] changes --- sdk/core/azure-core/azure/core/messaging.py | 23 +++++++---- .../tests/test_messaging_cloud_event.py | 41 +++++++++++++++++-- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 11f748e8033b..ba1389031848 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -7,6 +7,7 @@ import uuid from base64 import b64decode from datetime import datetime +import isodate try: from datetime import timezone @@ -39,8 +40,8 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :keyword data: Optional. Event data specific to the event type. If data is of bytes type, it will be sent as data_base64 in the outgoing request. :type data: object - :keyword time: Optional. The time (in UTC) the event was generated, in RFC3339 format. - :type time: str + :keyword time: Optional. The time (in UTC) the event was generated. + :type time: ~datetime.datetime :keyword dataschema: Optional. Identifies the schema that data adheres to. :type dataschema: str :keyword datacontenttype: Optional. Content type of data value. @@ -64,8 +65,8 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :vartype data: object :ivar type: Type of event related to the originating occurrence. :vartype type: str - :ivar time: The time (in UTC) the event was generated, in RFC3339 format. - :vartype time: str + :ivar time: The time (in UTC) the event was generated. + :vartype time: ~datetime.datetime :ivar dataschema: Identifies the schema that data adheres to. :vartype dataschema: str :ivar datacontenttype: Content type of data value. @@ -89,12 +90,16 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.type = type self.specversion = kwargs.pop("specversion", "1.0") self.id = kwargs.pop("id", str(uuid.uuid4())) - self.time = kwargs.pop("time", datetime.now(TZ_UTC).isoformat()) + self.time = kwargs.pop("time", datetime.now(TZ_UTC)) self.datacontenttype = kwargs.pop("datacontenttype", None) self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - self.extensions.update(dict(kwargs.pop('extensions', {}))) + _extensions = dict(kwargs.pop('extensions', {})) + for key in _extensions.keys(): + if not key.islower() or not key.isalnum(): + raise ValueError("Extension attributes should be lower cased and alphanumeric.") + self.extensions.update(_extensions) self.data = kwargs.pop("data", None) def __repr__(self): @@ -121,13 +126,17 @@ def from_dict(cls, event, **kwargs): data_base64 = event.pop("data_base64", None) if data and data_base64: raise ValueError("Invalid input. Only one of data and data_base64 must be present.") + try: + time = isodate.parse_datetime(event.pop("time", None)) + except AttributeError: + pass return cls( id=event.pop("id", None), source=event.pop("source", None), type=event.pop("type", None), specversion=event.pop("specversion", None), data=data or b64decode(data_base64), - time=event.pop("time", None), + time=time, dataschema=event.pop("dataschema", None), datacontenttype=event.pop("datacontenttype", None), subject=event.pop("subject", None), diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index a8a5fb72fbfd..e649e4227cf1 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -3,7 +3,7 @@ import os import pytest import json -from datetime import datetime +from datetime import datetime, timezone from azure.core.messaging import CloudEvent @@ -16,7 +16,7 @@ def test_cloud_event_constructor(): ) assert event.specversion == '1.0' - assert event.time.endswith('+00:00') + assert event.time.tzinfo == timezone.utc assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' @@ -56,7 +56,10 @@ def test_cloud_storage_dict(): "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} } assert event.specversion == "1.0" - assert event.time == "2020-08-07T01:11:49.765846Z" + assert event.time.__class__ == datetime + assert event.time.month == 8 + assert event.time.day == 7 + assert event.time.hour == 1 assert event.__class__ == CloudEvent @@ -108,3 +111,35 @@ def test_cloud_event_repr(): ) assert repr(event).startswith("CloudEvent(source=Azure.Core.Sample, type=SampleType, specversion=1.0,") + +def test_extensions_upper_case_value_error(): + with pytest.raises(ValueError): + event = CloudEvent( + source='sample', + type='type', + data='data', + extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} + ) + +def test_extensions_not_alphanumeric_value_error(): + with pytest.raises(ValueError): + event = CloudEvent( + source='sample', + type='type', + data='data', + extensions={"lowercase123": "accepted", "not@lph@nu^^3ic": "not allowed"} + ) + +def test_cloud_from_dict_with_invalid_extensions(): + cloud_custom_dict_with_extensions = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":{"team": "event grid squad"}, + "type":"Azure.Sdk.Sample", + "time":"2020-08-07T02:06:08.11969Z", + "specversion":"1.0", + "ext1": "example", + "BADext2": "example2" + } + with pytest.raises(ValueError): + event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) From f18f35ddd90af33f41a02169013cf4262e5bfc1e Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 19 Feb 2021 12:57:59 -0800 Subject: [PATCH 09/54] test fix --- .../azure-core/tests/test_messaging_cloud_event.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index e649e4227cf1..1147fb7fa114 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -3,7 +3,14 @@ import os import pytest import json -from datetime import datetime, timezone +from datetime import datetime + +try: + from datetime import timezone + TZ_UTC = timezone.utc # type: ignore +except ImportError: + from azure.core._utils import _FixedOffset + TZ_UTC = _FixedOffset(0) # type: ignore from azure.core.messaging import CloudEvent @@ -16,7 +23,7 @@ def test_cloud_event_constructor(): ) assert event.specversion == '1.0' - assert event.time.tzinfo == timezone.utc + assert event.time.tzinfo == TZ_UTC assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' From 9400e0e2a8c2ff4e4dbb6c51719e7a71b7b637b7 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 19 Feb 2021 13:38:39 -0800 Subject: [PATCH 10/54] test --- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 1147fb7fa114..52c90864572d 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -5,13 +5,6 @@ import json from datetime import datetime -try: - from datetime import timezone - TZ_UTC = timezone.utc # type: ignore -except ImportError: - from azure.core._utils import _FixedOffset - TZ_UTC = _FixedOffset(0) # type: ignore - from azure.core.messaging import CloudEvent # Cloud Event tests @@ -23,7 +16,7 @@ def test_cloud_event_constructor(): ) assert event.specversion == '1.0' - assert event.time.tzinfo == TZ_UTC + assert event.time.__class__ == datetime assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' From 70e08c097d18890d0ede6406d657ace289a1a096 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 21 Feb 2021 11:28:05 -0800 Subject: [PATCH 11/54] comments --- sdk/core/azure-core/azure/core/messaging.py | 50 ++++++++++++------- .../tests/test_messaging_cloud_event.py | 26 ++++++++++ 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index ba1389031848..f2f03f7e058c 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -22,7 +22,9 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict + from typing import Any, Dict, TypeVar + +CloudEventType = TypeVar('CloudEvent') __all__ = ["CloudEvent"] @@ -57,7 +59,7 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :keyword extensions: Optional. A CloudEvent MAY include any number of additional context attributes with distinct names represented as key - value pairs. Each extension must be alphanumeric, lower cased and must not exceed the length of 20 characters. - :type extensions: Optional[dict] + :type extensions: Optional[Dict] :ivar source: Identifies the context in which an event happened. The combination of id and source must be unique for each distinct event. If publishing to a domain topic, source must be the domain name. :vartype source: str @@ -85,7 +87,7 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :vartype extensions: dict """ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin - # type: (str, str, Any) -> None + # type: (str, str, **Any) -> None self.source = source self.type = type self.specversion = kwargs.pop("specversion", "1.0") @@ -115,31 +117,45 @@ def __repr__(self): @classmethod def from_dict(cls, event, **kwargs): - # type: (Dict, Any) -> CloudEvent + # type: (Type[CloudEventType], Dict, **Any) -> CloudEventType """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. :type event: dict :rtype: CloudEvent """ - data = event.pop("data", None) - data_base64 = event.pop("data_base64", None) + reserved_attr = [ + 'data', + 'data_base64', + 'id', + 'source', + 'type', + 'specversion', + 'time', + 'dataschema', + 'datacontenttype', + 'subject' + ] + + data = event.get("data", None) + data_base64 = event.get("data_base64", None) if data and data_base64: raise ValueError("Invalid input. Only one of data and data_base64 must be present.") + try: - time = isodate.parse_datetime(event.pop("time", None)) + time = isodate.parse_datetime(event.get("time", None)) except AttributeError: - pass + time = None return cls( - id=event.pop("id", None), - source=event.pop("source", None), - type=event.pop("type", None), - specversion=event.pop("specversion", None), - data=data or b64decode(data_base64), + id=event.get("id", None), + source=event.get("source", None), + type=event.get("type", None), + specversion=event.get("specversion", None), + data=data if data is not None else b64decode(data_base64), time=time, - dataschema=event.pop("dataschema", None), - datacontenttype=event.pop("datacontenttype", None), - subject=event.pop("subject", None), - extensions=event, + dataschema=event.get("dataschema", None), + datacontenttype=event.get("datacontenttype", None), + subject=event.get("subject", None), + extensions={k:v for k, v in event.items() if k not in reserved_attr}, **kwargs ) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 52c90864572d..a796db42033c 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -21,6 +21,19 @@ def test_cloud_event_constructor(): assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' +def test_cloud_event_constructor_blank_data(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data='' + ) + + assert event.specversion == '1.0' + assert event.time.__class__ == datetime + assert event.id is not None + assert event.source == 'Azure.Core.Sample' + assert event.data == '' + def test_cloud_storage_dict(): cloud_storage_dict = { "id":"a0517898-9fa4-4e70-b4a3-afda1dd68672", @@ -79,6 +92,19 @@ def test_cloud_custom_dict_with_extensions(): assert event.__class__ == CloudEvent assert event.extensions == {"ext1": "example", "ext2": "example2"} +def test_cloud_custom_dict_blank_data(): + cloud_custom_dict_with_extensions = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":'', + "type":"Azure.Sdk.Sample", + "time":"2020-08-07T02:06:08.11969Z", + "specversion":"1.0", + } + event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) + assert event.data == '' + assert event.__class__ == CloudEvent + def test_cloud_custom_dict_base64(): cloud_custom_dict_base64 = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", From b69cd73bdc40fdaddb95195299a2c79ef62defa6 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 21 Feb 2021 12:16:24 -0800 Subject: [PATCH 12/54] lint --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index f2f03f7e058c..b89b4d01f327 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -17,12 +17,12 @@ TZ_UTC = _FixedOffset(0) # type: ignore try: - from typing import TYPE_CHECKING + from typing import TYPE_CHECKING, TypeVar except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict, TypeVar + from typing import Any, Dict CloudEventType = TypeVar('CloudEvent') From 89c80b574b1ee1c386107d17ff4a6c0fa8040d63 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 21 Feb 2021 14:18:21 -0800 Subject: [PATCH 13/54] mypy --- sdk/core/azure-core/azure/core/messaging.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index b89b4d01f327..e138329a259e 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -17,15 +17,13 @@ TZ_UTC = _FixedOffset(0) # type: ignore try: - from typing import TYPE_CHECKING, TypeVar + from typing import TYPE_CHECKING except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: from typing import Any, Dict -CloudEventType = TypeVar('CloudEvent') - __all__ = ["CloudEvent"] @@ -117,7 +115,7 @@ def __repr__(self): @classmethod def from_dict(cls, event, **kwargs): - # type: (Type[CloudEventType], Dict, **Any) -> CloudEventType + # type: (CloudEvent, Dict, **Any) -> CloudEvent """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. From 10d81c305174a5d20b1d58a1614297d76d1b92f6 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 22 Feb 2021 10:23:23 -0800 Subject: [PATCH 14/54] type hint --- sdk/core/azure-core/azure/core/messaging.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index e138329a259e..613c264312c9 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -17,13 +17,15 @@ TZ_UTC = _FixedOffset(0) # type: ignore try: - from typing import TYPE_CHECKING + from typing import TYPE_CHECKING, Type, TypeVar except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: from typing import Any, Dict +CloudEventType = TypeVar("CloudEventType") + __all__ = ["CloudEvent"] @@ -115,7 +117,7 @@ def __repr__(self): @classmethod def from_dict(cls, event, **kwargs): - # type: (CloudEvent, Dict, **Any) -> CloudEvent + # type: (Type[CloudEventType], Dict, **Any) -> CloudEventType """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. From a121adc12fb103b9862fa5bb53d263f310a21b08 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 22 Feb 2021 10:56:39 -0800 Subject: [PATCH 15/54] Apply suggestions from code review --- sdk/core/azure-core/azure/core/messaging.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 613c264312c9..e674527f580b 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -17,14 +17,13 @@ TZ_UTC = _FixedOffset(0) # type: ignore try: - from typing import TYPE_CHECKING, Type, TypeVar + from typing import TYPE_CHECKING except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: from typing import Any, Dict -CloudEventType = TypeVar("CloudEventType") __all__ = ["CloudEvent"] @@ -117,7 +116,7 @@ def __repr__(self): @classmethod def from_dict(cls, event, **kwargs): - # type: (Type[CloudEventType], Dict, **Any) -> CloudEventType + # type: (Dict, **Any) -> CloudEvent """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. From 428e35c88ce9095b2a7ad360db44f0fbb4cc3eb9 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 23 Feb 2021 16:45:22 -0800 Subject: [PATCH 16/54] serialize date --- sdk/core/azure-core/azure/core/_utils.py | 23 +++++++++++++++++++ sdk/core/azure-core/azure/core/messaging.py | 9 +++----- .../tests/test_messaging_cloud_event.py | 17 ++++++++------ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index a8380c485de1..5b023ef0721d 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -5,6 +5,7 @@ # license information. # -------------------------------------------------------------------------- import datetime +import re class _FixedOffset(datetime.tzinfo): @@ -29,3 +30,25 @@ def __repr__(self): def dst(self, dt): return datetime.timedelta(0) + +def _convert_to_isoformat(date_time): + timestamp = re.split(r"([+|-])", re.sub(r"[:]|([-](?!((\d{2}[:]\d{2})|(\d{4}))$))", '', date_time)) + if len(timestamp) == 3: + time, sign, tzone = timestamp + else: + time = timestamp[0] + sign, tzone = None, None + + try: + deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%fZ") + except ValueError: + try: + deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%f") + except ValueError: + deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") + + if tzone: + delta = datetime.timedelta(hours=int(sign+tzone[:-2]), minutes=int(sign+tzone[-2:])) + deserialized = deserialized + delta + + return deserialized diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index e674527f580b..34346a1164d8 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -7,7 +7,6 @@ import uuid from base64 import b64decode from datetime import datetime -import isodate try: from datetime import timezone @@ -21,6 +20,8 @@ except ImportError: TYPE_CHECKING = False +from azure.core._utils import _convert_to_isoformat + if TYPE_CHECKING: from typing import Any, Dict @@ -141,17 +142,13 @@ def from_dict(cls, event, **kwargs): if data and data_base64: raise ValueError("Invalid input. Only one of data and data_base64 must be present.") - try: - time = isodate.parse_datetime(event.get("time", None)) - except AttributeError: - time = None return cls( id=event.get("id", None), source=event.get("source", None), type=event.get("type", None), specversion=event.get("specversion", None), data=data if data is not None else b64decode(data_base64), - time=time, + time=_convert_to_isoformat(event.get("time", None)), dataschema=event.get("dataschema", None), datacontenttype=event.get("datacontenttype", None), subject=event.get("subject", None), diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index a796db42033c..22790369f68a 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -51,7 +51,7 @@ def test_cloud_storage_dict(): "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} }, "type":"Microsoft.Storage.BlobCreated", - "time":"2020-08-07T01:11:49.765846Z", + "time":"2021-02-18T20:18:10.53986Z", "specversion":"1.0" } @@ -70,9 +70,9 @@ def test_cloud_storage_dict(): } assert event.specversion == "1.0" assert event.time.__class__ == datetime - assert event.time.month == 8 - assert event.time.day == 7 - assert event.time.hour == 1 + assert event.time.month == 2 + assert event.time.day == 18 + assert event.time.hour == 20 assert event.__class__ == CloudEvent @@ -82,7 +82,7 @@ def test_cloud_custom_dict_with_extensions(): "source":"https://egtest.dev/cloudcustomevent", "data":{"team": "event grid squad"}, "type":"Azure.Sdk.Sample", - "time":"2020-08-07T02:06:08.11969Z", + "time":"2021-02-18T20:18:10.53986+00:00", "specversion":"1.0", "ext1": "example", "ext2": "example2" @@ -90,6 +90,9 @@ def test_cloud_custom_dict_with_extensions(): event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) assert event.data == {"team": "event grid squad"} assert event.__class__ == CloudEvent + assert event.time.month == 2 + assert event.time.day == 18 + assert event.time.hour == 20 assert event.extensions == {"ext1": "example", "ext2": "example2"} def test_cloud_custom_dict_blank_data(): @@ -98,7 +101,7 @@ def test_cloud_custom_dict_blank_data(): "source":"https://egtest.dev/cloudcustomevent", "data":'', "type":"Azure.Sdk.Sample", - "time":"2020-08-07T02:06:08.11969Z", + "time":"2021-02-18T20:18:10+00:00", "specversion":"1.0", } event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) @@ -111,7 +114,7 @@ def test_cloud_custom_dict_base64(): "source":"https://egtest.dev/cloudcustomevent", "data_base64":'Y2xvdWRldmVudA==', "type":"Azure.Sdk.Sample", - "time":"2020-08-07T02:06:08.11969Z", + "time":"2021-02-18T20:18:10.345", "specversion":"1.0" } event = CloudEvent.from_dict(cloud_custom_dict_base64) From c88c1e99b0e14d5cd438974ec48a527a63a47422 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 23 Feb 2021 17:46:41 -0800 Subject: [PATCH 17/54] fix --- sdk/core/azure-core/azure/core/_utils.py | 6 +++--- .../azure-core/tests/test_messaging_cloud_event.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 5b023ef0721d..7731e4feb2a8 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -38,7 +38,7 @@ def _convert_to_isoformat(date_time): else: time = timestamp[0] sign, tzone = None, None - + try: deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%fZ") except ValueError: @@ -46,9 +46,9 @@ def _convert_to_isoformat(date_time): deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%f") except ValueError: deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") - + if tzone: delta = datetime.timedelta(hours=int(sign+tzone[:-2]), minutes=int(sign+tzone[-2:])) - deserialized = deserialized + delta + deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) return deserialized diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 22790369f68a..c057ebb572a0 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -3,7 +3,7 @@ import os import pytest import json -from datetime import datetime +import datetime from azure.core.messaging import CloudEvent @@ -16,7 +16,7 @@ def test_cloud_event_constructor(): ) assert event.specversion == '1.0' - assert event.time.__class__ == datetime + assert event.time.__class__ == datetime.datetime assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' @@ -29,7 +29,7 @@ def test_cloud_event_constructor_blank_data(): ) assert event.specversion == '1.0' - assert event.time.__class__ == datetime + assert event.time.__class__ == datetime.datetime assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == '' @@ -69,7 +69,7 @@ def test_cloud_storage_dict(): "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} } assert event.specversion == "1.0" - assert event.time.__class__ == datetime + assert event.time.__class__ == datetime.datetime assert event.time.month == 2 assert event.time.day == 18 assert event.time.hour == 20 @@ -114,12 +114,15 @@ def test_cloud_custom_dict_base64(): "source":"https://egtest.dev/cloudcustomevent", "data_base64":'Y2xvdWRldmVudA==', "type":"Azure.Sdk.Sample", - "time":"2021-02-18T20:18:10.345", + "time":"2021-02-23T17:11:13.308772-08:00", "specversion":"1.0" } event = CloudEvent.from_dict(cloud_custom_dict_base64) assert event.data == b'cloudevent' assert event.specversion == "1.0" + assert event.time.hour == 17 + assert event.time.day == 23 + assert event.time.tzinfo == datetime.timezone(datetime.timedelta(hours=-8)) assert event.__class__ == CloudEvent def test_data_and_base64_both_exist_raises(): From f6389618fb03b0db503201b0fa0be0b50aeca6f3 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 23 Feb 2021 18:19:16 -0800 Subject: [PATCH 18/54] fix --- sdk/core/azure-core/azure/core/_utils.py | 8 ++++++-- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 7731e4feb2a8..4543ae150f37 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -48,7 +48,11 @@ def _convert_to_isoformat(date_time): deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") if tzone: - delta = datetime.timedelta(hours=int(sign+tzone[:-2]), minutes=int(sign+tzone[-2:])) - deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) + hours=int(sign+tzone[:-2]) + delta = datetime.timedelta(minutes=int(sign+tzone[-2:])+hours*60) + try: + deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) + except AttributeError: + deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) return deserialized diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index c057ebb572a0..8dea394ae099 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -121,8 +121,9 @@ def test_cloud_custom_dict_base64(): assert event.data == b'cloudevent' assert event.specversion == "1.0" assert event.time.hour == 17 + assert event.time.minute == 11 assert event.time.day == 23 - assert event.time.tzinfo == datetime.timezone(datetime.timedelta(hours=-8)) + assert event.time.tzinfo is not None assert event.__class__ == CloudEvent def test_data_and_base64_both_exist_raises(): From 13335c110ee50a70432808b1c43254740c895309 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 23 Feb 2021 19:49:37 -0800 Subject: [PATCH 19/54] fix --- sdk/core/azure-core/azure/core/_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 4543ae150f37..620681f23ddd 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -48,11 +48,11 @@ def _convert_to_isoformat(date_time): deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") if tzone: - hours=int(sign+tzone[:-2]) - delta = datetime.timedelta(minutes=int(sign+tzone[-2:])+hours*60) + minutes = int(sign+tzone[:-2])*60 + int(sign+tzone[-2:]) + delta = datetime.timedelta(minutes=minutes) try: deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) except AttributeError: - deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) + deserialized = deserialized.replace(tzinfo=_FixedOffset(minutes)) return deserialized From 2dec99665dab4453d1e2cda1a75ec1c6b48cb28f Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 24 Feb 2021 09:41:36 -0800 Subject: [PATCH 20/54] Docstring Co-authored-by: Rakshith Bhyravabhotla --- sdk/core/azure-core/azure/core/_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 620681f23ddd..49cb91580da8 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -32,6 +32,9 @@ def dst(self, dt): return datetime.timedelta(0) def _convert_to_isoformat(date_time): + """Deserialize a date in RFC 3339 format to datetime object. + Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. + """ timestamp = re.split(r"([+|-])", re.sub(r"[:]|([-](?!((\d{2}[:]\d{2})|(\d{4}))$))", '', date_time)) if len(timestamp) == 3: time, sign, tzone = timestamp From c3368d5e206ef8cd4f33fbaf067190134f9352d7 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 24 Feb 2021 12:26:53 -0800 Subject: [PATCH 21/54] change util --- sdk/core/azure-core/azure/core/_utils.py | 31 ++++++++++-------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 49cb91580da8..6056cf5f1e15 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -35,27 +35,22 @@ def _convert_to_isoformat(date_time): """Deserialize a date in RFC 3339 format to datetime object. Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. """ - timestamp = re.split(r"([+|-])", re.sub(r"[:]|([-](?!((\d{2}[:]\d{2})|(\d{4}))$))", '', date_time)) - if len(timestamp) == 3: - time, sign, tzone = timestamp + if date_time[-1] == 'Z': + delta = 0 + timestamp = date_time[:-1] else: - time = timestamp[0] - sign, tzone = None, None + timestamp = date_time[:-6] + sign, offset = date_time[-6], date_time[-5:] + delta = int(sign+offset[:1])*60 + int(sign+offset[-2:]) try: - deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%fZ") + deserialized = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f') except ValueError: - try: - deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%f") - except ValueError: - deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") - - if tzone: - minutes = int(sign+tzone[:-2])*60 + int(sign+tzone[-2:]) - delta = datetime.timedelta(minutes=minutes) - try: - deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) - except AttributeError: - deserialized = deserialized.replace(tzinfo=_FixedOffset(minutes)) + deserialized = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S') + + try: + deserialized = deserialized.replace(tzinfo=datetime.timezone(datetime.timedelta(minutes=delta))) + except AttributeError: + deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) return deserialized From f461890d4dad54ca92990ce6ca79a35b8985350c Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 24 Feb 2021 12:56:52 -0800 Subject: [PATCH 22/54] lint --- sdk/core/azure-core/azure/core/_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 6056cf5f1e15..fd67bee91af9 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -5,7 +5,6 @@ # license information. # -------------------------------------------------------------------------- import datetime -import re class _FixedOffset(datetime.tzinfo): From 248db3dbb5cc3b220641d66f279725fd53ba0393 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 24 Feb 2021 15:58:35 -0800 Subject: [PATCH 23/54] apply black --- sdk/core/azure-core/azure/core/_utils.py | 23 ++++++-- sdk/core/azure-core/azure/core/messaging.py | 61 +++++++++------------ 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index fd67bee91af9..bbb1aef3f32d 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -22,7 +22,7 @@ def utcoffset(self, dt): return self.__offset def tzname(self, dt): - return str(self.__offset.total_seconds()/3600) + return str(self.__offset.total_seconds() / 3600) def __repr__(self): return "".format(self.tzname(None)) @@ -30,25 +30,36 @@ def __repr__(self): def dst(self, dt): return datetime.timedelta(0) + +try: + from datetime import timezone + + TZ_UTC = timezone.utc # type: ignore +except ImportError: + TZ_UTC = _FixedOffset(0) # type: ignore + + def _convert_to_isoformat(date_time): """Deserialize a date in RFC 3339 format to datetime object. Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. """ - if date_time[-1] == 'Z': + if date_time[-1] == "Z": delta = 0 timestamp = date_time[:-1] else: timestamp = date_time[:-6] sign, offset = date_time[-6], date_time[-5:] - delta = int(sign+offset[:1])*60 + int(sign+offset[-2:]) + delta = int(sign + offset[:1]) * 60 + int(sign + offset[-2:]) try: - deserialized = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f') + deserialized = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f") except ValueError: - deserialized = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S') + deserialized = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S") try: - deserialized = deserialized.replace(tzinfo=datetime.timezone(datetime.timedelta(minutes=delta))) + deserialized = deserialized.replace( + tzinfo=datetime.timezone(datetime.timedelta(minutes=delta)) + ) except AttributeError: deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 34346a1164d8..583e6f84f31b 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -5,23 +5,15 @@ # license information. # -------------------------------------------------------------------------- import uuid -from base64 import b64decode +from base64 import b64decode from datetime import datetime - -try: - from datetime import timezone - TZ_UTC = timezone.utc # type: ignore -except ImportError: - from azure.core._utils import _FixedOffset - TZ_UTC = _FixedOffset(0) # type: ignore +from azure.core._utils import _convert_to_isoformat, TZ_UTC try: from typing import TYPE_CHECKING except ImportError: TYPE_CHECKING = False -from azure.core._utils import _convert_to_isoformat - if TYPE_CHECKING: from typing import Any, Dict @@ -29,7 +21,7 @@ __all__ = ["CloudEvent"] -class CloudEvent(object): #pylint:disable=too-many-instance-attributes +class CloudEvent(object): # pylint:disable=too-many-instance-attributes """Properties of the CloudEvent 1.0 Schema. All required parameters must be populated in order to send to Azure. If data is of binary type, data_base64 can be used alternatively. Note that data and data_base64 @@ -86,7 +78,8 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes and must not exceed the length of 20 characters. :vartype extensions: dict """ - def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin + + def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, **Any) -> None self.source = source self.type = type @@ -97,23 +90,19 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - _extensions = dict(kwargs.pop('extensions', {})) + _extensions = dict(kwargs.pop("extensions", {})) for key in _extensions.keys(): if not key.islower() or not key.isalnum(): - raise ValueError("Extension attributes should be lower cased and alphanumeric.") + raise ValueError( + "Extension attributes should be lower cased and alphanumeric." + ) self.extensions.update(_extensions) self.data = kwargs.pop("data", None) def __repr__(self): - return ( - "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( - self.source, - self.type, - self.specversion, - self.id, - self.time - )[:1024] - ) + return "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( + self.source, self.type, self.specversion, self.id, self.time + )[:1024] @classmethod def from_dict(cls, event, **kwargs): @@ -125,22 +114,24 @@ def from_dict(cls, event, **kwargs): :rtype: CloudEvent """ reserved_attr = [ - 'data', - 'data_base64', - 'id', - 'source', - 'type', - 'specversion', - 'time', - 'dataschema', - 'datacontenttype', - 'subject' + "data", + "data_base64", + "id", + "source", + "type", + "specversion", + "time", + "dataschema", + "datacontenttype", + "subject", ] data = event.get("data", None) data_base64 = event.get("data_base64", None) if data and data_base64: - raise ValueError("Invalid input. Only one of data and data_base64 must be present.") + raise ValueError( + "Invalid input. Only one of data and data_base64 must be present." + ) return cls( id=event.get("id", None), @@ -152,6 +143,6 @@ def from_dict(cls, event, **kwargs): dataschema=event.get("dataschema", None), datacontenttype=event.get("datacontenttype", None), subject=event.get("subject", None), - extensions={k:v for k, v in event.items() if k not in reserved_attr}, + extensions={k: v for k, v in event.items() if k not in reserved_attr}, **kwargs ) From 32b25321da50a4294a0d59aeb828ea5091933133 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 24 Feb 2021 16:16:33 -0800 Subject: [PATCH 24/54] utilize tz utc --- sdk/core/azure-core/azure/core/_utils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index bbb1aef3f32d..9486795960ef 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -51,16 +51,18 @@ def _convert_to_isoformat(date_time): sign, offset = date_time[-6], date_time[-5:] delta = int(sign + offset[:1]) * 60 + int(sign + offset[-2:]) + if delta == 0: + tzinfo = TZ_UTC + else: + try: + tzinfo = datetime.timezone(datetime.timedelta(minutes=delta)) + except AttributeError: + tzinfo = _FixedOffset(delta) + try: deserialized = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f") except ValueError: deserialized = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S") - try: - deserialized = deserialized.replace( - tzinfo=datetime.timezone(datetime.timedelta(minutes=delta)) - ) - except AttributeError: - deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) - + deserialized = deserialized.replace(tzinfo=tzinfo) return deserialized From 244122283a32dac87814b8f1ee458e94fdceecf8 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 10:32:58 -0800 Subject: [PATCH 25/54] comments --- sdk/core/azure-core/azure/core/messaging.py | 38 ++++++++++--------- .../tests/test_messaging_cloud_event.py | 26 +++++++++++++ 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 583e6f84f31b..02f45f038743 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -24,8 +24,6 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes """Properties of the CloudEvent 1.0 Schema. All required parameters must be populated in order to send to Azure. - If data is of binary type, data_base64 can be used alternatively. Note that data and data_base64 - cannot be present at the same time. :param source: Required. Identifies the context in which an event happened. The combination of id and source must be unique for each distinct event. If publishing to a domain topic, source must be the domain name. :type source: str @@ -76,20 +74,20 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes :ivar extensions: A CloudEvent MAY include any number of additional context attributes with distinct names represented as key - value pairs. Each extension must be alphanumeric, lower cased and must not exceed the length of 20 characters. - :vartype extensions: dict + :vartype extensions: Dict """ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, **Any) -> None - self.source = source - self.type = type - self.specversion = kwargs.pop("specversion", "1.0") - self.id = kwargs.pop("id", str(uuid.uuid4())) - self.time = kwargs.pop("time", datetime.now(TZ_UTC)) - self.datacontenttype = kwargs.pop("datacontenttype", None) - self.dataschema = kwargs.pop("dataschema", None) - self.subject = kwargs.pop("subject", None) - self.extensions = {} + self.source = source # type: str + self.type = type # type: str + self.specversion = kwargs.pop("specversion", "1.0") # type: str + self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str + self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime + self.datacontenttype = kwargs.pop("datacontenttype", None) # type: str + self.dataschema = kwargs.pop("dataschema", None) # type: str + self.subject = kwargs.pop("subject", None) # type: str + self.extensions = {} # type: Dict _extensions = dict(kwargs.pop("extensions", {})) for key in _extensions.keys(): if not key.islower() or not key.isalnum(): @@ -97,7 +95,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin "Extension attributes should be lower cased and alphanumeric." ) self.extensions.update(_extensions) - self.data = kwargs.pop("data", None) + self.data = kwargs.pop("data", None) # type: object def __repr__(self): return "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( @@ -105,8 +103,8 @@ def __repr__(self): )[:1024] @classmethod - def from_dict(cls, event, **kwargs): - # type: (Dict, **Any) -> CloudEvent + def from_dict(cls, event): + # type: (Dict) -> CloudEvent """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. @@ -128,21 +126,25 @@ def from_dict(cls, event, **kwargs): data = event.get("data", None) data_base64 = event.get("data_base64", None) + if data and data_base64: raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) + elif data is None and data_base64 is None: + data = None + else: + data = data if data is not None else b64decode(data_base64) return cls( id=event.get("id", None), source=event.get("source", None), type=event.get("type", None), specversion=event.get("specversion", None), - data=data if data is not None else b64decode(data_base64), + data=data, time=_convert_to_isoformat(event.get("time", None)), dataschema=event.get("dataschema", None), datacontenttype=event.get("datacontenttype", None), subject=event.get("subject", None), - extensions={k: v for k, v in event.items() if k not in reserved_attr}, - **kwargs + extensions={k: v for k, v in event.items() if k not in reserved_attr} ) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 8dea394ae099..7cdcdc8a6a53 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -108,6 +108,32 @@ def test_cloud_custom_dict_blank_data(): assert event.data == '' assert event.__class__ == CloudEvent +def test_cloud_custom_dict_none_data(): + cloud_custom_dict_with_none_data = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":None, + "type":"Azure.Sdk.Sample", + "time":"2021-02-18T20:18:10+00:00", + "specversion":"1.0", + } + event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) + assert event.data == None + assert event.__class__ == CloudEvent + +def test_cloud_custom_dict_both_data_and_base64(): + cloud_custom_dict_with_data_and_base64 = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":"abc", + "data_base64":"Y2Wa==", + "type":"Azure.Sdk.Sample", + "time":"2021-02-18T20:18:10+00:00", + "specversion":"1.0", + } + with pytest.raises(ValueError): + event = CloudEvent.from_dict(cloud_custom_dict_with_data_and_base64) + def test_cloud_custom_dict_base64(): cloud_custom_dict_base64 = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", From 666bbd7efce8af0ace3627feda250bef2d167ffb Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 11:02:26 -0800 Subject: [PATCH 26/54] raise on unexpected kwargs --- sdk/core/azure-core/azure/core/messaging.py | 27 +++++++++++-------- .../tests/test_messaging_cloud_event.py | 9 +++++++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 02f45f038743..2540ab9b1ddc 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -79,15 +79,15 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, **Any) -> None - self.source = source # type: str - self.type = type # type: str - self.specversion = kwargs.pop("specversion", "1.0") # type: str - self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str - self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime - self.datacontenttype = kwargs.pop("datacontenttype", None) # type: str - self.dataschema = kwargs.pop("dataschema", None) # type: str - self.subject = kwargs.pop("subject", None) # type: str - self.extensions = {} # type: Dict + self.source = source # type: str + self.type = type # type: str + self.specversion = kwargs.pop("specversion", "1.0") # type: str + self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str + self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime + self.datacontenttype = kwargs.pop("datacontenttype", None) # type: str + self.dataschema = kwargs.pop("dataschema", None) # type: str + self.subject = kwargs.pop("subject", None) # type: str + self.extensions = {} # type: Dict _extensions = dict(kwargs.pop("extensions", {})) for key in _extensions.keys(): if not key.islower() or not key.isalnum(): @@ -95,7 +95,12 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin "Extension attributes should be lower cased and alphanumeric." ) self.extensions.update(_extensions) - self.data = kwargs.pop("data", None) # type: object + self.data = kwargs.pop("data", None) # type: object + + if kwargs: + raise ValueError( + "Unexpected keyword argument. Any extension attribures must be passed explicitly using extensions." + ) def __repr__(self): return "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( @@ -146,5 +151,5 @@ def from_dict(cls, event): dataschema=event.get("dataschema", None), datacontenttype=event.get("datacontenttype", None), subject=event.get("subject", None), - extensions={k: v for k, v in event.items() if k not in reserved_attr} + extensions={k: v for k, v in event.items() if k not in reserved_attr}, ) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 7cdcdc8a6a53..3f279c3e58ac 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -21,6 +21,15 @@ def test_cloud_event_constructor(): assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' +def test_cloud_event_constructor_unexpected_keyword(): + with pytest.raises(ValueError): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data='cloudevent', + unexpected_keyword="not allowed" + ) + def test_cloud_event_constructor_blank_data(): event = CloudEvent( source='Azure.Core.Sample', From 45d2a90a9ceba5c279dfedb088627b61c293a28c Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 11:03:37 -0800 Subject: [PATCH 27/54] doc --- sdk/core/azure-core/azure/core/messaging.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 2540ab9b1ddc..a0c5285bd2e0 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -29,8 +29,7 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes :type source: str :param type: Required. Type of event related to the originating occurrence. :type type: str - :keyword data: Optional. Event data specific to the event type. If data is of bytes type, it will be sent - as data_base64 in the outgoing request. + :keyword data: Optional. Event data specific to the event type. :type data: object :keyword time: Optional. The time (in UTC) the event was generated. :type time: ~datetime.datetime From 8c1f9fcf1d7de4eae6a523a38cca56c4de97ef7e Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 11:39:09 -0800 Subject: [PATCH 28/54] lint --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index a0c5285bd2e0..660be85c119d 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -137,7 +137,7 @@ def from_dict(cls, event): ) elif data is None and data_base64 is None: data = None - else: + elif data or data_base64: data = data if data is not None else b64decode(data_base64) return cls( From 8dcf54c9a6b16af4113e00203524df8ef31cc7b8 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 12:18:10 -0800 Subject: [PATCH 29/54] more lint --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 660be85c119d..40d0dd863197 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -135,7 +135,7 @@ def from_dict(cls, event): raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - elif data is None and data_base64 is None: + if data is None and data_base64 is None: data = None elif data or data_base64: data = data if data is not None else b64decode(data_base64) From f0d718fcb63021c3478b0ee2c491ce53832fe049 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 26 Feb 2021 12:56:16 -0800 Subject: [PATCH 30/54] attrs are optional --- sdk/core/azure-core/azure/core/_utils.py | 2 + sdk/core/azure-core/azure/core/messaging.py | 58 +++++++++++-------- .../tests/test_messaging_cloud_event.py | 21 ++++++- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 9486795960ef..d48581263be8 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -43,6 +43,8 @@ def _convert_to_isoformat(date_time): """Deserialize a date in RFC 3339 format to datetime object. Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. """ + if not date_time: + return if date_time[-1] == "Z": delta = 0 timestamp = date_time[:-1] diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 40d0dd863197..b4881f606a51 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -83,18 +83,22 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.specversion = kwargs.pop("specversion", "1.0") # type: str self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime - self.datacontenttype = kwargs.pop("datacontenttype", None) # type: str - self.dataschema = kwargs.pop("dataschema", None) # type: str - self.subject = kwargs.pop("subject", None) # type: str - self.extensions = {} # type: Dict - _extensions = dict(kwargs.pop("extensions", {})) - for key in _extensions.keys(): - if not key.islower() or not key.isalnum(): - raise ValueError( - "Extension attributes should be lower cased and alphanumeric." - ) - self.extensions.update(_extensions) - self.data = kwargs.pop("data", None) # type: object + + _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] + + for attr in _optional_attributes: + if attr in kwargs: + setattr(self, attr, kwargs.pop(attr)) + + if "extensions" in kwargs: + self.extensions = {} # type: Dict + _extensions = dict(kwargs.pop("extensions", {})) + for key in _extensions.keys(): + if not key.islower() or not key.isalnum(): + raise ValueError( + "Extension attributes should be lower cased and alphanumeric." + ) + self.extensions.update(_extensions) if kwargs: raise ValueError( @@ -115,6 +119,7 @@ def from_dict(cls, event): :type event: dict :rtype: CloudEvent """ + kwargs = {} reserved_attr = [ "data", "data_base64", @@ -128,27 +133,30 @@ def from_dict(cls, event): "subject", ] - data = event.get("data", None) - data_base64 = event.get("data_base64", None) - - if data and data_base64: + if "data" in event and "data_base64" in event: raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - if data is None and data_base64 is None: - data = None - elif data or data_base64: - data = data if data is not None else b64decode(data_base64) + + if "data" in event: + kwargs.setdefault("data", event.get("data")) + elif "data_base64" in event: + kwargs.setdefault("data", b64decode(event.get("data_base64"))) + + for item in ["datacontenttype", "dataschema", "subject"]: + if item in event: + kwargs.setdefault(item, event.get(item)) + + extensions={k: v for k, v in event.items() if k not in reserved_attr} + if extensions: + kwargs.setdefault("extensions", extensions) + return cls( id=event.get("id", None), source=event.get("source", None), type=event.get("type", None), specversion=event.get("specversion", None), - data=data, time=_convert_to_isoformat(event.get("time", None)), - dataschema=event.get("dataschema", None), - datacontenttype=event.get("datacontenttype", None), - subject=event.get("subject", None), - extensions={k: v for k, v in event.items() if k not in reserved_attr}, + **kwargs ) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 3f279c3e58ac..fbfe7f255cb3 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -43,6 +43,21 @@ def test_cloud_event_constructor_blank_data(): assert event.source == 'Azure.Core.Sample' assert event.data == '' +def test_cloud_event_constructor_no_data(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + ) + + with pytest.raises(AttributeError): + doesnt_exist = event.data + with pytest.raises(AttributeError): + doesnt_exist = event.datacontenttype + with pytest.raises(AttributeError): + doesnt_exist = event.dataschema + with pytest.raises(AttributeError): + doesnt_exist = event.subject + def test_cloud_storage_dict(): cloud_storage_dict = { "id":"a0517898-9fa4-4e70-b4a3-afda1dd68672", @@ -117,18 +132,18 @@ def test_cloud_custom_dict_blank_data(): assert event.data == '' assert event.__class__ == CloudEvent -def test_cloud_custom_dict_none_data(): +def test_cloud_custom_dict_no_data(): cloud_custom_dict_with_none_data = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", "source":"https://egtest.dev/cloudcustomevent", - "data":None, "type":"Azure.Sdk.Sample", "time":"2021-02-18T20:18:10+00:00", "specversion":"1.0", } event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) - assert event.data == None assert event.__class__ == CloudEvent + with pytest.raises(AttributeError): + missing = event.data def test_cloud_custom_dict_both_data_and_base64(): cloud_custom_dict_with_data_and_base64 = { From 950ffede75c82a146dcf35448cd349fbab0b8d8f Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 26 Feb 2021 15:56:07 -0800 Subject: [PATCH 31/54] add sentinel --- sdk/core/azure-core/CHANGELOG.md | 1 + sdk/core/azure-core/azure/core/_utils.py | 2 +- sdk/core/azure-core/azure/core/messaging.py | 19 ++++--- .../azure-core/azure/core/serialization.py | 10 ++++ .../tests/test_messaging_cloud_event.py | 54 +++++++++++++++---- 5 files changed, 66 insertions(+), 20 deletions(-) create mode 100644 sdk/core/azure-core/azure/core/serialization.py diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 3641c7733f78..7a2d0c81144d 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features - Added `azure.core.messaging.CloudEvent` model that follows the cloud event spec. +- Added `azure.core.serialization.NULL` sentinel value ## 1.11.0 (2021-02-08) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index d48581263be8..9178d4e5c7f1 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -44,7 +44,7 @@ def _convert_to_isoformat(date_time): Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. """ if not date_time: - return + return None if date_time[-1] == "Z": delta = 0 timestamp = date_time[:-1] diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index b4881f606a51..7ecb16263b86 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -8,6 +8,7 @@ from base64 import b64decode from datetime import datetime from azure.core._utils import _convert_to_isoformat, TZ_UTC +from azure.core.serialization import NULL try: from typing import TYPE_CHECKING @@ -87,8 +88,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] for attr in _optional_attributes: - if attr in kwargs: - setattr(self, attr, kwargs.pop(attr)) + setattr(self, attr, kwargs.pop(attr, None)) if "extensions" in kwargs: self.extensions = {} # type: Dict @@ -137,17 +137,20 @@ def from_dict(cls, event): raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - - if "data" in event: - kwargs.setdefault("data", event.get("data")) + if 'data' not in event and 'data_base64' not in event: + kwargs.setdefault("data", None) + elif "data" in event: + data = event.get("data") + kwargs.setdefault("data", data) if data is not None else kwargs.setdefault("data", NULL) elif "data_base64" in event: kwargs.setdefault("data", b64decode(event.get("data_base64"))) for item in ["datacontenttype", "dataschema", "subject"]: if item in event: - kwargs.setdefault(item, event.get(item)) - - extensions={k: v for k, v in event.items() if k not in reserved_attr} + val = event.get(item) + kwargs.setdefault(item, val) if val is not None else kwargs.setdefault(item, NULL) + + extensions = {k: v for k, v in event.items() if k not in reserved_attr} if extensions: kwargs.setdefault("extensions", extensions) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py new file mode 100644 index 000000000000..272f0239490a --- /dev/null +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -0,0 +1,10 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +__all__ = ["NULL"] + +NULL = False diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index fbfe7f255cb3..b15dc2689b65 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -6,6 +6,7 @@ import datetime from azure.core.messaging import CloudEvent +from azure.core.serialization import NULL # Cloud Event tests def test_cloud_event_constructor(): @@ -43,20 +44,37 @@ def test_cloud_event_constructor_blank_data(): assert event.source == 'Azure.Core.Sample' assert event.data == '' -def test_cloud_event_constructor_no_data(): +def test_cloud_event_constructor_NULL_data(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data=NULL + ) + + assert event.data == NULL + +def test_cloud_event_constructor_none_data(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data=None + ) + + assert event.data == None + assert event.datacontenttype == None + assert event.dataschema == None + assert event.subject == None + +def test_cloud_event_constructor_missing_data(): event = CloudEvent( source='Azure.Core.Sample', type='SampleType', ) - with pytest.raises(AttributeError): - doesnt_exist = event.data - with pytest.raises(AttributeError): - doesnt_exist = event.datacontenttype - with pytest.raises(AttributeError): - doesnt_exist = event.dataschema - with pytest.raises(AttributeError): - doesnt_exist = event.subject + assert event.data == None + assert event.datacontenttype == None + assert event.dataschema == None + assert event.subject == None def test_cloud_storage_dict(): cloud_storage_dict = { @@ -133,17 +151,31 @@ def test_cloud_custom_dict_blank_data(): assert event.__class__ == CloudEvent def test_cloud_custom_dict_no_data(): + cloud_custom_dict_with_missing_data = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "type":"Azure.Sdk.Sample", + "time":"2021-02-18T20:18:10+00:00", + "specversion":"1.0", + } + event = CloudEvent.from_dict(cloud_custom_dict_with_missing_data) + assert event.__class__ == CloudEvent + assert event.data == None + +def test_cloud_custom_dict_null_data(): cloud_custom_dict_with_none_data = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", "source":"https://egtest.dev/cloudcustomevent", "type":"Azure.Sdk.Sample", + "data":None, + "dataschema":None, "time":"2021-02-18T20:18:10+00:00", "specversion":"1.0", } event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) assert event.__class__ == CloudEvent - with pytest.raises(AttributeError): - missing = event.data + assert event.data == NULL + assert event.dataschema == NULL def test_cloud_custom_dict_both_data_and_base64(): cloud_custom_dict_with_data_and_base64 = { From c15a73acecb65b7146487fd876b023c6a7e34d59 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 28 Feb 2021 12:43:00 -0800 Subject: [PATCH 32/54] falsy object --- sdk/core/azure-core/azure/core/messaging.py | 21 ++++++++++++------- .../azure-core/azure/core/serialization.py | 8 ++++++- .../tests/test_messaging_cloud_event.py | 9 ++++---- .../azure-core/tests/test_serialization.py | 11 ++++++++++ 4 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 sdk/core/azure-core/tests/test_serialization.py diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 7ecb16263b86..7a7be166e3a1 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -11,12 +11,12 @@ from azure.core.serialization import NULL try: - from typing import TYPE_CHECKING + from typing import TYPE_CHECKING, cast except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict + from typing import Any, Dict, Union __all__ = ["CloudEvent"] @@ -88,7 +88,11 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] for attr in _optional_attributes: - setattr(self, attr, kwargs.pop(attr, None)) + if attr not in kwargs: + val = None + else: + val = kwargs.pop(attr, NULL) + setattr(self, attr, val) if "extensions" in kwargs: self.extensions = {} # type: Dict @@ -119,7 +123,7 @@ def from_dict(cls, event): :type event: dict :rtype: CloudEvent """ - kwargs = {} + kwargs = {} # type: Dict[Any, Any] reserved_attr = [ "data", "data_base64", @@ -138,12 +142,15 @@ def from_dict(cls, event): "Invalid input. Only one of data and data_base64 must be present." ) if 'data' not in event and 'data_base64' not in event: - kwargs.setdefault("data", None) + kwargs.setdefault("data", None) elif "data" in event: data = event.get("data") - kwargs.setdefault("data", data) if data is not None else kwargs.setdefault("data", NULL) + if data is not None: + kwargs.setdefault("data", data) + else: + kwargs.setdefault("data", NULL) elif "data_base64" in event: - kwargs.setdefault("data", b64decode(event.get("data_base64"))) + kwargs.setdefault("data", b64decode(cast(Union[str, bytes], event.get("data_base64")))) for item in ["datacontenttype", "dataschema", "subject"]: if item in event: diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 272f0239490a..a25c4e4002dc 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -7,4 +7,10 @@ __all__ = ["NULL"] -NULL = False +class _Null(object): + """To create a Falsy object + """ + def __bool__(self): + return False + +NULL = _Null() diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index b15dc2689b65..7c6fa70e41ac 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -1,3 +1,7 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ import logging import sys import os @@ -59,11 +63,8 @@ def test_cloud_event_constructor_none_data(): type='SampleType', data=None ) - + assert event.data == None - assert event.datacontenttype == None - assert event.dataschema == None - assert event.subject == None def test_cloud_event_constructor_missing_data(): event = CloudEvent( diff --git a/sdk/core/azure-core/tests/test_serialization.py b/sdk/core/azure-core/tests/test_serialization.py new file mode 100644 index 000000000000..8646ee1df2d9 --- /dev/null +++ b/sdk/core/azure-core/tests/test_serialization.py @@ -0,0 +1,11 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +from azure.core.serialization import NULL + +def test_NULL_is_falsy(): + assert NULL != False + assert (not NULL) + assert NULL is NULL \ No newline at end of file From a756fe3171c9a59dcf66f5c9487010bef12dfd87 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 28 Feb 2021 12:51:48 -0800 Subject: [PATCH 33/54] few more asserts --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 7a7be166e3a1..e620a81aa58f 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -11,12 +11,12 @@ from azure.core.serialization import NULL try: - from typing import TYPE_CHECKING, cast + from typing import TYPE_CHECKING, cast, Union except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict, Union + from typing import Any, Dict __all__ = ["CloudEvent"] diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 7c6fa70e41ac..d9f76320048d 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -117,6 +117,8 @@ def test_cloud_storage_dict(): assert event.time.day == 18 assert event.time.hour == 20 assert event.__class__ == CloudEvent + assert "id" in cloud_storage_dict + assert "data" in cloud_storage_dict def test_cloud_custom_dict_with_extensions(): From 6b4a31f88251254bdda84e69310fda71aabb4029 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 28 Feb 2021 13:16:32 -0800 Subject: [PATCH 34/54] lint --- sdk/core/azure-core/azure/core/messaging.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index e620a81aa58f..27437dd6dd0e 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -155,7 +155,10 @@ def from_dict(cls, event): for item in ["datacontenttype", "dataschema", "subject"]: if item in event: val = event.get(item) - kwargs.setdefault(item, val) if val is not None else kwargs.setdefault(item, NULL) + if val is not None: + kwargs.setdefault(item, val) + else: + kwargs.setdefault(item, NULL) extensions = {k: v for k, v in event.items() if k not in reserved_attr} if extensions: From 9371f77bd3937799cff7826f6e08f4619e019978 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 28 Feb 2021 13:54:45 -0800 Subject: [PATCH 35/54] pyt2 compat --- sdk/core/azure-core/azure/core/serialization.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index a25c4e4002dc..964fb8a64c12 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -13,4 +13,7 @@ class _Null(object): def __bool__(self): return False + __nonzero__ = __bool__ # Python2 compatibility + + NULL = _Null() From 78696ef0a557550a8740cbd58404f3fa003d2943 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 10:18:30 -0800 Subject: [PATCH 36/54] tests --- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 1 + sdk/core/azure-core/tests/test_serialization.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index d9f76320048d..7a0d122aa639 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -56,6 +56,7 @@ def test_cloud_event_constructor_NULL_data(): ) assert event.data == NULL + assert event.data is NULL def test_cloud_event_constructor_none_data(): event = CloudEvent( diff --git a/sdk/core/azure-core/tests/test_serialization.py b/sdk/core/azure-core/tests/test_serialization.py index 8646ee1df2d9..7ac58850cd91 100644 --- a/sdk/core/azure-core/tests/test_serialization.py +++ b/sdk/core/azure-core/tests/test_serialization.py @@ -6,6 +6,6 @@ from azure.core.serialization import NULL def test_NULL_is_falsy(): - assert NULL != False - assert (not NULL) + assert NULL is not False + assert bool(NULL) is False assert NULL is NULL \ No newline at end of file From 79e74d46c1c0b5b00e4ae39a549bf3446ec53641 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:11:57 -0800 Subject: [PATCH 37/54] comments --- sdk/core/azure-core/azure/core/messaging.py | 19 +++++++++---------- .../azure-core/azure/core/serialization.py | 4 ++++ .../tests/test_messaging_cloud_event.py | 6 ++++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 27437dd6dd0e..c879e979c664 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -88,25 +88,24 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] for attr in _optional_attributes: - if attr not in kwargs: - val = None - else: - val = kwargs.pop(attr, NULL) + val = None if attr not in kwargs else kwargs.pop(attr) setattr(self, attr, val) - if "extensions" in kwargs: - self.extensions = {} # type: Dict - _extensions = dict(kwargs.pop("extensions", {})) - for key in _extensions.keys(): + try: + self.extensions = kwargs.pop("extensions") # type: Dict + for key in self.extensions.keys(): if not key.islower() or not key.isalnum(): raise ValueError( "Extension attributes should be lower cased and alphanumeric." ) - self.extensions.update(_extensions) + except KeyError: + pass if kwargs: + remaining = ", ".join(kwargs.keys()) raise ValueError( - "Unexpected keyword argument. Any extension attribures must be passed explicitly using extensions." + "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions." + .format(remaining) ) def __repr__(self): diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 964fb8a64c12..98418a49c0c8 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -16,4 +16,8 @@ def __bool__(self): __nonzero__ = __bool__ # Python2 compatibility +""" +NULL is a falsy sentinel object which is supposed to be used to specify attributes +with no data. This gets serialized to `null` on the wire. +""" NULL = _Null() diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 7a0d122aa639..7023f0400664 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -27,12 +27,14 @@ def test_cloud_event_constructor(): assert event.data == 'cloudevent' def test_cloud_event_constructor_unexpected_keyword(): - with pytest.raises(ValueError): + remaining = "unexpected_keyword, another_bad_kwarg" + with pytest.raises(ValueError, match="Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions.".format(remaining)): event = CloudEvent( source='Azure.Core.Sample', type='SampleType', data='cloudevent', - unexpected_keyword="not allowed" + unexpected_keyword="not allowed", + another_bad_kwarg="not allowed either" ) def test_cloud_event_constructor_blank_data(): From a9c05f415c0480efcbd8d87c44d9f75baca1ab2f Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:31:49 -0800 Subject: [PATCH 38/54] update toc tree --- sdk/core/azure-core/doc/azure.core.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sdk/core/azure-core/doc/azure.core.rst b/sdk/core/azure-core/doc/azure.core.rst index 26b0607f3ca4..36716ad49c62 100644 --- a/sdk/core/azure-core/doc/azure.core.rst +++ b/sdk/core/azure-core/doc/azure.core.rst @@ -41,6 +41,14 @@ azure.core.exceptions :members: :undoc-members: +azure.core.messaging +------------------- + +.. automodule:: azure.core.messaging + :members: + :undoc-members: + :inherited-members: + azure.core.paging ----------------- @@ -57,3 +65,10 @@ azure.core.settings :undoc-members: :inherited-members: +azure.core.serialization +------------------- + +.. automodule:: azure.core.serialization + :members: + :undoc-members: + :inherited-members: From 57b1bd040a2f3e8136c8770b618adacd49bd93fa Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:49:14 -0800 Subject: [PATCH 39/54] doc --- sdk/core/azure-core/azure/core/serialization.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 98418a49c0c8..2877829ab767 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -15,9 +15,8 @@ def __bool__(self): __nonzero__ = __bool__ # Python2 compatibility - +NULL = _Null() """ -NULL is a falsy sentinel object which is supposed to be used to specify attributes +A falsy sentinel object which is supposed to be used to specify attributes with no data. This gets serialized to `null` on the wire. -""" -NULL = _Null() +""" \ No newline at end of file From e8b53b047b2d6fa528ec3cfd47930b8d14bf5286 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:49:26 -0800 Subject: [PATCH 40/54] doc --- sdk/core/azure-core/azure/core/serialization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 2877829ab767..c3422efa0c27 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -15,8 +15,9 @@ def __bool__(self): __nonzero__ = __bool__ # Python2 compatibility + NULL = _Null() """ A falsy sentinel object which is supposed to be used to specify attributes with no data. This gets serialized to `null` on the wire. -""" \ No newline at end of file +""" From 3a16ca9dcec64bc907cf7837a71d37ff67c406f9 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:51:01 -0800 Subject: [PATCH 41/54] doc --- sdk/core/azure-core/azure/core/messaging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index c879e979c664..ceb16781f899 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -25,6 +25,7 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes """Properties of the CloudEvent 1.0 Schema. All required parameters must be populated in order to send to Azure. + :param source: Required. Identifies the context in which an event happened. The combination of id and source must be unique for each distinct event. If publishing to a domain topic, source must be the domain name. :type source: str From 37e887cbdc159ad6a790ce5f61f1124f668a3a8d Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 14:05:16 -0800 Subject: [PATCH 42/54] unconditional --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index ceb16781f899..a8b3ace57add 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -100,7 +100,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin "Extension attributes should be lower cased and alphanumeric." ) except KeyError: - pass + self.extensions = None if kwargs: remaining = ", ".join(kwargs.keys()) From 8e196efb5f245440534af788dc231cb406bb3efc Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 14:26:53 -0800 Subject: [PATCH 43/54] test fix --- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 7023f0400664..96dd20758cd3 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -27,8 +27,7 @@ def test_cloud_event_constructor(): assert event.data == 'cloudevent' def test_cloud_event_constructor_unexpected_keyword(): - remaining = "unexpected_keyword, another_bad_kwarg" - with pytest.raises(ValueError, match="Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions.".format(remaining)): + with pytest.raises(ValueError) as e: event = CloudEvent( source='Azure.Core.Sample', type='SampleType', @@ -36,6 +35,8 @@ def test_cloud_event_constructor_unexpected_keyword(): unexpected_keyword="not allowed", another_bad_kwarg="not allowed either" ) + assert "unexpected_keyword" in e + assert "another_bad_kwarg" in e def test_cloud_event_constructor_blank_data(): event = CloudEvent( From b4c726e829fc4414a27dbdd83b6412ce6874b016 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 14:55:52 -0800 Subject: [PATCH 44/54] mypy --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index a8b3ace57add..53ba378db030 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -100,7 +100,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin "Extension attributes should be lower cased and alphanumeric." ) except KeyError: - self.extensions = None + self.extensions = cast(Dict, None) if kwargs: remaining = ", ".join(kwargs.keys()) From 3f49138620240dec5a07f03e9c3adb3a24e1f819 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 15:21:46 -0800 Subject: [PATCH 45/54] wrong import --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 53ba378db030..fa0a1b00aeb2 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -11,12 +11,12 @@ from azure.core.serialization import NULL try: - from typing import TYPE_CHECKING, cast, Union + from typing import TYPE_CHECKING, cast, Union, Dict except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict + from typing import Any __all__ = ["CloudEvent"] From 22db8b816a682c6d3c3c6303a6efcf2b55bf37b2 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 12:47:39 -0800 Subject: [PATCH 46/54] type annotations --- sdk/core/azure-core/azure/core/messaging.py | 47 ++++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index fa0a1b00aeb2..e26ed4a96d5f 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -11,12 +11,12 @@ from azure.core.serialization import NULL try: - from typing import TYPE_CHECKING, cast, Union, Dict + from typing import TYPE_CHECKING, cast, Union except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any + from typing import Any, Optional, Dict __all__ = ["CloudEvent"] @@ -82,31 +82,35 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, **Any) -> None self.source = source # type: str self.type = type # type: str - self.specversion = kwargs.pop("specversion", "1.0") # type: str - self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str - self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime + self.specversion = kwargs.pop("specversion", "1.0") # type: Optional[str] + self.id = kwargs.pop("id", str(uuid.uuid4())) # type: Optional[str] + self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: Optional[datetime] - _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] - - for attr in _optional_attributes: - val = None if attr not in kwargs else kwargs.pop(attr) - setattr(self, attr, val) + self.datacontenttype = kwargs.pop( + "datacontenttype", None + ) # type: Optional[str] + self.dataschema = kwargs.pop("dataschema", None) # type: Optional[str] + self.subject = kwargs.pop("subject", None) # type: Optional[str] + self.data = kwargs.pop("data", None) # type: Optional[object] try: - self.extensions = kwargs.pop("extensions") # type: Dict - for key in self.extensions.keys(): + self.extensions = kwargs.pop("extensions") # type: Optional[Dict] + for ( + key + ) in self.extensions.keys(): # type:ignore # extensions won't be None here if not key.islower() or not key.isalnum(): raise ValueError( "Extension attributes should be lower cased and alphanumeric." ) except KeyError: - self.extensions = cast(Dict, None) + self.extensions = None if kwargs: remaining = ", ".join(kwargs.keys()) raise ValueError( - "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions." - .format(remaining) + "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions.".format( + remaining + ) ) def __repr__(self): @@ -123,7 +127,7 @@ def from_dict(cls, event): :type event: dict :rtype: CloudEvent """ - kwargs = {} # type: Dict[Any, Any] + kwargs = {} # type: Dict[Any, Any] reserved_attr = [ "data", "data_base64", @@ -141,16 +145,18 @@ def from_dict(cls, event): raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - if 'data' not in event and 'data_base64' not in event: + if "data" not in event and "data_base64" not in event: kwargs.setdefault("data", None) elif "data" in event: data = event.get("data") if data is not None: - kwargs.setdefault("data", data) + kwargs["data"] = data else: - kwargs.setdefault("data", NULL) + kwargs["data"] = NULL elif "data_base64" in event: - kwargs.setdefault("data", b64decode(cast(Union[str, bytes], event.get("data_base64")))) + kwargs["data"] = b64decode( + cast(Union[str, bytes], event.get("data_base64")) + ) for item in ["datacontenttype", "dataschema", "subject"]: if item in event: @@ -164,7 +170,6 @@ def from_dict(cls, event): if extensions: kwargs.setdefault("extensions", extensions) - return cls( id=event.get("id", None), source=event.get("source", None), From 6e9ce14fdc2396f709e524509bf9ea3123ac1cf7 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 12:55:50 -0800 Subject: [PATCH 47/54] data --- sdk/core/azure-core/azure/core/messaging.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index e26ed4a96d5f..3a44d392b939 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -145,14 +145,10 @@ def from_dict(cls, event): raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - if "data" not in event and "data_base64" not in event: - kwargs.setdefault("data", None) - elif "data" in event: + + if "data" in event: data = event.get("data") - if data is not None: - kwargs["data"] = data - else: - kwargs["data"] = NULL + kwargs["data"] = data if data is not None else NULL elif "data_base64" in event: kwargs["data"] = b64decode( cast(Union[str, bytes], event.get("data_base64")) From 43a79c16af11df8efbcf31f726e6e2fc7da2a0be Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 13:02:29 -0800 Subject: [PATCH 48/54] coment --- sdk/core/azure-core/azure/core/messaging.py | 7 ++----- .../tests/test_messaging_cloud_event.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 3a44d392b939..f38fe163117b 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -157,14 +157,11 @@ def from_dict(cls, event): for item in ["datacontenttype", "dataschema", "subject"]: if item in event: val = event.get(item) - if val is not None: - kwargs.setdefault(item, val) - else: - kwargs.setdefault(item, NULL) + kwargs[item] = val if val is not None else NULL extensions = {k: v for k, v in event.items() if k not in reserved_attr} if extensions: - kwargs.setdefault("extensions", extensions) + kwargs["extensions"] = extensions return cls( id=event.get("id", None), diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 96dd20758cd3..1c5a5a104f8a 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -181,9 +181,24 @@ def test_cloud_custom_dict_null_data(): } event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) assert event.__class__ == CloudEvent - assert event.data == NULL + assert event.data is NULL assert event.dataschema == NULL +def test_cloud_custom_dict_valid_optional_attrs(): + cloud_custom_dict_with_none_data = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "type":"Azure.Sdk.Sample", + "data":None, + "dataschema":"exists", + "time":"2021-02-18T20:18:10+00:00", + "specversion":"1.0", + } + event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) + assert event.__class__ == CloudEvent + assert event.data is NULL + assert event.dataschema == "exists" + def test_cloud_custom_dict_both_data_and_base64(): cloud_custom_dict_with_data_and_base64 = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", From ba654a07e526e123f6dc123d1471b5af78c310ed Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 13:04:11 -0800 Subject: [PATCH 49/54] assets --- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 1c5a5a104f8a..c0a88b845488 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -167,7 +167,7 @@ def test_cloud_custom_dict_no_data(): } event = CloudEvent.from_dict(cloud_custom_dict_with_missing_data) assert event.__class__ == CloudEvent - assert event.data == None + assert event.data is None def test_cloud_custom_dict_null_data(): cloud_custom_dict_with_none_data = { @@ -181,8 +181,8 @@ def test_cloud_custom_dict_null_data(): } event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) assert event.__class__ == CloudEvent - assert event.data is NULL - assert event.dataschema == NULL + assert event.data == NULL + assert event.dataschema is NULL def test_cloud_custom_dict_valid_optional_attrs(): cloud_custom_dict_with_none_data = { From 4f9d80e171b23b2f45f4b0d4c153b93682c59e84 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 15:29:48 -0800 Subject: [PATCH 50/54] lint --- sdk/core/azure-core/azure/core/messaging.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index f38fe163117b..3a4e2295903a 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -108,9 +108,8 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin if kwargs: remaining = ", ".join(kwargs.keys()) raise ValueError( - "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions.".format( - remaining - ) + "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions." + .format(remaining) ) def __repr__(self): From 0a3aa87141463ddfc96c4489cbbceb4a615d9b56 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 17:29:55 -0800 Subject: [PATCH 51/54] unnecessary none --- sdk/core/azure-core/azure/core/messaging.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 3a4e2295903a..c02f975f95f5 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -163,10 +163,10 @@ def from_dict(cls, event): kwargs["extensions"] = extensions return cls( - id=event.get("id", None), - source=event.get("source", None), - type=event.get("type", None), - specversion=event.get("specversion", None), - time=_convert_to_isoformat(event.get("time", None)), + id=event.get("id"), + source=event.get("source"), + type=event.get("type"), + specversion=event.get("specversion"), + time=_convert_to_isoformat(event.get("time")), **kwargs ) From 50510341d01d8e466b2fe2af4de3bb606f7b42c2 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 17:36:21 -0800 Subject: [PATCH 52/54] format --- sdk/core/azure-core/azure/core/messaging.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index c02f975f95f5..1cdcb1062b44 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -86,18 +86,14 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.id = kwargs.pop("id", str(uuid.uuid4())) # type: Optional[str] self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: Optional[datetime] - self.datacontenttype = kwargs.pop( - "datacontenttype", None - ) # type: Optional[str] + self.datacontenttype = kwargs.pop("datacontenttype", None) # type: Optional[str] self.dataschema = kwargs.pop("dataschema", None) # type: Optional[str] self.subject = kwargs.pop("subject", None) # type: Optional[str] self.data = kwargs.pop("data", None) # type: Optional[object] try: self.extensions = kwargs.pop("extensions") # type: Optional[Dict] - for ( - key - ) in self.extensions.keys(): # type:ignore # extensions won't be None here + for key in self.extensions.keys(): # type:ignore # extensions won't be None here if not key.islower() or not key.isalnum(): raise ValueError( "Extension attributes should be lower cased and alphanumeric." From c7afd6e17b9567acde0bb46249f233aa387fd5e1 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 18:17:44 -0800 Subject: [PATCH 53/54] cast to str --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 1cdcb1062b44..f88ee89119b1 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -160,8 +160,8 @@ def from_dict(cls, event): return cls( id=event.get("id"), - source=event.get("source"), - type=event.get("type"), + source=cast(str, event.get("source")), + type=cast(str, event.get("type")), specversion=event.get("specversion"), time=_convert_to_isoformat(event.get("time")), **kwargs From 9613fd4370e1e94c41d351f7dd1a51e492eaaaf0 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 19:39:14 -0800 Subject: [PATCH 54/54] remove cast --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index f88ee89119b1..9131a7b46d69 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -160,8 +160,8 @@ def from_dict(cls, event): return cls( id=event.get("id"), - source=cast(str, event.get("source")), - type=cast(str, event.get("type")), + source=event["source"], + type=event["type"], specversion=event.get("specversion"), time=_convert_to_isoformat(event.get("time")), **kwargs