From 5bd7aa215c2609f81570e7d258ab167979a88a3f Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 17 Feb 2021 22:36:31 -0800 Subject: [PATCH 01/51] Add cloud event to core --- sdk/core/azure-core/CHANGELOG.md | 5 +- sdk/core/azure-core/azure/core/_version.py | 2 +- sdk/core/azure-core/azure/core/messaging.py | 127 ++++++++++++++++++ .../tests/test_messaging_from_dict.py | 76 +++++++++++ 4 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 sdk/core/azure-core/azure/core/messaging.py create mode 100644 sdk/core/azure-core/tests/test_messaging_from_dict.py diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 90e3b12f5520..3641c7733f78 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -1,7 +1,10 @@ # Release History -## 1.11.1 (Unreleased) +## 1.12.0 (Unreleased) +### Features + +- Added `azure.core.messaging.CloudEvent` model that follows the cloud event spec. ## 1.11.0 (2021-02-08) diff --git a/sdk/core/azure-core/azure/core/_version.py b/sdk/core/azure-core/azure/core/_version.py index 14d127f747d9..7643b787eff9 100644 --- a/sdk/core/azure-core/azure/core/_version.py +++ b/sdk/core/azure-core/azure/core/_version.py @@ -9,4 +9,4 @@ # regenerated. # -------------------------------------------------------------------------- -VERSION = "1.11.1" +VERSION = "1.12.0" diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py new file mode 100644 index 000000000000..599e7a9ec471 --- /dev/null +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -0,0 +1,127 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import uuid +import base64 +from datetime import tzinfo, timedelta, datetime + +try: + from datetime import timezone + TZ_UTC = timezone.utc # type: ignore +except ImportError: + class UTC(tzinfo): + """Time Zone info for handling UTC in python2""" + + def utcoffset(self, dt): + """UTF offset for UTC is 0.""" + return timedelta(0) + + def tzname(self, dt): + """Timestamp representation.""" + return "Z" + + def dst(self, dt): + """No daylight saving for UTC.""" + return timedelta(hours=1) + + TZ_UTC = UTC() # type: ignore + +try: + from typing import TYPE_CHECKING +except ImportError: + TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any, Dict + +__all__ = ["CloudEvent"] + + +class CloudEvent(object): #pylint:disable=too-many-instance-attributes + """Properties of the CloudEvent 1.0 Schema. + All required parameters must be populated in order to send to Azure. + If data is of binary type, data_base64 can be used alternatively. Note that data and data_base64 + cannot be present at the same time. + :param source: Required. Identifies the context in which an event happened. The combination of id and source must + be unique for each distinct event. If publishing to a domain topic, source must be the domain name. + :type source: str + :param type: Required. Type of event related to the originating occurrence. + :type type: str + :keyword data: Optional. Event data specific to the event type. If data is of bytes type, it will be sent + as data_base64 in the outgoing request. + :type data: object + :keyword time: Optional. The time (in UTC) the event was generated, in RFC3339 format. + :type time: ~datetime.datetime + :keyword dataschema: Optional. Identifies the schema that data adheres to. + :type dataschema: str + :keyword datacontenttype: Optional. Content type of data value. + :type datacontenttype: str + :keyword subject: Optional. This describes the subject of the event in the context of the event producer + (identified by source). + :type subject: str + :keyword specversion: Optional. The version of the CloudEvent spec. Defaults to "1.0" + :type specversion: str + :keyword id: Optional. An identifier for the event. The combination of id and source must be + unique for each distinct event. If not provided, a random UUID will be generated and used. + :type id: Optional[str] + :ivar source: Identifies the context in which an event happened. The combination of id and source must + be unique for each distinct event. If publishing to a domain topic, source must be the domain name. + :vartype source: str + :ivar data: Event data specific to the event type. + :vartype data: object + :ivar type: Type of event related to the originating occurrence. + :vartype type: str + :ivar time: The time (in UTC) the event was generated, in RFC3339 format. + :vartype time: ~datetime.datetime + :ivar dataschema: Identifies the schema that data adheres to. + :vartype dataschema: str + :ivar datacontenttype: Content type of data value. + :vartype datacontenttype: str + :ivar subject: This describes the subject of the event in the context of the event producer + (identified by source). + :vartype subject: str + :ivar specversion: Optional. The version of the CloudEvent spec. Defaults to "1.0" + :vartype specversion: str + :ivar id: An identifier for the event. The combination of id and source must be + unique for each distinct event. If not provided, a random UUID will be generated and used. + :vartype id: Optional[str] + """ + def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin + # type: (str, str, Any) -> None + self.source = source + self.type = type + self.specversion = kwargs.pop("specversion", "1.0") + self.id = kwargs.pop("id", str(uuid.uuid4())) + self.time = kwargs.pop("time", datetime.now(TZ_UTC).isoformat()) + self.datacontenttype = kwargs.pop("datacontenttype", None) + self.dataschema = kwargs.pop("dataschema", None) + self.subject = kwargs.pop("subject", None) + self.extensions = {} + self.extensions.update(dict(kwargs.pop('extensions', {}))) + self.data = kwargs.pop("data", None) + + @classmethod + def from_dict(cls, event, **kwargs): + # type: (Dict, Any) -> CloudEvent + """ + Returns the deserialized CloudEvent object when a dict is provided. + :param event: The dict representation of the event which needs to be deserialized. + :type event: dict + :rtype: CloudEvent + """ + return cls( + id=event.pop("id", None), + source=event.pop("source", None), + type=event.pop("type", None), + specversion=event.pop("specversion", None), + data=event.pop("data", None) or base64.b64decode(event.pop("data_base64", None)), + time=event.pop("time", None), + dataschema=event.pop("dataschema", None), + datacontenttype=event.pop("datacontenttype", None), + subject=event.pop("subject", None), + extensions=event, + **kwargs + ) diff --git a/sdk/core/azure-core/tests/test_messaging_from_dict.py b/sdk/core/azure-core/tests/test_messaging_from_dict.py new file mode 100644 index 000000000000..c868213073e6 --- /dev/null +++ b/sdk/core/azure-core/tests/test_messaging_from_dict.py @@ -0,0 +1,76 @@ +import logging +import sys +import os +import pytest +import json + +from azure.core.messaging import CloudEvent + +# Cloud Event tests +def test_cloud_storage_dict(): + cloud_storage_dict = { + "id":"a0517898-9fa4-4e70-b4a3-afda1dd68672", + "source":"/subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Storage/storageAccounts/{storage-account}", + "data":{ + "api":"PutBlockList", + "client_request_id":"6d79dbfb-0e37-4fc4-981f-442c9ca65760", + "request_id":"831e1650-001e-001b-66ab-eeb76e000000", + "e_tag":"0x8D4BCC2E4835CD0", + "content_type":"application/octet-stream", + "content_length":524288, + "blob_type":"BlockBlob", + "url":"https://oc2d2817345i60006.blob.core.windows.net/oc2d2817345i200097container/oc2d2817345i20002296blob", + "sequencer":"00000000000004420000000000028963", + "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} + }, + "type":"Microsoft.Storage.BlobCreated", + "time":"2020-08-07T01:11:49.765846Z", + "specversion":"1.0" + } + + event = CloudEvent.from_dict(cloud_storage_dict) + assert event.data == { + "api":"PutBlockList", + "client_request_id":"6d79dbfb-0e37-4fc4-981f-442c9ca65760", + "request_id":"831e1650-001e-001b-66ab-eeb76e000000", + "e_tag":"0x8D4BCC2E4835CD0", + "content_type":"application/octet-stream", + "content_length":524288, + "blob_type":"BlockBlob", + "url":"https://oc2d2817345i60006.blob.core.windows.net/oc2d2817345i200097container/oc2d2817345i20002296blob", + "sequencer":"00000000000004420000000000028963", + "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} + } + assert event.specversion == "1.0" + assert event.__class__ == CloudEvent + + +def test_cloud_custom_dict_with_extensions(): + cloud_custom_dict_with_extensions = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":{"team": "event grid squad"}, + "type":"Azure.Sdk.Sample", + "time":"2020-08-07T02:06:08.11969Z", + "specversion":"1.0", + "ext1": "example", + "ext2": "example2" + } + event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) + assert event.data == {"team": "event grid squad"} + assert event.__class__ == CloudEvent + assert event.extensions == {"ext1": "example", "ext2": "example2"} + +def test_cloud_custom_dict_base64(): + cloud_custom_dict_base64 = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data_base64":'Y2xvdWRldmVudA==', + "type":"Azure.Sdk.Sample", + "time":"2020-08-07T02:06:08.11969Z", + "specversion":"1.0" + } + event = CloudEvent.from_dict(cloud_custom_dict_base64) + assert event.data == b'cloudevent' + assert event.specversion == "1.0" + assert event.__class__ == CloudEvent From f4df9a8aa44f3276db906b6f413555f0b7cebf04 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 17 Feb 2021 23:30:34 -0800 Subject: [PATCH 02/51] extensions --- sdk/core/azure-core/azure/core/messaging.py | 6 +++++- ...saging_from_dict.py => test_messaging_cloud_event.py} | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) rename sdk/core/azure-core/tests/{test_messaging_from_dict.py => test_messaging_cloud_event.py} (91%) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 599e7a9ec471..891101dafeb8 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -100,7 +100,11 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - self.extensions.update(dict(kwargs.pop('extensions', {}))) + extensions = dict(kwargs.pop('extensions', {})) + for key in extensions.keys(): + if not key.islower() or not key.isalnum(): + raise ValueError("Extensions must be lower case and alphanumeric.") + self.extensions.update(extensions) self.data = kwargs.pop("data", None) @classmethod diff --git a/sdk/core/azure-core/tests/test_messaging_from_dict.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py similarity index 91% rename from sdk/core/azure-core/tests/test_messaging_from_dict.py rename to sdk/core/azure-core/tests/test_messaging_cloud_event.py index c868213073e6..b6641f101086 100644 --- a/sdk/core/azure-core/tests/test_messaging_from_dict.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -74,3 +74,12 @@ def test_cloud_custom_dict_base64(): assert event.data == b'cloudevent' assert event.specversion == "1.0" assert event.__class__ == CloudEvent + +def test_extensions_upper_case_value_error(): + with pytest.raises(ValueError): + event = CloudEvent( + source='sample', + type='type', + data='data', + extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} + ) From 5a3addd461f6d15526e96057fa4c4f08f69e6ce6 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 10:14:52 -0800 Subject: [PATCH 03/51] raise on both --- sdk/core/azure-core/azure/core/messaging.py | 12 ++++++++---- .../azure-core/tests/test_messaging_cloud_event.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 891101dafeb8..74d9995de594 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -100,11 +100,11 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - extensions = dict(kwargs.pop('extensions', {})) - for key in extensions.keys(): + _extensions = dict(kwargs.pop('extensions', {})) + for key in _extensions.keys(): if not key.islower() or not key.isalnum(): raise ValueError("Extensions must be lower case and alphanumeric.") - self.extensions.update(extensions) + self.extensions.update(_extensions) self.data = kwargs.pop("data", None) @classmethod @@ -116,12 +116,16 @@ def from_dict(cls, event, **kwargs): :type event: dict :rtype: CloudEvent """ + data = event.pop("data", None) + data_base64 = event.pop("data_base64", None) + if data and data_base64: + raise ValueError("Invalid input. Only one of data and data_base64 must be present.") return cls( id=event.pop("id", None), source=event.pop("source", None), type=event.pop("type", None), specversion=event.pop("specversion", None), - data=event.pop("data", None) or base64.b64decode(event.pop("data_base64", None)), + data=data or base64.b64decode(data_base64), time=event.pop("time", None), dataschema=event.pop("dataschema", None), datacontenttype=event.pop("datacontenttype", None), diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index b6641f101086..bb94799bce26 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -83,3 +83,13 @@ def test_extensions_upper_case_value_error(): data='data', extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} ) + +def test_data_and_base64_both_exist_raises(): + with pytest.raises(ValueError): + CloudEvent.from_dict( + {"source":'sample', + "type":'type', + "data":'data', + "data_base64":'Y2kQ==' + } + ) From 39da5de707ff36e4776d172a1a18c6bd75e0d34c Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 10:15:44 -0800 Subject: [PATCH 04/51] minor --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 74d9995de594..b9b7cc3880b1 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -5,7 +5,7 @@ # license information. # -------------------------------------------------------------------------- import uuid -import base64 +from base64 import b64decode from datetime import tzinfo, timedelta, datetime try: @@ -125,7 +125,7 @@ def from_dict(cls, event, **kwargs): source=event.pop("source", None), type=event.pop("type", None), specversion=event.pop("specversion", None), - data=data or base64.b64decode(data_base64), + data=data or b64decode(data_base64), time=event.pop("time", None), dataschema=event.pop("dataschema", None), datacontenttype=event.pop("datacontenttype", None), From 462762ca7a09d37b6e3f10844c4766d5f3e44389 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 12:12:24 -0800 Subject: [PATCH 05/51] more changes --- sdk/core/azure-core/azure/core/messaging.py | 31 ++++++++--------- .../tests/test_messaging_cloud_event.py | 33 +++++++++++++++++++ 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index b9b7cc3880b1..304dae01810d 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -12,22 +12,8 @@ from datetime import timezone TZ_UTC = timezone.utc # type: ignore except ImportError: - class UTC(tzinfo): - """Time Zone info for handling UTC in python2""" - - def utcoffset(self, dt): - """UTF offset for UTC is 0.""" - return timedelta(0) - - def tzname(self, dt): - """Timestamp representation.""" - return "Z" - - def dst(self, dt): - """No daylight saving for UTC.""" - return timedelta(hours=1) - - TZ_UTC = UTC() # type: ignore + from azure.core.pipeline.policies._utils import _FixedOffset + TZ_UTC = _FixedOffset(0) # type: ignore try: from typing import TYPE_CHECKING @@ -102,11 +88,22 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.extensions = {} _extensions = dict(kwargs.pop('extensions', {})) for key in _extensions.keys(): - if not key.islower() or not key.isalnum(): + if not key.islower() or not key.isalnum() or len(key) > 20: raise ValueError("Extensions must be lower case and alphanumeric.") self.extensions.update(_extensions) self.data = kwargs.pop("data", None) + def __repr__(self): + return ( + "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( + self.source, + self.type, + self.specversion, + self.id, + self.time + )[:1024] + ) + @classmethod def from_dict(cls, event, **kwargs): # type: (Dict, Any) -> CloudEvent diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index bb94799bce26..948d58c376b5 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -3,10 +3,24 @@ import os import pytest import json +from datetime import datetime from azure.core.messaging import CloudEvent # Cloud Event tests +def test_cloud_event_constructor(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data='cloudevent' + ) + + assert event.specversion == '1.0' + assert event.time.endswith('+00:00') + assert event.id is not None + assert event.source == 'Azure.Core.Sample' + assert event.data == 'cloudevent' + def test_cloud_storage_dict(): cloud_storage_dict = { "id":"a0517898-9fa4-4e70-b4a3-afda1dd68672", @@ -84,6 +98,16 @@ def test_extensions_upper_case_value_error(): extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} ) + +def test_extensions_name_too_long_value_error(): + with pytest.raises(ValueError): + event = CloudEvent( + source='sample', + type='type', + data='data', + extensions={"lowercase123": "accepted", "thisislowerandtoolongforaname": "not allowed"} + ) + def test_data_and_base64_both_exist_raises(): with pytest.raises(ValueError): CloudEvent.from_dict( @@ -93,3 +117,12 @@ def test_data_and_base64_both_exist_raises(): "data_base64":'Y2kQ==' } ) + +def test_cloud_event_repr(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data='cloudevent' + ) + + assert repr(event).startswith("CloudEvent(source=Azure.Core.Sample, type=SampleType, specversion=1.0,") From 59ea3b3e32191def96d5a68dc3cc994c5bf4e31e Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 13:49:20 -0800 Subject: [PATCH 06/51] Update sdk/core/azure-core/azure/core/messaging.py --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 304dae01810d..bcbf758e87bb 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -6,7 +6,7 @@ # -------------------------------------------------------------------------- import uuid from base64 import b64decode -from datetime import tzinfo, timedelta, datetime +from datetime import datetime try: from datetime import timezone From 9a911ad394c3aaca5a63d23149e53eac1f3b2e86 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 18 Feb 2021 15:55:04 -0800 Subject: [PATCH 07/51] comments --- sdk/core/azure-core/azure/core/_utils.py | 31 +++++++++++++ sdk/core/azure-core/azure/core/messaging.py | 44 ++++++++++--------- .../azure/core/pipeline/policies/_utils.py | 24 +--------- .../tests/test_messaging_cloud_event.py | 20 +-------- 4 files changed, 57 insertions(+), 62 deletions(-) create mode 100644 sdk/core/azure-core/azure/core/_utils.py diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py new file mode 100644 index 000000000000..a8380c485de1 --- /dev/null +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -0,0 +1,31 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import datetime + + +class _FixedOffset(datetime.tzinfo): + """Fixed offset in minutes east from UTC. + + Copy/pasted from Python doc + + :param int offset: offset in minutes + """ + + def __init__(self, offset): + self.__offset = datetime.timedelta(minutes=offset) + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return str(self.__offset.total_seconds()/3600) + + def __repr__(self): + return "".format(self.tzname(None)) + + def dst(self, dt): + return datetime.timedelta(0) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index bcbf758e87bb..11f748e8033b 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -12,7 +12,7 @@ from datetime import timezone TZ_UTC = timezone.utc # type: ignore except ImportError: - from azure.core.pipeline.policies._utils import _FixedOffset + from azure.core._utils import _FixedOffset TZ_UTC = _FixedOffset(0) # type: ignore try: @@ -40,7 +40,7 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes as data_base64 in the outgoing request. :type data: object :keyword time: Optional. The time (in UTC) the event was generated, in RFC3339 format. - :type time: ~datetime.datetime + :type time: str :keyword dataschema: Optional. Identifies the schema that data adheres to. :type dataschema: str :keyword datacontenttype: Optional. Content type of data value. @@ -53,6 +53,10 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :keyword id: Optional. An identifier for the event. The combination of id and source must be unique for each distinct event. If not provided, a random UUID will be generated and used. :type id: Optional[str] + :keyword extensions: Optional. A CloudEvent MAY include any number of additional context attributes + with distinct names represented as key - value pairs. Each extension must be alphanumeric, lower cased + and must not exceed the length of 20 characters. + :type extensions: Optional[dict] :ivar source: Identifies the context in which an event happened. The combination of id and source must be unique for each distinct event. If publishing to a domain topic, source must be the domain name. :vartype source: str @@ -61,7 +65,7 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :ivar type: Type of event related to the originating occurrence. :vartype type: str :ivar time: The time (in UTC) the event was generated, in RFC3339 format. - :vartype time: ~datetime.datetime + :vartype time: str :ivar dataschema: Identifies the schema that data adheres to. :vartype dataschema: str :ivar datacontenttype: Content type of data value. @@ -73,7 +77,11 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :vartype specversion: str :ivar id: An identifier for the event. The combination of id and source must be unique for each distinct event. If not provided, a random UUID will be generated and used. - :vartype id: Optional[str] + :vartype id: str + :ivar extensions: A CloudEvent MAY include any number of additional context attributes + with distinct names represented as key - value pairs. Each extension must be alphanumeric, lower cased + and must not exceed the length of 20 characters. + :vartype extensions: dict """ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, Any) -> None @@ -86,11 +94,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - _extensions = dict(kwargs.pop('extensions', {})) - for key in _extensions.keys(): - if not key.islower() or not key.isalnum() or len(key) > 20: - raise ValueError("Extensions must be lower case and alphanumeric.") - self.extensions.update(_extensions) + self.extensions.update(dict(kwargs.pop('extensions', {}))) self.data = kwargs.pop("data", None) def __repr__(self): @@ -118,15 +122,15 @@ def from_dict(cls, event, **kwargs): if data and data_base64: raise ValueError("Invalid input. Only one of data and data_base64 must be present.") return cls( - id=event.pop("id", None), - source=event.pop("source", None), - type=event.pop("type", None), - specversion=event.pop("specversion", None), - data=data or b64decode(data_base64), - time=event.pop("time", None), - dataschema=event.pop("dataschema", None), - datacontenttype=event.pop("datacontenttype", None), - subject=event.pop("subject", None), - extensions=event, - **kwargs + id=event.pop("id", None), + source=event.pop("source", None), + type=event.pop("type", None), + specversion=event.pop("specversion", None), + data=data or b64decode(data_base64), + time=event.pop("time", None), + dataschema=event.pop("dataschema", None), + datacontenttype=event.pop("datacontenttype", None), + subject=event.pop("subject", None), + extensions=event, + **kwargs ) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py index 173f19869804..76ee690c1d8f 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py @@ -26,29 +26,7 @@ import datetime import email.utils from requests.structures import CaseInsensitiveDict - -class _FixedOffset(datetime.tzinfo): - """Fixed offset in minutes east from UTC. - - Copy/pasted from Python doc - - :param int offset: offset in minutes - """ - - def __init__(self, offset): - self.__offset = datetime.timedelta(minutes=offset) - - def utcoffset(self, dt): - return self.__offset - - def tzname(self, dt): - return str(self.__offset.total_seconds()/3600) - - def __repr__(self): - return "".format(self.tzname(None)) - - def dst(self, dt): - return datetime.timedelta(0) +from ..._utils import _FixedOffset def _parse_http_date(text): """Parse a HTTP date format into datetime.""" diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 948d58c376b5..a8a5fb72fbfd 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -56,6 +56,7 @@ def test_cloud_storage_dict(): "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} } assert event.specversion == "1.0" + assert event.time == "2020-08-07T01:11:49.765846Z" assert event.__class__ == CloudEvent @@ -89,25 +90,6 @@ def test_cloud_custom_dict_base64(): assert event.specversion == "1.0" assert event.__class__ == CloudEvent -def test_extensions_upper_case_value_error(): - with pytest.raises(ValueError): - event = CloudEvent( - source='sample', - type='type', - data='data', - extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} - ) - - -def test_extensions_name_too_long_value_error(): - with pytest.raises(ValueError): - event = CloudEvent( - source='sample', - type='type', - data='data', - extensions={"lowercase123": "accepted", "thisislowerandtoolongforaname": "not allowed"} - ) - def test_data_and_base64_both_exist_raises(): with pytest.raises(ValueError): CloudEvent.from_dict( From 723bd77b51bd2fe8c5745bda10efb6c5f8b27517 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 19 Feb 2021 12:06:20 -0800 Subject: [PATCH 08/51] changes --- sdk/core/azure-core/azure/core/messaging.py | 23 +++++++---- .../tests/test_messaging_cloud_event.py | 41 +++++++++++++++++-- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 11f748e8033b..ba1389031848 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -7,6 +7,7 @@ import uuid from base64 import b64decode from datetime import datetime +import isodate try: from datetime import timezone @@ -39,8 +40,8 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :keyword data: Optional. Event data specific to the event type. If data is of bytes type, it will be sent as data_base64 in the outgoing request. :type data: object - :keyword time: Optional. The time (in UTC) the event was generated, in RFC3339 format. - :type time: str + :keyword time: Optional. The time (in UTC) the event was generated. + :type time: ~datetime.datetime :keyword dataschema: Optional. Identifies the schema that data adheres to. :type dataschema: str :keyword datacontenttype: Optional. Content type of data value. @@ -64,8 +65,8 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :vartype data: object :ivar type: Type of event related to the originating occurrence. :vartype type: str - :ivar time: The time (in UTC) the event was generated, in RFC3339 format. - :vartype time: str + :ivar time: The time (in UTC) the event was generated. + :vartype time: ~datetime.datetime :ivar dataschema: Identifies the schema that data adheres to. :vartype dataschema: str :ivar datacontenttype: Content type of data value. @@ -89,12 +90,16 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.type = type self.specversion = kwargs.pop("specversion", "1.0") self.id = kwargs.pop("id", str(uuid.uuid4())) - self.time = kwargs.pop("time", datetime.now(TZ_UTC).isoformat()) + self.time = kwargs.pop("time", datetime.now(TZ_UTC)) self.datacontenttype = kwargs.pop("datacontenttype", None) self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - self.extensions.update(dict(kwargs.pop('extensions', {}))) + _extensions = dict(kwargs.pop('extensions', {})) + for key in _extensions.keys(): + if not key.islower() or not key.isalnum(): + raise ValueError("Extension attributes should be lower cased and alphanumeric.") + self.extensions.update(_extensions) self.data = kwargs.pop("data", None) def __repr__(self): @@ -121,13 +126,17 @@ def from_dict(cls, event, **kwargs): data_base64 = event.pop("data_base64", None) if data and data_base64: raise ValueError("Invalid input. Only one of data and data_base64 must be present.") + try: + time = isodate.parse_datetime(event.pop("time", None)) + except AttributeError: + pass return cls( id=event.pop("id", None), source=event.pop("source", None), type=event.pop("type", None), specversion=event.pop("specversion", None), data=data or b64decode(data_base64), - time=event.pop("time", None), + time=time, dataschema=event.pop("dataschema", None), datacontenttype=event.pop("datacontenttype", None), subject=event.pop("subject", None), diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index a8a5fb72fbfd..e649e4227cf1 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -3,7 +3,7 @@ import os import pytest import json -from datetime import datetime +from datetime import datetime, timezone from azure.core.messaging import CloudEvent @@ -16,7 +16,7 @@ def test_cloud_event_constructor(): ) assert event.specversion == '1.0' - assert event.time.endswith('+00:00') + assert event.time.tzinfo == timezone.utc assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' @@ -56,7 +56,10 @@ def test_cloud_storage_dict(): "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} } assert event.specversion == "1.0" - assert event.time == "2020-08-07T01:11:49.765846Z" + assert event.time.__class__ == datetime + assert event.time.month == 8 + assert event.time.day == 7 + assert event.time.hour == 1 assert event.__class__ == CloudEvent @@ -108,3 +111,35 @@ def test_cloud_event_repr(): ) assert repr(event).startswith("CloudEvent(source=Azure.Core.Sample, type=SampleType, specversion=1.0,") + +def test_extensions_upper_case_value_error(): + with pytest.raises(ValueError): + event = CloudEvent( + source='sample', + type='type', + data='data', + extensions={"lowercase123": "accepted", "NOTlower123": "not allowed"} + ) + +def test_extensions_not_alphanumeric_value_error(): + with pytest.raises(ValueError): + event = CloudEvent( + source='sample', + type='type', + data='data', + extensions={"lowercase123": "accepted", "not@lph@nu^^3ic": "not allowed"} + ) + +def test_cloud_from_dict_with_invalid_extensions(): + cloud_custom_dict_with_extensions = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":{"team": "event grid squad"}, + "type":"Azure.Sdk.Sample", + "time":"2020-08-07T02:06:08.11969Z", + "specversion":"1.0", + "ext1": "example", + "BADext2": "example2" + } + with pytest.raises(ValueError): + event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) From f3f16f84790c146198c81089d6e6b14159daa369 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 19 Feb 2021 12:57:59 -0800 Subject: [PATCH 09/51] test fix --- .../azure-core/tests/test_messaging_cloud_event.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index e649e4227cf1..1147fb7fa114 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -3,7 +3,14 @@ import os import pytest import json -from datetime import datetime, timezone +from datetime import datetime + +try: + from datetime import timezone + TZ_UTC = timezone.utc # type: ignore +except ImportError: + from azure.core._utils import _FixedOffset + TZ_UTC = _FixedOffset(0) # type: ignore from azure.core.messaging import CloudEvent @@ -16,7 +23,7 @@ def test_cloud_event_constructor(): ) assert event.specversion == '1.0' - assert event.time.tzinfo == timezone.utc + assert event.time.tzinfo == TZ_UTC assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' From c5c58e8eb1e60b61f393710e116ad27c07124fa5 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 19 Feb 2021 13:38:39 -0800 Subject: [PATCH 10/51] test --- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 1147fb7fa114..52c90864572d 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -5,13 +5,6 @@ import json from datetime import datetime -try: - from datetime import timezone - TZ_UTC = timezone.utc # type: ignore -except ImportError: - from azure.core._utils import _FixedOffset - TZ_UTC = _FixedOffset(0) # type: ignore - from azure.core.messaging import CloudEvent # Cloud Event tests @@ -23,7 +16,7 @@ def test_cloud_event_constructor(): ) assert event.specversion == '1.0' - assert event.time.tzinfo == TZ_UTC + assert event.time.__class__ == datetime assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' From 1bea68159415c01468b991ef6c5147ca44c2a04a Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 21 Feb 2021 11:28:05 -0800 Subject: [PATCH 11/51] comments --- sdk/core/azure-core/azure/core/messaging.py | 50 ++++++++++++------- .../tests/test_messaging_cloud_event.py | 26 ++++++++++ 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index ba1389031848..f2f03f7e058c 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -22,7 +22,9 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict + from typing import Any, Dict, TypeVar + +CloudEventType = TypeVar('CloudEvent') __all__ = ["CloudEvent"] @@ -57,7 +59,7 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :keyword extensions: Optional. A CloudEvent MAY include any number of additional context attributes with distinct names represented as key - value pairs. Each extension must be alphanumeric, lower cased and must not exceed the length of 20 characters. - :type extensions: Optional[dict] + :type extensions: Optional[Dict] :ivar source: Identifies the context in which an event happened. The combination of id and source must be unique for each distinct event. If publishing to a domain topic, source must be the domain name. :vartype source: str @@ -85,7 +87,7 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes :vartype extensions: dict """ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin - # type: (str, str, Any) -> None + # type: (str, str, **Any) -> None self.source = source self.type = type self.specversion = kwargs.pop("specversion", "1.0") @@ -115,31 +117,45 @@ def __repr__(self): @classmethod def from_dict(cls, event, **kwargs): - # type: (Dict, Any) -> CloudEvent + # type: (Type[CloudEventType], Dict, **Any) -> CloudEventType """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. :type event: dict :rtype: CloudEvent """ - data = event.pop("data", None) - data_base64 = event.pop("data_base64", None) + reserved_attr = [ + 'data', + 'data_base64', + 'id', + 'source', + 'type', + 'specversion', + 'time', + 'dataschema', + 'datacontenttype', + 'subject' + ] + + data = event.get("data", None) + data_base64 = event.get("data_base64", None) if data and data_base64: raise ValueError("Invalid input. Only one of data and data_base64 must be present.") + try: - time = isodate.parse_datetime(event.pop("time", None)) + time = isodate.parse_datetime(event.get("time", None)) except AttributeError: - pass + time = None return cls( - id=event.pop("id", None), - source=event.pop("source", None), - type=event.pop("type", None), - specversion=event.pop("specversion", None), - data=data or b64decode(data_base64), + id=event.get("id", None), + source=event.get("source", None), + type=event.get("type", None), + specversion=event.get("specversion", None), + data=data if data is not None else b64decode(data_base64), time=time, - dataschema=event.pop("dataschema", None), - datacontenttype=event.pop("datacontenttype", None), - subject=event.pop("subject", None), - extensions=event, + dataschema=event.get("dataschema", None), + datacontenttype=event.get("datacontenttype", None), + subject=event.get("subject", None), + extensions={k:v for k, v in event.items() if k not in reserved_attr}, **kwargs ) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 52c90864572d..a796db42033c 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -21,6 +21,19 @@ def test_cloud_event_constructor(): assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' +def test_cloud_event_constructor_blank_data(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data='' + ) + + assert event.specversion == '1.0' + assert event.time.__class__ == datetime + assert event.id is not None + assert event.source == 'Azure.Core.Sample' + assert event.data == '' + def test_cloud_storage_dict(): cloud_storage_dict = { "id":"a0517898-9fa4-4e70-b4a3-afda1dd68672", @@ -79,6 +92,19 @@ def test_cloud_custom_dict_with_extensions(): assert event.__class__ == CloudEvent assert event.extensions == {"ext1": "example", "ext2": "example2"} +def test_cloud_custom_dict_blank_data(): + cloud_custom_dict_with_extensions = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":'', + "type":"Azure.Sdk.Sample", + "time":"2020-08-07T02:06:08.11969Z", + "specversion":"1.0", + } + event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) + assert event.data == '' + assert event.__class__ == CloudEvent + def test_cloud_custom_dict_base64(): cloud_custom_dict_base64 = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", From 3c567c22550170e7f2f816201b14c54b8c0e440b Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 21 Feb 2021 12:16:24 -0800 Subject: [PATCH 12/51] lint --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index f2f03f7e058c..b89b4d01f327 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -17,12 +17,12 @@ TZ_UTC = _FixedOffset(0) # type: ignore try: - from typing import TYPE_CHECKING + from typing import TYPE_CHECKING, TypeVar except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict, TypeVar + from typing import Any, Dict CloudEventType = TypeVar('CloudEvent') From e6409170eb54f145ab71751ddf5464ce6d47e6ad Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 21 Feb 2021 14:18:21 -0800 Subject: [PATCH 13/51] mypy --- sdk/core/azure-core/azure/core/messaging.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index b89b4d01f327..e138329a259e 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -17,15 +17,13 @@ TZ_UTC = _FixedOffset(0) # type: ignore try: - from typing import TYPE_CHECKING, TypeVar + from typing import TYPE_CHECKING except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: from typing import Any, Dict -CloudEventType = TypeVar('CloudEvent') - __all__ = ["CloudEvent"] @@ -117,7 +115,7 @@ def __repr__(self): @classmethod def from_dict(cls, event, **kwargs): - # type: (Type[CloudEventType], Dict, **Any) -> CloudEventType + # type: (CloudEvent, Dict, **Any) -> CloudEvent """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. From 191ad53ba98dbe3a978c911461be1e623219c400 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 22 Feb 2021 10:23:23 -0800 Subject: [PATCH 14/51] type hint --- sdk/core/azure-core/azure/core/messaging.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index e138329a259e..613c264312c9 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -17,13 +17,15 @@ TZ_UTC = _FixedOffset(0) # type: ignore try: - from typing import TYPE_CHECKING + from typing import TYPE_CHECKING, Type, TypeVar except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: from typing import Any, Dict +CloudEventType = TypeVar("CloudEventType") + __all__ = ["CloudEvent"] @@ -115,7 +117,7 @@ def __repr__(self): @classmethod def from_dict(cls, event, **kwargs): - # type: (CloudEvent, Dict, **Any) -> CloudEvent + # type: (Type[CloudEventType], Dict, **Any) -> CloudEventType """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. From f83b91c3ad5a629565f97deada1aa56c20a65c86 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 22 Feb 2021 10:56:39 -0800 Subject: [PATCH 15/51] Apply suggestions from code review --- sdk/core/azure-core/azure/core/messaging.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 613c264312c9..e674527f580b 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -17,14 +17,13 @@ TZ_UTC = _FixedOffset(0) # type: ignore try: - from typing import TYPE_CHECKING, Type, TypeVar + from typing import TYPE_CHECKING except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: from typing import Any, Dict -CloudEventType = TypeVar("CloudEventType") __all__ = ["CloudEvent"] @@ -117,7 +116,7 @@ def __repr__(self): @classmethod def from_dict(cls, event, **kwargs): - # type: (Type[CloudEventType], Dict, **Any) -> CloudEventType + # type: (Dict, **Any) -> CloudEvent """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. From 027f57b582c729c3455b24c3abbbc9a342cc7df6 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 23 Feb 2021 16:45:22 -0800 Subject: [PATCH 16/51] serialize date --- sdk/core/azure-core/azure/core/_utils.py | 23 +++++++++++++++++++ sdk/core/azure-core/azure/core/messaging.py | 9 +++----- .../tests/test_messaging_cloud_event.py | 17 ++++++++------ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index a8380c485de1..5b023ef0721d 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -5,6 +5,7 @@ # license information. # -------------------------------------------------------------------------- import datetime +import re class _FixedOffset(datetime.tzinfo): @@ -29,3 +30,25 @@ def __repr__(self): def dst(self, dt): return datetime.timedelta(0) + +def _convert_to_isoformat(date_time): + timestamp = re.split(r"([+|-])", re.sub(r"[:]|([-](?!((\d{2}[:]\d{2})|(\d{4}))$))", '', date_time)) + if len(timestamp) == 3: + time, sign, tzone = timestamp + else: + time = timestamp[0] + sign, tzone = None, None + + try: + deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%fZ") + except ValueError: + try: + deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%f") + except ValueError: + deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") + + if tzone: + delta = datetime.timedelta(hours=int(sign+tzone[:-2]), minutes=int(sign+tzone[-2:])) + deserialized = deserialized + delta + + return deserialized diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index e674527f580b..34346a1164d8 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -7,7 +7,6 @@ import uuid from base64 import b64decode from datetime import datetime -import isodate try: from datetime import timezone @@ -21,6 +20,8 @@ except ImportError: TYPE_CHECKING = False +from azure.core._utils import _convert_to_isoformat + if TYPE_CHECKING: from typing import Any, Dict @@ -141,17 +142,13 @@ def from_dict(cls, event, **kwargs): if data and data_base64: raise ValueError("Invalid input. Only one of data and data_base64 must be present.") - try: - time = isodate.parse_datetime(event.get("time", None)) - except AttributeError: - time = None return cls( id=event.get("id", None), source=event.get("source", None), type=event.get("type", None), specversion=event.get("specversion", None), data=data if data is not None else b64decode(data_base64), - time=time, + time=_convert_to_isoformat(event.get("time", None)), dataschema=event.get("dataschema", None), datacontenttype=event.get("datacontenttype", None), subject=event.get("subject", None), diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index a796db42033c..22790369f68a 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -51,7 +51,7 @@ def test_cloud_storage_dict(): "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} }, "type":"Microsoft.Storage.BlobCreated", - "time":"2020-08-07T01:11:49.765846Z", + "time":"2021-02-18T20:18:10.53986Z", "specversion":"1.0" } @@ -70,9 +70,9 @@ def test_cloud_storage_dict(): } assert event.specversion == "1.0" assert event.time.__class__ == datetime - assert event.time.month == 8 - assert event.time.day == 7 - assert event.time.hour == 1 + assert event.time.month == 2 + assert event.time.day == 18 + assert event.time.hour == 20 assert event.__class__ == CloudEvent @@ -82,7 +82,7 @@ def test_cloud_custom_dict_with_extensions(): "source":"https://egtest.dev/cloudcustomevent", "data":{"team": "event grid squad"}, "type":"Azure.Sdk.Sample", - "time":"2020-08-07T02:06:08.11969Z", + "time":"2021-02-18T20:18:10.53986+00:00", "specversion":"1.0", "ext1": "example", "ext2": "example2" @@ -90,6 +90,9 @@ def test_cloud_custom_dict_with_extensions(): event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) assert event.data == {"team": "event grid squad"} assert event.__class__ == CloudEvent + assert event.time.month == 2 + assert event.time.day == 18 + assert event.time.hour == 20 assert event.extensions == {"ext1": "example", "ext2": "example2"} def test_cloud_custom_dict_blank_data(): @@ -98,7 +101,7 @@ def test_cloud_custom_dict_blank_data(): "source":"https://egtest.dev/cloudcustomevent", "data":'', "type":"Azure.Sdk.Sample", - "time":"2020-08-07T02:06:08.11969Z", + "time":"2021-02-18T20:18:10+00:00", "specversion":"1.0", } event = CloudEvent.from_dict(cloud_custom_dict_with_extensions) @@ -111,7 +114,7 @@ def test_cloud_custom_dict_base64(): "source":"https://egtest.dev/cloudcustomevent", "data_base64":'Y2xvdWRldmVudA==', "type":"Azure.Sdk.Sample", - "time":"2020-08-07T02:06:08.11969Z", + "time":"2021-02-18T20:18:10.345", "specversion":"1.0" } event = CloudEvent.from_dict(cloud_custom_dict_base64) From 40065ed2d93121ae9f000279d5a1ad26480e173a Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 23 Feb 2021 17:46:41 -0800 Subject: [PATCH 17/51] fix --- sdk/core/azure-core/azure/core/_utils.py | 6 +++--- .../azure-core/tests/test_messaging_cloud_event.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 5b023ef0721d..7731e4feb2a8 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -38,7 +38,7 @@ def _convert_to_isoformat(date_time): else: time = timestamp[0] sign, tzone = None, None - + try: deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%fZ") except ValueError: @@ -46,9 +46,9 @@ def _convert_to_isoformat(date_time): deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%f") except ValueError: deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") - + if tzone: delta = datetime.timedelta(hours=int(sign+tzone[:-2]), minutes=int(sign+tzone[-2:])) - deserialized = deserialized + delta + deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) return deserialized diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 22790369f68a..c057ebb572a0 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -3,7 +3,7 @@ import os import pytest import json -from datetime import datetime +import datetime from azure.core.messaging import CloudEvent @@ -16,7 +16,7 @@ def test_cloud_event_constructor(): ) assert event.specversion == '1.0' - assert event.time.__class__ == datetime + assert event.time.__class__ == datetime.datetime assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' @@ -29,7 +29,7 @@ def test_cloud_event_constructor_blank_data(): ) assert event.specversion == '1.0' - assert event.time.__class__ == datetime + assert event.time.__class__ == datetime.datetime assert event.id is not None assert event.source == 'Azure.Core.Sample' assert event.data == '' @@ -69,7 +69,7 @@ def test_cloud_storage_dict(): "storage_diagnostics":{"batchId":"b68529f3-68cd-4744-baa4-3c0498ec19f0"} } assert event.specversion == "1.0" - assert event.time.__class__ == datetime + assert event.time.__class__ == datetime.datetime assert event.time.month == 2 assert event.time.day == 18 assert event.time.hour == 20 @@ -114,12 +114,15 @@ def test_cloud_custom_dict_base64(): "source":"https://egtest.dev/cloudcustomevent", "data_base64":'Y2xvdWRldmVudA==', "type":"Azure.Sdk.Sample", - "time":"2021-02-18T20:18:10.345", + "time":"2021-02-23T17:11:13.308772-08:00", "specversion":"1.0" } event = CloudEvent.from_dict(cloud_custom_dict_base64) assert event.data == b'cloudevent' assert event.specversion == "1.0" + assert event.time.hour == 17 + assert event.time.day == 23 + assert event.time.tzinfo == datetime.timezone(datetime.timedelta(hours=-8)) assert event.__class__ == CloudEvent def test_data_and_base64_both_exist_raises(): From e62acf613ab1d8994f8b807c78bcc9ebde6959e2 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 23 Feb 2021 18:19:16 -0800 Subject: [PATCH 18/51] fix --- sdk/core/azure-core/azure/core/_utils.py | 8 ++++++-- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 7731e4feb2a8..4543ae150f37 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -48,7 +48,11 @@ def _convert_to_isoformat(date_time): deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") if tzone: - delta = datetime.timedelta(hours=int(sign+tzone[:-2]), minutes=int(sign+tzone[-2:])) - deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) + hours=int(sign+tzone[:-2]) + delta = datetime.timedelta(minutes=int(sign+tzone[-2:])+hours*60) + try: + deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) + except AttributeError: + deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) return deserialized diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index c057ebb572a0..8dea394ae099 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -121,8 +121,9 @@ def test_cloud_custom_dict_base64(): assert event.data == b'cloudevent' assert event.specversion == "1.0" assert event.time.hour == 17 + assert event.time.minute == 11 assert event.time.day == 23 - assert event.time.tzinfo == datetime.timezone(datetime.timedelta(hours=-8)) + assert event.time.tzinfo is not None assert event.__class__ == CloudEvent def test_data_and_base64_both_exist_raises(): From fb9ff6d112d345e17b8eb2ccadf24224b32ff4ff Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 23 Feb 2021 19:49:37 -0800 Subject: [PATCH 19/51] fix --- sdk/core/azure-core/azure/core/_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 4543ae150f37..620681f23ddd 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -48,11 +48,11 @@ def _convert_to_isoformat(date_time): deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") if tzone: - hours=int(sign+tzone[:-2]) - delta = datetime.timedelta(minutes=int(sign+tzone[-2:])+hours*60) + minutes = int(sign+tzone[:-2])*60 + int(sign+tzone[-2:]) + delta = datetime.timedelta(minutes=minutes) try: deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) except AttributeError: - deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) + deserialized = deserialized.replace(tzinfo=_FixedOffset(minutes)) return deserialized From d9b5d31b71f1bd4ab425ce72bc27261a8a3fde05 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 24 Feb 2021 09:41:36 -0800 Subject: [PATCH 20/51] Docstring Co-authored-by: Rakshith Bhyravabhotla --- sdk/core/azure-core/azure/core/_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 620681f23ddd..49cb91580da8 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -32,6 +32,9 @@ def dst(self, dt): return datetime.timedelta(0) def _convert_to_isoformat(date_time): + """Deserialize a date in RFC 3339 format to datetime object. + Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. + """ timestamp = re.split(r"([+|-])", re.sub(r"[:]|([-](?!((\d{2}[:]\d{2})|(\d{4}))$))", '', date_time)) if len(timestamp) == 3: time, sign, tzone = timestamp From 7e22c2102eb03c3bf5716a53575c419793093220 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 24 Feb 2021 12:26:53 -0800 Subject: [PATCH 21/51] change util --- sdk/core/azure-core/azure/core/_utils.py | 31 ++++++++++-------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 49cb91580da8..6056cf5f1e15 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -35,27 +35,22 @@ def _convert_to_isoformat(date_time): """Deserialize a date in RFC 3339 format to datetime object. Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. """ - timestamp = re.split(r"([+|-])", re.sub(r"[:]|([-](?!((\d{2}[:]\d{2})|(\d{4}))$))", '', date_time)) - if len(timestamp) == 3: - time, sign, tzone = timestamp + if date_time[-1] == 'Z': + delta = 0 + timestamp = date_time[:-1] else: - time = timestamp[0] - sign, tzone = None, None + timestamp = date_time[:-6] + sign, offset = date_time[-6], date_time[-5:] + delta = int(sign+offset[:1])*60 + int(sign+offset[-2:]) try: - deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%fZ") + deserialized = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f') except ValueError: - try: - deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S.%f") - except ValueError: - deserialized = datetime.datetime.strptime(time, "%Y%m%dT%H%M%S") - - if tzone: - minutes = int(sign+tzone[:-2])*60 + int(sign+tzone[-2:]) - delta = datetime.timedelta(minutes=minutes) - try: - deserialized = deserialized.replace(tzinfo=datetime.timezone(delta)) - except AttributeError: - deserialized = deserialized.replace(tzinfo=_FixedOffset(minutes)) + deserialized = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S') + + try: + deserialized = deserialized.replace(tzinfo=datetime.timezone(datetime.timedelta(minutes=delta))) + except AttributeError: + deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) return deserialized From 66266962043f4944295744eff5c136064d3f1977 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 24 Feb 2021 12:56:52 -0800 Subject: [PATCH 22/51] lint --- sdk/core/azure-core/azure/core/_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 6056cf5f1e15..fd67bee91af9 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -5,7 +5,6 @@ # license information. # -------------------------------------------------------------------------- import datetime -import re class _FixedOffset(datetime.tzinfo): From e5b19cbccc0378ba53f9f81e4443bf182bcb93b3 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 24 Feb 2021 15:58:35 -0800 Subject: [PATCH 23/51] apply black --- sdk/core/azure-core/azure/core/_utils.py | 23 ++++++-- sdk/core/azure-core/azure/core/messaging.py | 61 +++++++++------------ 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index fd67bee91af9..bbb1aef3f32d 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -22,7 +22,7 @@ def utcoffset(self, dt): return self.__offset def tzname(self, dt): - return str(self.__offset.total_seconds()/3600) + return str(self.__offset.total_seconds() / 3600) def __repr__(self): return "".format(self.tzname(None)) @@ -30,25 +30,36 @@ def __repr__(self): def dst(self, dt): return datetime.timedelta(0) + +try: + from datetime import timezone + + TZ_UTC = timezone.utc # type: ignore +except ImportError: + TZ_UTC = _FixedOffset(0) # type: ignore + + def _convert_to_isoformat(date_time): """Deserialize a date in RFC 3339 format to datetime object. Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. """ - if date_time[-1] == 'Z': + if date_time[-1] == "Z": delta = 0 timestamp = date_time[:-1] else: timestamp = date_time[:-6] sign, offset = date_time[-6], date_time[-5:] - delta = int(sign+offset[:1])*60 + int(sign+offset[-2:]) + delta = int(sign + offset[:1]) * 60 + int(sign + offset[-2:]) try: - deserialized = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f') + deserialized = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f") except ValueError: - deserialized = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S') + deserialized = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S") try: - deserialized = deserialized.replace(tzinfo=datetime.timezone(datetime.timedelta(minutes=delta))) + deserialized = deserialized.replace( + tzinfo=datetime.timezone(datetime.timedelta(minutes=delta)) + ) except AttributeError: deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 34346a1164d8..583e6f84f31b 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -5,23 +5,15 @@ # license information. # -------------------------------------------------------------------------- import uuid -from base64 import b64decode +from base64 import b64decode from datetime import datetime - -try: - from datetime import timezone - TZ_UTC = timezone.utc # type: ignore -except ImportError: - from azure.core._utils import _FixedOffset - TZ_UTC = _FixedOffset(0) # type: ignore +from azure.core._utils import _convert_to_isoformat, TZ_UTC try: from typing import TYPE_CHECKING except ImportError: TYPE_CHECKING = False -from azure.core._utils import _convert_to_isoformat - if TYPE_CHECKING: from typing import Any, Dict @@ -29,7 +21,7 @@ __all__ = ["CloudEvent"] -class CloudEvent(object): #pylint:disable=too-many-instance-attributes +class CloudEvent(object): # pylint:disable=too-many-instance-attributes """Properties of the CloudEvent 1.0 Schema. All required parameters must be populated in order to send to Azure. If data is of binary type, data_base64 can be used alternatively. Note that data and data_base64 @@ -86,7 +78,8 @@ class CloudEvent(object): #pylint:disable=too-many-instance-attributes and must not exceed the length of 20 characters. :vartype extensions: dict """ - def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin + + def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, **Any) -> None self.source = source self.type = type @@ -97,23 +90,19 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.dataschema = kwargs.pop("dataschema", None) self.subject = kwargs.pop("subject", None) self.extensions = {} - _extensions = dict(kwargs.pop('extensions', {})) + _extensions = dict(kwargs.pop("extensions", {})) for key in _extensions.keys(): if not key.islower() or not key.isalnum(): - raise ValueError("Extension attributes should be lower cased and alphanumeric.") + raise ValueError( + "Extension attributes should be lower cased and alphanumeric." + ) self.extensions.update(_extensions) self.data = kwargs.pop("data", None) def __repr__(self): - return ( - "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( - self.source, - self.type, - self.specversion, - self.id, - self.time - )[:1024] - ) + return "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( + self.source, self.type, self.specversion, self.id, self.time + )[:1024] @classmethod def from_dict(cls, event, **kwargs): @@ -125,22 +114,24 @@ def from_dict(cls, event, **kwargs): :rtype: CloudEvent """ reserved_attr = [ - 'data', - 'data_base64', - 'id', - 'source', - 'type', - 'specversion', - 'time', - 'dataschema', - 'datacontenttype', - 'subject' + "data", + "data_base64", + "id", + "source", + "type", + "specversion", + "time", + "dataschema", + "datacontenttype", + "subject", ] data = event.get("data", None) data_base64 = event.get("data_base64", None) if data and data_base64: - raise ValueError("Invalid input. Only one of data and data_base64 must be present.") + raise ValueError( + "Invalid input. Only one of data and data_base64 must be present." + ) return cls( id=event.get("id", None), @@ -152,6 +143,6 @@ def from_dict(cls, event, **kwargs): dataschema=event.get("dataschema", None), datacontenttype=event.get("datacontenttype", None), subject=event.get("subject", None), - extensions={k:v for k, v in event.items() if k not in reserved_attr}, + extensions={k: v for k, v in event.items() if k not in reserved_attr}, **kwargs ) From dc99b5cc7a447209ffe34b12a0517d251d5242cd Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Wed, 24 Feb 2021 16:16:33 -0800 Subject: [PATCH 24/51] utilize tz utc --- sdk/core/azure-core/azure/core/_utils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index bbb1aef3f32d..9486795960ef 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -51,16 +51,18 @@ def _convert_to_isoformat(date_time): sign, offset = date_time[-6], date_time[-5:] delta = int(sign + offset[:1]) * 60 + int(sign + offset[-2:]) + if delta == 0: + tzinfo = TZ_UTC + else: + try: + tzinfo = datetime.timezone(datetime.timedelta(minutes=delta)) + except AttributeError: + tzinfo = _FixedOffset(delta) + try: deserialized = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f") except ValueError: deserialized = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S") - try: - deserialized = deserialized.replace( - tzinfo=datetime.timezone(datetime.timedelta(minutes=delta)) - ) - except AttributeError: - deserialized = deserialized.replace(tzinfo=_FixedOffset(delta)) - + deserialized = deserialized.replace(tzinfo=tzinfo) return deserialized From b8c61a825939e71f3a7bc150919933b8de886f6f Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 10:32:58 -0800 Subject: [PATCH 25/51] comments --- sdk/core/azure-core/azure/core/messaging.py | 38 ++++++++++--------- .../tests/test_messaging_cloud_event.py | 26 +++++++++++++ 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 583e6f84f31b..02f45f038743 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -24,8 +24,6 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes """Properties of the CloudEvent 1.0 Schema. All required parameters must be populated in order to send to Azure. - If data is of binary type, data_base64 can be used alternatively. Note that data and data_base64 - cannot be present at the same time. :param source: Required. Identifies the context in which an event happened. The combination of id and source must be unique for each distinct event. If publishing to a domain topic, source must be the domain name. :type source: str @@ -76,20 +74,20 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes :ivar extensions: A CloudEvent MAY include any number of additional context attributes with distinct names represented as key - value pairs. Each extension must be alphanumeric, lower cased and must not exceed the length of 20 characters. - :vartype extensions: dict + :vartype extensions: Dict """ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, **Any) -> None - self.source = source - self.type = type - self.specversion = kwargs.pop("specversion", "1.0") - self.id = kwargs.pop("id", str(uuid.uuid4())) - self.time = kwargs.pop("time", datetime.now(TZ_UTC)) - self.datacontenttype = kwargs.pop("datacontenttype", None) - self.dataschema = kwargs.pop("dataschema", None) - self.subject = kwargs.pop("subject", None) - self.extensions = {} + self.source = source # type: str + self.type = type # type: str + self.specversion = kwargs.pop("specversion", "1.0") # type: str + self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str + self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime + self.datacontenttype = kwargs.pop("datacontenttype", None) # type: str + self.dataschema = kwargs.pop("dataschema", None) # type: str + self.subject = kwargs.pop("subject", None) # type: str + self.extensions = {} # type: Dict _extensions = dict(kwargs.pop("extensions", {})) for key in _extensions.keys(): if not key.islower() or not key.isalnum(): @@ -97,7 +95,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin "Extension attributes should be lower cased and alphanumeric." ) self.extensions.update(_extensions) - self.data = kwargs.pop("data", None) + self.data = kwargs.pop("data", None) # type: object def __repr__(self): return "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( @@ -105,8 +103,8 @@ def __repr__(self): )[:1024] @classmethod - def from_dict(cls, event, **kwargs): - # type: (Dict, **Any) -> CloudEvent + def from_dict(cls, event): + # type: (Dict) -> CloudEvent """ Returns the deserialized CloudEvent object when a dict is provided. :param event: The dict representation of the event which needs to be deserialized. @@ -128,21 +126,25 @@ def from_dict(cls, event, **kwargs): data = event.get("data", None) data_base64 = event.get("data_base64", None) + if data and data_base64: raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) + elif data is None and data_base64 is None: + data = None + else: + data = data if data is not None else b64decode(data_base64) return cls( id=event.get("id", None), source=event.get("source", None), type=event.get("type", None), specversion=event.get("specversion", None), - data=data if data is not None else b64decode(data_base64), + data=data, time=_convert_to_isoformat(event.get("time", None)), dataschema=event.get("dataschema", None), datacontenttype=event.get("datacontenttype", None), subject=event.get("subject", None), - extensions={k: v for k, v in event.items() if k not in reserved_attr}, - **kwargs + extensions={k: v for k, v in event.items() if k not in reserved_attr} ) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 8dea394ae099..7cdcdc8a6a53 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -108,6 +108,32 @@ def test_cloud_custom_dict_blank_data(): assert event.data == '' assert event.__class__ == CloudEvent +def test_cloud_custom_dict_none_data(): + cloud_custom_dict_with_none_data = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":None, + "type":"Azure.Sdk.Sample", + "time":"2021-02-18T20:18:10+00:00", + "specversion":"1.0", + } + event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) + assert event.data == None + assert event.__class__ == CloudEvent + +def test_cloud_custom_dict_both_data_and_base64(): + cloud_custom_dict_with_data_and_base64 = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "data":"abc", + "data_base64":"Y2Wa==", + "type":"Azure.Sdk.Sample", + "time":"2021-02-18T20:18:10+00:00", + "specversion":"1.0", + } + with pytest.raises(ValueError): + event = CloudEvent.from_dict(cloud_custom_dict_with_data_and_base64) + def test_cloud_custom_dict_base64(): cloud_custom_dict_base64 = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", From c9db015b41d56cbc65d09c577504117130d53132 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 11:02:26 -0800 Subject: [PATCH 26/51] raise on unexpected kwargs --- sdk/core/azure-core/azure/core/messaging.py | 27 +++++++++++-------- .../tests/test_messaging_cloud_event.py | 9 +++++++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 02f45f038743..2540ab9b1ddc 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -79,15 +79,15 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, **Any) -> None - self.source = source # type: str - self.type = type # type: str - self.specversion = kwargs.pop("specversion", "1.0") # type: str - self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str - self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime - self.datacontenttype = kwargs.pop("datacontenttype", None) # type: str - self.dataschema = kwargs.pop("dataschema", None) # type: str - self.subject = kwargs.pop("subject", None) # type: str - self.extensions = {} # type: Dict + self.source = source # type: str + self.type = type # type: str + self.specversion = kwargs.pop("specversion", "1.0") # type: str + self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str + self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime + self.datacontenttype = kwargs.pop("datacontenttype", None) # type: str + self.dataschema = kwargs.pop("dataschema", None) # type: str + self.subject = kwargs.pop("subject", None) # type: str + self.extensions = {} # type: Dict _extensions = dict(kwargs.pop("extensions", {})) for key in _extensions.keys(): if not key.islower() or not key.isalnum(): @@ -95,7 +95,12 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin "Extension attributes should be lower cased and alphanumeric." ) self.extensions.update(_extensions) - self.data = kwargs.pop("data", None) # type: object + self.data = kwargs.pop("data", None) # type: object + + if kwargs: + raise ValueError( + "Unexpected keyword argument. Any extension attribures must be passed explicitly using extensions." + ) def __repr__(self): return "CloudEvent(source={}, type={}, specversion={}, id={}, time={})".format( @@ -146,5 +151,5 @@ def from_dict(cls, event): dataschema=event.get("dataschema", None), datacontenttype=event.get("datacontenttype", None), subject=event.get("subject", None), - extensions={k: v for k, v in event.items() if k not in reserved_attr} + extensions={k: v for k, v in event.items() if k not in reserved_attr}, ) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 7cdcdc8a6a53..3f279c3e58ac 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -21,6 +21,15 @@ def test_cloud_event_constructor(): assert event.source == 'Azure.Core.Sample' assert event.data == 'cloudevent' +def test_cloud_event_constructor_unexpected_keyword(): + with pytest.raises(ValueError): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data='cloudevent', + unexpected_keyword="not allowed" + ) + def test_cloud_event_constructor_blank_data(): event = CloudEvent( source='Azure.Core.Sample', From 317ccbb4894cc52d3cf52265548f15d91c9aa149 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 11:03:37 -0800 Subject: [PATCH 27/51] doc --- sdk/core/azure-core/azure/core/messaging.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 2540ab9b1ddc..a0c5285bd2e0 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -29,8 +29,7 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes :type source: str :param type: Required. Type of event related to the originating occurrence. :type type: str - :keyword data: Optional. Event data specific to the event type. If data is of bytes type, it will be sent - as data_base64 in the outgoing request. + :keyword data: Optional. Event data specific to the event type. :type data: object :keyword time: Optional. The time (in UTC) the event was generated. :type time: ~datetime.datetime From f0d7d34acc0938b7147e816390c0967ee27f1634 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 11:39:09 -0800 Subject: [PATCH 28/51] lint --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index a0c5285bd2e0..660be85c119d 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -137,7 +137,7 @@ def from_dict(cls, event): ) elif data is None and data_base64 is None: data = None - else: + elif data or data_base64: data = data if data is not None else b64decode(data_base64) return cls( From f21193b8e828ab609ac3fea6346989d57622d9dc Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Thu, 25 Feb 2021 12:18:10 -0800 Subject: [PATCH 29/51] more lint --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 660be85c119d..40d0dd863197 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -135,7 +135,7 @@ def from_dict(cls, event): raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - elif data is None and data_base64 is None: + if data is None and data_base64 is None: data = None elif data or data_base64: data = data if data is not None else b64decode(data_base64) From 6d32a120c52060ce462c0dbeaac571955c1dfbf8 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 26 Feb 2021 12:56:16 -0800 Subject: [PATCH 30/51] attrs are optional --- sdk/core/azure-core/azure/core/_utils.py | 2 + sdk/core/azure-core/azure/core/messaging.py | 58 +++++++++++-------- .../tests/test_messaging_cloud_event.py | 21 ++++++- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index 9486795960ef..d48581263be8 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -43,6 +43,8 @@ def _convert_to_isoformat(date_time): """Deserialize a date in RFC 3339 format to datetime object. Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. """ + if not date_time: + return if date_time[-1] == "Z": delta = 0 timestamp = date_time[:-1] diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 40d0dd863197..b4881f606a51 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -83,18 +83,22 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin self.specversion = kwargs.pop("specversion", "1.0") # type: str self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime - self.datacontenttype = kwargs.pop("datacontenttype", None) # type: str - self.dataschema = kwargs.pop("dataschema", None) # type: str - self.subject = kwargs.pop("subject", None) # type: str - self.extensions = {} # type: Dict - _extensions = dict(kwargs.pop("extensions", {})) - for key in _extensions.keys(): - if not key.islower() or not key.isalnum(): - raise ValueError( - "Extension attributes should be lower cased and alphanumeric." - ) - self.extensions.update(_extensions) - self.data = kwargs.pop("data", None) # type: object + + _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] + + for attr in _optional_attributes: + if attr in kwargs: + setattr(self, attr, kwargs.pop(attr)) + + if "extensions" in kwargs: + self.extensions = {} # type: Dict + _extensions = dict(kwargs.pop("extensions", {})) + for key in _extensions.keys(): + if not key.islower() or not key.isalnum(): + raise ValueError( + "Extension attributes should be lower cased and alphanumeric." + ) + self.extensions.update(_extensions) if kwargs: raise ValueError( @@ -115,6 +119,7 @@ def from_dict(cls, event): :type event: dict :rtype: CloudEvent """ + kwargs = {} reserved_attr = [ "data", "data_base64", @@ -128,27 +133,30 @@ def from_dict(cls, event): "subject", ] - data = event.get("data", None) - data_base64 = event.get("data_base64", None) - - if data and data_base64: + if "data" in event and "data_base64" in event: raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - if data is None and data_base64 is None: - data = None - elif data or data_base64: - data = data if data is not None else b64decode(data_base64) + + if "data" in event: + kwargs.setdefault("data", event.get("data")) + elif "data_base64" in event: + kwargs.setdefault("data", b64decode(event.get("data_base64"))) + + for item in ["datacontenttype", "dataschema", "subject"]: + if item in event: + kwargs.setdefault(item, event.get(item)) + + extensions={k: v for k, v in event.items() if k not in reserved_attr} + if extensions: + kwargs.setdefault("extensions", extensions) + return cls( id=event.get("id", None), source=event.get("source", None), type=event.get("type", None), specversion=event.get("specversion", None), - data=data, time=_convert_to_isoformat(event.get("time", None)), - dataschema=event.get("dataschema", None), - datacontenttype=event.get("datacontenttype", None), - subject=event.get("subject", None), - extensions={k: v for k, v in event.items() if k not in reserved_attr}, + **kwargs ) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 3f279c3e58ac..fbfe7f255cb3 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -43,6 +43,21 @@ def test_cloud_event_constructor_blank_data(): assert event.source == 'Azure.Core.Sample' assert event.data == '' +def test_cloud_event_constructor_no_data(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + ) + + with pytest.raises(AttributeError): + doesnt_exist = event.data + with pytest.raises(AttributeError): + doesnt_exist = event.datacontenttype + with pytest.raises(AttributeError): + doesnt_exist = event.dataschema + with pytest.raises(AttributeError): + doesnt_exist = event.subject + def test_cloud_storage_dict(): cloud_storage_dict = { "id":"a0517898-9fa4-4e70-b4a3-afda1dd68672", @@ -117,18 +132,18 @@ def test_cloud_custom_dict_blank_data(): assert event.data == '' assert event.__class__ == CloudEvent -def test_cloud_custom_dict_none_data(): +def test_cloud_custom_dict_no_data(): cloud_custom_dict_with_none_data = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", "source":"https://egtest.dev/cloudcustomevent", - "data":None, "type":"Azure.Sdk.Sample", "time":"2021-02-18T20:18:10+00:00", "specversion":"1.0", } event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) - assert event.data == None assert event.__class__ == CloudEvent + with pytest.raises(AttributeError): + missing = event.data def test_cloud_custom_dict_both_data_and_base64(): cloud_custom_dict_with_data_and_base64 = { From 12237211c68ff8369fb4a6bb134201c4eb7131ae Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Fri, 26 Feb 2021 15:56:07 -0800 Subject: [PATCH 31/51] add sentinel --- sdk/core/azure-core/CHANGELOG.md | 1 + sdk/core/azure-core/azure/core/_utils.py | 2 +- sdk/core/azure-core/azure/core/messaging.py | 19 ++++--- .../azure-core/azure/core/serialization.py | 10 ++++ .../tests/test_messaging_cloud_event.py | 54 +++++++++++++++---- 5 files changed, 66 insertions(+), 20 deletions(-) create mode 100644 sdk/core/azure-core/azure/core/serialization.py diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 3641c7733f78..7a2d0c81144d 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features - Added `azure.core.messaging.CloudEvent` model that follows the cloud event spec. +- Added `azure.core.serialization.NULL` sentinel value ## 1.11.0 (2021-02-08) diff --git a/sdk/core/azure-core/azure/core/_utils.py b/sdk/core/azure-core/azure/core/_utils.py index d48581263be8..9178d4e5c7f1 100644 --- a/sdk/core/azure-core/azure/core/_utils.py +++ b/sdk/core/azure-core/azure/core/_utils.py @@ -44,7 +44,7 @@ def _convert_to_isoformat(date_time): Check https://tools.ietf.org/html/rfc3339#section-5.8 for examples. """ if not date_time: - return + return None if date_time[-1] == "Z": delta = 0 timestamp = date_time[:-1] diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index b4881f606a51..7ecb16263b86 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -8,6 +8,7 @@ from base64 import b64decode from datetime import datetime from azure.core._utils import _convert_to_isoformat, TZ_UTC +from azure.core.serialization import NULL try: from typing import TYPE_CHECKING @@ -87,8 +88,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] for attr in _optional_attributes: - if attr in kwargs: - setattr(self, attr, kwargs.pop(attr)) + setattr(self, attr, kwargs.pop(attr, None)) if "extensions" in kwargs: self.extensions = {} # type: Dict @@ -137,17 +137,20 @@ def from_dict(cls, event): raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - - if "data" in event: - kwargs.setdefault("data", event.get("data")) + if 'data' not in event and 'data_base64' not in event: + kwargs.setdefault("data", None) + elif "data" in event: + data = event.get("data") + kwargs.setdefault("data", data) if data is not None else kwargs.setdefault("data", NULL) elif "data_base64" in event: kwargs.setdefault("data", b64decode(event.get("data_base64"))) for item in ["datacontenttype", "dataschema", "subject"]: if item in event: - kwargs.setdefault(item, event.get(item)) - - extensions={k: v for k, v in event.items() if k not in reserved_attr} + val = event.get(item) + kwargs.setdefault(item, val) if val is not None else kwargs.setdefault(item, NULL) + + extensions = {k: v for k, v in event.items() if k not in reserved_attr} if extensions: kwargs.setdefault("extensions", extensions) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py new file mode 100644 index 000000000000..272f0239490a --- /dev/null +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -0,0 +1,10 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +__all__ = ["NULL"] + +NULL = False diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index fbfe7f255cb3..b15dc2689b65 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -6,6 +6,7 @@ import datetime from azure.core.messaging import CloudEvent +from azure.core.serialization import NULL # Cloud Event tests def test_cloud_event_constructor(): @@ -43,20 +44,37 @@ def test_cloud_event_constructor_blank_data(): assert event.source == 'Azure.Core.Sample' assert event.data == '' -def test_cloud_event_constructor_no_data(): +def test_cloud_event_constructor_NULL_data(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data=NULL + ) + + assert event.data == NULL + +def test_cloud_event_constructor_none_data(): + event = CloudEvent( + source='Azure.Core.Sample', + type='SampleType', + data=None + ) + + assert event.data == None + assert event.datacontenttype == None + assert event.dataschema == None + assert event.subject == None + +def test_cloud_event_constructor_missing_data(): event = CloudEvent( source='Azure.Core.Sample', type='SampleType', ) - with pytest.raises(AttributeError): - doesnt_exist = event.data - with pytest.raises(AttributeError): - doesnt_exist = event.datacontenttype - with pytest.raises(AttributeError): - doesnt_exist = event.dataschema - with pytest.raises(AttributeError): - doesnt_exist = event.subject + assert event.data == None + assert event.datacontenttype == None + assert event.dataschema == None + assert event.subject == None def test_cloud_storage_dict(): cloud_storage_dict = { @@ -133,17 +151,31 @@ def test_cloud_custom_dict_blank_data(): assert event.__class__ == CloudEvent def test_cloud_custom_dict_no_data(): + cloud_custom_dict_with_missing_data = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "type":"Azure.Sdk.Sample", + "time":"2021-02-18T20:18:10+00:00", + "specversion":"1.0", + } + event = CloudEvent.from_dict(cloud_custom_dict_with_missing_data) + assert event.__class__ == CloudEvent + assert event.data == None + +def test_cloud_custom_dict_null_data(): cloud_custom_dict_with_none_data = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", "source":"https://egtest.dev/cloudcustomevent", "type":"Azure.Sdk.Sample", + "data":None, + "dataschema":None, "time":"2021-02-18T20:18:10+00:00", "specversion":"1.0", } event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) assert event.__class__ == CloudEvent - with pytest.raises(AttributeError): - missing = event.data + assert event.data == NULL + assert event.dataschema == NULL def test_cloud_custom_dict_both_data_and_base64(): cloud_custom_dict_with_data_and_base64 = { From b9b1bb90eb391622e5e7d1fca567ad88e9ec082c Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 28 Feb 2021 12:43:00 -0800 Subject: [PATCH 32/51] falsy object --- sdk/core/azure-core/azure/core/messaging.py | 21 ++++++++++++------- .../azure-core/azure/core/serialization.py | 8 ++++++- .../tests/test_messaging_cloud_event.py | 9 ++++---- .../azure-core/tests/test_serialization.py | 11 ++++++++++ 4 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 sdk/core/azure-core/tests/test_serialization.py diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 7ecb16263b86..7a7be166e3a1 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -11,12 +11,12 @@ from azure.core.serialization import NULL try: - from typing import TYPE_CHECKING + from typing import TYPE_CHECKING, cast except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict + from typing import Any, Dict, Union __all__ = ["CloudEvent"] @@ -88,7 +88,11 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] for attr in _optional_attributes: - setattr(self, attr, kwargs.pop(attr, None)) + if attr not in kwargs: + val = None + else: + val = kwargs.pop(attr, NULL) + setattr(self, attr, val) if "extensions" in kwargs: self.extensions = {} # type: Dict @@ -119,7 +123,7 @@ def from_dict(cls, event): :type event: dict :rtype: CloudEvent """ - kwargs = {} + kwargs = {} # type: Dict[Any, Any] reserved_attr = [ "data", "data_base64", @@ -138,12 +142,15 @@ def from_dict(cls, event): "Invalid input. Only one of data and data_base64 must be present." ) if 'data' not in event and 'data_base64' not in event: - kwargs.setdefault("data", None) + kwargs.setdefault("data", None) elif "data" in event: data = event.get("data") - kwargs.setdefault("data", data) if data is not None else kwargs.setdefault("data", NULL) + if data is not None: + kwargs.setdefault("data", data) + else: + kwargs.setdefault("data", NULL) elif "data_base64" in event: - kwargs.setdefault("data", b64decode(event.get("data_base64"))) + kwargs.setdefault("data", b64decode(cast(Union[str, bytes], event.get("data_base64")))) for item in ["datacontenttype", "dataschema", "subject"]: if item in event: diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 272f0239490a..a25c4e4002dc 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -7,4 +7,10 @@ __all__ = ["NULL"] -NULL = False +class _Null(object): + """To create a Falsy object + """ + def __bool__(self): + return False + +NULL = _Null() diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index b15dc2689b65..7c6fa70e41ac 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -1,3 +1,7 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ import logging import sys import os @@ -59,11 +63,8 @@ def test_cloud_event_constructor_none_data(): type='SampleType', data=None ) - + assert event.data == None - assert event.datacontenttype == None - assert event.dataschema == None - assert event.subject == None def test_cloud_event_constructor_missing_data(): event = CloudEvent( diff --git a/sdk/core/azure-core/tests/test_serialization.py b/sdk/core/azure-core/tests/test_serialization.py new file mode 100644 index 000000000000..8646ee1df2d9 --- /dev/null +++ b/sdk/core/azure-core/tests/test_serialization.py @@ -0,0 +1,11 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +from azure.core.serialization import NULL + +def test_NULL_is_falsy(): + assert NULL != False + assert (not NULL) + assert NULL is NULL \ No newline at end of file From d9b9bbb8ad4d554386895524ac1588e051aa7a58 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 28 Feb 2021 12:51:48 -0800 Subject: [PATCH 33/51] few more asserts --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 7a7be166e3a1..e620a81aa58f 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -11,12 +11,12 @@ from azure.core.serialization import NULL try: - from typing import TYPE_CHECKING, cast + from typing import TYPE_CHECKING, cast, Union except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict, Union + from typing import Any, Dict __all__ = ["CloudEvent"] diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 7c6fa70e41ac..d9f76320048d 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -117,6 +117,8 @@ def test_cloud_storage_dict(): assert event.time.day == 18 assert event.time.hour == 20 assert event.__class__ == CloudEvent + assert "id" in cloud_storage_dict + assert "data" in cloud_storage_dict def test_cloud_custom_dict_with_extensions(): From 45f1b8b216b9e226cbb0d00eb14b1b1fe3c2b52a Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 28 Feb 2021 13:16:32 -0800 Subject: [PATCH 34/51] lint --- sdk/core/azure-core/azure/core/messaging.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index e620a81aa58f..27437dd6dd0e 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -155,7 +155,10 @@ def from_dict(cls, event): for item in ["datacontenttype", "dataschema", "subject"]: if item in event: val = event.get(item) - kwargs.setdefault(item, val) if val is not None else kwargs.setdefault(item, NULL) + if val is not None: + kwargs.setdefault(item, val) + else: + kwargs.setdefault(item, NULL) extensions = {k: v for k, v in event.items() if k not in reserved_attr} if extensions: From 0b3608c99928c258880afb5bb8f4f26ad83aadca Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Sun, 28 Feb 2021 13:54:45 -0800 Subject: [PATCH 35/51] pyt2 compat --- sdk/core/azure-core/azure/core/serialization.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index a25c4e4002dc..964fb8a64c12 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -13,4 +13,7 @@ class _Null(object): def __bool__(self): return False + __nonzero__ = __bool__ # Python2 compatibility + + NULL = _Null() From 9acc262c4f99a47c609bda109e213784dbb210c3 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 10:18:30 -0800 Subject: [PATCH 36/51] tests --- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 1 + sdk/core/azure-core/tests/test_serialization.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index d9f76320048d..7a0d122aa639 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -56,6 +56,7 @@ def test_cloud_event_constructor_NULL_data(): ) assert event.data == NULL + assert event.data is NULL def test_cloud_event_constructor_none_data(): event = CloudEvent( diff --git a/sdk/core/azure-core/tests/test_serialization.py b/sdk/core/azure-core/tests/test_serialization.py index 8646ee1df2d9..7ac58850cd91 100644 --- a/sdk/core/azure-core/tests/test_serialization.py +++ b/sdk/core/azure-core/tests/test_serialization.py @@ -6,6 +6,6 @@ from azure.core.serialization import NULL def test_NULL_is_falsy(): - assert NULL != False - assert (not NULL) + assert NULL is not False + assert bool(NULL) is False assert NULL is NULL \ No newline at end of file From c5a7ee018435fa1f466aac2b2177776577feb082 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:11:57 -0800 Subject: [PATCH 37/51] comments --- sdk/core/azure-core/azure/core/messaging.py | 19 +++++++++---------- .../azure-core/azure/core/serialization.py | 4 ++++ .../tests/test_messaging_cloud_event.py | 6 ++++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 27437dd6dd0e..c879e979c664 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -88,25 +88,24 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] for attr in _optional_attributes: - if attr not in kwargs: - val = None - else: - val = kwargs.pop(attr, NULL) + val = None if attr not in kwargs else kwargs.pop(attr) setattr(self, attr, val) - if "extensions" in kwargs: - self.extensions = {} # type: Dict - _extensions = dict(kwargs.pop("extensions", {})) - for key in _extensions.keys(): + try: + self.extensions = kwargs.pop("extensions") # type: Dict + for key in self.extensions.keys(): if not key.islower() or not key.isalnum(): raise ValueError( "Extension attributes should be lower cased and alphanumeric." ) - self.extensions.update(_extensions) + except KeyError: + pass if kwargs: + remaining = ", ".join(kwargs.keys()) raise ValueError( - "Unexpected keyword argument. Any extension attribures must be passed explicitly using extensions." + "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions." + .format(remaining) ) def __repr__(self): diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 964fb8a64c12..98418a49c0c8 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -16,4 +16,8 @@ def __bool__(self): __nonzero__ = __bool__ # Python2 compatibility +""" +NULL is a falsy sentinel object which is supposed to be used to specify attributes +with no data. This gets serialized to `null` on the wire. +""" NULL = _Null() diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 7a0d122aa639..7023f0400664 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -27,12 +27,14 @@ def test_cloud_event_constructor(): assert event.data == 'cloudevent' def test_cloud_event_constructor_unexpected_keyword(): - with pytest.raises(ValueError): + remaining = "unexpected_keyword, another_bad_kwarg" + with pytest.raises(ValueError, match="Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions.".format(remaining)): event = CloudEvent( source='Azure.Core.Sample', type='SampleType', data='cloudevent', - unexpected_keyword="not allowed" + unexpected_keyword="not allowed", + another_bad_kwarg="not allowed either" ) def test_cloud_event_constructor_blank_data(): From a92dcc973a58f154cd3048a64a481cbf5956680a Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:31:49 -0800 Subject: [PATCH 38/51] update toc tree --- sdk/core/azure-core/doc/azure.core.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sdk/core/azure-core/doc/azure.core.rst b/sdk/core/azure-core/doc/azure.core.rst index 26b0607f3ca4..36716ad49c62 100644 --- a/sdk/core/azure-core/doc/azure.core.rst +++ b/sdk/core/azure-core/doc/azure.core.rst @@ -41,6 +41,14 @@ azure.core.exceptions :members: :undoc-members: +azure.core.messaging +------------------- + +.. automodule:: azure.core.messaging + :members: + :undoc-members: + :inherited-members: + azure.core.paging ----------------- @@ -57,3 +65,10 @@ azure.core.settings :undoc-members: :inherited-members: +azure.core.serialization +------------------- + +.. automodule:: azure.core.serialization + :members: + :undoc-members: + :inherited-members: From 6510b0bd722529e72926502910789efa7c483515 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:49:14 -0800 Subject: [PATCH 39/51] doc --- sdk/core/azure-core/azure/core/serialization.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 98418a49c0c8..2877829ab767 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -15,9 +15,8 @@ def __bool__(self): __nonzero__ = __bool__ # Python2 compatibility - +NULL = _Null() """ -NULL is a falsy sentinel object which is supposed to be used to specify attributes +A falsy sentinel object which is supposed to be used to specify attributes with no data. This gets serialized to `null` on the wire. -""" -NULL = _Null() +""" \ No newline at end of file From 33a008762bf70af287b2d4621ce0263e3d85782f Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:49:26 -0800 Subject: [PATCH 40/51] doc --- sdk/core/azure-core/azure/core/serialization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 2877829ab767..c3422efa0c27 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -15,8 +15,9 @@ def __bool__(self): __nonzero__ = __bool__ # Python2 compatibility + NULL = _Null() """ A falsy sentinel object which is supposed to be used to specify attributes with no data. This gets serialized to `null` on the wire. -""" \ No newline at end of file +""" From e372c6befc820cf72dce6af6ab6e59561ae26125 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 13:51:01 -0800 Subject: [PATCH 41/51] doc --- sdk/core/azure-core/azure/core/messaging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index c879e979c664..ceb16781f899 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -25,6 +25,7 @@ class CloudEvent(object): # pylint:disable=too-many-instance-attributes """Properties of the CloudEvent 1.0 Schema. All required parameters must be populated in order to send to Azure. + :param source: Required. Identifies the context in which an event happened. The combination of id and source must be unique for each distinct event. If publishing to a domain topic, source must be the domain name. :type source: str From f91b18c1a91fc84134eaca3f5ae059a6219c7a24 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 14:05:16 -0800 Subject: [PATCH 42/51] unconditional --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index ceb16781f899..a8b3ace57add 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -100,7 +100,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin "Extension attributes should be lower cased and alphanumeric." ) except KeyError: - pass + self.extensions = None if kwargs: remaining = ", ".join(kwargs.keys()) From 07ca924680b09ceb38df58f5f170dd864772ba57 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 14:26:53 -0800 Subject: [PATCH 43/51] test fix --- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 7023f0400664..96dd20758cd3 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -27,8 +27,7 @@ def test_cloud_event_constructor(): assert event.data == 'cloudevent' def test_cloud_event_constructor_unexpected_keyword(): - remaining = "unexpected_keyword, another_bad_kwarg" - with pytest.raises(ValueError, match="Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions.".format(remaining)): + with pytest.raises(ValueError) as e: event = CloudEvent( source='Azure.Core.Sample', type='SampleType', @@ -36,6 +35,8 @@ def test_cloud_event_constructor_unexpected_keyword(): unexpected_keyword="not allowed", another_bad_kwarg="not allowed either" ) + assert "unexpected_keyword" in e + assert "another_bad_kwarg" in e def test_cloud_event_constructor_blank_data(): event = CloudEvent( From bd72df477e1406bb1516a78e6bcc804459f7bb73 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 14:55:52 -0800 Subject: [PATCH 44/51] mypy --- sdk/core/azure-core/azure/core/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index a8b3ace57add..53ba378db030 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -100,7 +100,7 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin "Extension attributes should be lower cased and alphanumeric." ) except KeyError: - self.extensions = None + self.extensions = cast(Dict, None) if kwargs: remaining = ", ".join(kwargs.keys()) From 2f5f7a51bac33b73bee169787135331b3ac061fc Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 1 Mar 2021 15:21:46 -0800 Subject: [PATCH 45/51] wrong import --- sdk/core/azure-core/azure/core/messaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 53ba378db030..fa0a1b00aeb2 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -11,12 +11,12 @@ from azure.core.serialization import NULL try: - from typing import TYPE_CHECKING, cast, Union + from typing import TYPE_CHECKING, cast, Union, Dict except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any, Dict + from typing import Any __all__ = ["CloudEvent"] From 5bb4ac5547d1017aefecc25858637717ac0309eb Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 12:47:39 -0800 Subject: [PATCH 46/51] type annotations --- sdk/core/azure-core/azure/core/messaging.py | 47 ++++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index fa0a1b00aeb2..e26ed4a96d5f 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -11,12 +11,12 @@ from azure.core.serialization import NULL try: - from typing import TYPE_CHECKING, cast, Union, Dict + from typing import TYPE_CHECKING, cast, Union except ImportError: TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Any + from typing import Any, Optional, Dict __all__ = ["CloudEvent"] @@ -82,31 +82,35 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin # type: (str, str, **Any) -> None self.source = source # type: str self.type = type # type: str - self.specversion = kwargs.pop("specversion", "1.0") # type: str - self.id = kwargs.pop("id", str(uuid.uuid4())) # type: str - self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: datetime + self.specversion = kwargs.pop("specversion", "1.0") # type: Optional[str] + self.id = kwargs.pop("id", str(uuid.uuid4())) # type: Optional[str] + self.time = kwargs.pop("time", datetime.now(TZ_UTC)) # type: Optional[datetime] - _optional_attributes = ["datacontenttype", "dataschema", "subject", "data"] - - for attr in _optional_attributes: - val = None if attr not in kwargs else kwargs.pop(attr) - setattr(self, attr, val) + self.datacontenttype = kwargs.pop( + "datacontenttype", None + ) # type: Optional[str] + self.dataschema = kwargs.pop("dataschema", None) # type: Optional[str] + self.subject = kwargs.pop("subject", None) # type: Optional[str] + self.data = kwargs.pop("data", None) # type: Optional[object] try: - self.extensions = kwargs.pop("extensions") # type: Dict - for key in self.extensions.keys(): + self.extensions = kwargs.pop("extensions") # type: Optional[Dict] + for ( + key + ) in self.extensions.keys(): # type:ignore # extensions won't be None here if not key.islower() or not key.isalnum(): raise ValueError( "Extension attributes should be lower cased and alphanumeric." ) except KeyError: - self.extensions = cast(Dict, None) + self.extensions = None if kwargs: remaining = ", ".join(kwargs.keys()) raise ValueError( - "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions." - .format(remaining) + "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions.".format( + remaining + ) ) def __repr__(self): @@ -123,7 +127,7 @@ def from_dict(cls, event): :type event: dict :rtype: CloudEvent """ - kwargs = {} # type: Dict[Any, Any] + kwargs = {} # type: Dict[Any, Any] reserved_attr = [ "data", "data_base64", @@ -141,16 +145,18 @@ def from_dict(cls, event): raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - if 'data' not in event and 'data_base64' not in event: + if "data" not in event and "data_base64" not in event: kwargs.setdefault("data", None) elif "data" in event: data = event.get("data") if data is not None: - kwargs.setdefault("data", data) + kwargs["data"] = data else: - kwargs.setdefault("data", NULL) + kwargs["data"] = NULL elif "data_base64" in event: - kwargs.setdefault("data", b64decode(cast(Union[str, bytes], event.get("data_base64")))) + kwargs["data"] = b64decode( + cast(Union[str, bytes], event.get("data_base64")) + ) for item in ["datacontenttype", "dataschema", "subject"]: if item in event: @@ -164,7 +170,6 @@ def from_dict(cls, event): if extensions: kwargs.setdefault("extensions", extensions) - return cls( id=event.get("id", None), source=event.get("source", None), From ed2fe645732ae441e778384c271aa4633172782e Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 12:55:50 -0800 Subject: [PATCH 47/51] data --- sdk/core/azure-core/azure/core/messaging.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index e26ed4a96d5f..3a44d392b939 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -145,14 +145,10 @@ def from_dict(cls, event): raise ValueError( "Invalid input. Only one of data and data_base64 must be present." ) - if "data" not in event and "data_base64" not in event: - kwargs.setdefault("data", None) - elif "data" in event: + + if "data" in event: data = event.get("data") - if data is not None: - kwargs["data"] = data - else: - kwargs["data"] = NULL + kwargs["data"] = data if data is not None else NULL elif "data_base64" in event: kwargs["data"] = b64decode( cast(Union[str, bytes], event.get("data_base64")) From 35c8467f03cc2bb6a7990e5037c7730b941558bf Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 13:02:29 -0800 Subject: [PATCH 48/51] coment --- sdk/core/azure-core/azure/core/messaging.py | 7 ++----- .../tests/test_messaging_cloud_event.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index 3a44d392b939..f38fe163117b 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -157,14 +157,11 @@ def from_dict(cls, event): for item in ["datacontenttype", "dataschema", "subject"]: if item in event: val = event.get(item) - if val is not None: - kwargs.setdefault(item, val) - else: - kwargs.setdefault(item, NULL) + kwargs[item] = val if val is not None else NULL extensions = {k: v for k, v in event.items() if k not in reserved_attr} if extensions: - kwargs.setdefault("extensions", extensions) + kwargs["extensions"] = extensions return cls( id=event.get("id", None), diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 96dd20758cd3..1c5a5a104f8a 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -181,9 +181,24 @@ def test_cloud_custom_dict_null_data(): } event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) assert event.__class__ == CloudEvent - assert event.data == NULL + assert event.data is NULL assert event.dataschema == NULL +def test_cloud_custom_dict_valid_optional_attrs(): + cloud_custom_dict_with_none_data = { + "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", + "source":"https://egtest.dev/cloudcustomevent", + "type":"Azure.Sdk.Sample", + "data":None, + "dataschema":"exists", + "time":"2021-02-18T20:18:10+00:00", + "specversion":"1.0", + } + event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) + assert event.__class__ == CloudEvent + assert event.data is NULL + assert event.dataschema == "exists" + def test_cloud_custom_dict_both_data_and_base64(): cloud_custom_dict_with_data_and_base64 = { "id":"de0fd76c-4ef4-4dfb-ab3a-8f24a307e033", From d7bde2f9bc2059902a1aff39613e99c66f37a3a5 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 13:04:11 -0800 Subject: [PATCH 49/51] assets --- sdk/core/azure-core/tests/test_messaging_cloud_event.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/tests/test_messaging_cloud_event.py b/sdk/core/azure-core/tests/test_messaging_cloud_event.py index 1c5a5a104f8a..c0a88b845488 100644 --- a/sdk/core/azure-core/tests/test_messaging_cloud_event.py +++ b/sdk/core/azure-core/tests/test_messaging_cloud_event.py @@ -167,7 +167,7 @@ def test_cloud_custom_dict_no_data(): } event = CloudEvent.from_dict(cloud_custom_dict_with_missing_data) assert event.__class__ == CloudEvent - assert event.data == None + assert event.data is None def test_cloud_custom_dict_null_data(): cloud_custom_dict_with_none_data = { @@ -181,8 +181,8 @@ def test_cloud_custom_dict_null_data(): } event = CloudEvent.from_dict(cloud_custom_dict_with_none_data) assert event.__class__ == CloudEvent - assert event.data is NULL - assert event.dataschema == NULL + assert event.data == NULL + assert event.dataschema is NULL def test_cloud_custom_dict_valid_optional_attrs(): cloud_custom_dict_with_none_data = { From a3e7a0f42f1109bfabbf89b93c3c1fe8eab52b92 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 15:29:48 -0800 Subject: [PATCH 50/51] lint --- sdk/core/azure-core/azure/core/messaging.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/messaging.py b/sdk/core/azure-core/azure/core/messaging.py index f38fe163117b..3a4e2295903a 100644 --- a/sdk/core/azure-core/azure/core/messaging.py +++ b/sdk/core/azure-core/azure/core/messaging.py @@ -108,9 +108,8 @@ def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin if kwargs: remaining = ", ".join(kwargs.keys()) raise ValueError( - "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions.".format( - remaining - ) + "Unexpected keyword arguments {}. Any extension attributes must be passed explicitly using extensions." + .format(remaining) ) def __repr__(self): From f687fdbdeb8f5912ee690d27d3b5152f33dd5ee2 Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Tue, 2 Mar 2021 16:58:21 -0800 Subject: [PATCH 51/51] Adapt to cloud event's azure core --- sdk/eventgrid/azure-eventgrid/CHANGELOG.md | 2 + sdk/eventgrid/azure-eventgrid/README.md | 8 +- .../azure/eventgrid/__init__.py | 3 +- .../azure/eventgrid/_helpers.py | 26 ++++ .../azure/eventgrid/_models.py | 146 +----------------- .../azure/eventgrid/_publisher_client.py | 6 +- .../eventgrid/aio/_publisher_client_async.py | 6 +- ...le_publish_cloud_event_using_dict_async.py | 2 +- ...nts_using_cloud_events_1.0_schema_async.py | 2 +- ...consume_cloud_events_from_storage_queue.py | 2 +- ...ish_cloud_events_to_custom_topic_sample.py | 3 +- ...ish_cloud_events_to_domain_topic_sample.py | 3 +- ...ish_with_shared_access_signature_sample.py | 3 +- .../sample_consume_custom_payload.py | 2 +- ...sh_events_using_cloud_events_1.0_schema.py | 3 +- sdk/eventgrid/azure-eventgrid/setup.py | 2 +- .../tests/test_cloud_event_tracing.py | 2 +- .../tests/test_eg_publisher_client.py | 27 ++-- .../tests/test_eg_publisher_client_async.py | 24 ++- .../tests/test_serialization.py | 13 +- 20 files changed, 95 insertions(+), 190 deletions(-) diff --git a/sdk/eventgrid/azure-eventgrid/CHANGELOG.md b/sdk/eventgrid/azure-eventgrid/CHANGELOG.md index 7826d7f3b0a6..3db74e36cd88 100644 --- a/sdk/eventgrid/azure-eventgrid/CHANGELOG.md +++ b/sdk/eventgrid/azure-eventgrid/CHANGELOG.md @@ -2,6 +2,8 @@ ## 2.0.0b6 (Unreleased) + **Breaking Changes** + - `~azure.eventgrid.CloudEvent` is now removed in favor of `~azure.core.messaging.CloudEvent`. ## 2.0.0b5 (2021-02-10) diff --git a/sdk/eventgrid/azure-eventgrid/README.md b/sdk/eventgrid/azure-eventgrid/README.md index 9629b210e18b..813cf6208a5e 100644 --- a/sdk/eventgrid/azure-eventgrid/README.md +++ b/sdk/eventgrid/azure-eventgrid/README.md @@ -145,7 +145,8 @@ This example publishes a Cloud event. ```Python import os from azure.core.credentials import AzureKeyCredential -from azure.eventgrid import EventGridPublisherClient, CloudEvent +from azure.core.messaging import CloudEvent +from azure.eventgrid import EventGridPublisherClient key = os.environ["CLOUD_ACCESS_KEY"] endpoint = os.environ["CLOUD_TOPIC_HOSTNAME"] @@ -166,7 +167,7 @@ client.send(event) This example consumes a message received from storage queue and deserializes it to a CloudEvent object. ```Python -from azure.eventgrid import CloudEvent +from azure.core.messaging import CloudEvent from azure.storage.queue import QueueServiceClient, BinaryBase64DecodePolicy import os import json @@ -244,7 +245,8 @@ Once the `tracer` and `exporter` are set, please follow the example below to sta ```python import os -from azure.eventgrid import EventGridPublisherClient, CloudEvent +from azure.eventgrid import EventGridPublisherClient +from azure.core.messaging import CloudEvent from azure.core.credentials import AzureKeyCredential hostname = os.environ['CLOUD_TOPIC_HOSTNAME'] diff --git a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/__init__.py b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/__init__.py index bb031e5ec320..1dc3655a13bb 100644 --- a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/__init__.py +++ b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/__init__.py @@ -7,12 +7,11 @@ from ._publisher_client import EventGridPublisherClient from ._event_mappings import SystemEventNames from ._helpers import generate_sas -from ._models import CloudEvent, EventGridEvent +from ._models import EventGridEvent from ._version import VERSION __all__ = [ "EventGridPublisherClient", - "CloudEvent", "EventGridEvent", "generate_sas", "SystemEventNames", diff --git a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_helpers.py b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_helpers.py index 2780ceb569bb..a1a82d5bfac9 100644 --- a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_helpers.py +++ b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_helpers.py @@ -18,6 +18,10 @@ from ._signature_credential_policy import EventGridSasCredentialPolicy from . import _constants as constants +from ._generated.models import ( + CloudEvent as InternalCloudEvent, +) + if TYPE_CHECKING: from datetime import datetime @@ -134,3 +138,25 @@ def _eventgrid_data_typecheck(event): "Data in EventGridEvent cannot be bytes. Please refer to" "https://docs.microsoft.com/en-us/azure/event-grid/event-schema" ) + +def _cloud_event_to_generated(cloud_event, **kwargs): + if isinstance(cloud_event.data, six.binary_type): + data_base64 = cloud_event.data + data = None + else: + data = cloud_event.data + data_base64 = None + return InternalCloudEvent( + id=cloud_event.id, + source=cloud_event.source, + type=cloud_event.type, + specversion=cloud_event.specversion, + data=data, + data_base64=data_base64, + time=cloud_event.time, + dataschema=cloud_event.dataschema, + datacontenttype=cloud_event.datacontenttype, + subject=cloud_event.subject, + additional_properties=cloud_event.extensions, + **kwargs + ) diff --git a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py index 5154ed979fa4..61b74a4d6345 100644 --- a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py +++ b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- # pylint:disable=protected-access -from typing import Union, Any, Dict +from typing import Any import datetime as dt import uuid import json @@ -11,152 +11,10 @@ from msrest.serialization import UTC from ._generated.models import ( EventGridEvent as InternalEventGridEvent, - CloudEvent as InternalCloudEvent, ) -class EventMixin(object): - """ - Mixin for the event models comprising of some helper methods. - """ - - @staticmethod - def _from_json(event, encode): - """ - Load the event into the json - :param dict eventgrid_event: The event to be deserialized. - :type eventgrid_event: Union[str, dict, bytes] - :param str encode: The encoding to be used. Defaults to 'utf-8' - """ - if isinstance(event, six.binary_type): - event = json.loads(event.decode(encode)) - elif isinstance(event, six.string_types): - event = json.loads(event) - return event - - -class CloudEvent(EventMixin): # pylint:disable=too-many-instance-attributes - """Properties of an event published to an Event Grid topic using the CloudEvent 1.0 Schema. - - All required parameters must be populated in order to send to Azure. - If data is of binary type, data_base64 can be used alternatively. Note that data and data_base64 - cannot be present at the same time. - - :param source: Required. Identifies the context in which an event happened. The combination of id and source must - be unique for each distinct event. If publishing to a domain topic, source must be the domain name. - :type source: str - :param type: Required. Type of event related to the originating occurrence. - :type type: str - :keyword data: Optional. Event data specific to the event type. Only one of the `data` or `data_base64` - argument must be present. If data is of bytes type, it will be sent as data_base64 in the outgoing request. - :type data: object - :keyword time: Optional. The time (in UTC) the event was generated, in RFC3339 format. - :type time: ~datetime.datetime - :keyword dataschema: Optional. Identifies the schema that data adheres to. - :type dataschema: str - :keyword datacontenttype: Optional. Content type of data value. - :type datacontenttype: str - :keyword subject: Optional. This describes the subject of the event in the context of the event producer - (identified by source). - :type subject: str - :keyword specversion: Optional. The version of the CloudEvent spec. Defaults to "1.0" - :type specversion: str - :keyword id: Optional. An identifier for the event. The combination of id and source must be - unique for each distinct event. If not provided, a random UUID will be generated and used. - :type id: Optional[str] - :keyword data_base64: Optional. Event data specific to the event type if the data is of bytes type. - Only data of bytes type is accepted by `data-base64` and only one of the `data` or `data_base64` argument - must be present. - :type data_base64: bytes - :ivar source: Identifies the context in which an event happened. The combination of id and source must - be unique for each distinct event. If publishing to a domain topic, source must be the domain name. - :vartype source: str - :ivar data: Event data specific to the event type. - :vartype data: object - :ivar data_base64: Event data specific to the event type if the data is of bytes type. - :vartype data_base64: bytes - :ivar type: Type of event related to the originating occurrence. - :vartype type: str - :ivar time: The time (in UTC) the event was generated, in RFC3339 format. - :vartype time: ~datetime.datetime - :ivar dataschema: Identifies the schema that data adheres to. - :vartype dataschema: str - :ivar datacontenttype: Content type of data value. - :vartype datacontenttype: str - :ivar subject: This describes the subject of the event in the context of the event producer - (identified by source). - :vartype subject: str - :ivar specversion: Optional. The version of the CloudEvent spec. Defaults to "1.0" - :vartype specversion: str - :ivar id: An identifier for the event. The combination of id and source must be - unique for each distinct event. If not provided, a random UUID will be generated and used. - :vartype id: Optional[str] - """ - - def __init__(self, source, type, **kwargs): # pylint: disable=redefined-builtin - # type: (str, str, Any) -> None - self.source = source - self.type = type - self.specversion = kwargs.pop("specversion", "1.0") - self.id = kwargs.pop("id", str(uuid.uuid4())) - self.time = kwargs.pop("time", dt.datetime.now(UTC()).isoformat()) - self.data = kwargs.pop("data", None) - self.datacontenttype = kwargs.pop("datacontenttype", None) - self.dataschema = kwargs.pop("dataschema", None) - self.subject = kwargs.pop("subject", None) - self.data_base64 = kwargs.pop("data_base64", None) - self.extensions = {} - self.extensions.update(dict(kwargs.pop("extensions", {}))) - if self.data is not None and self.data_base64 is not None: - raise ValueError( - "data and data_base64 cannot be provided at the same time.\ - Use data_base64 only if you are sending bytes, and use data otherwise." - ) - - @classmethod - def _from_generated(cls, cloud_event, **kwargs): - # type: (Union[str, Dict, bytes], Any) -> CloudEvent - generated = InternalCloudEvent.deserialize(cloud_event) - if generated.additional_properties: - extensions = dict(generated.additional_properties) - kwargs.setdefault("extensions", extensions) - return cls( - id=generated.id, - source=generated.source, - type=generated.type, - specversion=generated.specversion, - data=generated.data or generated.data_base64, - time=generated.time, - dataschema=generated.dataschema, - datacontenttype=generated.datacontenttype, - subject=generated.subject, - **kwargs - ) - - def _to_generated(self, **kwargs): - if isinstance(self.data, six.binary_type): - data_base64 = self.data - data = None - else: - data = self.data - data_base64 = None - return InternalCloudEvent( - id=self.id, - source=self.source, - type=self.type, - specversion=self.specversion, - data=data, - data_base64=self.data_base64 or data_base64, - time=self.time, - dataschema=self.dataschema, - datacontenttype=self.datacontenttype, - subject=self.subject, - additional_properties=self.extensions, - **kwargs - ) - - -class EventGridEvent(InternalEventGridEvent, EventMixin): +class EventGridEvent(InternalEventGridEvent): """Properties of an event published to an Event Grid topic using the EventGrid Schema. Variables are only populated by the server, and will be ignored when sending a request. diff --git a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_publisher_client.py b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_publisher_client.py index 20b8b04df3b0..482699f9068a 100644 --- a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_publisher_client.py +++ b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/_publisher_client.py @@ -21,14 +21,16 @@ HttpLoggingPolicy, UserAgentPolicy, ) +from azure.core.messaging import CloudEvent -from ._models import CloudEvent, EventGridEvent +from ._models import EventGridEvent from ._helpers import ( _get_endpoint_only_fqdn, _get_authentication_policy, _is_cloud_event, _is_eventgrid_event, _eventgrid_data_typecheck, + _cloud_event_to_generated, ) from ._generated._event_grid_publisher_client import ( EventGridPublisherClient as EventGridPublisherClientImpl, @@ -179,7 +181,7 @@ def send(self, events, **kwargs): if isinstance(events[0], CloudEvent) or _is_cloud_event(events[0]): try: events = [ - cast(CloudEvent, e)._to_generated(**kwargs) for e in events # pylint: disable=protected-access + _cloud_event_to_generated(e, **kwargs) for e in events # pylint: disable=protected-access ] except AttributeError: pass # means it's a dictionary diff --git a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/aio/_publisher_client_async.py b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/aio/_publisher_client_async.py index 5029e9a5f963..2ae3d973bf7b 100644 --- a/sdk/eventgrid/azure-eventgrid/azure/eventgrid/aio/_publisher_client_async.py +++ b/sdk/eventgrid/azure-eventgrid/azure/eventgrid/aio/_publisher_client_async.py @@ -9,6 +9,7 @@ from typing import Any, Union, List, Dict, cast from azure.core.credentials import AzureKeyCredential, AzureSasCredential from azure.core.tracing.decorator_async import distributed_trace_async +from azure.core.messaging import CloudEvent from azure.core.pipeline.policies import ( RequestIdPolicy, HeadersPolicy, @@ -23,13 +24,14 @@ UserAgentPolicy, ) from .._policies import CloudEventDistributedTracingPolicy -from .._models import CloudEvent, EventGridEvent +from .._models import EventGridEvent from .._helpers import ( _get_endpoint_only_fqdn, _get_authentication_policy, _is_cloud_event, _is_eventgrid_event, _eventgrid_data_typecheck, + _cloud_event_to_generated, ) from .._generated.aio import EventGridPublisherClient as EventGridPublisherClientAsync from .._version import VERSION @@ -172,7 +174,7 @@ async def send(self, events: SendType, **kwargs: Any) -> None: if isinstance(events[0], CloudEvent) or _is_cloud_event(events[0]): try: events = [ - cast(CloudEvent, e)._to_generated(**kwargs) for e in events # pylint: disable=protected-access + _cloud_event_to_generated(e, **kwargs) for e in events # pylint: disable=protected-access ] except AttributeError: pass # means it's a dictionary diff --git a/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_cloud_event_using_dict_async.py b/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_cloud_event_using_dict_async.py index db958983fb53..63f36c5ecb80 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_cloud_event_using_dict_async.py +++ b/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_cloud_event_using_dict_async.py @@ -17,7 +17,7 @@ """ import os import asyncio -from azure.eventgrid import CloudEvent +from azure.core.messaging import CloudEvent from azure.eventgrid.aio import EventGridPublisherClient from azure.core.credentials import AzureKeyCredential diff --git a/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_events_using_cloud_events_1.0_schema_async.py b/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_events_using_cloud_events_1.0_schema_async.py index 24a881e51eb5..f67bf1a19df5 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_events_using_cloud_events_1.0_schema_async.py +++ b/sdk/eventgrid/azure-eventgrid/samples/async_samples/sample_publish_events_using_cloud_events_1.0_schema_async.py @@ -17,7 +17,7 @@ # [START publish_cloud_event_to_topic_async] import os import asyncio -from azure.eventgrid import CloudEvent +from azure.core.messaging import CloudEvent from azure.eventgrid.aio import EventGridPublisherClient from azure.core.credentials import AzureKeyCredential diff --git a/sdk/eventgrid/azure-eventgrid/samples/consume_samples/consume_cloud_events_from_storage_queue.py b/sdk/eventgrid/azure-eventgrid/samples/consume_samples/consume_cloud_events_from_storage_queue.py index 919183405d4d..511188b66df5 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/consume_samples/consume_cloud_events_from_storage_queue.py +++ b/sdk/eventgrid/azure-eventgrid/samples/consume_samples/consume_cloud_events_from_storage_queue.py @@ -14,7 +14,7 @@ 3) STORAGE_QUEUE_NAME: The name of the storage queue. """ -from azure.eventgrid import CloudEvent +from azure.core.messaging import CloudEvent from azure.storage.queue import QueueServiceClient, BinaryBase64DecodePolicy import os import json diff --git a/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_cloud_events_to_custom_topic_sample.py b/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_cloud_events_to_custom_topic_sample.py index 108904daa05c..fbd7aa72a024 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_cloud_events_to_custom_topic_sample.py +++ b/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_cloud_events_to_custom_topic_sample.py @@ -20,7 +20,8 @@ import time from azure.core.credentials import AzureKeyCredential -from azure.eventgrid import EventGridPublisherClient, CloudEvent +from azure.core.messaging import CloudEvent +from azure.eventgrid import EventGridPublisherClient key = os.environ.get("CLOUD_ACCESS_KEY") endpoint = os.environ["CLOUD_TOPIC_HOSTNAME"] diff --git a/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_cloud_events_to_domain_topic_sample.py b/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_cloud_events_to_domain_topic_sample.py index 9f71e67c752f..3cc781ac04be 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_cloud_events_to_domain_topic_sample.py +++ b/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_cloud_events_to_domain_topic_sample.py @@ -22,7 +22,8 @@ import time from azure.core.credentials import AzureKeyCredential -from azure.eventgrid import EventGridPublisherClient, CloudEvent +from azure.core.messaging import CloudEvent +from azure.eventgrid import EventGridPublisherClient domain_key = os.environ["DOMAIN_ACCESS_KEY"] domain_endpoint = os.environ["DOMAIN_TOPIC_HOSTNAME"] diff --git a/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_with_shared_access_signature_sample.py b/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_with_shared_access_signature_sample.py index 5782a2c65168..4d147a1aa75b 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_with_shared_access_signature_sample.py +++ b/sdk/eventgrid/azure-eventgrid/samples/publish_samples/publish_with_shared_access_signature_sample.py @@ -21,7 +21,8 @@ from datetime import datetime, timedelta from azure.core.credentials import AzureSasCredential -from azure.eventgrid import EventGridPublisherClient, CloudEvent, generate_sas +from azure.core.messaging import CloudEvent +from azure.eventgrid import EventGridPublisherClient, generate_sas key = os.environ["CLOUD_ACCESS_KEY"] endpoint = os.environ["CLOUD_TOPIC_HOSTNAME"] diff --git a/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_consume_custom_payload.py b/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_consume_custom_payload.py index 70ae25278024..b9559e94ffd7 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_consume_custom_payload.py +++ b/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_consume_custom_payload.py @@ -12,7 +12,7 @@ python sample_consume_custom_payload.py """ -from azure.eventgrid import CloudEvent +from azure.core.messaging import CloudEvent import json # all types of CloudEvents below produce same DeserializedEvent diff --git a/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_publish_events_using_cloud_events_1.0_schema.py b/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_publish_events_using_cloud_events_1.0_schema.py index c2147ce66238..f27cf822e6a5 100644 --- a/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_publish_events_using_cloud_events_1.0_schema.py +++ b/sdk/eventgrid/azure-eventgrid/samples/sync_samples/sample_publish_events_using_cloud_events_1.0_schema.py @@ -16,8 +16,9 @@ """ # [START publish_cloud_event_to_topic] import os -from azure.eventgrid import EventGridPublisherClient, CloudEvent +from azure.eventgrid import EventGridPublisherClient from azure.core.credentials import AzureKeyCredential +from azure.core.messaging import CloudEvent topic_key = os.environ["CLOUD_ACCESS_KEY"] endpoint = os.environ["CLOUD_TOPIC_HOSTNAME"] diff --git a/sdk/eventgrid/azure-eventgrid/setup.py b/sdk/eventgrid/azure-eventgrid/setup.py index 205d3ceec27c..36e7f2dee297 100644 --- a/sdk/eventgrid/azure-eventgrid/setup.py +++ b/sdk/eventgrid/azure-eventgrid/setup.py @@ -82,7 +82,7 @@ ]), install_requires=[ 'msrest>=0.6.19', - 'azure-core<2.0.0,>=1.10.0', + 'azure-core<2.0.0,>=1.12.0', ], extras_require={ ":python_version<'3.0'": ['azure-nspkg'], diff --git a/sdk/eventgrid/azure-eventgrid/tests/test_cloud_event_tracing.py b/sdk/eventgrid/azure-eventgrid/tests/test_cloud_event_tracing.py index 9032189d0658..feed7d9865d8 100644 --- a/sdk/eventgrid/azure-eventgrid/tests/test_cloud_event_tracing.py +++ b/sdk/eventgrid/azure-eventgrid/tests/test_cloud_event_tracing.py @@ -12,7 +12,7 @@ PipelineContext ) from azure.core.pipeline.transport import HttpRequest -from azure.eventgrid import CloudEvent +from azure.core.messaging import CloudEvent from azure.eventgrid._policies import CloudEventDistributedTracingPolicy from _mocks import ( cloud_storage_dict diff --git a/sdk/eventgrid/azure-eventgrid/tests/test_eg_publisher_client.py b/sdk/eventgrid/azure-eventgrid/tests/test_eg_publisher_client.py index 5792a07dec03..637c847249fa 100644 --- a/sdk/eventgrid/azure-eventgrid/tests/test_eg_publisher_client.py +++ b/sdk/eventgrid/azure-eventgrid/tests/test_eg_publisher_client.py @@ -18,7 +18,10 @@ from azure_devtools.scenario_tests import ReplayableTest from azure.core.credentials import AzureKeyCredential, AzureSasCredential -from azure.eventgrid import EventGridPublisherClient, CloudEvent, EventGridEvent, generate_sas +from azure.core.messaging import CloudEvent +from azure.core.serialization import NULL +from azure.eventgrid import EventGridPublisherClient, EventGridEvent, generate_sas +from azure.eventgrid._helpers import _cloud_event_to_generated from eventgrid_preparer import ( CachedEventGridTopicPreparer @@ -132,32 +135,27 @@ def test_send_cloud_event_data_dict(self, resource_group, eventgrid_topic, event ) client.send(cloud_event) + @pytest.mark.skip("https://github.com/Azure/azure-sdk-for-python/issues/16993") @CachedResourceGroupPreparer(name_prefix='eventgridtest') @CachedEventGridTopicPreparer(name_prefix='cloudeventgridtest') - def test_send_cloud_event_data_base64_using_data(self, resource_group, eventgrid_topic, eventgrid_topic_primary_key, eventgrid_topic_endpoint): + def test_send_cloud_event_data_NULL(self, resource_group, eventgrid_topic, eventgrid_topic_primary_key, eventgrid_topic_endpoint): akc_credential = AzureKeyCredential(eventgrid_topic_primary_key) client = EventGridPublisherClient(eventgrid_topic_endpoint, akc_credential) cloud_event = CloudEvent( source = "http://samplesource.dev", - data = b'cloudevent', + data = NULL, type="Sample.Cloud.Event" ) - - def callback(request): - req = json.loads(request.http_request.body) - assert req[0].get("data_base64") is not None - assert req[0].get("data") is None - - client.send(cloud_event, raw_response_hook=callback) + client.send(cloud_event) @CachedResourceGroupPreparer(name_prefix='eventgridtest') @CachedEventGridTopicPreparer(name_prefix='cloudeventgridtest') - def test_send_cloud_event_bytes_using_data_base64(self, resource_group, eventgrid_topic, eventgrid_topic_primary_key, eventgrid_topic_endpoint): + def test_send_cloud_event_data_base64_using_data(self, resource_group, eventgrid_topic, eventgrid_topic_primary_key, eventgrid_topic_endpoint): akc_credential = AzureKeyCredential(eventgrid_topic_primary_key) client = EventGridPublisherClient(eventgrid_topic_endpoint, akc_credential) cloud_event = CloudEvent( source = "http://samplesource.dev", - data_base64 = b'cloudevent', + data = b'cloudevent', type="Sample.Cloud.Event" ) @@ -168,9 +166,8 @@ def callback(request): client.send(cloud_event, raw_response_hook=callback) - def test_send_cloud_event_fails_on_providing_data_and_b64(self): - with pytest.raises(ValueError, match="data and data_base64 cannot be provided at the same time*"): + with pytest.raises(ValueError, match="Unexpected keyword arguments data_base64.*"): cloud_event = CloudEvent( source = "http://samplesource.dev", data_base64 = b'cloudevent', @@ -241,7 +238,7 @@ def test_send_cloud_event_data_with_extensions(self, resource_group, eventgrid_t } ) client.send([cloud_event]) - internal = cloud_event._to_generated().serialize() + internal = _cloud_event_to_generated(cloud_event).serialize() assert 'reason_code' in internal assert 'extension' in internal assert internal['reason_code'] == 204 diff --git a/sdk/eventgrid/azure-eventgrid/tests/test_eg_publisher_client_async.py b/sdk/eventgrid/azure-eventgrid/tests/test_eg_publisher_client_async.py index 040866a36ac2..6c2c53db9531 100644 --- a/sdk/eventgrid/azure-eventgrid/tests/test_eg_publisher_client_async.py +++ b/sdk/eventgrid/azure-eventgrid/tests/test_eg_publisher_client_async.py @@ -17,7 +17,9 @@ from azure_devtools.scenario_tests import ReplayableTest from azure.core.credentials import AzureKeyCredential, AzureSasCredential -from azure.eventgrid import CloudEvent, EventGridEvent, generate_sas +from azure.core.messaging import CloudEvent +from azure.core.serialization import NULL +from azure.eventgrid import EventGridEvent, generate_sas from azure.eventgrid.aio import EventGridPublisherClient from eventgrid_preparer import ( @@ -174,15 +176,15 @@ async def test_send_cloud_event_data_with_extensions(self, resource_group, event data = "cloudevent", type="Sample.Cloud.Event", extensions={ - 'reason_code':204, + 'reasonCode':204, 'extension':'hello' } ) await client.send([cloud_event]) internal = cloud_event._to_generated().serialize() - assert 'reason_code' in internal + assert 'reasonCode' in internal assert 'extension' in internal - assert internal['reason_code'] == 204 + assert internal['reasonCode'] == 204 @CachedResourceGroupPreparer(name_prefix='eventgridtest') @@ -213,6 +215,20 @@ async def test_send_cloud_event_data_none(self, resource_group, eventgrid_topic, ) await client.send(cloud_event) + @pytest.mark.skip("https://github.com/Azure/azure-sdk-for-python/issues/16993") + @CachedResourceGroupPreparer(name_prefix='eventgridtest') + @CachedEventGridTopicPreparer(name_prefix='cloudeventgridtest') + @pytest.mark.asyncio + async def test_send_cloud_event_data_NULL(self, resource_group, eventgrid_topic, eventgrid_topic_primary_key, eventgrid_topic_endpoint): + akc_credential = AzureKeyCredential(eventgrid_topic_primary_key) + client = EventGridPublisherClient(eventgrid_topic_endpoint, akc_credential) + cloud_event = CloudEvent( + source = "http://samplesource.dev", + data = NULL, + type="Sample.Cloud.Event" + ) + await client.send(cloud_event) + @CachedResourceGroupPreparer(name_prefix='eventgridtest') @CachedEventGridTopicPreparer(name_prefix='eventgridtest') @pytest.mark.asyncio diff --git a/sdk/eventgrid/azure-eventgrid/tests/test_serialization.py b/sdk/eventgrid/azure-eventgrid/tests/test_serialization.py index 5bed9c1c9212..cb455fc2c0a8 100644 --- a/sdk/eventgrid/azure-eventgrid/tests/test_serialization.py +++ b/sdk/eventgrid/azure-eventgrid/tests/test_serialization.py @@ -14,9 +14,10 @@ from devtools_testutils import AzureMgmtTestCase from msrest.serialization import UTC -from azure.eventgrid import CloudEvent, EventGridEvent +from azure.core.messaging import CloudEvent from azure.eventgrid._generated import models as internal_models -from azure.eventgrid import SystemEventNames +from azure.eventgrid._helpers import _cloud_event_to_generated +from azure.eventgrid import SystemEventNames, EventGridEvent from _mocks import ( cloud_storage_dict, cloud_storage_string, @@ -39,16 +40,14 @@ def test_cloud_event_serialization_extension_bytes(self, **kwargs): source="http://samplesource.dev", data=data, type="Sample.Cloud.Event", - foo="bar", extensions={'e1':1, 'e2':2} ) cloud_event.subject = "subject" # to test explicit setting of prop encoded = base64.b64encode(data).decode('utf-8') - internal = cloud_event._to_generated() + internal = _cloud_event_to_generated(cloud_event) assert internal.additional_properties is not None - assert 'foo' not in internal.additional_properties assert 'e1' in internal.additional_properties json = internal.serialize() @@ -72,15 +71,13 @@ def test_cloud_event_serialization_extension_string(self, **kwargs): source="http://samplesource.dev", data=data, type="Sample.Cloud.Event", - foo="bar", extensions={'e1':1, 'e2':2} ) cloud_event.subject = "subject" # to test explicit setting of prop - internal = cloud_event._to_generated() + internal = _cloud_event_to_generated(cloud_event) assert internal.additional_properties is not None - assert 'foo' not in internal.additional_properties assert 'e1' in internal.additional_properties json = internal.serialize()