[Tables] Add TableEntity decoder (Azure#36611)

* Add decoder

* Use decoder in sync ops

* Update __init__.py

* Update

* Update _decoder.py

* Fix IndexError

* Fix

* Fix

* Try without EdmType.INT64

* Add decoder sync tests

* Update assets.json

* Add cosmos decoder sync tests

* Support decoder in async ops

* Add storage and cosmos async tests

* Run black

* Update assets.json

* Fix mypy

* Fix a decoder bug

* Fix decoder bugs

* Move _prepare_key out of TableEntityEncoderABC

* Change to use convert_map for encoder and decoder

* Update CHANGELOG.md

* Update tests

* Fix bugs

* Fix pylint

* Update assets.json

* Add flatten entity metadata tests

* Add flatten entity metadata samples

* Address

* Update dataclass samples

* Create sample_encode_dataclass_model.py

* Address

* Address

* Address

* Address

* Add pydantic samples

* Address

* Mypy

* black

* Mypy

---------

Co-authored-by: antisch <[email protected]>

sdk/tables/azure-data-tables/CHANGELOG.md

@@ -1,19 +1,23 @@
 # Release History
 
-## 12.6.0b1 (Unreleased)
+## 12.7.0b1 (Unreleased)
 
 ### Features Added
-* Added to support custom encoder in entity CRUD operations.
-* Added to support custom Entity type.
+* Added to support customized encoding and decoding in entity CRUD operations.
 * Added to support Entity property in Tuple and Enum types.
-* Added support for Microsoft Entra auth with Azure Cosmos DB for Table's OAuth scope (`https://cosmos.azure.com/.default`).
+* Added to support flatten Entity metadata in entity deserialization by passing kwarg `flatten_result_entity` when creating clients.
 
 ### Bugs Fixed
-* Fixed a bug in encoder when Entity property has "@odata.type" provided.
+* Fixed duplicate odata tag bug in encoder when Entity property has "@odata.type" provided.
 * Fixed a bug in encoder that int32 and int64 are mapped to int32 when no "@odata.type" provided.
 
 ### Other Changes
-* Removed value range validation for Entity property in int32 and int64.
+* Removed value range validation for Entity property in int64.
+
+## 12.6.0 (2024-11-21)
+
+### Features Added
+* Added support for Microsoft Entra auth with Azure Cosmos DB for Table's OAuth scope (`https://cosmos.azure.com/.default`).
 
 ## 12.5.0 (2024-01-10)
 

sdk/tables/azure-data-tables/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/tables/azure-data-tables",
-  "Tag": "python/tables/azure-data-tables_032477adf3"
+  "Tag": "python/tables/azure-data-tables_48a9914a75"
 }

sdk/tables/azure-data-tables/azure/data/tables/__init__.py

@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from ._encoder import TableEntityEncoder, TableEntityEncoderABC
 from ._entity import TableEntity, EntityProperty, EdmType, EntityMetadata
 from ._error import RequestTooLargeError, TableTransactionError, TableErrorCode
 from ._table_shared_access_signature import generate_table_sas, generate_account_sas
@@ -32,8 +31,6 @@
     "TableServiceClient",
     "ResourceTypes",
     "AccountSasPermissions",
-    "TableEntityEncoder",
-    "TableEntityEncoderABC",
     "TableErrorCode",
     "TableSasPermissions",
     "TableAccessPolicy",

sdk/tables/azure-data-tables/azure/data/tables/_common_conversion.py

@@ -8,7 +8,7 @@
 import hmac
 from datetime import timezone
 from urllib.parse import ParseResult
-from typing import Optional, Tuple, List
+from typing import Optional, Tuple, List, Dict, Any, Union, cast
 
 
 def _to_str(value):
@@ -85,3 +85,51 @@ def _get_account(parsed_url: ParseResult) -> Tuple[List[str], Optional[str]]:
             account = parsed_url.netloc.split(".table.core.")
             account_name = account[0] if len(account) > 1 else None
     return account, account_name
+
+
+def _prepare_key(key: Union[str, int, float, None]) -> str:
+    """Duplicate the single quote char to escape.
+
+    :param str key: The entity PartitionKey or RowKey value in table entity.
+    :return: The entity PartitionKey or RowKey value in table entity.
+    :rtype: str
+    """
+    try:
+        return cast(str, key).replace("'", "''")
+    except (AttributeError, TypeError) as exc:
+        raise TypeError("PartitionKey or RowKey must be of type string.") from exc
+
+
+def _get_enum_value(value):
+    if value is None or value in ["None", ""]:
+        return None
+    try:
+        return value.value
+    except AttributeError:
+        return value
+
+
+def _normalize_headers(headers):
+    normalized = {}
+    for key, value in headers.items():
+        if key.startswith("x-ms-"):
+            key = key[5:]
+        normalized[key.lower().replace("-", "_")] = _get_enum_value(value)
+    return normalized
+
+
+def _return_headers_and_deserialized(_, deserialized, response_headers):
+    return _normalize_headers(response_headers), deserialized
+
+
+def _trim_service_metadata(metadata: Dict[str, str], content: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    result: Dict[str, Any] = {
+        "date": metadata.pop("date", None),
+        "etag": metadata.pop("etag", None),
+        "version": metadata.pop("version", None),
+    }
+    preference = metadata.pop("preference_applied", None)
+    if preference:
+        result["preference_applied"] = preference
+        result["content"] = content
+    return result

sdk/tables/azure-data-tables/azure/data/tables/_constants.py

@@ -23,3 +23,5 @@
 MIN_INT32 = -(2**31)  # -2147483648
 MAX_INT64 = (2**63) - 1  # 9223372036854775807
 MIN_INT64 = -(2**63)  # -9223372036854775808
+
+_ERROR_VALUE_TOO_LARGE = "{0} is too large to be cast to type {1}."

sdk/tables/azure-data-tables/azure/data/tables/_decoder.py

@@ -0,0 +1,190 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Any, Optional, Mapping, Union, Dict, Callable, cast
+from datetime import datetime, timezone
+from urllib.parse import quote
+from uuid import UUID
+
+from ._common_conversion import _decode_base64_to_bytes
+from ._entity import EntityProperty, EdmType, TableEntity, EntityMetadata
+
+DecoderMapType = Dict[EdmType, Callable[[Union[str, bool, int, float]], Any]]
+
+
+class TablesEntityDatetime(datetime):
+    _service_value: str
+
+    @property
+    def tables_service_value(self) -> str:
+        try:
+            return self._service_value
+        except AttributeError:
+            return ""
+
+
+NO_ODATA = {
+    int: EdmType.INT32,
+    str: EdmType.STRING,
+    bool: EdmType.BOOLEAN,
+    float: EdmType.DOUBLE,
+}
+
+
+class TableEntityDecoder:
+    def __init__(
+        self,
+        *,
+        flatten_result_entity: bool = False,
+        convert_map: Optional[DecoderMapType] = None,
+    ) -> None:
+        self.convert_map = convert_map
+        self.flatten_result_entity = flatten_result_entity
+
+    def __call__(  # pylint: disable=too-many-branches, too-many-statements
+        self, response_data: Mapping[str, Any]
+    ) -> TableEntity:
+        """Convert json response to entity.
+        The entity format is:
+        {
+        "Address":"Mountain View",
+        "Age":23,
+        "AmountDue":200.23,
+        "[email protected]":"Edm.Guid",
+        "CustomerCode":"c9da6455-213d-42c9-9a79-3e9149a57833",
+        "[email protected]":"Edm.DateTime",
+        "CustomerSince":"2008-07-10T00:00:00",
+        "IsActive":true,
+        "[email protected]":"Edm.Int64",
+        "NumberOfOrders":"255",
+        "PartitionKey":"my_partition_key",
+        "RowKey":"my_row_key"
+        }
+
+        :param response_data: The entity in response.
+        :type response_data: Mapping[str, Any]
+        :return: An entity dict with additional metadata.
+        :rtype: dict[str, Any]
+        """
+        entity = TableEntity()
+
+        properties = {}
+        edmtypes = {}
+        odata = {}
+
+        for name, value in response_data.items():
+            if name.startswith("odata."):
+                odata[name[6:]] = value
+            elif name.endswith("@odata.type"):
+                edmtypes[name[:-11]] = value
+            else:
+                properties[name] = value
+
+        # Partitionkey is a known property
+        partition_key = properties.pop("PartitionKey", None)
+        if partition_key is not None:
+            entity["PartitionKey"] = partition_key
+
+        # Timestamp is a known property
+        timestamp = properties.pop("Timestamp", None)
+
+        for name, value in properties.items():
+            mtype = edmtypes.get(name)
+
+            if not mtype:
+                mtype = NO_ODATA[type(value)]
+
+            convert = None
+            default_convert = None
+            if self.convert_map:
+                try:
+                    convert = self.convert_map[mtype]
+                except KeyError:
+                    pass
+            if convert:
+                new_property = convert(value)
+            else:
+                try:
+                    default_convert = _ENTITY_TO_PYTHON_CONVERSIONS[mtype]
+                except KeyError as e:
+                    raise TypeError(f"Unsupported edm type: {mtype}") from e
+                if default_convert is not None:
+                    new_property = default_convert(self, value)
+                else:
+                    new_property = EntityProperty(mtype, value)
+            entity[name] = new_property
+
+        # extract etag from entry
+        etag = odata.pop("etag", None)
+        odata.pop("metadata", None)
+        if timestamp:
+            if not etag:
+                etag = "W/\"datetime'" + quote(timestamp) + "'\""
+            timestamp = self.from_entity_datetime(timestamp)
+        odata.update({"etag": etag, "timestamp": timestamp})
+        if self.flatten_result_entity:
+            for name, value in odata.items():
+                entity[name] = value
+        entity._metadata = cast(EntityMetadata, odata)  # pylint: disable=protected-access
+        return entity
+
+    def from_entity_binary(self, value: str) -> bytes:
+        return _decode_base64_to_bytes(value)
+
+    def from_entity_int32(self, value: Union[int, str]) -> int:
+        return int(value)
+
+    def from_entity_int64(self, value: str) -> EntityProperty:
+        return EntityProperty(int(value), EdmType.INT64)
+
+    def from_entity_datetime(self, value: str) -> Optional[TablesEntityDatetime]:
+        return deserialize_iso(value)
+
+    def from_entity_guid(self, value: str) -> UUID:
+        return UUID(value)
+
+    def from_entity_str(self, value: Union[str, bytes]) -> str:
+        if isinstance(value, bytes):
+            return value.decode("utf-8")
+        return value
+
+
+_ENTITY_TO_PYTHON_CONVERSIONS = {
+    EdmType.BINARY: TableEntityDecoder.from_entity_binary,
+    EdmType.INT32: TableEntityDecoder.from_entity_int32,
+    EdmType.INT64: TableEntityDecoder.from_entity_int64,
+    EdmType.DOUBLE: lambda _, v: float(v),
+    EdmType.DATETIME: TableEntityDecoder.from_entity_datetime,
+    EdmType.GUID: TableEntityDecoder.from_entity_guid,
+    EdmType.STRING: TableEntityDecoder.from_entity_str,
+    EdmType.BOOLEAN: lambda _, v: v,
+}
+
+
+def deserialize_iso(value: Optional[str]) -> Optional[TablesEntityDatetime]:
+    if not value:
+        return None
+    # Cosmos returns this with a decimal point that throws an error on deserialization
+    cleaned_value = _clean_up_dotnet_timestamps(value)
+    try:
+        dt_obj = TablesEntityDatetime.strptime(cleaned_value, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
+    except ValueError:
+        dt_obj = TablesEntityDatetime.strptime(cleaned_value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
+    dt_obj._service_value = value  # pylint:disable=protected-access,assigning-non-slot
+    return dt_obj
+
+
+def _clean_up_dotnet_timestamps(value):
+    # .NET has more decimal places than Python supports in datetime objects, this truncates
+    # values after 6 decimal places.
+    value = value.split(".")
+    ms = ""
+    if len(value) == 2:
+        ms = value[-1].replace("Z", "")
+        if len(ms) > 6:
+            ms = ms[:6]
+        ms = ms + "Z"
+        return ".".join([value[0], ms])
+    return value[0]