forked from Azure/azure-sdk-for-python
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Cosmos] split proof queries sync client (Azure#22237)
* initial commit * Client Constructor (Azure#20310) * Removed some stuff * Looking at constructors * Updated request * Added client close * working client creation Co-authored-by: simorenoh <[email protected]> * read database database read works, but ignored exception is returned: Fatal error on SSL transport NoneType has no attribute 'send' (_loop._proactor.send) RuntimeError: Event loop is closed Unclosed connector/ connection * Update simon_testfile.py * with coroutine Added methods needed to use async with when initializing client, but logs output "Exception ignored... Runtime Error: Event loop is closed" * Update simon_testfile.py * small changes * async with returns no exceptions * async read container * async item read * cleaning up * create item/ database methods * item delete working * docs replace functionality missing upsert and other resources * upsert functionality missing read_all_items and both query methods for container class * missing query methods * CRUD for udf, sproc, triggers * initial query logic + container methods * missing some execution logic and tests * oops * fully working queries * small fix to query_items() also fixed README and added examples_async * Update _cosmos_client_connection_async.py * Update _cosmos_client_connection.py * documentation update * updated MIT dates and get_user_client() description * Update CHANGELOG.md * Delete simon_testfile.py * leftover retry utility * Update README.md * docs and removed six package * changes based on comments still missing discussion resolution on SSL verification and tests for async functionality under test module (apart from samples which are basically end to end tests) * small change in type hints * updated readme * fixes based on conversations * added missing type comments * update changelog for ci pipeline * added typehints, moved params into keywords, added decorators, made _connection_policy private * changes based on sync with central sdk * remove is_system_key from scripts (only used in execute_sproc) is_system_key verifies that an empty partition key is properly dealt with if ['partitionKey']['systemKey'] exists in the container options - however, we do not allow containers to be created with empty partition key values in the python sdk, so the functionality is needless * Revert "remove is_system_key from scripts (only used in execute_sproc)" Reverting last commit, will find way to init is_system_key for now * async script proxy using composition * pylint * capitalized constants * Apply suggestions from code review Clarifying comments for README Co-authored-by: Gahl Levy <[email protected]> * closing python code snippet * last doc updates * Update sdk/cosmos/azure-cosmos/CHANGELOG.md Co-authored-by: Simon Moreno <[email protected]> * version update * cosmos updates for release * current state gone_retry_policy might end up being unneccesary, based on what we feel is best from an arch standpoint * working split proof, need to remove prints * improving comments and removing print statements * removed last prints and used constants * Update CHANGELOG.md * small fixes based on comments * addressed more comments * added test, made slight changes * rename test and small changes * pylint * pylintpylintpylint * moved partition_range_gone check to exceptions since makes more sense * re use code Co-authored-by: annatisch <[email protected]> Co-authored-by: Gahl Levy <[email protected]> Co-authored-by: Travis Prescott <[email protected]>
- Loading branch information
1 parent
d67e7b4
commit 5026316
Showing
10 changed files
with
246 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
53 changes: 53 additions & 0 deletions
53
sdk/cosmos/azure-cosmos/azure/cosmos/_gone_retry_policy.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| # The MIT License (MIT) | ||
| # Copyright (c) 2021 Microsoft Corporation | ||
|
|
||
| # Permission is hereby granted, free of charge, to any person obtaining a copy | ||
| # of this software and associated documentation files (the "Software"), to deal | ||
| # in the Software without restriction, including without limitation the rights | ||
| # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
| # copies of the Software, and to permit persons to whom the Software is | ||
| # furnished to do so, subject to the following conditions: | ||
|
|
||
| # The above copyright notice and this permission notice shall be included in all | ||
| # copies or substantial portions of the Software. | ||
|
|
||
| # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
| # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
| # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
| # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
| # SOFTWARE. | ||
|
|
||
| """Internal class for connection reset retry policy implementation in the Azure | ||
| Cosmos database service. | ||
| """ | ||
|
|
||
|
|
||
| # pylint: disable=protected-access | ||
|
|
||
|
|
||
| class PartitionKeyRangeGoneRetryPolicy(object): | ||
|
|
||
| def __init__(self, client, *args): | ||
| self.retry_after_in_milliseconds = 1000 | ||
| self.refresh_partition_key_range_cache = True | ||
| self.args = args | ||
| self.client = client | ||
| self.exception = None | ||
|
|
||
| def ShouldRetry(self, exception): | ||
| """Returns true if should retry based on the passed-in exception. | ||
| :param (exceptions.CosmosHttpResponseError instance) exception: | ||
| :rtype: boolean | ||
| """ | ||
| self.exception = exception # needed for pylint | ||
| if self.refresh_partition_key_range_cache: | ||
| # refresh routing_map_provider to refresh partition key range cache | ||
| # make refresh_partition_key_range_cache False to skip this check on subsequent Gone exceptions | ||
| self.client.refresh_routing_map_provider() | ||
| self.refresh_partition_key_range_cache = False | ||
| # return False to raise error to multi_execution_aggregator and repair document producer context | ||
| return False |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
107 changes: 107 additions & 0 deletions
107
sdk/cosmos/azure-cosmos/test/test_partition_split_query.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| # The MIT License (MIT) | ||
| # Copyright (c) 2021 Microsoft Corporation | ||
|
|
||
| # Permission is hereby granted, free of charge, to any person obtaining a copy | ||
| # of this software and associated documentation files (the "Software"), to deal | ||
| # in the Software without restriction, including without limitation the rights | ||
| # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
| # copies of the Software, and to permit persons to whom the Software is | ||
| # furnished to do so, subject to the following conditions: | ||
|
|
||
| # The above copyright notice and this permission notice shall be included in all | ||
| # copies or substantial portions of the Software. | ||
|
|
||
| # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
| # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
| # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
| # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
| # SOFTWARE. | ||
|
|
||
| import unittest | ||
|
|
||
| import azure.cosmos.cosmos_client as cosmos_client | ||
| import pytest | ||
| import time | ||
| import random | ||
| import uuid | ||
| import test_config | ||
|
|
||
| # This test class serves to test partition splits within the query context | ||
|
|
||
| pytestmark = pytest.mark.cosmosEmulator | ||
|
|
||
|
|
||
| @pytest.mark.usefixtures("teardown") | ||
| class TestPartitionSplitQuery(unittest.TestCase): | ||
| configs = test_config._test_config | ||
| host = configs.host | ||
| masterKey = configs.masterKey | ||
| throughput = 400 | ||
|
|
||
| @classmethod | ||
| def setUpClass(cls): | ||
| cls.client = cosmos_client.CosmosClient(cls.host, cls.masterKey) | ||
| cls.database = test_config._test_config.create_database_if_not_exist_with_throughput(cls.client, cls.throughput) | ||
| cls.container = test_config._test_config.create_collection_no_custom_throughput(cls.client) | ||
|
|
||
| def test_partition_split_query(self): | ||
| for i in range(1000): | ||
| body = self.get_test_item() | ||
| self.container.create_item(body=body) | ||
|
|
||
| print("created items, changing offer to 11k and starting queries") | ||
| self.database.replace_throughput(11000) | ||
| offer_time = time.time() | ||
| print("changed offer to 11k") | ||
| print("--------------------------------") | ||
| print("now starting queries") | ||
|
|
||
| self.run_queries(self.container, 1000) # initial check for queries before partition split | ||
| print("initial check succeeded, now reading offer until replacing is done") | ||
| offer = self.database.read_offer() | ||
| while True: | ||
| if offer.properties['content'].get('isOfferReplacePending', False): | ||
| time.sleep(10) | ||
| offer = self.database.read_offer() | ||
| else: | ||
| print("offer replaced successfully, took around {} seconds".format(time.time() - offer_time)) | ||
| self.run_queries(self.container, 1000) # check queries work post partition split | ||
| print("test over") | ||
| self.assertTrue(offer.offer_throughput > self.throughput) | ||
| self.client.delete_database(self.configs.TEST_DATABASE_ID) | ||
| return | ||
|
|
||
| def run_queries(self, container, iterations): | ||
| ret_list = list() | ||
| for i in range(iterations): | ||
| curr = str(random.randint(0, 10)) | ||
| query = 'SELECT * FROM c WHERE c.attr1=' + curr + ' order by c.attr1' | ||
| qlist = list(container.query_items(query=query, enable_cross_partition_query=True)) | ||
| ret_list.append((curr, qlist)) | ||
| for ret in ret_list: | ||
| curr = ret[0] | ||
| if len(ret[1]) != 0: | ||
| for results in ret[1]: | ||
| attr_number = results['attr1'] | ||
| assert str(attr_number) == curr # verify that all results match their randomly generated attributes | ||
| print("validation succeeded for all query results") | ||
|
|
||
| def get_test_item(self): | ||
| async_item = { | ||
| 'id': 'Async_' + str(uuid.uuid4()), | ||
| 'address': { | ||
| 'state': 'WA', | ||
| 'city': 'Redmond', | ||
| 'street': '1 Microsoft Way' | ||
| }, | ||
| 'test_object': True, | ||
| 'lastName': 'Smith', | ||
| 'attr1': random.randint(0, 10) | ||
| } | ||
| return async_item | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| unittest.main() |