[Tables] Migrate to new test proxy (Azure#21321)

eng/scripts/Language-Settings.ps1

@@ -561,15 +561,15 @@ function Import-Dev-Cert-python
 }
 
 function Validate-Python-DocMsPackages
-{ 
+{
   Param(
     [Parameter(Mandatory=$true)]
     [PSCustomObject]$PackageInfo,
     [Parameter(Mandatory=$false)]
     [string]$PackageSourceOverride,
     [Parameter(Mandatory=$false)]
     [string]$DocValidationImageId
-  ) 
+  )
   $packageName = $packageInfo.Name
   $packageVersion = $packageInfo.Version
   ValidatePackage -packageName $packageName -packageVersion $packageVersion `

scripts/devops_tasks/trust_proxy_cert.py

@@ -36,4 +36,4 @@ def combine_cert_file():
 
     if os.getenv('TF_BUILD', False):
         print("##vso[task.setvariable variable=SSL_CERT_DIR]{}".format(os.path.dirname(COMBINED_LOCATION)))
-        print("##vso[task.setvariable variable=REQUESTS_CA_BUNDLE]{}".format(COMBINED_LOCATION))
+        print("##vso[task.setvariable variable=REQUESTS_CA_BUNDLE]{}".format(COMBINED_LOCATION))

sdk/tables/azure-data-tables/azure/data/tables/_error.py

@@ -67,7 +67,7 @@ def _validate_table_name(table_name):
         )
 
 
-def _decode_error(response, error_message=None, error_type=None, **kwargs):
+def _decode_error(response, error_message=None, error_type=None, **kwargs):  # pylint: disable=too-many-branches
     error_code = response.headers.get("x-ms-error-code")
     additional_data = {}
     try:
@@ -80,6 +80,14 @@ def _decode_error(response, error_message=None, error_type=None, **kwargs):
                     error_message = error_body["odata.error"][info]["value"]
                 else:
                     additional_data[info.tag] = info.text
+
+            # Special case: there was a playback error during test execution (test proxy only)
+            message = error_body.get("Message")
+            if message and message.startswith("Unable to find a record for the request"):
+                error = ResourceNotFoundError(message=error_message, response=response)
+                error.error_code = 404
+                error.additional_info = additional_data
+                return error
         else:
             if error_body:
                 for info in error_body.iter():

sdk/tables/azure-data-tables/tests/conftest.py

@@ -25,9 +25,27 @@
 # --------------------------------------------------------------------------
 import sys
 
+import pytest
+from devtools_testutils import add_general_regex_sanitizer, test_proxy
+
 # fixture needs to be visible from conftest
 
 # Ignore async tests for Python < 3.5
 collect_ignore_glob = []
 if sys.version_info < (3, 5):
     collect_ignore_glob.append("*_async.py")
+
+@pytest.fixture(scope="session", autouse=True)
+def add_sanitizers(test_proxy):
+    # sanitizes table/cosmos account names in URLs
+    add_general_regex_sanitizer(
+        value="fakeendpoint",
+        regex="(?<=\\/\\/)[a-z]+(?=(?:|-secondary)\\.(?:table|blob|queue)\\.(?:cosmos|core)\\."
+              "(?:azure|windows)\\.(?:com|net))",
+    )
+    # sanitizes random UUIDs that are sent in batch request headers and bodies
+    add_general_regex_sanitizer(
+        value="00000000-0000-0000-0000-000000000000",
+        regex="batch[a-z]*_([0-9a-f]{8}\\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\\b[0-9a-f]{12}\\b)",
+        group_for_replace="1",
+    )

sdk/tables/azure-data-tables/tests/preparers.py

@@ -9,14 +9,14 @@
 CosmosPreparer = functools.partial(
     PowerShellPreparer,
     "tables",
-    tables_cosmos_account_name="fake_cosmos_account",
+    tables_cosmos_account_name="fakecosmosaccount",
     tables_primary_cosmos_account_key="fakecosmosaccountkey",
 )
 
 TablesPreparer = functools.partial(
     PowerShellPreparer,
     "tables",
-    tables_storage_account_name="fake_table_account",
+    tables_storage_account_name="faketableaccount",
     tables_primary_storage_account_key="faketablesaccountkey",
 )
 

sdk/tables/azure-data-tables/tests/recordings/test_retry.pyTestStorageRetrytest_no_retry.json

@@ -0,0 +1,117 @@
+{
+  "Entries": [
+    {
+      "RequestUri": "https://fakeendpoint.table.core.windows.net/Tables",
+      "RequestMethod": "POST",
+      "RequestHeaders": {
+        "Accept": "application/json;odata=minimalmetadata",
+        "Accept-Encoding": "gzip, deflate",
+        "Authorization": "Sanitized",
+        "Connection": "keep-alive",
+        "Content-Length": "32",
+        "Content-Type": "application/json;odata=nometadata",
+        "DataServiceVersion": "3.0",
+        "Date": "Mon, 18 Oct 2021 21:58:35 GMT",
+        "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.2 (Windows-10-10.0.19041-SP0)",
+        "x-ms-client-request-id": "8b0bb1e3-305e-11ec-ad70-5cf37093a909",
+        "x-ms-date": "Mon, 18 Oct 2021 21:58:35 GMT",
+        "x-ms-version": "2019-02-02"
+      },
+      "RequestBody": "{\u0022TableName\u0022: \u0022uttablee5cc21c0\u0022}",
+      "StatusCode": 201,
+      "ResponseHeaders": {
+        "Cache-Control": "no-cache",
+        "Content-Type": "application/json; odata=minimalmetadata; streaming=true; charset=utf-8",
+        "Date": "Mon, 18 Oct 2021 21:58:29 GMT",
+        "Location": "https://fakeendpoint.table.core.windows.net/Tables(\u0027uttablee5cc21c0\u0027)",
+        "Server": [
+          "Windows-Azure-Table/1.0",
+          "Microsoft-HTTPAPI/2.0"
+        ],
+        "Transfer-Encoding": "chunked",
+        "X-Content-Type-Options": "nosniff",
+        "x-ms-client-request-id": "8b0bb1e3-305e-11ec-ad70-5cf37093a909",
+        "x-ms-request-id": "5762714b-2002-0015-386b-c4666e000000",
+        "x-ms-version": "2019-02-02"
+      },
+      "ResponseBody": {
+        "odata.metadata": "https://fakeendpoint.table.core.windows.net/$metadata#Tables/@Element",
+        "TableName": "uttablee5cc21c0"
+      }
+    },
+    {
+      "RequestUri": "https://fakeendpoint.table.core.windows.net/Tables(\u0027uttablee5cc21c0\u0027)",
+      "RequestMethod": "DELETE",
+      "RequestHeaders": {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip, deflate",
+        "Authorization": "Sanitized",
+        "Connection": "keep-alive",
+        "Content-Length": "0",
+        "Date": "Mon, 18 Oct 2021 21:58:36 GMT",
+        "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.2 (Windows-10-10.0.19041-SP0)",
+        "x-ms-client-request-id": "8b14d3a1-305e-11ec-b866-5cf37093a909",
+        "x-ms-date": "Mon, 18 Oct 2021 21:58:36 GMT",
+        "x-ms-version": "2019-02-02"
+      },
+      "RequestBody": null,
+      "StatusCode": 204,
+      "ResponseHeaders": {
+        "Cache-Control": "no-cache",
+        "Content-Length": "0",
+        "Date": "Mon, 18 Oct 2021 21:58:29 GMT",
+        "Server": [
+          "Windows-Azure-Table/1.0",
+          "Microsoft-HTTPAPI/2.0"
+        ],
+        "X-Content-Type-Options": "nosniff",
+        "x-ms-client-request-id": "8b14d3a1-305e-11ec-b866-5cf37093a909",
+        "x-ms-request-id": "57627152-2002-0015-3e6b-c4666e000000",
+        "x-ms-version": "2019-02-02"
+      },
+      "ResponseBody": null
+    },
+    {
+      "RequestUri": "https://fakeendpoint.table.core.windows.net/Tables(\u0027uttablee5cc21c0\u0027)",
+      "RequestMethod": "DELETE",
+      "RequestHeaders": {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip, deflate",
+        "Authorization": "Sanitized",
+        "Connection": "keep-alive",
+        "Content-Length": "0",
+        "Date": "Mon, 18 Oct 2021 21:58:36 GMT",
+        "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.2 (Windows-10-10.0.19041-SP0)",
+        "x-ms-client-request-id": "8b189c34-305e-11ec-b634-5cf37093a909",
+        "x-ms-date": "Mon, 18 Oct 2021 21:58:36 GMT",
+        "x-ms-version": "2019-02-02"
+      },
+      "RequestBody": null,
+      "StatusCode": 404,
+      "ResponseHeaders": {
+        "Cache-Control": "no-cache",
+        "Content-Type": "application/json; odata=minimalmetadata; streaming=true; charset=utf-8",
+        "Date": "Mon, 18 Oct 2021 21:58:29 GMT",
+        "Server": [
+          "Windows-Azure-Table/1.0",
+          "Microsoft-HTTPAPI/2.0"
+        ],
+        "Transfer-Encoding": "chunked",
+        "X-Content-Type-Options": "nosniff",
+        "x-ms-client-request-id": "8b189c34-305e-11ec-b634-5cf37093a909",
+        "x-ms-request-id": "57627156-2002-0015-426b-c4666e000000",
+        "x-ms-version": "2019-02-02"
+      },
+      "ResponseBody": {
+        "odata.error": {
+          "code": "ResourceNotFound",
+          "message": {
+            "lang": "en-US",
+            "value": "The specified resource does not exist.\nRequestId:57627156-2002-0015-426b-c4666e000000\nTime:2021-10-18T21:58:30.2926335Z"
+          }
+        }
+      }
+    }
+  ],
+  "Variables": {}
+}

sdk/tables/azure-data-tables/tests/recordings/test_retry.pyTestStorageRetrytest_retry_on_server_error.json

@@ -0,0 +1,160 @@
+{
+  "Entries": [
+    {
+      "RequestUri": "https://fakeendpoint.table.core.windows.net/Tables",
+      "RequestMethod": "POST",
+      "RequestHeaders": {
+        "Accept": "application/json;odata=minimalmetadata",
+        "Accept-Encoding": "gzip, deflate",
+        "Authorization": "Sanitized",
+        "Connection": "keep-alive",
+        "Content-Length": "32",
+        "Content-Type": "application/json;odata=nometadata",
+        "DataServiceVersion": "3.0",
+        "Date": "Mon, 18 Oct 2021 21:58:32 GMT",
+        "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.2 (Windows-10-10.0.19041-SP0)",
+        "x-ms-client-request-id": "8907585b-305e-11ec-9b7a-5cf37093a909",
+        "x-ms-date": "Mon, 18 Oct 2021 21:58:32 GMT",
+        "x-ms-version": "2019-02-02"
+      },
+      "RequestBody": "{\u0022TableName\u0022: \u0022uttablecee1273f\u0022}",
+      "StatusCode": 201,
+      "ResponseHeaders": {
+        "Cache-Control": "no-cache",
+        "Content-Type": "application/json; odata=minimalmetadata; streaming=true; charset=utf-8",
+        "Date": "Mon, 18 Oct 2021 21:58:26 GMT",
+        "Location": "https://fakeendpoint.table.core.windows.net/Tables(\u0027uttablecee1273f\u0027)",
+        "Server": [
+          "Windows-Azure-Table/1.0",
+          "Microsoft-HTTPAPI/2.0"
+        ],
+        "Transfer-Encoding": "chunked",
+        "X-Content-Type-Options": "nosniff",
+        "x-ms-client-request-id": "8907585b-305e-11ec-9b7a-5cf37093a909",
+        "x-ms-request-id": "57626ea4-2002-0015-6e6b-c4666e000000",
+        "x-ms-version": "2019-02-02"
+      },
+      "ResponseBody": {
+        "odata.metadata": "https://fakeendpoint.table.core.windows.net/$metadata#Tables/@Element",
+        "TableName": "uttablecee1273f"
+      }
+    },
+    {
+      "RequestUri": "https://fakeendpoint.table.core.windows.net/Tables",
+      "RequestMethod": "POST",
+      "RequestHeaders": {
+        "Accept": "application/json;odata=minimalmetadata",
+        "Accept-Encoding": "gzip, deflate",
+        "Authorization": "Sanitized",
+        "Connection": "keep-alive",
+        "Content-Length": "32",
+        "Content-Type": "application/json;odata=nometadata",
+        "DataServiceVersion": "3.0",
+        "Date": "Mon, 18 Oct 2021 21:58:32 GMT",
+        "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.2 (Windows-10-10.0.19041-SP0)",
+        "x-ms-client-request-id": "8907585b-305e-11ec-9b7a-5cf37093a909",
+        "x-ms-date": "Mon, 18 Oct 2021 21:58:32 GMT",
+        "x-ms-version": "2019-02-02"
+      },
+      "RequestBody": "{\u0022TableName\u0022: \u0022uttablecee1273f\u0022}",
+      "StatusCode": 409,
+      "ResponseHeaders": {
+        "Cache-Control": "no-cache",
+        "Content-Type": "application/json; odata=minimalmetadata; streaming=true; charset=utf-8",
+        "Date": "Mon, 18 Oct 2021 21:58:26 GMT",
+        "Server": [
+          "Windows-Azure-Table/1.0",
+          "Microsoft-HTTPAPI/2.0"
+        ],
+        "Transfer-Encoding": "chunked",
+        "X-Content-Type-Options": "nosniff",
+        "x-ms-client-request-id": "8907585b-305e-11ec-9b7a-5cf37093a909",
+        "x-ms-request-id": "57626eaf-2002-0015-776b-c4666e000000",
+        "x-ms-version": "2019-02-02"
+      },
+      "ResponseBody": {
+        "odata.error": {
+          "code": "TableAlreadyExists",
+          "message": {
+            "lang": "en-US",
+            "value": "The table specified already exists.\nRequestId:57626eaf-2002-0015-776b-c4666e000000\nTime:2021-10-18T21:58:27.3295071Z"
+          }
+        }
+      }
+    },
+    {
+      "RequestUri": "https://fakeendpoint.table.core.windows.net/Tables(\u0027uttablecee1273f\u0027)",
+      "RequestMethod": "DELETE",
+      "RequestHeaders": {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip, deflate",
+        "Authorization": "Sanitized",
+        "Connection": "keep-alive",
+        "Content-Length": "0",
+        "Date": "Mon, 18 Oct 2021 21:58:33 GMT",
+        "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.2 (Windows-10-10.0.19041-SP0)",
+        "x-ms-client-request-id": "8957e637-305e-11ec-9a4f-5cf37093a909",
+        "x-ms-date": "Mon, 18 Oct 2021 21:58:33 GMT",
+        "x-ms-version": "2019-02-02"
+      },
+      "RequestBody": null,
+      "StatusCode": 204,
+      "ResponseHeaders": {
+        "Cache-Control": "no-cache",
+        "Content-Length": "0",
+        "Date": "Mon, 18 Oct 2021 21:58:26 GMT",
+        "Server": [
+          "Windows-Azure-Table/1.0",
+          "Microsoft-HTTPAPI/2.0"
+        ],
+        "X-Content-Type-Options": "nosniff",
+        "x-ms-client-request-id": "8957e637-305e-11ec-9a4f-5cf37093a909",
+        "x-ms-request-id": "57626eb2-2002-0015-7a6b-c4666e000000",
+        "x-ms-version": "2019-02-02"
+      },
+      "ResponseBody": null
+    },
+    {
+      "RequestUri": "https://fakeendpoint.table.core.windows.net/Tables(\u0027uttablecee1273f\u0027)",
+      "RequestMethod": "DELETE",
+      "RequestHeaders": {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip, deflate",
+        "Authorization": "Sanitized",
+        "Connection": "keep-alive",
+        "Content-Length": "0",
+        "Date": "Mon, 18 Oct 2021 21:58:33 GMT",
+        "User-Agent": "azsdk-python-data-tables/12.1.1 Python/3.9.2 (Windows-10-10.0.19041-SP0)",
+        "x-ms-client-request-id": "895b8dc4-305e-11ec-9ce7-5cf37093a909",
+        "x-ms-date": "Mon, 18 Oct 2021 21:58:33 GMT",
+        "x-ms-version": "2019-02-02"
+      },
+      "RequestBody": null,
+      "StatusCode": 404,
+      "ResponseHeaders": {
+        "Cache-Control": "no-cache",
+        "Content-Type": "application/json; odata=minimalmetadata; streaming=true; charset=utf-8",
+        "Date": "Mon, 18 Oct 2021 21:58:26 GMT",
+        "Server": [
+          "Windows-Azure-Table/1.0",
+          "Microsoft-HTTPAPI/2.0"
+        ],
+        "Transfer-Encoding": "chunked",
+        "X-Content-Type-Options": "nosniff",
+        "x-ms-client-request-id": "895b8dc4-305e-11ec-9ce7-5cf37093a909",
+        "x-ms-request-id": "57626eb7-2002-0015-7e6b-c4666e000000",
+        "x-ms-version": "2019-02-02"
+      },
+      "ResponseBody": {
+        "odata.error": {
+          "code": "ResourceNotFound",
+          "message": {
+            "lang": "en-US",
+            "value": "The specified resource does not exist.\nRequestId:57626eb7-2002-0015-7e6b-c4666e000000\nTime:2021-10-18T21:58:27.3745385Z"
+          }
+        }
+      }
+    }
+  ],
+  "Variables": {}
+}