Add support for retrying unstructured 429 / 500 / 502 responses. (#6011)

Closes #5918.

bigquery/google/cloud/bigquery/retry.py

@@ -13,16 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from google.api_core import exceptions
 from google.api_core import retry
 
 
 _RETRYABLE_REASONS = frozenset([
-    'backendError',
     'rateLimitExceeded',
+    'backendError',
     'internalError',
     'badGateway',
 ])
 
+_UNSTRUCTURED_RETRYABLE_TYPES = (
+    exceptions.TooManyRequests,
+    exceptions.InternalServerError,
+    exceptions.BadGateway,
+)
+
 
 def _should_retry(exc):
     """Predicate for determining when to retry.
@@ -32,8 +39,11 @@ def _should_retry(exc):
     """
     if not hasattr(exc, 'errors'):
         return False
+
     if len(exc.errors) == 0:
-        return False
+        # Check for unstructured error returns, e.g. from GFE
+        return isinstance(exc, _UNSTRUCTURED_RETRYABLE_TYPES)
+
     reason = exc.errors[0]['reason']
     return reason in _RETRYABLE_REASONS
 

bigquery/tests/unit/test_retry.py

@@ -47,12 +47,30 @@ def test_w_rateLimitExceeded(self):
             errors=[{'reason': 'rateLimitExceeded'}], spec=['errors'])
         self.assertTrue(self._call_fut(exc))
 
+    def test_w_unstructured_too_many_requests(self):
+        from google.api_core.exceptions import TooManyRequests
+
+        exc = TooManyRequests('testing')
+        self.assertTrue(self._call_fut(exc))
+
     def test_w_internalError(self):
         exc = mock.Mock(
             errors=[{'reason': 'internalError'}], spec=['errors'])
         self.assertTrue(self._call_fut(exc))
 
+    def test_w_unstructured_internal_server_error(self):
+        from google.api_core.exceptions import InternalServerError
+
+        exc = InternalServerError('testing')
+        self.assertTrue(self._call_fut(exc))
+
     def test_w_badGateway(self):
         exc = mock.Mock(
             errors=[{'reason': 'badGateway'}], spec=['errors'])
         self.assertTrue(self._call_fut(exc))
+
+    def test_w_unstructured_bad_gateway(self):
+        from google.api_core.exceptions import BadGateway
+
+        exc = BadGateway('testing')
+        self.assertTrue(self._call_fut(exc))