cloud: add stream INTERNAL_ERROR to resumable HTTP errors

Currently, errors like
`stream error: stream ID <x>; INTERNAL_ERROR; received from peer`
are not being retried. Retry these errors assuggested by:

googleapis/google-cloud-go#3735
googleapis/google-cloud-go#784

Fixes: cockroachdb#85217, cockroachdb#85216, cockroachdb#85204, cockroachdb#84162

Release note: None

pkg/cloud/BUILD.bazel

@@ -31,5 +31,6 @@ go_library(
         "//pkg/util/tracing",
         "@com_github_cockroachdb_errors//:errors",
         "@com_github_stretchr_testify//require",
+        "@org_golang_x_net//http2",
     ],
 )

pkg/cloud/cloud_io.go

@@ -32,6 +32,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/sysutil"
 	"github.com/cockroachdb/cockroach/pkg/util/tracing"
 	"github.com/cockroachdb/errors"
+	"golang.org/x/net/http2"
 )
 
 // Timeout is a cluster setting used for cloud storage interactions.
@@ -147,10 +148,26 @@ func DelayedRetry(
 // In addition, we treat connection reset by peer errors (which can
 // happen if we didn't read from the connection too long due to e.g. load),
 // the same as unexpected eof errors.
+// We also retry on http2.StreamError if the error code is
+// http2.ErrCodeInternal.
 func IsResumableHTTPError(err error) bool {
-	return errors.Is(err, io.ErrUnexpectedEOF) ||
+	if errors.Is(err, io.ErrUnexpectedEOF) ||
 		sysutil.IsErrConnectionReset(err) ||
-		sysutil.IsErrConnectionRefused(err)
+		sysutil.IsErrConnectionRefused(err) {
+		return true
+	}
+
+	if e := (http2.StreamError{}); errors.As(err, &e) {
+		if e.Code == http2.ErrCodeInternal {
+			return true
+		}
+	}
+
+	if e := (errors.Wrapper)(nil); errors.As(err, &e) {
+		return IsResumableHTTPError(e.Unwrap())
+	}
+
+	return false
 }
 
 // Maximum number of times we can attempt to retry reading from external storage,