aws: Add default HTTP client instead of http.DefaultClient/Transport

Adds a new BuildableHTTPClient type to the SDK's aws package. The type
uses the builder pattern with immutable changes. Modifications to the
buildable client create copies of the client.

Adds a HTTPClient interface to the aws package that the SDK will use as
an abstraction over the specific HTTP client implementation. The SDK
will default to the BuildableHTTPClient, but a *http.Client can be also
provided for custom configuration.

When the SDK's aws.Config.HTTPClient value is a BuildableHTTPClient the
SDK will be able to use API client specific request timeout options.

Fix aws#279
Fix aws#269

aws/client.go

@@ -1,14 +1,10 @@
 package aws
 
-import (
-	"net/http"
-)
-
 // Metadata wraps immutable data from the Client structure.
 type Metadata struct {
 	ServiceName string
 	ServiceID   string
-	EndpointsID  string
+	EndpointsID string
 	APIVersion  string
 
 	Endpoint      string
@@ -36,7 +32,7 @@ type Client struct {
 	LogLevel LogLevel
 	Logger   Logger
 
-	HTTPClient *http.Client
+	HTTPClient HTTPClient
 }
 
 // NewClient will return a pointer to a new initialized service client.

aws/config.go

@@ -1,9 +1,5 @@
 package aws
 
-import (
-	"net/http"
-)
-
 // A Config provides service configuration for service clients.
 type Config struct {
 	// The region to send requests to. This parameter is required and must
@@ -24,9 +20,13 @@ type Config struct {
 	// to use based on region.
 	EndpointResolver EndpointResolver
 
-	// The HTTP client to use when sending requests. Defaults to
-	// `http.DefaultClient`.
-	HTTPClient *http.Client
+	// The HTTP Client the SDK's API clients will use to invoke HTTP requests.
+	// The SDK defaults to a BuildableHTTPClient allowing API clients to create
+	// copies of the HTTP Client for service specific customizations.
+	//
+	// Use a (*http.Client) for custom behavior. Using a custom http.Client
+	// will prevent the SDK from modifying the HTTP client.
+	HTTPClient HTTPClient
 
 	// TODO document
 	Handlers Handlers

aws/defaults/defaults.go

@@ -9,9 +9,7 @@ package defaults
 
 import (
 	"log"
-	"net/http"
 	"os"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/aws/endpoints"
@@ -55,17 +53,8 @@ func Config() aws.Config {
 // HTTPClient will return a new HTTP Client configured for the SDK.
 //
 // Does not use http.DefaultClient nor http.DefaultTransport.
-func HTTPClient() *http.Client {
-	return &http.Client{
-		Transport: &http.Transport{
-			Proxy:                 http.ProxyFromEnvironment,
-			MaxIdleConns:          100,
-			MaxIdleConnsPerHost:   10,
-			IdleConnTimeout:       30 * time.Second,
-			TLSHandshakeTimeout:   10 * time.Second,
-			ExpectContinueTimeout: 5 * time.Second,
-		},
-	}
+func HTTPClient() aws.HTTPClient {
+	return aws.NewBuildableHTTPClient()
 }
 
 // Handlers returns the default request handlers.

aws/defaults/handlers.go

@@ -91,9 +91,16 @@ var ValidateReqSigHandler = aws.NamedHandler{
 var SendHandler = aws.NamedHandler{
 	Name: "core.SendHandler",
 	Fn: func(r *aws.Request) {
-		sender := sendFollowRedirects
+		sender := r.Config.HTTPClient.Do
+
 		if r.DisableFollowRedirects {
-			sender = sendWithoutFollowRedirects
+			switch tv := r.Config.HTTPClient.(type) {
+			case aws.HTTPClientDoWithoutFollower:
+				sender = tv.DoWithoutFollow
+			case *http.Client:
+				wrapper := aws.WrapHTTPClientDoWithoutFollow{Client: tv}
+				sender = wrapper.DoWithoutFollow
+			}
 		}
 
 		if aws.NoBody == r.HTTPRequest.Body {
@@ -113,26 +120,13 @@ var SendHandler = aws.NamedHandler{
 		}
 
 		var err error
-		r.HTTPResponse, err = sender(r)
+		r.HTTPResponse, err = sender(r.HTTPRequest)
 		if err != nil {
 			handleSendError(r, err)
 		}
 	},
 }
 
-func sendFollowRedirects(r *aws.Request) (*http.Response, error) {
-	return r.Config.HTTPClient.Do(r.HTTPRequest)
-}
-
-func sendWithoutFollowRedirects(r *aws.Request) (*http.Response, error) {
-	transport := r.Config.HTTPClient.Transport
-	if transport == nil {
-		transport = http.DefaultTransport
-	}
-
-	return transport.RoundTrip(r.HTTPRequest)
-}
-
 func handleSendError(r *aws.Request, err error) {
 	// Prevent leaking if an HTTPResponse was returned. Clean up
 	// the body.

aws/ec2metadata/api_client.go

@@ -11,9 +11,11 @@ import (
 	"bytes"
 	"errors"
 	"io"
+	"net"
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/aws/awserr"
@@ -34,13 +36,23 @@ type Client struct {
 //     // Create a Client client from just a config.
 //     svc := ec2metadata.New(cfg)
 func New(config aws.Config) *Client {
+	if c, ok := config.HTTPClient.(*aws.BuildableHTTPClient); ok {
+		// Use a custom Dial timeout for the EC2 Metadata service to account
+		// for the possibility the application might not be running in an
+		// environment with the service present. The client should fail fast in
+		// this case.
+		config.HTTPClient = c.WithDialerOptions(func(d *net.Dialer) {
+			d.Timeout = 5 * time.Second
+		})
+	}
+
 	svc := &Client{
 		Client: aws.NewClient(
 			config,
 			aws.Metadata{
 				ServiceName: "EC2 Instance Metadata",
 				ServiceID:   "EC2InstanceMetadata",
-				EndpointsID:  "ec2metadata",
+				EndpointsID: "ec2metadata",
 				APIVersion:  "latest",
 			},
 		),
@@ -72,10 +84,6 @@ func New(config aws.Config) *Client {
 	return svc
 }
 
-func httpClientZero(c *http.Client) bool {
-	return c == nil || (c.Transport == nil && c.CheckRedirect == nil && c.Jar == nil && c.Timeout == 0)
-}
-
 type metadataOutput struct {
 	Content string
 }

aws/external/http_client.go

@@ -5,38 +5,25 @@ import (
 	"crypto/x509"
 	"net/http"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/aws/awserr"
 )
 
-func addHTTPClientCABundle(client *http.Client, pemCerts []byte) error {
-	var t *http.Transport
+func addHTTPClientCABundle(client *aws.BuildableHTTPClient, pemCerts []byte) (*aws.BuildableHTTPClient, error) {
+	var appendErr error
 
-	switch v := client.Transport.(type) {
-	case *http.Transport:
-		t = v
-	default:
-		if client.Transport != nil {
-			return awserr.New("LoadCustomCABundleError",
-				"unable to set custom CA bundle trasnsport must be http.Transport type", nil)
+	client = client.WithTransportOptions(func(tr *http.Transport) {
+		if tr.TLSClientConfig == nil {
+			tr.TLSClientConfig = &tls.Config{}
 		}
-	}
-
-	if t == nil {
-		t = &http.Transport{}
-	}
-	if t.TLSClientConfig == nil {
-		t.TLSClientConfig = &tls.Config{}
-	}
-	if t.TLSClientConfig.RootCAs == nil {
-		t.TLSClientConfig.RootCAs = x509.NewCertPool()
-	}
-
-	if !t.TLSClientConfig.RootCAs.AppendCertsFromPEM(pemCerts) {
-		return awserr.New("LoadCustomCABundleError",
-			"failed to load custom CA bundle PEM file", nil)
-	}
-
-	client.Transport = t
+		if tr.TLSClientConfig.RootCAs == nil {
+			tr.TLSClientConfig.RootCAs = x509.NewCertPool()
+		}
+		if !tr.TLSClientConfig.RootCAs.AppendCertsFromPEM(pemCerts) {
+			appendErr = awserr.New("LoadCustomCABundleError",
+				"failed to load custom CA bundle PEM file", nil)
+		}
+	})
 
-	return nil
+	return client, appendErr
 }

aws/external/http_client_test.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/internal/awstesting"
 )
 
@@ -27,8 +28,8 @@ func TestAddHTTPClientCABundle(t *testing.T) {
 		t.Fatalf("failed to read CA file, %v", err)
 	}
 
-	client := &http.Client{}
-	err = addHTTPClientCABundle(client, caPEM)
+	client := aws.NewBuildableHTTPClient()
+	client, err = addHTTPClientCABundle(client, caPEM)
 	if err != nil {
 		t.Fatalf("failed to add CA PEM to HTTP client, %v", err)
 	}

aws/external/resolve.go

@@ -2,7 +2,6 @@ package external
 
 import (
 	"fmt"
-	"net/http"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -41,7 +40,13 @@ func ResolveCustomCABundle(cfg *aws.Config, configs Configs) error {
 		return nil
 	}
 
-	return addHTTPClientCABundle(cfg.HTTPClient, v)
+	client, ok := cfg.HTTPClient.(*aws.BuildableHTTPClient)
+	if !ok {
+		return fmt.Errorf("unable to add custom RootCAs aws.Config.HTTPClient is not a *BuildableHTTPClient")
+	}
+
+	cfg.HTTPClient, err = addHTTPClientCABundle(client, v)
+	return err
 }
 
 // ResolveRegion extracts the first instance of a Region from the Configs slice.
@@ -195,8 +200,6 @@ func ResolveAssumeRoleCredentials(cfg *aws.Config, configs Configs) error {
 // use EC2 Instance Role always.
 func ResolveFallbackEC2Credentials(cfg *aws.Config, configs Configs) error {
 	cfgCp := cfg.Copy()
-	cfgCp.HTTPClient = shallowCopyHTTPClient(cfgCp.HTTPClient)
-	cfgCp.HTTPClient.Timeout = 5 * time.Second
 
 	provider := ec2rolecreds.NewProvider(ec2metadata.New(cfgCp))
 	provider.ExpiryWindow = 5 * time.Minute
@@ -205,12 +208,3 @@ func ResolveFallbackEC2Credentials(cfg *aws.Config, configs Configs) error {
 
 	return nil
 }
-
-func shallowCopyHTTPClient(client *http.Client) *http.Client {
-	return &http.Client{
-		Transport:     client.Transport,
-		CheckRedirect: client.CheckRedirect,
-		Jar:           client.Jar,
-		Timeout:       client.Timeout,
-	}
-}

aws/external/resolve_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/defaults"
 	"github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds"
 	"github.com/aws/aws-sdk-go-v2/aws/endpointcreds"
 	"github.com/aws/aws-sdk-go-v2/aws/stscreds"
@@ -19,20 +20,31 @@ func TestResolveCustomCABundle(t *testing.T) {
 		WithCustomCABundle(awstesting.TLSBundleCA),
 	}
 
-	cfg := aws.Config{
-		HTTPClient: &http.Client{Transport: &http.Transport{}},
-	}
-
+	cfg := defaults.Config()
 	if err := ResolveCustomCABundle(&cfg, configs); err != nil {
 		t.Fatalf("expect no error, got %v", err)
 	}
 
-	transport := cfg.HTTPClient.Transport.(*http.Transport)
+	client := cfg.HTTPClient.(*aws.BuildableHTTPClient).BuildHTTPClient()
+	transport := client.(*http.Client).Transport.(*http.Transport)
 	if transport.TLSClientConfig.RootCAs == nil {
 		t.Errorf("expect root CAs set")
 	}
 }
 
+func TestResolveCustomCABundle_ErrorCustomClient(t *testing.T) {
+	configs := Configs{
+		WithCustomCABundle(awstesting.TLSBundleCA),
+	}
+
+	cfg := aws.Config{
+		HTTPClient: &http.Client{},
+	}
+	if err := ResolveCustomCABundle(&cfg, configs); err == nil {
+		t.Fatalf("expect error, got none")
+	}
+}
+
 func TestResolveRegion(t *testing.T) {
 	configs := Configs{
 		WithRegion("mock-region"),