From 544807ba79f7e0c27f665761a3962c113601ec19 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Mon, 16 Dec 2019 10:56:32 +0100 Subject: [PATCH 1/2] env_aws: Disable Retries and set Session cfg Previously, Nomad used hand rolled HTTP requests to interact with the EC2 metadata API. Recently however, we switched to using the AWS SDK for this fingerprinting. The default behaviour of the AWS SDK is to perform retries with exponential backoff when a request fails. This is problematic for Nomad, because interacting with the EC2 API is in our client start path. Here we revert to our pre-existing behaviour of not performing retries in the fast path, as if the metadata service is unavailable, it's likely that nomad is not running in AWS. --- client/fingerprint/env_aws.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/client/fingerprint/env_aws.go b/client/fingerprint/env_aws.go index f468af39b3a..55163ff03cd 100644 --- a/client/fingerprint/env_aws.go +++ b/client/fingerprint/env_aws.go @@ -75,7 +75,10 @@ func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *F timeout = 1 * time.Millisecond } - ec2meta := ec2MetaClient(f.endpoint, timeout) + ec2meta, err := ec2MetaClient(f.endpoint, timeout) + if err != nil { + return fmt.Errorf("failed to setup ec2 client: %v", err) + } if !ec2meta.Available() { return nil @@ -191,15 +194,20 @@ func (f *EnvAWSFingerprint) linkSpeed(ec2meta *ec2metadata.EC2Metadata) int { return netSpeed } -func ec2MetaClient(endpoint string, timeout time.Duration) *ec2metadata.EC2Metadata { +func ec2MetaClient(endpoint string, timeout time.Duration) (*ec2metadata.EC2Metadata, error) { client := &http.Client{ Timeout: timeout, Transport: cleanhttp.DefaultTransport(), } - c := aws.NewConfig().WithHTTPClient(client) + c := aws.NewConfig().WithHTTPClient(client).WithMaxRetries(0) if endpoint != "" { c = c.WithEndpoint(endpoint) } - return ec2metadata.New(session.New(), c) + + session, err := session.NewSession(c) + if err != nil { + return nil, err + } + return ec2metadata.New(session, c), nil } From 59eb882197f1177e5b09f5aeb30b90bfd1709b66 Mon Sep 17 00:00:00 2001 From: Danielle Date: Mon, 16 Dec 2019 14:48:52 +0100 Subject: [PATCH 2/2] Update client/fingerprint/env_aws.go Co-Authored-By: Mahmood Ali --- client/fingerprint/env_aws.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/fingerprint/env_aws.go b/client/fingerprint/env_aws.go index 55163ff03cd..fdd46f5841b 100644 --- a/client/fingerprint/env_aws.go +++ b/client/fingerprint/env_aws.go @@ -77,7 +77,7 @@ func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *F ec2meta, err := ec2MetaClient(f.endpoint, timeout) if err != nil { - return fmt.Errorf("failed to setup ec2 client: %v", err) + return fmt.Errorf("failed to setup ec2Metadata client: %v", err) } if !ec2meta.Available() {