From b5f1ddefa7a24cebed52d8559d78aed65d241dd9 Mon Sep 17 00:00:00 2001 From: Zeleena Kearney Date: Fri, 1 Oct 2021 17:26:35 -0500 Subject: [PATCH] internal/resource: support S3 access point URLs Support S3 access point URLs in ARN format as a source. This allows valid, opaque S3 URLs such as `s3:arn:aws:s3:us-west-2:123456789012:accesspoint/test/object` Being able to use this format will allow S3 URLs on different partitions and lays the foundation to potentially support multi-region access points in the future. Fixes https://github.com/coreos/ignition/issues/1091 Signed-off-by: Zeleena Kearney --- config/shared/errors/errors.go | 1 + config/v3_4_experimental/types/url.go | 19 ++++++ config/v3_4_experimental/types/url_test.go | 32 +++++++++ docs/configuration-v3_4_experimental.md | 12 ++-- docs/supported-platforms.md | 2 +- internal/resource/url.go | 75 ++++++++++++++++++---- internal/resource/url_test.go | 14 ++++ 7 files changed, 135 insertions(+), 20 deletions(-) diff --git a/config/shared/errors/errors.go b/config/shared/errors/errors.go index 7761280d07..68340096fa 100644 --- a/config/shared/errors/errors.go +++ b/config/shared/errors/errors.go @@ -99,6 +99,7 @@ var ( ErrEngineConfiguration = errors.New("engine incorrectly configured") // AWS S3 specific errors + ErrInvalidS3ARN = errors.New("invalid S3 ARN format") ErrInvalidS3ObjectVersionId = errors.New("invalid S3 object VersionId") // Obsolete errors, left here for ABI compatibility diff --git a/config/v3_4_experimental/types/url.go b/config/v3_4_experimental/types/url.go index 0d8771bf6d..9f81bd6851 100644 --- a/config/v3_4_experimental/types/url.go +++ b/config/v3_4_experimental/types/url.go @@ -17,6 +17,7 @@ package types import ( "net/url" + "github.com/aws/aws-sdk-go/aws/arn" "github.com/vincent-petithory/dataurl" "github.com/coreos/ignition/v2/config/shared/errors" @@ -39,6 +40,24 @@ func validateURL(s string) error { } } return nil + case "arn": + fullURL := u.Scheme + ":" + u.Opaque + if !arn.IsARN(fullURL) { + return errors.ErrInvalidS3ARN + } + s3arn, err := arn.Parse(fullURL) + if err != nil { + return err + } + if s3arn.Service != "s3" { + return errors.ErrInvalidS3ARN + } + if v, ok := u.Query()["versionId"]; ok { + if len(v) == 0 || v[0] == "" { + return errors.ErrInvalidS3ObjectVersionId + } + } + return nil case "data": if _, err := dataurl.DecodeString(s); err != nil { return err diff --git a/config/v3_4_experimental/types/url_test.go b/config/v3_4_experimental/types/url_test.go index bfef261fe2..171a4e3f6e 100644 --- a/config/v3_4_experimental/types/url_test.go +++ b/config/v3_4_experimental/types/url_test.go @@ -66,6 +66,38 @@ func TestURLValidate(t *testing.T) { util.StrToPtr("s3://bucket/key?versionId=aVersionHash"), nil, }, + { + util.StrToPtr("Arn:"), + errors.ErrInvalidS3ARN, + }, + { + util.StrToPtr("arn:aws:iam:us-west-2:123456789012:resource"), + errors.ErrInvalidS3ARN, + }, + { + util.StrToPtr("arn:aws:s3:us-west-2:123456789012:bucket-name/object-key"), + nil, + }, + { + util.StrToPtr("arn:aws:s3:us-west-2:123456789012:bucket-name/object-key?versionId=aVersionHash"), + nil, + }, + { + util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/test/object"), + nil, + }, + { + util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/test/object?versionId=aVersionHash"), + nil, + }, + { + util.StrToPtr("arn:aws:s3:::bucket-name/object-key"), + nil, + }, + { + util.StrToPtr("arn:aws:s3:::bucket-name/object-key?versionId=aVersionHash"), + nil, + }, { util.StrToPtr("gs://bucket/object"), nil, diff --git a/docs/configuration-v3_4_experimental.md b/docs/configuration-v3_4_experimental.md index 1520e9605b..986570c45d 100644 --- a/docs/configuration-v3_4_experimental.md +++ b/docs/configuration-v3_4_experimental.md @@ -14,7 +14,7 @@ The Ignition configuration is a JSON document conforming to the following specif * **version** (string): the semantic version number of the spec. The spec version must be compatible with the latest version (`3.4.0-experimental`). Compatibility requires the major versions to match and the spec version be less than or equal to the latest version. `-experimental` versions compare less than the final version with the same number, and previous experimental versions are not accepted. * **_config_** (objects): options related to the configuration. * **_merge_** (list of objects): a list of the configs to be merged to the current config. - * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. * **_compression_** (string): the type of compression used on the config (null or gzip). Compression cannot be used with S3. * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only. * **name** (string): the header name. @@ -22,7 +22,7 @@ The Ignition configuration is a JSON document conforming to the following specif * **_verification_** (object): options related to the verification of the config. * **_hash_** (string): the hash of the config, in the form `-` where type is either `sha512` or `sha256`. * **_replace_** (object): the config that will replace the current. - * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. * **_compression_** (string): the type of compression used on the config (null or gzip). Compression cannot be used with S3. * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only. * **name** (string): the header name. @@ -35,7 +35,7 @@ The Ignition configuration is a JSON document conforming to the following specif * **_security_** (object): options relating to network security. * **_tls_** (object): options relating to TLS when fetching resources over `https`. * **_certificateAuthorities_** (list of objects): the list of additional certificate authorities (in addition to the system authorities) to be used for TLS verification when fetching over `https`. All certificate authorities must have a unique `source`. - * **source** (string): the URL of the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **source** (string): the URL of the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. * **_compression_** (string): the type of compression used on the certificate (null or gzip). Compression cannot be used with S3. * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only. * **name** (string): the header name. @@ -80,7 +80,7 @@ The Ignition configuration is a JSON document conforming to the following specif * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. `contents.source` must be specified if `overwrite` is true. Defaults to false. * **_contents_** (object): options related to the contents of the file. * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3. - * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created. + * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created. * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only. * **name** (string): the header name. * **_value_** (string): the header contents. @@ -88,7 +88,7 @@ The Ignition configuration is a JSON document conforming to the following specif * **_hash_** (string): the hash of the contents, in the form `-` where type is either `sha512` or `sha256`. * **_append_** (list of objects): list of contents to be appended to the file. Follows the same stucture as `contents` * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3. - * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only. * **name** (string): the header name. * **_value_** (string): the header contents. @@ -127,7 +127,7 @@ The Ignition configuration is a JSON document conforming to the following specif * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks. * **_keyFile_** (string): options related to the contents of the key file. * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3. - * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. + * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only. * **name** (string): the header name. * **_value_** (string): the header contents. diff --git a/docs/supported-platforms.md b/docs/supported-platforms.md index ee6a202596..bd20400351 100644 --- a/docs/supported-platforms.md +++ b/docs/supported-platforms.md @@ -16,7 +16,7 @@ Ignition is currently only supported for the following platforms: * [Exoscale] (`exoscale`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately. * [Google Cloud] (`gcp`) - Ignition will read its configuration from the instance metadata entry named "user-data". Cloud SSH keys are handled separately. * [IBM Cloud] (`ibmcloud`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately. -* Bare Metal (`metal`) - Use the `ignition.config.url` kernel parameter to provide a URL to the configuration. The URL can use the `http://`, `https://`, `tftp://`, `s3://`, or `gs://` schemes to specify a remote config. +* Bare Metal (`metal`) - Use the `ignition.config.url` kernel parameter to provide a URL to the configuration. The URL can use the `http://`, `https://`, `tftp://`, `s3://`, `arn:`, or `gs://` schemes to specify a remote config. * [Nutanix] (`nutanix`) - Ignition will read its configuration from the instance userdata via config drive. Cloud SSH keys are handled separately. * [OpenStack] (`openstack`) - Ignition will read its configuration from the instance userdata via either metadata service or config drive. Cloud SSH keys are handled separately. * [Equinix Metal] (`packet`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately. diff --git a/internal/resource/url.go b/internal/resource/url.go index 834236735d..c382b75e04 100644 --- a/internal/resource/url.go +++ b/internal/resource/url.go @@ -39,6 +39,7 @@ import ( "google.golang.org/api/option" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -135,7 +136,7 @@ func (f *Fetcher) FetchToBuffer(u url.URL, opts FetchOptions) ([]byte, error) { err = f.fetchFromTFTP(u, dest, opts) case "data": err = f.fetchFromDataURL(u, dest, opts) - case "s3": + case "s3", "arn": buf := &s3buf{ WriteAtBuffer: aws.NewWriteAtBuffer([]byte{}), } @@ -196,7 +197,7 @@ func (f *Fetcher) Fetch(u url.URL, dest *os.File, opts FetchOptions) error { return f.fetchFromTFTP(u, dest, opts) case "data": return f.fetchFromDataURL(u, dest, opts) - case "s3": + case "s3", "arn": return f.fetchFromS3(u, dest, opts) case "gs": return f.fetchFromGCS(u, dest, opts) @@ -407,17 +408,65 @@ func (f *Fetcher) fetchFromS3(u url.URL, dest s3target, opts FetchOptions) error } sess := f.AWSSession.Copy() - // Determine the partition and region this bucket is in - regionHint := "us-east-1" - if f.S3RegionHint != "" { - regionHint = f.S3RegionHint + // Determine the bucket and key based on the URL scheme + var bucket, key string + var s3arn arn.ARN + isAccessPoint := false + switch u.Scheme { + case "s3": + bucket = u.Host + key = u.Path + case "arn": + fullURL := u.Scheme + ":" + u.Opaque + if !arn.IsARN(fullURL) { + return configErrors.ErrInvalidS3ARN + } + s3arn, err := arn.Parse(fullURL) + if err != nil { + return err + } + if s3arn.Service != "s3" { + return configErrors.ErrInvalidS3ARN + } + + // Determine if the ARN is for an access point or a bucket. + urlSplit := strings.Split(fullURL, "/") + if strings.HasPrefix(s3arn.Resource, "accesspoint/") { + isAccessPoint = true + // When using GetObjectInput with an access point, + // you provide the access point ARN in place of the bucket name. + // For more information about access point ARNs, see Using access points + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html + bucket = strings.Join(urlSplit[:2], "/") + key = strings.Join(urlSplit[2:], "/") + } else { + // Use the full ARN as the bucket name. + // If specified, the key is part of the Relative ID which has the format "bucket-name/object-key" according to + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html + bucket = urlSplit[0] + key = strings.Join(urlSplit[1:], "/") + } + default: + return ErrSchemeUnsupported } - region, err := s3manager.GetBucketRegion(ctx, sess, u.Host, regionHint) - if err != nil { - if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" { - return fmt.Errorf("couldn't determine the region for bucket %q: %v", u.Host, err) + + // Determine the partition and region this bucket is in + var region string + var err error + if isAccessPoint { + region = s3arn.Region + } else { + regionHint := "us-east-1" + if f.S3RegionHint != "" { + regionHint = f.S3RegionHint + } + region, err = s3manager.GetBucketRegion(ctx, sess, bucket, regionHint) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" { + return fmt.Errorf("couldn't determine the region for bucket %q: %v", u.Host, err) + } + return err } - return err } sess.Config.Region = aws.String(region) @@ -428,8 +477,8 @@ func (f *Fetcher) fetchFromS3(u url.URL, dest s3target, opts FetchOptions) error } input := &s3.GetObjectInput{ - Bucket: &u.Host, - Key: &u.Path, + Bucket: &bucket, + Key: &key, VersionId: versionId, } err = f.fetchFromS3WithCreds(ctx, dest, input, sess) diff --git a/internal/resource/url_test.go b/internal/resource/url_test.go index 5701180d82..8f37e2a4c1 100644 --- a/internal/resource/url_test.go +++ b/internal/resource/url_test.go @@ -194,6 +194,20 @@ func TestFetchOffline(t *testing.T) { }, out: out{err: ErrNeedNet}, }, + // arn url specifying bucket + { + in: in{ + url: "arn:aws:s3:us-west-2:123456789012:bucket-name/object-key", + }, + out: out{err: ErrNeedNet}, + }, + // arn url specifying s3 access point + { + in: in{ + url: "arn:aws:s3:us-west-2:123456789012:accesspoint/test/object", + }, + out: out{err: ErrNeedNet}, + }, // gs url { in: in{