internal/resource: support S3 access point URLs

Support S3 access point URLs in ARN format as a source.
This allows valid, opaque S3 URLs such as
`s3:arn:aws:s3:us-west-2:123456789012:accesspoint/test/object`
Being able to use this format will allow S3 URLs on different
partitions and lays the foundation to potentially support
multi-region access points in the future.

Fixes #1091
Signed-off-by: Zeleena Kearney <[email protected]>

config/shared/errors/errors.go

@@ -99,6 +99,7 @@ var (
 	ErrEngineConfiguration             = errors.New("engine incorrectly configured")
 
 	// AWS S3 specific errors
+	ErrInvalidS3ARN             = errors.New("invalid S3 ARN format")
 	ErrInvalidS3ObjectVersionId = errors.New("invalid S3 object VersionId")
 
 	// Obsolete errors, left here for ABI compatibility

config/v3_4_experimental/types/url.go

@@ -17,6 +17,7 @@ package types
 import (
 	"net/url"
 
+	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/vincent-petithory/dataurl"
 
 	"github.com/coreos/ignition/v2/config/shared/errors"
@@ -39,6 +40,24 @@ func validateURL(s string) error {
 			}
 		}
 		return nil
+	case "arn":
+		fullURL := u.Scheme + ":" + u.Opaque
+		if !arn.IsARN(fullURL) {
+			return errors.ErrInvalidS3ARN
+		}
+		s3arn, err := arn.Parse(fullURL)
+		if err != nil {
+			return err
+		}
+		if s3arn.Service != "s3" {
+			return errors.ErrInvalidS3ARN
+		}
+		if v, ok := u.Query()["versionId"]; ok {
+			if len(v) == 0 || v[0] == "" {
+				return errors.ErrInvalidS3ObjectVersionId
+			}
+		}
+		return nil
 	case "data":
 		if _, err := dataurl.DecodeString(s); err != nil {
 			return err

config/v3_4_experimental/types/url_test.go

@@ -66,6 +66,38 @@ func TestURLValidate(t *testing.T) {
 			util.StrToPtr("s3://bucket/key?versionId=aVersionHash"),
 			nil,
 		},
+		{
+			util.StrToPtr("Arn:"),
+			errors.ErrInvalidS3ARN,
+		},
+		{
+			util.StrToPtr("arn:aws:iam:us-west-2:123456789012:resource"),
+			errors.ErrInvalidS3ARN,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:bucket-name/object-key"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:bucket-name/object-key?versionId=aVersionHash"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/test/object"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/test/object?versionId=aVersionHash"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:::bucket-name/object-key"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:::bucket-name/object-key?versionId=aVersionHash"),
+			nil,
+		},
 		{
 			util.StrToPtr("gs://bucket/object"),
 			nil,

docs/configuration-v3_4_experimental.md

@@ -14,15 +14,15 @@ The Ignition configuration is a JSON document conforming to the following specif
   * **version** (string): the semantic version number of the spec. The spec version must be compatible with the latest version (`3.4.0-experimental`). Compatibility requires the major versions to match and the spec version be less than or equal to the latest version. `-experimental` versions compare less than the final version with the same number, and previous experimental versions are not accepted.
   * **_config_** (objects): options related to the configuration.
     * **_merge_** (list of objects): a list of the configs to be merged to the current config.
-      * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+      * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
       * **_compression_** (string): the type of compression used on the config (null or gzip). Compression cannot be used with S3.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
         * **_value_** (string): the header contents.
       * **_verification_** (object): options related to the verification of the config.
         * **_hash_** (string): the hash of the config, in the form `<type>-<value>` where type is either `sha512` or `sha256`.
     * **_replace_** (object): the config that will replace the current.
-      * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+      * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
       * **_compression_** (string): the type of compression used on the config (null or gzip). Compression cannot be used with S3.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
@@ -35,7 +35,7 @@ The Ignition configuration is a JSON document conforming to the following specif
   * **_security_** (object): options relating to network security.
     * **_tls_** (object): options relating to TLS when fetching resources over `https`.
       * **_certificateAuthorities_** (list of objects): the list of additional certificate authorities (in addition to the system authorities) to be used for TLS verification when fetching over `https`. All certificate authorities must have a unique `source`.
-        * **source** (string): the URL of the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+        * **source** (string): the URL of the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
         * **_compression_** (string): the type of compression used on the certificate (null or gzip). Compression cannot be used with S3.
         * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
           * **name** (string): the header name.
@@ -80,15 +80,15 @@ The Ignition configuration is a JSON document conforming to the following specif
     * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. `contents.source` must be specified if `overwrite` is true. Defaults to false.
     * **_contents_** (object): options related to the contents of the file.
       * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3.
-      * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created.
+      * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
         * **_value_** (string): the header contents.
       * **_verification_** (object): options related to the verification of the file contents.
         * **_hash_** (string): the hash of the contents, in the form `<type>-<value>` where type is either `sha512` or `sha256`.
     * **_append_** (list of objects): list of contents to be appended to the file. Follows the same stucture as `contents`
       * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3.
-      * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+      * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
         * **_value_** (string): the header contents.
@@ -127,7 +127,7 @@ The Ignition configuration is a JSON document conforming to the following specif
     * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks.
     * **_keyFile_** (string): options related to the contents of the key file.
       * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3.
-      * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+      * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
         * **_value_** (string): the header contents.

docs/supported-platforms.md

@@ -16,7 +16,7 @@ Ignition is currently only supported for the following platforms:
 * [Exoscale] (`exoscale`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately.
 * [Google Cloud] (`gcp`) - Ignition will read its configuration from the instance metadata entry named "user-data". Cloud SSH keys are handled separately.
 * [IBM Cloud] (`ibmcloud`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately.
-* Bare Metal (`metal`) - Use the `ignition.config.url` kernel parameter to provide a URL to the configuration. The URL can use the `http://`, `https://`, `tftp://`, `s3://`, or `gs://` schemes to specify a remote config.
+* Bare Metal (`metal`) - Use the `ignition.config.url` kernel parameter to provide a URL to the configuration. The URL can use the `http://`, `https://`, `tftp://`, `s3://`, `arn:`, or `gs://` schemes to specify a remote config.
 * [Nutanix] (`nutanix`) - Ignition will read its configuration from the instance userdata via config drive. Cloud SSH keys are handled separately.
 * [OpenStack] (`openstack`) - Ignition will read its configuration from the instance userdata via either metadata service or config drive. Cloud SSH keys are handled separately.
 * [Equinix Metal] (`packet`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately.

internal/resource/url.go

@@ -39,6 +39,7 @@ import (
 	"google.golang.org/api/option"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -135,7 +136,7 @@ func (f *Fetcher) FetchToBuffer(u url.URL, opts FetchOptions) ([]byte, error) {
 		err = f.fetchFromTFTP(u, dest, opts)
 	case "data":
 		err = f.fetchFromDataURL(u, dest, opts)
-	case "s3":
+	case "s3", "arn":
 		buf := &s3buf{
 			WriteAtBuffer: aws.NewWriteAtBuffer([]byte{}),
 		}
@@ -196,7 +197,7 @@ func (f *Fetcher) Fetch(u url.URL, dest *os.File, opts FetchOptions) error {
 		return f.fetchFromTFTP(u, dest, opts)
 	case "data":
 		return f.fetchFromDataURL(u, dest, opts)
-	case "s3":
+	case "s3", "arn":
 		return f.fetchFromS3(u, dest, opts)
 	case "gs":
 		return f.fetchFromGCS(u, dest, opts)
@@ -407,17 +408,65 @@ func (f *Fetcher) fetchFromS3(u url.URL, dest s3target, opts FetchOptions) error
 	}
 	sess := f.AWSSession.Copy()
 
-	// Determine the partition and region this bucket is in
-	regionHint := "us-east-1"
-	if f.S3RegionHint != "" {
-		regionHint = f.S3RegionHint
+	// Determine the bucket and key based on the URL scheme
+	var bucket, key string
+	var s3arn arn.ARN
+	isAccessPoint := false
+	switch u.Scheme {
+	case "s3":
+		bucket = u.Host
+		key = u.Path
+	case "arn":
+		fullURL := u.Scheme + ":" + u.Opaque
+		if !arn.IsARN(fullURL) {
+			return configErrors.ErrInvalidS3ARN
+		}
+		s3arn, err := arn.Parse(fullURL)
+		if err != nil {
+			return err
+		}
+		if s3arn.Service != "s3" {
+			return configErrors.ErrInvalidS3ARN
+		}
+
+		// Determine if the ARN is for an access point or a bucket.
+		urlSplit := strings.Split(fullURL, "/")
+		if strings.HasPrefix(s3arn.Resource, "accesspoint/") {
+			isAccessPoint = true
+			// When using GetObjectInput with an access point,
+			// you provide the access point ARN in place of the bucket name.
+			// For more information about access point ARNs, see Using access points
+			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html
+			bucket = strings.Join(urlSplit[:2], "/")
+			key = strings.Join(urlSplit[2:], "/")
+		} else {
+			// Use the full ARN as the bucket name.
+			// If specified, the key is part of the Relative ID which has the format "bucket-name/object-key" according to
+			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html
+			bucket = urlSplit[0]
+			key = strings.Join(urlSplit[1:], "/")
+		}
+	default:
+		return ErrSchemeUnsupported
 	}
-	region, err := s3manager.GetBucketRegion(ctx, sess, u.Host, regionHint)
-	if err != nil {
-		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
-			return fmt.Errorf("couldn't determine the region for bucket %q: %v", u.Host, err)
+
+	// Determine the partition and region this bucket is in
+	var region string
+	var err error
+	if isAccessPoint {
+		region = s3arn.Region
+	} else {
+		regionHint := "us-east-1"
+		if f.S3RegionHint != "" {
+			regionHint = f.S3RegionHint
+		}
+		region, err = s3manager.GetBucketRegion(ctx, sess, bucket, regionHint)
+		if err != nil {
+			if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
+				return fmt.Errorf("couldn't determine the region for bucket %q: %v", u.Host, err)
+			}
+			return err
 		}
-		return err
 	}
 
 	sess.Config.Region = aws.String(region)
@@ -428,8 +477,8 @@ func (f *Fetcher) fetchFromS3(u url.URL, dest s3target, opts FetchOptions) error
 	}
 
 	input := &s3.GetObjectInput{
-		Bucket:    &u.Host,
-		Key:       &u.Path,
+		Bucket:    &bucket,
+		Key:       &key,
 		VersionId: versionId,
 	}
 	err = f.fetchFromS3WithCreds(ctx, dest, input, sess)

internal/resource/url_test.go

@@ -194,6 +194,20 @@ func TestFetchOffline(t *testing.T) {
 			},
 			out: out{err: ErrNeedNet},
 		},
+		// arn url specifying bucket
+		{
+			in: in{
+				url: "arn:aws:s3:us-west-2:123456789012:bucket-name/object-key",
+			},
+			out: out{err: ErrNeedNet},
+		},
+		// arn url specifying s3 access point
+		{
+			in: in{
+				url: "arn:aws:s3:us-west-2:123456789012:accesspoint/test/object",
+			},
+			out: out{err: ErrNeedNet},
+		},
 		// gs url
 		{
 			in: in{