internal/resource: support S3 access point URLs

config/shared/errors/errors.go

@@ -99,6 +99,7 @@ var (
 	ErrEngineConfiguration             = errors.New("engine incorrectly configured")
 
 	// AWS S3 specific errors
+	ErrInvalidS3ARN             = errors.New("invalid S3 ARN format")
 	ErrInvalidS3ObjectVersionId = errors.New("invalid S3 object VersionId")
 
 	// Obsolete errors, left here for ABI compatibility

config/v3_4_experimental/types/url.go

@@ -16,7 +16,9 @@ package types
 
 import (
 	"net/url"
+	"strings"
 
+	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/vincent-petithory/dataurl"
 
 	"github.com/coreos/ignition/v2/config/shared/errors"
@@ -39,6 +41,30 @@ func validateURL(s string) error {
 			}
 		}
 		return nil
+	case "arn":
+		fullURL := u.Scheme + ":" + u.Opaque
+		if !arn.IsARN(fullURL) {
+			return errors.ErrInvalidS3ARN
+		}
+		s3arn, err := arn.Parse(fullURL)
+		if err != nil {
+			return err
+		}
+		if s3arn.Service != "s3" {
+			return errors.ErrInvalidS3ARN
+		}
+		urlSplit := strings.Split(fullURL, "/")
+		if strings.HasPrefix(s3arn.Resource, "accesspoint/") && len(urlSplit) < 3 {
+			return errors.ErrInvalidS3ARN
+		} else if len(urlSplit) < 2 {
+			return errors.ErrInvalidS3ARN
+		}
+		if v, ok := u.Query()["versionId"]; ok {
+			if len(v) == 0 || v[0] == "" {
+				return errors.ErrInvalidS3ObjectVersionId
+			}
+		}
+		return nil
 	case "data":
 		if _, err := dataurl.DecodeString(s); err != nil {
 			return err

config/v3_4_experimental/types/url_test.go

@@ -66,6 +66,62 @@ func TestURLValidate(t *testing.T) {
 			util.StrToPtr("s3://bucket/key?versionId=aVersionHash"),
 			nil,
 		},
+		{
+			util.StrToPtr("Arn:"),
+			errors.ErrInvalidS3ARN,
+		},
+		{
+			util.StrToPtr("arn:aws:iam:us-west-2:123456789012:resource"),
+			errors.ErrInvalidS3ARN,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:::bucket-name-but-no-key"),
+			errors.ErrInvalidS3ARN,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint"),
+			errors.ErrInvalidS3ARN,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/"),
+			errors.ErrInvalidS3ARN,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/accesspoint-name-but-no-bucket"),
+			errors.ErrInvalidS3ARN,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:bucket-name/object-key"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:bucket-name/object-key?versionId=aVersionHash"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/accesspoint-name/object"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/accesspoint-name/some/nested/object"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:us-west-2:123456789012:accesspoint/accesspoint-name/object?versionId=aVersionHash"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:::bucket-name/object-key"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:::bucket-name/some/nested/object"),
+			nil,
+		},
+		{
+			util.StrToPtr("arn:aws:s3:::bucket-name/object-key?versionId=aVersionHash"),
+			nil,
+		},
 		{
 			util.StrToPtr("gs://bucket/object"),
 			nil,

docs/configuration-v3_4_experimental.md

@@ -14,15 +14,15 @@ The Ignition configuration is a JSON document conforming to the following specif
   * **version** (string): the semantic version number of the spec. The spec version must be compatible with the latest version (`3.4.0-experimental`). Compatibility requires the major versions to match and the spec version be less than or equal to the latest version. `-experimental` versions compare less than the final version with the same number, and previous experimental versions are not accepted.
   * **_config_** (objects): options related to the configuration.
     * **_merge_** (list of objects): a list of the configs to be merged to the current config.
-      * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+      * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
       * **_compression_** (string): the type of compression used on the config (null or gzip). Compression cannot be used with S3.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
         * **_value_** (string): the header contents.
       * **_verification_** (object): options related to the verification of the config.
         * **_hash_** (string): the hash of the config, in the form `<type>-<value>` where type is either `sha512` or `sha256`.
     * **_replace_** (object): the config that will replace the current.
-      * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+      * **source** (string): the URL of the config. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
       * **_compression_** (string): the type of compression used on the config (null or gzip). Compression cannot be used with S3.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
@@ -35,7 +35,7 @@ The Ignition configuration is a JSON document conforming to the following specif
   * **_security_** (object): options relating to network security.
     * **_tls_** (object): options relating to TLS when fetching resources over `https`.
       * **_certificateAuthorities_** (list of objects): the list of additional certificate authorities (in addition to the system authorities) to be used for TLS verification when fetching over `https`. All certificate authorities must have a unique `source`.
-        * **source** (string): the URL of the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates. Supported schemes are `http`, `https`, `s3`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+        * **source** (string): the URL of the certificate bundle (in PEM format). The bundle can contain multiple concatenated certificates. Supported schemes are `http`, `https`, `s3`, `arn`, `gs`, `tftp`, and [`data`][rfc2397]. Note: When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
         * **_compression_** (string): the type of compression used on the certificate (null or gzip). Compression cannot be used with S3.
         * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
           * **name** (string): the header name.
@@ -80,15 +80,15 @@ The Ignition configuration is a JSON document conforming to the following specif
     * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. `contents.source` must be specified if `overwrite` is true. Defaults to false.
     * **_contents_** (object): options related to the contents of the file.
       * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3.
-      * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created.
+      * **_source_** (string): the URL of the file contents. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a regular file already exists at the path, Ignition will do nothing. If source is omitted and no file exists, an empty file will be created.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
         * **_value_** (string): the header contents.
       * **_verification_** (object): options related to the verification of the file contents.
         * **_hash_** (string): the hash of the contents, in the form `<type>-<value>` where type is either `sha512` or `sha256`.
     * **_append_** (list of objects): list of contents to be appended to the file. Follows the same stucture as `contents`
       * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3.
-      * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+      * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
         * **_value_** (string): the header contents.
@@ -127,7 +127,7 @@ The Ignition configuration is a JSON document conforming to the following specif
     * **device** (string): the absolute path to the device. Devices are typically referenced by the `/dev/disk/by-*` symlinks.
     * **_keyFile_** (string): options related to the contents of the key file.
       * **_compression_** (string): the type of compression used on the contents (null or gzip). Compression cannot be used with S3.
-      * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
+      * **_source_** (string): the URL of the contents to append. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified.
       * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
         * **name** (string): the header name.
         * **_value_** (string): the header contents.

docs/supported-platforms.md

@@ -17,7 +17,7 @@ Ignition is currently only supported for the following platforms:
 * [Google Cloud] (`gcp`) - Ignition will read its configuration from the instance metadata entry named "user-data". Cloud SSH keys are handled separately.
 * [IBM Cloud] (`ibmcloud`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately.
 * [KubeVirt] (`kubevirt`) - Ignition will read its configuration from the instance userdata via config drive. Cloud SSH keys are handled separately.
-* Bare Metal (`metal`) - Use the `ignition.config.url` kernel parameter to provide a URL to the configuration. The URL can use the `http://`, `https://`, `tftp://`, `s3://`, or `gs://` schemes to specify a remote config.
+* Bare Metal (`metal`) - Use the `ignition.config.url` kernel parameter to provide a URL to the configuration. The URL can use the `http://`, `https://`, `tftp://`, `s3://`, `arn:`, or `gs://` schemes to specify a remote config.
 * [Nutanix] (`nutanix`) - Ignition will read its configuration from the instance userdata via config drive. Cloud SSH keys are handled separately.
 * [OpenStack] (`openstack`) - Ignition will read its configuration from the instance userdata via either metadata service or config drive. Cloud SSH keys are handled separately.
 * [Equinix Metal] (`packet`) - Ignition will read its configuration from the instance userdata. Cloud SSH keys are handled separately.

internal/resource/url.go

@@ -39,6 +39,7 @@ import (
 	"google.golang.org/api/option"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -135,7 +136,7 @@ func (f *Fetcher) FetchToBuffer(u url.URL, opts FetchOptions) ([]byte, error) {
 		err = f.fetchFromTFTP(u, dest, opts)
 	case "data":
 		err = f.fetchFromDataURL(u, dest, opts)
-	case "s3":
+	case "s3", "arn":
 		buf := &s3buf{
 			WriteAtBuffer: aws.NewWriteAtBuffer([]byte{}),
 		}
@@ -196,7 +197,7 @@ func (f *Fetcher) Fetch(u url.URL, dest *os.File, opts FetchOptions) error {
 		return f.fetchFromTFTP(u, dest, opts)
 	case "data":
 		return f.fetchFromDataURL(u, dest, opts)
-	case "s3":
+	case "s3", "arn":
 		return f.fetchFromS3(u, dest, opts)
 	case "gs":
 		return f.fetchFromGCS(u, dest, opts)
@@ -407,17 +408,39 @@ func (f *Fetcher) fetchFromS3(u url.URL, dest s3target, opts FetchOptions) error
 	}
 	sess := f.AWSSession.Copy()
 
-	// Determine the partition and region this bucket is in
-	regionHint := "us-east-1"
-	if f.S3RegionHint != "" {
-		regionHint = f.S3RegionHint
+	// Determine the bucket and key based on the URL scheme
+	var bucket, key, region string
+	var err error
+	switch u.Scheme {
+	case "s3":
+		bucket = u.Host
+		key = u.Path
+	case "arn":
+		fullURL := u.Scheme + ":" + u.Opaque
+		// Parse the bucket and key from the ARN Resource.
+		// Also set the region for accesspoints.
+		// S3 bucket ARNs don't include the region field.
+		bucket, key, region, err = f.parseARN(fullURL)
+		if err != nil {
+			return err
+		}
+	default:
+		return ErrSchemeUnsupported
 	}
-	region, err := s3manager.GetBucketRegion(ctx, sess, u.Host, regionHint)
-	if err != nil {
-		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
-			return fmt.Errorf("couldn't determine the region for bucket %q: %v", u.Host, err)
+
+	// Determine the partition and region this bucket is in
+	if region == "" {
+		regionHint := "us-east-1"
+		if f.S3RegionHint != "" {
+			regionHint = f.S3RegionHint
+		}
+		region, err = s3manager.GetBucketRegion(ctx, sess, bucket, regionHint)
+		if err != nil {
+			if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
+				return fmt.Errorf("couldn't determine the region for bucket %q: %v", u.Host, err)
+			}
+			return err
 		}
-		return err
 	}
 
 	sess.Config.Region = aws.String(region)
@@ -428,8 +451,8 @@ func (f *Fetcher) fetchFromS3(u url.URL, dest s3target, opts FetchOptions) error
 	}
 
 	input := &s3.GetObjectInput{
-		Bucket:    &u.Host,
-		Key:       &u.Path,
+		Bucket:    &bucket,
+		Key:       &key,
 		VersionId: versionId,
 	}
 	err = f.fetchFromS3WithCreds(ctx, dest, input, sess)
@@ -522,3 +545,53 @@ func (f *Fetcher) decompressCopyHashAndVerify(dest io.Writer, src io.Reader, opt
 	}
 	return nil
 }
+
+// parseARN is a custom wrapper around arn.Parse(); it takes arnURL, a full ARN URL,
+// and returns a bucket, a key, a potentially empty region, or an error if the ARN
+// is invalid or not for an S3 object.
+// If the given arnURL is an accesspoint ARN, the region is set.
+// The region is empty for S3 bucket ARNs because they don't include the region field.
+func (f *Fetcher) parseARN(arnURL string) (string, string, string, error) {
+	if !arn.IsARN(arnURL) {
+		return "", "", "", configErrors.ErrInvalidS3ARN
+	}
+	s3arn, err := arn.Parse(arnURL)
+	if err != nil {
+		return "", "", "", err
+	}
+	if s3arn.Service != "s3" {
+		return "", "", "", configErrors.ErrInvalidS3ARN
+	}
+	// Split the ARN bucket (or accesspoint) and key by separating on slashes.
+	// See https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arns-paths for more info.
+	urlSplit := strings.Split(arnURL, "/")
+
+	// Determine if the ARN is for an access point or a bucket.
+	var bucket, key string
+	if strings.HasPrefix(s3arn.Resource, "accesspoint/") {
+		// urlSplit must consist of arn, name of accesspoint, and key
+		if len(urlSplit) < 3 {
+			return "", "", "", configErrors.ErrInvalidS3ARN
+		}
+
+		// When using GetObjectInput with an access point,
+		// you provide the access point ARN in place of the bucket name.
+		// For more information about access point ARNs, see Using access points
+		// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html
+		bucket = strings.Join(urlSplit[:2], "/")
+		key = strings.Join(urlSplit[2:], "/")
+		return bucket, key, s3arn.Region, nil
+	}
+	// urlSplit must consist of name of bucket and key
+	if len(urlSplit) < 2 {
+		return "", "", "", configErrors.ErrInvalidS3ARN
+	}
+
+	// Parse out the bucket name in order to find the region with s3manager.GetBucketRegion.
+	// If specified, the key is part of the Relative ID which has the format "bucket-name/object-key" according to
+	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html
+	bucketUrlSplit := strings.Split(urlSplit[0], ":")
+	bucket = bucketUrlSplit[len(bucketUrlSplit)-1]
+	key = strings.Join(urlSplit[1:], "/")
+	return bucket, key, "", nil
+}