platform: add scaleway platform

cmd/kola/options.go

@@ -40,7 +40,7 @@ var (
 	kolaOffering       string
 	defaultTargetBoard = sdk.DefaultBoard()
 	kolaArchitectures  = []string{"amd64"}
-	kolaPlatforms      = []string{"aws", "azure", "brightbox", "do", "esx", "external", "gce", "openstack", "equinixmetal", "qemu", "qemu-unpriv"}
+	kolaPlatforms      = []string{"aws", "azure", "brightbox", "do", "esx", "external", "gce", "openstack", "equinixmetal", "qemu", "qemu-unpriv", "scaleway"}
 	kolaDistros        = []string{"cl", "fcos", "rhcos"}
 	kolaChannels       = []string{"alpha", "beta", "stable", "edge", "lts"}
 	kolaOfferings      = []string{"basic", "pro"}
@@ -233,6 +233,16 @@ func init() {
 	sv(&kola.BrightboxOptions.ClientSecret, "brightbox-client-secret", "", "Brightbox client secret")
 	sv(&kola.BrightboxOptions.Image, "brightbox-image", "", "Brightbox image ref")
 	sv(&kola.BrightboxOptions.ServerType, "brightbox-server-type", "2gb.ssd", "Brightbox server type")
+
+	// Scaleway specific options
+	sv(&kola.ScalewayOptions.OrganizationID, "scaleway-organization-id", "", "Scaleway organization ID")
+	sv(&kola.ScalewayOptions.ProjectID, "scaleway-project-id", "", "Scaleway organization ID")
+	sv(&kola.ScalewayOptions.Region, "scaleway-region", "fr-par", "Scaleway region")
+	sv(&kola.ScalewayOptions.Zone, "scaleway-zone", "fr-par-1", "Scaleway region")
+	sv(&kola.ScalewayOptions.AccessKey, "scaleway-access-key", "", "Scaleway credentials access key")
+	sv(&kola.ScalewayOptions.SecretKey, "scaleway-secret-key", "", "Scaleway credentials secret key")
+	sv(&kola.ScalewayOptions.Image, "scaleway-image", "", "Scaleway image ID")
+	sv(&kola.ScalewayOptions.InstanceType, "scaleway-instance-type", "DEV1-S", "Scaleway instance type")
 }
 
 // Sync up the command line options if there is dependency
@@ -252,6 +262,7 @@ func syncOptions() error {
 	kola.EquinixMetalOptions.Board = board
 	kola.EquinixMetalOptions.GSOptions = &kola.GCEOptions
 	kola.BrightboxOptions.Board = board
+	kola.ScalewayOptions.Board = board
 
 	validateOption := func(name, item string, valid []string) error {
 		for _, v := range valid {

cmd/ore/scaleway.go

@@ -0,0 +1,12 @@
+// Copyright The Mantle Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"github.com/flatcar/mantle/cmd/ore/scaleway"
+)
+
+func init() {
+	root.AddCommand(scaleway.Scaleway)
+}

cmd/ore/scaleway/create-image.go

@@ -0,0 +1,72 @@
+// Copyright The Mantle Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package scaleway
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+)
+
+const bucket = "flatcar-testing"
+
+var (
+	cmdCreate = &cobra.Command{
+		Use:   "create-image",
+		Short: "Create Scaleway image",
+		RunE:  runCreate,
+		Example: `IMAGE_ID=$(ore scaleway \
+  --scaleway-access-key "${SCALEWAY_ACCESS_KEY}" \
+  --scaleway-secret-key "${SCALEWAY_SECRET_KEY}" \
+  --scaleway-organization-id "${SCALEWAY_ORGANIZATION_ID}" \
+  create-image --channel beta)`,
+	}
+	channel string
+	version string
+	board   string
+	file    string
+)
+
+func init() {
+	Scaleway.AddCommand(cmdCreate)
+
+	cmdCreate.Flags().StringVar(&channel, "channel", "stable", "Flatcar channel")
+	cmdCreate.Flags().StringVar(&version, "version", "current", "Flatcar version")
+	cmdCreate.Flags().StringVar(&board, "board", "amd64-usr", "board used for naming with default prefix and AMI architecture")
+	cmdCreate.Flags().StringVar(&file, "file", "flatcar_production_scaleway_image.qcow2", "path to local Flatcar image (.qcow2)")
+}
+
+func runCreate(cmd *cobra.Command, args []string) error {
+	if err := API.InitializeBucket(bucket); err != nil {
+		return fmt.Errorf("creating bucket %s: %v", bucket, err)
+	}
+
+	f, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("opening Flatcar image file %s: %v", file, err)
+	}
+
+	defer f.Close()
+
+	key := fmt.Sprintf("%s/%s/%s/%s", channel, version, board, filepath.Base(file))
+	if err := API.UploadObject(f, bucket, key, true); err != nil {
+		return fmt.Errorf("uploading Flatcar image file %s: %v", file, err)
+	}
+
+	ID, err := API.CreateSnapshot(context.Background(), bucket, key)
+	if err != nil {
+		return fmt.Errorf("creating Flatcar image: %v", err)
+	}
+
+	if err := API.DeleteObject(bucket, key); err != nil {
+		return fmt.Errorf("deleting Flatcar image from s3 bucket: %s", fmt.Sprintf("s3://%s/%s", bucket, key))
+	}
+
+	fmt.Println(ID)
+
+	return nil
+}

cmd/ore/scaleway/gc.go

@@ -0,0 +1,36 @@
+// Copyright The Mantle Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package scaleway
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	cmdGC = &cobra.Command{
+		Use:   "gc",
+		Short: "GC resources in Scaleway",
+		Long:  `Delete instances and images created over the given duration ago`,
+		RunE:  runGC,
+	}
+
+	gcDuration time.Duration
+)
+
+func init() {
+	Scaleway.AddCommand(cmdGC)
+	cmdGC.Flags().DurationVar(&gcDuration, "duration", 5*time.Hour, "how old resources must be before they're considered garbage")
+}
+
+func runGC(cmd *cobra.Command, args []string) error {
+	if err := API.GC(context.Background(), gcDuration); err != nil {
+		return fmt.Errorf("running garbage collection: %w", err)
+	}
+
+	return nil
+}

cmd/ore/scaleway/scaleway.go

@@ -0,0 +1,60 @@
+// Copyright The Mantle Authors.
+// SPDX-License-Identifier: Apache-2.0
+package scaleway
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/coreos/pkg/capnslog"
+	"github.com/flatcar/mantle/cli"
+	"github.com/flatcar/mantle/platform"
+	"github.com/flatcar/mantle/platform/api/scaleway"
+	"github.com/spf13/cobra"
+)
+
+var (
+	plog = capnslog.NewPackageLogger("github.com/flatcar/mantle", "ore/scaleway")
+
+	Scaleway = &cobra.Command{
+		Use:   "scaleway [command]",
+		Short: "scaleway image utilities",
+	}
+
+	API            *scaleway.API
+	region         string
+	zone           string
+	accessKey      string
+	secretKey      string
+	organizationID string
+	projectID      string
+)
+
+func init() {
+	cli.WrapPreRun(Scaleway, preflightCheck)
+	Scaleway.PersistentFlags().StringVar(&region, "scaleway-region", "fr-par", "Scaleway region")
+	Scaleway.PersistentFlags().StringVar(&zone, "scaleway-zone", "fr-par-1", "Scaleway region")
+	Scaleway.PersistentFlags().StringVar(&accessKey, "scaleway-access-key", "", "Scaleway access key")
+	Scaleway.PersistentFlags().StringVar(&secretKey, "scaleway-secret-key", "", "Scaleway secret key")
+	Scaleway.PersistentFlags().StringVar(&organizationID, "scaleway-organization-id", "", "Scaleway organization ID")
+	Scaleway.PersistentFlags().StringVar(&projectID, "scaleway-project-id", "", "Scaleway project ID")
+}
+
+func preflightCheck(cmd *cobra.Command, args []string) error {
+	api, err := scaleway.New(&scaleway.Options{
+		Region:         region,
+		Zone:           zone,
+		AccessKey:      accessKey,
+		SecretKey:      secretKey,
+		OrganizationID: organizationID,
+		ProjectID:      projectID,
+		Options:        &platform.Options{},
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not create Scaleway API client: %v\n", err)
+		os.Exit(1)
+	}
+
+	API = api
+	return nil
+}

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/packethost/packngo v0.21.0
 	github.com/pborman/uuid v1.2.0
 	github.com/pin/tftp v2.1.0+incompatible
+	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.23
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace
 	github.com/stretchr/testify v1.9.0
@@ -135,6 +136,7 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
 	google.golang.org/grpc v1.59.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 replace github.com/Microsoft/azure-vhd-utils => github.com/kinvolk/azure-vhd-utils v0.0.0-20210818134022-97083698b75f

go.sum

@@ -478,6 +478,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.23 h1:zr2sP1pxJ+iPAmZipnwz+uXmpvMEJOndD9Y+F0Dn42A=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.23/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs=

kola/harness.go

@@ -46,6 +46,7 @@ import (
 	esxapi "github.com/flatcar/mantle/platform/api/esx"
 	gcloudapi "github.com/flatcar/mantle/platform/api/gcloud"
 	openstackapi "github.com/flatcar/mantle/platform/api/openstack"
+	scalewayapi "github.com/flatcar/mantle/platform/api/scaleway"
 	"github.com/flatcar/mantle/platform/conf"
 	"github.com/flatcar/mantle/platform/machine/aws"
 	"github.com/flatcar/mantle/platform/machine/azure"
@@ -57,6 +58,7 @@ import (
 	"github.com/flatcar/mantle/platform/machine/gcloud"
 	"github.com/flatcar/mantle/platform/machine/openstack"
 	"github.com/flatcar/mantle/platform/machine/qemu"
+	"github.com/flatcar/mantle/platform/machine/scaleway"
 	"github.com/flatcar/mantle/platform/machine/unprivqemu"
 	"github.com/flatcar/mantle/system"
 )
@@ -75,6 +77,7 @@ var (
 	OpenStackOptions    = openstackapi.Options{Options: &Options}    // glue to set platform options from main
 	EquinixMetalOptions = equinixmetalapi.Options{Options: &Options} // glue to set platform options from main
 	QEMUOptions         = qemu.Options{Options: &Options}            // glue to set platform options from main
+	ScalewayOptions     = scalewayapi.Options{Options: &Options}     // glue to set platform options from main
 
 	TestParallelism        int    //glue var to set test parallelism from main
 	TAPFile                string // if not "", write TAP results here
@@ -246,6 +249,8 @@ func NewFlight(pltfrm string) (flight platform.Flight, err error) {
 		flight, err = qemu.NewFlight(&QEMUOptions)
 	case "qemu-unpriv":
 		flight, err = unprivqemu.NewFlight(&QEMUOptions)
+	case "scaleway":
+		flight, err = scaleway.NewFlight(&ScalewayOptions)
 	default:
 		err = fmt.Errorf("invalid platform %q", pltfrm)
 	}

platform/api/aws/api.go

@@ -62,7 +62,7 @@ type API struct {
 	ec2         *ec2.EC2
 	iam         *iam.IAM
 	marketplace *marketplacecatalog.MarketplaceCatalog
-	s3          *s3.S3
+	S3          *s3.S3
 	opts        *Options
 }
 
@@ -98,7 +98,7 @@ func New(opts *Options) (*API, error) {
 		ec2:         ec2.New(sess),
 		marketplace: marketplacecatalog.New(sess),
 		iam:         iam.New(sess),
-		s3:          s3.New(sess),
+		S3:          s3.New(sess),
 		opts:        opts,
 	}
 

platform/api/aws/s3.go

@@ -55,10 +55,10 @@ func (a *API) UploadObject(r io.Reader, bucket, path string, force bool) error {
 
 // UploadObjectExt uploads an object to S3 with more control over options.
 func (a *API) UploadObjectExt(r io.Reader, bucket, path string, force bool, policy string, contentType string, max_age int) error {
-	s3uploader := s3manager.NewUploaderWithClient(a.s3)
+	s3uploader := s3manager.NewUploaderWithClient(a.S3)
 
 	if !force {
-		_, err := a.s3.HeadObject(&s3.HeadObjectInput{
+		_, err := a.S3.HeadObject(&s3.HeadObjectInput{
 			Bucket: &bucket,
 			Key:    &path,
 		})
@@ -76,8 +76,12 @@ func (a *API) UploadObjectExt(r io.Reader, bucket, path string, force bool, poli
 		Body:   r,
 		Bucket: aws.String(bucket),
 		Key:    aws.String(path),
-		ACL:    aws.String(policy),
 	}
+
+	if policy != "" {
+		input.ACL = aws.String(policy)
+	}
+
 	if max_age >= 0 {
 		input.CacheControl = aws.String(fmt.Sprintf("max-age=%d", max_age))
 	}
@@ -95,7 +99,7 @@ func (a *API) UploadObjectExt(r io.Reader, bucket, path string, force bool, poli
 
 func (a *API) DeleteObject(bucket, path string) error {
 	plog.Infof("Deleting s3://%v/%v", bucket, path)
-	_, err := a.s3.DeleteObject(&s3.DeleteObjectInput{
+	_, err := a.S3.DeleteObject(&s3.DeleteObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(path),
 	})
@@ -106,7 +110,7 @@ func (a *API) DeleteObject(bucket, path string) error {
 }
 
 func (a *API) InitializeBucket(bucket string) error {
-	_, err := a.s3.CreateBucket(&s3.CreateBucketInput{
+	_, err := a.S3.CreateBucket(&s3.CreateBucketInput{
 		Bucket: &bucket,
 	})
 	if err != nil {
@@ -121,7 +125,7 @@ func (a *API) InitializeBucket(bucket string) error {
 
 // This will modify the ACL on Objects to one of the canned ACL policies
 func (a *API) PutObjectAcl(bucket, path, policy string) error {
-	_, err := a.s3.PutObjectAcl(&s3.PutObjectAclInput{
+	_, err := a.S3.PutObjectAcl(&s3.PutObjectAclInput{
 		ACL:    aws.String(policy),
 		Bucket: aws.String(bucket),
 		Key:    aws.String(path),
@@ -138,7 +142,7 @@ func (a *API) CopyObject(srcBucket, srcPath, destBucket, destPath, policy string
 	if err != nil {
 		return fmt.Errorf("creating destination bucket: %v", err)
 	}
-	_, err = a.s3.CopyObject(&s3.CopyObjectInput{
+	_, err = a.S3.CopyObject(&s3.CopyObjectInput{
 		ACL:        aws.String(policy),
 		CopySource: aws.String(url.QueryEscape(fmt.Sprintf("%s/%s", srcBucket, srcPath))),
 		Bucket:     aws.String(destBucket),
@@ -156,7 +160,7 @@ func (a *API) CopyObject(srcBucket, srcPath, destBucket, destPath, policy string
 
 // Copies all objects in srcBucket to destBucket with a given canned ACL policy
 func (a *API) CopyBucket(srcBucket, prefix, destBucket, policy string) error {
-	objects, err := a.s3.ListObjects(&s3.ListObjectsInput{
+	objects, err := a.S3.ListObjects(&s3.ListObjectsInput{
 		Bucket: aws.String(srcBucket),
 		Prefix: aws.String(prefix),
 	})
@@ -183,7 +187,7 @@ func (a *API) CopyBucket(srcBucket, prefix, destBucket, policy string) error {
 // TODO: bikeshed this name
 // modifies the ACL of all objects of a given prefix in srcBucket to a given canned ACL policy
 func (a *API) UpdateBucketObjectsACL(srcBucket, prefix, policy string) error {
-	objects, err := a.s3.ListObjects(&s3.ListObjectsInput{
+	objects, err := a.S3.ListObjects(&s3.ListObjectsInput{
 		Bucket: aws.String(srcBucket),
 		Prefix: aws.String(prefix),
 	})