feat: add scaleway support

cmd/regioncheck/regioncheck.go

@@ -4,9 +4,11 @@ import (
 	"fmt"
 	"github.com/PuerkitoBio/goquery"
 	"github.com/sa7mon/s3scanner/provider"
+	"io"
 	"log"
 	"net/http"
 	"os"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
@@ -100,24 +102,51 @@ func GetRegionsLinode() ([]string, error) {
 	return regions, nil
 }
 
-func main() {
-	wg := sync.WaitGroup{}
-	wg.Add(2)
+func GetRegionsScaleway() ([]string, error) {
+	var re = regexp.MustCompile(`Region: \x60(.+)\x60`)
+	requestURL := "https://raw.githubusercontent.com/scaleway/docs-content/main/storage/object/how-to/create-a-bucket.mdx"
+	res, err := http.Get(requestURL)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
+	}
 
+	bytes, bErr := io.ReadAll(res.Body)
+	if bErr != nil {
+		return nil, bErr
+	}
+
+	var regions []string
+	for _, a := range re.FindAllSubmatch(bytes, -1) {
+		regions = append(regions, string(a[1]))
+	}
+	return regions, nil
+}
+
+func main() {
 	results := map[string][]string{}
 	errors := map[string]error{}
 
-	go func(w *sync.WaitGroup) {
-		results["digitalocean"], errors["digitalocean"] = GetRegionsDO()
-		wg.Done()
-	}(&wg)
-	go func(w *sync.WaitGroup) {
-		results["linode"], errors["linode"] = GetRegionsLinode()
-		wg.Done()
-	}(&wg)
+	p := map[string]func() ([]string, error){
+		"digitalocean": GetRegionsDO,
+		"linode":       GetRegionsLinode,
+		"scaleway":     GetRegionsScaleway,
+	}
 
-	// No region check for Dreamhost
-	results["dreamhost"] = provider.ProviderRegions["dreamhost"]
+	wg := sync.WaitGroup{}
+	wg.Add(len(p))
+
+	for name, get := range p {
+		name := name
+		get := get
+		go func(w *sync.WaitGroup) {
+			results[name], errors[name] = get()
+			wg.Done()
+		}(&wg)
+	}
 
 	wg.Wait()
 
@@ -136,7 +165,7 @@ func main() {
 			log.Printf("[%s] regions differ! Existing: %v, found: %v", p, knownRegions, foundRegions)
 			exit = 1
 		} else {
-			log.Printf("[%s} OK", p)
+			log.Printf("[%s] OK", p)
 		}
 	}
 	os.Exit(exit)

cmd/regioncheck/regioncheck_test.go

@@ -18,3 +18,10 @@ func TestGetRegionsLinode(t *testing.T) {
 	assert.GreaterOrEqual(t, len(r), 1)
 	assert.Contains(t, r, "us-east-1")
 }
+
+func TestGetRegionsScaleway(t *testing.T) {
+	r, err := GetRegionsScaleway()
+	assert.Nil(t, err)
+	assert.GreaterOrEqual(t, len(r), 1)
+	assert.Contains(t, r, "fr-par")
+}

provider/providers.go

@@ -40,13 +40,14 @@ type bucketCheckResult struct {
 }
 
 var AllProviders = []string{
-	"aws", "custom", "digitalocean", "dreamhost", "gcp", "linode",
+	"aws", "custom", "digitalocean", "dreamhost", "gcp", "linode", "scaleway",
 }
 
 var ProviderRegions = map[string][]string{
 	"digitalocean": {"ams3", "blr1", "fra1", "nyc3", "sfo2", "sfo3", "sgp1", "syd1"},
 	"dreamhost":    {"us-east-1"},
 	"linode":       {"ap-south-1", "eu-central-1", "fr-par-1", "se-sto-1", "us-east-1", "us-iad-1", "us-ord-1", "us-southeast-1"},
+	"scaleway":     {"fr-par", "nl-ams", "pl-waw"},
 }
 
 func NewProvider(name string) (StorageProvider, error) {
@@ -65,6 +66,8 @@ func NewProvider(name string) (StorageProvider, error) {
 		provider, err = NewProviderGCP()
 	case "linode":
 		provider, err = NewProviderLinode()
+	case "scaleway":
+		provider, err = NewProviderScaleway()
 	default:
 		err = fmt.Errorf("unknown provider: %s", name)
 	}
@@ -203,9 +206,8 @@ func checkPermissions(client *s3.Client, b *bucket.Bucket, doDestructiveChecks b
 	return nil
 }
 
+// bucketExists takes a bucket name and checks if it exists in any region contained in clients
 func bucketExists(clients *clientmap.ClientMap, b *bucket.Bucket) (bool, string, error) {
-	// TODO: Should this return a client or a region name? If region name, we'll need GetClient(region)
-	// TODO: Add region priority - order in which to check. maps are not ordered
 	results := make(chan bucketCheckResult, clients.Len())
 	e := make(chan error, 1)
 
@@ -216,18 +218,36 @@ func bucketExists(clients *clientmap.ClientMap, b *bucket.Bucket) (bool, string,
 				"region":      region,
 				"method":      "providers.bucketExists()",
 			}
-			_, regionErr := manager.GetBucketRegion(context.TODO(), client, bucketName)
+			var regionErr error
+
+			// Unlike other APIs, Scaleway returns '200 OK' to a HEAD request sent to the wrong region for a
+			// bucket that does exist in another region. So instead, we send a GET request for a list of 1 object.
+			// Scaleway will return 404 to the GET request in any region other than the one the bucket belongs to.
+			// See https://github.com/sa7mon/S3Scanner/issues/209 for a better way to fix this.
+			if b.Provider == "scaleway" {
+				_, regionErr = client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+					Bucket:  &b.Name,
+					MaxKeys: 1,
+				})
+			} else {
+				_, regionErr = manager.GetBucketRegion(context.TODO(), client, bucketName)
+			}
+
 			if regionErr == nil {
 				log.WithFields(logFields).Debugf("no error - bucket exists")
 				results <- bucketCheckResult{region: region, exists: true}
 				return
 			}
 
-			var bnf manager.BucketNotFound
+			var bnf manager.BucketNotFound // Can be returned from GetBucketRegion()
+			var nsb *types.NoSuchBucket    // Can be returned from ListObjectsV2()
 			var re2 *awshttp.ResponseError
 			if errors.As(regionErr, &bnf) {
 				log.WithFields(logFields).Debugf("BucketNotFound")
 				results <- bucketCheckResult{region: region, exists: false}
+			} else if errors.As(regionErr, &nsb) {
+				log.WithFields(logFields).Debugf("BucketNotFound")
+				results <- bucketCheckResult{region: region, exists: false}
 			} else if errors.As(regionErr, &re2) && re2.HTTPStatusCode() == 403 {
 				log.WithFields(logFields).Debugf("AccessDenied")
 				results <- bucketCheckResult{region: region, exists: true}

provider/providers_test.go

@@ -52,6 +52,12 @@ func TestMain(m *testing.M) {
 	}
 	providers["linode"] = provider
 
+	provider, err = NewProviderScaleway()
+	if err != nil {
+		panic(err)
+	}
+	providers["scaleway"] = provider
+
 	code := m.Run()
 	os.Exit(code)
 }
@@ -114,6 +120,7 @@ func Test_StorageProvider_Statics(t *testing.T) {
 		{name: "Dreamhost", provider: providers["dreamhost"], insecure: false, addressStyle: PathStyle},
 		{name: "GCP", provider: providers["gcp"], insecure: false, addressStyle: PathStyle},
 		{name: "Linode", provider: providers["linode"], insecure: false, addressStyle: VirtualHostStyle},
+		{name: "Scaleway", provider: providers["scaleway"], insecure: false, addressStyle: PathStyle},
 	}
 
 	for _, tt := range tests {
@@ -138,6 +145,7 @@ func Test_StorageProvider_BucketExists(t *testing.T) {
 		{name: "Dreamhost", provider: providers["dreamhost"], goodBucket: bucket.NewBucket("bitrix24"), badBucket: bucket.NewBucket("s3scanner-no-exist")},
 		{name: "GCP", provider: providers["gcp"], goodBucket: bucket.NewBucket("books"), badBucket: bucket.NewBucket("s3scanner-no-exist")},
 		{name: "Linode", provider: providers["linode"], goodBucket: bucket.NewBucket("vantage"), badBucket: bucket.NewBucket("s3scanner-no-exist")},
+		{name: "Scaleway", provider: providers["scaleway"], goodBucket: bucket.NewBucket("2017"), badBucket: bucket.NewBucket("s3scanner-no-exist")},
 	}
 
 	for _, tt := range tests {
@@ -170,6 +178,7 @@ func Test_StorageProvider_Enum(t *testing.T) {
 		{name: "Dreamhost", provider: providers["dreamhost"], goodBucket: bucket.NewBucket("bitrix24"), numObjects: 6},
 		{name: "GCP", provider: providers["gcp"], goodBucket: bucket.NewBucket("assets"), numObjects: 3},
 		{name: "Linode", provider: providers["linode"], goodBucket: bucket.NewBucket("vantage"), numObjects: 47},
+		{name: "Scaleway", provider: providers["scaleway"], goodBucket: bucket.NewBucket("3d-builder"), numObjects: 1},
 	}
 
 	for _, tt := range tests {
@@ -202,6 +211,7 @@ func Test_StorageProvider_Scan(t *testing.T) {
 		{name: "Dreamhost", provider: providers["dreamhost"], bucket: bucket.NewBucket("bitrix24"), permissions: "AuthUsers: [] | AllUsers: [READ]"},
 		{name: "GCP", provider: providers["gcp"], bucket: bucket.NewBucket("hatrioua"), permissions: "AuthUsers: [] | AllUsers: []"},
 		{name: "Linode", provider: providers["linode"], bucket: bucket.NewBucket("vantage"), permissions: "AuthUsers: [] | AllUsers: [READ]"},
+		{name: "Scaleway", provider: providers["scaleway"], bucket: bucket.NewBucket("3d-builder"), permissions: "AuthUsers: [] | AllUsers: [READ]"},
 	}
 
 	for _, tt := range tests {

provider/scaleway.go

@@ -0,0 +1,82 @@
+package provider
+
+import (
+	"errors"
+	"fmt"
+	"github.com/sa7mon/s3scanner/bucket"
+	"github.com/sa7mon/s3scanner/provider/clientmap"
+)
+
+type providerScaleway struct {
+	clients *clientmap.ClientMap
+}
+
+func NewProviderScaleway() (*providerScaleway, error) {
+	sc := new(providerScaleway)
+
+	clients, err := sc.newClients()
+	if err != nil {
+		return sc, err
+	}
+	sc.clients = clients
+	return sc, nil
+}
+
+func (sc *providerScaleway) newClients() (*clientmap.ClientMap, error) {
+	clients := clientmap.WithCapacity(len(ProviderRegions[sc.Name()]))
+	for _, r := range ProviderRegions[sc.Name()] {
+		client, err := newNonAWSClient(sc, fmt.Sprintf("https://s3.%s.scw.cloud", r))
+		if err != nil {
+			return nil, err
+		}
+		clients.Set(r, client)
+	}
+
+	return clients, nil
+}
+
+func (sc *providerScaleway) Scan(b *bucket.Bucket, doDestructiveChecks bool) error {
+	client := sc.clients.Get(b.Region)
+	return checkPermissions(client, b, doDestructiveChecks)
+}
+
+func (*providerScaleway) Insecure() bool {
+	return false
+}
+
+func (*providerScaleway) Name() string {
+	return "scaleway"
+}
+
+func (*providerScaleway) AddressStyle() int {
+	return PathStyle
+}
+
+func (sc *providerScaleway) BucketExists(b *bucket.Bucket) (*bucket.Bucket, error) {
+	b.Provider = sc.Name()
+	exists, region, err := bucketExists(sc.clients, b)
+	if err != nil {
+		return b, err
+	}
+	if exists {
+		b.Exists = bucket.BucketExists
+		b.Region = region
+	} else {
+		b.Exists = bucket.BucketNotExist
+	}
+
+	return b, nil
+}
+
+func (sc *providerScaleway) Enumerate(b *bucket.Bucket) error {
+	if b.Exists != bucket.BucketExists {
+		return errors.New("bucket might not exist")
+	}
+
+	client := sc.clients.Get(b.Region)
+	enumErr := enumerateListObjectsV2(client, b)
+	if enumErr != nil {
+		return enumErr
+	}
+	return nil
+}