feat: Upgrade to SDK v4 (#9)

* feat: Upgrade to SDK v4

* Update goreleaser variable ref

* update README

* linter: disable depguard

* use internalPlugin, revert goreleaser variable ref

* update golangci-lint version

* .github test: install cloudquery before make gen

* more .gitignore

---------

Co-authored-by: Kemal Hadimli <[email protected]>

.github/workflows/test.yaml

@@ -23,7 +23,12 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.50.1
+          version: v1.54.2
+      - name: Setup CloudQuery
+        if: github.event_name == 'pull_request'
+        uses: cloudquery/setup-cloudquery@v3
+        with:
+          version: v3.26.1
       - name: Get dependencies
         run: go get -t -d ./...
       - name: Build

.gitignore

@@ -2,5 +2,7 @@ dist/
 vendor/
 bin/
 cq-source-scaleway
+cloudquery
+cloudquery.log
 .DS_Store
 .idea

.golangci.yml

@@ -80,7 +80,6 @@ linters:
   enable:
     - asciicheck
     - bodyclose
-    - depguard
     - dupl
     - errcheck
     - gocritic

Makefile

@@ -1,16 +1,26 @@
+.PHONY: build
+build:
+	go build
+
 .PHONY: test
 test:
-	go test -timeout 3m ./...
+	go test -race -timeout 3m ./...
+
+.PHONY: gen-docs
+gen-docs: build
+	@command -v cloudquery >/dev/null 2>&1 || { \
+		echo "Error: 'cloudquery' command not found. Please install it before running gen-docs."; \
+		echo "You can install it by following the instructions at: https://www.cloudquery.io/docs/quickstart"; \
+		exit 1; \
+	}
+	rm -rf docs/tables
+	cloudquery tables --format markdown --output-dir docs/ test/config.yml
+	mv -vf docs/scaleway docs/tables
 
 .PHONY: lint
 lint:
 	@golangci-lint run --timeout 10m --verbose
 
-.PHONY: gen-docs
-gen-docs:
-	rm -rf ./docs/tables/*
-	go run main.go doc ./docs/tables
-
 # All gen targets
 .PHONY: gen
 gen: gen-docs

README.md

@@ -38,7 +38,10 @@ spec:
   name: "scaleway"
   path: "scaleway/scaleway"
   version: "${VERSION}"
-  # backend: "local" # use this to enable incremental syncing
+  # use this to enable incremental syncing
+  # backend_options:
+  #   table_name: "cq_state_scaleway"
+  #   connection: "@@plugins.DESTINATION_NAME.connection"
   tables: 
     - "*"
   skip_tables:
@@ -59,9 +62,12 @@ spec:
 - `zones` (list of string, optional. Defaults to all zones):
   List of zones to query.
 
-- `timeout_secs` (integer in seconds, optional. Default: 10):
+- `timeout_secs` (integer in seconds, optional. Default: `10`):
   Timeout for requests against the Scaleway API endpoint.
 
+- `concurrency` (int, optional, default: `1000`):
+  A best effort maximum number of Go routines to use. Lower this number to reduce memory usage.
+
 ## Development
 
 ### Run tests

client/client.go

@@ -1,116 +1,58 @@
 package client
 
 import (
-	"context"
-	"errors"
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/cloudquery/plugin-sdk/backend"
-	"github.com/cloudquery/plugin-sdk/plugins/source"
-	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/cloudquery/plugin-sdk/specs"
+	"github.com/cloudquery/plugin-sdk/v4/state"
 	"github.com/rs/zerolog"
 	"github.com/scaleway/scaleway-sdk-go/scw"
 )
 
 type Client struct {
 	Logger    zerolog.Logger
+	Spec      Spec
 	SCWClient *scw.Client
-	Backend   backend.Backend
-	OrgID     string
-	Region    scw.Region
-	Zone      scw.Zone
+	Backend   state.Client
+
+	OrgID  string
+	Region scw.Region
+	Zone   scw.Zone
+}
 
-	Spec       Spec
-	sourceSpec specs.Source
+func New(logger zerolog.Logger, spec Spec, orgID string, services *scw.Client, bk state.Client) *Client {
+	return &Client{
+		Logger:    logger,
+		Spec:      spec,
+		OrgID:     orgID,
+		SCWClient: services,
+		Backend:   bk,
+	}
 }
 
 func (c *Client) ID() string {
-	return c.sourceSpec.Name + ":" + string(c.Region) + ":" + string(c.Zone)
+	return "scaleway:" + string(c.Region) + ":" + string(c.Zone)
 }
 
 func (c *Client) WithRegion(r scw.Region) *Client {
 	return &Client{
-		Logger:     c.Logger.With().Str("region", string(r)).Logger(),
-		SCWClient:  c.SCWClient,
-		Backend:    c.Backend,
-		OrgID:      c.OrgID,
-		Region:     r,
-		Zone:       c.Zone,
-		Spec:       c.Spec,
-		sourceSpec: c.sourceSpec,
+		Logger:    c.Logger.With().Str("region", string(r)).Logger(),
+		Spec:      c.Spec,
+		SCWClient: c.SCWClient,
+		Backend:   c.Backend,
+
+		OrgID:  c.OrgID,
+		Region: r,
+		Zone:   c.Zone,
 	}
 }
 
 func (c *Client) WithZone(z scw.Zone) *Client {
 	return &Client{
-		Logger:     c.Logger.With().Str("zone", string(z)).Logger(),
-		SCWClient:  c.SCWClient,
-		Backend:    c.Backend,
-		OrgID:      c.OrgID,
-		Region:     c.Region,
-		Zone:       z,
-		Spec:       c.Spec,
-		sourceSpec: c.sourceSpec,
-	}
-}
+		Logger:    c.Logger.With().Str("zone", string(z)).Logger(),
+		Spec:      c.Spec,
+		SCWClient: c.SCWClient,
+		Backend:   c.Backend,
 
-func New(_ context.Context, logger zerolog.Logger, s specs.Source, opts source.Options) (schema.ClientMeta, error) {
-	var pluginSpec Spec
-	if err := s.UnmarshalSpec(&pluginSpec); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal plugin spec: %w", err)
-	}
-	err := pluginSpec.Validate()
-	if err != nil {
-		return nil, fmt.Errorf("failed to validate plugin spec: %w", err)
+		OrgID:  c.OrgID,
+		Region: c.Region,
+		Zone:   z,
 	}
-	pluginSpec.SetDefaults()
-
-	scwOpts := []scw.ClientOption{
-		scw.WithEnv(), // existing env variables may overwrite active profile
-
-		scw.WithHTTPClient(&http.Client{
-			Timeout: time.Duration(pluginSpec.Timeout) * time.Second,
-		}),
-		scw.WithUserAgent("cq-plugin-scaleway/" + s.Version),
-	}
-
-	cf, err := scw.LoadConfig()
-	if err != nil {
-		var configNotFoundError *scw.ConfigFileNotFoundError
-		if !errors.As(err, &configNotFoundError) {
-			return nil, err
-		}
-	}
-	if cf != nil {
-		p, err := cf.GetActiveProfile()
-		if err != nil {
-			return nil, err
-		}
-		scwOpts = append([]scw.ClientOption{
-			scw.WithProfile(p), // active profile applies first
-		}, scwOpts...)
-	}
-
-	// Create a Scaleway client
-	scwClient, err := scw.NewClient(scwOpts...)
-	if err != nil {
-		return nil, err
-	}
-
-	orgID, ok := scwClient.GetDefaultOrganizationID()
-	if !ok {
-		return nil, fmt.Errorf("SCW_DEFAULT_ORGANIZATION_ID or default_organization_id not set, get yours from https://console.scaleway.com/organization/settings")
-	}
-
-	return &Client{
-		Logger:     logger,
-		Backend:    opts.Backend,
-		OrgID:      orgID,
-		Spec:       pluginSpec,
-		SCWClient:  scwClient,
-		sourceSpec: s,
-	}, nil
 }

client/multiplexers.go

@@ -1,7 +1,7 @@
 package client
 
 import (
-	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/scaleway/scaleway-sdk-go/scw"
 )
 

client/resolvers.go

@@ -3,26 +3,23 @@ package client
 import (
 	"context"
 
-	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/apache/arrow/go/v14/arrow"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
 )
 
 var (
 	ZonePK = schema.Column{
-		Name:     "zone",
-		Type:     schema.TypeString,
-		Resolver: ResolveClientZone,
-		CreationOptions: schema.ColumnCreationOptions{
-			PrimaryKey: true,
-		},
+		Name:       "zone",
+		Type:       arrow.BinaryTypes.String,
+		Resolver:   ResolveClientZone,
+		PrimaryKey: true,
 	}
 
 	RegionPK = schema.Column{
-		Name:     "region",
-		Type:     schema.TypeString,
-		Resolver: ResolveClientRegion,
-		CreationOptions: schema.ColumnCreationOptions{
-			PrimaryKey: true,
-		},
+		Name:       "region",
+		Type:       arrow.BinaryTypes.String,
+		Resolver:   ResolveClientRegion,
+		PrimaryKey: true,
 	}
 )
 

client/spec.go

@@ -7,7 +7,8 @@ type Spec struct {
 	Zones   []scw.Zone   `json:"zones,omitempty"`
 
 	// Optional
-	Timeout int64 `json:"timeout_secs,omitempty"`
+	Timeout     int64 `json:"timeout_secs,omitempty"`
+	Concurrency int   `json:"concurrency,omitempty"`
 }
 
 func (Spec) Validate() error {
@@ -25,4 +26,7 @@ func (s *Spec) SetDefaults() {
 	if s.Timeout < 1 {
 		s.Timeout = 10
 	}
+	if s.Concurrency < 1 {
+		s.Concurrency = 1000
+	}
 }