Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Nuke users #2971

Merged
merged 4 commits into from
Aug 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions pkg/acceptance/testing.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@ import (
"github.com/snowflakedb/gosnowflake"
)

const AcceptanceTestPrefix = "acc_test_"

var (
TestDatabaseName = "acc_test_db_" + random.AcceptanceTestsSuffix
TestSchemaName = "acc_test_sc_" + random.AcceptanceTestsSuffix
TestWarehouseName = "acc_test_wh_" + random.AcceptanceTestsSuffix
TestDatabaseName = fmt.Sprintf("%sdb_%s", AcceptanceTestPrefix, random.AcceptanceTestsSuffix)
TestSchemaName = fmt.Sprintf("%ssc_%s", AcceptanceTestPrefix, random.AcceptanceTestsSuffix)
TestWarehouseName = fmt.Sprintf("%swh_%s", AcceptanceTestPrefix, random.AcceptanceTestsSuffix)
)

var (
Expand Down
2 changes: 2 additions & 0 deletions pkg/acceptance/testprofiles/testing_config_profiles.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,7 @@ package testprofiles
const (
Default = "default"
Secondary = "secondary_test_account"
Third = "third_test_account"
Fourth = "fourth_test_account"
IncorrectUserAndPassword = "incorrect_test_profile"
)
10 changes: 5 additions & 5 deletions pkg/sdk/client_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,26 +66,26 @@ func TestClient_NewClient(t *testing.T) {
}

func TestClient_ping(t *testing.T) {
client := testClient(t)
client := defaultTestClient(t)
err := client.Ping()
require.NoError(t, err)
}

func TestClient_close(t *testing.T) {
client := testClient(t)
client := defaultTestClient(t)
err := client.Close()
require.NoError(t, err)
}

func TestClient_exec(t *testing.T) {
client := testClient(t)
client := defaultTestClient(t)
ctx := context.Background()
_, err := client.exec(ctx, "SELECT 1")
require.NoError(t, err)
}

func TestClient_query(t *testing.T) {
client := testClient(t)
client := defaultTestClient(t)
ctx := context.Background()
rows := []struct {
One int `db:"ONE"`
Expand All @@ -98,7 +98,7 @@ func TestClient_query(t *testing.T) {
}

func TestClient_queryOne(t *testing.T) {
client := testClient(t)
client := defaultTestClient(t)
ctx := context.Background()
row := struct {
One int `db:"ONE"`
Expand Down
25 changes: 20 additions & 5 deletions pkg/sdk/helper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/acceptance/testprofiles"
)

func testClient(t *testing.T) *Client {
func defaultTestClient(t *testing.T) *Client {
t.Helper()

client, err := NewDefaultClient()
Expand All @@ -17,16 +17,31 @@ func testClient(t *testing.T) *Client {
return client
}

func testSecondaryClient(t *testing.T) *Client {
func secondaryTestClient(t *testing.T) *Client {
t.Helper()
return testClient(t, testprofiles.Secondary)
}

func thirdTestClient(t *testing.T) *Client {
t.Helper()
return testClient(t, testprofiles.Third)
}

func fourthTestClient(t *testing.T) *Client {
t.Helper()
return testClient(t, testprofiles.Fourth)
}

func testClient(t *testing.T, profile string) *Client {
t.Helper()

config, err := ProfileConfig(testprofiles.Secondary)
config, err := ProfileConfig(profile)
if err != nil {
t.Skipf("Snowflake secondary account not configured. Must be set in ~./snowflake/config.yml with profile name: %s", testprofiles.Secondary)
t.Skipf("Snowflake %s profile not configured. Must be set in ~./snowflake/config.yml", profile)
}
client, err := NewClient(config)
if err != nil {
t.Skipf("Snowflake secondary account not configured. Must be set in ~./snowflake/config.yml with profile name: %s", testprofiles.Secondary)
t.Skipf("Snowflake %s profile not configured. Must be set in ~./snowflake/config.yml", profile)
}

return client
Expand Down
154 changes: 123 additions & 31 deletions pkg/sdk/sweepers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package sdk

import (
"context"
"errors"
"fmt"
"log"
"slices"
"testing"
"time"

Expand All @@ -17,8 +19,8 @@ func TestSweepAll(t *testing.T) {
testenvs.AssertEnvSet(t, string(testenvs.TestObjectsSuffix))

t.Run("sweep after tests", func(t *testing.T) {
client := testClient(t)
secondaryClient := testSecondaryClient(t)
client := defaultTestClient(t)
secondaryClient := secondaryTestClient(t)

err := SweepAfterIntegrationTests(client, random.IntegrationTestsSuffix)
assert.NoError(t, err)
Expand All @@ -37,90 +39,180 @@ func TestSweepAll(t *testing.T) {
func Test_Sweeper_NukeStaleObjects(t *testing.T) {
_ = testenvs.GetOrSkipTest(t, testenvs.EnableSweep)

t.Run("sweep integration test precreated objects", func(t *testing.T) {
client := testClient(t)
secondaryClient := testSecondaryClient(t)
client := defaultTestClient(t)
secondaryClient := secondaryTestClient(t)
thirdClient := thirdTestClient(t)
fourthClient := fourthTestClient(t)

err := nukeWarehouses(client, "int_test_wh_%")()
assert.NoError(t, err)
allClients := []*Client{client, secondaryClient, thirdClient, fourthClient}

err = nukeWarehouses(secondaryClient, "int_test_wh_%")()
assert.NoError(t, err)
// can't use extracted IntegrationTestPrefix and AcceptanceTestPrefix until sweepers reside in the SDK package (cyclic)
const integrationTestPrefix = "int_test_"
const acceptanceTestPrefix = "acc_test_"

err = nukeDatabases(client, "int_test_db_%")()
assert.NoError(t, err)
t.Run("sweep integration test precreated objects", func(t *testing.T) {
integrationTestWarehousesPrefix := fmt.Sprintf("%swh_%%", integrationTestPrefix)
integrationTestDatabasesPrefix := fmt.Sprintf("%sdb_%%", integrationTestPrefix)

err = nukeDatabases(secondaryClient, "int_test_db_%")()
assert.NoError(t, err)
for _, c := range allClients {
err := nukeWarehouses(c, integrationTestWarehousesPrefix)()
assert.NoError(t, err)

err = nukeDatabases(c, integrationTestDatabasesPrefix)()
assert.NoError(t, err)
}
})

t.Run("sweep acceptance tests precreated objects", func(t *testing.T) {
client := testClient(t)
secondaryClient := testSecondaryClient(t)
acceptanceTestWarehousesPrefix := fmt.Sprintf("%swh_%%", acceptanceTestPrefix)
acceptanceTestDatabasesPrefix := fmt.Sprintf("%sdb_%%", acceptanceTestPrefix)

err := nukeWarehouses(client, "acc_test_wh_%")()
assert.NoError(t, err)
for _, c := range allClients {
err := nukeWarehouses(c, acceptanceTestWarehousesPrefix)()
assert.NoError(t, err)

err = nukeWarehouses(secondaryClient, "acc_test_wh_%")()
assert.NoError(t, err)
err = nukeDatabases(c, acceptanceTestDatabasesPrefix)()
assert.NoError(t, err)
}
})

err = nukeDatabases(client, "acc_test_db_%")()
assert.NoError(t, err)
t.Run("sweep users", func(t *testing.T) {
for _, c := range allClients {
err := nukeUsers(c)()
assert.NoError(t, err)
}
})

err = nukeDatabases(secondaryClient, "acc_test_db_%")()
assert.NoError(t, err)
// TODO [SNOW-955520]:
t.Run("sweep databases", func(t *testing.T) {
t.Skipf("Used for manual sweeping; will be addressed during SNOW-955520")
for _, c := range allClients {
err := nukeDatabases(c, "")()
assert.NoError(t, err)
}
})

// TODO [SNOW-955520]:
t.Run("sweep warehouses", func(t *testing.T) {
t.Skipf("Used for manual sweeping; will be addressed during SNOW-955520")
for _, c := range allClients {
err := nukeWarehouses(c, "")()
assert.NoError(t, err)
}
})

// TODO [SNOW-955520]: nuke stale objects (e.g. created more than 2 weeks ago)
}

// TODO [SNOW-955520]: generalize nuke methods (sweepers too)
func nukeWarehouses(client *Client, prefix string) func() error {
protectedWarehouses := []string{
"SNOWFLAKE",
"SYSTEM$STREAMLIT_NOTEBOOK_WH",
}

return func() error {
log.Printf("[DEBUG] Nuking warehouses with prefix %s\n", prefix)
ctx := context.Background()

whs, err := client.Warehouses.Show(ctx, &ShowWarehouseOptions{Like: &Like{Pattern: String(prefix)}})
var like *Like = nil
if prefix != "" {
like = &Like{Pattern: String(prefix)}
}

whs, err := client.Warehouses.Show(ctx, &ShowWarehouseOptions{Like: like})
if err != nil {
return fmt.Errorf("sweeping warehouses ended with error, err = %w", err)
}
var errs []error
log.Printf("[DEBUG] Found %d warehouses matching search criteria\n", len(whs))
for idx, wh := range whs {
log.Printf("[DEBUG] Processing warehouse [%d/%d]: %s...\n", idx+1, len(whs), wh.ID().FullyQualifiedName())
if wh.Name != "SNOWFLAKE" && wh.CreatedOn.Before(time.Now().Add(-4*time.Hour)) {
if !slices.Contains(protectedWarehouses, wh.Name) && wh.CreatedOn.Before(time.Now().Add(-2*time.Hour)) {
log.Printf("[DEBUG] Dropping warehouse %s, created at: %s\n", wh.ID().FullyQualifiedName(), wh.CreatedOn.String())
if err := client.Warehouses.Drop(ctx, wh.ID(), &DropWarehouseOptions{IfExists: Bool(true)}); err != nil {
return fmt.Errorf("sweeping warehouse %s ended with error, err = %w", wh.ID().FullyQualifiedName(), err)
log.Printf("[DEBUG] Dropping warehouse %s, resulted in error %v\n", wh.ID().FullyQualifiedName(), err)
errs = append(errs, fmt.Errorf("sweeping warehouse %s ended with error, err = %w", wh.ID().FullyQualifiedName(), err))
}
} else {
log.Printf("[DEBUG] Skipping warehouse %s, created at: %s\n", wh.ID().FullyQualifiedName(), wh.CreatedOn.String())
}
}
return nil
return errors.Join(errs...)
}
}

func nukeDatabases(client *Client, prefix string) func() error {
protectedDatabases := []string{
"SNOWFLAKE",
"MFA_ENFORCEMENT_POLICY",
}

return func() error {
log.Printf("[DEBUG] Nuking databases with prefix %s\n", prefix)
ctx := context.Background()

dbs, err := client.Databases.Show(ctx, &ShowDatabasesOptions{Like: &Like{Pattern: String(prefix)}})
var like *Like = nil
if prefix != "" {
like = &Like{Pattern: String(prefix)}
}
sfc-gh-jmichalak marked this conversation as resolved.
Show resolved Hide resolved
dbs, err := client.Databases.Show(ctx, &ShowDatabasesOptions{Like: like})
if err != nil {
return fmt.Errorf("sweeping databases ended with error, err = %w", err)
}
var errs []error
log.Printf("[DEBUG] Found %d databases matching search criteria\n", len(dbs))
for idx, db := range dbs {
log.Printf("[DEBUG] Processing database [%d/%d]: %s...\n", idx+1, len(dbs), db.ID().FullyQualifiedName())
if db.Name != "SNOWFLAKE" && db.CreatedOn.Before(time.Now().Add(-4*time.Hour)) {
if !slices.Contains(protectedDatabases, db.Name) && db.CreatedOn.Before(time.Now().Add(-2*time.Hour)) {
log.Printf("[DEBUG] Dropping database %s, created at: %s\n", db.ID().FullyQualifiedName(), db.CreatedOn.String())
if err := client.Databases.Drop(ctx, db.ID(), &DropDatabaseOptions{IfExists: Bool(true)}); err != nil {
return fmt.Errorf("sweeping database %s ended with error, err = %w", db.ID().FullyQualifiedName(), err)
log.Printf("[DEBUG] Dropping database %s, resulted in error %v\n", db.ID().FullyQualifiedName(), err)
errs = append(errs, fmt.Errorf("sweeping database %s ended with error, err = %w", db.ID().FullyQualifiedName(), err))
}
} else {
log.Printf("[DEBUG] Skipping database %s, created at: %s\n", db.ID().FullyQualifiedName(), db.CreatedOn.String())
}
}
return nil
return errors.Join(errs...)
}
}

func nukeUsers(client *Client) func() error {
protectedUsers := []string{
"SNOWFLAKE",
"ARTUR_SAWICKI",
"ARTUR_SAWICKI_LEGACY",
"JAKUB_MICHALAK",
"JAKUB_MICHALAK_LEGACY",
"JAN_CIESLAK",
"JAN_CIESLAK_LEGACY",
"TERRAFORM_SVC_ACCOUNT",
"TEST_CI_SERVICE_USER",
}

return func() error {
log.Println("[DEBUG] Nuking users")
ctx := context.Background()

users, err := client.Users.Show(ctx, &ShowUserOptions{})
if err != nil {
return fmt.Errorf("sweeping users ended with error, err = %w", err)
}
var errs []error
log.Printf("[DEBUG] Found %d users\n", len(users))
for idx, user := range users {
log.Printf("[DEBUG] Processing user [%d/%d]: %s...\n", idx+1, len(users), user.ID().FullyQualifiedName())
if !slices.Contains(protectedUsers, user.Name) && user.CreatedOn.Before(time.Now().Add(-2*time.Hour)) {
log.Printf("[DEBUG] Dropping user %s\n", user.ID().FullyQualifiedName())
if err := client.Users.Drop(ctx, user.ID(), &DropUserOptions{IfExists: Bool(true)}); err != nil {
log.Printf("[DEBUG] Dropping user %s, resulted in error %v\n", user.ID().FullyQualifiedName(), err)
errs = append(errs, fmt.Errorf("sweeping user %s ended with error, err = %w", user.ID().FullyQualifiedName(), err))
}
} else {
log.Printf("[DEBUG] Skipping user %s\n", user.ID().FullyQualifiedName())
}
}
return errors.Join(errs...)
}
}
8 changes: 5 additions & 3 deletions pkg/sdk/testint/setup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@ import (
"github.com/snowflakedb/gosnowflake"
)

const IntegrationTestPrefix = "int_test_"

var (
TestWarehouseName = "int_test_wh_" + random.IntegrationTestsSuffix
TestDatabaseName = "int_test_db_" + random.IntegrationTestsSuffix
TestSchemaName = "int_test_sc_" + random.IntegrationTestsSuffix
TestWarehouseName = fmt.Sprintf("%swh_%s", IntegrationTestPrefix, random.IntegrationTestsSuffix)
TestDatabaseName = fmt.Sprintf("%sdb_%s", IntegrationTestPrefix, random.IntegrationTestsSuffix)
TestSchemaName = fmt.Sprintf("%ssc_%s", IntegrationTestPrefix, random.IntegrationTestsSuffix)

NonExistingAccountObjectIdentifier = sdk.NewAccountObjectIdentifier("does_not_exist")
NonExistingDatabaseObjectIdentifier = sdk.NewDatabaseObjectIdentifier(TestDatabaseName, "does_not_exist")
Expand Down
Loading
Loading