Skip to content

Commit

Permalink
skip databases that are not available during auto discovery (#10699)
Browse files Browse the repository at this point in the history
  • Loading branch information
@greedy52
greedy52 committed Mar 8, 2022
1 parent 3db54b3 commit eb02f8b
Show file tree
Hide file tree
Showing 3 changed files with 141 additions and 8 deletions.
68 changes: 68 additions & 0 deletions lib/services/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,74 @@ func IsRDSClusterSupported(cluster *rds.DBCluster) bool {
return true
}

// IsRDSInstanceAvailable checks if the RDS instance is available.
func IsRDSInstanceAvailable(instance *rds.DBInstance) bool {
// For a full list of status values, see:
// https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/accessing-monitoring.html
switch aws.StringValue(instance.DBInstanceStatus) {
// Statuses marked as "Billed" in the above guide.
case "available", "backing-up", "configuring-enhanced-monitoring",
"configuring-iam-database-auth", "configuring-log-exports",
"converting-to-vpc", "incompatible-option-group",
"incompatible-parameters", "maintenance", "modifying", "moving-to-vpc",
"rebooting", "resetting-master-credentials", "renaming", "restore-error",
"storage-full", "storage-optimization", "upgrading":
return true

// Statuses marked as "Not billed" in the above guide.
case "creating", "deleting", "failed",
"inaccessible-encryption-credentials", "incompatible-network",
"incompatible-restore":
return false

// Statuses marked as "Billed for storage" in the above guide.
case "inaccessible-encryption-credentials-recoverable", "starting",
"stopped", "stopping":
return false

// Statuses that have no billing information in the above guide, but
// believed to be unavailable.
case "insufficient-capacity":
return false

default:
log.Warnf("Unknown status type: %q. Assuming RDS instance %q is available.",
aws.StringValue(instance.DBInstanceStatus),
aws.StringValue(instance.DBInstanceIdentifier),
)
return true
}
}

// IsRDSClusterAvailable checks if the RDS cluster is available.
func IsRDSClusterAvailable(cluster *rds.DBCluster) bool {
// For a full list of status values, see:
// https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/accessing-monitoring.html
switch aws.StringValue(cluster.Status) {
// Statuses marked as "Billed" in the above guide.
case "available", "backing-up", "backtracking", "failing-over",
"maintenance", "migrating", "modifying", "promoting", "renaming",
"resetting-master-credentials", "update-iam-db-auth", "upgrading":
return true

// Statuses marked as "Not billed" in the above guide.
case "cloning-failed", "creating", "deleting",
"inaccessible-encryption-credentials", "migration-failed":
return false

// Statuses marked as "Billed for storage" in the above guide.
case "starting", "stopped", "stopping":
return false

default:
log.Warnf("Unknown status type: %q. Assuming Aurora cluster %q is available.",
aws.StringValue(cluster.Status),
aws.StringValue(cluster.DBClusterIdentifier),
)
return true
}
}

// auroraMySQLVersion extracts aurora mysql version from engine version
func auroraMySQLVersion(cluster *rds.DBCluster) string {
// version guide: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Updates.Versions.html
Expand Down
14 changes: 14 additions & 0 deletions lib/srv/db/cloud/watchers/rds.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,13 @@ func (f *rdsDBInstancesFetcher) getRDSDatabases(ctx context.Context) (types.Data
}
databases := make(types.Databases, 0, len(instances))
for _, instance := range instances {
if !services.IsRDSInstanceAvailable(instance) {
f.log.Debugf("The current status of RDS instance %q is %q. Skipping.",
aws.StringValue(instance.DBInstanceIdentifier),
aws.StringValue(instance.DBInstanceStatus))
continue
}

database, err := services.NewDatabaseFromRDSInstance(instance)
if err != nil {
f.log.Warnf("Could not convert RDS instance %q to database resource: %v.",
Expand Down Expand Up @@ -195,6 +202,13 @@ func (f *rdsAuroraClustersFetcher) getAuroraDatabases(ctx context.Context) (type
continue
}

if !services.IsRDSClusterAvailable(cluster) {
f.log.Debugf("The current status of Aurora cluster %q is %q. Skipping.",
aws.StringValue(cluster.DBClusterIdentifier),
aws.StringValue(cluster.Status))
continue
}

// Add a database from primary endpoint
database, err := services.NewDatabaseFromRDSCluster(cluster)
if err != nil {
Expand Down
67 changes: 59 additions & 8 deletions lib/srv/db/cloud/watchers/watcher_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,15 @@ func TestWatcher(t *testing.T) {
rdsInstance2, _ := makeRDSInstance(t, "instance-2", "us-east-2", map[string]string{"env": "prod"})
rdsInstance3, _ := makeRDSInstance(t, "instance-3", "us-east-1", map[string]string{"env": "dev"})
rdsInstance4, rdsDatabase4 := makeRDSInstance(t, "instance-4", "us-west-1", nil)
rdsInstanceUnavailable, _ := makeRDSInstance(t, "instance-5", "us-west-1", nil, withRDSInstanceStatus("stopped"))
rdsInstanceUnknownStatus, rdsDatabaseUnknownStatus := makeRDSInstance(t, "instance-5", "us-west-6", nil, withRDSInstanceStatus("status-does-not-exist"))

auroraCluster1, auroraDatabase1 := makeRDSCluster(t, "cluster-1", "us-east-1", services.RDSEngineModeProvisioned, map[string]string{"env": "prod"})
auroraCluster1, auroraDatabase1 := makeRDSCluster(t, "cluster-1", "us-east-1", map[string]string{"env": "prod"})
auroraCluster2, auroraDatabases2 := makeRDSClusterWithExtraEndpoints(t, "cluster-2", "us-east-2", map[string]string{"env": "dev"})
auroraCluster3, _ := makeRDSCluster(t, "cluster-3", "us-east-2", services.RDSEngineModeProvisioned, map[string]string{"env": "prod"})
auroraClusterUnsupported, _ := makeRDSCluster(t, "serverless", "us-east-1", services.RDSEngineModeServerless, map[string]string{"env": "prod"})
auroraCluster3, _ := makeRDSCluster(t, "cluster-3", "us-east-2", map[string]string{"env": "prod"})
auroraClusterUnsupported, _ := makeRDSCluster(t, "serverless", "us-east-1", nil, withRDSClusterEngineMode("serverless"))
auroraClusterUnavailable, _ := makeRDSCluster(t, "cluster-4", "us-east-1", nil, withRDSClusterStatus("creating"))
auroraClusterUnknownStatus, auroraDatabaseUnknownStatus := makeRDSCluster(t, "cluster-5", "us-east-1", nil, withRDSClusterStatus("status-does-not-exist"))

tests := []struct {
name string
Expand All @@ -56,7 +60,7 @@ func TestWatcher(t *testing.T) {
expectedDatabases types.Databases
}{
{
name: "rds labels matching",
name: "RDS labels matching",
awsMatchers: []services.AWSMatcher{
{
Types: []string{services.AWSMatcherRDS},
Expand Down Expand Up @@ -84,7 +88,7 @@ func TestWatcher(t *testing.T) {
expectedDatabases: append(types.Databases{rdsDatabase1, auroraDatabase1}, auroraDatabases2...),
},
{
name: "rds aurora unsupported",
name: "RDS unsupported databases are skipped",
awsMatchers: []services.AWSMatcher{{
Types: []string{services.AWSMatcherRDS},
Regions: []string{"us-east-1"},
Expand All @@ -99,6 +103,21 @@ func TestWatcher(t *testing.T) {
},
expectedDatabases: types.Databases{auroraDatabase1},
},
{
name: "RDS unavailable databases are skipped",
awsMatchers: []services.AWSMatcher{{
Types: []string{services.AWSMatcherRDS},
Regions: []string{"us-east-1"},
Tags: types.Labels{"*": []string{"*"}},
}},
clients: &common.TestCloudClients{
RDS: &cloud.RDSMock{
DBInstances: []*rds.DBInstance{rdsInstance1, rdsInstanceUnavailable, rdsInstanceUnknownStatus},
DBClusters: []*rds.DBCluster{auroraCluster1, auroraClusterUnavailable, auroraClusterUnknownStatus},
},
},
expectedDatabases: types.Databases{rdsDatabase1, rdsDatabaseUnknownStatus, auroraDatabase1, auroraDatabaseUnknownStatus},
},
{
name: "skip access denied errors",
awsMatchers: []services.AWSMatcher{{
Expand Down Expand Up @@ -139,34 +158,44 @@ func TestWatcher(t *testing.T) {
}
}

func makeRDSInstance(t *testing.T, name, region string, labels map[string]string) (*rds.DBInstance, types.Database) {
func makeRDSInstance(t *testing.T, name, region string, labels map[string]string, opts ...func(*rds.DBInstance)) (*rds.DBInstance, types.Database) {
instance := &rds.DBInstance{
DBInstanceArn: aws.String(fmt.Sprintf("arn:aws:rds:%v:1234567890:db:%v", region, name)),
DBInstanceIdentifier: aws.String(name),
DbiResourceId: aws.String(uuid.New()),
Engine: aws.String(services.RDSEnginePostgres),
DBInstanceStatus: aws.String("available"),
Endpoint: &rds.Endpoint{
Address: aws.String("localhost"),
Port: aws.Int64(5432),
},
TagList: labelsToTags(labels),
}
for _, opt := range opts {
opt(instance)
}

database, err := services.NewDatabaseFromRDSInstance(instance)
require.NoError(t, err)
return instance, database
}

func makeRDSCluster(t *testing.T, name, region, engineMode string, labels map[string]string) (*rds.DBCluster, types.Database) {
func makeRDSCluster(t *testing.T, name, region string, labels map[string]string, opts ...func(*rds.DBCluster)) (*rds.DBCluster, types.Database) {
cluster := &rds.DBCluster{
DBClusterArn: aws.String(fmt.Sprintf("arn:aws:rds:%v:1234567890:cluster:%v", region, name)),
DBClusterIdentifier: aws.String(name),
DbClusterResourceId: aws.String(uuid.New()),
Engine: aws.String(services.RDSEngineAuroraMySQL),
EngineMode: aws.String(engineMode),
EngineMode: aws.String(services.RDSEngineModeProvisioned),
Status: aws.String("available"),
Endpoint: aws.String("localhost"),
Port: aws.Int64(3306),
TagList: labelsToTags(labels),
}
for _, opt := range opts {
opt(cluster)
}

database, err := services.NewDatabaseFromRDSCluster(cluster)
require.NoError(t, err)
return cluster, database
Expand All @@ -179,6 +208,7 @@ func makeRDSClusterWithExtraEndpoints(t *testing.T, name, region string, labels
DbClusterResourceId: aws.String(uuid.New()),
Engine: aws.String(services.RDSEngineAuroraMySQL),
EngineMode: aws.String(services.RDSEngineModeProvisioned),
Status: aws.String("available"),
Endpoint: aws.String("localhost"),
ReaderEndpoint: aws.String("reader.host"),
Port: aws.Int64(3306),
Expand All @@ -202,6 +232,27 @@ func makeRDSClusterWithExtraEndpoints(t *testing.T, name, region string, labels
return cluster, append(types.Databases{primaryDatabase, readerDatabase}, customDatabases...)
}

// withRDSInstanceStatus returns an option function for makeRDSInstance to overwrite status.
func withRDSInstanceStatus(status string) func(*rds.DBInstance) {
return func(instance *rds.DBInstance) {
instance.DBInstanceStatus = aws.String(status)
}
}

// withRDSClusterEngineMode returns an option function for makeRDSCluster to overwrite engine mode.
func withRDSClusterEngineMode(mode string) func(*rds.DBCluster) {
return func(cluster *rds.DBCluster) {
cluster.EngineMode = aws.String(mode)
}
}

// withRDSClusterStatus returns an option function for makeRDSCluster to overwrite status.
func withRDSClusterStatus(status string) func(*rds.DBCluster) {
return func(cluster *rds.DBCluster) {
cluster.Status = aws.String(status)
}
}

func labelsToTags(labels map[string]string) (tags []*rds.Tag) {
for key, val := range labels {
tags = append(tags, &rds.Tag{
Expand Down

0 comments on commit eb02f8b

Please sign in to comment.