[v8] skip databases that are not available during auto discovery (#10699)

lib/services/database.go

@@ -335,6 +335,74 @@ func IsRDSClusterSupported(cluster *rds.DBCluster) bool {
 	return true
 }
 
+// IsRDSInstanceAvailable checks if the RDS instance is available.
+func IsRDSInstanceAvailable(instance *rds.DBInstance) bool {
+	// For a full list of status values, see:
+	// https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/accessing-monitoring.html
+	switch aws.StringValue(instance.DBInstanceStatus) {
+	// Statuses marked as "Billed" in the above guide.
+	case "available", "backing-up", "configuring-enhanced-monitoring",
+		"configuring-iam-database-auth", "configuring-log-exports",
+		"converting-to-vpc", "incompatible-option-group",
+		"incompatible-parameters", "maintenance", "modifying", "moving-to-vpc",
+		"rebooting", "resetting-master-credentials", "renaming", "restore-error",
+		"storage-full", "storage-optimization", "upgrading":
+		return true
+
+	// Statuses marked as "Not billed" in the above guide.
+	case "creating", "deleting", "failed",
+		"inaccessible-encryption-credentials", "incompatible-network",
+		"incompatible-restore":
+		return false
+
+	// Statuses marked as "Billed for storage" in the above guide.
+	case "inaccessible-encryption-credentials-recoverable", "starting",
+		"stopped", "stopping":
+		return false
+
+	// Statuses that have no billing information in the above guide, but
+	// believed to be unavailable.
+	case "insufficient-capacity":
+		return false
+
+	default:
+		log.Warnf("Unknown status type: %q. Assuming RDS instance %q is available.",
+			aws.StringValue(instance.DBInstanceStatus),
+			aws.StringValue(instance.DBInstanceIdentifier),
+		)
+		return true
+	}
+}
+
+// IsRDSClusterAvailable checks if the RDS cluster is available.
+func IsRDSClusterAvailable(cluster *rds.DBCluster) bool {
+	// For a full list of status values, see:
+	// https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/accessing-monitoring.html
+	switch aws.StringValue(cluster.Status) {
+	// Statuses marked as "Billed" in the above guide.
+	case "available", "backing-up", "backtracking", "failing-over",
+		"maintenance", "migrating", "modifying", "promoting", "renaming",
+		"resetting-master-credentials", "update-iam-db-auth", "upgrading":
+		return true
+
+	// Statuses marked as "Not billed" in the above guide.
+	case "cloning-failed", "creating", "deleting",
+		"inaccessible-encryption-credentials", "migration-failed":
+		return false
+
+	// Statuses marked as "Billed for storage" in the above guide.
+	case "starting", "stopped", "stopping":
+		return false
+
+	default:
+		log.Warnf("Unknown status type: %q. Assuming Aurora cluster %q is available.",
+			aws.StringValue(cluster.Status),
+			aws.StringValue(cluster.DBClusterIdentifier),
+		)
+		return true
+	}
+}
+
 // auroraMySQLVersion extracts aurora mysql version from engine version
 func auroraMySQLVersion(cluster *rds.DBCluster) string {
 	// version guide: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Updates.Versions.html

lib/srv/db/cloud/watchers/rds.go

@@ -106,6 +106,13 @@ func (f *rdsDBInstancesFetcher) getRDSDatabases(ctx context.Context) (types.Data
 	}
 	databases := make(types.Databases, 0, len(instances))
 	for _, instance := range instances {
+		if !services.IsRDSInstanceAvailable(instance) {
+			f.log.Debugf("The current status of RDS instance %q is %q. Skipping.",
+				aws.StringValue(instance.DBInstanceIdentifier),
+				aws.StringValue(instance.DBInstanceStatus))
+			continue
+		}
+
 		database, err := services.NewDatabaseFromRDSInstance(instance)
 		if err != nil {
 			f.log.Warnf("Could not convert RDS instance %q to database resource: %v.",
@@ -195,6 +202,13 @@ func (f *rdsAuroraClustersFetcher) getAuroraDatabases(ctx context.Context) (type
 			continue
 		}
 
+		if !services.IsRDSClusterAvailable(cluster) {
+			f.log.Debugf("The current status of Aurora cluster %q is %q. Skipping.",
+				aws.StringValue(cluster.DBClusterIdentifier),
+				aws.StringValue(cluster.Status))
+			continue
+		}
+
 		// Add a database from primary endpoint
 		database, err := services.NewDatabaseFromRDSCluster(cluster)
 		if err != nil {

lib/srv/db/cloud/watchers/watcher_test.go

@@ -43,11 +43,15 @@ func TestWatcher(t *testing.T) {
 	rdsInstance2, _ := makeRDSInstance(t, "instance-2", "us-east-2", map[string]string{"env": "prod"})
 	rdsInstance3, _ := makeRDSInstance(t, "instance-3", "us-east-1", map[string]string{"env": "dev"})
 	rdsInstance4, rdsDatabase4 := makeRDSInstance(t, "instance-4", "us-west-1", nil)
+	rdsInstanceUnavailable, _ := makeRDSInstance(t, "instance-5", "us-west-1", nil, withRDSInstanceStatus("stopped"))
+	rdsInstanceUnknownStatus, rdsDatabaseUnknownStatus := makeRDSInstance(t, "instance-5", "us-west-6", nil, withRDSInstanceStatus("status-does-not-exist"))
 
-	auroraCluster1, auroraDatabase1 := makeRDSCluster(t, "cluster-1", "us-east-1", services.RDSEngineModeProvisioned, map[string]string{"env": "prod"})
+	auroraCluster1, auroraDatabase1 := makeRDSCluster(t, "cluster-1", "us-east-1", map[string]string{"env": "prod"})
 	auroraCluster2, auroraDatabases2 := makeRDSClusterWithExtraEndpoints(t, "cluster-2", "us-east-2", map[string]string{"env": "dev"})
-	auroraCluster3, _ := makeRDSCluster(t, "cluster-3", "us-east-2", services.RDSEngineModeProvisioned, map[string]string{"env": "prod"})
-	auroraClusterUnsupported, _ := makeRDSCluster(t, "serverless", "us-east-1", services.RDSEngineModeServerless, map[string]string{"env": "prod"})
+	auroraCluster3, _ := makeRDSCluster(t, "cluster-3", "us-east-2", map[string]string{"env": "prod"})
+	auroraClusterUnsupported, _ := makeRDSCluster(t, "serverless", "us-east-1", nil, withRDSClusterEngineMode("serverless"))
+	auroraClusterUnavailable, _ := makeRDSCluster(t, "cluster-4", "us-east-1", nil, withRDSClusterStatus("creating"))
+	auroraClusterUnknownStatus, auroraDatabaseUnknownStatus := makeRDSCluster(t, "cluster-5", "us-east-1", nil, withRDSClusterStatus("status-does-not-exist"))
 
 	tests := []struct {
 		name              string
@@ -56,7 +60,7 @@ func TestWatcher(t *testing.T) {
 		expectedDatabases types.Databases
 	}{
 		{
-			name: "rds labels matching",
+			name: "RDS labels matching",
 			awsMatchers: []services.AWSMatcher{
 				{
 					Types:   []string{services.AWSMatcherRDS},
@@ -84,7 +88,7 @@ func TestWatcher(t *testing.T) {
 			expectedDatabases: append(types.Databases{rdsDatabase1, auroraDatabase1}, auroraDatabases2...),
 		},
 		{
-			name: "rds aurora unsupported",
+			name: "RDS unsupported databases are skipped",
 			awsMatchers: []services.AWSMatcher{{
 				Types:   []string{services.AWSMatcherRDS},
 				Regions: []string{"us-east-1"},
@@ -99,6 +103,21 @@ func TestWatcher(t *testing.T) {
 			},
 			expectedDatabases: types.Databases{auroraDatabase1},
 		},
+		{
+			name: "RDS unavailable databases are skipped",
+			awsMatchers: []services.AWSMatcher{{
+				Types:   []string{services.AWSMatcherRDS},
+				Regions: []string{"us-east-1"},
+				Tags:    types.Labels{"*": []string{"*"}},
+			}},
+			clients: &common.TestCloudClients{
+				RDS: &cloud.RDSMock{
+					DBInstances: []*rds.DBInstance{rdsInstance1, rdsInstanceUnavailable, rdsInstanceUnknownStatus},
+					DBClusters:  []*rds.DBCluster{auroraCluster1, auroraClusterUnavailable, auroraClusterUnknownStatus},
+				},
+			},
+			expectedDatabases: types.Databases{rdsDatabase1, rdsDatabaseUnknownStatus, auroraDatabase1, auroraDatabaseUnknownStatus},
+		},
 		{
 			name: "skip access denied errors",
 			awsMatchers: []services.AWSMatcher{{
@@ -139,34 +158,44 @@ func TestWatcher(t *testing.T) {
 	}
 }
 
-func makeRDSInstance(t *testing.T, name, region string, labels map[string]string) (*rds.DBInstance, types.Database) {
+func makeRDSInstance(t *testing.T, name, region string, labels map[string]string, opts ...func(*rds.DBInstance)) (*rds.DBInstance, types.Database) {
 	instance := &rds.DBInstance{
 		DBInstanceArn:        aws.String(fmt.Sprintf("arn:aws:rds:%v:1234567890:db:%v", region, name)),
 		DBInstanceIdentifier: aws.String(name),
 		DbiResourceId:        aws.String(uuid.New()),
 		Engine:               aws.String(services.RDSEnginePostgres),
+		DBInstanceStatus:     aws.String("available"),
 		Endpoint: &rds.Endpoint{
 			Address: aws.String("localhost"),
 			Port:    aws.Int64(5432),
 		},
 		TagList: labelsToTags(labels),
 	}
+	for _, opt := range opts {
+		opt(instance)
+	}
+
 	database, err := services.NewDatabaseFromRDSInstance(instance)
 	require.NoError(t, err)
 	return instance, database
 }
 
-func makeRDSCluster(t *testing.T, name, region, engineMode string, labels map[string]string) (*rds.DBCluster, types.Database) {
+func makeRDSCluster(t *testing.T, name, region string, labels map[string]string, opts ...func(*rds.DBCluster)) (*rds.DBCluster, types.Database) {
 	cluster := &rds.DBCluster{
 		DBClusterArn:        aws.String(fmt.Sprintf("arn:aws:rds:%v:1234567890:cluster:%v", region, name)),
 		DBClusterIdentifier: aws.String(name),
 		DbClusterResourceId: aws.String(uuid.New()),
 		Engine:              aws.String(services.RDSEngineAuroraMySQL),
-		EngineMode:          aws.String(engineMode),
+		EngineMode:          aws.String(services.RDSEngineModeProvisioned),
+		Status:              aws.String("available"),
 		Endpoint:            aws.String("localhost"),
 		Port:                aws.Int64(3306),
 		TagList:             labelsToTags(labels),
 	}
+	for _, opt := range opts {
+		opt(cluster)
+	}
+
 	database, err := services.NewDatabaseFromRDSCluster(cluster)
 	require.NoError(t, err)
 	return cluster, database
@@ -179,6 +208,7 @@ func makeRDSClusterWithExtraEndpoints(t *testing.T, name, region string, labels
 		DbClusterResourceId: aws.String(uuid.New()),
 		Engine:              aws.String(services.RDSEngineAuroraMySQL),
 		EngineMode:          aws.String(services.RDSEngineModeProvisioned),
+		Status:              aws.String("available"),
 		Endpoint:            aws.String("localhost"),
 		ReaderEndpoint:      aws.String("reader.host"),
 		Port:                aws.Int64(3306),
@@ -202,6 +232,27 @@ func makeRDSClusterWithExtraEndpoints(t *testing.T, name, region string, labels
 	return cluster, append(types.Databases{primaryDatabase, readerDatabase}, customDatabases...)
 }
 
+// withRDSInstanceStatus returns an option function for makeRDSInstance to overwrite status.
+func withRDSInstanceStatus(status string) func(*rds.DBInstance) {
+	return func(instance *rds.DBInstance) {
+		instance.DBInstanceStatus = aws.String(status)
+	}
+}
+
+// withRDSClusterEngineMode returns an option function for makeRDSCluster to overwrite engine mode.
+func withRDSClusterEngineMode(mode string) func(*rds.DBCluster) {
+	return func(cluster *rds.DBCluster) {
+		cluster.EngineMode = aws.String(mode)
+	}
+}
+
+// withRDSClusterStatus returns an option function for makeRDSCluster to overwrite status.
+func withRDSClusterStatus(status string) func(*rds.DBCluster) {
+	return func(cluster *rds.DBCluster) {
+		cluster.Status = aws.String(status)
+	}
+}
+
 func labelsToTags(labels map[string]string) (tags []*rds.Tag) {
 	for key, val := range labels {
 		tags = append(tags, &rds.Tag{