diff --git a/lib/services/database.go b/lib/services/database.go index eed6b7bea19a6..12faa8aa21613 100644 --- a/lib/services/database.go +++ b/lib/services/database.go @@ -390,6 +390,107 @@ func IsRDSClusterSupported(cluster *rds.DBCluster) bool { return true } +// IsRDSInstanceAvailable checks if the RDS instance is available. +func IsRDSInstanceAvailable(instance *rds.DBInstance) bool { + // For a full list of status values, see: + // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/accessing-monitoring.html + switch aws.StringValue(instance.DBInstanceStatus) { + // Statuses marked as "Billed" in the above guide. + case "available", "backing-up", "configuring-enhanced-monitoring", + "configuring-iam-database-auth", "configuring-log-exports", + "converting-to-vpc", "incompatible-option-group", + "incompatible-parameters", "maintenance", "modifying", "moving-to-vpc", + "rebooting", "resetting-master-credentials", "renaming", "restore-error", + "storage-full", "storage-optimization", "upgrading": + return true + + // Statuses marked as "Not billed" in the above guide. + case "creating", "deleting", "failed", + "inaccessible-encryption-credentials", "incompatible-network", + "incompatible-restore": + return false + + // Statuses marked as "Billed for storage" in the above guide. + case "inaccessible-encryption-credentials-recoverable", "starting", + "stopped", "stopping": + return false + + // Statuses that have no billing information in the above guide, but + // believed to be unavailable. + case "insufficient-capacity": + return false + + default: + log.Warnf("Unknown status type: %q. Assuming RDS instance %q is available.", + aws.StringValue(instance.DBInstanceStatus), + aws.StringValue(instance.DBInstanceIdentifier), + ) + return true + } +} + +// IsRDSClusterAvailable checks if the RDS cluster is available. +func IsRDSClusterAvailable(cluster *rds.DBCluster) bool { + // For a full list of status values, see: + // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/accessing-monitoring.html + switch aws.StringValue(cluster.Status) { + // Statuses marked as "Billed" in the above guide. + case "available", "backing-up", "backtracking", "failing-over", + "maintenance", "migrating", "modifying", "promoting", "renaming", + "resetting-master-credentials", "update-iam-db-auth", "upgrading": + return true + + // Statuses marked as "Not billed" in the above guide. + case "cloning-failed", "creating", "deleting", + "inaccessible-encryption-credentials", "migration-failed": + return false + + // Statuses marked as "Billed for storage" in the above guide. + case "starting", "stopped", "stopping": + return false + + default: + log.Warnf("Unknown status type: %q. Assuming Aurora cluster %q is available.", + aws.StringValue(cluster.Status), + aws.StringValue(cluster.DBClusterIdentifier), + ) + return true + } +} + +// IsRedshiftClusterAvailable checks if the Redshift cluster is available. +func IsRedshiftClusterAvailable(cluster *redshift.Cluster) bool { + // For a full list of status values, see: + // https://docs.aws.amazon.com/redshift/latest/mgmt/working-with-clusters.html#rs-mgmt-cluster-status + // + // Note that the Redshift guide does not specify billing information like + // the RDS and Aurora guides do. Most Redshift statuses are + // cross-referenced with similar statuses from RDS and Aurora guides to + // determine the availability. + // + // For "incompatible-xxx" statuses, the cluster is assumed to be available + // if the status is resulted by modifying the cluster, and the cluster is + // assumed to be unavailable if the cluster cannot be created or restored. + switch aws.StringValue(cluster.ClusterStatus) { + case "available", "available, prep-for-resize", "available, resize-cleanup", + "cancelling-resize", "final-snapshot", "modifying", "rebooting", + "renaming", "resizing", "rotating-keys", "storage-full", "updating-hsm", + "incompatible-parameters", "incompatible-hsm": + return true + + case "creating", "deleting", "hardware-failure", "paused", + "incompatible-network": + return false + + default: + log.Warnf("Unknown status type: %q. Assuming Redshift cluster %q is available.", + aws.StringValue(cluster.ClusterStatus), + aws.StringValue(cluster.ClusterIdentifier), + ) + return true + } +} + // auroraMySQLVersion extracts aurora mysql version from engine version func auroraMySQLVersion(cluster *rds.DBCluster) string { // version guide: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Updates.Versions.html diff --git a/lib/srv/db/cloud/watchers/rds.go b/lib/srv/db/cloud/watchers/rds.go index 8d11c09d28334..3f697b9cbeda7 100644 --- a/lib/srv/db/cloud/watchers/rds.go +++ b/lib/srv/db/cloud/watchers/rds.go @@ -106,6 +106,13 @@ func (f *rdsDBInstancesFetcher) getRDSDatabases(ctx context.Context) (types.Data } databases := make(types.Databases, 0, len(instances)) for _, instance := range instances { + if !services.IsRDSInstanceAvailable(instance) { + f.log.Debugf("The current status of RDS instance %q is %q. Skipping.", + aws.StringValue(instance.DBInstanceIdentifier), + aws.StringValue(instance.DBInstanceStatus)) + continue + } + database, err := services.NewDatabaseFromRDSInstance(instance) if err != nil { f.log.Warnf("Could not convert RDS instance %q to database resource: %v.", @@ -195,6 +202,13 @@ func (f *rdsAuroraClustersFetcher) getAuroraDatabases(ctx context.Context) (type continue } + if !services.IsRDSClusterAvailable(cluster) { + f.log.Debugf("The current status of Aurora cluster %q is %q. Skipping.", + aws.StringValue(cluster.DBClusterIdentifier), + aws.StringValue(cluster.Status)) + continue + } + // Add a database from primary endpoint database, err := services.NewDatabaseFromRDSCluster(cluster) if err != nil { diff --git a/lib/srv/db/cloud/watchers/redshift.go b/lib/srv/db/cloud/watchers/redshift.go index 2a8f8a98ba7db..33baed413046d 100644 --- a/lib/srv/db/cloud/watchers/redshift.go +++ b/lib/srv/db/cloud/watchers/redshift.go @@ -86,6 +86,13 @@ func (f *redshiftFetcher) Get(ctx context.Context) (types.Databases, error) { var databases types.Databases for _, cluster := range clusters { + if !services.IsRedshiftClusterAvailable(cluster) { + f.log.Debugf("The current status of Redshift cluster %q is %q. Skipping.", + aws.StringValue(cluster.ClusterIdentifier), + aws.StringValue(cluster.ClusterStatus)) + continue + } + database, err := services.NewDatabaseFromRedshiftCluster(cluster) if err != nil { f.log.Infof("Could not convert Redshift cluster %q to database resource: %v.", diff --git a/lib/srv/db/cloud/watchers/watcher_test.go b/lib/srv/db/cloud/watchers/watcher_test.go index f8ab8f50e662d..bcb3ddb8fa7e7 100644 --- a/lib/srv/db/cloud/watchers/watcher_test.go +++ b/lib/srv/db/cloud/watchers/watcher_test.go @@ -44,14 +44,20 @@ func TestWatcher(t *testing.T) { rdsInstance2, _ := makeRDSInstance(t, "instance-2", "us-east-2", map[string]string{"env": "prod"}) rdsInstance3, _ := makeRDSInstance(t, "instance-3", "us-east-1", map[string]string{"env": "dev"}) rdsInstance4, rdsDatabase4 := makeRDSInstance(t, "instance-4", "us-west-1", nil) + rdsInstanceUnavailable, _ := makeRDSInstance(t, "instance-5", "us-west-1", nil, withRDSInstanceStatus("stopped")) + rdsInstanceUnknownStatus, rdsDatabaseUnknownStatus := makeRDSInstance(t, "instance-5", "us-west-6", nil, withRDSInstanceStatus("status-does-not-exist")) - auroraCluster1, auroraDatabase1 := makeRDSCluster(t, "cluster-1", "us-east-1", services.RDSEngineModeProvisioned, map[string]string{"env": "prod"}) + auroraCluster1, auroraDatabase1 := makeRDSCluster(t, "cluster-1", "us-east-1", map[string]string{"env": "prod"}) auroraCluster2, auroraDatabases2 := makeRDSClusterWithExtraEndpoints(t, "cluster-2", "us-east-2", map[string]string{"env": "dev"}) - auroraCluster3, _ := makeRDSCluster(t, "cluster-3", "us-east-2", services.RDSEngineModeProvisioned, map[string]string{"env": "prod"}) - auroraClusterUnsupported, _ := makeRDSCluster(t, "serverless", "us-east-1", services.RDSEngineModeServerless, map[string]string{"env": "prod"}) + auroraCluster3, _ := makeRDSCluster(t, "cluster-3", "us-east-2", map[string]string{"env": "prod"}) + auroraClusterUnsupported, _ := makeRDSCluster(t, "serverless", "us-east-1", nil, withRDSClusterEngineMode("serverless")) + auroraClusterUnavailable, _ := makeRDSCluster(t, "cluster-4", "us-east-1", nil, withRDSClusterStatus("creating")) + auroraClusterUnknownStatus, auroraDatabaseUnknownStatus := makeRDSCluster(t, "cluster-5", "us-east-1", nil, withRDSClusterStatus("status-does-not-exist")) redshiftUse1Prod, redshiftDatabaseUse1Prod := makeRedshiftCluster(t, "us-east-1", "prod") redshiftUse1Dev, _ := makeRedshiftCluster(t, "us-east-1", "dev") + redshiftUse1Unavailable, _ := makeRedshiftCluster(t, "us-east-1", "qa", withRedshiftStatus("paused")) + redshiftUse1UnknownStatus, redshiftDatabaseUnknownStatus := makeRedshiftCluster(t, "us-east-1", "test", withRedshiftStatus("status-does-not-exist")) tests := []struct { name string @@ -60,7 +66,7 @@ func TestWatcher(t *testing.T) { expectedDatabases types.Databases }{ { - name: "rds labels matching", + name: "RDS labels matching", awsMatchers: []services.AWSMatcher{ { Types: []string{services.AWSMatcherRDS}, @@ -88,7 +94,7 @@ func TestWatcher(t *testing.T) { expectedDatabases: append(types.Databases{rdsDatabase1, auroraDatabase1}, auroraDatabases2...), }, { - name: "rds aurora unsupported", + name: "RDS unsupported databases are skipped", awsMatchers: []services.AWSMatcher{{ Types: []string{services.AWSMatcherRDS}, Regions: []string{"us-east-1"}, @@ -103,6 +109,21 @@ func TestWatcher(t *testing.T) { }, expectedDatabases: types.Databases{auroraDatabase1}, }, + { + name: "RDS unavailable databases are skipped", + awsMatchers: []services.AWSMatcher{{ + Types: []string{services.AWSMatcherRDS}, + Regions: []string{"us-east-1"}, + Tags: types.Labels{"*": []string{"*"}}, + }}, + clients: &common.TestCloudClients{ + RDS: &cloud.RDSMock{ + DBInstances: []*rds.DBInstance{rdsInstance1, rdsInstanceUnavailable, rdsInstanceUnknownStatus}, + DBClusters: []*rds.DBCluster{auroraCluster1, auroraClusterUnavailable, auroraClusterUnknownStatus}, + }, + }, + expectedDatabases: types.Databases{rdsDatabase1, rdsDatabaseUnknownStatus, auroraDatabase1, auroraDatabaseUnknownStatus}, + }, { name: "skip access denied errors", awsMatchers: []services.AWSMatcher{{ @@ -126,7 +147,7 @@ func TestWatcher(t *testing.T) { expectedDatabases: types.Databases{rdsDatabase4, auroraDatabase1}, }, { - name: "redshift", + name: "Redshift labels matching", awsMatchers: []services.AWSMatcher{ { Types: []string{services.AWSMatcherRedshift}, @@ -141,6 +162,22 @@ func TestWatcher(t *testing.T) { }, expectedDatabases: types.Databases{redshiftDatabaseUse1Prod}, }, + { + name: "Redshift unavailable databases are skipped", + awsMatchers: []services.AWSMatcher{ + { + Types: []string{services.AWSMatcherRedshift}, + Regions: []string{"us-east-1"}, + Tags: types.Labels{"*": []string{"*"}}, + }, + }, + clients: &common.TestCloudClients{ + Redshift: &cloud.RedshiftMock{ + Clusters: []*redshift.Cluster{redshiftUse1Prod, redshiftUse1Unavailable, redshiftUse1UnknownStatus}, + }, + }, + expectedDatabases: types.Databases{redshiftDatabaseUse1Prod, redshiftDatabaseUnknownStatus}, + }, { name: "matcher with multiple types", awsMatchers: []services.AWSMatcher{ @@ -178,43 +215,54 @@ func TestWatcher(t *testing.T) { } } -func makeRDSInstance(t *testing.T, name, region string, labels map[string]string) (*rds.DBInstance, types.Database) { +func makeRDSInstance(t *testing.T, name, region string, labels map[string]string, opts ...func(*rds.DBInstance)) (*rds.DBInstance, types.Database) { instance := &rds.DBInstance{ DBInstanceArn: aws.String(fmt.Sprintf("arn:aws:rds:%v:1234567890:db:%v", region, name)), DBInstanceIdentifier: aws.String(name), DbiResourceId: aws.String(uuid.New().String()), Engine: aws.String(services.RDSEnginePostgres), + DBInstanceStatus: aws.String("available"), Endpoint: &rds.Endpoint{ Address: aws.String("localhost"), Port: aws.Int64(5432), }, TagList: labelsToTags(labels), } + for _, opt := range opts { + opt(instance) + } + database, err := services.NewDatabaseFromRDSInstance(instance) require.NoError(t, err) return instance, database } -func makeRDSCluster(t *testing.T, name, region, engineMode string, labels map[string]string) (*rds.DBCluster, types.Database) { +func makeRDSCluster(t *testing.T, name, region string, labels map[string]string, opts ...func(*rds.DBCluster)) (*rds.DBCluster, types.Database) { cluster := &rds.DBCluster{ DBClusterArn: aws.String(fmt.Sprintf("arn:aws:rds:%v:1234567890:cluster:%v", region, name)), DBClusterIdentifier: aws.String(name), DbClusterResourceId: aws.String(uuid.New().String()), Engine: aws.String(services.RDSEngineAuroraMySQL), - EngineMode: aws.String(engineMode), + EngineMode: aws.String(services.RDSEngineModeProvisioned), + Status: aws.String("available"), Endpoint: aws.String("localhost"), Port: aws.Int64(3306), TagList: labelsToTags(labels), } + for _, opt := range opts { + opt(cluster) + } + database, err := services.NewDatabaseFromRDSCluster(cluster) require.NoError(t, err) return cluster, database } -func makeRedshiftCluster(t *testing.T, region, env string) (*redshift.Cluster, types.Database) { +func makeRedshiftCluster(t *testing.T, region, env string, opts ...func(*redshift.Cluster)) (*redshift.Cluster, types.Database) { cluster := &redshift.Cluster{ ClusterIdentifier: aws.String(env), ClusterNamespaceArn: aws.String(fmt.Sprintf("arn:aws:redshift:%s:1234567890:namespace:%s", region, env)), + ClusterStatus: aws.String("available"), Endpoint: &redshift.Endpoint{ Address: aws.String("localhost"), Port: aws.Int64(5439), @@ -224,6 +272,10 @@ func makeRedshiftCluster(t *testing.T, region, env string) (*redshift.Cluster, t Value: aws.String(env), }}, } + for _, opt := range opts { + opt(cluster) + } + database, err := services.NewDatabaseFromRedshiftCluster(cluster) require.NoError(t, err) return cluster, database @@ -236,6 +288,7 @@ func makeRDSClusterWithExtraEndpoints(t *testing.T, name, region string, labels DbClusterResourceId: aws.String(uuid.New().String()), Engine: aws.String(services.RDSEngineAuroraMySQL), EngineMode: aws.String(services.RDSEngineModeProvisioned), + Status: aws.String("available"), Endpoint: aws.String("localhost"), ReaderEndpoint: aws.String("reader.host"), Port: aws.Int64(3306), @@ -259,6 +312,34 @@ func makeRDSClusterWithExtraEndpoints(t *testing.T, name, region string, labels return cluster, append(types.Databases{primaryDatabase, readerDatabase}, customDatabases...) } +// withRDSInstanceStatus returns an option function for makeRDSInstance to overwrite status. +func withRDSInstanceStatus(status string) func(*rds.DBInstance) { + return func(instance *rds.DBInstance) { + instance.DBInstanceStatus = aws.String(status) + } +} + +// withRDSClusterEngineMode returns an option function for makeRDSCluster to overwrite engine mode. +func withRDSClusterEngineMode(mode string) func(*rds.DBCluster) { + return func(cluster *rds.DBCluster) { + cluster.EngineMode = aws.String(mode) + } +} + +// withRDSClusterStatus returns an option function for makeRDSCluster to overwrite status. +func withRDSClusterStatus(status string) func(*rds.DBCluster) { + return func(cluster *rds.DBCluster) { + cluster.Status = aws.String(status) + } +} + +// withRedshiftStatus returns an option function for makeRedshiftCluster to overwrite status. +func withRedshiftStatus(status string) func(*redshift.Cluster) { + return func(cluster *redshift.Cluster) { + cluster.ClusterStatus = aws.String(status) + } +} + func labelsToTags(labels map[string]string) (tags []*rds.Tag) { for key, val := range labels { tags = append(tags, &rds.Tag{