From 57ec89026860f81f5ce45d51067f495bc3e8bb90 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Mon, 25 Nov 2024 23:35:54 -0600 Subject: [PATCH] Infer managedServiceSourceSnapshotEnabled when not set Signed-off-by: Andre Kurait --- .../lib/stack-composer.ts | 5 +- .../opensearch-service-migration/options.md | 2 +- .../test/stack-composer.test.ts | 74 ++++++++++++++++++- 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index c49c9674c..b9ae37dbd 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -214,7 +214,7 @@ export class StackComposer { const reindexFromSnapshotMaxShardSizeGiB = this.getContextForType('reindexFromSnapshotMaxShardSizeGiB', 'number', defaultValues, contextJSON) const reindexFromSnapshotWorkerSize = this.getContextForType('reindexFromSnapshotWorkerSize', 'string', defaultValues, contextJSON) const albAcmCertArn = this.getContextForType('albAcmCertArn', 'string', defaultValues, contextJSON); - const managedServiceSourceSnapshotEnabled = this.getContextForType('managedServiceSourceSnapshotEnabled', 'boolean', defaultValues, contextJSON) + let managedServiceSourceSnapshotEnabled = this.getContextForType('managedServiceSourceSnapshotEnabled', 'boolean', defaultValues, contextJSON) // We're in a transition state from an older model with limited, individually defined fields and heading towards objects // that fully define the source and target cluster configurations. For the time being, we're supporting both. @@ -245,6 +245,9 @@ export class StackComposer { if (managedServiceSourceSnapshotEnabled && !sourceCluster?.auth.sigv4) { throw new Error("A managed service source snapshot is only compatible with sigv4 authentication. If you would like to proceed" + " please disable `managedServiceSourceSnapshotEnabled` and provide your own snapshot of the source cluster.") + } else if (sourceCluster?.auth.sigv4 && managedServiceSourceSnapshotEnabled == null) { + managedServiceSourceSnapshotEnabled = true; + CdkLogger.info("`managedServiceSourceSnapshotEnabled` is not set with source cluster set with sigv4 auth, defaulting to true.") } const targetClusterEndpointField = this.getContextForType('targetClusterEndpoint', 'string', defaultValues, contextJSON) diff --git a/deployment/cdk/opensearch-service-migration/options.md b/deployment/cdk/opensearch-service-migration/options.md index d7b7826a6..fce98987e 100644 --- a/deployment/cdk/opensearch-service-migration/options.md +++ b/deployment/cdk/opensearch-service-migration/options.md @@ -56,7 +56,7 @@ In all other cases, the required components of each cluster object are: | reindexFromSnapshotServiceEnabled | boolean | true | Create resources for deploying and configuring the RFS ECS service | | reindexFromSnapshotExtraArgs | string | "--target-aws-region us-east-1 --target-aws-service-signing-name es" | Extra arguments to provide to the Document Migration command with space separation. See [RFS Arguments](../../../DocumentsFromSnapshotMigration/README.md#Arguments). [^1] | | sourceClusterEndpoint | string | `"https://source-cluster.elb.us-east-1.endpoint.com"` | The endpoint for the source cluster from which RFS will take a snapshot | -| managedServiceSourceSnapshotEnabled | boolean | true | Create the necessary roles and trust relationships to take a snapshot of a managed service source cluster. This is only compatible with SigV4 auth. | +| managedServiceSourceSnapshotEnabled | boolean | true | Create the necessary roles and trust relationships to take a snapshot of a managed service source cluster. This is only compatible with SigV4 auth. Default as true if not specified and source cluster is set with sigv4 auth. | | reindexFromSnapshotMaxShardSizeGiB | integer | 80 | OPTIONAL: The size, in whole GiB, of the largest shard you want to migrate across all indices; used to ensure we have enough disk space reserved to perform the migration. Default: 80 GiB | | reindexFromSnapshotWorkerSize | enum | default | maximum | OPTIONAL: default provisions a 2vCPU worker balancing speed with cost efficiency designed for most migrations with horizontal scaling, maximum provisions a 16vCPU worker for high throughput migrations when parallelization is limited (low source shard count). Default: default | diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts index 590edc1d0..47ec84e2d 100644 --- a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts @@ -3,7 +3,7 @@ import { OpenSearchDomainStack } from "../lib/opensearch-domain-stack"; import { createStackComposer, createStackComposerOnlyPassedContext } from "./test-utils"; import { App } from "aws-cdk-lib"; import { StackComposer } from "../lib/stack-composer"; -import { KafkaStack } from "../lib"; +import { KafkaStack, MigrationConsoleStack } from "../lib"; import { describe, beforeEach, afterEach, test, expect, jest } from '@jest/globals'; import { ContainerImage } from "aws-cdk-lib/aws-ecs"; @@ -30,6 +30,78 @@ describe('Stack Composer Tests', () => { domainTemplate.resourceCountIs("AWS::OpenSearchService::Domain", 1) }) + function testManagedServiceSourceSnapshot( + { sourceAuth, additionalOptions }: { sourceAuth: Record; additionalOptions: Record }, + expectedRoleCount: number, + description: string + ) { + test(description, () => { + const contextOptions = { + sourceCluster: { + "endpoint": "https://test-cluster", + "auth": sourceAuth, + "version": "ES_7.10" + }, + targetCluster: { + "endpoint": "https://test-cluster", + "auth": {"type": "none"}, + "version": "OS_1.3" + }, + vpcEnabled: true, + migrationConsoleServiceEnabled: true, + migrationAssistanceEnabled: true, + reindexFromSnapshotServiceEnabled: true, + ...additionalOptions + }; + + const openSearchStacks = createStackComposer(contextOptions); + const migrationConsoleStack = openSearchStacks.stacks.filter((s) => s instanceof MigrationConsoleStack)[0]; + const migrationConsoleTemplate = Template.fromStack(migrationConsoleStack); + migrationConsoleTemplate.resourceCountIs("AWS::IAM::Role", expectedRoleCount); + if (expectedRoleCount === 3) { + migrationConsoleTemplate.hasResourceProperties("AWS::IAM::Role", { + RoleName: "OSMigrations-unit-test-us-east-1-default-SnapshotRole" + }); + } + }); + } + + const sigv4Auth = { + "type": "sigv4", + "region": "us-east-1", + "serviceSigningName": "es" + }; + + const noAuth = {"type": "none"}; + + testManagedServiceSourceSnapshot( + { + sourceAuth: sigv4Auth, + additionalOptions: {} + }, + 3, + 'Test sigv4 source cluster with no managedServiceSourceSnapshotEnabled, defaults to true' + ); + + testManagedServiceSourceSnapshot( + { + sourceAuth: sigv4Auth, + additionalOptions: { managedServiceSourceSnapshotEnabled: false } + }, + 2, + 'Test sigv4 source cluster with false managedServiceSourceSnapshotEnabled, does not create snapshot role' + ); + + testManagedServiceSourceSnapshot( + { + sourceAuth: noAuth, + additionalOptions: {} + }, + 2, + 'Test no auth source cluster with no managedServiceSourceSnapshotEnabled, defaults to false' + ); + + test('Test invalid engine version format throws error', () => { const contextOptions = { // Should be OS_1.3