From 6d9261d238032dbf2d56d8b18bdbf1296bdcec76 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 23 Feb 2024 17:55:41 -0600 Subject: [PATCH 1/9] Add support for textFixtures imports with maven local publishing Signed-off-by: Andre Kurait --- TrafficCapture/README.md | 7 +++++++ TrafficCapture/build.gradle | 6 +----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/TrafficCapture/README.md b/TrafficCapture/README.md index e51026746..8d7c81799 100644 --- a/TrafficCapture/README.md +++ b/TrafficCapture/README.md @@ -172,7 +172,14 @@ jsonMessageTransformers nettyWireLogging openSearch23PlusTargetTransformerProvider replayerPlugins +testUtilities trafficCaptureProxyServer trafficCaptureProxyServerTest trafficReplayer +``` + +To include a testFixture dependency, define the import like + +```groovy +testImplementation testFixtures('org.opensearch.migrations.trafficcapture:trafficReplayer:0.1.0-SNAPSHOT') ``` \ No newline at end of file diff --git a/TrafficCapture/build.gradle b/TrafficCapture/build.gradle index 223230055..dba36c155 100644 --- a/TrafficCapture/build.gradle +++ b/TrafficCapture/build.gradle @@ -15,18 +15,14 @@ subprojects { def excludedProjects = [ 'buildSrc', 'dockerSolution', - // TODO: Get testFixtures exported to Maven - 'testUtilities', ] if (!(project.name in excludedProjects)) { publishing { publications { mavenJava(MavenPublication) { - artifactId = project.name - from components.java - groupId = 'org.opensearch.migrations.trafficcapture' + group = 'org.opensearch.migrations.trafficcapture' version = '0.1.0-SNAPSHOT' // Suppress POM metadata warnings for test fixtures From f1750308f3e33a6a72271b5db0915a41d66c344c Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 23 Feb 2024 19:56:11 -0600 Subject: [PATCH 2/9] Fix ServiceConnectService import to the base aws-ecs library export Signed-off-by: Andre Kurait --- .../lib/service-stacks/capture-proxy-es-stack.ts | 3 +-- .../lib/service-stacks/capture-proxy-stack.ts | 3 +-- .../lib/service-stacks/elasticsearch-stack.ts | 3 +-- .../lib/service-stacks/kafka-broker-stack.ts | 3 +-- .../lib/service-stacks/kafka-zookeeper-stack.ts | 3 +-- .../lib/service-stacks/migration-service-core.ts | 5 ++--- .../lib/service-stacks/opensearch-container-stack.ts | 3 +-- 7 files changed, 8 insertions(+), 15 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts index 38fcee01e..036fa3976 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts @@ -1,10 +1,9 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; +import {PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; -import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; import {StreamingSourceType} from "../streaming-source-type"; import {createMSKProducerIAMPolicies} from "../common-utilities"; diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts index 91f40e44b..c4b01be51 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts @@ -1,10 +1,9 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; +import {PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; -import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; import {StreamingSourceType} from "../streaming-source-type"; import {createMSKProducerIAMPolicies} from "../common-utilities"; diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts index ca1bbca88..38e98a58b 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts @@ -1,10 +1,9 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; +import {PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; -import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts index 7276d41ac..72873462d 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts @@ -1,12 +1,11 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; import { - PortMapping, Protocol + PortMapping, Protocol, ServiceConnectService } from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {MigrationServiceCore} from "./migration-service-core"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; -import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; export interface KafkaBrokerProps extends StackPropsExt { readonly vpc: IVpc diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts index e204b226e..255704fb9 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts @@ -1,12 +1,11 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; import { - PortMapping, Protocol, + PortMapping, Protocol, ServiceConnectService, } from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {MigrationServiceCore} from "./migration-service-core"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; -import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; export interface KafkaZookeeperProps extends StackPropsExt { readonly vpc: IVpc diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts index c419609dc..d17b76601 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts @@ -1,21 +1,20 @@ import {StackPropsExt} from "../stack-composer"; import {ISecurityGroup, IVpc, SubnetType} from "aws-cdk-lib/aws-ec2"; import { - CfnService as FargateCfnService, + CfnService as FargateCfnService, CloudMapOptions, Cluster, ContainerImage, FargateService, FargateTaskDefinition, LogDrivers, MountPoint, - PortMapping, Ulimit, + PortMapping, ServiceConnectService, Ulimit, Volume } from "aws-cdk-lib/aws-ecs"; import {DockerImageAsset} from "aws-cdk-lib/aws-ecr-assets"; import {RemovalPolicy, Stack} from "aws-cdk-lib"; import {LogGroup, RetentionDays} from "aws-cdk-lib/aws-logs"; import {PolicyStatement} from "aws-cdk-lib/aws-iam"; -import {CloudMapOptions, ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; import {CfnService as DiscoveryCfnService, PrivateDnsNamespace} from "aws-cdk-lib/aws-servicediscovery"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; import {createDefaultECSTaskRole} from "../common-utilities"; diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts index 4bfc13b2b..dbef90fab 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts @@ -1,10 +1,9 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol, Ulimit, UlimitName} from "aws-cdk-lib/aws-ecs"; +import {PortMapping, Protocol, ServiceConnectService, Ulimit, UlimitName} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {MigrationServiceCore} from "./migration-service-core"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; -import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; export interface OpenSearchContainerProps extends StackPropsExt { readonly vpc: IVpc From 345a3168c756fe84d4b04493fa0fc9812f7e1844 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 23 Feb 2024 19:56:43 -0600 Subject: [PATCH 3/9] Define outDir for typescript compilation artifacts Signed-off-by: Andre Kurait --- deployment/cdk/opensearch-service-migration/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/cdk/opensearch-service-migration/tsconfig.json b/deployment/cdk/opensearch-service-migration/tsconfig.json index 4c83b7c87..8d0615d5a 100644 --- a/deployment/cdk/opensearch-service-migration/tsconfig.json +++ b/deployment/cdk/opensearch-service-migration/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { + "outDir": "dist", "target": "ES2020", "module": "commonjs", "lib": [ From fa49b82eb1e424f23561294a841b7cd4e8f6df86 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 23 Feb 2024 20:12:09 -0600 Subject: [PATCH 4/9] Increase extensibility by moving aws-cdk dependencies to peer dependencies Signed-off-by: Andre Kurait --- .../package-lock.json | 380 +++--------------- .../opensearch-service-migration/package.json | 15 +- 2 files changed, 65 insertions(+), 330 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/package-lock.json b/deployment/cdk/opensearch-service-migration/package-lock.json index 4fb714506..354b6d3fe 100644 --- a/deployment/cdk/opensearch-service-migration/package-lock.json +++ b/deployment/cdk/opensearch-service-migration/package-lock.json @@ -9,25 +9,30 @@ "version": "1.0.0", "license": "Apache-2.0", "dependencies": { - "@aws-cdk/aws-servicecatalogappregistry-alpha": "2.100.0-alpha.0", "@aws-sdk/client-kafka": "^3.410.0", "@aws-sdk/client-lambda": "^3.359.0", "@types/aws-lambda": "^8.10.117", - "aws-cdk-lib": "2.100.0", "aws-lambda": "^1.0.7", - "constructs": "^10.0.0", "esbuild": "^0.19.5", "semver": "^7.5.4", "source-map-support": "^0.5.21" }, "devDependencies": { + "@aws-cdk/aws-servicecatalogappregistry-alpha": "2.100.0-alpha.0", "@types/jest": "^29.2.5", "@types/node": "18.11.18", - "aws-cdk": "2.62.2", + "aws-cdk": "2.100.0", + "aws-cdk-lib": "2.100.0", + "constructs": "^10.0.0", "jest": "^29.3.1", "ts-jest": "^29.0.3", "ts-node": "^10.9.1", "typescript": "~4.9.4" + }, + "peerDependencies": { + "@aws-cdk/aws-servicecatalogappregistry-alpha": "^2.100.0-alpha.0", + "aws-cdk-lib": "^2.100.0", + "constructs": "^10.0.0" } }, "node_modules/@ampproject/remapping": { @@ -46,22 +51,26 @@ "node_modules/@aws-cdk/asset-awscli-v1": { "version": "2.2.200", "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.200.tgz", - "integrity": "sha512-Kf5J8DfJK4wZFWT2Myca0lhwke7LwHcHBo+4TvWOGJrFVVKVuuiLCkzPPRBQQVDj0Vtn2NBokZAz8pfMpAqAKg==" + "integrity": "sha512-Kf5J8DfJK4wZFWT2Myca0lhwke7LwHcHBo+4TvWOGJrFVVKVuuiLCkzPPRBQQVDj0Vtn2NBokZAz8pfMpAqAKg==", + "dev": true }, "node_modules/@aws-cdk/asset-kubectl-v20": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.2.tgz", - "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==" + "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==", + "dev": true }, "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.1.tgz", - "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==" + "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==", + "dev": true }, "node_modules/@aws-cdk/aws-servicecatalogappregistry-alpha": { "version": "2.100.0-alpha.0", "resolved": "https://registry.npmjs.org/@aws-cdk/aws-servicecatalogappregistry-alpha/-/aws-servicecatalogappregistry-alpha-2.100.0-alpha.0.tgz", "integrity": "sha512-+CVSZ9KVspdXENqD8TV3inD1TNybvgdBwq00t0b9xbFCZpsRHKeD3bkU9ZTd/0+zXhxDDkJyAr09TJ3iJUrKFw==", + "dev": true, "engines": { "node": ">= 14.15.0" }, @@ -1325,51 +1334,6 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, - "node_modules/@esbuild/android-arm": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.19.5.tgz", - "integrity": "sha512-bhvbzWFF3CwMs5tbjf3ObfGqbl/17ict2/uwOSfr3wmxDE6VdS2GqY/FuzIPe0q0bdhj65zQsvqfArI9MY6+AA==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.19.5.tgz", - "integrity": "sha512-5d1OkoJxnYQfmC+Zd8NBFjkhyCNYwM4n9ODrycTFY6Jk1IGiZ+tjVJDDSwDt77nK+tfpGP4T50iMtVi4dEGzhQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.19.5.tgz", - "integrity": "sha512-9t+28jHGL7uBdkBjL90QFxe7DVA+KGqWlHCF8ChTKyaKO//VLuoBricQCgwhOjA1/qOczsw843Fy4cbs4H3DVA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, "node_modules/@esbuild/darwin-arm64": { "version": "0.19.5", "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.19.5.tgz", @@ -1385,276 +1349,6 @@ "node": ">=12" } }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.19.5.tgz", - "integrity": "sha512-Ly8cn6fGLNet19s0X4unjcniX24I0RqjPv+kurpXabZYSXGM4Pwpmf85WHJN3lAgB8GSth7s5A0r856S+4DyiA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.5.tgz", - "integrity": "sha512-GGDNnPWTmWE+DMchq1W8Sd0mUkL+APvJg3b11klSGUDvRXh70JqLAO56tubmq1s2cgpVCSKYywEiKBfju8JztQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.19.5.tgz", - "integrity": "sha512-1CCwDHnSSoA0HNwdfoNY0jLfJpd7ygaLAp5EHFos3VWJCRX9DMwWODf96s9TSse39Br7oOTLryRVmBoFwXbuuQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.19.5.tgz", - "integrity": "sha512-lrWXLY/vJBzCPC51QN0HM71uWgIEpGSjSZZADQhq7DKhPcI6NH1IdzjfHkDQws2oNpJKpR13kv7/pFHBbDQDwQ==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.19.5.tgz", - "integrity": "sha512-o3vYippBmSrjjQUCEEiTZ2l+4yC0pVJD/Dl57WfPwwlvFkrxoSO7rmBZFii6kQB3Wrn/6GwJUPLU5t52eq2meA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.19.5.tgz", - "integrity": "sha512-MkjHXS03AXAkNp1KKkhSKPOCYztRtK+KXDNkBa6P78F8Bw0ynknCSClO/ztGszILZtyO/lVKpa7MolbBZ6oJtQ==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.19.5.tgz", - "integrity": "sha512-42GwZMm5oYOD/JHqHska3Jg0r+XFb/fdZRX+WjADm3nLWLcIsN27YKtqxzQmGNJgu0AyXg4HtcSK9HuOk3v1Dw==", - "cpu": [ - "loong64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.19.5.tgz", - "integrity": "sha512-kcjndCSMitUuPJobWCnwQ9lLjiLZUR3QLQmlgaBfMX23UEa7ZOrtufnRds+6WZtIS9HdTXqND4yH8NLoVVIkcg==", - "cpu": [ - "mips64el" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.19.5.tgz", - "integrity": "sha512-yJAxJfHVm0ZbsiljbtFFP1BQKLc8kUF6+17tjQ78QjqjAQDnhULWiTA6u0FCDmYT1oOKS9PzZ2z0aBI+Mcyj7Q==", - "cpu": [ - "ppc64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.19.5.tgz", - "integrity": "sha512-5u8cIR/t3gaD6ad3wNt1MNRstAZO+aNyBxu2We8X31bA8XUNyamTVQwLDA1SLoPCUehNCymhBhK3Qim1433Zag==", - "cpu": [ - "riscv64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.19.5.tgz", - "integrity": "sha512-Z6JrMyEw/EmZBD/OFEFpb+gao9xJ59ATsoTNlj39jVBbXqoZm4Xntu6wVmGPB/OATi1uk/DB+yeDPv2E8PqZGw==", - "cpu": [ - "s390x" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.19.5.tgz", - "integrity": "sha512-psagl+2RlK1z8zWZOmVdImisMtrUxvwereIdyJTmtmHahJTKb64pAcqoPlx6CewPdvGvUKe2Jw+0Z/0qhSbG1A==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.19.5.tgz", - "integrity": "sha512-kL2l+xScnAy/E/3119OggX8SrWyBEcqAh8aOY1gr4gPvw76la2GlD4Ymf832UCVbmuWeTf2adkZDK+h0Z/fB4g==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.19.5.tgz", - "integrity": "sha512-sPOfhtzFufQfTBgRnE1DIJjzsXukKSvZxloZbkJDG383q0awVAq600pc1nfqBcl0ice/WN9p4qLc39WhBShRTA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.19.5.tgz", - "integrity": "sha512-dGZkBXaafuKLpDSjKcB0ax0FL36YXCvJNnztjKV+6CO82tTYVDSH2lifitJ29jxRMoUhgkg9a+VA/B03WK5lcg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.19.5.tgz", - "integrity": "sha512-dWVjD9y03ilhdRQ6Xig1NWNgfLtf2o/STKTS+eZuF90fI2BhbwD6WlaiCGKptlqXlURVB5AUOxUj09LuwKGDTg==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.19.5.tgz", - "integrity": "sha512-4liggWIA4oDgUxqpZwrDhmEfAH4d0iljanDOK7AnVU89T6CzHon/ony8C5LeOdfgx60x5cnQJFZwEydVlYx4iw==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.19.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.19.5.tgz", - "integrity": "sha512-czTrygUsB/jlM8qEW5MD8bgYU2Xg14lo6kBDXW6HdxKjh8M5PzETGiSHaz9MtbXBYDloHNUAUW2tMiKW4KM9Mw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -2811,9 +2505,9 @@ } }, "node_modules/aws-cdk": { - "version": "2.62.2", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.62.2.tgz", - "integrity": "sha512-fACUQEdc43AfiqKKXUZLuHB2VADLU965hl50Fn6BhMemFmUsc+iS/GFfOwvfCUO3/iaRtsn/gk+dQ3zZ+snKRw==", + "version": "2.100.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.100.0.tgz", + "integrity": "sha512-Gt/4wPuEiBYw2tl0+cN0EbLxxJEvltcJxSQAcVHgNbqvDj49KUJ/oCbZ335dF0gK/hrVVb70xfNiYbBSPOsmvg==", "dev": true, "bin": { "cdk": "bin/cdk" @@ -2841,6 +2535,7 @@ "table", "yaml" ], + "dev": true, "dependencies": { "@aws-cdk/asset-awscli-v1": "^2.2.200", "@aws-cdk/asset-kubectl-v20": "^2.1.2", @@ -2865,11 +2560,13 @@ }, "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { "version": "1.0.2", + "dev": true, "inBundle": true, "license": "Apache-2.0" }, "node_modules/aws-cdk-lib/node_modules/ajv": { "version": "8.12.0", + "dev": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -2885,6 +2582,7 @@ }, "node_modules/aws-cdk-lib/node_modules/ansi-regex": { "version": "5.0.1", + "dev": true, "inBundle": true, "license": "MIT", "engines": { @@ -2893,6 +2591,7 @@ }, "node_modules/aws-cdk-lib/node_modules/ansi-styles": { "version": "4.3.0", + "dev": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -2907,6 +2606,7 @@ }, "node_modules/aws-cdk-lib/node_modules/astral-regex": { "version": "2.0.0", + "dev": true, "inBundle": true, "license": "MIT", "engines": { @@ -2915,11 +2615,13 @@ }, "node_modules/aws-cdk-lib/node_modules/balanced-match": { "version": "1.0.2", + "dev": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/brace-expansion": { "version": "1.1.11", + "dev": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -2929,6 +2631,7 @@ }, "node_modules/aws-cdk-lib/node_modules/case": { "version": "1.6.3", + "dev": true, "inBundle": true, "license": "(MIT OR GPL-3.0-or-later)", "engines": { @@ -2937,6 +2640,7 @@ }, "node_modules/aws-cdk-lib/node_modules/color-convert": { "version": "2.0.1", + "dev": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -2948,26 +2652,31 @@ }, "node_modules/aws-cdk-lib/node_modules/color-name": { "version": "1.1.4", + "dev": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/concat-map": { "version": "0.0.1", + "dev": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/emoji-regex": { "version": "8.0.0", + "dev": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { "version": "3.1.3", + "dev": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/fs-extra": { "version": "11.1.1", + "dev": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -2981,11 +2690,13 @@ }, "node_modules/aws-cdk-lib/node_modules/graceful-fs": { "version": "4.2.11", + "dev": true, "inBundle": true, "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/ignore": { "version": "5.2.4", + "dev": true, "inBundle": true, "license": "MIT", "engines": { @@ -2994,6 +2705,7 @@ }, "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { "version": "3.0.0", + "dev": true, "inBundle": true, "license": "MIT", "engines": { @@ -3002,11 +2714,13 @@ }, "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { "version": "1.0.0", + "dev": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/jsonfile": { "version": "6.1.0", + "dev": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -3018,6 +2732,7 @@ }, "node_modules/aws-cdk-lib/node_modules/jsonschema": { "version": "1.4.1", + "dev": true, "inBundle": true, "license": "MIT", "engines": { @@ -3026,11 +2741,13 @@ }, "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { "version": "4.4.2", + "dev": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/lru-cache": { "version": "6.0.0", + "dev": true, "inBundle": true, "license": "ISC", "dependencies": { @@ -3042,6 +2759,7 @@ }, "node_modules/aws-cdk-lib/node_modules/minimatch": { "version": "3.1.2", + "dev": true, "inBundle": true, "license": "ISC", "dependencies": { @@ -3053,6 +2771,7 @@ }, "node_modules/aws-cdk-lib/node_modules/punycode": { "version": "2.3.0", + "dev": true, "inBundle": true, "license": "MIT", "engines": { @@ -3061,6 +2780,7 @@ }, "node_modules/aws-cdk-lib/node_modules/require-from-string": { "version": "2.0.2", + "dev": true, "inBundle": true, "license": "MIT", "engines": { @@ -3069,6 +2789,7 @@ }, "node_modules/aws-cdk-lib/node_modules/semver": { "version": "7.5.4", + "dev": true, "inBundle": true, "license": "ISC", "dependencies": { @@ -3083,6 +2804,7 @@ }, "node_modules/aws-cdk-lib/node_modules/slice-ansi": { "version": "4.0.0", + "dev": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -3099,6 +2821,7 @@ }, "node_modules/aws-cdk-lib/node_modules/string-width": { "version": "4.2.3", + "dev": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -3112,6 +2835,7 @@ }, "node_modules/aws-cdk-lib/node_modules/strip-ansi": { "version": "6.0.1", + "dev": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -3123,6 +2847,7 @@ }, "node_modules/aws-cdk-lib/node_modules/table": { "version": "6.8.1", + "dev": true, "inBundle": true, "license": "BSD-3-Clause", "dependencies": { @@ -3138,6 +2863,7 @@ }, "node_modules/aws-cdk-lib/node_modules/universalify": { "version": "2.0.0", + "dev": true, "inBundle": true, "license": "MIT", "engines": { @@ -3146,6 +2872,7 @@ }, "node_modules/aws-cdk-lib/node_modules/uri-js": { "version": "4.4.1", + "dev": true, "inBundle": true, "license": "BSD-2-Clause", "dependencies": { @@ -3154,11 +2881,13 @@ }, "node_modules/aws-cdk-lib/node_modules/yallist": { "version": "4.0.0", + "dev": true, "inBundle": true, "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/yaml": { "version": "1.10.2", + "dev": true, "inBundle": true, "license": "ISC", "engines": { @@ -3602,6 +3331,7 @@ "version": "10.2.70", "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.2.70.tgz", "integrity": "sha512-z6zr1E8K/9tzJbCQzY0UGX0/oVKPFKu9C/mzEnghCG6TAJINnvlq0CMKm63XqqeMleadZYm5T3sZGJKcxJS/Pg==", + "dev": true, "engines": { "node": ">= 16.14.0" } diff --git a/deployment/cdk/opensearch-service-migration/package.json b/deployment/cdk/opensearch-service-migration/package.json index 7c2cd7e5e..a0fdc5f95 100644 --- a/deployment/cdk/opensearch-service-migration/package.json +++ b/deployment/cdk/opensearch-service-migration/package.json @@ -5,7 +5,7 @@ "license": "Apache-2.0", "author": { "name": "Amazon Web Services", - "url": "https://github.com/opensearch-project" + "url": "https://github.com/opensearch-project/opensearch-migrations" }, "scripts": { "build": "tsc", @@ -14,26 +14,31 @@ "cdk": "cdk" }, "devDependencies": { + "@aws-cdk/aws-servicecatalogappregistry-alpha": "2.100.0-alpha.0", "@types/jest": "^29.2.5", "@types/node": "18.11.18", - "aws-cdk": "2.62.2", + "aws-cdk": "2.100.0", + "aws-cdk-lib": "2.100.0", + "constructs": "^10.0.0", "jest": "^29.3.1", "ts-jest": "^29.0.3", "ts-node": "^10.9.1", "typescript": "~4.9.4" }, "dependencies": { - "@aws-cdk/aws-servicecatalogappregistry-alpha": "2.100.0-alpha.0", "@aws-sdk/client-kafka": "^3.410.0", "@aws-sdk/client-lambda": "^3.359.0", "@types/aws-lambda": "^8.10.117", - "aws-cdk-lib": "2.100.0", "aws-lambda": "^1.0.7", - "constructs": "^10.0.0", "esbuild": "^0.19.5", "semver": "^7.5.4", "source-map-support": "^0.5.21" }, + "peerDependencies": { + "@aws-cdk/aws-servicecatalogappregistry-alpha": "^2.100.0-alpha.0", + "aws-cdk-lib": "^2.100.0", + "constructs": "^10.0.0" + }, "overrides": { "semver": "^7.5.4" } From f0e549f55303050304cf38b47c4c1146f5f4265a Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 23 Feb 2024 20:25:29 -0600 Subject: [PATCH 5/9] add dist directory to gitignore in cdk directory Signed-off-by: Andre Kurait --- deployment/cdk/opensearch-service-migration/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/cdk/opensearch-service-migration/.gitignore b/deployment/cdk/opensearch-service-migration/.gitignore index 5cf701542..1c13263cc 100644 --- a/deployment/cdk/opensearch-service-migration/.gitignore +++ b/deployment/cdk/opensearch-service-migration/.gitignore @@ -4,7 +4,7 @@ node_modules cdk.context.json coverage - +dist # CDK asset staging directory .cdk.staging cdk.out From 1c0c95e0b74eb525ce8361da24616f4526caab8d Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 23 Feb 2024 22:31:04 -0600 Subject: [PATCH 6/9] Update to use index.ts for simplifying external imports Signed-off-by: Andre Kurait --- deployment/cdk/opensearch-service-migration/index.ts | 1 + .../cdk/opensearch-service-migration/lib/index.ts | 10 ++++++++++ .../opensearch-service-migration/lib/lambda/index.ts | 2 ++ .../lib/service-stacks/index.ts | 10 ++++++++++ .../cdk/opensearch-service-migration/package.json | 3 +++ .../cdk/opensearch-service-migration/tsconfig.json | 5 ++++- 6 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 deployment/cdk/opensearch-service-migration/index.ts create mode 100644 deployment/cdk/opensearch-service-migration/lib/index.ts create mode 100644 deployment/cdk/opensearch-service-migration/lib/lambda/index.ts create mode 100644 deployment/cdk/opensearch-service-migration/lib/service-stacks/index.ts diff --git a/deployment/cdk/opensearch-service-migration/index.ts b/deployment/cdk/opensearch-service-migration/index.ts new file mode 100644 index 000000000..cc80ecd28 --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/index.ts @@ -0,0 +1 @@ +export * from './lib' \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/index.ts b/deployment/cdk/opensearch-service-migration/lib/index.ts new file mode 100644 index 000000000..979dc05c3 --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/lib/index.ts @@ -0,0 +1,10 @@ +export * from './common-utilities'; +export * from './fetch-migration-stack'; +export * from './lambda' +export * from './migration-assistance-stack'; +export * from './msk-utility-stack'; +export * from './network-stack'; +export * from './opensearch-domain-stack'; +export * from './service-stacks'; +export * from './stack-composer'; +export * from './streaming-source-type'; \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/lambda/index.ts b/deployment/cdk/opensearch-service-migration/lib/lambda/index.ts new file mode 100644 index 000000000..72d1343f0 --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/lib/lambda/index.ts @@ -0,0 +1,2 @@ +export * as msk_ordered_endpoints_handler from './msk-ordered-endpoints-handler' +export * as msk_public_endpoint_handler from './msk-public-endpoint-handler' \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/index.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/index.ts new file mode 100644 index 000000000..e98879c68 --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/index.ts @@ -0,0 +1,10 @@ +export * from './capture-proxy-es-stack' +export * from './capture-proxy-stack' +export * from './elasticsearch-stack' +export * from './kafka-broker-stack' +export * from './kafka-zookeeper-stack' +export * from './migration-analytics-stack' +export * from './migration-console-stack' +export * from './migration-service-core' +export * from './opensearch-container-stack' +export * from './traffic-replayer-stack' diff --git a/deployment/cdk/opensearch-service-migration/package.json b/deployment/cdk/opensearch-service-migration/package.json index a0fdc5f95..758743c0e 100644 --- a/deployment/cdk/opensearch-service-migration/package.json +++ b/deployment/cdk/opensearch-service-migration/package.json @@ -3,6 +3,8 @@ "version": "1.0.0", "description": "CDK infrastructure for migrating to AWS OpenSearch", "license": "Apache-2.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", "author": { "name": "Amazon Web Services", "url": "https://github.com/opensearch-project/opensearch-migrations" @@ -11,6 +13,7 @@ "build": "tsc", "watch": "tsc -w", "test": "jest --coverage", + "clean": "rm -rf dist", "cdk": "cdk" }, "devDependencies": { diff --git a/deployment/cdk/opensearch-service-migration/tsconfig.json b/deployment/cdk/opensearch-service-migration/tsconfig.json index 8d0615d5a..015bb0b78 100644 --- a/deployment/cdk/opensearch-service-migration/tsconfig.json +++ b/deployment/cdk/opensearch-service-migration/tsconfig.json @@ -27,6 +27,9 @@ }, "exclude": [ "node_modules", - "cdk.out" + "cdk.out", + "dist", + "test", + "bin" ] } From d8763bb4744dec9bd3786730e1282dd19750492d Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Mon, 26 Feb 2024 17:06:28 -0500 Subject: [PATCH 7/9] MIGRATIONS-1519: Add support for providing Fargate CPU arch in CDK (#520) * MIGRATIONS-1519: Add support for providing CPU arch Signed-off-by: Tanner Lewis --- TrafficCapture/dockerSolution/build.gradle | 2 +- .../lib/common-utilities.ts | 19 ++++++++++ .../lib/fetch-migration-stack.ts | 11 ++++-- .../service-stacks/capture-proxy-es-stack.ts | 4 ++- .../lib/service-stacks/capture-proxy-stack.ts | 4 ++- .../lib/service-stacks/elasticsearch-stack.ts | 4 ++- .../lib/service-stacks/kafka-broker-stack.ts | 5 ++- .../service-stacks/kafka-zookeeper-stack.ts | 5 ++- .../migration-analytics-stack.ts | 4 ++- .../service-stacks/migration-console-stack.ts | 4 ++- .../service-stacks/migration-service-core.ts | 9 +++-- .../opensearch-container-stack.ts | 6 ++-- .../service-stacks/traffic-replayer-stack.ts | 4 ++- .../lib/stack-composer.ts | 14 ++++++++ .../opensearch-service-migration/options.md | 1 + .../test/common-utilities.test.ts | 35 +++++++++++++++++++ .../test/default-values-test.json | 3 +- .../test/fetch-migration-stack.test.ts | 4 ++- 18 files changed, 120 insertions(+), 18 deletions(-) create mode 100644 deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index 672ecc89c..d0885e887 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -68,7 +68,7 @@ dockerCompose { "${project.getProperty('otel-collector')}" : "otel-prometheus-jaeger-opensearch.yml"), "${extensionsDir}" + (project.hasProperty("multiProxy") ? "proxy-multi.yml" : "proxy-single.yml") - ] + ] } task buildDockerImages { diff --git a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts index a28fa2b61..62512fcd8 100644 --- a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts +++ b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts @@ -1,6 +1,7 @@ import {Effect, PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam"; import {Construct} from "constructs"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; +import {CpuArchitecture} from "aws-cdk-lib/aws-ecs"; export function createOpenSearchIAMAccessPolicy(region: string, accountId: string): PolicyStatement { return new PolicyStatement({ @@ -120,4 +121,22 @@ export function createDefaultECSTaskRole(scope: Construct, serviceName: string): ] })) return serviceTaskRole +} + +export function validateFargateCpuArch(cpuArch?: string): CpuArchitecture { + const desiredArch = cpuArch ? cpuArch : process.arch + const desiredArchUpper = desiredArch.toUpperCase() + + if (desiredArchUpper === "X86_64" || desiredArchUpper === "X64") { + return CpuArchitecture.X86_64 + } else if (desiredArchUpper === "ARM64") { + return CpuArchitecture.ARM64 + } else { + if (cpuArch) { + throw new Error(`Unknown Fargate cpu architecture provided: ${desiredArch}`) + } + else { + throw new Error(`Unsupported process cpu architecture detected: ${desiredArch}, CDK requires X64 or ARM64 for Docker image compatability`) + } + } } \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts index dd712543d..dff7e65fb 100644 --- a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts @@ -3,9 +3,9 @@ import {IVpc} from "aws-cdk-lib/aws-ec2"; import {Construct} from "constructs"; import { Cluster, - ContainerImage, + ContainerImage, CpuArchitecture, FargateTaskDefinition, - LogDrivers, + LogDrivers, OperatingSystemFamily, Secret as ECSSecret } from "aws-cdk-lib/aws-ecs"; import {Secret as SMSecret} from "aws-cdk-lib/aws-secretsmanager"; @@ -22,7 +22,8 @@ import { export interface FetchMigrationProps extends StackPropsExt { readonly vpc: IVpc, readonly dpPipelineTemplatePath: string, - readonly sourceEndpoint: string + readonly sourceEndpoint: string, + readonly fargateCpuArch: CpuArchitecture } export class FetchMigrationStack extends Stack { @@ -49,6 +50,10 @@ export class FetchMigrationStack extends Stack { ecsTaskRole.addToPolicy(openSearchServerlessPolicy) // ECS Task Definition const fetchMigrationFargateTask = new FargateTaskDefinition(this, "fetchMigrationFargateTask", { + runtimePlatform: { + operatingSystemFamily: OperatingSystemFamily.LINUX, + cpuArchitecture: props.fargateCpuArch + }, family: `migration-${props.stage}-${serviceName}`, memoryLimitMiB: 8192, cpu: 2048, diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts index 38fcee01e..37204c403 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -14,6 +14,7 @@ export interface CaptureProxyESProps extends StackPropsExt { readonly vpc: IVpc, readonly streamingSourceType: StreamingSourceType, readonly analyticsServiceEnabled: boolean, + readonly fargateCpuArch: CpuArchitecture, readonly extraArgs?: string, } @@ -75,6 +76,7 @@ export class CaptureProxyESStack extends MigrationServiceCore { serviceConnectServices: [serviceConnectService, esServiceConnectService], serviceDiscoveryEnabled: true, serviceDiscoveryPort: 19200, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 1024, taskMemoryLimitMiB: 4096, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts index 91f40e44b..66842ca84 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -13,6 +13,7 @@ import {createMSKProducerIAMPolicies} from "../common-utilities"; export interface CaptureProxyProps extends StackPropsExt { readonly vpc: IVpc, readonly streamingSourceType: StreamingSourceType, + readonly fargateCpuArch: CpuArchitecture, readonly customSourceClusterEndpoint?: string, readonly analyticsServiceEnabled?: boolean, readonly extraArgs?: string, @@ -60,6 +61,7 @@ export class CaptureProxyStack extends MigrationServiceCore { taskRolePolicies: servicePolicies, portMappings: [servicePort], serviceConnectServices: [serviceConnectService], + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 512, taskMemoryLimitMiB: 2048, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts index ca1bbca88..a14513c64 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -10,6 +10,7 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm"; export interface ElasticsearchProps extends StackPropsExt { readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture } /** @@ -45,6 +46,7 @@ export class ElasticsearchStack extends MigrationServiceCore { serviceConnectServices: [serviceConnectService], serviceDiscoveryEnabled: true, serviceDiscoveryPort: 9200, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 512, taskMemoryLimitMiB: 2048, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts index 7276d41ac..d8bf72b95 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts @@ -1,6 +1,7 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; import { + CpuArchitecture, PortMapping, Protocol } from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; @@ -9,7 +10,8 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm"; import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; export interface KafkaBrokerProps extends StackPropsExt { - readonly vpc: IVpc + readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture } /** @@ -62,6 +64,7 @@ export class KafkaBrokerStack extends MigrationServiceCore { }, portMappings: [servicePort], serviceConnectServices: [serviceConnectService], + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 256, taskMemoryLimitMiB: 2048, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts index e204b226e..405af42eb 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts @@ -1,6 +1,7 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; import { + CpuArchitecture, PortMapping, Protocol, } from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; @@ -9,7 +10,8 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm"; import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; export interface KafkaZookeeperProps extends StackPropsExt { - readonly vpc: IVpc + readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture } /** @@ -46,6 +48,7 @@ export class KafkaZookeeperStack extends MigrationServiceCore { }, portMappings: [servicePort], serviceConnectServices: [serviceConnectService], + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 256, taskMemoryLimitMiB: 512, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts index ca5cd62b0..22075a0ca 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts @@ -6,7 +6,7 @@ import { SecurityGroup, IVpc, } from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -15,6 +15,7 @@ import {createAwsDistroForOtelPushInstrumentationPolicy} from "../common-utiliti export interface MigrationAnalyticsProps extends StackPropsExt { readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture, readonly bastionHostEnabled?: boolean } @@ -82,6 +83,7 @@ export class MigrationAnalyticsStack extends MigrationServiceCore { dockerDirectoryPath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/src/main/docker/otelCollector"), dockerImageCommand: ["--config=/etc/otel-config-aws.yaml"], securityGroups: securityGroups, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 1024, taskMemoryLimitMiB: 4096, portMappings: [otelCollectorPort, otelHealthCheckPort], diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts index 16013cb40..e1d931e0a 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -17,6 +17,7 @@ export interface MigrationConsoleProps extends StackPropsExt { readonly vpc: IVpc, readonly streamingSourceType: StreamingSourceType, readonly fetchMigrationEnabled: boolean, + readonly fargateCpuArch: CpuArchitecture, readonly migrationAnalyticsEnabled: boolean } @@ -154,6 +155,7 @@ export class MigrationConsoleStack extends MigrationServiceCore { mountPoints: [replayerOutputMountPoint], environment: environment, taskRolePolicies: servicePolicies, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 512, taskMemoryLimitMiB: 1024, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts index c419609dc..d030179e2 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts @@ -3,11 +3,11 @@ import {ISecurityGroup, IVpc, SubnetType} from "aws-cdk-lib/aws-ec2"; import { CfnService as FargateCfnService, Cluster, - ContainerImage, + ContainerImage, CpuArchitecture, FargateService, FargateTaskDefinition, LogDrivers, - MountPoint, + MountPoint, OperatingSystemFamily, PortMapping, Ulimit, Volume } from "aws-cdk-lib/aws-ecs"; @@ -25,6 +25,7 @@ export interface MigrationServiceCoreProps extends StackPropsExt { readonly serviceName: string, readonly vpc: IVpc, readonly securityGroups: ISecurityGroup[], + readonly cpuArchitecture: CpuArchitecture, readonly dockerFilePath?: string, readonly dockerDirectoryPath?: string, readonly dockerImageRegistryName?: string, @@ -87,6 +88,10 @@ export class MigrationServiceCore extends Stack { const serviceTaskDef = new FargateTaskDefinition(this, "ServiceTaskDef", { ephemeralStorageGiB: 75, + runtimePlatform: { + operatingSystemFamily: OperatingSystemFamily.LINUX, + cpuArchitecture: props.cpuArchitecture + }, family: `migration-${props.stage}-${props.serviceName}`, memoryLimitMiB: props.taskMemoryLimitMiB ? props.taskMemoryLimitMiB : 1024, cpu: props.taskCpuUnits ? props.taskCpuUnits : 256, diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts index 4bfc13b2b..420f94bbe 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts @@ -1,13 +1,14 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol, Ulimit, UlimitName} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol, Ulimit, UlimitName} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {MigrationServiceCore} from "./migration-service-core"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; export interface OpenSearchContainerProps extends StackPropsExt { - readonly vpc: IVpc + readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture } /** @@ -61,6 +62,7 @@ export class OpenSearchContainerStack extends MigrationServiceCore { serviceConnectServices: [serviceConnectService], taskCpuUnits: 1024, taskMemoryLimitMiB: 4096, + cpuArchitecture: props.fargateCpuArch, ulimits: ulimits, ...props }); diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts index a290b86c7..b6a183367 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -18,6 +18,7 @@ export interface TrafficReplayerProps extends StackPropsExt { readonly vpc: IVpc, readonly enableClusterFGACAuth: boolean, readonly streamingSourceType: StreamingSourceType, + readonly fargateCpuArch: CpuArchitecture, readonly addOnMigrationId?: string, readonly customKafkaGroupId?: string, readonly userAgentSuffix?: string, @@ -102,6 +103,7 @@ export class TrafficReplayerStack extends MigrationServiceCore { environment: { "TUPLE_DIR_PATH": `/shared-replayer-output/traffic-replayer-${deployId}` }, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 1024, taskMemoryLimitMiB: 4096, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index 87d432714..9147caab4 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -18,6 +18,7 @@ import {KafkaZookeeperStack} from "./service-stacks/kafka-zookeeper-stack"; import {Application} from "@aws-cdk/aws-servicecatalogappregistry-alpha"; import {OpenSearchContainerStack} from "./service-stacks/opensearch-container-stack"; import {determineStreamingSourceType, StreamingSourceType} from "./streaming-source-type"; +import {validateFargateCpuArch} from "./common-utilities"; export interface StackPropsExt extends StackProps { readonly stage: string, @@ -159,6 +160,7 @@ export class StackComposer { const mskBrokerNodeCount = this.getContextForType('mskBrokerNodeCount', 'number', defaultValues, contextJSON) const mskSubnetIds = this.getContextForType('mskSubnetIds', 'object', defaultValues, contextJSON) const addOnMigrationDeployId = this.getContextForType('addOnMigrationDeployId', 'string', defaultValues, contextJSON) + const defaultFargateCpuArch = this.getContextForType('defaultFargateCpuArch', 'string', defaultValues, contextJSON) const captureProxyESServiceEnabled = this.getContextForType('captureProxyESServiceEnabled', 'boolean', defaultValues, contextJSON) const captureProxyESExtraArgs = this.getContextForType('captureProxyESExtraArgs', 'string', defaultValues, contextJSON) const migrationConsoleServiceEnabled = this.getContextForType('migrationConsoleServiceEnabled', 'boolean', defaultValues, contextJSON) @@ -211,6 +213,8 @@ export class StackComposer { } else if (targetClusterEndpoint || osContainerServiceEnabled) { targetEndpoint = targetClusterEndpoint ? targetClusterEndpoint : "https://opensearch:9200" } + + const fargateCpuArch = validateFargateCpuArch(defaultFargateCpuArch) const streamingSourceType = determineStreamingSourceType(kafkaBrokerServiceEnabled) const engineVersion = this.getContextForType('engineVersion', 'string', defaultValues, contextJSON) @@ -384,6 +388,7 @@ export class StackComposer { migrationAnalyticsStack = new MigrationAnalyticsStack(scope, "migration-analytics", { stackName: `OSMigrations-${stage}-${region}-MigrationAnalytics`, description: "This stack contains the OpenTelemetry Collector and Bastion Host", + fargateCpuArch: fargateCpuArch, bastionHostEnabled: migrationAnalyticsBastionHostEnabled, vpc:networkStack.vpc, stage: stage, @@ -403,6 +408,7 @@ export class StackComposer { description: "This stack contains resources for the OpenSearch Container ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(osContainerStack, [migrationStack]) @@ -417,6 +423,7 @@ export class StackComposer { description: "This stack contains resources for the Kafka Zookeeper ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(kafkaZookeeperStack, [migrationStack]) @@ -431,6 +438,7 @@ export class StackComposer { description: "This stack contains resources for the Kafka Broker ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(kafkaBrokerStack, [migrationStack, kafkaZookeeperStack]) @@ -448,6 +456,7 @@ export class StackComposer { description: "This stack contains resources to assist migrating historical data to an OpenSearch Service domain", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(fetchMigrationStack, [migrationStack, openSearchStack, osContainerStack]) @@ -465,6 +474,7 @@ export class StackComposer { description: "This stack contains resources for the Capture Proxy/Elasticsearch ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) // The analytics stack dependency is necessary to ensure the otel collector is available (and can be found via service connect) @@ -487,6 +497,7 @@ export class StackComposer { description: "This stack contains resources for the Traffic Replayer ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) // The analytics stack dependency is necessary to ensure the otel collector is available (and can be found via service connect) @@ -503,6 +514,7 @@ export class StackComposer { description: "This stack contains resources for a testing mock Elasticsearch single node cluster ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(elasticsearchStack, [migrationStack]) @@ -521,6 +533,7 @@ export class StackComposer { description: "This stack contains resources for the Capture Proxy ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) // The analytics stack dependency is necessary to ensure the otel collector is available (and can be found via service connect) @@ -540,6 +553,7 @@ export class StackComposer { description: "This stack contains resources for the Migration Console ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) // To enable the Migration Console to make requests to other service endpoints with Service Connect, diff --git a/deployment/cdk/opensearch-service-migration/options.md b/deployment/cdk/opensearch-service-migration/options.md index 68005b7ba..68a8c847d 100644 --- a/deployment/cdk/opensearch-service-migration/options.md +++ b/deployment/cdk/opensearch-service-migration/options.md @@ -7,6 +7,7 @@ These tables list all CDK context configuration values a user can specify for th |--------------------------------------|---------|------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | migrationAssistanceEnabled | boolean | true | Flag which controls deploying common Migration Service resources such as MSK, EFS, and an ECS cluster. **Note**: This option must be enabled to use any Migration service | | addOnMigrationDeployId | string | "cit2-replay" | Specify an ID string to use for an additional replay [scenario](./README.md#how-to-run-multiple-traffic-replayer-scenarios). **Note**: This option should not be used for initial deployments | +| defaultFargateCpuArch | string | "X86_64", "ARM64" | Provide a default CPU architecture that should be used for all containers. Defaults to using `process.arch` to determine the proper architecture to use | | captureProxyESServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | | captureProxyESExtraArgs | string | `"--suppressCaptureForHeaderMatch user-agent .*elastic-java/7.17.0.*"` | Extra arguments to provide to the Capture Proxy command. This includes available arguments specified by the [Capture Proxy](../../../TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java). | | migrationConsoleServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | diff --git a/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts new file mode 100644 index 000000000..cfbf2a8be --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts @@ -0,0 +1,35 @@ +import {CpuArchitecture} from "aws-cdk-lib/aws-ecs"; +import {validateFargateCpuArch} from "../lib/common-utilities"; + +test('Test valid fargate cpu arch strings can be parsed', () => { + const cpuArch1 = "arm64" + const detectedArch1 = validateFargateCpuArch(cpuArch1) + expect(detectedArch1).toEqual(CpuArchitecture.ARM64) + + const cpuArch2 = "ARM64" + const detectedArch2 = validateFargateCpuArch(cpuArch2) + expect(detectedArch2).toEqual(CpuArchitecture.ARM64) + + const cpuArch3 = "x86_64" + const detectedArch3 = validateFargateCpuArch(cpuArch3) + expect(detectedArch3).toEqual(CpuArchitecture.X86_64) + + const cpuArch4 = "X86_64" + const detectedArch4 = validateFargateCpuArch(cpuArch4) + expect(detectedArch4).toEqual(CpuArchitecture.X86_64) +}) + +test('Test invalid fargate cpu arch strings throws error', () => { + const cpuArch = "arm32" + const getArchFunction = () => validateFargateCpuArch(cpuArch) + expect(getArchFunction).toThrowError() +}) + +test('Test detected fargate cpu arch is valid', () => { + const detectedArch = process.arch + const detectedArchUpper = detectedArch.toUpperCase() + + const expectedCpuArch = detectedArchUpper === "X64" ? CpuArchitecture.X86_64 : CpuArchitecture.ARM64 + const cpuArch = validateFargateCpuArch() + expect(cpuArch).toEqual(expectedCpuArch) +}) \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/test/default-values-test.json b/deployment/cdk/opensearch-service-migration/test/default-values-test.json index d3ddac369..53c4c8ec9 100644 --- a/deployment/cdk/opensearch-service-migration/test/default-values-test.json +++ b/deployment/cdk/opensearch-service-migration/test/default-values-test.json @@ -1,5 +1,6 @@ { "engineVersion": "OS_1.0", "domainName": "sample-cdk-unit-test-domain", - "dpPipelineTemplatePath": "./dp_pipeline_template.yaml" + "dpPipelineTemplatePath": "./dp_pipeline_template.yaml", + "defaultFargateCpuArch": "X86_64" } \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/test/fetch-migration-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/fetch-migration-stack.test.ts index aa2bba94e..911f420ff 100644 --- a/deployment/cdk/opensearch-service-migration/test/fetch-migration-stack.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/fetch-migration-stack.test.ts @@ -2,6 +2,7 @@ import {App} from 'aws-cdk-lib'; import {FetchMigrationStack} from "../lib/fetch-migration-stack"; import {Template} from "aws-cdk-lib/assertions"; import {NetworkStack} from "../lib/network-stack"; +import {CpuArchitecture} from "aws-cdk-lib/aws-ecs"; test('Test default fetch migration stack creates required resources', () => { const app = new App(); @@ -16,7 +17,8 @@ test('Test default fetch migration stack creates required resources', () => { dpPipelineTemplatePath: "./dp_pipeline_template.yaml", sourceEndpoint: "https://test-cluster", defaultDeployId: "default", - stage: "unit-test" + stage: "unit-test", + fargateCpuArch: CpuArchitecture.X86_64, }) From 4af7f4cafba41de13453324f1046f07abacd0666 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Tue, 27 Feb 2024 11:56:17 -0600 Subject: [PATCH 8/9] Add fields as protected to allow for class extension Signed-off-by: Andre Kurait --- .../proxyserver/CaptureProxy.java | 18 +++++++++--------- .../netty/NettyScanningHttpProxy.java | 8 ++++---- .../netty/ProxyChannelInitializer.java | 10 +++++----- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index a705845ff..d7f749428 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -55,7 +55,7 @@ public class CaptureProxy { private static final String HTTPS_CONFIG_PREFIX = "plugins.security.ssl.http."; public static final String DEFAULT_KAFKA_CLIENT_ID = "HttpCaptureProxyProducer"; - static class Parameters { + public static class Parameters { @Parameter(required = false, names = {"--traceDirectory"}, arity = 1, @@ -142,7 +142,7 @@ static class Parameters { description = "The header name (which will be interpreted in a case-insensitive manner) and a regex " + "pattern. When the incoming request has a header that matches the regex, it will be passed " + "through to the service but will NOT be captured. E.g. user-agent 'healthcheck'.") - private List suppressCaptureHeaderPairs = new ArrayList<>(); + List suppressCaptureHeaderPairs = new ArrayList<>(); } static Parameters parseArgs(String[] args) { @@ -167,7 +167,7 @@ static Parameters parseArgs(String[] args) { } @SneakyThrows - private static Settings getSettings(@NonNull String configFile) { + protected static Settings getSettings(@NonNull String configFile) { var builder = Settings.builder(); try (var lines = Files.lines(Paths.get(configFile))) { lines @@ -184,7 +184,7 @@ private static Settings getSettings(@NonNull String configFile) { return builder.build(); } - private static IConnectionCaptureFactory getNullConnectionCaptureFactory() { + protected static IConnectionCaptureFactory getNullConnectionCaptureFactory() { System.err.println("No trace log directory specified. Logging to /dev/null"); return ctx -> new StreamChannelConnectionCaptureSerializer<>(null, ctx.getConnectionId(), new StreamLifecycleManager<>() { @@ -201,7 +201,7 @@ public CompletableFuture closeStream(CodedOutputStreamHolder outputStrea }); } - private static String getNodeId() { + protected static String getNodeId() { return UUID.randomUUID().toString(); } @@ -234,7 +234,7 @@ static Properties buildKafkaProperties(Parameters params) throws IOException { return kafkaProps; } - private static IConnectionCaptureFactory + protected static IConnectionCaptureFactory getConnectionCaptureFactory(Parameters params, RootCaptureContext rootContext) throws IOException { var nodeId = getNodeId(); // Resist the urge for now though until it comes in as a request/need. @@ -252,7 +252,7 @@ static Properties buildKafkaProperties(Parameters params) throws IOException { // Utility method for converting uri string to an actual URI object. Similar logic is placed in the trafficReplayer // module: TrafficReplayer.java - private static URI convertStringToUri(String uriString) { + protected static URI convertStringToUri(String uriString) { URI serverUri; try { serverUri = new URI(uriString); @@ -274,7 +274,7 @@ private static URI convertStringToUri(String uriString) { return serverUri; } - private static SslContext loadBacksideSslContext(URI serverUri, boolean allowInsecureConnections) throws + protected static SslContext loadBacksideSslContext(URI serverUri, boolean allowInsecureConnections) throws SSLException { if (serverUri.getScheme().equalsIgnoreCase("https")) { var sslContextBuilder = SslContextBuilder.forClient(); @@ -287,7 +287,7 @@ private static SslContext loadBacksideSslContext(URI serverUri, boolean allowIns } } - private static Map convertPairListToMap(List list) { + protected static Map convertPairListToMap(List list) { var map = new TreeMap(); for (int i=0; i extends ChannelInitializer { - private final IConnectionCaptureFactory connectionCaptureFactory; - private final Supplier sslEngineProvider; - private final IRootWireLoggingContext rootContext; - private final BacksideConnectionPool backsideConnectionPool; - private final RequestCapturePredicate requestCapturePredicate; + protected final IConnectionCaptureFactory connectionCaptureFactory; + protected final Supplier sslEngineProvider; + protected final IRootWireLoggingContext rootContext; + protected final BacksideConnectionPool backsideConnectionPool; + protected final RequestCapturePredicate requestCapturePredicate; public ProxyChannelInitializer(IRootWireLoggingContext rootContext, BacksideConnectionPool backsideConnectionPool, From 3feb0ae0df7384aa52d70b20911eb5a8b38cb2f1 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Tue, 27 Feb 2024 12:15:52 -0600 Subject: [PATCH 9/9] Make CaptureProxy.Parameters field members public Signed-off-by: Andre Kurait --- .../proxyserver/CaptureProxy.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index d7f749428..6c8afd3e0 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -60,89 +60,89 @@ public static class Parameters { names = {"--traceDirectory"}, arity = 1, description = "Directory to store trace files in.") - String traceDirectory; + public String traceDirectory; @Parameter(required = false, names = {"--noCapture"}, arity = 0, description = "If enabled, Does NOT capture traffic to ANY sink.") - boolean noCapture; + public boolean noCapture; @Parameter(required = false, names = {"--kafkaConfigFile"}, arity = 1, description = "Kafka properties file for additional client customization.") - String kafkaPropertiesFile; + public String kafkaPropertiesFile; @Parameter(required = false, names = {"--kafkaClientId"}, arity = 1, description = "clientId to use for interfacing with Kafka.") - String kafkaClientId = DEFAULT_KAFKA_CLIENT_ID; + public String kafkaClientId = DEFAULT_KAFKA_CLIENT_ID; @Parameter(required = false, names = {"--kafkaConnection"}, arity = 1, description = "Sequence of values delimited by ','.") - String kafkaConnection; + public String kafkaConnection; @Parameter(required = false, names = {"--enableMSKAuth"}, arity = 0, description = "Enables SASL Kafka properties required for connecting to MSK with IAM auth.") - boolean mskAuthEnabled = false; + public boolean mskAuthEnabled = false; @Parameter(required = false, names = {"--sslConfigFile"}, arity = 1, description = "YAML configuration of the HTTPS settings. When this is not set, the proxy will not use TLS.") - String sslConfigFilePath; + public String sslConfigFilePath; @Parameter(required = false, names = {"--maxTrafficBufferSize"}, arity = 1, description = "The maximum number of bytes that will be written to a single TrafficStream object.") - int maximumTrafficStreamSize = 1024*1024; + public int maximumTrafficStreamSize = 1024*1024; @Parameter(required = false, names = {"--insecureDestination"}, arity = 0, description = "Do not check the destination server's certificate") - boolean allowInsecureConnectionsToBackside; + public boolean allowInsecureConnectionsToBackside; @Parameter(required = true, names = {"--destinationUri"}, arity = 1, description = "URI of the server that the proxy is capturing traffic for.") - String backsideUriString; + public String backsideUriString; @Parameter(required = true, names = {"--listenPort"}, arity = 1, description = "Exposed port for clients to connect to this proxy.") - int frontsidePort = 0; + public int frontsidePort = 0; @Parameter(required = false, names = {"--numThreads"}, arity = 1, description = "How many threads netty should create in its event loop group") - int numThreads = 1; + public int numThreads = 1; @Parameter(required = false, names = {"--destinationConnectionPoolSize"}, arity = 1, description = "Number of socket connections that should be maintained to the destination server " + "to reduce the perceived latency to clients. Each thread will have its own cache, so the " + "total number of outstanding warm connections will be multiplied by numThreads.") - int destinationConnectionPoolSize = 0; + public int destinationConnectionPoolSize = 0; @Parameter(required = false, names = {"--destinationConnectionPoolTimeout"}, arity = 1, description = "Of the socket connections maintained by the destination connection pool, " + "how long after connection should the be recycled " + "(closed with a new connection taking its place)") - String destinationConnectionPoolTimeout = "PT30S"; + public String destinationConnectionPoolTimeout = "PT30S"; @Parameter(required = false, names = {"--otelCollectorEndpoint"}, arity = 1, description = "Endpoint (host:port) for the OpenTelemetry Collector to which metrics logs should be forwarded." + "If this is not provided, metrics will not be sent to a collector.") - String otelCollectorEndpoint; + public String otelCollectorEndpoint; @Parameter(required = false, names = "--suppressCaptureForHeaderMatch", arity = 2, description = "The header name (which will be interpreted in a case-insensitive manner) and a regex " + "pattern. When the incoming request has a header that matches the regex, it will be passed " + "through to the service but will NOT be captured. E.g. user-agent 'healthcheck'.") - List suppressCaptureHeaderPairs = new ArrayList<>(); + public List suppressCaptureHeaderPairs = new ArrayList<>(); } static Parameters parseArgs(String[] args) {