Merge pull request #127 from grapl-security/cdk-naming

CDK - Add support for multiple Grapl deployments to single AWS account
grapl-security · Jun 26, 2020 · 0d772fd · 0d772fd
2 parents e6bd9f8 + 2b8cffa
commit 0d772fd
Show file tree

Hide file tree

Showing 24 changed files with 1,210 additions and 1,069 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -71,6 +71,15 @@ services:
       - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
       - "SOURCE_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-node-identifier-queue"
       - "GRAPH_MERGER_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-graph-merger-queue"
+      - STATIC_MAPPING_TABLE=local-grapl-static_mapping_table
+      - DYNAMIC_SESSION_TABLE=local-grapl-dynamic_session_table
+      - PROCESS_HISTORY_TABLE=local-grapl-process_history_table
+      - FILE_HISTORY_TABLE=local-grapl-file_history_table
+      - INBOUND_CONNECTION_HISTORY_TABLE=local-grapl-inbound_connection_history_table
+      - OUTBOUND_CONNECTION_HISTORY_TABLE=local-grapl-outbound_connection_history_table
+      - NETWORK_CONNECTION_HISTORY_TABLE=local-grapl-network_connection_history_table
+      - IP_CONNECTION_HISTORY_TABLE=local-grapl-ip_connection_history_table
+      - ASSET_ID_MAPPINGS=local-grapl-asset_id_mappings
       - RUST_LOG=${RUST_LOG:-ERROR}
     tty: false
     links:
@@ -237,6 +246,7 @@ services:
       - "8124:8124"
     environment:
       - "IS_LOCAL=True"
+      - MG_ALPHAS=master_graph:9080
       - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
       - GRAPL_DGRAPH_TTL_S=${GRAPL_DGRAPH_TTL_S:-600}
       - GRAPL_TTL_DELETE_BATCH_SIZE=${GRAPL_TTL_DELETE_BATCH_SIZE:-10}

diff --git a/etc/local_grapl/local_dynamodb/provision_local_identity_table.py b/etc/local_grapl/local_dynamodb/provision_local_identity_table.py
@@ -3,20 +3,19 @@
 import boto3
 
 table_names = [
-    "process_history_table",
-    "file_history_table",
-    "node_id_retry_table",
-    "outbound_connection_history_table",
-    "inbound_connection_history_table",
-    "network_connection_history_table",
-    "ip_connection_history_table",
-    "asset_id_mappings",
-    "dynamic_session_table",
-    "static_mapping_table",
+    "local-grapl-process_history_table",
+    "local-grapl-file_history_table",
+    "local-grapl-outbound_connection_history_table",
+    "local-grapl-inbound_connection_history_table",
+    "local-grapl-network_connection_history_table",
+    "local-grapl-ip_connection_history_table",
+    "local-grapl-asset_id_mappings",
+    "local-grapl-dynamic_session_table",
+    "local-grapl-static_mapping_table",
 ]
 
 table_defs = {
-    "network_connection_history_table": {
+    "local-grapl-network_connection_history_table": {
         "key_schema": [
             {"KeyType": "HASH", "AttributeName": "pseudo_key"},
             {"KeyType": "RANGE", "AttributeName": "create_time"},
@@ -26,7 +25,7 @@
             {"AttributeName": "pseudo_key", "AttributeType": "S"},
         ],
     },
-    "inbound_connection_history_table": {
+    "local-grapl-inbound_connection_history_table": {
         "key_schema": [
             {"KeyType": "HASH", "AttributeName": "pseudo_key"},
             {"KeyType": "RANGE", "AttributeName": "create_time"},
@@ -36,19 +35,13 @@
             {"AttributeName": "pseudo_key", "AttributeType": "S"},
         ],
     },
-    "node_id_retry_table": {
+    "local-grapl-static_mapping_table": {
         "key_schema": [{"KeyType": "HASH", "AttributeName": "pseudo_key"}],
         "attribute_definitions": [
             {"AttributeName": "pseudo_key", "AttributeType": "S"}
         ],
     },
-    "static_mapping_table": {
-        "key_schema": [{"KeyType": "HASH", "AttributeName": "pseudo_key"}],
-        "attribute_definitions": [
-            {"AttributeName": "pseudo_key", "AttributeType": "S"}
-        ],
-    },
-    "dynamic_session_table": {
+    "local-grapl-dynamic_session_table": {
         "key_schema": [
             {"KeyType": "HASH", "AttributeName": "pseudo_key"},
             {"KeyType": "RANGE", "AttributeName": "create_time"},
@@ -58,7 +51,7 @@
             {"AttributeName": "pseudo_key", "AttributeType": "S"},
         ],
     },
-    "process_history_table": {
+    "local-grapl-process_history_table": {
         "key_schema": [
             {"KeyType": "HASH", "AttributeName": "pseudo_key"},
             {"KeyType": "RANGE", "AttributeName": "create_time"},
@@ -68,7 +61,7 @@
             {"AttributeName": "pseudo_key", "AttributeType": "S"},
         ],
     },
-    "file_history_table": {
+    "local-grapl-file_history_table": {
         "key_schema": [
             {"KeyType": "HASH", "AttributeName": "pseudo_key"},
             {"KeyType": "RANGE", "AttributeName": "create_time"},
@@ -78,7 +71,7 @@
             {"AttributeName": "pseudo_key", "AttributeType": "S"},
         ],
     },
-    "outbound_connection_history_table": {
+    "local-grapl-outbound_connection_history_table": {
         "key_schema": [
             {"KeyType": "HASH", "AttributeName": "pseudo_key"},
             {"KeyType": "RANGE", "AttributeName": "create_time"},
@@ -88,7 +81,7 @@
             {"AttributeName": "pseudo_key", "AttributeType": "S"},
         ],
     },
-    "asset_id_mappings": {
+    "local-grapl-asset_id_mappings": {
         "key_schema": [
             {"KeyType": "HASH", "AttributeName": "pseudo_key"},
             {"KeyType": "RANGE", "AttributeName": "c_timestamp"},
@@ -98,7 +91,7 @@
             {"AttributeName": "pseudo_key", "AttributeType": "S"},
         ],
     },
-    "ip_connection_history_table": {
+    "local-grapl-ip_connection_history_table": {
         "key_schema": [
             {"KeyType": "HASH", "AttributeName": "pseudo_key"},
             {"KeyType": "RANGE", "AttributeName": "create_time"},

diff --git a/src/js/grapl-cdk/.gitignore b/src/js/grapl-cdk/.gitignore
@@ -2,7 +2,6 @@
 !jest.config.js
 *.d.ts
 node_modules
-.env
 
 # CDK asset staging directory
 .cdk.staging

diff --git a/src/js/grapl-cdk/bin/grapl-cdk.ts b/src/js/grapl-cdk/bin/grapl-cdk.ts
@@ -3,32 +3,23 @@ import 'source-map-support/register';
 import * as cdk from '@aws-cdk/core';
 
 import { GraplCdkStack } from '../lib/grapl-cdk-stack';
-import { EngagementUx, EngagementEdge } from '../lib/engagement';
-import { GraphQLEndpoint } from '../lib/graphql';
+import { EngagementUx } from '../lib/engagement';
 
-const env = require('node-env-file');
-
-env(__dirname + '/../.env');
+const deployName = 'Grapl-MYDEPLOYMENT';
 
 const app = new cdk.App();
-const grapl = new GraplCdkStack(app, 'Grapl');
-
-const engagement_edge = new EngagementEdge(
-    app,
-    'EngagementEdge',
-    grapl.grapl_env
-);
-
-const graphql_endpoint = new GraphQLEndpoint(
-    app,
-    'GraphqlEndpoint',
-    grapl.grapl_env
-);
+const grapl = new GraplCdkStack(app, 'Grapl', {
+    version: 'latest',
+    stackName: deployName,
+    tags: {'grapl deployment': deployName},
+});
 
-const ux = new EngagementUx(
+new EngagementUx(
     app,
-    'EngagementUX',
-    grapl.grapl_env.prefix,
-    engagement_edge,
-    graphql_endpoint
+    'EngagementUX', {
+        prefix: grapl.prefix,
+        engagement_edge: grapl.engagement_edge,
+        graphql_endpoint: grapl.graphql_endpoint,
+        stackName: deployName + '-EngagementUX',
+    }
 );
diff --git a/src/js/grapl-cdk/cdk.json b/src/js/grapl-cdk/cdk.json
@@ -1,3 +1,6 @@
 {
-  "app": "npx ts-node bin/grapl-cdk.ts"
+  "app": "npx ts-node bin/grapl-cdk.ts",
+  "context": {
+    "@aws-cdk/core:newStyleStackSynthesis": true
+  }
 }
diff --git a/src/js/grapl-cdk/deploy_all.sh b/src/js/grapl-cdk/deploy_all.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 npm run build &&
-cdk deploy --require-approval=never Grapl EngagementEdge GraphqlEndpoint && \
+cdk deploy --require-approval=never Grapl && \
 rm -rf ./edge_ux_package && \
 cdk synth && \
 cdk deploy --require-approval=never EngagementUX && \

diff --git a/src/js/grapl-cdk/lib/dgraph.ts b/src/js/grapl-cdk/lib/dgraph.ts
@@ -115,39 +115,44 @@ class Alpha extends cdk.Construct {
     }
 }
 
+export interface DGraphEcsProps {
+    prefix: string,
+    vpc: ec2.Vpc,
+    alphaCount: number,
+    alphaPort: number
+    zeroCount: number,
+}
+
 export class DGraphEcs extends cdk.Construct {
     readonly alphas: [string, number][];
 
     constructor(
         scope: cdk.Construct,
         id: string,
-        vpc: ec2.Vpc,
-        zeroCount: number,
-        alphaCount: number,
-        alphaPort: number
+        props: DGraphEcsProps,
     ) {
         super(scope, id);
 
         const cluster = new ecs.Cluster(this, 'EcsCluster', {
-            clusterName: `Grapl-${id}-EcsCluster`,
-            vpc: vpc
+            clusterName: `${props.prefix}-${id}-EcsCluster`,
+            vpc: props.vpc
         });
 
         cluster.connections.allowInternally(ec2.Port.allTcp());
 
-        const namespace = cluster.addDefaultCloudMapNamespace(
+        cluster.addDefaultCloudMapNamespace(
             {
                 name: id + '.grapl',
                 type: servicediscovery.NamespaceType.DNS_PRIVATE,
-                vpc
+                vpc: props.vpc,
             }
         );
 
         cluster.addCapacity('ZeroGroupCapacity',
             {
                 instanceType: new ec2.InstanceType("t3a.small"),
-                minCapacity: zeroCount,
-                maxCapacity: zeroCount,
+                minCapacity: props.zeroCount,
+                maxCapacity: props.zeroCount,
             }
         );
 
@@ -160,7 +165,7 @@ export class DGraphEcs extends cdk.Construct {
             1
         );
 
-        for (let i = 1; i < zeroCount; i++) {
+        for (let i = 1; i < props.zeroCount; i++) {
             new Zero(
                 this,
                 id,
@@ -176,12 +181,12 @@ export class DGraphEcs extends cdk.Construct {
         cluster.addCapacity('AlphaGroupCapacity',
             {
                 instanceType: new ec2.InstanceType("t3a.2xlarge"),
-                minCapacity: alphaCount,
-                maxCapacity: alphaCount,
+                minCapacity: props.alphaCount,
+                maxCapacity: props.alphaCount,
             }
         );
 
-        for (let i = 0; i < alphaCount; i++) {
+        for (let i = 0; i < props.alphaCount; i++) {
 
             const alpha = new Alpha(
                 this,
@@ -191,7 +196,7 @@ export class DGraphEcs extends cdk.Construct {
                 "zero0"
             );
 
-            this.alphas.push([alpha.name, alphaPort]);
+            this.alphas.push([alpha.name, props.alphaPort]);
         }
     };