Implementing CDK Pipelines for resource orchestration

PetAdoptions/cdk/pet_stack/.vscode/launch.json

@@ -11,7 +11,12 @@
                 "./node_modules/ts-node/register/transpile-only"
             ],
             // Entry point of your stack
-            "args": ["${workspaceFolder}/app/pet_stack.ts"]
+            "args": [
+                        "${workspaceFolder}/app/pet_stack.ts"
+                    ],
+            "env": {
+                "CONFIG_PATH": "${workspaceFolder}/config.yaml"
+            }
         }
     ]
 }

PetAdoptions/cdk/pet_stack/app/pet_stack.ts

@@ -1,26 +1,22 @@
 #!/usr/bin/env node
 import 'source-map-support/register';
-import { Services } from '../lib/services';
-import { Applications } from '../lib/applications';
-//import { EKSPetsite } from '../lib/ekspetsite'
 import { App, Tags, Aspects } from 'aws-cdk-lib';
+import { CDKPipeline } from '../lib/pipeline';
 //import { AwsSolutionsChecks } from 'cdk-nag';
 
 
-const stackName = "Services";
+const stackName = "OneObservabilityWorkshop";
 const app = new App();
 
-const stack = new Services(app, stackName, { 
+const pipelineStack = new CDKPipeline(app, stackName+"Pipeline", {
+  sourceBucketName: process.env.SOURCE_BUCKET_NAME || "fake-bucket-wont-work",
+  branchName: process.env.GITHUB_BRANCH || "main",
   env: { 
     account: process.env.CDK_DEFAULT_ACCOUNT, 
     region: process.env.CDK_DEFAULT_REGION 
-}});
+  }
+});
 
-const applications = new Applications(app, "Applications", {
-  env: { 
-    account: process.env.CDK_DEFAULT_ACCOUNT, 
-    region: process.env.CDK_DEFAULT_REGION 
-}});
 
 Tags.of(app).add("Workshop","true")
 //Aspects.of(stack).add(new AwsSolutionsChecks({verbose: true}));

PetAdoptions/cdk/pet_stack/config.yaml

@@ -0,0 +1 @@
+createXRayGroup: true

PetAdoptions/cdk/pet_stack/lib/applicationsStage.ts

@@ -0,0 +1,17 @@
+
+import { Stage, StageProps } from "aws-cdk-lib";
+import { Construct } from "constructs";
+import { Applications } from "./stacks/applications";
+
+export class ApplicationsStage extends Stage {
+    constructor(scope: Construct, id: string, props: StageProps) {
+        super(scope, id, props);
+
+        const stackName = "Applications";
+        const stack = new Applications(this, stackName, { 
+            env: { 
+              account: props.env?.account,
+              region:  props.env?.region
+          }});
+    }
+}

PetAdoptions/cdk/pet_stack/lib/common/config.ts

@@ -0,0 +1,32 @@
+import * as fs from 'fs';
+import path = require('path');
+import * as yaml from 'js-yaml';
+import { log } from 'console';
+import { Construct } from 'constructs';
+
+export interface WorkshopConfig
+{
+    readonly createXRayGroup : boolean;
+}
+
+export function getConfig(app: Construct) : WorkshopConfig {
+
+    // Default configuration
+    let config = {
+        createXRayGroup: false
+    };
+    if (process.env.CONFIG_PATH) {
+        let configPath = process.env.CONFIG_PATH;
+        log(`Using config file: ${configPath}`);
+        /// Check if the file exists and is not empty
+        if (!fs.existsSync(configPath)) {
+            throw new Error(`Config file ${configPath} does not exist`);
+        }
+        /// Check if configPath exists. If it exists read the content of the file as YAML and convert the result into an object using WorkshopConfig interface
+        let configContent = fs.readFileSync(configPath, 'utf8');
+        config = yaml.load(configContent) as WorkshopConfig;
+    }
+
+
+    return config;
+}

PetAdoptions/cdk/pet_stack/lib/constructs/imageBuiltStep.ts

@@ -0,0 +1,50 @@
+import { PolicyStatement } from "aws-cdk-lib/aws-iam";
+import { CodeBuildStep, CodePipelineSource } from "aws-cdk-lib/pipelines";
+
+export interface ImageBuildStepProps {
+    repositoryName: string;
+    repositoryUri: string;
+    source: CodePipelineSource;
+    account: string;
+    region: string;
+    branchName: string;
+
+}
+
+export class ImageBuildStep extends CodeBuildStep {
+    constructor(name: string, props: ImageBuildStepProps) {
+        super(name, {
+            commands: [
+                'nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2 &',
+                'timeout 15 sh -c "until docker info; do echo .; sleep 1; done"',
+                'cd ${BASE_PATH}',
+                'aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com',
+                'docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG .',
+                'docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG',
+                'docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG'
+            ],
+            rolePolicyStatements: [
+                new PolicyStatement({
+                    actions: [
+                      'ecr:*',
+                    ],
+                    resources: ['*'],
+                  }),
+            ],
+            input: props.source,
+            buildEnvironment: {
+                privileged: true
+            },
+            env: {
+                'AWS_ACCOUNT_ID': props.account,
+                'AWS_DEFAULT_REGION': props.region,
+                'IMAGE_TAG': "latest",
+                'ECR_REPOSITORY_URL': props.repositoryUri,
+                'IMAGE_REPO_NAME': props.repositoryName,
+                'BASE_PATH': `one-observability-demo-${props.branchName}/PetAdoptions/${props.repositoryName}`
+            }
+        });
+
+        this.consumedStackOutputs.push()
+    }
+}

PetAdoptions/cdk/pet_stack/lib/constructs/network.ts

@@ -0,0 +1,69 @@
+import { Construct } from "constructs";
+import * as ec2 from 'aws-cdk-lib/aws-ec2';
+import * as logs from 'aws-cdk-lib/aws-logs';
+import * as iam from 'aws-cdk-lib/aws-iam';
+
+export interface WorkshopNetworkProps {
+    name: string;
+    cidrRange: string;
+}
+
+export class WorkshopNetwork extends Construct {
+    public readonly vpc : ec2.Vpc;
+    constructor(scope: Construct, id: string, props: WorkshopNetworkProps) {
+        super(scope, id);  
+
+        // Create a VPC with public and private subnets
+        // The VPC where all the microservices will be deployed into
+        this.vpc = new ec2.Vpc(this, 'VPC-' + props.name, {
+            ipAddresses: ec2.IpAddresses.cidr(props.cidrRange),
+            natGateways: 1,
+            maxAzs: 2
+        });
+
+        const flowLogGroup = new logs.LogGroup(this, 'FlowLogGroup', {
+            logGroupName: '/aws/vpcflowlogs/' + this.vpc.vpcId,
+            retention: logs.RetentionDays.ONE_WEEK
+        });
+
+        const role = new iam.Role(this, 'VPCFlowLogRole', {
+            assumedBy: new iam.ServicePrincipal('vpc-flow-logs.amazonaws.com')
+        });
+
+        const flowLog = new ec2.FlowLog(this, 'VPCFlowLog', {
+            destination: ec2.FlowLogDestination.toCloudWatchLogs(flowLogGroup, role),
+            resourceType: ec2.FlowLogResourceType.fromVpc(this.vpc),
+            logFormat: [
+                ec2.LogFormat.ACCOUNT_ID,
+                ec2.LogFormat.ACTION,
+                ec2.LogFormat.AZ_ID,
+                ec2.LogFormat.BYTES,
+                ec2.LogFormat.DST_ADDR,
+                ec2.LogFormat.DST_PORT,
+                ec2.LogFormat.END_TIMESTAMP,
+                ec2.LogFormat.FLOW_DIRECTION,
+                ec2.LogFormat.INSTANCE_ID,
+                ec2.LogFormat.INTERFACE_ID,
+                ec2.LogFormat.LOG_STATUS,
+                ec2.LogFormat.PACKETS,
+                ec2.LogFormat.PKT_DST_AWS_SERVICE,
+                ec2.LogFormat.PKT_DST_ADDR,
+                ec2.LogFormat.PKT_SRC_AWS_SERVICE,
+                ec2.LogFormat.PKT_SRC_ADDR,
+                ec2.LogFormat.PROTOCOL,
+                ec2.LogFormat.REGION,
+                ec2.LogFormat.SRC_ADDR,
+                ec2.LogFormat.SRC_PORT,
+                ec2.LogFormat.START_TIMESTAMP,
+                ec2.LogFormat.SUBLOCATION_ID,
+                ec2.LogFormat.SUBLOCATION_TYPE,
+                ec2.LogFormat.SUBNET_ID,
+                ec2.LogFormat.TCP_FLAGS,
+                ec2.LogFormat.TRAFFIC_PATH,
+                ec2.LogFormat.TRAFFIC_TYPE,
+                ec2.LogFormat.VERSION,
+                ec2.LogFormat.VPC_ID
+            ]
+        });
+    }   
+}

PetAdoptions/cdk/pet_stack/lib/constructs/repository.ts

@@ -0,0 +1,36 @@
+import { RemovalPolicy, Stack } from 'aws-cdk-lib';
+import { Construct  } from "constructs";
+import * as ecr from "aws-cdk-lib/aws-ecr";
+import * as iam from "aws-cdk-lib/aws-iam";
+import { NagSuppressions } from 'cdk-nag';
+import { CodeBuildStep } from 'aws-cdk-lib/pipelines';
+
+export interface RepositoryProps {
+    name: string;
+    enableScanOnPush: boolean;
+    initialCodePath: string;
+}
+
+
+export class Repository extends Construct {
+    public readonly imageRepo: ecr.Repository
+    public readonly codeBuildStep: CodeBuildStep;
+
+    constructor(scope: Construct, id: string, props: RepositoryProps) {
+        super(scope, id);  
+
+        this.imageRepo = new ecr.Repository(scope, props.name + "ImageRepo", {
+            repositoryName: props.name, 
+            imageScanOnPush: props.enableScanOnPush,
+            imageTagMutability: ecr.TagMutability.MUTABLE,  // Set to Mutable to allow the Pipeline to run multiple times. An alternative solution can be used to delete the latest before pushing the new build.
+            removalPolicy: RemovalPolicy.DESTROY,
+            encryption: ecr.RepositoryEncryption.AES_256,
+            autoDeleteImages: true
+        });
+
+    }
+
+    public getECRUri() {
+        return this.imageRepo.repositoryUri;
+    }
+}

PetAdoptions/cdk/pet_stack/lib/imageBuilderStage.ts

@@ -0,0 +1,20 @@
+import { Stage, StageProps } from "aws-cdk-lib";
+import { Construct } from "constructs";
+import { ImageBuilderStack } from "./stacks/imageBuilder";
+
+export class ImageBuilderStage extends Stage {
+    public readonly repoList = new Map<string, string>();
+    constructor(scope: Construct, id: string, props: StageProps) {
+        super(scope, id, props);
+
+        const stackName = "ImageBuilder";
+        const coreStack = new ImageBuilderStack(this, stackName, { 
+            env: { 
+                account: props.env?.account,
+                region:  props.env?.region
+            },
+        });
+
+        this.repoList = coreStack.repoList;
+    }
+}