diff --git a/packages/deployment-service/cdk/lib/cdk-stack.ts b/packages/deployment-service/cdk/lib/cdk-stack.ts index 77f5e3ae98..1d7241ce43 100644 --- a/packages/deployment-service/cdk/lib/cdk-stack.ts +++ b/packages/deployment-service/cdk/lib/cdk-stack.ts @@ -24,7 +24,7 @@ import { createLambda } from './create-lambda' import { createS3Buckets } from './create-S3-bucket' import { createSqsQueues, QueueNames } from './create-sqs' import { createPolicies } from './create-policies' -import { Role } from 'aws-cdk-lib/aws-iam' +import { Effect, Role } from 'aws-cdk-lib/aws-iam' import config from '../../config.json' import * as cdk from 'aws-cdk-lib' @@ -71,11 +71,21 @@ export const createStack = async () => { const api = createApi(stack, 'apigateway', undefined) const vpc = createVpc(stack, 'vpc') - const buckets = createS3Buckets(stack, usercodeStack, envStage) + const buckets = createS3Buckets(usercodeStack, envStage) const queues = createSqsQueues(stack) const database = createDatabase(stack, 'database', databaseName, vpc) const secretManager = database.secret + const AOC = new cdk.aws_cloudfront.CfnOriginAccessControl(usercodeStack, 's3-origin', { + originAccessControlConfig: { + name: 'distro-to-s3', + originAccessControlOriginType: 's3', + signingBehavior: 'always', + signingProtocol: 'sigv4', + description: 'Allow distros to access S3', + }, + }) + if (!secretManager) { throw new Error('Failed to create rds secret') } @@ -101,7 +111,12 @@ export const createStack = async () => { const functionSetups: { [s: string]: FunctionSetup } = { sqs: { handler: createFileLoc('sqs', 'handle'), - policies: [...policies.commonBackendPolicies, policies.cloudFrontPolicy, policies.route53Policy], + policies: [ + ...policies.commonBackendPolicies, + policies.cloudFrontPolicy, + policies.route53Policy, + policies.originAccessControlPolicy, + ], queues: [ queues[QueueNames.CODEBUILD_EXECUTOR], queues[QueueNames.CODEBUILD_VERSION_DEPLOY], @@ -239,23 +254,23 @@ export const createStack = async () => { * is to add the migration script to a second stack which required the first stack */ - // const migrationHandler = createLambda({ - // stack, - // name: 'cloud-deployment-migration', - // entrypoint: 'bundle/migration-run.zip', - // handler: createFileLoc('migration-run', 'migrationRun'), - // runtime: aws_lambda.Runtime.NODEJS_18_X, - // env, - // vpc, - // }) + const migrationHandler = createLambda({ + stack, + name: 'cloud-deployment-migration', + entrypoint: 'bundle/migration-run.zip', + handler: createFileLoc('migration-run', 'migrationRun'), + runtime: aws_lambda.Runtime.NODEJS_18_X, + env, + vpc, + }) - // policies.commonBackendPolicies.forEach((policy) => migrationHandler.addToRolePolicy(policy)) + policies.commonBackendPolicies.forEach((policy) => migrationHandler.addToRolePolicy(policy)) - // Object.values(policies) - // .filter((policy) => policy instanceof PolicyStatement) - // .forEach((policy) => migrationHandler.addToRolePolicy(policy as PolicyStatement)) + Object.values(policies) + .filter((policy) => policy instanceof PolicyStatement) + .forEach((policy) => migrationHandler.addToRolePolicy(policy as PolicyStatement)) - // const numberOfMigrations = await getNumberOfMigrations() + const numberOfMigrations = await getNumberOfMigrations() - // createStackEventHandler(stack, 'migration-event', migrationHandler, `${numberOfMigrations}`) + createStackEventHandler(stack, 'migration-event', migrationHandler, `${numberOfMigrations}`) } diff --git a/packages/deployment-service/cdk/lib/create-S3-bucket.ts b/packages/deployment-service/cdk/lib/create-S3-bucket.ts index d1be9c3992..c2b9ca9d74 100644 --- a/packages/deployment-service/cdk/lib/create-S3-bucket.ts +++ b/packages/deployment-service/cdk/lib/create-S3-bucket.ts @@ -1,5 +1,5 @@ -import { Stack, Bucket, BucketOptions, PolicyStatement } from '@reapit/ts-scripts/src/cdk' -import { aws_s3, PhysicalName, aws_iam } from 'aws-cdk-lib' +import { aws_s3, PhysicalName, aws_iam, Stack } from 'aws-cdk-lib' +import { ServicePrincipal } from 'aws-cdk-lib/aws-iam' export enum BucketNames { LIVE = 'v2-cloud-deployment-live', @@ -8,20 +8,33 @@ export enum BucketNames { REPO_CACHE = 'v2-cloud-deployment-repo-cache', } -export const createBucket = (stack: Stack, bucketName: string, options?: BucketOptions): aws_s3.Bucket => { +type BucketOptions = { + public?: boolean + get?: boolean + list?: boolean + put?: boolean + stack?: Stack +} + +export const createBucket = ({ + stack, + bucketName, + options, +}: { + stack: Stack + bucketName: string + options?: BucketOptions +}): aws_s3.Bucket => { const bucket = new aws_s3.Bucket(options?.stack || stack, bucketName, { - publicReadAccess: true, websiteIndexDocument: options?.public ? 'index.html' : undefined, bucketName: bucketName || PhysicalName.GENERATE_IF_NEEDED, - // blockPublicAccess: aws_s3.BlockPublicAccess.BLOCK_ALL, - // accessControl: aws_s3.BucketAccessControl.PRIVATE, - // objectOwnership: aws_s3.ObjectOwnership.OBJECT_WRITER, - blockPublicAccess: new aws_s3.BlockPublicAccess({ - blockPublicAcls: false, - ignorePublicAcls: false, + blockPublicAccess: { blockPublicPolicy: false, + blockPublicAcls: false, restrictPublicBuckets: false, - }), + ignorePublicAcls: false, + }, + publicReadAccess: false, }) const actions: string[] = [] @@ -36,17 +49,37 @@ export const createBucket = (stack: Stack, bucketName: string, options?: BucketO } bucket.addToResourcePolicy( - new PolicyStatement({ + new aws_iam.PolicyStatement({ effect: aws_iam.Effect.ALLOW, actions, resources: [bucket.arnForObjects('*')], principals: [new aws_iam.ArnPrincipal('*')], }), ) + + // allows cloudfront to access this bucket + bucket.addToResourcePolicy( + new aws_iam.PolicyStatement({ + effect: aws_iam.Effect.ALLOW, + actions: ['s3:Get*'], + resources: [bucket.arnForObjects('*')], + principals: [new ServicePrincipal('cloudfront.amazonaws.com')], + }), + ) + + bucket.addToResourcePolicy( + new aws_iam.PolicyStatement({ + effect: aws_iam.Effect.ALLOW, + actions: ['s3:Get*'], + resources: [bucket.arnForObjects('*')], + principals: [new ServicePrincipal('codebuild.amazonaws.com')], + }), + ) + return bucket } -export const createS3Buckets = (stack: Stack, usercodeStack: Stack, envStage: string): Record => { +export const createS3Buckets = (usercodeStack: Stack, envStage: string): Record => { const bucketOptions: { [k in BucketNames]: BucketOptions } = { @@ -55,32 +88,32 @@ export const createS3Buckets = (stack: Stack, usercodeStack: Stack, envStage: st get: true, list: true, put: true, - stack: usercodeStack, }, [BucketNames.LOG]: { put: true, - stack: usercodeStack, }, [BucketNames.REPO_CACHE]: { put: true, get: true, - stack: usercodeStack, }, [BucketNames.VERSION]: { get: true, list: true, put: true, - stack: usercodeStack, }, } - return (Object.keys(bucketOptions) as Array).reduce<{ [k in BucketNames]: Bucket }>( + return (Object.keys(bucketOptions) as Array).reduce<{ [k in BucketNames]: aws_s3.IBucket }>( (buckets, bucketName) => { - buckets[bucketName] = createBucket( - bucketOptions[bucketName].stack || stack, - `${bucketName}-${envStage}`, - bucketOptions[bucketName], - ) + // const existingBucket = + // Bucket.fromBucketName(usercodeStack, `lookup-${bucketName}`, `${bucketName}-${envStage}`) + + // buckets[bucketName] = existingBucket ?? createBucket({ + buckets[bucketName] = createBucket({ + stack: usercodeStack, + bucketName: `${bucketName}-${envStage}`, + options: bucketOptions[bucketName], + }) return buckets }, {} as any, diff --git a/packages/deployment-service/cdk/lib/create-policies.ts b/packages/deployment-service/cdk/lib/create-policies.ts index 7a3d77db2e..6ea2f8a2e4 100644 --- a/packages/deployment-service/cdk/lib/create-policies.ts +++ b/packages/deployment-service/cdk/lib/create-policies.ts @@ -1,8 +1,9 @@ -import { Project, ISecret, Effect, PolicyStatement, Bucket, Stack, Topic } from '@reapit/ts-scripts/src/cdk' +import { Project, ISecret, Effect, PolicyStatement, Stack, Topic } from '@reapit/ts-scripts/src/cdk' import { AccountPrincipal, ArnPrincipal, CompositePrincipal, Policy, Role } from 'aws-cdk-lib/aws-iam' import config from '../../config.json' import { aws_sqs as sqs } from 'aws-cdk-lib' import { BucketNames } from './create-S3-bucket' +import { IBucket } from 'aws-cdk-lib/aws-s3' export enum PolicyNames { // lambdaInvoke = 'lambdaInvoke', @@ -12,6 +13,7 @@ export enum PolicyNames { sqsPolices = 'sqsPolicies', secretManagerPolicy = 'secretManagerPolicy', S3BucketPolicy = 'S3BucketPolicy', + originAccessControlPolicy = 'originAccessControlPolicy', } type namedPolicyType = { @@ -32,7 +34,7 @@ export const createPolicies = ({ codebuildSnsTopic, githubPemSecretArn, }: { - buckets: { [s: string]: Bucket } + buckets: { [s: string]: IBucket } queues: { [s: string]: sqs.IQueue } secretManager: ISecret codeBuild: Project @@ -164,6 +166,12 @@ export const createPolicies = ({ ], }) + const originAccessControlPolicy = new PolicyStatement({ + actions: ['cloudfront:ListOriginAccessControls'], + effect: Effect.ALLOW, + resources: ['*'], + }) + // create a policy that allows the lambda to do what it needs to do in the usercode stack const usercodePolicy = new Policy(usercodeStack, 'UsercodePolicy') usercodePolicy.addStatements( @@ -173,6 +181,7 @@ export const createPolicies = ({ codebuildSnssubscriptionPolicy, codebuildExecPolicy, parameterStorePolicy, + originAccessControlPolicy, ) const usercodeStackRoleName = `${usercodeStack.stackName}-UsercodeStackRole` // create a role that lambdas can assume in the usercode stack, with the policy we just created @@ -216,6 +225,7 @@ export const createPolicies = ({ commonBackendPolicies, codebuildExecPolicy, cloudFrontPolicy, + originAccessControlPolicy, route53Policy, sqsPolicies, secretManagerPolicy, diff --git a/packages/deployment-service/jest.config.js b/packages/deployment-service/jest.config.js index fe0efc12c6..d70d390e5b 100644 --- a/packages/deployment-service/jest.config.js +++ b/packages/deployment-service/jest.config.js @@ -7,7 +7,15 @@ module.exports = { testPathIgnorePatterns: ['/dist'], collectCoverageFrom: ['/src/**/*.ts'], coverageDirectory: './src/tests/coverage', - coveragePathIgnorePatterns: ['[/\\\\](node_modules|src/types|src/tests|src/scripts)[/\\\\]', '.d.ts'], + coveragePathIgnorePatterns: [ + '[/\\\\](node_modules|src/types|src/tests|src/scripts)[/\\\\]', + '.d.ts', + 'src/http.ts', + 'src/sqs.ts', + 'src/sns.ts', + 'src/local-http.ts', + 'src/migration-run.ts', + ], reporters: ['default', 'github-actions'], coverageThreshold: { global: { diff --git a/packages/deployment-service/src/deployment/deploy-provider.ts b/packages/deployment-service/src/deployment/deploy-provider.ts index be6c1d8c66..d4cd1ab1ea 100644 --- a/packages/deployment-service/src/deployment/deploy-provider.ts +++ b/packages/deployment-service/src/deployment/deploy-provider.ts @@ -136,7 +136,7 @@ export class DeployProvider { Bucket: process.env.DEPLOYMENT_LIVE_BUCKET_NAME as string, Key: `${prefix}/${fileName}`, Body: buffer, - ACL: 'public-read', + // ACL: 'public-read', ContentType: mimeType, Metadata: { ['Content-Type']: mimeType, diff --git a/packages/deployment-service/src/pipeline/__test__/pipeline-setup-workflow.test.ts b/packages/deployment-service/src/pipeline/__test__/pipeline-setup-workflow.test.ts index f41e0e29ca..cbca7dbc77 100644 --- a/packages/deployment-service/src/pipeline/__test__/pipeline-setup-workflow.test.ts +++ b/packages/deployment-service/src/pipeline/__test__/pipeline-setup-workflow.test.ts @@ -3,7 +3,7 @@ import { PipelineSetupWorkflow } from '../pipeline-setup-workflow' import { PusherProvider, SqsProvider } from '../../events' import { PipelineProvider } from '../pipeline-provider' import { SQS, S3 } from 'aws-sdk' -import { CloudFrontClient } from '@aws-sdk/client-cloudfront' +import { CloudFrontClient, ListOriginAccessControlsCommand } from '@aws-sdk/client-cloudfront' import { Route53Client } from '@aws-sdk/client-route-53' import { S3Provider } from '../../s3' import { v4 as uuid } from 'uuid' @@ -84,12 +84,28 @@ describe('PipelineSetupWorkflow', () => { const pipelineId = uuid() const developerId = uuid() - mockCloudFrontClient.send.mockImplementationOnce(() => ({ - Distribution: { - DomainName: 'domain', - Id: 'id', - }, - })) + mockCloudFrontClient.send.mockImplementation((event) => { + console.log('event', event) + if (event instanceof ListOriginAccessControlsCommand) { + return { + OriginAccessControlList: { + Items: [ + { + Name: 'distro-to-s3', + Id: 'AOCID', + }, + ], + }, + } + } + + return { + Distribution: { + DomainName: 'domain', + Id: 'id', + }, + } + }) mockRoute53Client.send.mockImplementationOnce(() => ({ ChangeInfo: { diff --git a/packages/deployment-service/src/pipeline/pipeline-setup-workflow.ts b/packages/deployment-service/src/pipeline/pipeline-setup-workflow.ts index b8670dc268..909541abc9 100644 --- a/packages/deployment-service/src/pipeline/pipeline-setup-workflow.ts +++ b/packages/deployment-service/src/pipeline/pipeline-setup-workflow.ts @@ -1,7 +1,11 @@ import { AbstractWorkflow, PusherProvider, SqsProvider, Workflow } from '../events' import { S3Provider } from '../s3' import { PipelineProvider } from './pipeline-provider' -import { CloudFrontClient, CreateDistributionCommand } from '@aws-sdk/client-cloudfront' +import { + CloudFrontClient, + CreateDistributionCommand, + ListOriginAccessControlsCommand, +} from '@aws-sdk/client-cloudfront' import { PipelineEntity } from '../entities/pipeline.entity' import { v4 as uuid } from 'uuid' import { ChangeResourceRecordSetsCommand, Route53Client } from '@aws-sdk/client-route-53' @@ -64,9 +68,19 @@ export class PipelineSetupWorkflow extends AbstractWorkflow { } } + private async findOACId(): Promise { + const result = await this.cloudfrontClient.send(new ListOriginAccessControlsCommand({})) + + return result.OriginAccessControlList?.Items?.find((item) => item.Name === 'distro-to-s3')?.Id + } + private async createDistro(pipeline: PipelineEntity) { const id = uuid() + const AOCId = await this.findOACId() + + if (!AOCId) throw new Error('Could not resolve AOCId') + const distroCommand = new CreateDistributionCommand({ DistributionConfig: { DefaultRootObject: 'index.html', @@ -85,12 +99,13 @@ export class PipelineSetupWorkflow extends AbstractWorkflow { S3OriginConfig: { OriginAccessIdentity: '', }, + OriginAccessControlId: AOCId, }, ], }, Aliases: { Quantity: 1, - Items: [`${pipeline.subDomain}.iaas.paas.reapit.cloud`], + Items: [`${pipeline.subDomain}.iaas${process.env.NODE_ENV === 'production' ? '' : '.dev'}.paas.reapit.cloud`], }, Comment: `Cloudfront distribution for pipeline [${pipeline.id}] [${ process.env.NODE_ENV === 'production' ? 'prod' : 'dev' @@ -143,7 +158,7 @@ export class PipelineSetupWorkflow extends AbstractWorkflow { Action: 'UPSERT', ResourceRecordSet: { Type: 'A', - Name: `${pipeline.subDomain}.iaas.paas.reapit.cloud`, + Name: `${pipeline.subDomain}.iaas${process.env.NODE_ENV === 'production' ? '' : '.dev'}.paas.reapit.cloud`, AliasTarget: { DNSName: frontDomain, EvaluateTargetHealth: false, diff --git a/packages/developer-portal/src/components/apps/pipeline/__tests__/__snapshots__/pipeline-info.test.tsx.snap b/packages/developer-portal/src/components/apps/pipeline/__tests__/__snapshots__/pipeline-info.test.tsx.snap index cdd7ce30fe..187aef1d39 100644 --- a/packages/developer-portal/src/components/apps/pipeline/__tests__/__snapshots__/pipeline-info.test.tsx.snap +++ b/packages/developer-portal/src/components/apps/pipeline/__tests__/__snapshots__/pipeline-info.test.tsx.snap @@ -287,11 +287,11 @@ exports[`Pipelineinfo Should match snapshot 1`] = ` class="el-text-base el-body-text el-has-grey-text el-has-no-margin" > - https://beautiful-land.iaas.paas.reapit.cloud + https://beautiful-land.dev.iaas.paas.reapit.cloud

@@ -687,11 +687,11 @@ exports[`Pipelineinfo Should match snapshot 1`] = ` class="el-text-base el-body-text el-has-grey-text el-has-no-margin" > - https://beautiful-land.iaas.paas.reapit.cloud + https://beautiful-land.dev.iaas.paas.reapit.cloud

diff --git a/packages/developer-portal/src/components/apps/pipeline/__tests__/__snapshots__/pipeline-page.test.tsx.snap b/packages/developer-portal/src/components/apps/pipeline/__tests__/__snapshots__/pipeline-page.test.tsx.snap index a8f8787bf9..75429b2b67 100644 --- a/packages/developer-portal/src/components/apps/pipeline/__tests__/__snapshots__/pipeline-page.test.tsx.snap +++ b/packages/developer-portal/src/components/apps/pipeline/__tests__/__snapshots__/pipeline-page.test.tsx.snap @@ -292,11 +292,11 @@ exports[`PipelinePage should match snapshot 1`] = ` class="el-text-base el-body-text el-has-grey-text el-has-no-margin" > - https://beautiful-land.iaas.paas.reapit.cloud + https://beautiful-land.dev.iaas.paas.reapit.cloud

@@ -740,11 +740,11 @@ exports[`PipelinePage should match snapshot 1`] = ` class="el-text-base el-body-text el-has-grey-text el-has-no-margin" > - https://beautiful-land.iaas.paas.reapit.cloud + https://beautiful-land.dev.iaas.paas.reapit.cloud

@@ -1250,11 +1250,11 @@ exports[`PipelinePage should match snapshot for mobile view 1`] = ` class="el-text-base el-body-text el-has-grey-text el-has-no-margin" > - https://beautiful-land.iaas.paas.reapit.cloud + https://beautiful-land.dev.iaas.paas.reapit.cloud

@@ -1698,11 +1698,11 @@ exports[`PipelinePage should match snapshot for mobile view 1`] = ` class="el-text-base el-body-text el-has-grey-text el-has-no-margin" > - https://beautiful-land.iaas.paas.reapit.cloud + https://beautiful-land.dev.iaas.paas.reapit.cloud

diff --git a/packages/developer-portal/src/components/apps/pipeline/pipeline-info.tsx b/packages/developer-portal/src/components/apps/pipeline/pipeline-info.tsx index de97cf75ca..b017ceb967 100644 --- a/packages/developer-portal/src/components/apps/pipeline/pipeline-info.tsx +++ b/packages/developer-portal/src/components/apps/pipeline/pipeline-info.tsx @@ -17,8 +17,8 @@ import { PipelineDeploymentTable } from './pipeline-deployments-table' import { PipelineTabs } from './pipeline-tabs' export const getAppFromPipeline = (isGithub: boolean) => { - const isProd = process.env.appEnv === 'production' const isBitbucket = !isGithub + const isProd = process.env.appEnv === 'production' if (isBitbucket && isProd) return 'https://bitbucket.org/site/addons/authorize?addon_key=reapit' if (isBitbucket) return 'https://bitbucket.org/site/addons/authorize?addon_key=reapit-dev' @@ -27,9 +27,10 @@ export const getAppFromPipeline = (isGithub: boolean) => { } export const PipelineInfo: FC = () => { + const isProd = process.env.appEnv === 'production' const { appPipelineState } = useAppState() const { appPipeline } = appPipelineState - const pipelineUri = `https://${appPipeline?.subDomain}.iaas.paas.reapit.cloud` + const pipelineUri = `https://${appPipeline?.subDomain}${isProd ? '' : '.dev'}.iaas.paas.reapit.cloud` const isGithub = Boolean(appPipeline?.repository?.repositoryUrl?.includes('github')) const isBitbucket = Boolean(appPipeline?.repository?.repositoryUrl?.includes('bitbucket')) const hasGithubApp = Boolean(appPipeline?.repository?.installationId) diff --git a/packages/ts-scripts/src/cdk/components/rds-database.ts b/packages/ts-scripts/src/cdk/components/rds-database.ts index 0bdab32744..6f58908a55 100644 --- a/packages/ts-scripts/src/cdk/components/rds-database.ts +++ b/packages/ts-scripts/src/cdk/components/rds-database.ts @@ -29,12 +29,6 @@ export const createDatabase = ( db.connections.allowFromAnyIpv4(ec2.Port.allTcp()) - if (!bastion) { - bastion = new ec2.BastionHostLinux(stack, 'bastion', { - vpc, - }) - } - return db }