From e7ad990a4d752c9d7a57d27687628430d818da0c Mon Sep 17 00:00:00 2001 From: Sander Knape Date: Sat, 18 May 2019 00:08:38 +0200 Subject: [PATCH] feat(codebuild): add support for local cache modes (#2529) Fixes #1956 --- packages/@aws-cdk/aws-codebuild/README.md | 30 +++++++ packages/@aws-cdk/aws-codebuild/lib/cache.ts | 83 +++++++++++++++++++ packages/@aws-cdk/aws-codebuild/lib/index.ts | 1 + .../@aws-cdk/aws-codebuild/lib/project.ts | 35 +++----- .../aws-codebuild/test/integ.caching.ts | 3 +- .../aws-codebuild/test/test.project.ts | 79 +++++++++++++++++- 6 files changed, 206 insertions(+), 25 deletions(-) create mode 100644 packages/@aws-cdk/aws-codebuild/lib/cache.ts diff --git a/packages/@aws-cdk/aws-codebuild/README.md b/packages/@aws-cdk/aws-codebuild/README.md index 5e3602d611718..3f1a07a2c3830 100644 --- a/packages/@aws-cdk/aws-codebuild/README.md +++ b/packages/@aws-cdk/aws-codebuild/README.md @@ -109,6 +109,36 @@ aws codebuild import-source-credentials --server-type GITHUB --auth-type PERSONA This source type can be used to build code from a BitBucket repository. +## Caching + +You can save time when your project builds by using a cache. A cache can store reusable pieces of your build environment and use them across multiple builds. Your build project can use one of two types of caching: Amazon S3 or local. In general, S3 caching is a good option for small and intermediate build artifacts that are more expensive to build than to download. Local caching is a good option for large intermediate build artifacts because the cache is immediately available on the build host. + +### S3 Caching + +With S3 caching, the cache is stored in an S3 bucket which is available from multiple hosts. + +```typescript +new codebuild.Project(this, 'Project', { + source: new codebuild.CodePipelineSource(), + cache: codebuild.Cache.bucket(new Bucket(this, 'Bucket')) +}); +``` + +### Local Caching + +With local caching, the cache is stored on the codebuild instance itself. CodeBuild cannot guarantee a reuse of instance. For example, when a build starts and caches files locally, if two subsequent builds start at the same time afterwards only one of those builds would get the cache. Three different cache modes are supported: + +* `LocalCacheMode.Source` caches Git metadata for primary and secondary sources. +* `LocalCacheMode.DockerLayer` caches existing Docker layers. +* `LocalCacheMode.Custom` caches directories you specify in the buildspec file. + +```typescript +new codebuild.Project(this, 'Project', { + source: new codebuild.CodePipelineSource(), + cache: codebuild.Cache.local(LocalCacheMode.DockerLayer, LocalCacheMode.Custom) +}); +``` + ## Environment By default, projects use a small instance with an Ubuntu 18.04 image. You diff --git a/packages/@aws-cdk/aws-codebuild/lib/cache.ts b/packages/@aws-cdk/aws-codebuild/lib/cache.ts new file mode 100644 index 0000000000000..aac93ec29ebe9 --- /dev/null +++ b/packages/@aws-cdk/aws-codebuild/lib/cache.ts @@ -0,0 +1,83 @@ +import { IBucket } from "@aws-cdk/aws-s3"; +import { Aws, Fn } from "@aws-cdk/cdk"; +import { CfnProject } from "./codebuild.generated"; +import { IProject } from "./project"; + +export interface BucketCacheOptions { + /** + * The prefix to use to store the cache in the bucket + */ + readonly prefix?: string; +} + +/** + * Local cache modes to enable for the CodeBuild Project + */ +export enum LocalCacheMode { + /** + * Caches Git metadata for primary and secondary sources + */ + Source = 'LOCAL_SOURCE_CACHE', + + /** + * Caches existing Docker layers + */ + DockerLayer = 'LOCAL_DOCKER_LAYER_CACHE', + + /** + * Caches directories you specify in the buildspec file + */ + Custom = 'LOCAL_CUSTOM_CACHE', +} + +/** + * Cache options for CodeBuild Project. + * A cache can store reusable pieces of your build environment and use them across multiple builds. + * @see https://docs.aws.amazon.com/codebuild/latest/userguide/build-caching.html + */ +export abstract class Cache { + public static none(): Cache { + return { _toCloudFormation: () => undefined, _bind: () => { return; } }; + } + + /** + * Create a local caching strategy. + * @param modes the mode(s) to enable for local caching + */ + public static local(...modes: LocalCacheMode[]): Cache { + return { + _toCloudFormation: () => ({ + type: 'LOCAL', + modes + }), + _bind: () => { return; } + }; + } + + /** + * Create an S3 caching strategy. + * @param bucket the S3 bucket to use for caching + * @param options additional options to pass to the S3 caching + */ + public static bucket(bucket: IBucket, options?: BucketCacheOptions): Cache { + return { + _toCloudFormation: () => ({ + type: 'S3', + location: Fn.join('/', [bucket.bucketName, options && options.prefix || Aws.noValue]) + }), + _bind: (project) => { + bucket.grantReadWrite(project); + } + }; + } + + /** + * @internal + */ + public abstract _toCloudFormation(): CfnProject.ProjectCacheProperty | undefined; + + /** + * @internal + */ + public abstract _bind(project: IProject): void; +} diff --git a/packages/@aws-cdk/aws-codebuild/lib/index.ts b/packages/@aws-cdk/aws-codebuild/lib/index.ts index 7fbe814d26429..2cb640b7cc9a0 100644 --- a/packages/@aws-cdk/aws-codebuild/lib/index.ts +++ b/packages/@aws-cdk/aws-codebuild/lib/index.ts @@ -2,6 +2,7 @@ export * from './pipeline-project'; export * from './project'; export * from './source'; export * from './artifacts'; +export * from './cache'; // AWS::CodeBuild CloudFormation Resources: export * from './codebuild.generated'; diff --git a/packages/@aws-cdk/aws-codebuild/lib/project.ts b/packages/@aws-cdk/aws-codebuild/lib/project.ts index 1707887f6b995..54e967296d2dd 100644 --- a/packages/@aws-cdk/aws-codebuild/lib/project.ts +++ b/packages/@aws-cdk/aws-codebuild/lib/project.ts @@ -6,9 +6,9 @@ import ecr = require('@aws-cdk/aws-ecr'); import events = require('@aws-cdk/aws-events'); import iam = require('@aws-cdk/aws-iam'); import kms = require('@aws-cdk/aws-kms'); -import s3 = require('@aws-cdk/aws-s3'); -import { Aws, CfnOutput, Construct, Fn, IResource, Resource, Token } from '@aws-cdk/cdk'; +import { Aws, CfnOutput, Construct, IResource, Resource, Token } from '@aws-cdk/cdk'; import { BuildArtifacts, CodePipelineBuildArtifacts, NoBuildArtifacts } from './artifacts'; +import { Cache } from './cache'; import { CfnProject } from './codebuild.generated'; import { BuildSource, NoSource, SourceType } from './source'; @@ -392,21 +392,16 @@ export interface CommonProjectProps { readonly role?: iam.IRole; /** - * Encryption key to use to read and write artifacts + * Encryption key to use to read and write artifacts. * If not specified, a role will be created. */ readonly encryptionKey?: kms.IEncryptionKey; /** - * Bucket to store cached source artifacts - * If not specified, source artifacts will not be cached. + * Caching strategy to use. + * @default Cache.none */ - readonly cacheBucket?: s3.IBucket; - - /** - * Subdirectory to store cached artifacts - */ - readonly cacheDir?: string; + readonly cache?: Cache; /** * Build environment to use for the build. @@ -618,17 +613,6 @@ export class Project extends ProjectBase { }); this.grantPrincipal = this.role; - let cache: CfnProject.ProjectCacheProperty | undefined; - if (props.cacheBucket) { - const cacheDir = props.cacheDir != null ? props.cacheDir : Aws.noValue; - cache = { - type: 'S3', - location: Fn.join('/', [props.cacheBucket.bucketName, cacheDir]), - }; - - props.cacheBucket.grantReadWrite(this.role); - } - this.buildImage = (props.environment && props.environment.buildImage) || LinuxBuildImage.STANDARD_1_0; // let source "bind" to the project. this usually involves granting permissions @@ -639,6 +623,11 @@ export class Project extends ProjectBase { const artifacts = this.parseArtifacts(props); artifacts._bind(this); + const cache = props.cache || Cache.none(); + + // give the caching strategy the option to grant permissions to any required resources + cache._bind(this); + // Inject download commands for asset if requested const environmentVariables = props.environmentVariables || {}; const buildSpec = props.buildSpec || {}; @@ -696,7 +685,7 @@ export class Project extends ProjectBase { environment: this.renderEnvironment(props.environment, environmentVariables), encryptionKey: props.encryptionKey && props.encryptionKey.keyArn, badgeEnabled: props.badge, - cache, + cache: cache._toCloudFormation(), name: props.projectName, timeoutInMinutes: props.timeout, secondarySources: new Token(() => this.renderSecondarySources()), diff --git a/packages/@aws-cdk/aws-codebuild/test/integ.caching.ts b/packages/@aws-cdk/aws-codebuild/test/integ.caching.ts index fc1a34c0b1a8d..a381a88f36eb8 100644 --- a/packages/@aws-cdk/aws-codebuild/test/integ.caching.ts +++ b/packages/@aws-cdk/aws-codebuild/test/integ.caching.ts @@ -2,6 +2,7 @@ import s3 = require('@aws-cdk/aws-s3'); import cdk = require('@aws-cdk/cdk'); import codebuild = require('../lib'); +import { Cache } from '../lib/cache'; const app = new cdk.App(); @@ -12,7 +13,7 @@ const bucket = new s3.Bucket(stack, 'CacheBucket', { }); new codebuild.Project(stack, 'MyProject', { - cacheBucket: bucket, + cache: Cache.bucket(bucket), buildSpec: { build: { commands: ['echo Hello'] diff --git a/packages/@aws-cdk/aws-codebuild/test/test.project.ts b/packages/@aws-cdk/aws-codebuild/test/test.project.ts index 520d6aaf0a192..27715110cce8f 100644 --- a/packages/@aws-cdk/aws-codebuild/test/test.project.ts +++ b/packages/@aws-cdk/aws-codebuild/test/test.project.ts @@ -1,8 +1,10 @@ -import { expect, haveResource, haveResourceLike } from '@aws-cdk/assert'; +import { expect, haveResource, haveResourceLike, not } from '@aws-cdk/assert'; import assets = require('@aws-cdk/assets'); +import { Bucket } from '@aws-cdk/aws-s3'; import cdk = require('@aws-cdk/cdk'); import { Test } from 'nodeunit'; import codebuild = require('../lib'); +import { Cache, LocalCacheMode } from '../lib/cache'; // tslint:disable:object-literal-key-quotes @@ -161,4 +163,79 @@ export = { test.done(); }, + + 'project with s3 cache bucket'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new codebuild.Project(stack, 'Project', { + source: new codebuild.CodePipelineSource(), + cache: Cache.bucket(new Bucket(stack, 'Bucket'), { + prefix: "cache-prefix" + }) + }); + + // THEN + expect(stack).to(haveResourceLike('AWS::CodeBuild::Project', { + Cache: { + Type: "S3", + Location: { + "Fn::Join": [ + "/", + [ + { + "Ref": "Bucket83908E77" + }, + "cache-prefix" + ] + ] + } + }, + })); + + test.done(); + }, + + 'project with local cache modes'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new codebuild.Project(stack, 'Project', { + source: new codebuild.CodePipelineSource(), + cache: Cache.local(LocalCacheMode.Custom, LocalCacheMode.DockerLayer, LocalCacheMode.Source) + }); + + // THEN + expect(stack).to(haveResourceLike('AWS::CodeBuild::Project', { + Cache: { + Type: "LOCAL", + Modes: [ + "LOCAL_CUSTOM_CACHE", + "LOCAL_DOCKER_LAYER_CACHE", + "LOCAL_SOURCE_CACHE" + ] + }, + })); + + test.done(); + }, + + 'project by default has no cache modes'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new codebuild.Project(stack, 'Project', { + source: new codebuild.CodePipelineSource() + }); + + // THEN + expect(stack).to(not(haveResourceLike('AWS::CodeBuild::Project', { + Cache: {} + }))); + + test.done(); + }, };