feat(lambda-python): bundle dependencies in a lambda layer

packages/@aws-cdk/aws-lambda-python/README.md

@@ -32,11 +32,38 @@ All other properties of `lambda.Function` are supported, see also the [AWS Lambd
 
 ### Module Dependencies
 
-If `requirements.txt` exists at the entry path, the construct will handle installing
+If `requirements.txt` or `Pipfile` exists at the entry path, the construct will handle installing
 all required modules in a [Lambda compatible Docker container](https://hub.docker.com/r/amazon/aws-sam-cli-build-image-python3.7)
 according to the `runtime`.
+
+**Lambda with a requirements.txt**
 ```
 .
 ├── lambda_function.py # exports a function named 'handler'
 ├── requirements.txt # has to be present at the entry path
 ```
+
+**Lambda with a Pipfile**
+```
+.
+├── lambda_function.py # exports a function named 'handler'
+├── Pipfile # has to be present at the entry path
+├── Pipfile.lock # your lock file
+```
+
+**Lambda Layer Support**
+
+You may create a python-based lambda layer with `PythonLayerVersion`. If `PythonLayerVersion` detects a `requirements.txt`
+or `Pipfile` at the entry path, then `PythonLayerVersion` will include the dependencies inline with your code in the
+layer.
+
+```ts
+new lambda.PythonFunction(this, 'MyFunction', {
+  entry: '/path/to/my/function',
+  layers: [
+    new lambda.PythonLayerVersion(this, 'MyLayer', {
+      entry: '/path/to/my/layer', // point this to your library's directory
+    }),
+  ],
+});
+```

packages/@aws-cdk/aws-lambda-python/lib/Dockerfile.dependencies

@@ -0,0 +1,18 @@
+# The correct AWS SAM build image based on the runtime of the function will be
+# passed as build arg. The default allows to do `docker build .` when testing.
+ARG IMAGE=amazon/aws-sam-cli-build-image-python3.7
+FROM $IMAGE
+
+# Ensure rsync is installed
+RUN yum -q list installed rsync &>/dev/null || yum install -y rsync
+
+# Install pipenv so we can create a requirements.txt if we detect pipfile
+RUN pip install pipenv
+
+# Install the dependencies in a cacheable layer
+WORKDIR /var/dependencies
+COPY Pipfile* requirements.tx[t] ./
+RUN [ -f 'Pipfile' ] && pipenv lock -r >requirements.txt; \
+    [ -f 'requirements.txt' ] && pip install -r requirements.txt -t .;
+
+CMD [ "python" ]

packages/@aws-cdk/aws-lambda-python/lib/bundling.ts

@@ -3,6 +3,16 @@ import * as path from 'path';
 import * as lambda from '@aws-cdk/aws-lambda';
 import * as cdk from '@aws-cdk/core';
 
+/**
+ * Dependency files to exclude from the asset hash.
+ */
+export const DEPENDENCY_EXCLUDES = ['*.pyc'];
+
+/**
+ * The location in the image that the bundler image caches dependencies.
+ */
+export const BUNDLER_DEPENDENCIES_CACHE = '/var/dependencies';
+
 /**
  * Options for bundling
  */
@@ -16,39 +26,65 @@ export interface BundlingOptions {
    * The runtime of the lambda function
    */
   readonly runtime: lambda.Runtime;
+
+  /**
+   * Output path suffix ('python' for a layer, '.' otherwise)
+   */
+  readonly outputPathSuffix: string;
 }
 
 /**
  * Produce bundled Lambda asset code
  */
 export function bundle(options: BundlingOptions): lambda.AssetCode {
-  // Bundling image derived from runtime bundling image (AWS SAM docker image)
-  const image = cdk.BundlingDockerImage.fromAsset(__dirname, {
-    buildArgs: {
-      IMAGE: options.runtime.bundlingDockerImage.image,
-    },
-  });
-
-  let installer = options.runtime === lambda.Runtime.PYTHON_2_7 ? Installer.PIP : Installer.PIP3;
+  const { entry, runtime, outputPathSuffix } = options;
 
-  let hasRequirements = fs.existsSync(path.join(options.entry, 'requirements.txt'));
+  const hasDeps = hasDependencies(entry);
 
-  let depsCommand = chain([
-    hasRequirements ? `${installer} install -r requirements.txt -t ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}` : '',
-    `rsync -r . ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}`,
+  const depsCommand = chain([
+    hasDeps ? `rsync -r ${BUNDLER_DEPENDENCIES_CACHE}/. ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}/${outputPathSuffix}` : '',
+    `rsync -r . ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}/${outputPathSuffix}`,
   ]);
 
-  return lambda.Code.fromAsset(options.entry, {
+  // Determine which dockerfile to use. When dependencies are present, we use a
+  // Dockerfile that can create a cacheable layer. We can't use this Dockerfile
+  // if there aren't dependencies or the Dockerfile will complain about missing
+  // sources.
+  const dockerfile = hasDeps
+    ? 'Dockerfile.dependencies'
+    : 'Dockerfile';
+
+  const image = cdk.BundlingDockerImage.fromAsset(entry, {
+    buildArgs: {
+      IMAGE: runtime.bundlingDockerImage.image,
+    },
+    file: path.join(__dirname, dockerfile),
+  });
+
+  return lambda.Code.fromAsset(entry, {
+    assetHashType: cdk.AssetHashType.BUNDLE,
+    exclude: DEPENDENCY_EXCLUDES,
     bundling: {
       image,
       command: ['bash', '-c', depsCommand],
     },
   });
 }
 
-enum Installer {
-  PIP = 'pip',
-  PIP3 = 'pip3',
+/**
+ * Checks to see if the `entry` directory contains a type of dependency that
+ * we know how to install.
+ */
+export function hasDependencies(entry: string): boolean {
+  if (fs.existsSync(path.join(entry, 'Pipfile'))) {
+    return true;
+  }
+
+  if (fs.existsSync(path.join(entry, 'requirements.txt'))) {
+    return true;
+  }
+
+  return false;
 }
 
 function chain(commands: string[]): string {

packages/@aws-cdk/aws-lambda-python/lib/function.ts

@@ -64,9 +64,9 @@ export class PythonFunction extends lambda.Function {
       ...props,
       runtime,
       code: bundle({
-        ...props,
-        entry,
         runtime,
+        entry,
+        outputPathSuffix: '.',
       }),
       handler: `${index.slice(0, -3)}.${handler}`,
     });

packages/@aws-cdk/aws-lambda-python/lib/index.ts

@@ -1 +1,2 @@
 export * from './function';
+export * from './layer';

packages/@aws-cdk/aws-lambda-python/lib/layer.ts

@@ -0,0 +1,54 @@
+import * as path from 'path';
+import * as lambda from '@aws-cdk/aws-lambda';
+import * as cdk from '@aws-cdk/core';
+import { bundle } from './bundling';
+
+/**
+ * Properties for PythonLayerVersion
+ */
+export interface PythonLayerVersionProps extends lambda.LayerVersionOptions {
+  /**
+   * The path to the root directory of the lambda layer.
+   */
+  readonly entry: string;
+
+  /**
+   * The runtimes compatible with the python layer.
+   *
+   * @default - All runtimes are supported.
+   */
+  readonly compatibleRuntimes?: lambda.Runtime[];
+}
+
+/**
+ * A lambda layer version.
+ *
+ * @experimental
+ */
+export class PythonLayerVersion extends lambda.LayerVersion {
+  constructor(scope: cdk.Construct, id: string, props: PythonLayerVersionProps) {
+    const compatibleRuntimes = props.compatibleRuntimes ?? [lambda.Runtime.PYTHON_3_7];
+
+    // Ensure that all compatible runtimes are python
+    for (const runtime of compatibleRuntimes) {
+      if (runtime && runtime.family !== lambda.RuntimeFamily.PYTHON) {
+        throw new Error('Only `PYTHON` runtimes are supported.');
+      }
+    }
+
+    // Entry and defaults
+    const entry = path.resolve(props.entry);
+    // Pick the first compatibleRuntime to use for bundling or PYTHON_3_7
+    const runtime = compatibleRuntimes[0] ?? lambda.Runtime.PYTHON_3_7;
+
+    super(scope, id, {
+      ...props,
+      compatibleRuntimes,
+      code: bundle({
+        entry,
+        runtime,
+        outputPathSuffix: 'python',
+      }),
+    });
+  }
+}