diff --git a/README.md b/README.md index 1f89893f..bd4bc229 100644 --- a/README.md +++ b/README.md @@ -403,6 +403,11 @@ source_path = [ "!vendor/colorful-.+.dist-info/.*", "!vendor/colorful/__pycache__/?.*", ] + }, { + path = "src/nodejs14.x-app1", + npm_requirements = true, + npm_tmp_dir = "/tmp/dir/location" + prefix_in_zip = "foo/bar1", }, { path = "src/python3.8-app3", commands = [ @@ -424,8 +429,9 @@ source_path = [ ] ``` -Few notes: +*Few notes:* +- If you specify a source path as a string that references a folder and the runtime is either python or nodejs, the build process will automatically build python and nodejs dependencies if `requirements.txt` or `package.json` file will be found in the source folder. If you want to customize this behavior, please use the object notation as explained below. - All arguments except `path` are optional. - `patterns` - List of Python regex filenames should satisfy. Default value is "include everything" which is equal to `patterns = [".*"]`. This can also be specified as multiline heredoc string (no comments allowed). Some examples of valid patterns: @@ -442,10 +448,12 @@ Few notes: !abc/def/hgk/.* # Filter out again in abc/def/hgk sub folder ``` -- `commands` - List of commands to run. If specified, this argument overrides `pip_requirements`. +- `commands` - List of commands to run. If specified, this argument overrides `pip_requirements` and `npm_requirements`. - `:zip [source] [destination]` is a special command which creates content of current working directory (first argument) and places it inside of path (second argument). - `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead. - `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds. +- `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead. +- `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds. - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive. ### Building in Docker diff --git a/examples/build-package/README.md b/examples/build-package/README.md index 7ee75c69..fc463a3d 100644 --- a/examples/build-package/README.md +++ b/examples/build-package/README.md @@ -38,11 +38,14 @@ Note that this example may create resources which cost money. Run `terraform des | [lambda\_layer\_pip\_requirements](#module\_lambda\_layer\_pip\_requirements) | ../.. | n/a | | [package\_dir](#module\_package\_dir) | ../../ | n/a | | [package\_dir\_pip\_dir](#module\_package\_dir\_pip\_dir) | ../../ | n/a | +| [package\_dir\_with\_npm\_install](#module\_package\_dir\_with\_npm\_install) | ../../ | n/a | +| [package\_dir\_without\_npm\_install](#module\_package\_dir\_without\_npm\_install) | ../../ | n/a | | [package\_dir\_without\_pip\_install](#module\_package\_dir\_without\_pip\_install) | ../../ | n/a | | [package\_file](#module\_package\_file) | ../../ | n/a | | [package\_file\_with\_pip\_requirements](#module\_package\_file\_with\_pip\_requirements) | ../../ | n/a | | [package\_with\_commands\_and\_patterns](#module\_package\_with\_commands\_and\_patterns) | ../../ | n/a | | [package\_with\_docker](#module\_package\_with\_docker) | ../../ | n/a | + [package\_with\_npm\_requirements\_in\_docker](#module\_package\_with\_npm\_requirements\_in\_docker) | ../../ | n/a | | [package\_with\_patterns](#module\_package\_with\_patterns) | ../../ | n/a | | [package\_with\_pip\_requirements\_in\_docker](#module\_package\_with\_pip\_requirements\_in\_docker) | ../../ | n/a | diff --git a/examples/build-package/main.tf b/examples/build-package/main.tf index 636ac8d8..6ab09b1a 100644 --- a/examples/build-package/main.tf +++ b/examples/build-package/main.tf @@ -223,6 +223,44 @@ module "package_with_docker" { docker_image = "lambci/lambda:build-python3.8" } +# Create zip-archive of a single directory where "npm install" will also be executed (default for nodejs runtime) +module "package_dir_with_npm_install" { + source = "../../" + + create_function = false + + runtime = "nodejs14.x" + source_path = "${path.module}/../fixtures/nodejs14.x-app1" +} + +# Create zip-archive of a single directory without running "npm install" (which is the default for nodejs runtime) +module "package_dir_without_npm_install" { + source = "../../" + + create_function = false + + runtime = "nodejs14.x" + source_path = [ + { + path = "${path.module}/../fixtures/nodejs14.x-app1" + npm_requirements = false + # npm_requirements = true # Will run "npm install" with default requirements.txt + } + ] +} + +# Create zip-archive of a single directory where "npm install" will also be executed using docker +module "package_with_npm_requirements_in_docker" { + source = "../../" + + create_function = false + + runtime = "nodejs14.x" + source_path = "${path.module}/../fixtures/nodejs14.x-app1" + build_in_docker = true + hash_extra = "something-unique-to-not-conflict-with-module.package_dir_with_npm_install" +} + ################################ # Build package in Docker and # use it to deploy Lambda Layer diff --git a/examples/fixtures/nodejs14.x-app1/index.js b/examples/fixtures/nodejs14.x-app1/index.js new file mode 100644 index 00000000..97968e4a --- /dev/null +++ b/examples/fixtures/nodejs14.x-app1/index.js @@ -0,0 +1,16 @@ +'use strict'; + +module.exports.hello = async (event) => { + console.log(event); + return { + statusCode: 200, + body: JSON.stringify( + { + message: `Go Serverless.tf! Your Nodejs function executed successfully!`, + input: event, + }, + null, + 2 + ), + }; +}; diff --git a/examples/fixtures/nodejs14.x-app1/package.json b/examples/fixtures/nodejs14.x-app1/package.json new file mode 100644 index 00000000..89c23f36 --- /dev/null +++ b/examples/fixtures/nodejs14.x-app1/package.json @@ -0,0 +1,8 @@ +{ + "name": "nodejs14.x-app1", + "version": "1.0.0", + "main": "index.js", + "dependencies": { + "requests": "^0.3.0" + } +} diff --git a/package.py b/package.py index b9fccf28..7bd8dd2d 100644 --- a/package.py +++ b/package.py @@ -660,6 +660,18 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): step('pip', runtime, requirements, prefix, tmp_dir) hash(requirements) + def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None): + requirements = path + if os.path.isdir(path): + requirements = os.path.join(path, 'package.json') + if not os.path.isfile(requirements): + if required: + raise RuntimeError( + 'File not found: {}'.format(requirements)) + else: + step('npm', runtime, requirements, prefix, tmp_dir) + hash(requirements) + def commands_step(path, commands): if not commands: return @@ -717,6 +729,9 @@ def commands_step(path, commands): if runtime.startswith('python'): pip_requirements_step( os.path.join(path, 'requirements.txt')) + elif runtime.startswith('nodejs'): + npm_requirements_step( + os.path.join(path, 'package.json')) step('zip', path, None) hash(path) @@ -731,6 +746,7 @@ def commands_step(path, commands): else: prefix = claim.get('prefix_in_zip') pip_requirements = claim.get('pip_requirements') + npm_requirements = claim.get('npm_package_json') runtime = claim.get('runtime', query.runtime) if pip_requirements and runtime.startswith('python'): @@ -740,6 +756,13 @@ def commands_step(path, commands): pip_requirements_step(pip_requirements, prefix, required=True, tmp_dir=claim.get('pip_tmp_dir')) + if npm_requirements and runtime.startswith('nodejs'): + if isinstance(npm_requirements, bool) and path: + npm_requirements_step(path, prefix, required=True, tmp_dir=claim.get('npm_tmp_dir')) + else: + npm_requirements_step(npm_requirements, prefix, + required=True, tmp_dir=claim.get('npm_tmp_dir')) + if path: step('zip', path, prefix) if patterns: @@ -793,6 +816,16 @@ def execute(self, build_plan, zip_stream, query): else: # XXX: timestamp=0 - what actually do with it? zs.write_dirs(rd, prefix=prefix, timestamp=0) + elif cmd == 'npm': + runtime, npm_requirements, prefix, tmp_dir = action[1:] + with install_npm_requirements(query, npm_requirements, tmp_dir) as rd: + if rd: + if pf: + self._zip_write_with_filter(zs, pf, rd, prefix, + timestamp=0) + else: + # XXX: timestamp=0 - what actually do with it? + zs.write_dirs(rd, prefix=prefix, timestamp=0) elif cmd == 'sh': r, w = os.pipe() side_ch = os.fdopen(r) @@ -934,6 +967,89 @@ def install_pip_requirements(query, requirements_file, tmp_dir): yield temp_dir +@contextmanager +def install_npm_requirements(query, requirements_file, tmp_dir): + # TODO: + # 1. Emit files instead of temp_dir + + if not os.path.exists(requirements_file): + yield + return + + runtime = query.runtime + artifacts_dir = query.artifacts_dir + temp_dir = query.temp_dir + docker = query.docker + docker_image_tag_id = None + + if docker: + docker_file = docker.docker_file + docker_image = docker.docker_image + docker_build_root = docker.docker_build_root + + if docker_image: + ok = False + while True: + output = check_output(docker_image_id_command(docker_image)) + if output: + docker_image_tag_id = output.decode().strip() + log.debug("DOCKER TAG ID: %s -> %s", + docker_image, docker_image_tag_id) + ok = True + if ok: + break + docker_cmd = docker_build_command( + build_root=docker_build_root, + docker_file=docker_file, + tag=docker_image, + ) + check_call(docker_cmd) + ok = True + elif docker_file or docker_build_root: + raise ValueError('docker_image must be specified ' + 'for a custom image future references') + + log.info('Installing npm requirements: %s', requirements_file) + with tempdir(tmp_dir) as temp_dir: + requirements_filename = os.path.basename(requirements_file) + target_file = os.path.join(temp_dir, requirements_filename) + shutil.copyfile(requirements_file, target_file) + + subproc_env = None + if not docker and OSX: + subproc_env = os.environ.copy() + + # Install dependencies into the temporary directory. + with cd(temp_dir): + npm_command = ['npm', 'install'] + if docker: + with_ssh_agent = docker.with_ssh_agent + chown_mask = '{}:{}'.format(os.getuid(), os.getgid()) + shell_command = [shlex_join(npm_command), '&&', + shlex_join(['chown', '-R', + chown_mask, '.'])] + shell_command = [' '.join(shell_command)] + check_call(docker_run_command( + '.', shell_command, runtime, + image=docker_image_tag_id, + shell=True, ssh_agent=with_ssh_agent + )) + else: + cmd_log.info(shlex_join(npm_command)) + log_handler and log_handler.flush() + try: + check_call(npm_command, env=subproc_env) + except FileNotFoundError as e: + raise RuntimeError( + "Nodejs interpreter version equal " + "to defined lambda runtime ({}) should be " + "available in system PATH".format(runtime) + ) from e + + os.remove(target_file) + yield temp_dir + + def docker_image_id_command(tag): """""" docker_cmd = ['docker', 'images', '--format={{.ID}}', tag] @@ -1011,7 +1127,7 @@ def docker_run_command(build_root, command, runtime, ]) if not image: - image = 'lambci/lambda:build-{}'.format(runtime) + image = 'public.ecr.aws/sam/build-{}'.format(runtime) docker_cmd.append(image) @@ -1128,7 +1244,7 @@ def prepare_command(args): def build_command(args): """ Builds a zip file from the source_dir or source_file. - Installs dependencies with pip automatically. + Installs dependencies with pip or npm automatically. """ log = logging.getLogger('build')