diff --git a/samcli/commands/build/build_context.py b/samcli/commands/build/build_context.py index 163ccdee64..39bae9861f 100644 --- a/samcli/commands/build/build_context.py +++ b/samcli/commands/build/build_context.py @@ -9,6 +9,7 @@ import click +from samcli.commands.build.utils import prompt_user_to_enable_mount_with_write_if_needed, MountMode from samcli.lib.build.bundler import EsbuildBundlerManager from samcli.lib.providers.sam_api_provider import SamApiProvider from samcli.lib.telemetry.event import EventTracker @@ -35,7 +36,6 @@ ApplicationBuilder, BuildError, UnsupportedBuilderLibraryVersionError, - ContainerBuildNotSupported, ApplicationBuildResult, ) from samcli.commands._utils.constants import DEFAULT_BUILD_DIR @@ -78,6 +78,7 @@ def __init__( locate_layer_nested: bool = False, hook_name: Optional[str] = None, build_in_source: Optional[bool] = None, + mount_with: str = MountMode.READ.value, ) -> None: """ Initialize the class @@ -133,6 +134,8 @@ def __init__( Name of the hook package build_in_source: Optional[bool] Set to True to build in the source directory. + mount_with: + Mount mode of source code directory when building inside container, READ ONLY by default """ self._resource_identifier = resource_identifier @@ -172,6 +175,7 @@ def __init__( self._locate_layer_nested = locate_layer_nested self._hook_name = hook_name self._build_in_source = build_in_source + self._mount_with = MountMode(mount_with) def __enter__(self) -> "BuildContext": self.set_up() @@ -232,6 +236,19 @@ def run(self): self._stacks = self._handle_build_pre_processing() + # boolean value indicates if mount with write or not, defaults to READ ONLY + mount_with_write = False + if self._use_container: + if self._mount_with == MountMode.WRITE: + mount_with_write = True + else: + # if self._mount_with is NOT WRITE + # check the need of mounting with write permissions and prompt user to enable it if needed + mount_with_write = prompt_user_to_enable_mount_with_write_if_needed( + self.get_resources_to_build(), + self.base_dir, + ) + try: builder = ApplicationBuilder( self.get_resources_to_build(), @@ -249,6 +266,7 @@ def run(self): build_images=self._build_images, combine_dependencies=not self._create_auto_dependency_layer, build_in_source=self._build_in_source, + mount_with_write=mount_with_write, ) except FunctionNotFound as ex: raise UserException(str(ex), wrapped_from=ex.__class__.__name__) from ex @@ -293,7 +311,6 @@ def run(self): BuildError, BuildInsideContainerError, UnsupportedBuilderLibraryVersionError, - ContainerBuildNotSupported, InvalidBuildGraphException, ) as ex: click.secho("\nBuild Failed", fg="red") diff --git a/samcli/commands/build/command.py b/samcli/commands/build/command.py index 9518280e7f..d5fdeace17 100644 --- a/samcli/commands/build/command.py +++ b/samcli/commands/build/command.py @@ -28,6 +28,7 @@ from samcli.cli.cli_config_file import configuration_option, TomlProvider from samcli.lib.utils.version_checker import check_newer_version from samcli.commands.build.click_container import ContainerOptions +from samcli.commands.build.utils import MountMode LOG = logging.getLogger(__name__) @@ -136,6 +137,16 @@ help="Enabled parallel builds. Use this flag to build your AWS SAM template's functions and layers in parallel. " "By default the functions and layers are built in sequence", ) +@click.option( + "--mount-with", + "-mw", + type=click.Choice(MountMode.values(), case_sensitive=False), + default=MountMode.READ.value, + help="Optional. Specify mount mode for building functions/layers inside container. " + "If it is mounted with write permissions, some files in source code directory may " + "be changed/added by the build process. By default the source code directory is read only.", + cls=ContainerOptions, +) @build_dir_option @cache_dir_option @base_dir_option @@ -175,6 +186,7 @@ def cli( config_env: str, hook_name: Optional[str], skip_prepare_infra: bool, + mount_with, ) -> None: """ `sam build` command entry point @@ -205,6 +217,7 @@ def cli( exclude, hook_name, None, # TODO: replace with build_in_source once it's added as a click option + mount_with, ) # pragma: no cover @@ -230,6 +243,7 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements exclude: Optional[Tuple[str, ...]], hook_name: Optional[str], build_in_source: Optional[bool], + mount_with, ) -> None: """ Implementation of the ``cli`` method @@ -275,6 +289,7 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements aws_region=click_ctx.region, hook_name=hook_name, build_in_source=build_in_source, + mount_with=mount_with, ) as ctx: ctx.run() diff --git a/samcli/commands/build/utils.py b/samcli/commands/build/utils.py new file mode 100644 index 0000000000..25d2980b0b --- /dev/null +++ b/samcli/commands/build/utils.py @@ -0,0 +1,110 @@ +""" +Utilities for sam build command +""" +import pathlib +from enum import Enum +from typing import List + +import click + +from samcli.lib.build.workflow_config import CONFIG, get_workflow_config +from samcli.lib.providers.provider import ResourcesToBuildCollector + + +class MountMode(Enum): + """ + Enums that represent mount mode used when build lambda functions/layers inside container + """ + + READ = "READ" + WRITE = "WRITE" + + @classmethod + def values(cls) -> List[str]: + """ + A getter to retrieve the accepted value list for mount mode + + Returns: List[str] + The accepted mount mode list + """ + return [e.value for e in cls] + + +def prompt_user_to_enable_mount_with_write_if_needed( + resources_to_build: ResourcesToBuildCollector, + base_dir: str, +) -> bool: + """ + First check if mounting with write permissions is needed for building inside container or not. If it is needed, then + prompt user to choose if enables mounting with write permissions or not. + + Parameters + ---------- + resources_to_build: + Resource to build inside container + + base_dir : str + Path to the base directory + + Returns + ------- + bool + True, if user enabled mounting with write permissions. + """ + + for function in resources_to_build.functions: + code_uri = function.codeuri + if not code_uri: + continue + runtime = function.runtime + code_dir = str(pathlib.Path(base_dir, code_uri).resolve()) + # get specified_workflow if metadata exists + metadata = function.metadata + specified_workflow = metadata.get("BuildMethod", None) if metadata else None + config = get_workflow_config(runtime, code_dir, base_dir, specified_workflow) + # at least one function needs mount with write, return with prompting + if not config.must_mount_with_write_in_container: + continue + return prompt(config, code_dir) + + for layer in resources_to_build.layers: + code_uri = layer.codeuri + if not code_uri: + continue + code_dir = str(pathlib.Path(base_dir, code_uri).resolve()) + specified_workflow = layer.build_method + config = get_workflow_config(None, code_dir, base_dir, specified_workflow) + # at least one layer needs mount with write, return with prompting + if not config.must_mount_with_write_in_container: + continue + return prompt(config, code_dir) + + return False + + +def prompt(config: CONFIG, source_dir: str) -> bool: + """ + Prompt user to choose if enables mounting with write permissions or not when building lambda functions/layers + + Parameters + ---------- + config: namedtuple(Capability) + Config specifying the particular build workflow + + source_dir : str + Path to the function source code + + Returns + ------- + bool + True, if user enabled mounting with write permissions. + """ + if click.confirm( + f"\nBuilding functions with {config.language} inside containers needs " + f"mounting with write permissions to the source code directory {source_dir}. " + f"Some files in this directory may be changed or added by the build process. " + f"Pass `--mount-with WRITE` to `sam build` CLI to avoid this confirmation. " + f"\nWould you like to enable mounting with write permissions? " + ): + return True + return False diff --git a/samcli/lib/build/app_builder.py b/samcli/lib/build/app_builder.py index a0ba880d4b..8949f66e2b 100644 --- a/samcli/lib/build/app_builder.py +++ b/samcli/lib/build/app_builder.py @@ -7,7 +7,6 @@ import logging import pathlib from typing import List, Optional, Dict, cast, NamedTuple - import docker import docker.errors from aws_lambda_builders import ( @@ -15,7 +14,6 @@ ) from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import LambdaBuilderError - from samcli.lib.build.build_graph import FunctionBuildDefinition, LayerBuildDefinition, BuildGraph from samcli.lib.build.build_strategy import ( DefaultBuildStrategy, @@ -51,14 +49,12 @@ DockerBuildFailed, BuildError, BuildInsideContainerError, - ContainerBuildNotSupported, UnsupportedBuilderLibraryVersionError, ) - from samcli.lib.build.workflow_config import ( get_workflow_config, + supports_specified_workflow, get_layer_subfolder, - supports_build_in_container, CONFIG, UnsupportedRuntimeException, ) @@ -101,6 +97,7 @@ def __init__( build_images: Optional[Dict] = None, combine_dependencies: bool = True, build_in_source: Optional[bool] = None, + mount_with_write: bool = False, ) -> None: """ Initialize the class @@ -144,6 +141,8 @@ def __init__( dependencies or not. build_in_source: Optional[bool] Set to True to build in the source directory. + mount_with_write: bool + Mount source code directory with write permissions when building inside container. """ self._resources_to_build = resources_to_build self._build_dir = build_dir @@ -166,6 +165,7 @@ def __init__( self._build_images = build_images or {} self._combine_dependencies = combine_dependencies self._build_in_source = build_in_source + self._mount_with_write = mount_with_write def build(self) -> ApplicationBuildResult: """ @@ -544,6 +544,8 @@ def _build_layer( build_runtime = compatible_runtimes[0] global_image = self._build_images.get(None) image = self._build_images.get(layer_name, global_image) + # pass to container only when specified workflow is supported to overwrite runtime to get image + supported_specified_workflow = supports_specified_workflow(specified_workflow) self._build_function_on_container( config, code_dir, @@ -555,6 +557,7 @@ def _build_layer( container_env_vars, image, is_building_layer=True, + specified_workflow=specified_workflow if supported_specified_workflow else None, ) else: self._build_function_in_process( @@ -644,11 +647,9 @@ def _build_function( # pylint: disable=R1710 # Create the arguments to pass to the builder # Code is always relative to the given base directory. code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve()) - # Determine if there was a build workflow that was specified directly in the template. - specified_build_workflow = metadata.get("BuildMethod", None) if metadata else None - - config = get_workflow_config(runtime, code_dir, self._base_dir, specified_workflow=specified_build_workflow) + specified_workflow = metadata.get("BuildMethod", None) if metadata else None + config = get_workflow_config(runtime, code_dir, self._base_dir, specified_workflow=specified_workflow) if config.language == "provided" and isinstance(metadata, dict) and metadata.get("ProjectRootDirectory"): code_dir = str(pathlib.Path(self._base_dir, metadata.get("ProjectRootDirectory", code_dir)).resolve()) @@ -682,7 +683,8 @@ def _build_function( # pylint: disable=R1710 # None represents the global build image for all functions/layers global_image = self._build_images.get(None) image = self._build_images.get(function_name, global_image) - + # pass to container only when specified workflow is supported to overwrite runtime to get image + supported_specified_workflow = supports_specified_workflow(specified_workflow) return self._build_function_on_container( config, code_dir, @@ -693,6 +695,7 @@ def _build_function( # pylint: disable=R1710 options, container_env_vars, image, + specified_workflow=specified_workflow if supported_specified_workflow else None, ) return self._build_function_in_process( @@ -883,6 +886,7 @@ def _build_function_on_container( container_env_vars: Optional[Dict] = None, build_image: Optional[str] = None, is_building_layer: bool = False, + specified_workflow: Optional[str] = None, ) -> str: # _build_function_on_container() is only called when self._container_manager if not None if not self._container_manager: @@ -893,10 +897,6 @@ def _build_function_on_container( "Docker is unreachable. Docker needs to be running to build inside a container." ) - container_build_supported, reason = supports_build_in_container(config) - if not container_build_supported: - raise ContainerBuildNotSupported(reason) - # If we are printing debug logs in SAM CLI, the builder library should also print debug logs log_level = LOG.getEffectiveLevel() @@ -911,6 +911,7 @@ def _build_function_on_container( manifest_path, runtime, architecture, + specified_workflow=specified_workflow, log_level=log_level, optimizations=None, options=options, @@ -920,6 +921,8 @@ def _build_function_on_container( image=build_image, is_building_layer=is_building_layer, build_in_source=self._build_in_source, + mount_with_write=self._mount_with_write, + build_dir=self._build_dir, ) try: @@ -930,7 +933,6 @@ def _build_function_on_container( raise UnsupportedBuilderLibraryVersionError( container.image, "{} executable not found in container".format(container.executable_name) ) from ex - # Container's output provides status of whether the build succeeded or failed # stdout contains the result of JSON-RPC call stdout_stream = io.BytesIO() diff --git a/samcli/lib/build/exceptions.py b/samcli/lib/build/exceptions.py index 321302c677..0bab416224 100644 --- a/samcli/lib/build/exceptions.py +++ b/samcli/lib/build/exceptions.py @@ -12,10 +12,6 @@ def __init__(self, container_name: str, error_msg: str) -> None: Exception.__init__(self, msg.format(container_name=container_name, error_msg=error_msg)) -class ContainerBuildNotSupported(Exception): - pass - - class BuildError(Exception): def __init__(self, wrapped_from: str, msg: str) -> None: self.wrapped_from = wrapped_from diff --git a/samcli/lib/build/workflow_config.py b/samcli/lib/build/workflow_config.py index 3756f58dc1..bdfd42be36 100644 --- a/samcli/lib/build/workflow_config.py +++ b/samcli/lib/build/workflow_config.py @@ -222,40 +222,25 @@ def get_workflow_config( ) from ex -def supports_build_in_container(config: CONFIG) -> Tuple[bool, Optional[str]]: +def supports_specified_workflow(specified_workflow: str) -> bool: """ - Given a workflow config, this method provides a boolean on whether the workflow can run within a container or not. + Given a specified workflow, returns whether it is supported in container builds, + can be used to overwrite runtime and get docker image or not Parameters ---------- - config namedtuple(Capability) - Config specifying the particular build workflow + specified_workflow + Workflow specified in the template Returns ------- - tuple(bool, str) - True, if this workflow can be built inside a container. False, along with a reason message if it cannot be. + bool + True, if this workflow is supported, can be used to overwrite runtime and get docker image """ - def _key(c: CONFIG) -> str: - return str(c.language) + str(c.dependency_manager) + str(c.application_framework) + supported_specified_workflow = ["dotnet7"] - # This information could have beeen bundled inside the Workflow Config object. But we this way because - # ultimately the workflow's implementation dictates whether it can run within a container or not. - # A "workflow config" is like a primary key to identify the workflow. So we use the config as a key in the - # map to identify which workflows can support building within a container. - - unsupported = { - _key(DOTNET_CLIPACKAGE_CONFIG): "We do not support building .NET Core Lambda functions within a container. " - "Try building without the container. Most .NET Core functions will build " - "successfully.", - } - - thiskey = _key(config) - if thiskey in unsupported: - return False, unsupported[thiskey] - - return True, None + return specified_workflow in supported_specified_workflow class BasicWorkflowSelector: diff --git a/samcli/lib/build/workflows.py b/samcli/lib/build/workflows.py index e8d969a15a..66d8529e4a 100644 --- a/samcli/lib/build/workflows.py +++ b/samcli/lib/build/workflows.py @@ -5,7 +5,14 @@ CONFIG = namedtuple( "Capability", - ["language", "dependency_manager", "application_framework", "manifest_name", "executable_search_paths"], + [ + "language", + "dependency_manager", + "application_framework", + "manifest_name", + "executable_search_paths", + "must_mount_with_write_in_container", + ], ) PYTHON_PIP_CONFIG = CONFIG( @@ -14,6 +21,7 @@ application_framework=None, manifest_name="requirements.txt", executable_search_paths=None, + must_mount_with_write_in_container=False, ) NODEJS_NPM_CONFIG = CONFIG( @@ -22,6 +30,7 @@ application_framework=None, manifest_name="package.json", executable_search_paths=None, + must_mount_with_write_in_container=False, ) RUBY_BUNDLER_CONFIG = CONFIG( @@ -30,6 +39,7 @@ application_framework=None, manifest_name="Gemfile", executable_search_paths=None, + must_mount_with_write_in_container=False, ) JAVA_GRADLE_CONFIG = CONFIG( @@ -38,6 +48,7 @@ application_framework=None, manifest_name="build.gradle", executable_search_paths=None, + must_mount_with_write_in_container=False, ) JAVA_KOTLIN_GRADLE_CONFIG = CONFIG( @@ -46,6 +57,7 @@ application_framework=None, manifest_name="build.gradle.kts", executable_search_paths=None, + must_mount_with_write_in_container=False, ) JAVA_MAVEN_CONFIG = CONFIG( @@ -54,14 +66,17 @@ application_framework=None, manifest_name="pom.xml", executable_search_paths=None, + must_mount_with_write_in_container=False, ) +# dotnet must mount with write for container builds because it outputs to source code directory by default DOTNET_CLIPACKAGE_CONFIG = CONFIG( language="dotnet", dependency_manager="cli-package", application_framework=None, manifest_name=".csproj", executable_search_paths=None, + must_mount_with_write_in_container=True, ) GO_MOD_CONFIG = CONFIG( @@ -70,6 +85,7 @@ application_framework=None, manifest_name="go.mod", executable_search_paths=None, + must_mount_with_write_in_container=False, ) PROVIDED_MAKE_CONFIG = CONFIG( @@ -78,6 +94,7 @@ application_framework=None, manifest_name="Makefile", executable_search_paths=None, + must_mount_with_write_in_container=False, ) NODEJS_NPM_ESBUILD_CONFIG = CONFIG( @@ -86,6 +103,7 @@ application_framework=None, manifest_name="package.json", executable_search_paths=None, + must_mount_with_write_in_container=False, ) RUST_CARGO_LAMBDA_CONFIG = CONFIG( @@ -94,6 +112,7 @@ application_framework=None, manifest_name="Cargo.toml", executable_search_paths=None, + must_mount_with_write_in_container=False, ) ALL_CONFIGS: List[CONFIG] = [ diff --git a/samcli/local/docker/container.py b/samcli/local/docker/container.py index c2ae95054d..f3020cc51e 100644 --- a/samcli/local/docker/container.py +++ b/samcli/local/docker/container.py @@ -3,10 +3,13 @@ """ import logging import os +import pathlib +import shutil import socket import tempfile import threading import time +from typing import Optional import docker import requests @@ -14,6 +17,7 @@ from samcli.lib.utils.retry import retry from samcli.lib.utils.tar import extract_tarfile +from samcli.local.docker.effective_user import ROOT_USER_ID, EffectiveUser from .exceptions import ContainerNotStartableException from .utils import NoFreePortsError, find_free_port, to_posix_path @@ -68,6 +72,8 @@ def __init__( additional_volumes=None, container_host="localhost", container_host_interface="127.0.0.1", + mount_with_write: bool = False, + host_tmp_dir: Optional[str] = None, ): """ Initializes the class with given configuration. This does not automatically create or run the container. @@ -86,6 +92,9 @@ def __init__( :param additional_volumes: Optional list of additional volumes :param string container_host: Optional. Host of locally emulated Lambda container :param string container_host_interface: Optional. Interface that Docker host binds ports to + :param bool mount_with_write: Optional. Mount source code directory with write permissions when + building on container + :param string host_tmp_dir: Optional. Temporary directory on the host when mounting with write permissions. """ self._image = image @@ -114,6 +123,8 @@ def __init__( self._container_host = container_host self._container_host_interface = container_host_interface + self._mount_with_write = mount_with_write + self._host_tmp_dir = host_tmp_dir try: self.rapid_port_host = find_free_port(start=self._start_port_range, end=self._end_port_range) @@ -135,15 +146,15 @@ def create(self): _volumes = {} if self._host_dir: - LOG.info("Mounting %s as %s:ro,delegated inside runtime container", self._host_dir, self._working_dir) + mount_mode = "rw,delegated" if self._mount_with_write else "ro,delegated" + LOG.info("Mounting %s as %s:%s, inside runtime container", self._host_dir, self._working_dir, mount_mode) _volumes = { self._host_dir: { - # Mount the host directory as "read only" directory inside container at working_dir + # Mount the host directory inside container at working_dir # https://docs.docker.com/storage/bind-mounts - # Mount the host directory as "read only" inside container "bind": self._working_dir, - "mode": "ro,delegated", + "mode": mount_mode, } } @@ -157,6 +168,15 @@ def create(self): "use_config_proxy": True, } + # Get effective user when building lambda and mounting with write permissions + # Pass effective user to docker run CLI as "--user" option in the format of uid[:gid] + # to run docker as current user instead of root + # Skip if current user is root on posix systems or non-posix systems + effective_user = EffectiveUser.get_current_effective_user().to_effective_user_str() + if self._mount_with_write and effective_user and effective_user != ROOT_USER_ID: + LOG.debug("Detect non-root user, will pass argument '--user %s' to container", effective_user) + kwargs["user"] = effective_user + if self._container_opts: kwargs.update(self._container_opts) @@ -256,6 +276,13 @@ def delete(self): if not removal_in_progress: raise ex LOG.debug("Container removal is in progress, skipping exception: %s", msg) + finally: + # Remove tmp dir on the host + if self._host_tmp_dir: + host_tmp_dir_path = pathlib.Path(self._host_tmp_dir) + if host_tmp_dir_path.exists(): + shutil.rmtree(self._host_tmp_dir) + LOG.debug("Successfully removed temporary directory %s on the host.", self._host_tmp_dir) self.id = None @@ -277,6 +304,11 @@ def start(self, input_data=None): if not self.is_created(): raise RuntimeError("Container does not exist. Cannot start this container") + # Make tmp dir on the host + if self._mount_with_write and self._host_tmp_dir and not os.path.exists(self._host_tmp_dir): + os.makedirs(self._host_tmp_dir) + LOG.debug("Successfully created temporary directory %s on the host.", self._host_tmp_dir) + # Get the underlying container instance from Docker API real_container = self.docker_client.containers.get(self.id) diff --git a/samcli/local/docker/effective_user.py b/samcli/local/docker/effective_user.py new file mode 100644 index 0000000000..b6ed8ec9b3 --- /dev/null +++ b/samcli/local/docker/effective_user.py @@ -0,0 +1,41 @@ +""" +Representation of an effective user +""" +import os +from dataclasses import dataclass +from typing import Optional + +# constant for root user id +ROOT_USER_ID = "0" + + +@dataclass(frozen=True) +class EffectiveUser: + user_id: Optional[str] + group_id: Optional[str] + + def to_effective_user_str(self) -> Optional[str]: + """ + Return String representation of the posix effective user, or None for non posix systems + """ + if not self.user_id: + # Return None for non-posix systems + return None + + if self.user_id == ROOT_USER_ID or not self.group_id: + # Return only user id if root or no group id + return str(self.user_id) + + return f"{self.user_id}:{self.group_id}" + + @staticmethod + def get_current_effective_user(): + """ + Get the posix effective user and group id for current user + """ + if os.name.lower() == "posix": + user_id = os.getuid() + group_ids = os.getgroups() + return EffectiveUser(str(user_id), str(group_ids[0]) if len(group_ids) > 0 else None) + + return EffectiveUser(None, None) diff --git a/samcli/local/docker/lambda_build_container.py b/samcli/local/docker/lambda_build_container.py index 7ce1002855..c2c20e54ad 100644 --- a/samcli/local/docker/lambda_build_container.py +++ b/samcli/local/docker/lambda_build_container.py @@ -4,7 +4,10 @@ import json import logging +import os import pathlib +from typing import List +from uuid import uuid4 from samcli.commands._utils.experimental import get_enabled_experimental_flags from samcli.local.docker.container import Container @@ -33,6 +36,7 @@ def __init__( # pylint: disable=too-many-locals manifest_path, runtime, architecture, + specified_workflow=None, optimizations=None, options=None, executable_search_paths=None, @@ -42,6 +46,8 @@ def __init__( # pylint: disable=too-many-locals image=None, is_building_layer=False, build_in_source=None, + mount_with_write: bool = False, + build_dir=None, ): abs_manifest_path = pathlib.Path(manifest_path).resolve() manifest_file_name = abs_manifest_path.name @@ -83,16 +89,25 @@ def __init__( # pylint: disable=too-many-locals ) if image is None: - image = LambdaBuildContainer._get_image(runtime, architecture) + # use specified_workflow to get image if exists, otherwise use runtime + runtime_to_get_image = specified_workflow if specified_workflow else runtime + image = LambdaBuildContainer._get_image(runtime_to_get_image, architecture) entry = LambdaBuildContainer._get_entrypoint(request_json) - cmd = [] + cmd: List[str] = [] + mount_mode = "rw" if mount_with_write else "ro" additional_volumes = { # Manifest is mounted separately in order to support the case where manifest # is outside of source directory - manifest_dir: {"bind": container_dirs["manifest_dir"], "mode": "ro"} + manifest_dir: {"bind": container_dirs["manifest_dir"], "mode": mount_mode} } + host_tmp_dir = None + if mount_with_write and build_dir: + # Mounting tmp dir on the host as ``/tmp/samcli`` on container, which gives current user write permissions + host_tmp_dir = os.path.join(build_dir, f"tmp-{uuid4().hex}") + additional_volumes.update({host_tmp_dir: {"bind": container_dirs["base_dir"], "mode": mount_mode}}) + if log_level: env_vars["LAMBDA_BUILDERS_LOG_LEVEL"] = log_level @@ -104,6 +119,8 @@ def __init__( # pylint: disable=too-many-locals additional_volumes=additional_volumes, entrypoint=entry, env_vars=env_vars, + mount_with_write=mount_with_write, + host_tmp_dir=host_tmp_dir, ) @property @@ -183,6 +200,7 @@ def get_container_dirs(source_dir, manifest_dir): """ base = "/tmp/samcli" result = { + "base_dir": base, "source_dir": "{}/source".format(base), "artifacts_dir": "{}/artifacts".format(base), "scratch_dir": "{}/scratch".format(base), diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index 59b8b514ae..0453474435 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -14,6 +14,7 @@ import jmespath from pathlib import Path +from samcli.commands.build.utils import MountMode from samcli.lib.utils import osutils from samcli.lib.utils.architecture import X86_64, has_runtime_multi_arch_image from samcli.local.docker.lambda_build_container import LambdaBuildContainer @@ -81,6 +82,7 @@ def get_command_list( hook_name=None, beta_features=None, build_in_source=None, + mount_with=None, ): command_list = [self.cmd, "build"] @@ -125,6 +127,9 @@ def get_command_list( if build_image: command_list += ["--build-image", build_image] + if mount_with: + command_list += ["--mount-with", mount_with.value] + if exclude: for f in exclude: command_list += ["--exclude", f] diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index 5706cb0198..742ba5cf10 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -12,6 +12,7 @@ import pytest from parameterized import parameterized, parameterized_class +from samcli.commands.build.utils import MountMode from samcli.lib.utils import osutils from samcli.yamlhelper import yaml_parse from tests.testing_utils import ( @@ -975,7 +976,7 @@ class TestBuildCommand_Dotnet_cli_package(BuildIntegBase): ] ) @pytest.mark.flaky(reruns=3) - def test_with_dotnetcore(self, runtime, code_uri, mode, architecture="x86_64"): + def test_dotnetcore_in_process(self, runtime, code_uri, mode, architecture="x86_64"): # dotnet7 requires docker to build the function if code_uri == "Dotnet7" and (SKIP_DOCKER_TESTS or SKIP_DOCKER_BUILD): self.skipTest(SKIP_DOCKER_MESSAGE) @@ -1035,10 +1036,160 @@ def test_with_dotnetcore(self, runtime, code_uri, mode, architecture="x86_64"): ) self.verify_docker_container_cleanedup(runtime) + @parameterized.expand( + [ + ("dotnetcore3.1", "Dotnetcore3.1", None), + ("dotnet6", "Dotnet6", None), + ("dotnetcore3.1", "Dotnetcore3.1", "debug"), + ("dotnet6", "Dotnet6", "debug"), + # force to run tests on arm64 machines may cause dotnet7 test failing + # because Native AOT Lambda functions require the host and lambda architectures to match + ("provided.al2", "Dotnet7", None), + ] + ) + @skipIf(SKIP_DOCKER_TESTS or SKIP_DOCKER_BUILD, SKIP_DOCKER_MESSAGE) + @pytest.mark.flaky(reruns=3) + def test_dotnetcore_in_container_mount_with_write_explicit(self, runtime, code_uri, mode, architecture="x86_64"): + overrides = { + "Runtime": runtime, + "CodeUri": code_uri, + "Handler": "HelloWorld::HelloWorld.Function::FunctionHandler", + "Architectures": architecture, + } + + if runtime == "provided.al2": + self.template_path = self.template_path.replace("template.yaml", "template_build_method_dotnet_7.yaml") + + # test with explicit mount_with_write flag + cmdlist = self.get_command_list(use_container=True, parameter_overrides=overrides, mount_with=MountMode.WRITE) + # env vars needed for testing unless set by dotnet images on public.ecr.aws + cmdlist += ["--container-env-var", "DOTNET_CLI_HOME=/tmp/dotnet"] + cmdlist += ["--container-env-var", "XDG_DATA_HOME=/tmp/xdg"] + + LOG.info("Running Command: {}".format(cmdlist)) + LOG.info("Running with SAM_BUILD_MODE={}".format(mode)) + + newenv = os.environ.copy() + if mode: + newenv["SAM_BUILD_MODE"] = mode + + run_command(cmdlist, cwd=self.working_dir, env=newenv) + + self._verify_built_artifact( + self.default_build_dir, + self.FUNCTION_LOGICAL_ID, + self.EXPECTED_FILES_PROJECT_MANIFEST + if runtime != "provided.al2" + else self.EXPECTED_FILES_PROJECT_MANIFEST_PROVIDED, + ) + + self._verify_resource_property( + str(self.built_template), + "OtherRelativePathResource", + "BodyS3Location", + os.path.relpath( + os.path.normpath(os.path.join(str(self.test_data_path), "SomeRelativePath")), + str(self.default_build_dir), + ), + ) + + self._verify_resource_property( + str(self.built_template), + "GlueResource", + "Command.ScriptLocation", + os.path.relpath( + os.path.normpath(os.path.join(str(self.test_data_path), "SomeRelativePath")), + str(self.default_build_dir), + ), + ) + + expected = "{'message': 'Hello World'}" + self._verify_invoke_built_function( + self.built_template, self.FUNCTION_LOGICAL_ID, self._make_parameter_override_arg(overrides), expected + ) + self.verify_docker_container_cleanedup(runtime) + + @parameterized.expand( + [ + ("dotnetcore3.1", "Dotnetcore3.1", None), + ("dotnet6", "Dotnet6", None), + ("dotnetcore3.1", "Dotnetcore3.1", "debug"), + ("dotnet6", "Dotnet6", "debug"), + # force to run tests on arm64 machines may cause dotnet7 test failing + # because Native AOT Lambda functions require the host and lambda architectures to match + ("provided.al2", "Dotnet7", None), + ] + ) + @skipIf(SKIP_DOCKER_TESTS or SKIP_DOCKER_BUILD, SKIP_DOCKER_MESSAGE) + @pytest.mark.flaky(reruns=3) + def test_dotnetcore_in_container_mount_with_write_interactive( + self, + runtime, + code_uri, + mode, + architecture="x86_64", + ): + overrides = { + "Runtime": runtime, + "CodeUri": code_uri, + "Handler": "HelloWorld::HelloWorld.Function::FunctionHandler", + "Architectures": architecture, + } + + if runtime == "provided.al2": + self.template_path = self.template_path.replace("template.yaml", "template_build_method_dotnet_7.yaml") + + # test without explicit mount_with_write flag + cmdlist = self.get_command_list(use_container=True, parameter_overrides=overrides) + # env vars needed for testing unless set by dotnet images on public.ecr.aws + cmdlist += ["--container-env-var", "DOTNET_CLI_HOME=/tmp/dotnet"] + cmdlist += ["--container-env-var", "XDG_DATA_HOME=/tmp/xdg"] + + LOG.info("Running Command: {}".format(cmdlist)) + LOG.info("Running with SAM_BUILD_MODE={}".format(mode)) + + # mock user input to mount with write + user_click_confirm_input = "y" + run_command_with_input(cmdlist, user_click_confirm_input.encode(), cwd=self.working_dir) + + self._verify_built_artifact( + self.default_build_dir, + self.FUNCTION_LOGICAL_ID, + self.EXPECTED_FILES_PROJECT_MANIFEST + if runtime != "provided.al2" + else self.EXPECTED_FILES_PROJECT_MANIFEST_PROVIDED, + ) + + self._verify_resource_property( + str(self.built_template), + "OtherRelativePathResource", + "BodyS3Location", + os.path.relpath( + os.path.normpath(os.path.join(str(self.test_data_path), "SomeRelativePath")), + str(self.default_build_dir), + ), + ) + + self._verify_resource_property( + str(self.built_template), + "GlueResource", + "Command.ScriptLocation", + os.path.relpath( + os.path.normpath(os.path.join(str(self.test_data_path), "SomeRelativePath")), + str(self.default_build_dir), + ), + ) + + expected = "{'message': 'Hello World'}" + self._verify_invoke_built_function( + self.built_template, self.FUNCTION_LOGICAL_ID, self._make_parameter_override_arg(overrides), expected + ) + self.verify_docker_container_cleanedup(runtime) + @parameterized.expand([("dotnetcore3.1", "Dotnetcore3.1"), ("dotnet6", "Dotnet6")]) @skipIf(SKIP_DOCKER_TESTS or SKIP_DOCKER_BUILD, SKIP_DOCKER_MESSAGE) @pytest.mark.flaky(reruns=3) - def test_must_fail_with_container(self, runtime, code_uri): + def test_must_fail_on_container_mount_without_write_interactive(self, runtime, code_uri): use_container = True overrides = { "Runtime": runtime, @@ -1048,9 +1199,11 @@ def test_must_fail_with_container(self, runtime, code_uri): cmdlist = self.get_command_list(use_container=use_container, parameter_overrides=overrides) LOG.info("Running Command: {}".format(cmdlist)) - process_execute = run_command(cmdlist, cwd=self.working_dir) + # mock user input to not allow mounting with write + user_click_confirm_input = "N" + process_execute = run_command_with_input(cmdlist, user_click_confirm_input.encode()) - # Must error out, because container builds are not supported + # Must error out, because mounting with write is not allowed self.assertEqual(process_execute.process.returncode, 1) def _verify_built_artifact(self, build_dir, function_logical_id, expected_files): diff --git a/tests/unit/commands/buildcmd/test_build_context.py b/tests/unit/commands/buildcmd/test_build_context.py index d5c9fbae6a..3cf556a9a1 100644 --- a/tests/unit/commands/buildcmd/test_build_context.py +++ b/tests/unit/commands/buildcmd/test_build_context.py @@ -4,6 +4,7 @@ from parameterized import parameterized +from samcli.commands.build.utils import MountMode from samcli.lib.build.build_graph import DEFAULT_DEPENDENCIES_DIR from samcli.lib.build.bundler import EsbuildBundlerManager from samcli.lib.utils.osutils import BUILD_DIR_PERMISSIONS @@ -16,7 +17,6 @@ BuildError, UnsupportedBuilderLibraryVersionError, BuildInsideContainerError, - ContainerBuildNotSupported, ApplicationBuildResult, ) from samcli.lib.build.workflow_config import UnsupportedRuntimeException @@ -987,6 +987,7 @@ def test_run_build_context( build_images={}, create_auto_dependency_layer=auto_dependency_layer, build_in_source=False, + mount_with=MountMode.READ, ) as build_context: build_context.run() is_sam_template_mock.assert_called_once_with() @@ -1007,6 +1008,7 @@ def test_run_build_context( build_images=build_context._build_images, combine_dependencies=not auto_dependency_layer, build_in_source=build_context._build_in_source, + mount_with_write=False, ) builder_mock.build.assert_called_once() builder_mock.update_template.assert_has_calls( @@ -1062,7 +1064,6 @@ def test_run_build_context( (UnsupportedRuntimeException(), "UnsupportedRuntimeException"), (BuildInsideContainerError(), "BuildInsideContainerError"), (BuildError(wrapped_from=DeepWrap().__class__.__name__, msg="Test"), "DeepWrap"), - (ContainerBuildNotSupported(), "ContainerBuildNotSupported"), ( UnsupportedBuilderLibraryVersionError(container_name="name", error_msg="msg"), "UnsupportedBuilderLibraryVersionError", diff --git a/tests/unit/commands/buildcmd/test_command.py b/tests/unit/commands/buildcmd/test_command.py index 8bf6c0a570..080ca7cbb4 100644 --- a/tests/unit/commands/buildcmd/test_command.py +++ b/tests/unit/commands/buildcmd/test_command.py @@ -5,6 +5,7 @@ from unittest.mock import Mock, patch from samcli.commands.build.command import do_cli, _get_mode_value_from_envvar +from samcli.commands.build.utils import MountMode class TestDoCli(TestCase): @@ -37,6 +38,7 @@ def test_must_succeed_build(self, os_mock, BuildContextMock, mock_build_click): (), hook_name=None, build_in_source=False, + mount_with=MountMode.READ, ) BuildContextMock.assert_called_with( @@ -61,6 +63,7 @@ def test_must_succeed_build(self, os_mock, BuildContextMock, mock_build_click): aws_region=ctx_mock.region, hook_name=None, build_in_source=False, + mount_with=MountMode.READ, ) ctx_mock.run.assert_called_with() self.assertEqual(ctx_mock.run.call_count, 1) @@ -94,6 +97,7 @@ def test_build_exits_supplied_hook_name(self, BuildContextMock, is_experimental_ (), hook_name="terraform", build_in_source=None, + mount_with=MountMode.READ, ) self.assertEqual(ctx_mock.call_count, 0) self.assertEqual(ctx_mock.run.call_count, 0) diff --git a/tests/unit/commands/buildcmd/test_utils.py b/tests/unit/commands/buildcmd/test_utils.py new file mode 100644 index 0000000000..476b8b6935 --- /dev/null +++ b/tests/unit/commands/buildcmd/test_utils.py @@ -0,0 +1,213 @@ +""" +Unit tests for build command utils +""" +from unittest import TestCase +from unittest.mock import patch + +from samcli.commands.build.utils import prompt_user_to_enable_mount_with_write_if_needed +from samcli.lib.utils.architecture import X86_64 +from samcli.lib.utils.packagetype import ZIP +from samcli.lib.providers.provider import ResourcesToBuildCollector, Function, LayerVersion + + +class TestBuildUtils(TestCase): + @patch("samcli.commands.build.utils.prompt") + def test_must_prompt_for_layer(self, prompt_mock): + base_dir = "/mybase" + + # dotnet6 need write permissions + metadata1 = {"BuildMethod": "dotnet6"} + metadata2 = {"BuildMethod": "python3.8"} + layer1 = LayerVersion( + "layer1", + codeuri="codeuri", + metadata=metadata1, + ) + layer2 = LayerVersion( + "layer2", + codeuri="codeuri", + metadata=metadata2, + ) + + function = Function( + stack_path="somepath", + function_id="function_id", + name="logical_id", + functionname="function_name", + runtime="python3.8", + memory=1234, + timeout=12, + handler="handler", + codeuri="codeuri", + environment=None, + rolearn=None, + layers=[layer1, layer2], + events=None, + metadata={}, + inlinecode=None, + imageuri=None, + imageconfig=None, + packagetype=ZIP, + architectures=[X86_64], + codesign_config_arn=None, + function_url_config=None, + runtime_management_config=None, + ) + + resources_to_build = ResourcesToBuildCollector() + resources_to_build.add_function(function) + resources_to_build.add_layers([layer1, layer2]) + + prompt_user_to_enable_mount_with_write_if_needed(resources_to_build, base_dir) + + prompt_mock.assert_called() + + @patch("samcli.commands.build.utils.prompt") + def test_must_prompt_for_function(self, prompt_mock): + base_dir = "/mybase" + + metadata = {"BuildMethod": "python3.8"} + layer1 = LayerVersion( + "layer1", + codeuri="codeuri", + metadata=metadata, + ) + layer2 = LayerVersion( + "layer2", + codeuri="codeuri", + metadata=metadata, + ) + + function = Function( + stack_path="somepath", + function_id="function_id", + name="logical_id", + functionname="function_name", + runtime="dotnet6", + memory=1234, + timeout=12, + handler="handler", + codeuri="codeuri", + environment=None, + rolearn=None, + layers=[layer1, layer2], + events=None, + metadata=None, + inlinecode=None, + imageuri=None, + imageconfig=None, + packagetype=ZIP, + architectures=[X86_64], + codesign_config_arn=None, + function_url_config=None, + runtime_management_config=None, + ) + + resources_to_build = ResourcesToBuildCollector() + resources_to_build.add_function(function) + resources_to_build.add_layers([layer1, layer2]) + + prompt_user_to_enable_mount_with_write_if_needed(resources_to_build, base_dir) + + prompt_mock.assert_called() + + @patch("samcli.commands.build.utils.prompt") + def test_must_prompt_for_function_with_specified_workflow(self, prompt_mock): + base_dir = "/mybase" + + metadata1 = {"BuildMethod": "python3.8"} + layer1 = LayerVersion( + "layer1", + codeuri="codeuri", + metadata=metadata1, + ) + layer2 = LayerVersion( + "layer2", + codeuri="codeuri", + metadata=metadata1, + ) + + metadata2 = {"BuildMethod": "dotnet7"} + + function = Function( + stack_path="somepath", + function_id="function_id", + name="logical_id", + functionname="function_name", + runtime="provided.al2", + memory=1234, + timeout=12, + handler="handler", + codeuri="codeuri", + environment=None, + rolearn=None, + layers=[layer1, layer2], + events=None, + metadata=metadata2, + inlinecode=None, + imageuri=None, + imageconfig=None, + packagetype=ZIP, + architectures=[X86_64], + codesign_config_arn=None, + function_url_config=None, + runtime_management_config=None, + ) + + resources_to_build = ResourcesToBuildCollector() + resources_to_build.add_function(function) + resources_to_build.add_layers([layer1, layer2]) + + prompt_user_to_enable_mount_with_write_if_needed(resources_to_build, base_dir) + + prompt_mock.assert_called() + + @patch("samcli.commands.build.utils.prompt") + def test_must_not_prompt(self, prompt_mock): + base_dir = "/mybase" + + metadata = {"BuildMethod": "python3.8"} + layer1 = LayerVersion( + "layer1", + codeuri="codeuri", + metadata=metadata, + ) + layer2 = LayerVersion( + "layer2", + codeuri="codeuri", + metadata=metadata, + ) + + function = Function( + stack_path="somepath", + function_id="function_id", + name="logical_id", + functionname="function_name", + runtime="python3.8", + memory=1234, + timeout=12, + handler="handler", + codeuri="codeuri", + environment=None, + rolearn=None, + layers=[layer1, layer2], + events=None, + metadata=None, + inlinecode=None, + imageuri=None, + imageconfig=None, + packagetype=ZIP, + architectures=[X86_64], + codesign_config_arn=None, + function_url_config=None, + runtime_management_config=None, + ) + + resources_to_build = ResourcesToBuildCollector() + resources_to_build.add_function(function) + resources_to_build.add_layers([layer1, layer2]) + + prompt_user_to_enable_mount_with_write_if_needed(resources_to_build, base_dir) + mount_with_write = prompt_user_to_enable_mount_with_write_if_needed(resources_to_build, base_dir) + prompt_mock.assert_not_called() + self.assertFalse(mount_with_write) diff --git a/tests/unit/commands/samconfig/test_samconfig.py b/tests/unit/commands/samconfig/test_samconfig.py index 8cc5b37aae..dbe5a99580 100644 --- a/tests/unit/commands/samconfig/test_samconfig.py +++ b/tests/unit/commands/samconfig/test_samconfig.py @@ -120,6 +120,7 @@ def test_build(self, do_cli_mock): "container_env_var_file": "file", "build_image": [("")], "exclude": [("")], + "mount_with": "read", } with samconfig_parameters(["build"], self.scratch_dir, **config_values) as config_path: @@ -157,6 +158,7 @@ def test_build(self, do_cli_mock): ("",), None, None, + "READ", ) @patch("samcli.commands.build.command.do_cli") @@ -178,6 +180,7 @@ def test_build_with_no_cached_override(self, do_cli_mock): "container_env_var_file": "file", "build_image": [("")], "exclude": [("")], + "mount_with": "read", } with samconfig_parameters(["build"], self.scratch_dir, **config_values) as config_path: @@ -215,6 +218,7 @@ def test_build_with_no_cached_override(self, do_cli_mock): ("",), None, None, + "READ", ) @patch("samcli.commands.build.command.do_cli") @@ -233,6 +237,7 @@ def test_build_with_container_env_vars(self, do_cli_mock): "parameter_overrides": "ParameterKey=Key,ParameterValue=Value ParameterKey=Key2,ParameterValue=Value2", "container_env_var": [("")], "container_env_var_file": "env_vars_file", + "mount_with": "read", } with samconfig_parameters(["build"], self.scratch_dir, **config_values) as config_path: @@ -270,6 +275,7 @@ def test_build_with_container_env_vars(self, do_cli_mock): (), None, None, + "READ", ) @patch("samcli.commands.build.command.do_cli") @@ -287,6 +293,7 @@ def test_build_with_build_images(self, do_cli_mock): "skip_pull_image": True, "parameter_overrides": "ParameterKey=Key,ParameterValue=Value ParameterKey=Key2,ParameterValue=Value2", "build_image": ["Function1=image_1", "image_2"], + "mount_with": "read", } with samconfig_parameters(["build"], self.scratch_dir, **config_values) as config_path: @@ -324,6 +331,7 @@ def test_build_with_build_images(self, do_cli_mock): (), None, None, + "READ", ) @patch("samcli.commands.local.invoke.cli.do_cli") diff --git a/tests/unit/lib/build_module/test_app_builder.py b/tests/unit/lib/build_module/test_app_builder.py index 984a693b3b..ba1f47e50b 100644 --- a/tests/unit/lib/build_module/test_app_builder.py +++ b/tests/unit/lib/build_module/test_app_builder.py @@ -18,7 +18,6 @@ UnsupportedBuilderLibraryVersionError, BuildError, LambdaBuilderError, - ContainerBuildNotSupported, BuildInsideContainerError, DockerfileOutSideOfContext, DockerBuildFailed, @@ -990,6 +989,7 @@ def test_must_build_layer_in_container(self, get_layer_subfolder_mock, osutils_m None, None, is_building_layer=True, + specified_workflow=None, ) @patch("samcli.lib.build.app_builder.get_workflow_config") @@ -1025,6 +1025,7 @@ def test_must_build_layer_in_container_with_global_build_image( None, "test_image", is_building_layer=True, + specified_workflow=None, ) @patch("samcli.lib.build.app_builder.get_workflow_config") @@ -1060,6 +1061,46 @@ def test_must_build_layer_in_container_with_specific_build_image( None, "test_image", is_building_layer=True, + specified_workflow=None, + ) + + @patch("samcli.lib.build.app_builder.supports_specified_workflow") + @patch("samcli.lib.build.app_builder.get_workflow_config") + @patch("samcli.lib.build.app_builder.osutils") + @patch("samcli.lib.build.app_builder.get_layer_subfolder") + def test_must_build_layer_in_container_with_specified_workflow_if_supported( + self, get_layer_subfolder_mock, osutils_mock, get_workflow_config_mock, supports_specified_workflow_mock + ): + self.builder._container_manager = self.container_manager + get_layer_subfolder_mock.return_value = "python" + config_mock = Mock() + config_mock.manifest_name = "manifest_name" + + scratch_dir = "scratch" + osutils_mock.mkdir_temp.return_value.__enter__ = Mock(return_value=scratch_dir) + osutils_mock.mkdir_temp.return_value.__exit__ = Mock() + + get_workflow_config_mock.return_value = config_mock + build_function_on_container_mock = Mock() + + build_images = {"layer_name": "test_image"} + self.builder._build_images = build_images + self.builder._build_function_on_container = build_function_on_container_mock + supports_specified_workflow_mock.return_value = True + + self.builder._build_layer("layer_name", "code_uri", "python3.8", ["python3.8"], ARM64, "full_path") + build_function_on_container_mock.assert_called_once_with( + config_mock, + PathValidator("code_uri"), + PathValidator("python"), + PathValidator("manifest_name"), + "python3.8", + ARM64, + None, + None, + "test_image", + is_building_layer=True, + specified_workflow="python3.8", ) @@ -2248,7 +2289,16 @@ def test_must_build_in_container(self, osutils_mock, get_workflow_config_mock): self.builder._build_function(function_name, codeuri, packagetype, runtime, architecture, handler, artifacts_dir) self.builder._build_function_on_container.assert_called_with( - config_mock, code_dir, artifacts_dir, manifest_path, runtime, architecture, None, None, None + config_mock, + code_dir, + artifacts_dir, + manifest_path, + runtime, + architecture, + None, + None, + None, + specified_workflow=None, ) @patch("samcli.lib.build.app_builder.get_workflow_config") @@ -2288,7 +2338,16 @@ def test_must_build_in_container_with_env_vars(self, osutils_mock, get_workflow_ ) self.builder._build_function_on_container.assert_called_with( - config_mock, code_dir, artifacts_dir, manifest_path, runtime, architecture, None, {"TEST": "test"}, None + config_mock, + code_dir, + artifacts_dir, + manifest_path, + runtime, + architecture, + None, + {"TEST": "test"}, + None, + specified_workflow=None, ) @patch("samcli.lib.build.app_builder.get_workflow_config") @@ -2323,7 +2382,16 @@ def test_must_build_in_container_with_custom_specified_build_image(self, osutils ) self.builder._build_function_on_container.assert_called_with( - config_mock, code_dir, artifacts_dir, manifest_path, runtime, architecture, None, None, image_uri + config_mock, + code_dir, + artifacts_dir, + manifest_path, + runtime, + architecture, + None, + None, + image_uri, + specified_workflow=None, ) @patch("samcli.lib.build.app_builder.get_workflow_config") @@ -2358,7 +2426,16 @@ def test_must_build_in_container_with_custom_default_build_image(self, osutils_m ) self.builder._build_function_on_container.assert_called_with( - config_mock, code_dir, artifacts_dir, manifest_path, runtime, architecture, None, None, image_uri + config_mock, + code_dir, + artifacts_dir, + manifest_path, + runtime, + architecture, + None, + None, + image_uri, + specified_workflow=None, ) @@ -2521,7 +2598,12 @@ def tearDown(self): @patch("samcli.lib.build.app_builder.LOG") @patch("samcli.lib.build.app_builder.osutils") def test_must_build_in_container( - self, osutils_mock, LOGMock, protocol_version_mock, LambdaBuildContainerMock, event_mock + self, + osutils_mock, + LOGMock, + protocol_version_mock, + LambdaBuildContainerMock, + event_mock, ): event_mock.return_value = "runtime" config = Mock() @@ -2551,6 +2633,7 @@ def mock_wait_for_logs(stdout, stderr): "manifest_path", "runtime", X86_64, + specified_workflow=None, image=None, log_level=log_level, optimizations=None, @@ -2560,6 +2643,8 @@ def mock_wait_for_logs(stdout, stderr): env_vars={}, is_building_layer=False, build_in_source=False, + mount_with_write=False, + build_dir="/build/dir", ) self.container_manager.run.assert_called_with(container_mock) @@ -2607,20 +2692,6 @@ def test_must_raise_on_docker_not_running(self): str(ctx.exception), "Docker is unreachable. Docker needs to be running to build inside a container." ) - @patch("samcli.lib.build.app_builder.supports_build_in_container") - def test_must_raise_on_unsupported_container_build(self, supports_build_in_container_mock): - config = Mock() - - reason = "my reason" - supports_build_in_container_mock.return_value = (False, reason) - - with self.assertRaises(ContainerBuildNotSupported) as ctx: - self.builder._build_function_on_container( - config, "source_dir", "artifacts_dir", "scratch_dir", "manifest_path", "runtime", X86_64, {} - ) - - self.assertEqual(str(ctx.exception), reason) - class TestApplicationBuilder_parse_builder_response(TestCase): def setUp(self): diff --git a/tests/unit/lib/build_module/test_workflow_config.py b/tests/unit/lib/build_module/test_workflow_config.py index 556eae69d2..c8a1fad900 100644 --- a/tests/unit/lib/build_module/test_workflow_config.py +++ b/tests/unit/lib/build_module/test_workflow_config.py @@ -26,6 +26,7 @@ def test_must_work_for_python(self, runtime): self.assertIsNone(result.executable_search_paths) self.assertEqual(len(EventTracker.get_tracked_events()), 1) self.assertIn(Event("BuildWorkflowUsed", "python-pip"), EventTracker.get_tracked_events()) + self.assertFalse(result.must_mount_with_write_in_container) @parameterized.expand([("nodejs12.x",), ("nodejs14.x",), ("nodejs16.x",), ("nodejs18.x",)]) def test_must_work_for_nodejs(self, runtime): @@ -37,6 +38,7 @@ def test_must_work_for_nodejs(self, runtime): self.assertIsNone(result.executable_search_paths) self.assertEqual(len(EventTracker.get_tracked_events()), 1) self.assertIn(Event("BuildWorkflowUsed", "nodejs-npm"), EventTracker.get_tracked_events()) + self.assertFalse(result.must_mount_with_write_in_container) @parameterized.expand([("provided",)]) def test_must_work_for_provided(self, runtime): @@ -48,6 +50,7 @@ def test_must_work_for_provided(self, runtime): self.assertIsNone(result.executable_search_paths) self.assertEqual(len(EventTracker.get_tracked_events()), 1) self.assertIn(Event("BuildWorkflowUsed", "provided-None"), EventTracker.get_tracked_events()) + self.assertFalse(result.must_mount_with_write_in_container) @parameterized.expand([("provided.al2",)]) def test_must_work_for_provided_with_build_method_dotnet7(self, runtime): @@ -59,6 +62,12 @@ def test_must_work_for_provided_with_build_method_dotnet7(self, runtime): self.assertIsNone(result.executable_search_paths) self.assertEqual(len(EventTracker.get_tracked_events()), 1) self.assertIn(Event("BuildWorkflowUsed", "dotnet-cli-package"), EventTracker.get_tracked_events()) + self.assertTrue(result.must_mount_with_write_in_container) + + @parameterized.expand([("dotnetcore3.1",), ("dotnet6",), ("provided.al2", "dotnet7")]) + def test_must_mount_with_write_for_dotnet_in_container(self, runtime, specified_workflow=None): + result = get_workflow_config(runtime, self.code_dir, self.project_dir, specified_workflow) + self.assertTrue(result.must_mount_with_write_in_container) @parameterized.expand([("provided.al2",)]) def test_must_work_for_provided_with_build_method_rustcargolambda(self, runtime): @@ -70,6 +79,7 @@ def test_must_work_for_provided_with_build_method_rustcargolambda(self, runtime) self.assertIsNone(result.executable_search_paths) self.assertEqual(len(EventTracker.get_tracked_events()), 1) self.assertIn(Event("BuildWorkflowUsed", "rust-cargo"), EventTracker.get_tracked_events()) + self.assertFalse(result.must_mount_with_write_in_container) @parameterized.expand([("provided",)]) def test_must_work_for_provided_with_no_specified_workflow(self, runtime): @@ -82,6 +92,7 @@ def test_must_work_for_provided_with_no_specified_workflow(self, runtime): self.assertIsNone(result.executable_search_paths) self.assertEqual(len(EventTracker.get_tracked_events()), 1) self.assertIn(Event("BuildWorkflowUsed", "provided-None"), EventTracker.get_tracked_events()) + self.assertFalse(result.must_mount_with_write_in_container) @parameterized.expand([("provided",)]) def test_raise_exception_for_bad_specified_workflow(self, runtime): @@ -98,6 +109,7 @@ def test_must_work_for_ruby(self, runtime): self.assertIsNone(result.executable_search_paths) self.assertEqual(len(EventTracker.get_tracked_events()), 1) self.assertIn(Event("BuildWorkflowUsed", "ruby-bundler"), EventTracker.get_tracked_events()) + self.assertFalse(result.must_mount_with_write_in_container) @parameterized.expand( [("java8", "build.gradle", "gradle"), ("java8", "build.gradle.kts", "gradle"), ("java8", "pom.xml", "maven")] @@ -113,6 +125,7 @@ def test_must_work_for_java(self, runtime, build_file, dep_manager, os_mock): self.assertEqual(result.application_framework, None) self.assertEqual(result.manifest_name, build_file) self.assertEqual(len(EventTracker.get_tracked_events()), 1) + self.assertFalse(result.must_mount_with_write_in_container) if dep_manager == "gradle": self.assertEqual(result.executable_search_paths, [self.code_dir, self.project_dir]) @@ -131,6 +144,7 @@ def test_must_get_workflow_for_esbuild(self): self.assertIsNone(result.executable_search_paths) self.assertEqual(len(EventTracker.get_tracked_events()), 1) self.assertIn(Event("BuildWorkflowUsed", "nodejs-npm-esbuild"), EventTracker.get_tracked_events()) + self.assertFalse(result.must_mount_with_write_in_container) @parameterized.expand([("java8", "unknown.manifest")]) @patch("samcli.lib.build.workflow_config.os") diff --git a/tests/unit/local/docker/test_container.py b/tests/unit/local/docker/test_container.py index 6e4772d1bd..61ccad4c9f 100644 --- a/tests/unit/local/docker/test_container.py +++ b/tests/unit/local/docker/test_container.py @@ -457,6 +457,17 @@ def test_must_skip_if_container_is_not_created(self): self.container.delete() self.mock_docker_client.containers.get.assert_not_called() + @patch("samcli.local.docker.container.pathlib.Path.exists") + @patch("samcli.local.docker.container.shutil") + def test_must_remove_host_tmp_dir_after_mount_with_write_container_build(self, mock_shutil, mock_exists): + self.container.is_created.return_value = True + self.container._mount_with_write = True + self.container._host_tmp_dir = "host_tmp_dir" + + mock_exists.return_value = True + self.container.delete() + mock_shutil.rmtree.assert_called_with(self.container._host_tmp_dir) + class TestContainer_start(TestCase): def setUp(self): @@ -500,6 +511,17 @@ def test_must_not_support_input_data(self): with self.assertRaises(ValueError): self.container.start(input_data="some input data") + @patch("samcli.local.docker.container.os.path") + @patch("samcli.local.docker.container.os") + def test_must_make_host_tmp_dir_if_mount_with_write_container_build(self, mock_os, mock_path): + self.container.is_created.return_value = True + self.container._mount_with_write = True + self.container._host_tmp_dir = "host_tmp_dir" + mock_path.exists.return_value = False + + self.container.start() + mock_os.makedirs.assert_called_with(self.container._host_tmp_dir) + class TestContainer_wait_for_result(TestCase): def setUp(self): diff --git a/tests/unit/local/docker/test_effective_user.py b/tests/unit/local/docker/test_effective_user.py new file mode 100644 index 0000000000..cdb255bbb5 --- /dev/null +++ b/tests/unit/local/docker/test_effective_user.py @@ -0,0 +1,82 @@ +""" +Unit test for EffectiveUser class +""" +from unittest import TestCase +from unittest.mock import patch + +from samcli.local.docker.effective_user import EffectiveUser, ROOT_USER_ID + + +class TestEffectiveUser(TestCase): + @patch("samcli.local.docker.effective_user.os.name.lower") + @patch("samcli.local.docker.effective_user.os") + def test_return_effective_user_if_posix(self, mock_os, mock_os_name): + mock_os_name.return_value = "posix" + mock_os.getuid.return_value = 1000 + mock_os.getgroups.return_value = [1000, 2000, 3000] + + result = EffectiveUser.get_current_effective_user() + + mock_os.getuid.assert_called_once() + mock_os.getgroups.assert_called_once() + self.assertEqual("1000", result.user_id) + self.assertEqual("1000", result.group_id) + + @patch("samcli.local.docker.effective_user.os.name.lower") + @patch("samcli.local.docker.effective_user.os") + def test_return_none_if_non_posix(self, mock_os, mock_os_name): + mock_os_name.return_value = "nt" + mock_os.getuid.return_value = 1000 + mock_os.getgroups.return_value = [1000, 2000, 3000] + + result = EffectiveUser.get_current_effective_user() + + mock_os.getuid.assert_not_called() + mock_os.getgroups.assert_not_called() + self.assertIsNone(result.user_id) + self.assertIsNone(result.group_id) + + @patch("samcli.local.docker.effective_user.os.name.lower") + @patch("samcli.local.docker.effective_user.os") + def test_to_effective_user_str(self, mock_os, mock_os_name): + mock_os_name.return_value = "posix" + mock_os.getuid.return_value = 1000 + mock_os.getgroups.return_value = [1000, 2000, 3000] + + result = EffectiveUser.get_current_effective_user().to_effective_user_str() + + self.assertEqual("1000:1000", result) + + @patch("samcli.local.docker.effective_user.os.name.lower") + @patch("samcli.local.docker.effective_user.os") + def test_to_effective_user_str_if_root(self, mock_os, mock_os_name): + mock_os_name.return_value = "posix" + # 0 means current user is root + mock_os.getuid.return_value = 0 + mock_os.getgroups.return_value = [1000, 2000, 3000] + + result = EffectiveUser.get_current_effective_user().to_effective_user_str() + + self.assertEqual(ROOT_USER_ID, result) + + @patch("samcli.local.docker.effective_user.os.name.lower") + @patch("samcli.local.docker.effective_user.os") + def test_to_effective_user_str_if_no_group_id(self, mock_os, mock_os_name): + mock_os_name.return_value = "posix" + mock_os.getuid.return_value = 1000 + mock_os.getgroups.return_value = [] + + result = EffectiveUser.get_current_effective_user().to_effective_user_str() + + self.assertEqual("1000", result) + + @patch("samcli.local.docker.effective_user.os.name.lower") + @patch("samcli.local.docker.effective_user.os") + def test_to_effective_user_str_if_non_posix(self, mock_os, mock_os_name): + mock_os_name.return_value = "nt" + mock_os.getuid.return_value = 1000 + mock_os.getgroups.return_value = [1000, 2000, 3000] + + result = EffectiveUser.get_current_effective_user().to_effective_user_str() + + self.assertIsNone(result) diff --git a/tests/unit/local/docker/test_lambda_build_container.py b/tests/unit/local/docker/test_lambda_build_container.py index 46e16bd799..c72e808f13 100644 --- a/tests/unit/local/docker/test_lambda_build_container.py +++ b/tests/unit/local/docker/test_lambda_build_container.py @@ -24,6 +24,7 @@ def test_must_init_class(self, get_container_dirs_mock, get_entrypoint_mock, get entry = get_entrypoint_mock.return_value = "entrypoint" image = get_image_mock.return_value = "imagename" container_dirs = get_container_dirs_mock.return_value = { + "base_dir": "/mybase", "source_dir": "/mysource", "manifest_dir": "/mymanifest", "artifacts_dir": "/myartifacts", @@ -77,6 +78,7 @@ def test_must_make_request_object_string(self, is_building_layer, experimental_f patched_experimental_flags.return_value = experimental_flags container_dirs = { + "base_dir": "base_dir", "source_dir": "source_dir", "artifacts_dir": "artifacts_dir", "scratch_dir": "scratch_dir", @@ -142,6 +144,7 @@ def test_must_return_dirs(self): self.assertEqual( result, { + "base_dir": "/tmp/samcli", "source_dir": "/tmp/samcli/source", "manifest_dir": "/tmp/samcli/manifest", "artifacts_dir": "/tmp/samcli/artifacts", @@ -158,6 +161,7 @@ def test_must_override_manifest_if_equal_to_source(self): self.assertEqual( result, { + "base_dir": "/tmp/samcli", # When source & manifest directories are the same, manifest_dir must be equal to source "source_dir": "/tmp/samcli/source", "manifest_dir": "/tmp/samcli/source", @@ -177,6 +181,31 @@ class TestLambdaBuildContainer_get_image(TestCase): def test_must_get_image_name(self, runtime, architecture, expected_image_name): self.assertEqual(expected_image_name, LambdaBuildContainer._get_image(runtime, architecture)) + @patch("samcli.lib.build.workflow_config.supports_specified_workflow") + @patch.object(LambdaBuildContainer, "_get_image") + def test_get_image_by_specified_workflow_if_supported(self, get_image_mock, supports_specified_workflow_mock): + architecture = "arm64" + specified_workflow = "specified_workflow" + + supports_specified_workflow_mock.return_value = True + + LambdaBuildContainer( + "protocol", + "language", + "dependency", + "application", + "/foo/source", + "/bar/manifest.txt", + "runtime", + optimizations="optimizations", + options="options", + log_level="log-level", + mode="mode", + architecture=architecture, + specified_workflow=specified_workflow, + ) + get_image_mock.assert_called_once_with(specified_workflow, architecture) + class TestLambdaBuildContainer_get_image_tag(TestCase): @parameterized.expand(