Skip to content

Commit

Permalink
feat: support dotnet lambda container builds (#4665)
Browse files Browse the repository at this point in the history
* allow use container for dotnet with write permission to source code directory

* fix typo

* major code changes to allow mount with write

* remove dotnet env vars and add logic to make tmp dir on host

* integration tests

* consist help text and click confirm

* cleaner approach to make tmp dir on host, refactor regarding dev guide

* update and add new unit tests

* fix failed unit tests

* update test comments

* update CONFIG and remove unnecessary Json dump

* update integration tests

* pass ruff tests

* change mount with click to enum

* add debug logging and remove unused exception

* update unit and integration tests

* update click

* add MountMode enum and set default to READ

* add unit tests for prompting

* default mount_with to READ in all places

* early exit

* better use of enum and constant, update unit tests

* make tmp dir on sam build dir on host

---------

Co-authored-by: Mehmet Nuri Deveci <[email protected]>
  • Loading branch information
mrkdeng and mndeveci authored Mar 15, 2023
1 parent cf03826 commit bd76bfd
Show file tree
Hide file tree
Showing 21 changed files with 916 additions and 79 deletions.
21 changes: 19 additions & 2 deletions samcli/commands/build/build_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import click

from samcli.commands.build.utils import prompt_user_to_enable_mount_with_write_if_needed, MountMode
from samcli.lib.build.bundler import EsbuildBundlerManager
from samcli.lib.providers.sam_api_provider import SamApiProvider
from samcli.lib.telemetry.event import EventTracker
Expand All @@ -35,7 +36,6 @@
ApplicationBuilder,
BuildError,
UnsupportedBuilderLibraryVersionError,
ContainerBuildNotSupported,
ApplicationBuildResult,
)
from samcli.commands._utils.constants import DEFAULT_BUILD_DIR
Expand Down Expand Up @@ -78,6 +78,7 @@ def __init__(
locate_layer_nested: bool = False,
hook_name: Optional[str] = None,
build_in_source: Optional[bool] = None,
mount_with: str = MountMode.READ.value,
) -> None:
"""
Initialize the class
Expand Down Expand Up @@ -133,6 +134,8 @@ def __init__(
Name of the hook package
build_in_source: Optional[bool]
Set to True to build in the source directory.
mount_with:
Mount mode of source code directory when building inside container, READ ONLY by default
"""

self._resource_identifier = resource_identifier
Expand Down Expand Up @@ -172,6 +175,7 @@ def __init__(
self._locate_layer_nested = locate_layer_nested
self._hook_name = hook_name
self._build_in_source = build_in_source
self._mount_with = MountMode(mount_with)

def __enter__(self) -> "BuildContext":
self.set_up()
Expand Down Expand Up @@ -232,6 +236,19 @@ def run(self):

self._stacks = self._handle_build_pre_processing()

# boolean value indicates if mount with write or not, defaults to READ ONLY
mount_with_write = False
if self._use_container:
if self._mount_with == MountMode.WRITE:
mount_with_write = True
else:
# if self._mount_with is NOT WRITE
# check the need of mounting with write permissions and prompt user to enable it if needed
mount_with_write = prompt_user_to_enable_mount_with_write_if_needed(
self.get_resources_to_build(),
self.base_dir,
)

try:
builder = ApplicationBuilder(
self.get_resources_to_build(),
Expand All @@ -249,6 +266,7 @@ def run(self):
build_images=self._build_images,
combine_dependencies=not self._create_auto_dependency_layer,
build_in_source=self._build_in_source,
mount_with_write=mount_with_write,
)
except FunctionNotFound as ex:
raise UserException(str(ex), wrapped_from=ex.__class__.__name__) from ex
Expand Down Expand Up @@ -293,7 +311,6 @@ def run(self):
BuildError,
BuildInsideContainerError,
UnsupportedBuilderLibraryVersionError,
ContainerBuildNotSupported,
InvalidBuildGraphException,
) as ex:
click.secho("\nBuild Failed", fg="red")
Expand Down
15 changes: 15 additions & 0 deletions samcli/commands/build/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from samcli.cli.cli_config_file import configuration_option, TomlProvider
from samcli.lib.utils.version_checker import check_newer_version
from samcli.commands.build.click_container import ContainerOptions
from samcli.commands.build.utils import MountMode

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -136,6 +137,16 @@
help="Enabled parallel builds. Use this flag to build your AWS SAM template's functions and layers in parallel. "
"By default the functions and layers are built in sequence",
)
@click.option(
"--mount-with",
"-mw",
type=click.Choice(MountMode.values(), case_sensitive=False),
default=MountMode.READ.value,
help="Optional. Specify mount mode for building functions/layers inside container. "
"If it is mounted with write permissions, some files in source code directory may "
"be changed/added by the build process. By default the source code directory is read only.",
cls=ContainerOptions,
)
@build_dir_option
@cache_dir_option
@base_dir_option
Expand Down Expand Up @@ -175,6 +186,7 @@ def cli(
config_env: str,
hook_name: Optional[str],
skip_prepare_infra: bool,
mount_with,
) -> None:
"""
`sam build` command entry point
Expand Down Expand Up @@ -205,6 +217,7 @@ def cli(
exclude,
hook_name,
None, # TODO: replace with build_in_source once it's added as a click option
mount_with,
) # pragma: no cover


Expand All @@ -230,6 +243,7 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
exclude: Optional[Tuple[str, ...]],
hook_name: Optional[str],
build_in_source: Optional[bool],
mount_with,
) -> None:
"""
Implementation of the ``cli`` method
Expand Down Expand Up @@ -275,6 +289,7 @@ def do_cli( # pylint: disable=too-many-locals, too-many-statements
aws_region=click_ctx.region,
hook_name=hook_name,
build_in_source=build_in_source,
mount_with=mount_with,
) as ctx:
ctx.run()

Expand Down
110 changes: 110 additions & 0 deletions samcli/commands/build/utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
"""
Utilities for sam build command
"""
import pathlib
from enum import Enum
from typing import List

import click

from samcli.lib.build.workflow_config import CONFIG, get_workflow_config
from samcli.lib.providers.provider import ResourcesToBuildCollector


class MountMode(Enum):
"""
Enums that represent mount mode used when build lambda functions/layers inside container
"""

READ = "READ"
WRITE = "WRITE"

@classmethod
def values(cls) -> List[str]:
"""
A getter to retrieve the accepted value list for mount mode
Returns: List[str]
The accepted mount mode list
"""
return [e.value for e in cls]


def prompt_user_to_enable_mount_with_write_if_needed(
resources_to_build: ResourcesToBuildCollector,
base_dir: str,
) -> bool:
"""
First check if mounting with write permissions is needed for building inside container or not. If it is needed, then
prompt user to choose if enables mounting with write permissions or not.
Parameters
----------
resources_to_build:
Resource to build inside container
base_dir : str
Path to the base directory
Returns
-------
bool
True, if user enabled mounting with write permissions.
"""

for function in resources_to_build.functions:
code_uri = function.codeuri
if not code_uri:
continue
runtime = function.runtime
code_dir = str(pathlib.Path(base_dir, code_uri).resolve())
# get specified_workflow if metadata exists
metadata = function.metadata
specified_workflow = metadata.get("BuildMethod", None) if metadata else None
config = get_workflow_config(runtime, code_dir, base_dir, specified_workflow)
# at least one function needs mount with write, return with prompting
if not config.must_mount_with_write_in_container:
continue
return prompt(config, code_dir)

for layer in resources_to_build.layers:
code_uri = layer.codeuri
if not code_uri:
continue
code_dir = str(pathlib.Path(base_dir, code_uri).resolve())
specified_workflow = layer.build_method
config = get_workflow_config(None, code_dir, base_dir, specified_workflow)
# at least one layer needs mount with write, return with prompting
if not config.must_mount_with_write_in_container:
continue
return prompt(config, code_dir)

return False


def prompt(config: CONFIG, source_dir: str) -> bool:
"""
Prompt user to choose if enables mounting with write permissions or not when building lambda functions/layers
Parameters
----------
config: namedtuple(Capability)
Config specifying the particular build workflow
source_dir : str
Path to the function source code
Returns
-------
bool
True, if user enabled mounting with write permissions.
"""
if click.confirm(
f"\nBuilding functions with {config.language} inside containers needs "
f"mounting with write permissions to the source code directory {source_dir}. "
f"Some files in this directory may be changed or added by the build process. "
f"Pass `--mount-with WRITE` to `sam build` CLI to avoid this confirmation. "
f"\nWould you like to enable mounting with write permissions? "
):
return True
return False
32 changes: 17 additions & 15 deletions samcli/lib/build/app_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,13 @@
import logging
import pathlib
from typing import List, Optional, Dict, cast, NamedTuple

import docker
import docker.errors
from aws_lambda_builders import (
RPC_PROTOCOL_VERSION as lambda_builders_protocol_version,
)
from aws_lambda_builders.builder import LambdaBuilder
from aws_lambda_builders.exceptions import LambdaBuilderError

from samcli.lib.build.build_graph import FunctionBuildDefinition, LayerBuildDefinition, BuildGraph
from samcli.lib.build.build_strategy import (
DefaultBuildStrategy,
Expand Down Expand Up @@ -51,14 +49,12 @@
DockerBuildFailed,
BuildError,
BuildInsideContainerError,
ContainerBuildNotSupported,
UnsupportedBuilderLibraryVersionError,
)

from samcli.lib.build.workflow_config import (
get_workflow_config,
supports_specified_workflow,
get_layer_subfolder,
supports_build_in_container,
CONFIG,
UnsupportedRuntimeException,
)
Expand Down Expand Up @@ -101,6 +97,7 @@ def __init__(
build_images: Optional[Dict] = None,
combine_dependencies: bool = True,
build_in_source: Optional[bool] = None,
mount_with_write: bool = False,
) -> None:
"""
Initialize the class
Expand Down Expand Up @@ -144,6 +141,8 @@ def __init__(
dependencies or not.
build_in_source: Optional[bool]
Set to True to build in the source directory.
mount_with_write: bool
Mount source code directory with write permissions when building inside container.
"""
self._resources_to_build = resources_to_build
self._build_dir = build_dir
Expand All @@ -166,6 +165,7 @@ def __init__(
self._build_images = build_images or {}
self._combine_dependencies = combine_dependencies
self._build_in_source = build_in_source
self._mount_with_write = mount_with_write

def build(self) -> ApplicationBuildResult:
"""
Expand Down Expand Up @@ -544,6 +544,8 @@ def _build_layer(
build_runtime = compatible_runtimes[0]
global_image = self._build_images.get(None)
image = self._build_images.get(layer_name, global_image)
# pass to container only when specified workflow is supported to overwrite runtime to get image
supported_specified_workflow = supports_specified_workflow(specified_workflow)
self._build_function_on_container(
config,
code_dir,
Expand All @@ -555,6 +557,7 @@ def _build_layer(
container_env_vars,
image,
is_building_layer=True,
specified_workflow=specified_workflow if supported_specified_workflow else None,
)
else:
self._build_function_in_process(
Expand Down Expand Up @@ -644,11 +647,9 @@ def _build_function( # pylint: disable=R1710
# Create the arguments to pass to the builder
# Code is always relative to the given base directory.
code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve())

# Determine if there was a build workflow that was specified directly in the template.
specified_build_workflow = metadata.get("BuildMethod", None) if metadata else None

config = get_workflow_config(runtime, code_dir, self._base_dir, specified_workflow=specified_build_workflow)
specified_workflow = metadata.get("BuildMethod", None) if metadata else None
config = get_workflow_config(runtime, code_dir, self._base_dir, specified_workflow=specified_workflow)

if config.language == "provided" and isinstance(metadata, dict) and metadata.get("ProjectRootDirectory"):
code_dir = str(pathlib.Path(self._base_dir, metadata.get("ProjectRootDirectory", code_dir)).resolve())
Expand Down Expand Up @@ -682,7 +683,8 @@ def _build_function( # pylint: disable=R1710
# None represents the global build image for all functions/layers
global_image = self._build_images.get(None)
image = self._build_images.get(function_name, global_image)

# pass to container only when specified workflow is supported to overwrite runtime to get image
supported_specified_workflow = supports_specified_workflow(specified_workflow)
return self._build_function_on_container(
config,
code_dir,
Expand All @@ -693,6 +695,7 @@ def _build_function( # pylint: disable=R1710
options,
container_env_vars,
image,
specified_workflow=specified_workflow if supported_specified_workflow else None,
)

return self._build_function_in_process(
Expand Down Expand Up @@ -883,6 +886,7 @@ def _build_function_on_container(
container_env_vars: Optional[Dict] = None,
build_image: Optional[str] = None,
is_building_layer: bool = False,
specified_workflow: Optional[str] = None,
) -> str:
# _build_function_on_container() is only called when self._container_manager if not None
if not self._container_manager:
Expand All @@ -893,10 +897,6 @@ def _build_function_on_container(
"Docker is unreachable. Docker needs to be running to build inside a container."
)

container_build_supported, reason = supports_build_in_container(config)
if not container_build_supported:
raise ContainerBuildNotSupported(reason)

# If we are printing debug logs in SAM CLI, the builder library should also print debug logs
log_level = LOG.getEffectiveLevel()

Expand All @@ -911,6 +911,7 @@ def _build_function_on_container(
manifest_path,
runtime,
architecture,
specified_workflow=specified_workflow,
log_level=log_level,
optimizations=None,
options=options,
Expand All @@ -920,6 +921,8 @@ def _build_function_on_container(
image=build_image,
is_building_layer=is_building_layer,
build_in_source=self._build_in_source,
mount_with_write=self._mount_with_write,
build_dir=self._build_dir,
)

try:
Expand All @@ -930,7 +933,6 @@ def _build_function_on_container(
raise UnsupportedBuilderLibraryVersionError(
container.image, "{} executable not found in container".format(container.executable_name)
) from ex

# Container's output provides status of whether the build succeeded or failed
# stdout contains the result of JSON-RPC call
stdout_stream = io.BytesIO()
Expand Down
Loading

0 comments on commit bd76bfd

Please sign in to comment.