Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: link the API gateway resource parent to either rest api or another gateway resource #5697

Merged
merged 1 commit into from
Aug 3, 2023
Merged

fix: link the API gateway resource parent to either rest api or another gateway resource #5697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions samcli/hook_packages/terraform/hooks/prepare/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,13 @@
CFN_AWS_LAMBDA_LAYER_VERSION: "Content",
}
COMPILED_REGULAR_EXPRESSION = re.compile(r"\[[^\[\]]*\]")
REMOTE_DUMMY_VALUE = "<<REMOTE DUMMY VALUE - RAISE ERROR IF IT IS STILL THERE>>"
TF_AWS_LAMBDA_FUNCTION = "aws_lambda_function"
TF_AWS_LAMBDA_LAYER_VERSION = "aws_lambda_layer_version"
TF_AWS_API_GATEWAY_RESOURCE = "aws_api_gateway_resource"
TF_AWS_API_GATEWAY_REST_API = "aws_api_gateway_rest_api"
TF_AWS_API_GATEWAY_STAGE = "aws_api_gateway_stage"
TF_AWS_API_GATEWAY_METHOD = "aws_api_gateway_method"
TF_AWS_API_GATEWAY_INTEGRATION = "aws_api_gateway_integration"
TF_AWS_API_GATEWAY_AUTHORIZER = "aws_api_gateway_authorizer"
TF_AWS_API_GATEWAY_INTEGRATION_RESPONSE = "aws_api_gateway_method_response"
21 changes: 21 additions & 0 deletions samcli/hook_packages/terraform/hooks/prepare/exceptions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ def __init__(self, message):
UserException.__init__(self, msg)


class UnexpectedDestinationResource(InvalidResourceLinkingException):
"""
Exception that will be thrown while doing terraform linking logic in case if the found destination resource is not
of the expected type, or the field used for linking is not the expected field.
"""


class ApplyLimitationException(UserException):
def __init__(self, message):
fmt = "{message}{line_sep}{line_sep}{apply_work_around}"
Expand Down Expand Up @@ -85,6 +92,20 @@ class GatewayResourceToGatewayRestApiLocalVariablesLinkingLimitationException(Lo
"""


class OneGatewayResourceToParentResourceLinkingLimitationException(OneResourceLinkingLimitationException):
"""
Exception specific for Gateway Resource linking to more than one Parent Resources which is either Rest API or
Gateway Resource
"""


class GatewayResourceToParentResourceLocalVariablesLinkingLimitationException(LocalVariablesLinkingLimitationException):
"""
Exception specific for Gateway Resource linking to Parent Resources which is either Rest API or Gateway Resource
using locals.
"""


class OneRestApiToApiGatewayMethodLinkingLimitationException(OneResourceLinkingLimitationException):
"""
Exception specific for Gateway Method linking to more than Rest API
Expand Down
24 changes: 12 additions & 12 deletions samcli/hook_packages/terraform/hooks/prepare/property_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,18 @@
from json.decoder import JSONDecodeError
from typing import Any, Dict, Optional

from samcli.hook_packages.terraform.hooks.prepare.constants import (
REMOTE_DUMMY_VALUE,
TF_AWS_API_GATEWAY_AUTHORIZER,
TF_AWS_API_GATEWAY_INTEGRATION,
TF_AWS_API_GATEWAY_INTEGRATION_RESPONSE,
TF_AWS_API_GATEWAY_METHOD,
TF_AWS_API_GATEWAY_RESOURCE,
TF_AWS_API_GATEWAY_REST_API,
TF_AWS_API_GATEWAY_STAGE,
TF_AWS_LAMBDA_FUNCTION,
TF_AWS_LAMBDA_LAYER_VERSION,
)
from samcli.hook_packages.terraform.hooks.prepare.resource_linking import _resolve_resource_attribute
from samcli.hook_packages.terraform.hooks.prepare.resources.internal import (
INTERNAL_API_GATEWAY_INTEGRATION,
Expand All @@ -29,18 +41,6 @@

LOG = logging.getLogger(__name__)

REMOTE_DUMMY_VALUE = "<<REMOTE DUMMY VALUE - RAISE ERROR IF IT IS STILL THERE>>"
TF_AWS_LAMBDA_FUNCTION = "aws_lambda_function"
TF_AWS_LAMBDA_LAYER_VERSION = "aws_lambda_layer_version"

TF_AWS_API_GATEWAY_RESOURCE = "aws_api_gateway_resource"
TF_AWS_API_GATEWAY_REST_API = "aws_api_gateway_rest_api"
TF_AWS_API_GATEWAY_STAGE = "aws_api_gateway_stage"
TF_AWS_API_GATEWAY_METHOD = "aws_api_gateway_method"
TF_AWS_API_GATEWAY_INTEGRATION = "aws_api_gateway_integration"
TF_AWS_API_GATEWAY_AUTHORIZER = "aws_api_gateway_authorizer"
TF_AWS_API_GATEWAY_INTEGRATION_RESPONSE = "aws_api_gateway_method_response"


def _build_code_property(tf_properties: dict, resource: TFResource) -> Any:
"""
Expand Down
382 changes: 250 additions & 132 deletions samcli/hook_packages/terraform/hooks/prepare/resource_linking.py
View file
Open in desktop

Large diffs are not rendered by default.

16 changes: 14 additions & 2 deletions samcli/hook_packages/terraform/hooks/prepare/resources/resource_links.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from typing import List

from samcli.hook_packages.terraform.hooks.prepare.property_builder import (
from samcli.hook_packages.terraform.hooks.prepare.constants import (
TF_AWS_API_GATEWAY_AUTHORIZER,
TF_AWS_API_GATEWAY_INTEGRATION,
TF_AWS_API_GATEWAY_INTEGRATION_RESPONSE,
Expand All @@ -23,10 +23,14 @@
_link_gateway_method_to_gateway_resource,
_link_gateway_methods_to_gateway_rest_apis,
_link_gateway_resources_to_gateway_rest_apis,
_link_gateway_resources_to_parents,
_link_gateway_stage_to_rest_api,
_link_lambda_functions_to_layers,
)
from samcli.hook_packages.terraform.hooks.prepare.types import LinkingPairCaller
from samcli.hook_packages.terraform.hooks.prepare.types import (
LinkingMultipleDestinationsOptionsCaller,
LinkingPairCaller,
)

RESOURCE_LINKS: List[LinkingPairCaller] = [
LinkingPairCaller(
Expand Down Expand Up @@ -91,3 +95,11 @@
linking_func=_link_gateway_method_to_gateway_authorizer,
),
]

MULTIPLE_DESTINATIONS_RESOURCE_LINKS: List[LinkingMultipleDestinationsOptionsCaller] = [
LinkingMultipleDestinationsOptionsCaller(
source=TF_AWS_API_GATEWAY_RESOURCE,
destinations=[TF_AWS_API_GATEWAY_REST_API, TF_AWS_API_GATEWAY_RESOURCE],
linking_func=_link_gateway_resources_to_parents,
Comment on lines +100 to +103
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The onus is then on us to know in which cases there could be multiple destinations, correct? I'm assuming there's similar case with V2 APIs for nested resources that we missed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, I am going to review all links we have to make sure we did not miss anything else

),
]
2 changes: 1 addition & 1 deletion samcli/hook_packages/terraform/hooks/prepare/resources/resource_properties.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Module for getting the resource property mappings for various resource types"""
from typing import Dict

from samcli.hook_packages.terraform.hooks.prepare.property_builder import (
from samcli.hook_packages.terraform.hooks.prepare.constants import (
TF_AWS_API_GATEWAY_AUTHORIZER,
TF_AWS_API_GATEWAY_INTEGRATION,
TF_AWS_API_GATEWAY_INTEGRATION_RESPONSE,
Expand Down
39 changes: 28 additions & 11 deletions samcli/hook_packages/terraform/hooks/prepare/translate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,16 @@

from samcli.hook_packages.terraform.hooks.prepare.constants import (
CFN_CODE_PROPERTIES,
SAM_METADATA_RESOURCE_NAME_ATTRIBUTE,
)
from samcli.hook_packages.terraform.hooks.prepare.enrich import enrich_resources_and_generate_makefile
from samcli.hook_packages.terraform.hooks.prepare.property_builder import (
REMOTE_DUMMY_VALUE,
RESOURCE_TRANSLATOR_MAPPING,
SAM_METADATA_RESOURCE_NAME_ATTRIBUTE,
TF_AWS_API_GATEWAY_INTEGRATION,
TF_AWS_API_GATEWAY_INTEGRATION_RESPONSE,
TF_AWS_API_GATEWAY_METHOD,
TF_AWS_API_GATEWAY_REST_API,
)
from samcli.hook_packages.terraform.hooks.prepare.enrich import enrich_resources_and_generate_makefile
from samcli.hook_packages.terraform.hooks.prepare.property_builder import (
RESOURCE_TRANSLATOR_MAPPING,
PropertyBuilderMapping,
)
from samcli.hook_packages.terraform.hooks.prepare.resource_linking import (
Expand All @@ -31,7 +31,10 @@
add_integrations_to_methods,
)
from samcli.hook_packages.terraform.hooks.prepare.resources.internal import INTERNAL_PREFIX
from samcli.hook_packages.terraform.hooks.prepare.resources.resource_links import RESOURCE_LINKS
from samcli.hook_packages.terraform.hooks.prepare.resources.resource_links import (
MULTIPLE_DESTINATIONS_RESOURCE_LINKS,
RESOURCE_LINKS,
)
from samcli.hook_packages.terraform.hooks.prepare.resources.resource_properties import get_resource_property_mapping
from samcli.hook_packages.terraform.hooks.prepare.types import (
CodeResourceProperties,
Expand Down Expand Up @@ -339,11 +342,25 @@ def translate_to_cfn(tf_json: dict, output_directory_path: str, terraform_applic


def _handle_linking(resource_property_mapping: Dict[str, ResourceProperties]) -> None:
for links in RESOURCE_LINKS:
links.linking_func(
resource_property_mapping[links.source].terraform_config,
resource_property_mapping[links.source].cfn_resources,
resource_property_mapping[links.dest].terraform_resources,
for link in RESOURCE_LINKS:
link.linking_func(
resource_property_mapping[link.source].terraform_config,
resource_property_mapping[link.source].cfn_resources,
resource_property_mapping[link.dest].terraform_resources,
)

for multiple_destinations_link in MULTIPLE_DESTINATIONS_RESOURCE_LINKS:
destinations: Dict[str, Dict] = {}
for dest_resource_type in multiple_destinations_link.destinations:
destinations = {
**destinations,
**resource_property_mapping[dest_resource_type].terraform_resources,
}

multiple_destinations_link.linking_func(
resource_property_mapping[multiple_destinations_link.source].terraform_config,
resource_property_mapping[multiple_destinations_link.source].cfn_resources,
destinations,
)


Expand Down
6 changes: 6 additions & 0 deletions samcli/hook_packages/terraform/hooks/prepare/types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,12 @@ class LinkingPairCaller(NamedTuple):
linking_func: Callable[[Dict[str, TFResource], Dict[str, List], Dict[str, Dict]], None]


class LinkingMultipleDestinationsOptionsCaller(NamedTuple):
source: str
destinations: List[str]
linking_func: Callable[[Dict[str, TFResource], Dict[str, List], Dict[str, Dict[str, Dict]]], None]


class ResourceTranslationValidator:
"""
Base class for a validation class to be used when translating Terraform resources to a metadata file
Expand Down
38 changes: 30 additions & 8 deletions tests/integration/local/start_api/test_start_api_with_terraform_application.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,18 +94,29 @@ def tearDownClass(cls) -> None:
not CI_OVERRIDE,
"Skip Terraform test cases unless running in CI",
)
@parameterized_class(
[
{
"terraform_application": "terraform-v1-nested-apis",
"testing_urls": ["parent/hello", "parent"],
},
{
"terraform_application": "terraform-v1-api-simple",
"testing_urls": ["hello"],
},
]
)
@pytest.mark.flaky(reruns=3)
class TestStartApiTerraformApplication(TerraformStartApiIntegrationBase):
terraform_application = "terraform-v1-api-simple"

def setUp(self):
self.url = "http://127.0.0.1:{}".format(self.port)

def test_successful_request(self):
response = requests.get(self.url + "/hello", timeout=300)
for url in self.testing_urls:
response = requests.get(f"{self.url}/{url}", timeout=300)

self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {"message": "hello world"})
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {"message": "hello world"})


@skipIf(
Expand Down Expand Up @@ -172,11 +183,21 @@ def test_unsupported_limitations(self):
@pytest.mark.flaky(reruns=3)
@parameterized_class(
[
{
"terraform_application": "terraform-v1-nested-apis",
"testing_urls": ["parent/hello", "parent"],
},
{
"terraform_application": "terraform-v1-api-simple",
"testing_urls": ["hello"],
},
{
"terraform_application": "terraform-api-simple-multiple-resources-limitation",
"testing_urls": ["hello"],
},
{
"terraform_application": "terraform-api-simple-local-variables-limitation",
"testing_urls": ["hello"],
},
]
)
Expand All @@ -185,10 +206,11 @@ def setUp(self):
self.url = "http://127.0.0.1:{}".format(self.port)

def test_successful_request(self):
response = requests.get(self.url + "/hello", timeout=300)
for url in self.testing_urls:
response = requests.get(f"{self.url}/{url}", timeout=300)

self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {"message": "hello world"})
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {"message": "hello world"})


@skipIf(
Expand Down
40 changes: 30 additions & 10 deletions tests/integration/testdata/start_api/terraform/terraform-v1-api-simple/main.tf
View file
Open in desktop
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice catch for making the resource names unique

Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
provider "aws" {
}

resource "random_uuid" "unique_id" {
keepers = {
my_key = "my_key"
}
}

resource "aws_iam_role" "iam_for_lambda" {
name = "iam_for_lambda"
name = "iam_for_lambda_${random_uuid.unique_id.result}"

assume_role_policy = <<EOF
{
Expand All @@ -22,13 +28,14 @@ EOF
}

resource "aws_s3_bucket" "lambda_code_bucket" {
bucket = "lambda_code_bucket"
bucket = "lambda-code-bucket-${random_uuid.unique_id.result}"
}

resource "aws_s3_object" "s3_lambda_code" {
bucket = "lambda_code_bucket"
bucket = "lambda-code-bucket-${random_uuid.unique_id.result}"
key = "s3_lambda_code_key"
source = "HelloWorldFunction.zip"
depends_on = [aws_s3_bucket.lambda_code_bucket]
}

resource "aws_lambda_layer_version" "MyAwesomeLayer" {
Expand All @@ -38,25 +45,26 @@ resource "aws_lambda_layer_version" "MyAwesomeLayer" {
}

resource "aws_lambda_function" "HelloWorldFunction" {
s3_bucket = "lambda_code_bucket"
s3_bucket = "lambda-code-bucket-${random_uuid.unique_id.result}"
s3_key = "s3_lambda_code_key"
handler = "app.lambda_handler"
runtime = "python3.8"
function_name = "HelloWorldFunction"
function_name = "HelloWorldFunction-${random_uuid.unique_id.result}"
timeout = 500
role = aws_iam_role.iam_for_lambda.arn
layers = [aws_lambda_layer_version.MyAwesomeLayer.arn]
depends_on = [aws_s3_object.s3_lambda_code]
}

resource "aws_lambda_function" "HelloWorldFunction2" {
s3_bucket = "lambda_code_bucket"
s3_bucket = "lambda-code-bucket-${random_uuid.unique_id.result}"
s3_key = "s3_lambda_code_key"
handler = "app.lambda_handler"
runtime = "python3.8"
function_name = "HelloWorldFunction2"
function_name = "HelloWorldFunction2-${random_uuid.unique_id.result}"
timeout = 500
role = aws_iam_role.iam_for_lambda.arn
layers = ["arn:aws:lambda:us-east-1:178733185316:layer:01383708b0:1"]
depends_on = [aws_s3_object.s3_lambda_code]
}

resource "aws_api_gateway_rest_api" "MyDemoAPI" {
Expand Down Expand Up @@ -85,23 +93,35 @@ resource "aws_api_gateway_method" "PostMethod" {
}

resource "aws_api_gateway_stage" "MyDemoStage" {
stage_name = "prod"
stage_name = "prod-${random_uuid.unique_id.result}"
rest_api_id = aws_api_gateway_rest_api.MyDemoAPI.id
deployment_id = aws_api_gateway_deployment.MyDemoDeployment.id
}

resource "aws_api_gateway_deployment" "MyDemoDeployment" {
rest_api_id = aws_api_gateway_rest_api.MyDemoAPI.id
stage_name = "prod"
triggers = {
redeployment = sha1(jsonencode([
aws_api_gateway_resource.MyDemoResource.id,
aws_api_gateway_method.GetMethod.http_method,
aws_api_gateway_integration.MyDemoIntegration.id,
]))
}

lifecycle {
create_before_destroy = true
}
Comment on lines +103 to +113
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Curious what this configuration is for.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this part might be not needed, i was trying to fix some failures for terraform apply command, but in general the triggers is to let TF know when to update the stage resource even if the stage itself does not change, and lifecycle helped me to destroy the project when It was partially created.

}

resource "aws_api_gateway_integration" "MyDemoIntegration" {
rest_api_id = aws_api_gateway_rest_api.MyDemoAPI.id
resource_id = aws_api_gateway_resource.MyDemoResource.id
http_method = aws_api_gateway_method.GetMethod.http_method
integration_http_method = "POST"
type = "AWS_PROXY"
content_handling = "CONVERT_TO_TEXT"
uri = aws_lambda_function.HelloWorldFunction.invoke_arn
depends_on = [aws_api_gateway_method.GetMethod]
}

resource "aws_api_gateway_integration" "MyDemoIntegrationMock" {
Expand Down
Binary file added .../integration/testdata/start_api/terraform/terraform-v1-nested-apis/HelloWorldFunction.zip
View file
Open in desktop
Binary file not shown.
Loading