From d37cb6c7892e71c03f24254db209c3e934a1daa2 Mon Sep 17 00:00:00 2001 From: Kevin DeJong Date: Thu, 15 Aug 2024 09:55:56 -0700 Subject: [PATCH 1/4] Script to create release JSON schemas --- scripts/release_schemas/_translator.py | 95 ++++++++++++++++ scripts/release_schemas/generator.py | 151 +++++++++++++++++++++++++ 2 files changed, 246 insertions(+) create mode 100644 scripts/release_schemas/_translator.py create mode 100755 scripts/release_schemas/generator.py diff --git a/scripts/release_schemas/_translator.py b/scripts/release_schemas/_translator.py new file mode 100644 index 0000000000..18501c994c --- /dev/null +++ b/scripts/release_schemas/_translator.py @@ -0,0 +1,95 @@ +""" +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: MIT-0 +""" + +from __future__ import annotations + +# Translate cfn-lint unique keywords into json schema keywords +import logging +from collections import deque +from typing import Any, Iterator + +from cfnlint.schema import PROVIDER_SCHEMA_MANAGER + +logger = logging.getLogger(__name__) + + +def required_xor(properties: list[str]) -> dict[str, list[Any]]: + + return {"oneOf": [{"required": [p]} for p in properties]} + + +def dependent_excluded(properties: dict[str, list[str]]) -> dict[str, list[Any]]: + dependencies: dict[str, Any] = {"dependencies": {}} + for prop, exclusions in properties.items(): + dependencies["dependencies"][prop] = {"not": {"anyOf": []}} + for exclusion in exclusions: + dependencies["dependencies"][prop]["not"]["anyOf"].append( + {"required": exclusion} + ) + + return dependencies + + +_keywords = { + "requiredXor": required_xor, + "dependentExcluded": dependent_excluded, +} + + +def _find_keywords(schema: Any) -> Iterator[deque[str | int]]: + + if isinstance(schema, list): + for i, item in enumerate(schema): + for path in _find_keywords(item): + path.appendleft(i) + yield path + elif isinstance(schema, dict): + for key, value in schema.items(): + if key in _keywords: + yield deque([key, value]) + else: + for path in _find_keywords(value): + path.appendleft(key) + yield path + + +def translator(resource_type: str, region: str): + keywords = list( + _find_keywords( + PROVIDER_SCHEMA_MANAGER.get_resource_schema( + region=region, resource_type=resource_type + ).schema + ) + ) + + for keyword in keywords: + value = keyword.pop() + key = keyword.pop() + if not keyword: + path = "" + else: + path = f"/{'/'.join(str(k) for k in keyword)}" + + patch = [ + { + "op": "add", + "path": f"{path}/allOf", + "value": [], + } + ] + + logger.info(f"Patch {resource_type} add allOf for {key}") + PROVIDER_SCHEMA_MANAGER._schemas[region][resource_type].patch(patches=patch) + + patch = [ + { + "op": "remove", + "path": f"{path}/{key}", + }, + {"op": "add", "path": f"{path}/allOf/-", "value": _keywords[key](value)}, # type: ignore + ] + + logger.info(f"Patch {resource_type} replace for {key}") + PROVIDER_SCHEMA_MANAGER._schemas[region][resource_type].patch(patches=patch) diff --git a/scripts/release_schemas/generator.py b/scripts/release_schemas/generator.py new file mode 100755 index 0000000000..6000a93185 --- /dev/null +++ b/scripts/release_schemas/generator.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +""" +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +SPDX-License-Identifier: MIT-0 +""" +import logging +from collections import deque +from pathlib import Path + +import _translator + +from cfnlint.helpers import REGIONS, ToPy, format_json_string, load_plugins +from cfnlint.schema import PROVIDER_SCHEMA_MANAGER + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +def _get_schema_path(schema, path): + s = schema.schema + schema_path = deque([]) + while path: + key = path.popleft() + if key == "*": + schema_path.append("items") + s = s["items"] + else: + s = s["properties"][key] + schema_path.extend(["properties", key]) + + pointer = s.get("$ref") + if pointer: + _, s = schema.resolver.resolve(pointer) + schema_path = deque(pointer.split("/")[1:]) + + return schema_path + + +def _build_patch(path, patch): + if not path: + path_str = "/allOf" + else: + path_str = f"/{'/'.join(path)}/allOf" + + return ( + [ + { + "op": "add", + "path": path_str, + "value": [], + } + ], + [ + { + "op": "add", + "path": f"{path_str}/-", + "value": patch, + } + ], + ) + + +schemas = {} + +########################## +# +# Build the definitive list of all resource types across all regions +# +########################### + +for region in ["us-east-1"] + list((set(REGIONS) - set(["us-east-1"]))): + for resource_type in PROVIDER_SCHEMA_MANAGER.get_resource_types(region): + if resource_type in ["AWS::CDK::Metadata", "Module"]: + continue + if resource_type not in schemas: + schemas[resource_type] = region + + +########################## +# +# Merge in rule schemas into the resource schemas +# +########################### + +rules_folder = Path("src") / "cfnlint" / "rules" + +rules = load_plugins( + rules_folder, + name="CfnLintJsonSchema", + modules=( + "cfnlint.rules.jsonschema.CfnLintJsonSchema", + "cfnlint.rules.jsonschema.CfnLintJsonSchema.CfnLintJsonSchema", + ), +) + +for rule in rules: + if rule.__class__.__base__ == ( + "cfnlint.rules.jsonschema." + "CfnLintJsonSchemaRegional.CfnLintJsonSchemaRegional" + ): + continue + if not rule.id or rule.schema == {}: + continue + + for keyword in rule.keywords: + if not keyword.startswith("Resources/"): + continue + path = deque(keyword.split("/")) + + if len(path) < 3: + continue + + path.popleft() + resource_type = path.popleft() + resource_properties = path.popleft() + if resource_type not in schemas and resource_properties != "Properties": + continue + + schema_path = _get_schema_path( + PROVIDER_SCHEMA_MANAGER.get_resource_schema( + schemas[resource_type], resource_type + ), + path, + ) + all_of_patch, schema_patch = _build_patch(schema_path, rule.schema) + + PROVIDER_SCHEMA_MANAGER._schemas[schemas[resource_type]][resource_type].patch( + patches=all_of_patch + ) + PROVIDER_SCHEMA_MANAGER._schemas[schemas[resource_type]][resource_type].patch( + patches=schema_patch + ) + + logger.info(f"Patch {rule.id} for {resource_type} in {schemas[resource_type]}") + + +for resource_type, region in schemas.items(): + rt_py = ToPy(resource_type) + + _translator.translator(resource_type, region) + + with open(f"local/release_schemas/{rt_py.py}.json", "w") as f: + f.write( + format_json_string( + PROVIDER_SCHEMA_MANAGER.get_resource_schema( + region, resource_type + ).schema + ) + ) From 5f7d05c0f0939a4ee2b87404e9c916549d9374f1 Mon Sep 17 00:00:00 2001 From: Kevin DeJong Date: Wed, 21 Aug 2024 10:40:03 -0700 Subject: [PATCH 2/4] Build release artfiacts --- scripts/release_schemas/generator.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/release_schemas/generator.py b/scripts/release_schemas/generator.py index 6000a93185..987441a6ae 100755 --- a/scripts/release_schemas/generator.py +++ b/scripts/release_schemas/generator.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: MIT-0 """ import logging +import tarfile from collections import deque from pathlib import Path @@ -136,12 +137,29 @@ def _build_patch(path, patch): logger.info(f"Patch {rule.id} for {resource_type} in {schemas[resource_type]}") +build_dir = Path("build") +schemas_dir = build_dir / "schemas" +schemas_cfnlint_dir = schemas_dir / "cfnlint" +schemas_cfnlint_dir.mkdir(parents=True, exist_ok=True) + +schemas_draft7_dir = schemas_dir / "draft7" +schemas_draft7_dir.mkdir(parents=True, exist_ok=True) + for resource_type, region in schemas.items(): rt_py = ToPy(resource_type) + with open(schemas_cfnlint_dir / f"{rt_py.py}.json", "w") as f: + f.write( + format_json_string( + PROVIDER_SCHEMA_MANAGER.get_resource_schema( + region, resource_type + ).schema + ) + ) + _translator.translator(resource_type, region) - with open(f"local/release_schemas/{rt_py.py}.json", "w") as f: + with open(schemas_draft7_dir / f"{rt_py.py}.json", "w") as f: f.write( format_json_string( PROVIDER_SCHEMA_MANAGER.get_resource_schema( @@ -149,3 +167,10 @@ def _build_patch(path, patch): ).schema ) ) + +logger.info("Create schema package") +with tarfile.open(build_dir / "schemas-cfnlint.zip", "w:gz") as tar: + tar.add(schemas_cfnlint_dir, arcname="schemas") + +with tarfile.open(build_dir / "schemas-draft7.zip", "w:gz") as tar: + tar.add(schemas_draft7_dir, arcname="schemas") From bca28ead7a4a0780adb927923d8d612054dd0576 Mon Sep 17 00:00:00 2001 From: Kevin DeJong Date: Wed, 21 Aug 2024 15:10:30 -0700 Subject: [PATCH 3/4] Build a better Changelog --- scripts/release_schemas/_translator.py | 95 ------------- scripts/release_schemas/generator.py | 176 ------------------------- 2 files changed, 271 deletions(-) delete mode 100644 scripts/release_schemas/_translator.py delete mode 100755 scripts/release_schemas/generator.py diff --git a/scripts/release_schemas/_translator.py b/scripts/release_schemas/_translator.py deleted file mode 100644 index 18501c994c..0000000000 --- a/scripts/release_schemas/_translator.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -SPDX-License-Identifier: MIT-0 -""" - -from __future__ import annotations - -# Translate cfn-lint unique keywords into json schema keywords -import logging -from collections import deque -from typing import Any, Iterator - -from cfnlint.schema import PROVIDER_SCHEMA_MANAGER - -logger = logging.getLogger(__name__) - - -def required_xor(properties: list[str]) -> dict[str, list[Any]]: - - return {"oneOf": [{"required": [p]} for p in properties]} - - -def dependent_excluded(properties: dict[str, list[str]]) -> dict[str, list[Any]]: - dependencies: dict[str, Any] = {"dependencies": {}} - for prop, exclusions in properties.items(): - dependencies["dependencies"][prop] = {"not": {"anyOf": []}} - for exclusion in exclusions: - dependencies["dependencies"][prop]["not"]["anyOf"].append( - {"required": exclusion} - ) - - return dependencies - - -_keywords = { - "requiredXor": required_xor, - "dependentExcluded": dependent_excluded, -} - - -def _find_keywords(schema: Any) -> Iterator[deque[str | int]]: - - if isinstance(schema, list): - for i, item in enumerate(schema): - for path in _find_keywords(item): - path.appendleft(i) - yield path - elif isinstance(schema, dict): - for key, value in schema.items(): - if key in _keywords: - yield deque([key, value]) - else: - for path in _find_keywords(value): - path.appendleft(key) - yield path - - -def translator(resource_type: str, region: str): - keywords = list( - _find_keywords( - PROVIDER_SCHEMA_MANAGER.get_resource_schema( - region=region, resource_type=resource_type - ).schema - ) - ) - - for keyword in keywords: - value = keyword.pop() - key = keyword.pop() - if not keyword: - path = "" - else: - path = f"/{'/'.join(str(k) for k in keyword)}" - - patch = [ - { - "op": "add", - "path": f"{path}/allOf", - "value": [], - } - ] - - logger.info(f"Patch {resource_type} add allOf for {key}") - PROVIDER_SCHEMA_MANAGER._schemas[region][resource_type].patch(patches=patch) - - patch = [ - { - "op": "remove", - "path": f"{path}/{key}", - }, - {"op": "add", "path": f"{path}/allOf/-", "value": _keywords[key](value)}, # type: ignore - ] - - logger.info(f"Patch {resource_type} replace for {key}") - PROVIDER_SCHEMA_MANAGER._schemas[region][resource_type].patch(patches=patch) diff --git a/scripts/release_schemas/generator.py b/scripts/release_schemas/generator.py deleted file mode 100755 index 987441a6ae..0000000000 --- a/scripts/release_schemas/generator.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python -""" -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -SPDX-License-Identifier: MIT-0 -""" -import logging -import tarfile -from collections import deque -from pathlib import Path - -import _translator - -from cfnlint.helpers import REGIONS, ToPy, format_json_string, load_plugins -from cfnlint.schema import PROVIDER_SCHEMA_MANAGER - -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" -) -logger = logging.getLogger(__name__) - - -def _get_schema_path(schema, path): - s = schema.schema - schema_path = deque([]) - while path: - key = path.popleft() - if key == "*": - schema_path.append("items") - s = s["items"] - else: - s = s["properties"][key] - schema_path.extend(["properties", key]) - - pointer = s.get("$ref") - if pointer: - _, s = schema.resolver.resolve(pointer) - schema_path = deque(pointer.split("/")[1:]) - - return schema_path - - -def _build_patch(path, patch): - if not path: - path_str = "/allOf" - else: - path_str = f"/{'/'.join(path)}/allOf" - - return ( - [ - { - "op": "add", - "path": path_str, - "value": [], - } - ], - [ - { - "op": "add", - "path": f"{path_str}/-", - "value": patch, - } - ], - ) - - -schemas = {} - -########################## -# -# Build the definitive list of all resource types across all regions -# -########################### - -for region in ["us-east-1"] + list((set(REGIONS) - set(["us-east-1"]))): - for resource_type in PROVIDER_SCHEMA_MANAGER.get_resource_types(region): - if resource_type in ["AWS::CDK::Metadata", "Module"]: - continue - if resource_type not in schemas: - schemas[resource_type] = region - - -########################## -# -# Merge in rule schemas into the resource schemas -# -########################### - -rules_folder = Path("src") / "cfnlint" / "rules" - -rules = load_plugins( - rules_folder, - name="CfnLintJsonSchema", - modules=( - "cfnlint.rules.jsonschema.CfnLintJsonSchema", - "cfnlint.rules.jsonschema.CfnLintJsonSchema.CfnLintJsonSchema", - ), -) - -for rule in rules: - if rule.__class__.__base__ == ( - "cfnlint.rules.jsonschema." - "CfnLintJsonSchemaRegional.CfnLintJsonSchemaRegional" - ): - continue - if not rule.id or rule.schema == {}: - continue - - for keyword in rule.keywords: - if not keyword.startswith("Resources/"): - continue - path = deque(keyword.split("/")) - - if len(path) < 3: - continue - - path.popleft() - resource_type = path.popleft() - resource_properties = path.popleft() - if resource_type not in schemas and resource_properties != "Properties": - continue - - schema_path = _get_schema_path( - PROVIDER_SCHEMA_MANAGER.get_resource_schema( - schemas[resource_type], resource_type - ), - path, - ) - all_of_patch, schema_patch = _build_patch(schema_path, rule.schema) - - PROVIDER_SCHEMA_MANAGER._schemas[schemas[resource_type]][resource_type].patch( - patches=all_of_patch - ) - PROVIDER_SCHEMA_MANAGER._schemas[schemas[resource_type]][resource_type].patch( - patches=schema_patch - ) - - logger.info(f"Patch {rule.id} for {resource_type} in {schemas[resource_type]}") - - -build_dir = Path("build") -schemas_dir = build_dir / "schemas" -schemas_cfnlint_dir = schemas_dir / "cfnlint" -schemas_cfnlint_dir.mkdir(parents=True, exist_ok=True) - -schemas_draft7_dir = schemas_dir / "draft7" -schemas_draft7_dir.mkdir(parents=True, exist_ok=True) - -for resource_type, region in schemas.items(): - rt_py = ToPy(resource_type) - - with open(schemas_cfnlint_dir / f"{rt_py.py}.json", "w") as f: - f.write( - format_json_string( - PROVIDER_SCHEMA_MANAGER.get_resource_schema( - region, resource_type - ).schema - ) - ) - - _translator.translator(resource_type, region) - - with open(schemas_draft7_dir / f"{rt_py.py}.json", "w") as f: - f.write( - format_json_string( - PROVIDER_SCHEMA_MANAGER.get_resource_schema( - region, resource_type - ).schema - ) - ) - -logger.info("Create schema package") -with tarfile.open(build_dir / "schemas-cfnlint.zip", "w:gz") as tar: - tar.add(schemas_cfnlint_dir, arcname="schemas") - -with tarfile.open(build_dir / "schemas-draft7.zip", "w:gz") as tar: - tar.add(schemas_draft7_dir, arcname="schemas") From dbc6ab02bbd86eb93ff878d61e4b9a31e66284ae Mon Sep 17 00:00:00 2001 From: Kevin DeJong Date: Thu, 22 Aug 2024 23:12:11 -0700 Subject: [PATCH 4/4] fix integration testing to make it actually work --- .../results/integration/ref-no-value.json | 30 +++---- .../fixtures/results/quickstart/nist_iam.json | 4 +- test/integration/__init__.py | 36 +++++--- test/integration/test_good_templates.py | 84 +++++-------------- .../integration/test_integration_templates.py | 10 +-- test/integration/test_quickstart_templates.py | 10 +-- 6 files changed, 74 insertions(+), 100 deletions(-) diff --git a/test/fixtures/results/integration/ref-no-value.json b/test/fixtures/results/integration/ref-no-value.json index d2beca5cf3..27e15f5294 100644 --- a/test/fixtures/results/integration/ref-no-value.json +++ b/test/fixtures/results/integration/ref-no-value.json @@ -61,7 +61,7 @@ }, { "Filename": "test/fixtures/templates/integration/ref-no-value.yaml", - "Id": "2ed7922b-1cd5-30ae-0c5f-918f69c9b782", + "Id": "93c41f0b-56b3-2349-8fb6-4be356befb25", "Level": "Error", "Location": { "End": { @@ -71,25 +71,26 @@ "Path": [ "Resources", "IamRole2", - "Properties" + "Properties", + "Ref" ], "Start": { "ColumnNumber": 5, "LineNumber": 26 } }, - "Message": "'AssumeRolePolicyDocument' is a required property", + "Message": "{'Ref': 'AWS::NoValue'} is not of type object", "ParentId": null, "Rule": { - "Description": "Make sure that Resources properties that are required exist", - "Id": "E3003", - "ShortDescription": "Required Resource properties are missing", - "Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required" + "Description": "Checks resource property values with Primitive Types for values that match those types.", + "Id": "E3012", + "ShortDescription": "Check resource properties values", + "Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type" } }, { "Filename": "test/fixtures/templates/integration/ref-no-value.yaml", - "Id": "edfc8e64-ad37-e697-8fb7-5c89d2ff1735", + "Id": "5714e7bb-7c4b-573d-88cb-cbda0df6276d", "Level": "Error", "Location": { "End": { @@ -99,20 +100,21 @@ "Path": [ "Resources", "CloudFront1", - "Properties" + "Properties", + "Ref" ], "Start": { "ColumnNumber": 5, "LineNumber": 39 } }, - "Message": "'DistributionConfig' is a required property", + "Message": "{'Ref': 'AWS::NoValue'} is not of type object", "ParentId": null, "Rule": { - "Description": "Make sure that Resources properties that are required exist", - "Id": "E3003", - "ShortDescription": "Required Resource properties are missing", - "Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required" + "Description": "Checks resource property values with Primitive Types for values that match those types.", + "Id": "E3012", + "ShortDescription": "Check resource properties values", + "Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type" } }, { diff --git a/test/fixtures/results/quickstart/nist_iam.json b/test/fixtures/results/quickstart/nist_iam.json index 92a45bf532..3ed569fc70 100644 --- a/test/fixtures/results/quickstart/nist_iam.json +++ b/test/fixtures/results/quickstart/nist_iam.json @@ -57,7 +57,7 @@ }, { "Filename": "test/fixtures/templates/quickstart/nist_iam.yaml", - "Id": "99273eea-4208-d5b5-e641-516b59429087", + "Id": "2902b820-10fb-7008-c4f2-b74be7c678f6", "Level": "Warning", "Location": { "End": { @@ -78,7 +78,7 @@ "LineNumber": 165 } }, - "Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack']", + "Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createthemeforstack', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deletethemeforstack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describethemeforstack', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack', 'updatethemeforstack']", "ParentId": null, "Rule": { "Description": "Check for valid IAM Permissions", diff --git a/test/integration/__init__.py b/test/integration/__init__.py index b45aa01b5b..9d57623163 100644 --- a/test/integration/__init__.py +++ b/test/integration/__init__.py @@ -6,11 +6,15 @@ import json import subprocess import unittest +from copy import deepcopy +from io import StringIO from pathlib import Path from typing import Any, Dict, List +from unittest.mock import patch from cfnlint.config import configure_logging from cfnlint.decode import cfn_yaml +from cfnlint.formatters import JsonFormatter from cfnlint.runner import Runner @@ -102,16 +106,22 @@ def run_module_integration_scenarios(self, config): for result in expected_results: result["Filename"] = str(Path(result.get("Filename"))) - template = cfn_yaml.load(filename) - - runner = Runner(config) - matches = list(runner.validate_template(filename, template)) - - # Only check that the error count matches as the formats are different - self.assertEqual( - len(expected_results), - len(matches), - "Expected {} failures, got {} on {}".format( - len(expected_results), matches, filename - ), - ) + # template = cfn_yaml.load(filename) + scenario_config = deepcopy(config) + scenario_config.cli_args.template_alt = [filename] + scenario_config.cli_args.format = "json" + + runner = Runner(scenario_config) + + print(f"Running test for {filename!r}") + with patch("sys.exit") as exit: + with patch("sys.stdout", new=StringIO()) as out: + runner.cli() + exit.assert_called_once_with(scenario.get("exit_code", 0)) + + output = json.loads(out.getvalue()) + self.assertEqual( + expected_results, + output, + f"Test for {filename!r} got results: {output!r}", + ) diff --git a/test/integration/test_good_templates.py b/test/integration/test_good_templates.py index 0fbdd5fff3..e799d5d8d1 100644 --- a/test/integration/test_good_templates.py +++ b/test/integration/test_good_templates.py @@ -40,96 +40,58 @@ class TestQuickStartTemplates(BaseCliTestCase): ), "results": [ { - "Filename": str( - Path( - "test/fixtures/templates/bad/transform_serverless_template.yaml" - ) - ), - "Id": "74181426-e865-10eb-96fd-908dfd30a358", + "Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml", + "Id": "9e05773a-b0d0-f157-2955-596d9bd54749", + "Level": "Error", "Location": { - "Start": {"ColumnNumber": 1, "LineNumber": 1}, "End": {"ColumnNumber": 2, "LineNumber": 1}, "Path": None, + "Start": {"ColumnNumber": 1, "LineNumber": 1}, }, + "Message": "Error transforming template: Resource with id [myFunctionMyTimer] is invalid. Missing required property 'Schedule'.", "ParentId": None, "Rule": { + "Description": "Errors found when performing transformation on the template", "Id": "E0001", - "Description": ( - "Errors found when performing transformation on the" - " template" - ), - "Source": ("https://github.com/aws-cloudformation/cfn-lint"), - "ShortDescription": ( - "Error found when transforming the template" - ), + "ShortDescription": "Error found when transforming the template", + "Source": "https://github.com/aws-cloudformation/cfn-lint", }, - "Level": "Error", - "Message": ( - "Error transforming template: Resource with id [AppName] is" - " invalid. Resource is missing the required [Location]" - " property." - ), }, { - "Filename": str( - Path( - "test/fixtures/templates/bad/transform_serverless_template.yaml" - ) - ), + "Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml", "Id": "fd751fa3-7d1f-e194-7108-eb08352814c8", + "Level": "Error", "Location": { - "Start": {"ColumnNumber": 1, "LineNumber": 1}, "End": {"ColumnNumber": 2, "LineNumber": 1}, "Path": None, + "Start": {"ColumnNumber": 1, "LineNumber": 1}, }, + "Message": "Error transforming template: Resource with id [ExampleLayer] is invalid. Missing required property 'ContentUri'.", "ParentId": None, "Rule": { + "Description": "Errors found when performing transformation on the template", "Id": "E0001", - "Description": ( - "Errors found when performing transformation on the" - " template" - ), - "Source": ("https://github.com/aws-cloudformation/cfn-lint"), - "ShortDescription": ( - "Error found when transforming the template" - ), + "ShortDescription": "Error found when transforming the template", + "Source": "https://github.com/aws-cloudformation/cfn-lint", }, - "Level": "Error", - "Message": ( - "Error transforming template: Resource with id [ExampleLayer]" - " is invalid. Missing required property 'ContentUri'." - ), }, { - "Filename": str( - Path( - "test/fixtures/templates/bad/transform_serverless_template.yaml" - ) - ), - "Id": "9e05773a-b0d0-f157-2955-596d9bd54749", + "Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml", + "Id": "74181426-e865-10eb-96fd-908dfd30a358", + "Level": "Error", "Location": { - "Start": {"ColumnNumber": 1, "LineNumber": 1}, "End": {"ColumnNumber": 2, "LineNumber": 1}, "Path": None, + "Start": {"ColumnNumber": 1, "LineNumber": 1}, }, + "Message": "Error transforming template: Resource with id [AppName] is invalid. Resource is missing the required [Location] property.", "ParentId": None, "Rule": { + "Description": "Errors found when performing transformation on the template", "Id": "E0001", - "Description": ( - "Errors found when performing transformation on the" - " template" - ), - "Source": ("https://github.com/aws-cloudformation/cfn-lint"), - "ShortDescription": ( - "Error found when transforming the template" - ), + "ShortDescription": "Error found when transforming the template", + "Source": "https://github.com/aws-cloudformation/cfn-lint", }, - "Level": "Error", - "Message": ( - "Error transforming template: Resource with id" - " [myFunctionMyTimer] is invalid. Missing required property" - " 'Schedule'." - ), }, ], "exit_code": 2, diff --git a/test/integration/test_integration_templates.py b/test/integration/test_integration_templates.py index 959b6b2221..18ef3ad0eb 100644 --- a/test/integration/test_integration_templates.py +++ b/test/integration/test_integration_templates.py @@ -24,21 +24,21 @@ class TestQuickStartTemplates(BaseCliTestCase): "exit_code": 0, }, { - "filename": ("test/fixtures/templates/integration/dynamic-references.yaml"), + "filename": "test/fixtures/templates/integration/dynamic-references.yaml", "results_filename": ( - "test/fixtures/results/integration/dynamic-references.json" + "test/fixtures/results/integration" "/dynamic-references.json" ), "exit_code": 2, }, { - "filename": ("test/fixtures/templates/integration/ref-no-value.yaml"), + "filename": "test/fixtures/templates/integration/ref-no-value.yaml", "results_filename": ("test/fixtures/results/integration/ref-no-value.json"), "exit_code": 2, }, { - "filename": ("test/fixtures/templates/integration/metdata.yaml"), + "filename": "test/fixtures/templates/integration/metdata.yaml", "results_filename": ("test/fixtures/results/integration/metadata.json"), - "exit_code": 2, + "exit_code": 4, }, ] diff --git a/test/integration/test_quickstart_templates.py b/test/integration/test_quickstart_templates.py index 6f7c55e41b..bd7d94e15c 100644 --- a/test/integration/test_quickstart_templates.py +++ b/test/integration/test_quickstart_templates.py @@ -30,14 +30,14 @@ class TestQuickStartTemplates(BaseCliTestCase): { "filename": "test/fixtures/templates/quickstart/nist_application.yaml", "results_filename": ( - "test/fixtures/results/quickstart/nist_application.json" + "test/fixtures/results/quickstart" "/nist_application.json" ), "exit_code": 14, }, { "filename": "test/fixtures/templates/quickstart/nist_config_rules.yaml", "results_filename": ( - "test/fixtures/results/quickstart/nist_config_rules.json" + "test/fixtures/results/quickstart/" "nist_config_rules.json" ), "exit_code": 6, }, @@ -54,21 +54,21 @@ class TestQuickStartTemplates(BaseCliTestCase): { "filename": "test/fixtures/templates/quickstart/nist_vpc_management.yaml", "results_filename": ( - "test/fixtures/results/quickstart/nist_vpc_management.json" + "test/fixtures/results/quickstart/" "nist_vpc_management.json" ), "exit_code": 14, }, { "filename": "test/fixtures/templates/quickstart/nist_vpc_production.yaml", "results_filename": ( - "test/fixtures/results/quickstart/nist_vpc_production.json" + "test/fixtures/results/quickstart/" "nist_vpc_production.json" ), "exit_code": 14, }, { "filename": "test/fixtures/templates/quickstart/openshift_master.yaml", "results_filename": ( - "test/fixtures/results/quickstart/openshift_master.json" + "test/fixtures/results/quickstart" "/openshift_master.json" ), "exit_code": 8, },