diff --git a/test/fixtures/results/integration/ref-no-value.json b/test/fixtures/results/integration/ref-no-value.json index d2beca5cf3..27e15f5294 100644 --- a/test/fixtures/results/integration/ref-no-value.json +++ b/test/fixtures/results/integration/ref-no-value.json @@ -61,7 +61,7 @@ }, { "Filename": "test/fixtures/templates/integration/ref-no-value.yaml", - "Id": "2ed7922b-1cd5-30ae-0c5f-918f69c9b782", + "Id": "93c41f0b-56b3-2349-8fb6-4be356befb25", "Level": "Error", "Location": { "End": { @@ -71,25 +71,26 @@ "Path": [ "Resources", "IamRole2", - "Properties" + "Properties", + "Ref" ], "Start": { "ColumnNumber": 5, "LineNumber": 26 } }, - "Message": "'AssumeRolePolicyDocument' is a required property", + "Message": "{'Ref': 'AWS::NoValue'} is not of type object", "ParentId": null, "Rule": { - "Description": "Make sure that Resources properties that are required exist", - "Id": "E3003", - "ShortDescription": "Required Resource properties are missing", - "Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required" + "Description": "Checks resource property values with Primitive Types for values that match those types.", + "Id": "E3012", + "ShortDescription": "Check resource properties values", + "Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type" } }, { "Filename": "test/fixtures/templates/integration/ref-no-value.yaml", - "Id": "edfc8e64-ad37-e697-8fb7-5c89d2ff1735", + "Id": "5714e7bb-7c4b-573d-88cb-cbda0df6276d", "Level": "Error", "Location": { "End": { @@ -99,20 +100,21 @@ "Path": [ "Resources", "CloudFront1", - "Properties" + "Properties", + "Ref" ], "Start": { "ColumnNumber": 5, "LineNumber": 39 } }, - "Message": "'DistributionConfig' is a required property", + "Message": "{'Ref': 'AWS::NoValue'} is not of type object", "ParentId": null, "Rule": { - "Description": "Make sure that Resources properties that are required exist", - "Id": "E3003", - "ShortDescription": "Required Resource properties are missing", - "Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required" + "Description": "Checks resource property values with Primitive Types for values that match those types.", + "Id": "E3012", + "ShortDescription": "Check resource properties values", + "Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type" } }, { diff --git a/test/fixtures/results/quickstart/nist_iam.json b/test/fixtures/results/quickstart/nist_iam.json index 92a45bf532..3ed569fc70 100644 --- a/test/fixtures/results/quickstart/nist_iam.json +++ b/test/fixtures/results/quickstart/nist_iam.json @@ -57,7 +57,7 @@ }, { "Filename": "test/fixtures/templates/quickstart/nist_iam.yaml", - "Id": "99273eea-4208-d5b5-e641-516b59429087", + "Id": "2902b820-10fb-7008-c4f2-b74be7c678f6", "Level": "Warning", "Location": { "End": { @@ -78,7 +78,7 @@ "LineNumber": 165 } }, - "Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack']", + "Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createthemeforstack', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deletethemeforstack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describethemeforstack', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack', 'updatethemeforstack']", "ParentId": null, "Rule": { "Description": "Check for valid IAM Permissions", diff --git a/test/integration/__init__.py b/test/integration/__init__.py index b45aa01b5b..9d57623163 100644 --- a/test/integration/__init__.py +++ b/test/integration/__init__.py @@ -6,11 +6,15 @@ import json import subprocess import unittest +from copy import deepcopy +from io import StringIO from pathlib import Path from typing import Any, Dict, List +from unittest.mock import patch from cfnlint.config import configure_logging from cfnlint.decode import cfn_yaml +from cfnlint.formatters import JsonFormatter from cfnlint.runner import Runner @@ -102,16 +106,22 @@ def run_module_integration_scenarios(self, config): for result in expected_results: result["Filename"] = str(Path(result.get("Filename"))) - template = cfn_yaml.load(filename) - - runner = Runner(config) - matches = list(runner.validate_template(filename, template)) - - # Only check that the error count matches as the formats are different - self.assertEqual( - len(expected_results), - len(matches), - "Expected {} failures, got {} on {}".format( - len(expected_results), matches, filename - ), - ) + # template = cfn_yaml.load(filename) + scenario_config = deepcopy(config) + scenario_config.cli_args.template_alt = [filename] + scenario_config.cli_args.format = "json" + + runner = Runner(scenario_config) + + print(f"Running test for {filename!r}") + with patch("sys.exit") as exit: + with patch("sys.stdout", new=StringIO()) as out: + runner.cli() + exit.assert_called_once_with(scenario.get("exit_code", 0)) + + output = json.loads(out.getvalue()) + self.assertEqual( + expected_results, + output, + f"Test for {filename!r} got results: {output!r}", + ) diff --git a/test/integration/test_good_templates.py b/test/integration/test_good_templates.py index 0fbdd5fff3..e799d5d8d1 100644 --- a/test/integration/test_good_templates.py +++ b/test/integration/test_good_templates.py @@ -40,96 +40,58 @@ class TestQuickStartTemplates(BaseCliTestCase): ), "results": [ { - "Filename": str( - Path( - "test/fixtures/templates/bad/transform_serverless_template.yaml" - ) - ), - "Id": "74181426-e865-10eb-96fd-908dfd30a358", + "Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml", + "Id": "9e05773a-b0d0-f157-2955-596d9bd54749", + "Level": "Error", "Location": { - "Start": {"ColumnNumber": 1, "LineNumber": 1}, "End": {"ColumnNumber": 2, "LineNumber": 1}, "Path": None, + "Start": {"ColumnNumber": 1, "LineNumber": 1}, }, + "Message": "Error transforming template: Resource with id [myFunctionMyTimer] is invalid. Missing required property 'Schedule'.", "ParentId": None, "Rule": { + "Description": "Errors found when performing transformation on the template", "Id": "E0001", - "Description": ( - "Errors found when performing transformation on the" - " template" - ), - "Source": ("https://github.com/aws-cloudformation/cfn-lint"), - "ShortDescription": ( - "Error found when transforming the template" - ), + "ShortDescription": "Error found when transforming the template", + "Source": "https://github.com/aws-cloudformation/cfn-lint", }, - "Level": "Error", - "Message": ( - "Error transforming template: Resource with id [AppName] is" - " invalid. Resource is missing the required [Location]" - " property." - ), }, { - "Filename": str( - Path( - "test/fixtures/templates/bad/transform_serverless_template.yaml" - ) - ), + "Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml", "Id": "fd751fa3-7d1f-e194-7108-eb08352814c8", + "Level": "Error", "Location": { - "Start": {"ColumnNumber": 1, "LineNumber": 1}, "End": {"ColumnNumber": 2, "LineNumber": 1}, "Path": None, + "Start": {"ColumnNumber": 1, "LineNumber": 1}, }, + "Message": "Error transforming template: Resource with id [ExampleLayer] is invalid. Missing required property 'ContentUri'.", "ParentId": None, "Rule": { + "Description": "Errors found when performing transformation on the template", "Id": "E0001", - "Description": ( - "Errors found when performing transformation on the" - " template" - ), - "Source": ("https://github.com/aws-cloudformation/cfn-lint"), - "ShortDescription": ( - "Error found when transforming the template" - ), + "ShortDescription": "Error found when transforming the template", + "Source": "https://github.com/aws-cloudformation/cfn-lint", }, - "Level": "Error", - "Message": ( - "Error transforming template: Resource with id [ExampleLayer]" - " is invalid. Missing required property 'ContentUri'." - ), }, { - "Filename": str( - Path( - "test/fixtures/templates/bad/transform_serverless_template.yaml" - ) - ), - "Id": "9e05773a-b0d0-f157-2955-596d9bd54749", + "Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml", + "Id": "74181426-e865-10eb-96fd-908dfd30a358", + "Level": "Error", "Location": { - "Start": {"ColumnNumber": 1, "LineNumber": 1}, "End": {"ColumnNumber": 2, "LineNumber": 1}, "Path": None, + "Start": {"ColumnNumber": 1, "LineNumber": 1}, }, + "Message": "Error transforming template: Resource with id [AppName] is invalid. Resource is missing the required [Location] property.", "ParentId": None, "Rule": { + "Description": "Errors found when performing transformation on the template", "Id": "E0001", - "Description": ( - "Errors found when performing transformation on the" - " template" - ), - "Source": ("https://github.com/aws-cloudformation/cfn-lint"), - "ShortDescription": ( - "Error found when transforming the template" - ), + "ShortDescription": "Error found when transforming the template", + "Source": "https://github.com/aws-cloudformation/cfn-lint", }, - "Level": "Error", - "Message": ( - "Error transforming template: Resource with id" - " [myFunctionMyTimer] is invalid. Missing required property" - " 'Schedule'." - ), }, ], "exit_code": 2, diff --git a/test/integration/test_integration_templates.py b/test/integration/test_integration_templates.py index 959b6b2221..18ef3ad0eb 100644 --- a/test/integration/test_integration_templates.py +++ b/test/integration/test_integration_templates.py @@ -24,21 +24,21 @@ class TestQuickStartTemplates(BaseCliTestCase): "exit_code": 0, }, { - "filename": ("test/fixtures/templates/integration/dynamic-references.yaml"), + "filename": "test/fixtures/templates/integration/dynamic-references.yaml", "results_filename": ( - "test/fixtures/results/integration/dynamic-references.json" + "test/fixtures/results/integration" "/dynamic-references.json" ), "exit_code": 2, }, { - "filename": ("test/fixtures/templates/integration/ref-no-value.yaml"), + "filename": "test/fixtures/templates/integration/ref-no-value.yaml", "results_filename": ("test/fixtures/results/integration/ref-no-value.json"), "exit_code": 2, }, { - "filename": ("test/fixtures/templates/integration/metdata.yaml"), + "filename": "test/fixtures/templates/integration/metdata.yaml", "results_filename": ("test/fixtures/results/integration/metadata.json"), - "exit_code": 2, + "exit_code": 4, }, ] diff --git a/test/integration/test_quickstart_templates.py b/test/integration/test_quickstart_templates.py index 6f7c55e41b..bd7d94e15c 100644 --- a/test/integration/test_quickstart_templates.py +++ b/test/integration/test_quickstart_templates.py @@ -30,14 +30,14 @@ class TestQuickStartTemplates(BaseCliTestCase): { "filename": "test/fixtures/templates/quickstart/nist_application.yaml", "results_filename": ( - "test/fixtures/results/quickstart/nist_application.json" + "test/fixtures/results/quickstart" "/nist_application.json" ), "exit_code": 14, }, { "filename": "test/fixtures/templates/quickstart/nist_config_rules.yaml", "results_filename": ( - "test/fixtures/results/quickstart/nist_config_rules.json" + "test/fixtures/results/quickstart/" "nist_config_rules.json" ), "exit_code": 6, }, @@ -54,21 +54,21 @@ class TestQuickStartTemplates(BaseCliTestCase): { "filename": "test/fixtures/templates/quickstart/nist_vpc_management.yaml", "results_filename": ( - "test/fixtures/results/quickstart/nist_vpc_management.json" + "test/fixtures/results/quickstart/" "nist_vpc_management.json" ), "exit_code": 14, }, { "filename": "test/fixtures/templates/quickstart/nist_vpc_production.yaml", "results_filename": ( - "test/fixtures/results/quickstart/nist_vpc_production.json" + "test/fixtures/results/quickstart/" "nist_vpc_production.json" ), "exit_code": 14, }, { "filename": "test/fixtures/templates/quickstart/openshift_master.yaml", "results_filename": ( - "test/fixtures/results/quickstart/openshift_master.json" + "test/fixtures/results/quickstart" "/openshift_master.json" ), "exit_code": 8, },