Skip to content

Commit

Permalink
Fix integration tests (#3615)
Browse files Browse the repository at this point in the history 
* fix integration testing to make it actually work
  • Loading branch information
@kddejong
kddejong authored Aug 23, 2024
1 parent fccb4d3 commit 37aaa9e
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 100 deletions.
30 changes: 16 additions & 14 deletions test/fixtures/results/integration/ref-no-value.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
},
{
"Filename": "test/fixtures/templates/integration/ref-no-value.yaml",
"Id": "2ed7922b-1cd5-30ae-0c5f-918f69c9b782",
"Id": "93c41f0b-56b3-2349-8fb6-4be356befb25",
"Level": "Error",
"Location": {
"End": {
Expand All @@ -71,25 +71,26 @@
"Path": [
"Resources",
"IamRole2",
"Properties"
"Properties",
"Ref"
],
"Start": {
"ColumnNumber": 5,
"LineNumber": 26
}
},
"Message": "'AssumeRolePolicyDocument' is a required property",
"Message": "{'Ref': 'AWS::NoValue'} is not of type object",
"ParentId": null,
"Rule": {
"Description": "Make sure that Resources properties that are required exist",
"Id": "E3003",
"ShortDescription": "Required Resource properties are missing",
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required"
"Description": "Checks resource property values with Primitive Types for values that match those types.",
"Id": "E3012",
"ShortDescription": "Check resource properties values",
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type"
}
},
{
"Filename": "test/fixtures/templates/integration/ref-no-value.yaml",
"Id": "edfc8e64-ad37-e697-8fb7-5c89d2ff1735",
"Id": "5714e7bb-7c4b-573d-88cb-cbda0df6276d",
"Level": "Error",
"Location": {
"End": {
Expand All @@ -99,20 +100,21 @@
"Path": [
"Resources",
"CloudFront1",
"Properties"
"Properties",
"Ref"
],
"Start": {
"ColumnNumber": 5,
"LineNumber": 39
}
},
"Message": "'DistributionConfig' is a required property",
"Message": "{'Ref': 'AWS::NoValue'} is not of type object",
"ParentId": null,
"Rule": {
"Description": "Make sure that Resources properties that are required exist",
"Id": "E3003",
"ShortDescription": "Required Resource properties are missing",
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required"
"Description": "Checks resource property values with Primitive Types for values that match those types.",
"Id": "E3012",
"ShortDescription": "Check resource properties values",
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type"
}
},
{
Expand Down
4 changes: 2 additions & 2 deletions test/fixtures/results/quickstart/nist_iam.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
},
{
"Filename": "test/fixtures/templates/quickstart/nist_iam.yaml",
"Id": "99273eea-4208-d5b5-e641-516b59429087",
"Id": "2902b820-10fb-7008-c4f2-b74be7c678f6",
"Level": "Warning",
"Location": {
"End": {
Expand All @@ -78,7 +78,7 @@
"LineNumber": 165
}
},
"Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack']",
"Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createthemeforstack', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deletethemeforstack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describethemeforstack', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack', 'updatethemeforstack']",
"ParentId": null,
"Rule": {
"Description": "Check for valid IAM Permissions",
Expand Down
36 changes: 23 additions & 13 deletions test/integration/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,15 @@
import json
import subprocess
import unittest
from copy import deepcopy
from io import StringIO
from pathlib import Path
from typing import Any, Dict, List
from unittest.mock import patch

from cfnlint.config import configure_logging
from cfnlint.decode import cfn_yaml
from cfnlint.formatters import JsonFormatter
from cfnlint.runner import Runner


Expand Down Expand Up @@ -102,16 +106,22 @@ def run_module_integration_scenarios(self, config):
for result in expected_results:
result["Filename"] = str(Path(result.get("Filename")))

template = cfn_yaml.load(filename)

runner = Runner(config)
matches = list(runner.validate_template(filename, template))

# Only check that the error count matches as the formats are different
self.assertEqual(
len(expected_results),
len(matches),
"Expected {} failures, got {} on {}".format(
len(expected_results), matches, filename
),
)
# template = cfn_yaml.load(filename)
scenario_config = deepcopy(config)
scenario_config.cli_args.template_alt = [filename]
scenario_config.cli_args.format = "json"

runner = Runner(scenario_config)

print(f"Running test for {filename!r}")
with patch("sys.exit") as exit:
with patch("sys.stdout", new=StringIO()) as out:
runner.cli()
exit.assert_called_once_with(scenario.get("exit_code", 0))

output = json.loads(out.getvalue())
self.assertEqual(
expected_results,
output,
f"Test for {filename!r} got results: {output!r}",
)
84 changes: 23 additions & 61 deletions test/integration/test_good_templates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,96 +40,58 @@ class TestQuickStartTemplates(BaseCliTestCase):
),
"results": [
{
"Filename": str(
Path(
"test/fixtures/templates/bad/transform_serverless_template.yaml"
)
),
"Id": "74181426-e865-10eb-96fd-908dfd30a358",
"Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml",
"Id": "9e05773a-b0d0-f157-2955-596d9bd54749",
"Level": "Error",
"Location": {
"Start": {"ColumnNumber": 1, "LineNumber": 1},
"End": {"ColumnNumber": 2, "LineNumber": 1},
"Path": None,
"Start": {"ColumnNumber": 1, "LineNumber": 1},
},
"Message": "Error transforming template: Resource with id [myFunctionMyTimer] is invalid. Missing required property 'Schedule'.",
"ParentId": None,
"Rule": {
"Description": "Errors found when performing transformation on the template",
"Id": "E0001",
"Description": (
"Errors found when performing transformation on the"
" template"
),
"Source": ("https://github.com/aws-cloudformation/cfn-lint"),
"ShortDescription": (
"Error found when transforming the template"
),
"ShortDescription": "Error found when transforming the template",
"Source": "https://github.com/aws-cloudformation/cfn-lint",
},
"Level": "Error",
"Message": (
"Error transforming template: Resource with id [AppName] is"
" invalid. Resource is missing the required [Location]"
" property."
),
},
{
"Filename": str(
Path(
"test/fixtures/templates/bad/transform_serverless_template.yaml"
)
),
"Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml",
"Id": "fd751fa3-7d1f-e194-7108-eb08352814c8",
"Level": "Error",
"Location": {
"Start": {"ColumnNumber": 1, "LineNumber": 1},
"End": {"ColumnNumber": 2, "LineNumber": 1},
"Path": None,
"Start": {"ColumnNumber": 1, "LineNumber": 1},
},
"Message": "Error transforming template: Resource with id [ExampleLayer] is invalid. Missing required property 'ContentUri'.",
"ParentId": None,
"Rule": {
"Description": "Errors found when performing transformation on the template",
"Id": "E0001",
"Description": (
"Errors found when performing transformation on the"
" template"
),
"Source": ("https://github.com/aws-cloudformation/cfn-lint"),
"ShortDescription": (
"Error found when transforming the template"
),
"ShortDescription": "Error found when transforming the template",
"Source": "https://github.com/aws-cloudformation/cfn-lint",
},
"Level": "Error",
"Message": (
"Error transforming template: Resource with id [ExampleLayer]"
" is invalid. Missing required property 'ContentUri'."
),
},
{
"Filename": str(
Path(
"test/fixtures/templates/bad/transform_serverless_template.yaml"
)
),
"Id": "9e05773a-b0d0-f157-2955-596d9bd54749",
"Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml",
"Id": "74181426-e865-10eb-96fd-908dfd30a358",
"Level": "Error",
"Location": {
"Start": {"ColumnNumber": 1, "LineNumber": 1},
"End": {"ColumnNumber": 2, "LineNumber": 1},
"Path": None,
"Start": {"ColumnNumber": 1, "LineNumber": 1},
},
"Message": "Error transforming template: Resource with id [AppName] is invalid. Resource is missing the required [Location] property.",
"ParentId": None,
"Rule": {
"Description": "Errors found when performing transformation on the template",
"Id": "E0001",
"Description": (
"Errors found when performing transformation on the"
" template"
),
"Source": ("https://github.com/aws-cloudformation/cfn-lint"),
"ShortDescription": (
"Error found when transforming the template"
),
"ShortDescription": "Error found when transforming the template",
"Source": "https://github.com/aws-cloudformation/cfn-lint",
},
"Level": "Error",
"Message": (
"Error transforming template: Resource with id"
" [myFunctionMyTimer] is invalid. Missing required property"
" 'Schedule'."
),
},
],
"exit_code": 2,
Expand Down
10 changes: 5 additions & 5 deletions test/integration/test_integration_templates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,21 @@ class TestQuickStartTemplates(BaseCliTestCase):
"exit_code": 0,
},
{
"filename": ("test/fixtures/templates/integration/dynamic-references.yaml"),
"filename": "test/fixtures/templates/integration/dynamic-references.yaml",
"results_filename": (
"test/fixtures/results/integration/dynamic-references.json"
"test/fixtures/results/integration" "/dynamic-references.json"
),
"exit_code": 2,
},
{
"filename": ("test/fixtures/templates/integration/ref-no-value.yaml"),
"filename": "test/fixtures/templates/integration/ref-no-value.yaml",
"results_filename": ("test/fixtures/results/integration/ref-no-value.json"),
"exit_code": 2,
},
{
"filename": ("test/fixtures/templates/integration/metdata.yaml"),
"filename": "test/fixtures/templates/integration/metdata.yaml",
"results_filename": ("test/fixtures/results/integration/metadata.json"),
"exit_code": 2,
"exit_code": 4,
},
]

Expand Down
10 changes: 5 additions & 5 deletions test/integration/test_quickstart_templates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,14 @@ class TestQuickStartTemplates(BaseCliTestCase):
{
"filename": "test/fixtures/templates/quickstart/nist_application.yaml",
"results_filename": (
"test/fixtures/results/quickstart/nist_application.json"
"test/fixtures/results/quickstart" "/nist_application.json"
),
"exit_code": 14,
},
{
"filename": "test/fixtures/templates/quickstart/nist_config_rules.yaml",
"results_filename": (
"test/fixtures/results/quickstart/nist_config_rules.json"
"test/fixtures/results/quickstart/" "nist_config_rules.json"
),
"exit_code": 6,
},
Expand All @@ -54,21 +54,21 @@ class TestQuickStartTemplates(BaseCliTestCase):
{
"filename": "test/fixtures/templates/quickstart/nist_vpc_management.yaml",
"results_filename": (
"test/fixtures/results/quickstart/nist_vpc_management.json"
"test/fixtures/results/quickstart/" "nist_vpc_management.json"
),
"exit_code": 14,
},
{
"filename": "test/fixtures/templates/quickstart/nist_vpc_production.yaml",
"results_filename": (
"test/fixtures/results/quickstart/nist_vpc_production.json"
"test/fixtures/results/quickstart/" "nist_vpc_production.json"
),
"exit_code": 14,
},
{
"filename": "test/fixtures/templates/quickstart/openshift_master.yaml",
"results_filename": (
"test/fixtures/results/quickstart/openshift_master.json"
"test/fixtures/results/quickstart" "/openshift_master.json"
),
"exit_code": 8,
},
Expand Down

0 comments on commit 37aaa9e

Please sign in to comment.