Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix integration tests #3615

Merged
merged 4 commits into from
Aug 23, 2024
Merged

Fix integration tests #3615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d37cb6c
Script to create release JSON schemas
kddejong Aug 15, 2024
5f7d05c
Build release artfiacts
kddejong Aug 21, 2024
bca28ea
Build a better Changelog
kddejong Aug 21, 2024
dbc6ab0
fix integration testing to make it actually work
kddejong Aug 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 16 additions & 14 deletions test/fixtures/results/integration/ref-no-value.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
},
{
"Filename": "test/fixtures/templates/integration/ref-no-value.yaml",
"Id": "2ed7922b-1cd5-30ae-0c5f-918f69c9b782",
"Id": "93c41f0b-56b3-2349-8fb6-4be356befb25",
"Level": "Error",
"Location": {
"End": {
Expand All @@ -71,25 +71,26 @@
"Path": [
"Resources",
"IamRole2",
"Properties"
"Properties",
"Ref"
],
"Start": {
"ColumnNumber": 5,
"LineNumber": 26
}
},
"Message": "'AssumeRolePolicyDocument' is a required property",
"Message": "{'Ref': 'AWS::NoValue'} is not of type object",
"ParentId": null,
"Rule": {
"Description": "Make sure that Resources properties that are required exist",
"Id": "E3003",
"ShortDescription": "Required Resource properties are missing",
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required"
"Description": "Checks resource property values with Primitive Types for values that match those types.",
"Id": "E3012",
"ShortDescription": "Check resource properties values",
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type"
}
},
{
"Filename": "test/fixtures/templates/integration/ref-no-value.yaml",
"Id": "edfc8e64-ad37-e697-8fb7-5c89d2ff1735",
"Id": "5714e7bb-7c4b-573d-88cb-cbda0df6276d",
"Level": "Error",
"Location": {
"End": {
Expand All @@ -99,20 +100,21 @@
"Path": [
"Resources",
"CloudFront1",
"Properties"
"Properties",
"Ref"
],
"Start": {
"ColumnNumber": 5,
"LineNumber": 39
}
},
"Message": "'DistributionConfig' is a required property",
"Message": "{'Ref': 'AWS::NoValue'} is not of type object",
"ParentId": null,
"Rule": {
"Description": "Make sure that Resources properties that are required exist",
"Id": "E3003",
"ShortDescription": "Required Resource properties are missing",
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required"
"Description": "Checks resource property values with Primitive Types for values that match those types.",
"Id": "E3012",
"ShortDescription": "Check resource properties values",
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type"
}
},
{
Expand Down
4 changes: 2 additions & 2 deletions test/fixtures/results/quickstart/nist_iam.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
},
{
"Filename": "test/fixtures/templates/quickstart/nist_iam.yaml",
"Id": "99273eea-4208-d5b5-e641-516b59429087",
"Id": "2902b820-10fb-7008-c4f2-b74be7c678f6",
"Level": "Warning",
"Location": {
"End": {
Expand All @@ -78,7 +78,7 @@
"LineNumber": 165
}
},
"Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack']",
"Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createthemeforstack', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deletethemeforstack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describethemeforstack', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack', 'updatethemeforstack']",
"ParentId": null,
"Rule": {
"Description": "Check for valid IAM Permissions",
Expand Down
36 changes: 23 additions & 13 deletions test/integration/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,15 @@
import json
import subprocess
import unittest
from copy import deepcopy
from io import StringIO
from pathlib import Path
from typing import Any, Dict, List
from unittest.mock import patch

from cfnlint.config import configure_logging
from cfnlint.decode import cfn_yaml
from cfnlint.formatters import JsonFormatter
from cfnlint.runner import Runner


Expand Down Expand Up @@ -102,16 +106,22 @@ def run_module_integration_scenarios(self, config):
for result in expected_results:
result["Filename"] = str(Path(result.get("Filename")))

template = cfn_yaml.load(filename)

runner = Runner(config)
matches = list(runner.validate_template(filename, template))

# Only check that the error count matches as the formats are different
self.assertEqual(
len(expected_results),
len(matches),
"Expected {} failures, got {} on {}".format(
len(expected_results), matches, filename
),
)
# template = cfn_yaml.load(filename)
scenario_config = deepcopy(config)
scenario_config.cli_args.template_alt = [filename]
scenario_config.cli_args.format = "json"

runner = Runner(scenario_config)

print(f"Running test for {filename!r}")
with patch("sys.exit") as exit:
with patch("sys.stdout", new=StringIO()) as out:
runner.cli()
exit.assert_called_once_with(scenario.get("exit_code", 0))

output = json.loads(out.getvalue())
self.assertEqual(
expected_results,
output,
f"Test for {filename!r} got results: {output!r}",
)
84 changes: 23 additions & 61 deletions test/integration/test_good_templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,96 +40,58 @@ class TestQuickStartTemplates(BaseCliTestCase):
),
"results": [
{
"Filename": str(
Path(
"test/fixtures/templates/bad/transform_serverless_template.yaml"
)
),
"Id": "74181426-e865-10eb-96fd-908dfd30a358",
"Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml",
"Id": "9e05773a-b0d0-f157-2955-596d9bd54749",
"Level": "Error",
"Location": {
"Start": {"ColumnNumber": 1, "LineNumber": 1},
"End": {"ColumnNumber": 2, "LineNumber": 1},
"Path": None,
"Start": {"ColumnNumber": 1, "LineNumber": 1},
},
"Message": "Error transforming template: Resource with id [myFunctionMyTimer] is invalid. Missing required property 'Schedule'.",
"ParentId": None,
"Rule": {
"Description": "Errors found when performing transformation on the template",
"Id": "E0001",
"Description": (
"Errors found when performing transformation on the"
" template"
),
"Source": ("https://github.com/aws-cloudformation/cfn-lint"),
"ShortDescription": (
"Error found when transforming the template"
),
"ShortDescription": "Error found when transforming the template",
"Source": "https://github.com/aws-cloudformation/cfn-lint",
},
"Level": "Error",
"Message": (
"Error transforming template: Resource with id [AppName] is"
" invalid. Resource is missing the required [Location]"
" property."
),
},
{
"Filename": str(
Path(
"test/fixtures/templates/bad/transform_serverless_template.yaml"
)
),
"Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml",
"Id": "fd751fa3-7d1f-e194-7108-eb08352814c8",
"Level": "Error",
"Location": {
"Start": {"ColumnNumber": 1, "LineNumber": 1},
"End": {"ColumnNumber": 2, "LineNumber": 1},
"Path": None,
"Start": {"ColumnNumber": 1, "LineNumber": 1},
},
"Message": "Error transforming template: Resource with id [ExampleLayer] is invalid. Missing required property 'ContentUri'.",
"ParentId": None,
"Rule": {
"Description": "Errors found when performing transformation on the template",
"Id": "E0001",
"Description": (
"Errors found when performing transformation on the"
" template"
),
"Source": ("https://github.com/aws-cloudformation/cfn-lint"),
"ShortDescription": (
"Error found when transforming the template"
),
"ShortDescription": "Error found when transforming the template",
"Source": "https://github.com/aws-cloudformation/cfn-lint",
},
"Level": "Error",
"Message": (
"Error transforming template: Resource with id [ExampleLayer]"
" is invalid. Missing required property 'ContentUri'."
),
},
{
"Filename": str(
Path(
"test/fixtures/templates/bad/transform_serverless_template.yaml"
)
),
"Id": "9e05773a-b0d0-f157-2955-596d9bd54749",
"Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml",
"Id": "74181426-e865-10eb-96fd-908dfd30a358",
"Level": "Error",
"Location": {
"Start": {"ColumnNumber": 1, "LineNumber": 1},
"End": {"ColumnNumber": 2, "LineNumber": 1},
"Path": None,
"Start": {"ColumnNumber": 1, "LineNumber": 1},
},
"Message": "Error transforming template: Resource with id [AppName] is invalid. Resource is missing the required [Location] property.",
"ParentId": None,
"Rule": {
"Description": "Errors found when performing transformation on the template",
"Id": "E0001",
"Description": (
"Errors found when performing transformation on the"
" template"
),
"Source": ("https://github.com/aws-cloudformation/cfn-lint"),
"ShortDescription": (
"Error found when transforming the template"
),
"ShortDescription": "Error found when transforming the template",
"Source": "https://github.com/aws-cloudformation/cfn-lint",
},
"Level": "Error",
"Message": (
"Error transforming template: Resource with id"
" [myFunctionMyTimer] is invalid. Missing required property"
" 'Schedule'."
),
},
],
"exit_code": 2,
Expand Down
10 changes: 5 additions & 5 deletions test/integration/test_integration_templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,21 @@ class TestQuickStartTemplates(BaseCliTestCase):
"exit_code": 0,
},
{
"filename": ("test/fixtures/templates/integration/dynamic-references.yaml"),
"filename": "test/fixtures/templates/integration/dynamic-references.yaml",
"results_filename": (
"test/fixtures/results/integration/dynamic-references.json"
"test/fixtures/results/integration" "/dynamic-references.json"
),
"exit_code": 2,
},
{
"filename": ("test/fixtures/templates/integration/ref-no-value.yaml"),
"filename": "test/fixtures/templates/integration/ref-no-value.yaml",
"results_filename": ("test/fixtures/results/integration/ref-no-value.json"),
"exit_code": 2,
},
{
"filename": ("test/fixtures/templates/integration/metdata.yaml"),
"filename": "test/fixtures/templates/integration/metdata.yaml",
"results_filename": ("test/fixtures/results/integration/metadata.json"),
"exit_code": 2,
"exit_code": 4,
},
]

Expand Down
10 changes: 5 additions & 5 deletions test/integration/test_quickstart_templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,14 @@ class TestQuickStartTemplates(BaseCliTestCase):
{
"filename": "test/fixtures/templates/quickstart/nist_application.yaml",
"results_filename": (
"test/fixtures/results/quickstart/nist_application.json"
"test/fixtures/results/quickstart" "/nist_application.json"
),
"exit_code": 14,
},
{
"filename": "test/fixtures/templates/quickstart/nist_config_rules.yaml",
"results_filename": (
"test/fixtures/results/quickstart/nist_config_rules.json"
"test/fixtures/results/quickstart/" "nist_config_rules.json"
),
"exit_code": 6,
},
Expand All @@ -54,21 +54,21 @@ class TestQuickStartTemplates(BaseCliTestCase):
{
"filename": "test/fixtures/templates/quickstart/nist_vpc_management.yaml",
"results_filename": (
"test/fixtures/results/quickstart/nist_vpc_management.json"
"test/fixtures/results/quickstart/" "nist_vpc_management.json"
),
"exit_code": 14,
},
{
"filename": "test/fixtures/templates/quickstart/nist_vpc_production.yaml",
"results_filename": (
"test/fixtures/results/quickstart/nist_vpc_production.json"
"test/fixtures/results/quickstart/" "nist_vpc_production.json"
),
"exit_code": 14,
},
{
"filename": "test/fixtures/templates/quickstart/openshift_master.yaml",
"results_filename": (
"test/fixtures/results/quickstart/openshift_master.json"
"test/fixtures/results/quickstart" "/openshift_master.json"
),
"exit_code": 8,
},
Expand Down
Loading