From e6eb7570e0c12fe8bc6fb150c794d47f51eabcd0 Mon Sep 17 00:00:00 2001 From: Kevin DeJong Date: Wed, 4 Sep 2024 07:41:33 -0700 Subject: [PATCH] Allow for patch in place (#3649) * Allow for patch in place --- scripts/update_schemas_manually.py | 14 ++++ src/cfnlint/config.py | 10 +++ .../extensions/all/aws_ec2_subnet/manual.json | 14 ++++ .../providers/af_south_1/aws-ec2-subnet.json | 8 ++ .../providers/ap_east_1/aws-ec2-subnet.json | 8 ++ .../ap_northeast_2/aws-ec2-subnet.json | 8 ++ .../providers/ap_south_2/aws-ec2-subnet.json | 8 ++ .../ap_southeast_3/aws-ec2-subnet.json | 8 ++ .../ap_southeast_4/aws-ec2-subnet.json | 8 ++ .../ap_southeast_5/aws-ec2-subnet.json | 8 ++ .../providers/ca_west_1/aws-ec2-subnet.json | 8 ++ .../cn_northwest_1/aws-ec2-subnet.json | 8 ++ .../eu_central_2/aws-ec2-subnet.json | 8 ++ .../providers/eu_north_1/aws-ec2-subnet.json | 8 ++ .../providers/eu_south_1/aws-ec2-subnet.json | 8 ++ .../providers/eu_south_2/aws-ec2-subnet.json | 8 ++ .../il_central_1/aws-ec2-subnet.json | 8 ++ .../providers/me_south_1/aws-ec2-subnet.json | 8 ++ .../providers/us_east_1/aws-ec2-subnet.json | 8 ++ .../us_gov_east_1/aws-ec2-subnet.json | 8 ++ .../us_gov_west_1/aws-ec2-subnet.json | 8 ++ src/cfnlint/maintenance.py | 9 +++ src/cfnlint/runner.py | 4 + src/cfnlint/schema/manager.py | 75 ++++++++++++------- test/unit/module/schema/test_manager.py | 4 +- 25 files changed, 247 insertions(+), 27 deletions(-) create mode 100644 src/cfnlint/data/schemas/patches/extensions/all/aws_ec2_subnet/manual.json diff --git a/scripts/update_schemas_manually.py b/scripts/update_schemas_manually.py index 62167c0756..dde8809c82 100755 --- a/scripts/update_schemas_manually.py +++ b/scripts/update_schemas_manually.py @@ -1574,6 +1574,20 @@ ), ], ), + ResourcePatch( + resource_type="AWS::EC2::Subnet", + patches=[ + Patch( + values={ + "dependentExcluded": { + "AvailabilityZone": ["AvailabilityZoneId"], + "AvailabilityZoneId": ["AvailabilityZone"], + }, + }, + path="/", + ), + ], + ), ] ) diff --git a/src/cfnlint/config.py b/src/cfnlint/config.py index fd8934e39e..aca246c0ed 100644 --- a/src/cfnlint/config.py +++ b/src/cfnlint/config.py @@ -533,6 +533,12 @@ def __call__(self, parser, namespace, values, option_string=None): help="Update the CloudFormation Specs", action="store_true", ) + advanced.add_argument( + "-p", + "--patch-specs", + help="Patch the CloudFormation Specs in place", + action="store_true", + ) advanced.add_argument( "--update-documentation", help=argparse.SUPPRESS, action="store_true" ) @@ -792,6 +798,10 @@ def custom_rules(self): def update_specs(self): return self._get_argument_value("update_specs", False, False) + @property + def patch_specs(self): + return self._get_argument_value("patch_specs", False, False) + @property def update_documentation(self): return self._get_argument_value("update_documentation", False, False) diff --git a/src/cfnlint/data/schemas/patches/extensions/all/aws_ec2_subnet/manual.json b/src/cfnlint/data/schemas/patches/extensions/all/aws_ec2_subnet/manual.json new file mode 100644 index 0000000000..853dd65adc --- /dev/null +++ b/src/cfnlint/data/schemas/patches/extensions/all/aws_ec2_subnet/manual.json @@ -0,0 +1,14 @@ +[ + { + "op": "add", + "path": "/dependentExcluded", + "value": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + } + } +] diff --git a/src/cfnlint/data/schemas/providers/af_south_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/af_south_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/af_south_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/af_south_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/ap_east_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/ap_east_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/ap_east_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/ap_east_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/ap_northeast_2/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/ap_northeast_2/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/ap_northeast_2/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/ap_northeast_2/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/ap_south_2/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/ap_south_2/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/ap_south_2/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/ap_south_2/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/ap_southeast_3/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/ap_southeast_3/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/ap_southeast_3/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/ap_southeast_3/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/ap_southeast_4/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/ap_southeast_4/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/ap_southeast_4/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/ap_southeast_4/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/ap_southeast_5/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/ap_southeast_5/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/ap_southeast_5/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/ap_southeast_5/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/ca_west_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/ca_west_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/ca_west_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/ca_west_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/cn_northwest_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/cn_northwest_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/cn_northwest_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/cn_northwest_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/eu_central_2/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/eu_central_2/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/eu_central_2/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/eu_central_2/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/eu_north_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/eu_north_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/eu_north_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/eu_north_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/eu_south_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/eu_south_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/eu_south_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/eu_south_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/eu_south_2/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/eu_south_2/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/eu_south_2/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/eu_south_2/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/il_central_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/il_central_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/il_central_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/il_central_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/me_south_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/me_south_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/me_south_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/me_south_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/us_east_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/us_east_1/aws-ec2-subnet.json index 7b7398ef96..3ac49a874a 100644 --- a/src/cfnlint/data/schemas/providers/us_east_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/us_east_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/us_gov_east_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/us_gov_east_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/us_gov_east_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/us_gov_east_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/data/schemas/providers/us_gov_west_1/aws-ec2-subnet.json b/src/cfnlint/data/schemas/providers/us_gov_west_1/aws-ec2-subnet.json index ab454061a3..e8e64cc364 100644 --- a/src/cfnlint/data/schemas/providers/us_gov_west_1/aws-ec2-subnet.json +++ b/src/cfnlint/data/schemas/providers/us_gov_west_1/aws-ec2-subnet.json @@ -33,6 +33,14 @@ "type": "object" } }, + "dependentExcluded": { + "AvailabilityZone": [ + "AvailabilityZoneId" + ], + "AvailabilityZoneId": [ + "AvailabilityZone" + ] + }, "handlers": { "create": { "permissions": [ diff --git a/src/cfnlint/maintenance.py b/src/cfnlint/maintenance.py index c240d4f44c..1bb57e4a9a 100644 --- a/src/cfnlint/maintenance.py +++ b/src/cfnlint/maintenance.py @@ -29,6 +29,15 @@ def update_resource_specs(force: bool = False): PROVIDER_SCHEMA_MANAGER.update(force) +def patch_resource_specs(): + # Pool() uses cpu count if no number of processors is specified + # Pool() only implements the Context Manager protocol from Python3.3 onwards, + # so it will fail Python2.7 style linting, as well as throw AttributeError + + # Update provider Schemas + PROVIDER_SCHEMA_MANAGER.patch_schemas() + + def update_documentation(rules): # Update the overview of all rules in the linter filename = "docs/rules.md" diff --git a/src/cfnlint/runner.py b/src/cfnlint/runner.py index 442ff91b24..e6cc4fe32a 100644 --- a/src/cfnlint/runner.py +++ b/src/cfnlint/runner.py @@ -417,6 +417,10 @@ def cli(self) -> None: cfnlint.maintenance.update_resource_specs(self.config.force) sys.exit(0) + if self.config.patch_specs: + cfnlint.maintenance.patch_resource_specs() + sys.exit(0) + if self.config.update_iam_policies: cfnlint.maintenance.update_iam_policies() sys.exit(0) diff --git a/src/cfnlint/schema/manager.py b/src/cfnlint/schema/manager.py index 498d2ea474..3872ebafea 100644 --- a/src/cfnlint/schema/manager.py +++ b/src/cfnlint/schema/manager.py @@ -320,31 +320,8 @@ def _update_provider_schema(self, region: str, force: bool = False) -> None: with open(f"{directory}{filename}", "r+", encoding="utf-8") as fh: spec = json.load(fh) all_types.append(spec["typeName"]) - try: - spec = self._remove_descriptions(spec) - spec = self._patch_provider_schema(spec, filename, "all") - spec = self._patch_provider_schema( - spec, filename, region=reg.py - ) - except Exception as e: # pylint: disable=broad-except - LOGGER.info( - "Issuing patching schema for %s in %s: %s", - filename, - reg.name, - e, - ) - # Back to zero to write spec - fh.seek(0) - json.dump( - spec, - fh, - indent=1, - separators=(",", ": "), - sort_keys=True, - ) - fh.write("\n") - # Resize doc as needed - fh.truncate() + + self._patch_region_schemas(region) # if the region is not us-east-1 compare the files to those in us-east-1 # symlink if the files are the same @@ -392,6 +369,54 @@ def _update_provider_schema(self, region: str, force: bool = False) -> None: except Exception as e: # pylint: disable=broad-except LOGGER.info("Issuing updating schemas for %s: %s", region, e) + def patch_schemas(self) -> None: + + self._patch_region_schemas(self._region_primary.name) + # pylint: disable=not-context-manager + with multiprocessing.Pool() as pool: + # Patch from registry schema + provider_pool_tuple = [ + (k,) for k in REGIONS if k != self._region_primary.name + ] + pool.starmap(self._patch_region_schemas, provider_pool_tuple) + + def _patch_region_schemas(self, region: str) -> None: + reg = ToPy(region) + directory = os.path.join(f"{self._root.path_relative}/{reg.py}/") + + filenames = [ + f + for f in os.listdir(directory) + if os.path.isfile(os.path.join(directory, f)) and f != "__init__.py" + ] + + for filename in filenames: + with open(f"{directory}{filename}", "r+", encoding="utf-8") as fh: + spec = json.load(fh) + try: + spec = self._remove_descriptions(spec) + spec = self._patch_provider_schema(spec, filename, "all") + spec = self._patch_provider_schema(spec, filename, region=reg.py) + except Exception as e: # pylint: disable=broad-except + LOGGER.info( + "Issuing patching schema for %s in %s: %s", + filename, + reg.name, + e, + ) + # Back to zero to write spec + fh.seek(0) + json.dump( + spec, + fh, + indent=1, + separators=(",", ": "), + sort_keys=True, + ) + fh.write("\n") + # Resize doc as needed + fh.truncate() + def _patch_provider_schema( self, content: Dict, source_filename: str, region: str ) -> Dict: diff --git a/test/unit/module/schema/test_manager.py b/test/unit/module/schema/test_manager.py index dd50be42d9..74b9ce0e86 100644 --- a/test/unit/module/schema/test_manager.py +++ b/test/unit/module/schema/test_manager.py @@ -67,7 +67,7 @@ def test_update_resource_spec( "aws_lambda_codesigningconfig.json", "__init__.py", ] - mock_os_path_isfile.side_effect = [True, True] + mock_os_path_isfile.side_effect = [True, True, True, True] mock_load_resource.return_value = self.schema_patch mock_os_walk.return_value = iter( [("all", [], ["aws_lambda_codesigningconfig.json"])] @@ -132,7 +132,7 @@ def test_update_resource_spec_cache( "aws-lambda-codesigningconfig.json", "__init__.py", ] - mock_os_path_isfile.side_effect = [True, True] + mock_os_path_isfile.side_effect = [True, True, True, True] mock_filecmp_cmp.side_effect = [True] mock_load_resource.return_value = self.schema_patch mock_os_walk.return_value = iter(