Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Increase logic of boto automation #3818

Merged
merged 4 commits into from
Nov 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
60 changes: 59 additions & 1 deletion scripts/boto/_automated_patches.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@

_fields = ["pattern", "enum"]

_visited_paths = []


def renamer(name):
manual_fixes = {
Expand All @@ -43,6 +45,9 @@ def renamer(name):
"kafka": "MSK",
"firehose": "KinesisFirehose",
"es": "ElasticSearch",
"elbv2": "ElasticLoadBalancingV2",
"elb": "ElasticLoadBalancing",
"ds": "DirectoryService",
}
if name in manual_fixes:
return manual_fixes[name].lower()
Expand Down Expand Up @@ -92,6 +97,45 @@ def get_last_date(service_dir: Path) -> str:
return last_date


def _nested_arrays(
resolver: RefResolver,
schema_data: dict[str, Any],
boto_data: dict[str, Any],
shape_data: dict[str, Any],
start_path: str,
source: list[str],
):

shape = shape_data.get("member", {}).get("shape")
if not shape:
return []

array_shap_data = boto_data.get("shapes", {}).get(shape)

path = f"{start_path}/items"
schema_data = schema_data.get("items", {})
while True:
if "$ref" not in schema_data:
break
path = schema_data["$ref"][1:]
schema_data = resolver.resolve_from_url(schema_data["$ref"])

if array_shap_data.get("type") == "structure":
return _nested_objects(
resolver, schema_data, boto_data, array_shap_data, path, source
)
else:
# skip if we already have an enum or pattern
if any([schema_data.get(field) for field in _fields]):
return {}
return {
path: Patch(
source=source,
shape=shape,
)
}


def _nested_objects(
resolver: RefResolver,
schema_data: dict[str, Any],
Expand All @@ -106,9 +150,14 @@ def _nested_objects(
if p_name in skip_property_names:
continue
if p_name.lower() == member.lower():

path = f"{start_path}/properties/{p_name}"

global _visited_paths
if path in _visited_paths:
continue

_visited_paths.append(path)

while True:
if "$ref" not in p_data:
break
Expand All @@ -129,6 +178,13 @@ def _nested_objects(
resolver, p_data, boto_data, member_shape, path, source
)
)
elif member_shape.get("type") == "list":
if p_data.get("type") == "array":
results.update(
_nested_arrays(
resolver, p_data, boto_data, member_shape, path, source
)
)

if not any([member_shape.get(field) for field in _fields]):
continue
Expand Down Expand Up @@ -158,6 +214,8 @@ def _per_resource_patch(
.get("shape")
)

global _visited_paths
_visited_paths = []
results.update(
_nested_objects(
resolver,
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"5357ad155752331d8c927fa32fdb28c9\"", "url": "https://schema.cloudformation.eu-south-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"06f2c54452239a0c3847294aada34e90\"", "url": "https://schema.cloudformation.eu-south-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"9c09a8e86cff20c3da1dbd5301bd0ebe\"", "url": "https://schema.cloudformation.cn-north-1.amazonaws.com.cn/CloudformationSchema.zip"}
{"etag": "\"571c99dd6238c071f50aaf1bf42ccd9d\"", "url": "https://schema.cloudformation.cn-north-1.amazonaws.com.cn/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"e883cba4b52bd486689be2590182302e\"", "url": "https://schema.cloudformation.us-gov-east-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"4b96f860d808197aca85a75c0860f90d\"", "url": "https://schema.cloudformation.us-gov-east-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"dabda94e45dfc9d5c84a459da91edd2c\"", "url": "https://schema.cloudformation.me-south-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"1bfe0784146f65cb03510a26c85265c9\"", "url": "https://schema.cloudformation.me-south-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"b6a93fd4200882e33932f3977cf81d2d\"", "url": "https://schema.cloudformation.us-gov-west-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"4d412930f3e193d65e0d9ef55e20f4ab\"", "url": "https://schema.cloudformation.us-gov-west-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"1fafcf4ca44c438a68bf190ec5046b30\"", "url": "https://schema.cloudformation.me-central-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"bcb41574922a37d844518ce09c881715\"", "url": "https://schema.cloudformation.me-central-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"679ee3121552f93003a6865d0301f95f\"", "url": "https://schema.cloudformation.eu-west-2.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"44dc07de94943dc1863442f7e633234f\"", "url": "https://schema.cloudformation.eu-west-2.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"11d1b4b6102578b479d6e21c3363b07a\"", "url": "https://schema.cloudformation.cn-northwest-1.amazonaws.com.cn/CloudformationSchema.zip"}
{"etag": "\"0e30ac23646d4089331909c1842afc5d\"", "url": "https://schema.cloudformation.cn-northwest-1.amazonaws.com.cn/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"fb5b87fa2d3233312282562bcd3bfbc9\"", "url": "https://schema.cloudformation.af-south-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"9c0307b20bdb79fdb999ecfc15aab500\"", "url": "https://schema.cloudformation.af-south-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"4c9cb5a4d498a6466009723774049dbe\"", "url": "https://schema.cloudformation.us-west-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"6dd4ce495a596673838985e198828649\"", "url": "https://schema.cloudformation.us-west-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"721ed22e90284fd92477dcd7becf2876\"", "url": "https://schema.cloudformation.ap-southeast-5.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"fffa224d25659985ba594d96ef13f6a6\"", "url": "https://schema.cloudformation.ap-southeast-5.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"520442db355318cdc3026536d0986cc4\"", "url": "https://schema.cloudformation.eu-central-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"360ed14fcaebf7e6b93fff1e01c88fe1\"", "url": "https://schema.cloudformation.eu-central-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"d273c61b83b327ae7062f9e0252c380e\"", "url": "https://schema.cloudformation.ap-south-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"a53d9259f44296bbf227cb5e0f09fa61\"", "url": "https://schema.cloudformation.ap-south-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"2c5ccaaac2f27b06d22c2a84f14eb02c\"", "url": "https://schema.cloudformation.ap-southeast-4.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"be2e47da051983abd207d8f7f29f17cc\"", "url": "https://schema.cloudformation.ap-southeast-4.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"57c64212488b8f143b59162adffc2be9\"", "url": "https://schema.cloudformation.us-east-2.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"af0ecea0abcd6023ad9b1f3c3647f24f\"", "url": "https://schema.cloudformation.us-east-2.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"b17b43b6c7bd34f3af6de3f406c397fe\"", "url": "https://schema.cloudformation.ap-southeast-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"f329158ba2f26e94fe4581d32c379fa0\"", "url": "https://schema.cloudformation.ap-southeast-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"895a42c168d6397357dbf1ddba3d12b9\"", "url": "https://schema.cloudformation.ap-northeast-2.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"dfe2dd7e6375dbb9d9264d71c89d4bc3\"", "url": "https://schema.cloudformation.ap-northeast-2.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"29da5b62780e15423d7c2526d453d938\"", "url": "https://schema.cloudformation.ap-southeast-3.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"64dca5234e69de998a9e8e68d03f0d07\"", "url": "https://schema.cloudformation.ap-southeast-3.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"10122da694bede181ecd4161716c4a1a\"", "url": "https://schema.cloudformation.ap-east-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"b20066453b5a5701f09eda8ca7d06b68\"", "url": "https://schema.cloudformation.ap-east-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"ee2fafec85a78e0c4e65284b8c3ed8f9\"", "url": "https://schema.cloudformation.sa-east-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"d10f35b073d40372a94fc10b14a96d00\"", "url": "https://schema.cloudformation.sa-east-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"87c18247cfa1e1cc3005309803025962\"", "url": "https://schema.cloudformation.ap-southeast-2.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"c28472dd23783a0639cb6a73ff4b90a1\"", "url": "https://schema.cloudformation.ap-southeast-2.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"617e4f25d14fa2ae621304bfeb35c742\"", "url": "https://schema.cloudformation.ca-west-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"36c33a027d70d16fdca8a8a3f6be2710\"", "url": "https://schema.cloudformation.ca-west-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"8e8aef22a73df8aac450d1c66839de7c\"", "url": "https://schema.cloudformation.eu-central-2.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"887c79022fb5071dc5cc450d7ac68759\"", "url": "https://schema.cloudformation.eu-central-2.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"abff77b737994051678f96052059e865\"", "url": "https://schema.cloudformation.eu-north-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"37b1cfd2da163a623317213c5a57ca4d\"", "url": "https://schema.cloudformation.eu-north-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"ac577518b13bc5b9460aea32b546bc9d\"", "url": "https://schema.cloudformation.eu-south-2.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"1404496addd5d32cf7f28a1e29b2f08a\"", "url": "https://schema.cloudformation.eu-south-2.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"17e8c7adc1cf749c360ecdc7aab897f4\"", "url": "https://schema.cloudformation.ca-central-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"b2b86e1097162ce014d5f4301c3d7042\"", "url": "https://schema.cloudformation.ca-central-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"7bf14c08f1c6a5630ce8d8f85059a5e0\"", "url": "https://schema.cloudformation.eu-west-3.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"dc516309342d8ecbd22396aa9e7daacf\"", "url": "https://schema.cloudformation.eu-west-3.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"dd65610a40912779b8e996dca8a6e526\"", "url": "https://schema.cloudformation.ap-northeast-3.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"ca695fc4aa5a2cb7e2fbfad3d362db38\"", "url": "https://schema.cloudformation.ap-northeast-3.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"e639191636cd225fbe9ac6488edd9fbc\"", "url": "https://schema.cloudformation.us-west-2.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"f52c69e38743b137a6fac383369f6053\"", "url": "https://schema.cloudformation.us-west-2.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"d152e80c902be01477cb4446c400c753\"", "url": "https://schema.cloudformation.ap-south-2.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"f492a5987a8ea775191545aabfa6353b\"", "url": "https://schema.cloudformation.ap-south-2.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"ebb8bf034d08a1d5fa72fbef92356773\"", "url": "https://schema.cloudformation.us-east-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"fe7fc848615367d3262bfa060f92bf8e\"", "url": "https://schema.cloudformation.us-east-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"9a85fb289ed577c7c5e49c678f0b77c3\"", "url": "https://schema.cloudformation.il-central-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"94f53ec7340dfd8b734cbbef7d13d22f\"", "url": "https://schema.cloudformation.il-central-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"c42f41acfb73c2b6701b83c459ee2009\"", "url": "https://schema.cloudformation.eu-west-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"e27a1b9b1c17fdc0a212b2e9c4fee982\"", "url": "https://schema.cloudformation.eu-west-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"etag": "\"5d7672eafd26eff88359b68742c5212e\"", "url": "https://schema.cloudformation.ap-northeast-1.amazonaws.com/CloudformationSchema.zip"}
{"etag": "\"1c546c6b46008aa70ff7ad3cd63d133a\"", "url": "https://schema.cloudformation.ap-northeast-1.amazonaws.com/CloudformationSchema.zip"}
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,10 @@
"ORGANIZATION",
"ORGANIZATION_UNUSED_ACCESS"
]
},
{
"op": "add",
"path": "/definitions/ArchiveRule/properties/RuleName/pattern",
"value": "[A-Za-z][A-Za-z0-9_.-]*"
}
]
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,16 @@
"FIPS_140_2_LEVEL_3_OR_HIGHER"
]
},
{
"op": "add",
"path": "/definitions/Tag/properties/Key/pattern",
"value": "([\\p{L}\\p{Z}\\p{N}_.:/=+\\-@]*)"
},
{
"op": "add",
"path": "/definitions/Tag/properties/Value/pattern",
"value": "([\\p{L}\\p{Z}\\p{N}_.:/=+\\-@]*)"
},
{
"op": "add",
"path": "/properties/UsageMode/enum",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,14 @@
"op": "add",
"path": "/properties/SourceAccount/pattern",
"value": "[0-9]+"
},
{
"op": "add",
"path": "/properties/Actions/items/enum",
"value": [
"GetCertificate",
"IssueCertificate",
"ListPermissions"
]
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"op": "add",
"path": "/properties/AutoSubDomainCreationPatterns/items/pattern",
"value": "(?s).+"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[
{
"op": "add",
"path": "/definitions/Validators/properties/Type/enum",
"value": [
"JSON_SCHEMA",
"LAMBDA"
]
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[
{
"op": "add",
"path": "/definitions/SalesforceDestinationProperties/properties/IdFieldNames/items/pattern",
"value": "\\S+"
},
{
"op": "add",
"path": "/definitions/ZendeskDestinationProperties/properties/IdFieldNames/items/pattern",
"value": "\\S+"
},
{
"op": "add",
"path": "/definitions/CustomConnectorDestinationProperties/properties/IdFieldNames/items/pattern",
"value": "\\S+"
},
{
"op": "add",
"path": "/definitions/SAPODataDestinationProperties/properties/IdFieldNames/items/pattern",
"value": "\\S+"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
[
{
"op": "add",
"path": "/definitions/ApprovedOrigins/pattern",
"value": "^\\w+\\:\\/\\/.*$"
},
{
"op": "add",
"path": "/definitions/Permissions/pattern",
"value": "^[a-zA-Z0-9\\/\\._\\-\\*]+$"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"op": "add",
"path": "/definitions/FileConfiguration/properties/Folders/items/pattern",
"value": ".*\\S.*"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
[
{
"op": "add",
"path": "/definitions/Tag/properties/Key/pattern",
"value": "^([\\p{L}\\p{Z}\\p{N}_.:/=+\\-@]*)$"
},
{
"op": "add",
"path": "/definitions/Tag/properties/Value/pattern",
"value": "^([\\p{L}\\p{Z}\\p{N}_.:/=+\\-@]*)$"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"op": "add",
"path": "/definitions/Tag/properties/Key/pattern",
"value": "^(?!aws:).+"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"op": "add",
"path": "/definitions/Tag/properties/Key/pattern",
"value": "^(?!aws:).+"
}
]
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,10 @@
"op": "add",
"path": "/definitions/ImageConfiguration/properties/StartCommand/pattern",
"value": "[^\\x0a\\x0d]+"
},
{
"op": "add",
"path": "/definitions/Tag/properties/Key/pattern",
"value": "^(?!aws:).+"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"op": "add",
"path": "/definitions/Tag/properties/Key/pattern",
"value": "^(?!aws:).+"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"op": "add",
"path": "/definitions/Tag/properties/Key/pattern",
"value": "^(?!aws:).+"
}
]
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,12 @@
"op": "add",
"path": "/properties/IamRoleArn/pattern",
"value": "^arn:aws(?:\\-cn|\\-iso\\-b|\\-iso|\\-us\\-gov)?:[A-Za-z0-9][A-Za-z0-9_/.-]{0,62}:[A-Za-z0-9_/.-]{0,63}:[A-Za-z0-9_/.-]{0,63}:[A-Za-z0-9][A-Za-z0-9:_/+=,@.\\\\-]{0,1023}$"
},
{
"op": "add",
"path": "/definitions/AccessEndpoint/properties/EndpointType/enum",
"value": [
"STREAMING"
]
}
]
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@
"path": "/definitions/S3Location/properties/S3Bucket/pattern",
"value": "^[0-9a-z\\.\\-]*(?<!\\.)$"
},
{
"op": "add",
"path": "/definitions/PlatformType/enum",
"value": [
"AMAZON_LINUX2",
"RHEL8",
"WINDOWS",
"WINDOWS_SERVER_2016",
"WINDOWS_SERVER_2019",
"WINDOWS_SERVER_2022"
]
},
{
"op": "add",
"path": "/definitions/Arn/pattern",
Expand Down
Loading
Loading