Tools: Enforce SPDX headers (awsdocs#5945)

* Tools: enforce SPDX check

.tools/validation/project_validator.py

@@ -29,12 +29,13 @@
 
 from file_utils import get_files
 from metadata_errors import MetadataErrors, MetadataError, DuplicateItemException
+from spdx import verify_spdx
 import validator_config
 
 logger = logging.getLogger(__name__)
 
 
-def check_files(root: Path, errors: MetadataErrors, check_spdx: bool):
+def check_files(root: Path, errors: MetadataErrors, do_check_spdx: bool):
     """
     Walk a folder system, scanning all files with specified extensions.
     Errors are logged and counted and the count of errors is returned.
@@ -54,7 +55,7 @@ def check_files(root: Path, errors: MetadataErrors, check_spdx: bool):
         verify_no_secret_keys(file_contents, file_path, errors)
         verify_no_secret_keys(file_contents, file_path, errors)
         verify_snippet_start_end(file_contents, file_path, errors)
-        if check_spdx:
+        if do_check_spdx:
             verify_spdx(file_contents, file_path, errors)
 
     print(f"{file_count} files scanned in {root}.\n")
@@ -181,80 +182,6 @@ def verify_no_secret_keys(
         errors.append(PossibleSecretKey(file=str(file_location), word=word))
 
 
-@dataclass
-class InvalidSPDX(MetadataError):
-    has_copyright: bool = True
-    has_license: bool = True
-    has_bom: bool = False
-
-    def message(self):
-        message = "Invalid SPDX"
-        if not self.has_copyright:
-            message += " Missing Copyright line"
-        if not self.has_license:
-            message += " Missing License line"
-        if self.has_bom:
-            message += " Has BOM"
-        return message
-
-
-@dataclass
-class MissingSPDX(MetadataError):
-    def message(self):
-        return "Missing SPDX"
-
-
-def verify_spdx(file_contents: str, file_location: Path, errors: MetadataErrors):
-    """Verify the file starts with an SPDX comment, possibly following a shebang line"""
-    if file_location.suffix in validator_config.IGNORE_SPDX_SUFFIXES:
-        return
-    has_bom = file_contents.startswith("\uFEFF")
-    lines = file_contents.splitlines()
-    if len(lines) < 2:
-        return
-    if (
-        lines[0].startswith("#!")
-        or lines[0] == "<?php"
-        or lines[0].startswith("// swift-tools-version:")
-    ):
-        lines = lines[1:]
-    if len(lines) < 2:
-        return
-    # First line may be a start of comment
-    has_copyright = (
-        False if re.match(validator_config.SPDX_COPYRIGHT, lines[0]) is None else True
-    )
-    has_license = (
-        False if re.match(validator_config.SPDX_LICENSE, lines[1]) is None else True
-    )
-    if not (has_copyright and has_license) or has_bom:
-        file_has_copyright = (
-            False
-            if re.match(validator_config.SPDX_COPYRIGHT, file_contents) is None
-            else True
-        )
-        file_has_license = (
-            False
-            if re.match(validator_config.SPDX_LICENSE, file_contents) is None
-            else True
-        )
-        if file_has_copyright or file_has_license or has_bom:
-            errors.append(
-                InvalidSPDX(
-                    file=file_location,
-                    has_copyright=has_copyright,
-                    has_license=has_license,
-                    has_bom=has_bom,
-                )
-            )
-        else:
-            errors.append(
-                MissingSPDX(
-                    file=file_location,
-                )
-            )
-
-
 @dataclass
 class SnippetParseError(MetadataError):
     tag: str = field(default="")

.tools/validation/spdx.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from dataclasses import dataclass
+from pathlib import Path
+import re
+from sys import argv
+
+from metadata_errors import MetadataError, MetadataErrors
+import validator_config
+
+
+@dataclass
+class InvalidSPDX(MetadataError):
+    has_copyright: bool = True
+    has_license: bool = True
+
+    def message(self):
+        message = "Invalid SPDX"
+        if self.has_copyright:
+            message += " Invalid Copyright line"
+        if self.has_license:
+            message += " Invalid License line"
+        return message
+
+
+@dataclass
+class MissingSPDX(MetadataError):
+    def message(self):
+        return "Missing SPDX"
+
+
+def verify_spdx(file_contents: str, file_location: Path, errors: MetadataErrors):
+    """Verify the file starts with an SPDX comment, possibly following a shebang line"""
+    if file_location.suffix in validator_config.IGNORE_SPDX_SUFFIXES:
+        return
+    lines = file_contents.splitlines()
+    if len(lines) < 2:
+        return
+    if (
+        lines[0].startswith("#!")
+        or lines[0].startswith("// swift-tools-version:")
+        or lines[0] == "<?php"
+    ):
+        lines = lines[1:]
+    if len(lines) < 2:
+        return
+    # First line may be a start of comment
+    has_copyright = (
+        False
+        if re.match(
+            validator_config.SPDX_LEADER + validator_config.SPDX_COPYRIGHT, lines[0]
+        )
+        is None
+        else True
+    )
+    has_license = (
+        False
+        if re.match(
+            validator_config.SPDX_LEADER + validator_config.SPDX_LICENSE, lines[1]
+        )
+        is None
+        else True
+    )
+    if not (has_copyright and has_license):
+        file_has_copyright = (
+            True
+            if re.match(validator_config.SPDX_COPYRIGHT, file_contents) is None
+            else False
+        )
+        file_has_license = (
+            True
+            if re.match(validator_config.SPDX_LICENSE, file_contents) is None
+            else False
+        )
+        if file_has_copyright or file_has_license:
+            errors.append(
+                InvalidSPDX(
+                    file=file_location,
+                    has_copyright=has_copyright,
+                    has_license=has_license,
+                )
+            )
+        else:
+            errors.append(
+                MissingSPDX(
+                    file=file_location,
+                )
+            )
+
+
+def main():
+    for p in argv[1:]:
+        p = Path(p)
+        with open(p, encoding="utf8") as f:
+            contents = f.readlines()
+        prefix = "#" if p.suffix == ".py" or p.suffix == ".sh" else "//"
+        offset = (
+            1
+            if contents[0].startswith("#!")
+            or contents[0].startswith("<?php")
+            or contents[0].startswith("// swift-tools-version")
+            else 0
+        )
+        contents.insert(
+            offset,
+            prefix
+            + " Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.\n",
+        )
+        contents.insert(offset + 1, prefix + " SPDX-License-Identifier: Apache-2.0\n")
+        with open(p, "wt", encoding="utf-8-sig") as f:
+            f.writelines(contents)
+
+
+if __name__ == "__main__":
+    main()

.tools/validation/validate.py

@@ -40,7 +40,7 @@ def main():
     )
     parser.add_argument(
         "--check-spdx",
-        default=False,
+        default=True,
         help="Verify all files start with SPDX header",
         required=False,
     )

.tools/validation/validator_config.py

@@ -61,10 +61,9 @@ def skip(path):
     ".yml",
 }
 
-SPDX_COPYRIGHT = (
-    r"(#|//) Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved."
-)
-SPDX_LICENSE = r"(#|//) SPDX-License-Identifier: (Apache-2.0|MIT-0)"
+SPDX_LEADER = r"^(#|//) "
+SPDX_COPYRIGHT = r"Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved."
+SPDX_LICENSE = r"SPDX-License-Identifier: (Apache-2.0|MIT-0)"
 
 GOOD_WORDS = {
     "crash",

javascriptv3/example_code/bedrock/actions/get-foundation-model.js

@@ -1,11 +1,9 @@
-/*
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- * SPDX-License-Identifier: Apache-2.0
- */
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 import { fileURLToPath } from 'url';
 
-import { BedrockClient, GetFoundationModelCommand  } from '@aws-sdk/client-bedrock';
+import { BedrockClient, GetFoundationModelCommand } from '@aws-sdk/client-bedrock';
 
 /**
  * Get details about an Amazon Bedrock foundation model.

javascriptv3/example_code/bedrock/actions/list-foundation-models.js

@@ -1,7 +1,5 @@
-/*
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- * SPDX-License-Identifier: Apache-2.0
- */
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 import { fileURLToPath } from 'url';
 

javascriptv3/example_code/bedrock/hello.js

@@ -1,14 +1,12 @@
-/*
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- * SPDX-License-Identifier: Apache-2.0
- */
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 import { fileURLToPath } from 'url';
 
 import { BedrockClient, ListFoundationModelsCommand } from '@aws-sdk/client-bedrock';
 
 const REGION = 'us-east-1';
-const client = new BedrockClient( { region: REGION } );
+const client = new BedrockClient({ region: REGION });
 
 export const main = async () => {
     const command = new ListFoundationModelsCommand({});

javascriptv3/example_code/bedrock/tests/get-foundation-model.integration.test.js

@@ -1,7 +1,5 @@
-/*
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- * SPDX-License-Identifier: Apache-2.0
- */
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 import { z } from 'zod';
 import { describe, it, expect } from 'vitest';

javascriptv3/example_code/bedrock/tests/list-foundation-models.integration.test.js

@@ -1,7 +1,5 @@
-/*
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- * SPDX-License-Identifier: Apache-2.0
- */
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 import { z } from 'zod';
 import { describe, it, expect } from 'vitest';

javav2/usecases/creating_amazon_cognito_app/src/main/java/com/example/GreetingApplication.java

@@ -1,7 +1,5 @@
-/*
-   Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-   SPDX-License-Identifier: Apache-2.0
-*/
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 package com.example;
 

javav2/usecases/creating_amazon_cognito_app/src/main/java/com/example/handlingformsubmission/DynamoDBEnhanced.java

@@ -1,7 +1,5 @@
-/*
-   Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-   SPDX-License-Identifier: Apache-2.0
-*/
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 package com.example.handlingformsubmission;
 
@@ -18,7 +16,7 @@
 @Component("DynamoDBEnhanced")
 public class DynamoDBEnhanced {
 
-     public void injectDynamoItem(Greeting item){
+    public void injectDynamoItem(Greeting item) {
 
         Region region = Region.US_EAST_1;
         DynamoDbClient ddb = DynamoDbClient.builder()
@@ -31,7 +29,8 @@ public void injectDynamoItem(Greeting item){
                     .dynamoDbClient(ddb)
                     .build();
 
-            DynamoDbTable<GreetingItems> mappedTable = enhancedClient.table("Greeting", TableSchema.fromBean(GreetingItems.class));
+            DynamoDbTable<GreetingItems> mappedTable = enhancedClient.table("Greeting",
+                    TableSchema.fromBean(GreetingItems.class));
             GreetingItems gi = new GreetingItems();
             gi.setName(item.getName());
             gi.setMessage(item.getBody());

javav2/usecases/creating_amazon_cognito_app/src/main/java/com/example/handlingformsubmission/Greeting.java

@@ -1,7 +1,5 @@
-/*
-   Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-   SPDX-License-Identifier: Apache-2.0
-*/
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 package com.example.handlingformsubmission;