Generic SBOM Generation Script (#46)

* add sbom generator action

* modify test.yml

* pyyaml

* update action

* store sbom

* add shell

* pr

* add api

* fix pr

* api

* h

* add script

* Update action.yml

* Update action.yml

* Update action.yml

* Update action.yml

* Update action.yml

* Update action.yml

* Update action.yml

* Update test.yml

* Update test.yml

* add kernel script

* Update scan_corePKCS11.py

* clean up

* clean up

* Update scan_overTheAir.py

* Update scan_overTheAir.py

* Update scan_kernel.py

* Update scan_corePKCS11.py

* Update scan_overTheAir.py

* generic python script

* fix url

* Fix URL

* Update action.yml

* Fi xurkl

sbom-generator/action.yml

@@ -0,0 +1,22 @@
+name: 'generate-sbom'
+description: 'Generate SBOM for FreeRTOS libraries'
+inputs:
+  repo_path:
+    description: 'Path to repository folder containing manifest.yml to verify.'
+    required: false
+    default: ./
+  source_path:
+    description: 'Path to source code'
+    required: false
+    default: ./source
+runs:
+  using: "composite"
+  steps:
+      - name: Install dependencies
+        run: pip install -r $GITHUB_ACTION_PATH/requirements.txt
+        shell: bash 
+      - name: Run generator script
+        working-directory: ${{ inputs.repo_path }}
+        run: |
+          python3 $GITHUB_ACTION_PATH/scan_dir.py --source-path ${{ inputs.source_path }}
+        shell: bash

sbom-generator/requirements.txt

@@ -0,0 +1 @@
+pyyaml

sbom-generator/sbom_utils.py

@@ -0,0 +1,59 @@
+import hashlib
+from datetime import datetime
+
+SPDX_VERSION = 'SPDX-2.2'
+DATA_LICENSE = 'CC0-1.0'
+CREATOR = 'Amazon Web Services'
+
+def hash_sha1(file_path: str) -> str:
+    BLOCKSIZE = 65536
+    hasher = hashlib.sha1()
+    with open(file_path, 'rb') as afile:
+        buf = afile.read(BLOCKSIZE)
+        while len(buf) > 0:
+            hasher.update(buf)
+            buf = afile.read(BLOCKSIZE)
+    return hasher.hexdigest()
+
+def package_hash(file_list: str) -> str:
+    sorted(file_list)
+    h = hashlib.sha1("".join(file_list).encode())
+    return h.hexdigest()
+
+def file_writer(output, filepath: str, filename: str, sha1: str, license: str, copyright='NOASSERTION', comment='NOASSERTION'):
+    output.write('FileName: '+ filename + '\n')
+    output.write('SPDXID: SPDXRef-File-'+ filename.replace('/', '-') + '\n')
+    output.write('FileChecksum: SHA1: '+ hash_sha1(filepath) + '\n')
+    output.write('LicenseConcluded: '+ license + '\n')
+    output.write('FileCopyrightText: '+ copyright + '\n')
+    output.write('FileComment: '+ comment + '\n')
+    output.write('\n')
+
+def pacakge_writer(output, packageName: str, version: str, url: str, license: str, ver_code: str, file_analyzed=True, 
+                   copyright='NOASSERTION', summary='NOASSERTION', description='NOASSERTION'):
+    output.write('PackageName: '+ packageName + '\n')
+    output.write('SPDXID: SPDXRef-Package-'+ packageName + '\n')
+    output.write('PackageVersion: '+ version + '\n')
+    output.write('PackageDownloadLocation: '+ url + '\n')
+    output.write('PackageLicenseConcluded: '+ license + '\n')
+    output.write('FilesAnalyzed: '+ str(file_analyzed) + '\n')
+    output.write('PackageVerificationCode: '+ ver_code + '\n')
+    output.write('PackageCopyrightText: '+ copyright + '\n')
+    output.write('PackageSummary: '+ summary + '\n')
+    output.write('PackageDescription: '+ description + '\n')
+    output.write('\n')
+
+def doc_writer(output, version: str, name: str, creator_comment='NOASSERTION', 
+               doc_comment='NOASSERTION'):
+    today = datetime.now()
+    namespace = 'https://github.com/FreeRTOS/'+name+'/blob/'+version+'/sbom.spdx'
+    output.write('SPDXVersion: ' + SPDX_VERSION + '\n')
+    output.write('DataLicense: ' + DATA_LICENSE  + '\n')
+    output.write('SPDXID: SPDXRef-DOCUMENT\n')
+    output.write('DocumentName: ' + name + '\n')
+    output.write('DocumentNamespace: ' + namespace + '\n')
+    output.write('Creator: ' + CREATOR + '\n')
+    output.write('Created: ' + today.isoformat()[:-7] + 'Z\n')
+    output.write('CreatorComment: ' + creator_comment + '\n')
+    output.write('DocumentComment: ' + doc_comment + '\n')
+    output.write('\n')

sbom-generator/scan_dir.py

@@ -0,0 +1,139 @@
+import yaml
+from yaml.loader import SafeLoader
+import io
+import os
+import hashlib
+from datetime import datetime
+from argparse import ArgumentParser
+from sbom_utils import *
+
+REPO_PATH = ''
+SOURCE_PATH = ''
+
+def scan_dir():
+    dependency_path = os.path.join(REPO_PATH, 'source/dependency')
+    path_3rdparty = os.path.join(REPO_PATH, 'source/dependency/3rdparty')
+    manifest_path = os.path.join(REPO_PATH, 'manifest.yml')
+    url = 'https://github.com/' + os.getenv('GITHUB_REPOSITORY')
+
+    output_buffer = {}
+    total_file_list = []
+    dependency_info = {}
+    dependency_file_list = {}
+    with open(manifest_path) as f:
+        manifest = yaml.load(f, Loader=SafeLoader)
+    root_license = manifest['license']
+    root_name = manifest['name']
+    url += '/tree/' + manifest['version']
+    output_buffer[root_name] = io.StringIO()
+
+    try:
+        for dependency in manifest['dependencies']:
+            dependency_info[dependency['name']] = dependency
+    except Exception:
+        pass
+
+    #scan the source code
+    for subdir, dirs, files in os.walk(SOURCE_PATH):
+        if 'dependency' in subdir:
+            continue
+        for file in files:
+            if file.endswith('.spdx'):
+                continue
+            filepath = os.path.join(subdir, file)
+            file_checksum = hash_sha1(filepath)
+            total_file_list.append(file_checksum)
+            if file.endswith('.c'):
+                file_writer(output_buffer[root_name], filepath, file, file_checksum, root_license)
+
+    #scan dependencies
+    if os.path.exists(dependency_path):
+        for library in os.listdir(dependency_path):
+            if library.startswith('.') or library == '3rdparty':
+                continue
+
+            #cross check with manifest file
+            if library in dependency_info.keys():
+                output_buffer[library] = io.StringIO()
+                buffer = output_buffer[library]
+                library_lic = dependency_info[library]['license']
+                dependency_file_list[library] = []
+                temp_list = dependency_file_list[library]
+            else:
+                library_lic = root_license
+                buffer = output_buffer[root_name]
+                temp_list = []
+
+            for subdir, dirs, files in os.walk(os.path.join(dependency_path, library)):
+                for file in files:
+                    filepath = os.path.join(subdir, file)
+                    file_checksum = hash_sha1(filepath)
+                    if file.endswith('.c'):
+                        file_writer(buffer, filepath, file, file_checksum, library_lic)
+                    total_file_list.append(file_checksum)
+                    temp_list.append(file_checksum)
+
+    #scan 3rd party code
+    if os.path.exists(path_3rdparty):
+        for library in os.listdir(path_3rdparty):
+            if library.startswith('.'):
+                continue
+
+            #cross check with manifest file
+            if library in dependency_info.keys():
+                output_buffer[library] = io.StringIO() 
+                buffer = output_buffer[library]
+                library_lic = dependency_info[library]['license']
+                dependency_file_list[library] = []
+                temp_list = dependency_file_list[library]
+            else:
+                library_lic = root_license
+                buffer = output_buffer[root_name]
+                temp_list = []
+
+            for subdir, dirs, files in os.walk(os.path.join(path_3rdparty, library)):
+                for file in files:
+                    filepath = os.path.join(subdir, file)
+                    file_checksum = hash_sha1(filepath)
+                    if file.endswith('.c'):
+                        file_writer(buffer, filepath, file, file_checksum, library_lic)
+                    total_file_list.append(file_checksum)
+                    temp_list.append(file_checksum)
+
+    #print sbom file to sbom.spdx
+    output = open('sbom.spdx', 'w')
+    doc_writer(output, manifest['version'], manifest['name'])
+    pacakge_writer(output, manifest['name'], manifest['version'], url, root_license, package_hash(total_file_list), description=manifest['description'])
+    output.write(output_buffer[root_name].getvalue())
+
+    #print dependencies
+    for library_name in output_buffer.keys():
+        if library_name == root_name:
+            continue
+        info = dependency_info[library_name]
+        pacakge_writer(output, library_name, info['version'], info['repository']['url'], info['license'], package_hash(dependency_file_list[library_name]))
+        output.write(output_buffer[library_name].getvalue())
+
+    #print relationships
+    for library_name in output_buffer.keys():
+        if library_name == root_name:
+            continue
+        output.write('Relationship: SPDXRef-Package-' + manifest['name'] + ' DEPENDS_ON SPDXRef-Package-' + library_name + '\n')
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description='SBOM generator')
+    parser.add_argument('--repo-root-path',
+                        type=str,
+                        required=None,
+                        default=os.getcwd(),
+                        help='Path to the repository root.')
+    parser.add_argument('--source-path',
+                        type=str,
+                        required=None,
+                        default=os.path.join(os.getcwd(), 'source'),
+                        help='Path to the source code dir.')
+    args = parser.parse_args()
+    REPO_PATH = os.path.abspath(args.repo_root_path)
+    SOURCE_PATH = os.path.abspath(args.source_path)
+
+    scan_dir()