From 89bf6cc54767a9a5fb02d33f7ad7e4ee7ce51d29 Mon Sep 17 00:00:00 2001 From: Xin Lin <47510956+xlin7799@users.noreply.github.com> Date: Fri, 29 Jul 2022 13:57:54 -0700 Subject: [PATCH] Generic SBOM Generation Script (#46) * add sbom generator action * modify test.yml * pyyaml * update action * store sbom * add shell * pr * add api * fix pr * api * h * add script * Update action.yml * Update action.yml * Update action.yml * Update action.yml * Update action.yml * Update action.yml * Update action.yml * Update test.yml * Update test.yml * add kernel script * Update scan_corePKCS11.py * clean up * clean up * Update scan_overTheAir.py * Update scan_overTheAir.py * Update scan_kernel.py * Update scan_corePKCS11.py * Update scan_overTheAir.py * generic python script * fix url * Fix URL * Update action.yml * Fi xurkl --- sbom-generator/action.yml | 22 +++++ sbom-generator/requirements.txt | 1 + sbom-generator/sbom_utils.py | 59 ++++++++++++++ sbom-generator/scan_dir.py | 139 ++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+) create mode 100644 sbom-generator/action.yml create mode 100644 sbom-generator/requirements.txt create mode 100644 sbom-generator/sbom_utils.py create mode 100644 sbom-generator/scan_dir.py diff --git a/sbom-generator/action.yml b/sbom-generator/action.yml new file mode 100644 index 00000000..5cbfd78f --- /dev/null +++ b/sbom-generator/action.yml @@ -0,0 +1,22 @@ +name: 'generate-sbom' +description: 'Generate SBOM for FreeRTOS libraries' +inputs: + repo_path: + description: 'Path to repository folder containing manifest.yml to verify.' + required: false + default: ./ + source_path: + description: 'Path to source code' + required: false + default: ./source +runs: + using: "composite" + steps: + - name: Install dependencies + run: pip install -r $GITHUB_ACTION_PATH/requirements.txt + shell: bash + - name: Run generator script + working-directory: ${{ inputs.repo_path }} + run: | + python3 $GITHUB_ACTION_PATH/scan_dir.py --source-path ${{ inputs.source_path }} + shell: bash diff --git a/sbom-generator/requirements.txt b/sbom-generator/requirements.txt new file mode 100644 index 00000000..c3726e8b --- /dev/null +++ b/sbom-generator/requirements.txt @@ -0,0 +1 @@ +pyyaml diff --git a/sbom-generator/sbom_utils.py b/sbom-generator/sbom_utils.py new file mode 100644 index 00000000..58de830a --- /dev/null +++ b/sbom-generator/sbom_utils.py @@ -0,0 +1,59 @@ +import hashlib +from datetime import datetime + +SPDX_VERSION = 'SPDX-2.2' +DATA_LICENSE = 'CC0-1.0' +CREATOR = 'Amazon Web Services' + +def hash_sha1(file_path: str) -> str: + BLOCKSIZE = 65536 + hasher = hashlib.sha1() + with open(file_path, 'rb') as afile: + buf = afile.read(BLOCKSIZE) + while len(buf) > 0: + hasher.update(buf) + buf = afile.read(BLOCKSIZE) + return hasher.hexdigest() + +def package_hash(file_list: str) -> str: + sorted(file_list) + h = hashlib.sha1("".join(file_list).encode()) + return h.hexdigest() + +def file_writer(output, filepath: str, filename: str, sha1: str, license: str, copyright='NOASSERTION', comment='NOASSERTION'): + output.write('FileName: '+ filename + '\n') + output.write('SPDXID: SPDXRef-File-'+ filename.replace('/', '-') + '\n') + output.write('FileChecksum: SHA1: '+ hash_sha1(filepath) + '\n') + output.write('LicenseConcluded: '+ license + '\n') + output.write('FileCopyrightText: '+ copyright + '\n') + output.write('FileComment: '+ comment + '\n') + output.write('\n') + +def pacakge_writer(output, packageName: str, version: str, url: str, license: str, ver_code: str, file_analyzed=True, + copyright='NOASSERTION', summary='NOASSERTION', description='NOASSERTION'): + output.write('PackageName: '+ packageName + '\n') + output.write('SPDXID: SPDXRef-Package-'+ packageName + '\n') + output.write('PackageVersion: '+ version + '\n') + output.write('PackageDownloadLocation: '+ url + '\n') + output.write('PackageLicenseConcluded: '+ license + '\n') + output.write('FilesAnalyzed: '+ str(file_analyzed) + '\n') + output.write('PackageVerificationCode: '+ ver_code + '\n') + output.write('PackageCopyrightText: '+ copyright + '\n') + output.write('PackageSummary: '+ summary + '\n') + output.write('PackageDescription: '+ description + '\n') + output.write('\n') + +def doc_writer(output, version: str, name: str, creator_comment='NOASSERTION', + doc_comment='NOASSERTION'): + today = datetime.now() + namespace = 'https://github.com/FreeRTOS/'+name+'/blob/'+version+'/sbom.spdx' + output.write('SPDXVersion: ' + SPDX_VERSION + '\n') + output.write('DataLicense: ' + DATA_LICENSE + '\n') + output.write('SPDXID: SPDXRef-DOCUMENT\n') + output.write('DocumentName: ' + name + '\n') + output.write('DocumentNamespace: ' + namespace + '\n') + output.write('Creator: ' + CREATOR + '\n') + output.write('Created: ' + today.isoformat()[:-7] + 'Z\n') + output.write('CreatorComment: ' + creator_comment + '\n') + output.write('DocumentComment: ' + doc_comment + '\n') + output.write('\n') diff --git a/sbom-generator/scan_dir.py b/sbom-generator/scan_dir.py new file mode 100644 index 00000000..577e35b8 --- /dev/null +++ b/sbom-generator/scan_dir.py @@ -0,0 +1,139 @@ +import yaml +from yaml.loader import SafeLoader +import io +import os +import hashlib +from datetime import datetime +from argparse import ArgumentParser +from sbom_utils import * + +REPO_PATH = '' +SOURCE_PATH = '' + +def scan_dir(): + dependency_path = os.path.join(REPO_PATH, 'source/dependency') + path_3rdparty = os.path.join(REPO_PATH, 'source/dependency/3rdparty') + manifest_path = os.path.join(REPO_PATH, 'manifest.yml') + url = 'https://github.com/' + os.getenv('GITHUB_REPOSITORY') + + output_buffer = {} + total_file_list = [] + dependency_info = {} + dependency_file_list = {} + with open(manifest_path) as f: + manifest = yaml.load(f, Loader=SafeLoader) + root_license = manifest['license'] + root_name = manifest['name'] + url += '/tree/' + manifest['version'] + output_buffer[root_name] = io.StringIO() + + try: + for dependency in manifest['dependencies']: + dependency_info[dependency['name']] = dependency + except Exception: + pass + + #scan the source code + for subdir, dirs, files in os.walk(SOURCE_PATH): + if 'dependency' in subdir: + continue + for file in files: + if file.endswith('.spdx'): + continue + filepath = os.path.join(subdir, file) + file_checksum = hash_sha1(filepath) + total_file_list.append(file_checksum) + if file.endswith('.c'): + file_writer(output_buffer[root_name], filepath, file, file_checksum, root_license) + + #scan dependencies + if os.path.exists(dependency_path): + for library in os.listdir(dependency_path): + if library.startswith('.') or library == '3rdparty': + continue + + #cross check with manifest file + if library in dependency_info.keys(): + output_buffer[library] = io.StringIO() + buffer = output_buffer[library] + library_lic = dependency_info[library]['license'] + dependency_file_list[library] = [] + temp_list = dependency_file_list[library] + else: + library_lic = root_license + buffer = output_buffer[root_name] + temp_list = [] + + for subdir, dirs, files in os.walk(os.path.join(dependency_path, library)): + for file in files: + filepath = os.path.join(subdir, file) + file_checksum = hash_sha1(filepath) + if file.endswith('.c'): + file_writer(buffer, filepath, file, file_checksum, library_lic) + total_file_list.append(file_checksum) + temp_list.append(file_checksum) + + #scan 3rd party code + if os.path.exists(path_3rdparty): + for library in os.listdir(path_3rdparty): + if library.startswith('.'): + continue + + #cross check with manifest file + if library in dependency_info.keys(): + output_buffer[library] = io.StringIO() + buffer = output_buffer[library] + library_lic = dependency_info[library]['license'] + dependency_file_list[library] = [] + temp_list = dependency_file_list[library] + else: + library_lic = root_license + buffer = output_buffer[root_name] + temp_list = [] + + for subdir, dirs, files in os.walk(os.path.join(path_3rdparty, library)): + for file in files: + filepath = os.path.join(subdir, file) + file_checksum = hash_sha1(filepath) + if file.endswith('.c'): + file_writer(buffer, filepath, file, file_checksum, library_lic) + total_file_list.append(file_checksum) + temp_list.append(file_checksum) + + #print sbom file to sbom.spdx + output = open('sbom.spdx', 'w') + doc_writer(output, manifest['version'], manifest['name']) + pacakge_writer(output, manifest['name'], manifest['version'], url, root_license, package_hash(total_file_list), description=manifest['description']) + output.write(output_buffer[root_name].getvalue()) + + #print dependencies + for library_name in output_buffer.keys(): + if library_name == root_name: + continue + info = dependency_info[library_name] + pacakge_writer(output, library_name, info['version'], info['repository']['url'], info['license'], package_hash(dependency_file_list[library_name])) + output.write(output_buffer[library_name].getvalue()) + + #print relationships + for library_name in output_buffer.keys(): + if library_name == root_name: + continue + output.write('Relationship: SPDXRef-Package-' + manifest['name'] + ' DEPENDS_ON SPDXRef-Package-' + library_name + '\n') + +if __name__ == "__main__": + parser = ArgumentParser(description='SBOM generator') + parser.add_argument('--repo-root-path', + type=str, + required=None, + default=os.getcwd(), + help='Path to the repository root.') + parser.add_argument('--source-path', + type=str, + required=None, + default=os.path.join(os.getcwd(), 'source'), + help='Path to the source code dir.') + args = parser.parse_args() + REPO_PATH = os.path.abspath(args.repo_root_path) + SOURCE_PATH = os.path.abspath(args.source_path) + + scan_dir()