From a1deac0c64dbe14e7b315a674c55c4dd0a795102 Mon Sep 17 00:00:00 2001 From: Soren Ptak Date: Wed, 16 Aug 2023 10:36:02 -0700 Subject: [PATCH] Updates to the manifest verifier (#74) Make the Manifest Verifier not fail if the repo doesn't have submodules --- manifest-verifier/action.yml | 43 +++++++++++++++++---- manifest-verifier/verify_manifest.py | 56 +++++++++++++++++++--------- 2 files changed, 75 insertions(+), 24 deletions(-) diff --git a/manifest-verifier/action.yml b/manifest-verifier/action.yml index acbb98e7..a7c9c58a 100644 --- a/manifest-verifier/action.yml +++ b/manifest-verifier/action.yml @@ -9,23 +9,52 @@ inputs: description: 'List of comma-separated relative path to submodules that should not be present in manifest.yml. Eg libraries/thirdparty/tinycbor,libraries/thirdparty/mbedtls' required: false default: '' - fail-on-incorrect-version: + fail-on-incorrect-version: description: 'Boolean flag to indicate if verification should fail if any submodule version in manifest.yml file is incorrect or stale.' required: false default: 'false' runs: using: "composite" - steps: - - name: Install dependencies - run: pip install -r $GITHUB_ACTION_PATH/requirements.txt + steps: + - env: + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0m + stepName: Install Manifest Verification Dependencies + name: ${{ env.stepName }} shell: bash - - name: Run verifier script + run: | + # ${{ env.stepName }} + echo "::group::${{ env.stepName }}" + pip install -r $GITHUB_ACTION_PATH/requirements.txt + exitStatus=$? + echo "::endgroup::" + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + + - env: + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0m + stepName: Run Manifest Verification Script + name: ${{ env.stepName }} working-directory: ${{ inputs.path }} + shell: bash run: | + # ${{ env.stepName }} + echo "::group::${{ env.stepName }}" if [[ "${{ inputs.fail-on-incorrect-version }}" == "true" ]]; then - echo 'Value of flag is ${{ inputs.fail-on-incorrect-version }}' + echo -e "${{ env.bashInfo}} Value of flag is ${{ inputs.fail-on-incorrect-version }} ${{ env.bashEnd }}" python3 $GITHUB_ACTION_PATH/verify_manifest.py --ignore-submodule-path=${{ inputs.exclude-submodules }} --fail-on-incorrect-version else python3 $GITHUB_ACTION_PATH/verify_manifest.py --ignore-submodule-path=${{ inputs.exclude-submodules }} fi - shell: bash + exitStatus=$? + echo "::endgroup::" + if [ "$exitStatus" = "0" ]; then + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" + exit 1 + fi diff --git a/manifest-verifier/verify_manifest.py b/manifest-verifier/verify_manifest.py index 0389ed21..9bfbf75e 100644 --- a/manifest-verifier/verify_manifest.py +++ b/manifest-verifier/verify_manifest.py @@ -7,22 +7,33 @@ from git import Repo from argparse import ArgumentParser +# At time of writing GitHub Runners will select Bash by default. +bashPass= "\033[32;1mPASSED -" +bashInfo= "\033[33;1mINFO -" +bashFail= "\033[31;1mFAILED -" +bashEnd= "\033[0m" + REPO_PATH='' # List of submodules excluded from manifest.yml file IGNORE_SUBMODULES_LIST = [] # Obtain submodule path of all entries in manifest.yml file. -def read_manifest(): +def read_manifest(git_modules, path_manifest): dict = {} - # Read YML file - path_manifest = os.path.join(REPO_PATH, 'manifest.yml') - assert os.path.exists(path_manifest), 'Missing manifest.yml' + with open(git_modules, 'r') as fp: + module_lines = fp.read() + if "submodule" not in module_lines: + print("{0} No submodules in the repo. Exiting. {1}".format(bashInfo, bashEnd)) + exit(0) + with open(path_manifest, 'r') as fp: manifest_data = fp.read() yml = load(manifest_data, Loader=Loader) - assert 'dependencies' in yml, 'Manifest YML parsing error' + if "dependencies" not in yml: + print("{0} No dependencies in {1}. Exiting {2}".format(bashInfo, path_manifest, bashEnd)) + exit(0) # Iterate over all the "dependencies" entries, verify that # each contains entries for the following hierarchy: @@ -34,8 +45,10 @@ def read_manifest(): # path: # for dep in yml['dependencies']: - assert 'version' in dep, "Failed to parse 'version/tag' for submodule" - assert 'repository' in dep and 'path' in dep['repository'] and 'url' in dep['repository'], "Failed to parse 'repository' object for submodule" + assert 'version' in dep, f"Failed to parse 'version/tag' for submodule {dep}" + assert 'repository' in dep, f"Failed to parse 'repository' for {dep}" + assert 'path' in dep['repository'], f"Failed to parse 'path' for {dep}" + assert 'url' in dep['repository'], f"Failed to parse 'repository' object for {dep}" dict[dep['name']] = (dep['repository']['path'], dep['version']) return dict @@ -75,30 +88,35 @@ def get_all_submodules(): # Convert any relative path (like './') in passed argument to absolute path. REPO_PATH = os.path.abspath(args.repo_root_path) + # Read YML file + path_manifest = os.path.join(REPO_PATH, 'manifest.yml') + assert os.path.exists(path_manifest), f"{bashFail} NO FILE {REPO_PATH}/manifest.yml {bashEnd}" - submodules_info_from_manifest = read_manifest() + git_modules = os.path.join(REPO_PATH, '.gitmodules') + assert os.path.exists(path_manifest), f"{bashFail} NO FILE {REPO_PATH}/.gitmodules {bashEnd}" + + submodules_info_from_manifest = read_manifest(git_modules, path_manifest) submodule_path_from_manifest = [pair[0] for pair in submodules_info_from_manifest.values()] submodule_path_from_manifest = sorted(submodule_path_from_manifest) submodules_info_from_git = get_all_submodules() submodule_path_from_git = sorted(submodules_info_from_git.keys()) - print(REPO_PATH) - print('\nList of submoduled libraries being verified:', submodule_path_from_git) + print("{0} CHECKING PATH: {1}{2}".format(bashInfo, REPO_PATH, bashEnd)) + print("{0} List of submodules being verified: {1}{2}".format(bashInfo, submodule_path_from_git, bashEnd)) # Check that manifest.yml contains entries for all submodules # present in repository. if submodule_path_from_manifest != submodule_path_from_git: - print('manifest.yml is missing entries for:') # Find list of library submodules missing in manifest.yml for git_path in submodule_path_from_git: if git_path not in submodule_path_from_manifest: - print(git_path) + print(f"{bashFail} {git_path} not in {submodule_path_from_manifest} {bashEnd}") sys.exit(1) # Verify that manifest contains correct versions of submodules pointers. mismatch_flag = False - print('\nVerifying that manifest.yml versions are up-to-date.....') + print(f"{bashInfo} Verifying that manifest.yml versions are up-to-date..... {bashEnd}") for submodule_name, submodule_info in submodules_info_from_manifest.items(): relative_path = submodule_info[0] manifest_commit = submodule_info[1] @@ -106,13 +124,17 @@ def get_all_submodules(): submodule.remote('origin').fetch() submodule.git.checkout(manifest_commit) if (submodules_info_from_git[relative_path] != submodule.head.commit): - print('manifest.yml does not have correct commit ID for', submodule_name,'manifest Commit=(',manifest_commit, submodule.head.commit,') Actual Commit=',submodules_info_from_git[relative_path]) + print("{0} manifest.yml does not have correct commit ID for {1} manifest Commit=({2},{3}) Actual Commit={4} {5}" + .format(bashFail, submodule_name, manifest_commit, submodule.head.commit, submodules_info_from_git[relative_path], bashEnd)) mismatch_flag = True - - if mismatch_flag and args.fail_on_incorrect_version: + + if ( True == mismatch_flag ) and args.fail_on_incorrect_version: + print(f"{bashFail} MISMATCHES WERE FOUND IN THE MANIFEST. EXITING WITH FAILURE DUE {bashEnd}") sys.exit(1) + elif ( True == mismatch_flag ): + print(f"{bashInfo} MISMATCHES WERE FOUND IN THE MANIFEST. EXITING WITH SUCCESS AS FAIL ON INCORRECT VERSION WAS NOT SET {bashEnd}") - print('\nmanifest.yml file has been verified!') + print("{0} manifest.yml file has been verified!{1}". format(bashPass, bashEnd)) sys.exit(0)