From 312236ef6283f295137f759d99c893011dc170bf Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Wed, 11 Dec 2024 11:24:23 -0800 Subject: [PATCH] DAOS-15677 cq: add copyright GHA and remove required-githooks (#15552) Add GHA to check for copyright update. Move core logic from update-copyright githook into check_update_copyright.sh so the logic is shared between the githook and GHA. Remove required-githooks watermark since it did not work in all scenarios and the GHA checks are more secure than client-side githooks. Combination of #15552, #15596, #15636, #15639 Skip-build: true Signed-off-by: Dalton Bohning --- .github/workflows/linting.yml | 15 +++ Jenkinsfile | 19 --- utils/cq/check_update_copyright.sh | 111 ++++++++++++++++++ utils/githooks/commit-msg | 3 - utils/githooks/commit-msg.d/10-watermark.py | 87 -------------- utils/githooks/hook_base.sh | 6 - .../githooks/pre-commit.d/10-update-copyright | 81 ------------- .../pre-commit.d/10-update-copyright.sh | 17 +++ .../{30-Jenkinsfile => 30-Jenkinsfile.sh} | 4 +- .../{50-clang-format => 50-clang-format.sh} | 0 10 files changed, 145 insertions(+), 198 deletions(-) create mode 100755 utils/cq/check_update_copyright.sh delete mode 100755 utils/githooks/commit-msg delete mode 100755 utils/githooks/commit-msg.d/10-watermark.py delete mode 100755 utils/githooks/pre-commit.d/10-update-copyright create mode 100755 utils/githooks/pre-commit.d/10-update-copyright.sh rename utils/githooks/pre-commit.d/{30-Jenkinsfile => 30-Jenkinsfile.sh} (83%) rename utils/githooks/pre-commit.d/{50-clang-format => 50-clang-format.sh} (100%) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 8c1553f10d4..c6350efceeb 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -197,6 +197,20 @@ jobs: - name: Run check run: yamllint --format github . + copyright: + name: Copyright check + runs-on: ubuntu-24.04 + steps: + - name: Check out source repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Run check + run: | + \[ ! -x ./utils/cq/check_update_copyright.sh \] || ./utils/cq/check_update_copyright.sh \ + $(git merge-base HEAD ${{ github.event.pull_request.base.sha || github.ref }}) gha + linting-summary: name: Linting Summary runs-on: ubuntu-24.04 @@ -211,6 +225,7 @@ jobs: - codespell # - clang-format # not required - yaml-lint + - copyright if: (!cancelled()) steps: - name: Check if any job failed diff --git a/Jenkinsfile b/Jenkinsfile index 8b2c027b15f..0eb77395fbf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -365,25 +365,6 @@ pipeline { stage('Check PR') { when { changeRequest() } parallel { - stage('Used Required Git Hooks') { - steps { - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS', - message: 'PR did not get committed with required git hooks. ' + - 'Please see utils/githooks/README.md.') { - sh 'if ! ' + cachedCommitPragma('Required-githooks', 'false') + '''; then - echo 'PR did not get committed with required git hooks. ' + - 'Please see utils/githooks/README.md.' - exit 1 - fi''' - } - } - post { - unsuccessful { - echo 'PR did not get committed with required git hooks. ' + - 'Please see utils/githooks/README.md.' - } - } - } // stage('Used Required Git Hooks') stage('Branch name check') { when { changeRequest() } steps { diff --git a/utils/cq/check_update_copyright.sh b/utils/cq/check_update_copyright.sh new file mode 100755 index 00000000000..5bb4016915f --- /dev/null +++ b/utils/cq/check_update_copyright.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# +# Copyright 2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +# Check or update copyright date in modified files. +# Usage: check_update_copyright.sh +# mode "githook" will update copyright dates in place. +# mode "gha" will just print a warning in a GHA-compatible format. + +set -e + +git_target="$1" +mode="$2" +case "$mode" in + "githook" | "gha") + ;; + *) + echo "Usage: check_update_copyright.sh " + exit 1 +esac + +# Navigate to repo root +PARENT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +cd "$PARENT_DIR"/../../ + + +regex='(^[[:blank:]]*[\*/]*.*)((Copyright[[:blank:]]*)([0-9]{4})(-([0-9]{4}))?)([[:blank:]]*(Intel.*$))' +year=$(date +%Y) +errors=0 +targets=( + # Entries with wildcard. These must be first and start with '*' or + # older versions of git will return files that were not changed. + '*.c' + '*.h' + '*.go' + '*.py' + '*.proto' + '*.java' + '*.yml' + '*.yaml' + '*.sh' + '*.bash' + '*Dockerfile*' + '*README*' + '*LICENSE*' + '*NOTICE*' + '*.txt' + '*.md' + # Entries without a wildcard + 'Makefile' + 'Jenkinsfile' + 'SConscript' + 'SConstruct' + 'copyright' + '.env' +) + +if [ -z "$files" ]; then + files=$(git diff "$git_target" --cached --diff-filter=AM --name-only -- "${targets[@]}") +else + echo " Checking against custom files" +fi + +os=$(uname -s) + +. utils/githooks/git-version.sh + +for file in $files; do + if [[ "$file" == *vendor* ]] || [[ "$file" == *pb.go ]] || + [[ "$file" == *_string.go ]] || [[ "$file" == *pb-c* ]] || + { [ "$mode" == "githook" ] && + [ "$git_vercode" -ge 2030000 ] && + [ "$(git diff --cached -I Copyright "$file")" = '' ]; }; then + continue + fi + read -r y1 y2 <<< "$(sed -nre "s/^.*$regex.*$/\4 \6/p" "$file")" + if [[ -z $y1 ]] ; then + # Print warning but don't error on non-existent copyright + echo " Copyright Information not found in: $file" + elif [[ $y1 -ne $year && $year -ne $y2 ]] ; then + if [[ "$mode" == "githook" ]]; then + # Update copyright in place + if ! git reset "$file"; then + echo " Unable to un-stage $file" + errors=$((errors + 1)) + fi + if [[ "$os" == 'Linux' ]]; then + sed -i -re "s/$regex/\1Copyright $y1-$year \8/" "$file" + else + sed -i '' -re "s/$regex/\1Copyright $y1-$year \8/" "$file" + fi + + if ! git add "$file"; then + echo " Unable to re-stage $file" + errors=$((errors + 1)) + fi + elif [[ "$mode" == "gha" ]]; then + # Print error but do not update + lineno="$(grep -nE "$regex" "$file" | cut -f1 -d:)" + echo "::error file=$file,line=$lineno::Copyright out of date" + errors=$((errors + 1)) + fi + fi +done + +if [[ $errors -ne 0 ]]; then + echo " $errors errors while checking/fixing copyrights." + exit 1 +fi diff --git a/utils/githooks/commit-msg b/utils/githooks/commit-msg deleted file mode 100755 index 59aec7c8578..00000000000 --- a/utils/githooks/commit-msg +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -. utils/githooks/hook_base.sh diff --git a/utils/githooks/commit-msg.d/10-watermark.py b/utils/githooks/commit-msg.d/10-watermark.py deleted file mode 100755 index 667b27afa78..00000000000 --- a/utils/githooks/commit-msg.d/10-watermark.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -# Disable check for module name. disable-next doesn't work here. -# pylint: disable=invalid-name -# pylint: enable=invalid-name -"""Add watermark to commit message""" -import os -import sys - - -def check_if_run(name): - """Check existence of a file, removes file""" - if not os.path.exists(f".{name}"): - print(f"Required githook {name} is not installed. See utils/githooks/README.md") - return False - os.remove(f".{name}") - return True - - -def find_hooks(): - """Find the required git hooks""" - hooks = [] - for fname in os.listdir('utils/githooks'): - if fname == "commit-msg.d": - continue - if not fname.endswith('.d'): - continue - if not os.path.isdir(os.path.join("utils/githooks", fname)): - continue - hooks.append(fname[:-2]) - return hooks - - -def emit_watermark(msg): - """Print the watermark to the commit message""" - msg.write("Required-githooks: true\n") - skipped = os.environ.get("DAOS_GITHOOK_SKIP", None) - if skipped: - msg.write(f"Skipped-githooks: {skipped}\n") - msg.write("\n") - - -def run_check(): - """Run the checks for the required commit hooks""" - empty_commit = True - msg = [] - with open(sys.argv[1], "r") as commit_msg: - msg = commit_msg.readlines() - for line in msg: - if line.startswith("Required-githooks: true"): - sys.exit(0) - elif line.startswith("Signed-off-by"): - continue - elif line.startswith("Change-Id"): - continue - elif line.strip() == "": - continue - elif line.startswith("# ------------------------ >8 ------------------------"): - break - elif line.startswith("#"): - continue - empty_commit = False - if empty_commit: - sys.exit(0) - - hooks = find_hooks() - if not all(list(map(check_if_run, hooks))): - sys.exit(0) - - with open(sys.argv[1], "w") as commit_msg: - hook_emitted = False - for line in msg: - if not hook_emitted: - if line.startswith("Change-Id"): - emit_watermark(commit_msg) - hook_emitted = True - if line.startswith("Signed-off-by"): - emit_watermark(commit_msg) - hook_emitted = True - elif line.startswith("#"): - emit_watermark(commit_msg) - hook_emitted = True - commit_msg.write(line) - - -if __name__ == "__main__": - print("Checking that required hooks ran") - run_check() diff --git a/utils/githooks/hook_base.sh b/utils/githooks/hook_base.sh index 3def155df68..7e3a61cc544 100755 --- a/utils/githooks/hook_base.sh +++ b/utils/githooks/hook_base.sh @@ -54,9 +54,3 @@ run-parts() { } run-parts utils/githooks/"${hook}".d "$@" 1>&2 - -# Create temp file for the commit-msg watermark to indicate this hook was ran. -# But not for the commit-msg itself. -if [ "${hook}" != "commit-msg" ]; then - touch ".${hook}" -fi diff --git a/utils/githooks/pre-commit.d/10-update-copyright b/utils/githooks/pre-commit.d/10-update-copyright deleted file mode 100755 index e2641848cd7..00000000000 --- a/utils/githooks/pre-commit.d/10-update-copyright +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -# -# Copyright 2022-2024 Intel Corporation. -# -# SPDX-License-Identifier: BSD-2-Clause-Patent -# -# A git hook to validate and correct the copyright date in source files. - -_print_githook_header "Copyright" -if [ -e .git/MERGE_HEAD ]; then - echo "Merge commit. Skipping" - exit 0 -fi - -echo "Updating copyright headers" - -regex='(^[[:blank:]]*[\*/]*.*)((Copyright[[:blank:]]*)([0-9]{4})(-([0-9]{4}))?)([[:blank:]]*(Intel.*$))' -year=$(date +%Y) -errors=0 -targets=( - # Entries with wildcard. These must be first and start with '*' or - # older versions of git will return files that were not changed. - '*.c' - '*.h' - '*.go' - '*.py' - '*.proto' - '*.java' - '*.yml' - '*.yaml' - '*.sh' - '*.bash' - '*Dockerfile*' - '*README*' - '*LICENSE*' - '*NOTICE*' - '*.txt' - '*.md' - # Entries without a wildcard - 'Makefile' - 'Jenkinsfile' - 'SConscript' - 'SConstruct' - 'copyright' - '.env' -) - -if [ -z "$files" ]; then - files=$(git diff "$TARGET" --cached --diff-filter=AM --name-only -- "${targets[@]}") -else - echo " Checking against custom files" -fi - -os=$(uname -s) - -. utils/githooks/git-version.sh - -for file in $files; do - if [[ "$file" == *vendor* ]] || [[ "$file" == *pb.go ]] || - [[ "$file" == *_string.go ]] || [[ "$file" == *pb-c* ]] || - { [ "$git_vercode" -ge 2030000 ] && - [ "$(git diff --cached -I Copyright "$file")" = '' ]; }; then - continue - fi - read -r y1 y2 <<< "$(sed -nre "s/^.*$regex.*$/\4 \6/p" "$file")" - if [[ -z $y1 ]] ; then - echo " Copyright Information not found in: $file" - errors=$((errors + 1)) - elif [[ $y1 -ne $year && $year -ne $y2 ]] ; then - git reset "$file" || (echo " Unable to un-stage $file" && exit 1) - if [ "$os" == 'Linux' ] - then - sed -i -re "s/$regex/\1Copyright $y1-$year \8/" "$file" - else - sed -i '' -re "s/$regex/\1Copyright $y1-$year \8/" "$file" - fi - - git add "$file" || (echo " Unable to re-stage $file" && exit 1) - fi -done -[[ $errors = 0 ]] || (echo " $errors errors while checking/fixing copyrights.") diff --git a/utils/githooks/pre-commit.d/10-update-copyright.sh b/utils/githooks/pre-commit.d/10-update-copyright.sh new file mode 100755 index 00000000000..b88cce8e634 --- /dev/null +++ b/utils/githooks/pre-commit.d/10-update-copyright.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2022-2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +# A git hook to validate and correct the copyright date in source files. + +_print_githook_header "Copyright" +if [ -e .git/MERGE_HEAD ]; then + echo "Merge commit. Skipping" + exit 0 +fi + +echo "Updating copyright headers" + +utils/cq/check_update_copyright.sh "$TARGET" githook diff --git a/utils/githooks/pre-commit.d/30-Jenkinsfile b/utils/githooks/pre-commit.d/30-Jenkinsfile.sh similarity index 83% rename from utils/githooks/pre-commit.d/30-Jenkinsfile rename to utils/githooks/pre-commit.d/30-Jenkinsfile.sh index 6ee2efed4aa..97df5e44222 100755 --- a/utils/githooks/pre-commit.d/30-Jenkinsfile +++ b/utils/githooks/pre-commit.d/30-Jenkinsfile.sh @@ -23,9 +23,9 @@ echo "Checking syntax" HOST="${HOST:-build.hpdd.intel.com}" CURL_VERBOSE=${CURL_VERBOSE:-""} CURL_PROXY="${CURL_PROXY:+-x }${CURL_PROXY:-}" -CURL_OPTS="$CURL_PROXY $CURL_VERBOSE -s" +CURL_OPTS=("$CURL_PROXY" "$CURL_VERBOSE" -s) URL="https://$HOST/pipeline-model-converter/validate" -if ! output=$(curl $CURL_OPTS -s -X POST -F "jenkinsfile=<${1:-Jenkinsfile}" "$URL"); then +if ! output=$(curl "${CURL_OPTS[@]}" -s -X POST -F "jenkinsfile=<${1:-Jenkinsfile}" "$URL"); then echo " Failed to access $URL. Skipping" exit 0 fi diff --git a/utils/githooks/pre-commit.d/50-clang-format b/utils/githooks/pre-commit.d/50-clang-format.sh similarity index 100% rename from utils/githooks/pre-commit.d/50-clang-format rename to utils/githooks/pre-commit.d/50-clang-format.sh