From a418b2ba4648147e7d4578644f23f090d96e1ffd Mon Sep 17 00:00:00 2001 From: Soren Ptak Date: Tue, 5 Sep 2023 16:42:16 -0400 Subject: [PATCH] CSpell Action (#75) * Update the spell check action to use cSpell, add in exclude dirs and files directly, update tests to account for it, move the .cSpellWords.txt file to live in .github * Changes to the cspell action to account for running in a different directory, updating the config file as well --- .github/workflows/test.yml | 49 ++++++- spellings/action.yml | 134 +++++++++++++++--- spellings/cspell.config.yaml | 31 +++++ spellings/tools/README.md | 30 ---- spellings/tools/ablexicon | 88 ------------ spellings/tools/extract-comments | 41 ------ spellings/tools/find-unknown-comment-words | 152 --------------------- 7 files changed, 190 insertions(+), 335 deletions(-) create mode 100644 spellings/cspell.config.yaml delete mode 100644 spellings/tools/README.md delete mode 100755 spellings/tools/ablexicon delete mode 100755 spellings/tools/extract-comments delete mode 100755 spellings/tools/find-unknown-comment-words diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 596d0888..30c97181 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,17 +83,58 @@ jobs: test-spell-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + + - uses: actions/checkout@v3 with: - repository: FreeRTOS/coreMQTT - ref: main + repository: skptak/coreMQTT + ref: CI-CD-Updates path: coreMQTT + - name: Test spell check action uses: ./spellings with: path: coreMQTT + test-spell-checker-find-mistake: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/checkout@v3 + with: + repository: skptak/coreMQTT + ref: CI-CD-Updates + path: coreMQTT + + - name: Empty the lexicon + shell: bash + working-directory: coreMQTT + run: file=$(find . -name .cSpellWords.txt); readlink -f "$file" ; > "$file" + + - name: Test Spell Check Fails on Misspelled Word + continue-on-error: true + id: test-spellings-find-mistake + uses: ./spellings + with: + path: coreMQTT + + - env: + stepName: Check Failure Test Case + name: ${{ env.stepName }} + id: check-failure-test-cases + shell: bash + run: | + # ${{ env.stepName }} + exitStatus=0 + if [ "${{ steps.test-spellings-find-mistake.outcome}}" = "failure" ]; then + echo -e "${{ env.bashPass }} Functional | Failure | Fail on Misspelled Word | Had Expected "failure" ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} Functional | Failure | Fail on Misspelled Word | Had Unexpected "success" ${{ env.bashEnd }}" + exitStatus=1 + fi + exit $exitStatus + test-coverage-cop: runs-on: ubuntu-latest steps: diff --git a/spellings/action.yml b/spellings/action.yml index a4e7e291..833bb4fc 100644 --- a/spellings/action.yml +++ b/spellings/action.yml @@ -1,29 +1,123 @@ name: 'spellings' -description: 'CI spellings check' +description: 'cSpell CI spelling check' inputs: path: description: 'Path to repository folder to check spellings in.' required: false default: ./ + exclude-dirs: + description: "Comma separated list of directories to not spell check" + required: false + exclude-files: + description: "Comma separated list of files to not spell check" + required: false + include-extensions: + description: "Comma separated list of files to match to regex" + required: false + + runs: using: "composite" steps: - - name: Install spell - run: | - sudo apt-get install spell - sudo apt-get install util-linux - shell: bash - - name: Check spelling - working-directory: ${{ inputs.path }} - run: | - PATH=$PATH:$GITHUB_ACTION_PATH/tools - for lexfile in `find ./ -name lexicon.txt` - do dir=${lexfile%/lexicon.txt} - echo $dir - find-unknown-comment-words --directory $dir - if [ $? -ne "0" ] - then - exit 1 - fi - done - shell: bash + - env: + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0m + stepName: Set-Up The Spell Checker + name: ${{ env.stepName }} + id: spell-checker-setup + shell: bash + run: | + # ${{ env.stepName }} + echo -e "::group::${{ env.bashInfo }} ${{ env.stepName }} ${{ env.bashEnd }}" + + # Install the Dependencies we need to run the spell-checker + sudo apt-get install util-linux -y + sudo apt-get install fd-find -y + sudo apt-get install npm -y + sudo npm install -g cspell + echo -e "::endgroup::" + + # Add the Github Action Path to PATH + export PATH="$GITHUB_ACTION_PATH:$PATH" + + # Account for starting with an alternate path in a repository + # Do this by copying the cspell config and wordlist to the desired path + # Wrap in a set +e so Github doesn't throw an error if the file or + # directory already exists. + set +e + cp cspell.config.yaml ${{ inputs.path }} + mkdir ${{ inputs.path }}/.github + cp .github/.cSpellWords.txt ${{ inputs.path }}/.github + cd ${{ inputs.path }} + set -e + + # Make sure we have all the commands we need. + echo -e "${{ env.bashInfo }} fdfind --version $(fdfind --version) ${{ env.bashEnd }}" + echo -e "${{ env.bashInfo }} cspell --version $(cspell --version) ${{ env.bashEnd }}" + + # Only reach this line if no errors were hit above + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + + - env: + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0m + stepName: Spell Checker + name: ${{ env.stepName }} + id: run-spell-checker + working-directory: ${{ inputs.path }} + shell: bash + run: | + # ${{ env.stepName }} + #echo -e "::group::${{ env.stepName }}" + export PATH="$GITHUB_ACTION_PATH:$PATH" + exitStatus=0 + + # Parse the optional inputs + args="" + + # fd-find uses -E to exclude a file or directory + if [ -n "${{ inputs.exclude-dirs }}" ]; then + dirs=" -E " + dirs+="${{ inputs.exclude-dirs }}" + dirs="${dirs//,/ -E }" + args+=" ${dirs}" + fi + + # fd-find uses -E to exclude a file or directory + if [ -n "${{ inputs.exclude-files }}" ]; then + files=" -E " + files+="${{ inputs.exclude-files }}" + files="${files//,/ -E }" + args+=" ${files}" + fi + + # fd-find uses -e to exclude a file extension + if [ -n "${{ inputs.include-file-types }}" ]; then + file_types=" -e " + file_types+="${{ inputs.include-file-types }}" + file_types="${file_types//,/ -e }" + args+=" ${file_types}" + fi + + echo -e "${{ env.bashInfo }} Running: fdfind -e c -e h -e md -e txt -e readme ${args} --exec cspell lint --language-id C --color --show-context --show-suggestions --no-must-find-files -c cspell.config.yaml ${{ env.bashEnd }}" + + # Wrap in a set +e so Github doesn't stop the spell check from running + set +e + + # Find all relevant files, then check them for spelling mistakes + fdfind -e c -e h -e md -e txt -e readme ${args} --exec-batch \ + cspell lint --language-id C --color --show-context --show-suggestions --no-must-find-files -c cspell.config.yaml + exitStatus=$? + set -e + + echo -e "::endgroup::" + if [ $exitStatus -eq 0 ]; then + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" + fi + exit $exitStatus diff --git a/spellings/cspell.config.yaml b/spellings/cspell.config.yaml new file mode 100644 index 00000000..911ce1d8 --- /dev/null +++ b/spellings/cspell.config.yaml @@ -0,0 +1,31 @@ +--- +$schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json +version: '0.2' +# Allows things like stringLength +allowCompoundWords: true + +# Read files not to spell check from the git ignore +useGitignore: true + +# Language settings for C +languageSettings: + - caseSensitive: false + enabled: true + languageId: c + locale: "*" + +# Add a dictionary, and the path to the word list +dictionaryDefinitions: + - name: freertos-words + path: '.github/.cSpellWords.txt' + addWords: true + +dictionaries: + - freertos-words + +# Paths and files to ignore +ignorePaths: + - 'dependency' + - 'docs' + - 'ThirdParty' + - 'History.txt' diff --git a/spellings/tools/README.md b/spellings/tools/README.md deleted file mode 100644 index b0bd89aa..00000000 --- a/spellings/tools/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Pre-requisites to running the spell check scripts - -1. In your GNU environment, install the *spell* and *getopt* programs. Use the following commands in Debian distributions, to install the packages (*getopt* is part of the `util-linux` package): - ```shell - apt-get install spell - apt-get install util-linux - ``` - -1. Add the folder containing the **spellings/tools/ablexicon**, **spellings/tools/extract-comments**, and **spellings/tools/find-unknown-comment-words** scripts to your system's PATH. - ```shell - export PATH=/spellings/tools:$PATH - ``` - -# How to create a lexicon.txt for a new library. - -1. Ensure there does not exist a file called "lexicon.txt" in your library's root directory. Run the following command to create a lexicon.txt for your library: - ```shell - find-unknown-comment-words -d /path/to/your/library/root > /path/to/your/library/root/lexicon.txt - ``` - -1. Check the contents of */path/to/your/library/root/lexicon.txt* for any misspelled words. Fix them in your library's source code and delete them from the lexicon.txt. - -# How to run for changes to an existing library. - -1. If there exists a lexicon.txt in the library's root directory, run the following command: - ```shell - find-unknown-comment-words -d /path/to/your/library/root/lexicon.txt - ``` - -1. Add any non-dictionary correctly spelled words to */path/to/your/library/root/lexicon.txt*. Fix any misspelled words in your code comment change. diff --git a/spellings/tools/ablexicon b/spellings/tools/ablexicon deleted file mode 100755 index de790a93..00000000 --- a/spellings/tools/ablexicon +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# -# ablexicon - Compare an input list of words against a dictionary and -# optional lexicon. If any words are in neither the dictionary nor the -# lexicon, log them to stdout. -# -set -e -set -f - -function usage () { - echo "Find occurrences of non-dictionary/lexicon words" - echo "" - echo "Usage:" - echo " ${0##*/} [options]" - echo "" - echo "Options:" - echo " -f, --file source text (defaults to /dev/fd/0)" - echo " -l, --lexicon lexicon file (one word per line)" - echo " -h, --help display this help" - exit 1 -} - -# -# Verify that required commands are present -# -REQUIRED=( "spell" "getopt" ) -for i in "${REQUIRED[@]}" -do - command -v $i"" >/dev/null - if [ $? -ne "0" ] - then - echo "'"$i"' must be installed, exiting...">&2 - exit 1 - fi -done - -GETOPT_OUT=`getopt -o hf:l: --long help,file:,lexicon: -n "${0##*/}" -- "$@"` -if [ $? != 0 ] -then - echo "Exiting..." >&2 - exit 1 -fi - -eval set -- "$GETOPT_OUT" - -INFILE=/dev/fd/0 -LEXICON=/dev/null -while true; do - case "$1" in - -h | --help ) usage $0 ;; - -f | --file ) INFILE="$2"; shift 2 ;; - -l | --lexicon ) LEXICON="$2"; shift 2 ;; - -- ) shift; break ;; - * ) break ;; - esac -done - -if [ ! -f $INFILE"" ] && [ $INFILE"" != /dev/fd/0 ] -then - echo "Invalid input file" - usage -fi - -# -# Search for all input words, sort them removing duplicate words in -# the process and then find them in the dictionary. -# -for word in `cat $INFILE"" | sort -u | spell` -do - # - # Search for each remaining word in the lexicon - # - if [ $LEXICON"" != /dev/null ] - then - if ! grep -w -q "$word" $LEXICON"" - then - # - # The word is neither in the dictionary nor the lexicon, send - # it to stdout. - # - echo $word - fi - else - # if the lexicon is /dev/null i.e. it is not provided by the user, - # then we should just echo the word. - echo $word - fi -done diff --git a/spellings/tools/extract-comments b/spellings/tools/extract-comments deleted file mode 100755 index f52cb303..00000000 --- a/spellings/tools/extract-comments +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# -# Extract comments from C/C++ files -# -set -e -set -f - -function usage () { - echo "Extract comments from C/C++ files" - echo "" - echo "usage: "${0##*/}" file-list" - exit 1 -} - -if [ $# -lt 1 ] -then - usage $0 -fi - -if [ $1 = "-h" ] || [ $1 == "--help" ] -then - usage $0 -fi - -while test $# -gt 0 -do - if [ ! -f $1 ] - then - echo $0": '"$1"' is not a file." 2>/dev/null - exit 1 - fi -# -# Extract all words from C/C++ language comments; add line -# numbers to aid in searching. -# -# NOTE: This has some limitations. For example, it prints -# non-comment text at the beginning of a comment line. -# - nl -ba $1 | awk '/\/\// {print $0}; /\/\*/ {comment=1}; {if(comment) print $0}; /\*\// {comment=0}' - shift -done diff --git a/spellings/tools/find-unknown-comment-words b/spellings/tools/find-unknown-comment-words deleted file mode 100755 index 228cb65c..00000000 --- a/spellings/tools/find-unknown-comment-words +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash -# -# Locate unknown words in C/C++ comments. Uses "extract-comments" -# and "ablexicon" scripts. -# -set -o nounset -set -o pipefail -set -o errexit -set -f - -BLUE="\e[1;34m" -GREEN="\e[1;32m" -DEFAULTFG="\e[39m" - -function usage () { - echo "Find unknown words in C/C++ comments" - echo "" - echo "Usage:" - echo " ${0##*/} [options]" - echo "" - echo "Options:" - echo " -d, --directory directory to scan (defaults to .)" - echo " -l, --lexicon lexicon file (one word per line, default 'lexicon.txt')" - echo " -t, --terse terse output only (enabled if no lexicon available)" - echo " -h, --help display this help" - exit 1 -} - -# -# Verify that required commands are present -# -REQUIRED=( "extract-comments" "ablexicon" "getopt" ) -for i in "${REQUIRED[@]}" -do - command -v $i"" >/dev/null - if [ $? -ne "0" ] - then - echo "Can't find '"$i"' , exiting...">&2 - exit 1 - fi -done - -GETOPT_OUT=`getopt -o htd:l: --long help,terse,directory:,lexicon: -n "${0##*/}" -- "$@"` -if [ $? != 0 ] -then - echo "Exiting..." >&2 - exit 1 -fi - -eval set -- "$GETOPT_OUT" - -DIRNAME=/dev/fd/0 -LEXICON= -STATUS= -TERSE= -while true; do - case "$1" in - -h | --help ) usage $0 ;; - -t | --terse ) TERSE=1; shift ;; - -d | --directory ) DIRNAME="$2"; shift 2 ;; - -l | --lexicon ) LEXICON="$2"; shift 2 ;; - -- ) shift; break ;; - * ) break ;; - esac -done - -if [ ! -d $DIRNAME"" ] -then - echo "Invalid directory: "$DIRNAME - usage -fi - -if [ $LEXICON"" = "" ] -then - if [ -f $DIRNAME/lexicon.txt ] - then - LEXICON=$DIRNAME/lexicon.txt - else - LEXICON=/dev/null - TERSE=1 - fi -fi - -TMPFILE=${0##*/}-$USER-$RANDOM - -unknowns=( "not-used" ) # get around empty array with nounset -# Symlinks will be ignored in this spell check. `-type f` switch in `find` command will ignore symlinks. - -extract-comments `find $DIRNAME \( -iname \*.[ch] -o -iname \*.dox \) -type f` | - tr [:upper:] [:lower:] | - grep -o -E '[a-zA-Z]+' | - ablexicon -l $LEXICON > $TMPFILE - -readarray -O 1 -t unknowns < $TMPFILE -rm -f $TMPFILE - -for word in "${unknowns[@]}" -do - if [ $word"" == "not-used" ] - then - continue - fi - - if [ $TERSE"" != "" ] - then - echo $word - continue - fi - - # Symlinks will be ignored in this spell check. `-type f` switch in `find` command will ignore symlinks. - for file in `find $DIRNAME \( -iname \*.[ch] -o -iname \*.dox \) -type f` - do - if [[ $file == *"third_party"* || $file == *"CMock"* ]] - then - continue - fi - # Disable errexit here, extract-comments can return non-zero - set +e - # - # A little inefficient here; we will grep twice, once to detect - # the unknown word and another to print it with color highlighting. - # If there's a way to preserve ANSI color output with the first - # search and reuse it within the if statement (I gave up trying - # to find one after a few minutes), that would be nice. - # - extract-comments $file | grep -iw $word > /dev/null - if [ $? == "0" ] - then - if [ $STATUS"" != "1" ] - then - echo -e $GREEN"############################################################################"$DEFAULTFG - echo -e $GREEN"#"$DEFAULTFG - echo -e $GREEN"# Unknown word(s) found. Please either correct the spelling or add them"$DEFAULTFG - echo -e $GREEN"# to the lexicon file '"$LEXICON"'".$DEFAULTFG - echo -e $GREEN"#"$DEFAULTFG - echo -e $GREEN"############################################################################"$DEFAULTFG - STATUS=1 # Return non-zero status if any unidentified words are found - fi - echo "" - echo -e $BLUE$file$DEFAULTFG - echo "" - extract-comments $file | grep --color=always -iw $word | GREP_COLORS="mt=01;32" grep --color=always -E -e '^[ \t]*[0-9]+' - fi - # Re-enable errexit - set -o errexit - done -done - -if [ $STATUS"" = "1" ] -then - exit 1 -fi