Fix the tests

.github/workflows/testing.yml

@@ -8,12 +8,14 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: ${{ matrix.experimental }}
     strategy:
+      fail-fast: false
       matrix:
         codespell_pip_version: ['codespell']
         experimental: [false]
         include:
           - codespell_pip_version: 'git+https://github.com/codespell-project/codespell.git'
-            experimental: true
+            # Set this to true if git master is breaking the action's tests
+            experimental: false
 
     steps:
       - name: Checkout
@@ -62,12 +64,25 @@ jobs:
   diagnose_bats:
     name: Diagnose bats
     runs-on: ubuntu-latest
+    continue-on-error: true
     steps:
       - uses: actions/checkout@v1
       - name: Set up Python
         uses: actions/setup-python@v2
       - run: pip3 --quiet --quiet install git+https://github.com/codespell-project/codespell.git
       - run: |
+          # Skip the bats diagnostics
+          exit 0
+          # Simulate the Dockerfile COPY command
+          [ -d "${RUNNER_TEMP}/code/" ] || sudo mkdir -p ${RUNNER_TEMP}/code/
+          [ -f "${RUNNER_TEMP}/code/codespell-matcher.json" ] || sudo cp codespell-problem-matcher/codespell-matcher.json ${RUNNER_TEMP}/code/
+          #ls -alR ${RUNNER_TEMP}/code/
+          [ -d "/code/" ] || sudo mkdir -p /code/
+          [ -f "/code/codespell-matcher.json" ] || sudo cp codespell-problem-matcher/codespell-matcher.json /code/
+          #ls -alR /code/
+          # Add a random place BATS tries to put it
+          [ -d "/github/workflow/" ] || sudo mkdir -p /github/workflow/ && sudo chmod 777 /github/workflow/
+          #ls -alR /github/workflow/
           export INPUT_CHECK_FILENAMES=""
           export INPUT_CHECK_HIDDEN=""
           export INPUT_EXCLUDE_FILE=""
@@ -76,4 +91,4 @@ jobs:
           export INPUT_IGNORE_WORDS_LIST=""
           export INPUT_PATH="./test/testdata"
           export INPUT_ONLY_WARN=""
-          ./entrypoint.sh
+          ./entrypoint.sh || echo $?

entrypoint.sh

@@ -42,7 +42,7 @@ if [ "x${INPUT_IGNORE_WORDS_LIST}" != "x" ]; then
 fi
 echo "Resulting CLI options ${command_args}"
 exec 5>&1
-res=`{ { codespell ${command_args} ${INPUT_PATH}; echo $? 1>&4; } 1>&5; } 4>&1`
+res=`{ { codespell --count ${command_args} ${INPUT_PATH}; echo $? 1>&4; } 1>&5; } 4>&1`
 if [ "$res" = "0" ]; then
     echo "Codespell found no problems"
 else

requirements.txt

@@ -1 +1 @@
-codespell>=1.17.0
+codespell>=2.0.0

test/teardown.bash

@@ -0,0 +1,12 @@
+teardown() {
+  if ((BATS_ERROR_STATUS));then
+    echo "Failed with status $BATS_ERROR_STATUS"
+    if [[ "${output}" != "" ]];then
+      echo "Output at that time was:"
+      echo "$output"
+    else
+      echo "Output was empty"
+    fi
+    return $BATS_ERROR_STATUS
+  fi
+}

test/test.bats

@@ -3,12 +3,16 @@
 # Tests using the Bats testing framework
 # https://github.com/bats-core/bats-core
 
+# Add some test debug from https://github.com/bats-core/bats-core/issues/199
+# This currently breaks the tests, so only enabling when troubleshooting
+#load teardown
+
 ROOT_MISSPELLING_COUNT=5
 FILENAME_MISSPELLING_COUNT=1
 HIDDEN_MISSPELLING_COUNT=1
 EXCLUDED_MISSPELLING_COUNT=1
 BUILTIN_NAMES_MISSPELLING_COUNT=1
-IGNORE_WORDS_MISSPELLING_COUNT=6
+IGNORE_WORDS_MISSPELLING_COUNT=5
 SUBFOLDER_MISSPELLING_COUNT=1
 # From all files called example.txt
 EXAMPLE_MISSPELLING_COUNT=5
@@ -17,6 +21,17 @@ export RUNNER_TEMP="/foo/runner_temp"
 
 # This function runs before every test
 function setup() {
+    # Simulate the Dockerfile COPY command
+    [ -d "${RUNNER_TEMP}/code/" ] || sudo mkdir -p ${RUNNER_TEMP}/code/
+    [ -f "${RUNNER_TEMP}/code/codespell-matcher.json" ] || sudo cp codespell-problem-matcher/codespell-matcher.json ${RUNNER_TEMP}/code/
+    #ls -alR ${RUNNER_TEMP}/code/
+    [ -d "/code/" ] || sudo mkdir -p /code/
+    [ -f "/code/codespell-matcher.json" ] || sudo cp codespell-problem-matcher/codespell-matcher.json /code/
+    #ls -alR /code/
+    # Add a random place BATS tries to put it
+    [ -d "/github/workflow/" ] || sudo mkdir -p /github/workflow/ && sudo chmod 777 /github/workflow/
+    #ls -alR /github/workflow/
+
     # Set default input values
     export INPUT_CHECK_FILENAMES=""
     export INPUT_CHECK_HIDDEN=""
@@ -30,91 +45,119 @@ function setup() {
 }
 
 @test "Run with defaults" {
-    # codespell's exit status is the number of misspelled words found
-    expectedExitStatus=$((ROOT_MISSPELLING_COUNT + HIDDEN_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT))
+    errorCount=$((ROOT_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT))
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
 
     # Check output
-    [ "${lines[1]}" == "::add-matcher::${RUNNER_TEMP}/_github_workflow/codespell-matcher.json" ]
+    [ "${lines[0]}" == "::add-matcher::${RUNNER_TEMP}/_github_workflow/codespell-matcher.json" ]
     outputRegex="^Running codespell on '${INPUT_PATH}'"
-    [[ "${lines[2]}" =~ $outputRegex ]]
+    [[ "${lines[1]}" =~ $outputRegex ]]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
     [ "${lines[-3]}" == "Codespell found one or more problems" ]
     [ "${lines[-2]}" == "::remove-matcher owner=codespell-matcher-default::" ]
     [ "${lines[-1]}" == "::remove-matcher owner=codespell-matcher-specified::" ]
 }
 
 @test "Check file names" {
-    expectedExitStatus=$((ROOT_MISSPELLING_COUNT + HIDDEN_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT + FILENAME_MISSPELLING_COUNT))
+    errorCount=$((ROOT_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT + FILENAME_MISSPELLING_COUNT))
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_CHECK_FILENAMES=true
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }
 
 @test "Check a hidden file" {
-    # codespell --check-hidden has a somewhat unintuitive behavior. When run on
-    # a directory, codespell works on hidden files even if this option is not
-    # enabled. It's only when INPUT_PATH points directly at a hidden file that
-    # codespell ignores it by default.
-    expectedExitStatus=$HIDDEN_MISSPELLING_COUNT
+    errorCount=$HIDDEN_MISSPELLING_COUNT
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_CHECK_HIDDEN=true
     INPUT_PATH="./test/testdata/.hidden"
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }
 
 @test "Check a hidden file without INPUT_CHECK_HIDDEN set" {
-    expectedExitStatus=0
+    errorCount=0
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_PATH="./test/testdata/.hidden"
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }
 
 @test "Use an exclude file" {
-    expectedExitStatus=$((ROOT_MISSPELLING_COUNT + HIDDEN_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT - EXCLUDED_MISSPELLING_COUNT))
+    errorCount=$((ROOT_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT - EXCLUDED_MISSPELLING_COUNT))
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_EXCLUDE_FILE="./test/exclude-file.txt"
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }
 
 @test "Check the skip option" {
-    expectedExitStatus=$((ROOT_MISSPELLING_COUNT + HIDDEN_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT - EXAMPLE_MISSPELLING_COUNT))
+    errorCount=$((ROOT_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT - EXAMPLE_MISSPELLING_COUNT))
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_SKIP="example.txt"
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }
 
 @test "Use an additional builtin dictionary" {
-    expectedExitStatus=$((ROOT_MISSPELLING_COUNT + HIDDEN_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT + BUILTIN_NAMES_MISSPELLING_COUNT))
+    errorCount=$((ROOT_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT + BUILTIN_NAMES_MISSPELLING_COUNT))
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_BUILTIN="clear,rare,names"
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }
 
 @test "Use an ignore words file" {
-    expectedExitStatus=$((ROOT_MISSPELLING_COUNT + HIDDEN_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT - IGNORE_WORDS_MISSPELLING_COUNT))
+    errorCount=$((ROOT_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT - IGNORE_WORDS_MISSPELLING_COUNT))
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_IGNORE_WORDS_FILE="./test/ignore-words-file.txt"
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }
 
 @test "Use an ignore words list" {
-    expectedExitStatus=$((ROOT_MISSPELLING_COUNT + HIDDEN_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT - IGNORE_WORDS_MISSPELLING_COUNT))
+    errorCount=$((ROOT_MISSPELLING_COUNT + SUBFOLDER_MISSPELLING_COUNT - IGNORE_WORDS_MISSPELLING_COUNT))
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_IGNORE_WORDS_LIST="abandonned"
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }
 
 @test "Custom path" {
-    expectedExitStatus=$((SUBFOLDER_MISSPELLING_COUNT))
+    errorCount=$((SUBFOLDER_MISSPELLING_COUNT))
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_PATH="./test/testdata/subfolder"
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    [ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }
 
 @test "Only warn" {
-    expectedExitStatus=0
+    errorCount=0
+    # codespell's exit status is 0, or 65 if there are errors found
+    if [ $errorCount -eq 0 ]; then expectedExitStatus=0; else expectedExitStatus=65; fi
     INPUT_ONLY_WARN=true
     run "./entrypoint.sh"
     [ $status -eq $expectedExitStatus ]
+    #[ "${lines[-4 - $errorCount]}" == "$errorCount" ]
 }