From 79a15d072ad7cb3232416af031f077e3b3610617 Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Mon, 3 Mar 2025 22:16:28 +0100 Subject: [PATCH 01/14] Skip build test for non essentials #690 --- .github/workflows/build-and-test.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 4ab3d8c8..51aec1d6 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -14,28 +14,46 @@ jobs: RM_TS_DIR: "/tmp/rmlint-unit-testdir" steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Check changes" + # TODO also include skipping build but allow docs to be processed + run: | + if git diff --exit-code --name-only origin/master . ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then + echo "SKIP_CHANGES=false" >> "$GITHUB_ENV" + else + echo "SKIP_CHANGES=true" >> "$GITHUB_ENV" + fi + - name: "Prepare build environment" run: | - sudo apt update - sudo apt install -y --no-install-recommends \ + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ scons python3-sphinx gettext python3-setuptools \ libblkid-dev libelf-dev libglib2.0-dev libjson-glib-dev \ clang python3-pip python3-cffi python3-dev libffi-dev pip3 install -r tests/requirements.txt + - name: "Build" # Todo: eventually run tests with valgrind (RM_TS_USE_VALGRIND) # Todo enable slow tests in pytest run: | scons config scons VERBOSE=1 DEBUG=1 O=release + - name: "Prepare test environment" + # TODO Ignore linter warning. See https://github.com/github/vscode-github-actions/issues/222 + if: ${{ env.SKIP_CHANGES == 'true' }} # The test suite is seriously disk-intensive. Given that linux # instances hosted in GitHub have 16G of RAM available we will # use it to speed up the run. run: | sudo mkdir "${RM_TS_DIR}" sudo mount -o size=12G,nr_inodes=0 -t tmpfs tmpfs "${RM_TS_DIR}" - - name: "Test" + + - name: "Test it" + if: ${{ env.SKIP_CHANGES == 'true' }} run: | RM_TS_PRINT_CMD=1 RM_TS_PEDANTIC=0 python -m pytest -s -v - name: CoW tests From 06de9eabd822fdafce5fabb09b6de28528baab06 Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Thu, 6 Mar 2025 20:01:10 +0100 Subject: [PATCH 02/14] Support diff on any branch Echo diff result --- .github/workflows/build-and-test.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 51aec1d6..b3538029 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -20,12 +20,14 @@ jobs: - name: "Check changes" # TODO also include skipping build but allow docs to be processed run: | - if git diff --exit-code --name-only origin/master . ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then - echo "SKIP_CHANGES=false" >> "$GITHUB_ENV" + if git diff --exit-code --name-only HEAD^ HEAD ':!.gitignore' ':!docs' ':!*.md' ':!*.txt'; then + echo "RUN_TEST=false" >> "$GITHUB_ENV" + echo "RUN_TEST = false" else - echo "SKIP_CHANGES=true" >> "$GITHUB_ENV" + echo "RUN_TEST=true" >> "$GITHUB_ENV" + echo "RUN_TEST = true" fi - + - name: "Prepare build environment" run: | sudo apt-get update @@ -43,8 +45,8 @@ jobs: scons VERBOSE=1 DEBUG=1 O=release - name: "Prepare test environment" - # TODO Ignore linter warning. See https://github.com/github/vscode-github-actions/issues/222 - if: ${{ env.SKIP_CHANGES == 'true' }} + # TODO Ignore linter warning. See https://github.com/github/vscode-github-actions/issues/222 + if: ${{ env.RUN_TEST == 'true' }} # The test suite is seriously disk-intensive. Given that linux # instances hosted in GitHub have 16G of RAM available we will # use it to speed up the run. @@ -53,10 +55,11 @@ jobs: sudo mount -o size=12G,nr_inodes=0 -t tmpfs tmpfs "${RM_TS_DIR}" - name: "Test it" - if: ${{ env.SKIP_CHANGES == 'true' }} + if: ${{ env.RUN_TEST == 'true' }} run: | RM_TS_PRINT_CMD=1 RM_TS_PEDANTIC=0 python -m pytest -s -v - name: CoW tests + if: ${{ env.RUN_TEST == 'true' }} shell: bash run: | sudo umount "${RM_TS_DIR}" @@ -80,6 +83,7 @@ jobs: EOF RM_TS_PRINT_CMD=1 RM_TS_PEDANTIC=0 python -m pytest -s -v - name: "Cleanup" + if: ${{ env.RUN_TEST == 'true' }} run: | - sudo umount "${RM_TS_DIR}" - sudo rmdir "${RM_TS_DIR}" + sudo umount "${RM_TS_DIR}" + sudo rmdir "${RM_TS_DIR}" From 25986ddbb21a2bef81f81a010525eb34a8229bac Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Thu, 6 Mar 2025 21:39:48 +0100 Subject: [PATCH 03/14] . --- .github/workflows/build-and-test.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index b3538029..4512bfd5 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -17,17 +17,6 @@ jobs: with: fetch-depth: 0 - - name: "Check changes" - # TODO also include skipping build but allow docs to be processed - run: | - if git diff --exit-code --name-only HEAD^ HEAD ':!.gitignore' ':!docs' ':!*.md' ':!*.txt'; then - echo "RUN_TEST=false" >> "$GITHUB_ENV" - echo "RUN_TEST = false" - else - echo "RUN_TEST=true" >> "$GITHUB_ENV" - echo "RUN_TEST = true" - fi - - name: "Prepare build environment" run: | sudo apt-get update @@ -43,7 +32,18 @@ jobs: run: | scons config scons VERBOSE=1 DEBUG=1 O=release - + + - name: "Check need for testing" + # TODO also include skipping build but allow docs to be processed + run: | + if git diff --exit-code --name-only HEAD^ HEAD ':!.gitignore' ':!docs' ':!*.md' ':!*.txt'; then + echo "RUN_TEST=false" >> "$GITHUB_ENV" + echo "RUN_TEST = false" + else + echo "RUN_TEST=true" >> "$GITHUB_ENV" + echo "RUN_TEST = true" + fi + - name: "Prepare test environment" # TODO Ignore linter warning. See https://github.com/github/vscode-github-actions/issues/222 if: ${{ env.RUN_TEST == 'true' }} From e224716b4fe20be3d9cadc417d18af97447c8206 Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Mon, 3 Mar 2025 22:16:28 +0100 Subject: [PATCH 04/14] Skip build test for non essentials #690 Support diff on any branch --- .github/workflows/build-and-test.yml | 85 ++++++---------------------- tests/test_formatters/test_csv.py | 4 +- tests/test_formatters/test_json.py | 3 +- 3 files changed, 22 insertions(+), 70 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 4512bfd5..bbc7c7e4 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -9,7 +9,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 env: RM_TS_DIR: "/tmp/rmlint-unit-testdir" steps: @@ -17,73 +17,24 @@ jobs: with: fetch-depth: 0 - - name: "Prepare build environment" - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - scons python3-sphinx gettext python3-setuptools \ - libblkid-dev libelf-dev libglib2.0-dev libjson-glib-dev \ - clang python3-pip python3-cffi python3-dev libffi-dev - pip3 install -r tests/requirements.txt - - - name: "Build" - # Todo: eventually run tests with valgrind (RM_TS_USE_VALGRIND) - # Todo enable slow tests in pytest - run: | - scons config - scons VERBOSE=1 DEBUG=1 O=release - - - name: "Check need for testing" - # TODO also include skipping build but allow docs to be processed - run: | - if git diff --exit-code --name-only HEAD^ HEAD ':!.gitignore' ':!docs' ':!*.md' ':!*.txt'; then - echo "RUN_TEST=false" >> "$GITHUB_ENV" - echo "RUN_TEST = false" - else - echo "RUN_TEST=true" >> "$GITHUB_ENV" - echo "RUN_TEST = true" - fi - - - name: "Prepare test environment" - # TODO Ignore linter warning. See https://github.com/github/vscode-github-actions/issues/222 - if: ${{ env.RUN_TEST == 'true' }} - # The test suite is seriously disk-intensive. Given that linux - # instances hosted in GitHub have 16G of RAM available we will - # use it to speed up the run. - run: | - sudo mkdir "${RM_TS_DIR}" - sudo mount -o size=12G,nr_inodes=0 -t tmpfs tmpfs "${RM_TS_DIR}" - - - name: "Test it" - if: ${{ env.RUN_TEST == 'true' }} - run: | - RM_TS_PRINT_CMD=1 RM_TS_PEDANTIC=0 python -m pytest -s -v - - name: CoW tests - if: ${{ env.RUN_TEST == 'true' }} + - name: "Check changes" + # TODO also include skipping build but allow docs to be processed shell: bash run: | - sudo umount "${RM_TS_DIR}" - sudo modprobe brd rd_nr=1 rd_size=12582912 - sudo mkfs.btrfs -f /dev/ram0 - sudo mount /dev/ram0 "${RM_TS_DIR}" - sudo chmod 0777 "${RM_TS_DIR}" - cat <<'EOF' >> tests/conftest.py + echo "Base: $GITHUB_BASE_REF" + echo "Head: $GITHUB_HEAD_REF" + if git diff --exit-code --name-only origin/$GITHUB_BASE_REF HEAD ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then + echo "RUN_TEST=false" >> $GITHUB_ENV + echo "RUN_TEST = false" + else + echo "RUN_TEST=true" >> $GITHUB_ENV + echo "RUN_TEST = true" + fi + - name: "Should we test" + run: + echo "the answer is $RUN_TEST" - def pytest_collection_modifyitems(items, config): - selected_items = [] - deselected_items = [] - - for item in items: - if "needs_reflink_fs" in getattr(item, "fixturenames", ()): - selected_items.append(item) - else: - deselected_items.append(item) - config.hook.pytest_deselected(items=deselected_items) - items[:] = selected_items - EOF - RM_TS_PRINT_CMD=1 RM_TS_PEDANTIC=0 python -m pytest -s -v - - name: "Cleanup" - if: ${{ env.RUN_TEST == 'true' }} + - name: "Test it" + if: ! ${{ env.RUN_TEST == 'true' }} run: | - sudo umount "${RM_TS_DIR}" - sudo rmdir "${RM_TS_DIR}" + echo "RUNNING IT... " diff --git a/tests/test_formatters/test_csv.py b/tests/test_formatters/test_csv.py index 8d19cb64..4659525a 100644 --- a/tests/test_formatters/test_csv.py +++ b/tests/test_formatters/test_csv.py @@ -8,8 +8,8 @@ def csv_string_to_data(csv_dump): data = list(csv.reader(csv_dump.splitlines())) return data[1:] - -def test_simple(usual_setup_usual_teardown): +@pytest.mark.usefixtures('usual_setup_usual_teardown') +def test_simple(): create_file('1234', 'a') create_file('1234', 'b') create_file('1234', 'stupid\'file,name') diff --git a/tests/test_formatters/test_json.py b/tests/test_formatters/test_json.py index 2e72e2ee..d04a1396 100644 --- a/tests/test_formatters/test_json.py +++ b/tests/test_formatters/test_json.py @@ -2,7 +2,8 @@ from tests.utils import * -def test_simple(usual_setup_usual_teardown): +@pytest.mark.usefixtures('usual_setup_usual_teardown') +def test_simple(): full_path_a = create_file('x', '\t\r\"\b\f\\') full_path_b = create_file('x', '\"\t\n2134124') head, *data, footer = run_rmlint('-S a') From 50a49ffd15cd25e385126d3a9e823310ed40d76e Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Thu, 6 Mar 2025 21:58:45 +0100 Subject: [PATCH 05/14] Commit2 --- tests/test_formatters/test_others.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_formatters/test_others.py b/tests/test_formatters/test_others.py index 3f16a637..746166cf 100644 --- a/tests/test_formatters/test_others.py +++ b/tests/test_formatters/test_others.py @@ -2,7 +2,8 @@ from tests.utils import * -def test_just_call_it(usual_setup_usual_teardown): +@pytest.mark.usefixtures('usual_setup_usual_teardown') +def test_just_call_it(): create_file('1234', 'a') create_file('1234', 'b') From b9f0a12cdf083d18e8009f69a5cf5b3bcdb8d1e9 Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Thu, 6 Mar 2025 21:59:20 +0100 Subject: [PATCH 06/14] commit3 --- tests/test_formatters/test_sh.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_formatters/test_sh.py b/tests/test_formatters/test_sh.py index 3b332c89..6ead303f 100644 --- a/tests/test_formatters/test_sh.py +++ b/tests/test_formatters/test_sh.py @@ -19,7 +19,8 @@ def filter_part_of_directory(data): return [e for e in data if e['type'] != 'part_of_directory'] -def test_basic(usual_setup_usual_teardown, shell): +@pytest.mark.usefixtures('usual_setup_usual_teardown') +def test_basic(shell): create_file('xxx', 'a') create_file('xxx', 'b') From c7f89a787ee8b2811a89ba0d8fa58af38676528c Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Thu, 6 Mar 2025 22:36:51 +0100 Subject: [PATCH 07/14] . --- tests/.python-version | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/.python-version diff --git a/tests/.python-version b/tests/.python-version new file mode 100644 index 00000000..251b3507 --- /dev/null +++ b/tests/.python-version @@ -0,0 +1 @@ +3.10.16 From c4c7de0bd9fca888f4a17ca0098a332267430070 Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:51:08 +0100 Subject: [PATCH 08/14] test --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index bbc7c7e4..cabc1b0e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -23,7 +23,7 @@ jobs: run: | echo "Base: $GITHUB_BASE_REF" echo "Head: $GITHUB_HEAD_REF" - if git diff --exit-code --name-only origin/$GITHUB_BASE_REF HEAD ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then + if git diff --exit-code --name-only origin/$GITHUB_BASE_REF...$GITHUB_SHA ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then echo "RUN_TEST=false" >> $GITHUB_ENV echo "RUN_TEST = false" else From 31bc6a653916fbc5ff19942f53cafb4a78a4f005 Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:59:37 +0100 Subject: [PATCH 09/14] info --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index dc7ffefe..33b5c618 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -23,7 +23,7 @@ jobs: shell: bash run: | echo "Base: $GITHUB_BASE_REF" - echo "Head: $GITHUB_HEAD_REF" + echo "Head: $GITHUB_SHA" if git diff --exit-code --name-only origin/$GITHUB_BASE_REF...$GITHUB_SHA ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then echo "RUN_TEST=false" >> $GITHUB_ENV echo "RUN_TEST = false" From 5f59a60dc72c0cfd37463d593b9f0586675f9565 Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:04:54 +0100 Subject: [PATCH 10/14] more info --- .github/workflows/build-and-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 33b5c618..2c300fe9 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -23,6 +23,7 @@ jobs: shell: bash run: | echo "Base: $GITHUB_BASE_REF" + echo "Base: $GITHUB_REF" echo "Head: $GITHUB_SHA" if git diff --exit-code --name-only origin/$GITHUB_BASE_REF...$GITHUB_SHA ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then echo "RUN_TEST=false" >> $GITHUB_ENV From d31db10b65a8fc12a43155fdc01fb8ddbdfad608 Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:06:04 +0100 Subject: [PATCH 11/14] But --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2c300fe9..b4e1c876 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,7 +25,7 @@ jobs: echo "Base: $GITHUB_BASE_REF" echo "Base: $GITHUB_REF" echo "Head: $GITHUB_SHA" - if git diff --exit-code --name-only origin/$GITHUB_BASE_REF...$GITHUB_SHA ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then + if git diff --exit-code --name-only $GITHUB_BASE_REF...$GITHUB_SHA ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then echo "RUN_TEST=false" >> $GITHUB_ENV echo "RUN_TEST = false" else From a347cb79f053d90d506bfa58ec1dba20f69cdbcf Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:10:08 +0100 Subject: [PATCH 12/14] . --- .github/workflows/build-and-test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index b4e1c876..345a955f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -22,9 +22,10 @@ jobs: # TODO also include skippin shell: bash run: | - echo "Base: $GITHUB_BASE_REF" - echo "Base: $GITHUB_REF" - echo "Head: $GITHUB_SHA" + echo "BASE: $GITHUB_BASE_REF" + echo "REF: $GITHUB_REF" + echo "MERGECOMMIT: $GITHUB_SHA" + echo "HEAD: $github.head_ref" if git diff --exit-code --name-only $GITHUB_BASE_REF...$GITHUB_SHA ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then echo "RUN_TEST=false" >> $GITHUB_ENV echo "RUN_TEST = false" From 851753d944e050d06b1e8960a735a2c3401c5aa5 Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:11:17 +0100 Subject: [PATCH 13/14] . --- .github/workflows/build-and-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 345a955f..38b57676 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,7 +25,8 @@ jobs: echo "BASE: $GITHUB_BASE_REF" echo "REF: $GITHUB_REF" echo "MERGECOMMIT: $GITHUB_SHA" - echo "HEAD: $github.head_ref" + echo "HEAD1: $GITHUB.HEAD_REF" + echo "HEAD2: $HEAD_REF" if git diff --exit-code --name-only $GITHUB_BASE_REF...$GITHUB_SHA ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then echo "RUN_TEST=false" >> $GITHUB_ENV echo "RUN_TEST = false" From 6b5f9af697510179030c36257a0d7a6409fdf89f Mon Sep 17 00:00:00 2001 From: Ray Oei <10501445+RayOei@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:14:14 +0100 Subject: [PATCH 14/14] . --- .github/workflows/build-and-test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 38b57676..24e78d64 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,9 +25,8 @@ jobs: echo "BASE: $GITHUB_BASE_REF" echo "REF: $GITHUB_REF" echo "MERGECOMMIT: $GITHUB_SHA" - echo "HEAD1: $GITHUB.HEAD_REF" - echo "HEAD2: $HEAD_REF" - if git diff --exit-code --name-only $GITHUB_BASE_REF...$GITHUB_SHA ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then + echo "HEAD_REF: ${{ GITHUB.HEAD_REF }}" + if git diff --exit-code --name-only origin/$GITHUB_BASE_REF...$GITHUB_SHA ':!.gitignore' ':!docs' ':!*.md' ':!*.txt' ':!*.yml'; then echo "RUN_TEST=false" >> $GITHUB_ENV echo "RUN_TEST = false" else