diff --git a/.github/workflows/test-platforms.yml b/.github/workflows/test-platforms.yml new file mode 100644 index 00000000..ada31133 --- /dev/null +++ b/.github/workflows/test-platforms.yml @@ -0,0 +1,344 @@ +# Run kickstart tests in a PR triggered by a "/test-platforms" command from an organization member +# +# /test-platforms - tests all tests affected by the PR or smoke tests if there is no such. +# Or specify tests by combination of test names and/or options: +# /test-platforms --testtype TESTTYPE --skip-testtypes TYPE[,TYPE..] TEST1 TEST2 +# +# For dry run use: +# /test-platforms-dry +name: test-platforms +on: + issue_comment: + types: [created] + +permissions: + contents: read + statuses: write + +jobs: + pr-info: + if: startsWith(github.event.comment.body, '/test-platforms') + runs-on: ubuntu-latest + steps: + - name: Query comment author repository permissions + uses: octokit/request-action@v2.x + id: user_permission + with: + route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # restrict running of tests to users with admin or write permission for the repository + # see https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#get-repository-permissions-for-a-user + # store output if user is allowed in allowed_user job output so it has to be checked in downstream job + - name: Check if user does have correct permissions + if: contains('admin write', fromJson(steps.user_permission.outputs.data).permission) + id: check_user_perm + run: | + echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'" + echo "allowed_user=true" >> $GITHUB_OUTPUT + + - name: Get information for pull request + uses: octokit/request-action@v2.x + id: pr_api + with: + route: GET /repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Parse launch arguments + id: parse_launch_args + # Do not use comment body directly in the shell command to avoid possible code injection. + env: + BODY: ${{ github.event.comment.body }} + run: | + # extract first line and cut out the "/test-platforms" first word + LAUNCH_ARGS=$(echo "$BODY" | sed -n '1 s/^[^ ]* *//p' | sed 's/[[:space:]]*$//') + echo "launch arguments are: $LAUNCH_ARGS" + echo "launch_args=${LAUNCH_ARGS}" >> $GITHUB_OUTPUT + # check for dry run mode + DRY_RUN=False + echo "$BODY" | grep -q "/test-platforms-dry" && DRY_RUN=True + echo "Dry run: ${DRY_RUN}" + echo "dry_run=${DRY_RUN}" >> $GITHUB_OUTPUT + + outputs: + allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }} + base_ref: ${{ fromJson(steps.pr_api.outputs.data).base.ref }} + sha: ${{ fromJson(steps.pr_api.outputs.data).head.sha }} + launch_args: ${{ steps.parse_launch_args.outputs.launch_args }} + dry_run: ${{ steps.parse_launch_args.outputs.dry_run }} + + + platform: + needs: pr-info + if: needs.pr-info.outputs.allowed_user == 'true' + name: Run tests on the platform + runs-on: [self-hosted, kstest-test] + env: + STATUS_NAME: test-platforms + TARGET_BRANCH: ${{ needs.pr-info.outputs.base_ref }} + TEST_JOBS: 16 + GITHUB_TOKEN: /home/github/github-token + strategy: + matrix: + platform: [daily-iso, rawhide, rhel8, rhel9] + fail-fast: false + + steps: + # self-hosted runners don't do this automatically; also useful to keep stuff around for debugging + # need to run sudo as the launch script and the container create root/other user owned files + - name: Clean up previous run + run: | + sudo podman ps -q --all --filter='ancestor=kstest-runner' | xargs -tr sudo podman rm -f + sudo podman volume rm --all || true + sudo rm -rf * .git + + - name: Clone repository + uses: actions/checkout@v4 + with: + ref: ${{ needs.pr-info.outputs.sha }} + path: kickstart-tests + + - name: Check out kickstart-tests + uses: actions/checkout@v4 + with: + #TODO change + #repository: rhinstaller/kickstart-tests + repository: rvykydal/kickstart-tests + path: kickstart-tests + + - name: Generate test cases + working-directory: ./kickstart-tests + run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases + + - name: Clone Permian repository + uses: actions/checkout@v4 + with: + repository: rhinstaller/permian + path: permian + ref: main + + - name: Clone tplib repository + uses: actions/checkout@v4 + with: + repository: rhinstaller/tplib + path: tplib + + - name: Ensure http proxy is running + run: sudo containers/squid.sh start + working-directory: ./kickstart-tests + + - name: Update container images used here + run: | + sudo podman pull quay.io/rhinstaller/kstest-runner:latest + + - name: Get changed tests + working-directory: ./kickstart-tests + id: get_changed_tests + run: | + git fetch origin + CHANGED_TESTS=$(git diff --name-only origin/$TARGET_BRANCH ${{ needs.pr-info.outputs.sha }} -- *.ks.in $(find -maxdepth 1 -name '*.sh' -perm -u+x) | sed 's/\.ks\.in$//; s/\.sh$//' | sort -u | tr '\n' ' ') + echo "changed_tests=${CHANGED_TESTS}" >> $GITHUB_OUTPUT + + - name: Get skipped tests for platform ${{ matrix.platform }} + id: get_platform_specs + working-directory: ./kickstart-tests + run: | + set -eux + + source ./containers/runner/skip-testtypes + if [ ${{ matrix.platform }} == "daily-iso" ]; then + echo "skip_tests=skip-on-fedora" >> $GITHUB_OUTPUT + echo "disabled_testtypes=$SKIP_TESTTYPES_DAILY_ISO" >> $GITHUB_OUTPUT + echo "platform=rawhide" >> $GITHUB_OUTPUT + elif [ ${{ matrix.platform }} == "rawhide" ]; then + echo "skip_tests=skip-on-fedora" >> $GITHUB_OUTPUT + echo "disabled_testtypes=$SKIP_TESTTYPES_RAWHIDE" >> $GITHUB_OUTPUT + echo "platform=rawhide" >> $GITHUB_OUTPUT + elif [ ${{ matrix.platform }} == "rhel8" ]; then + echo "skip_tests=skip-on-rhel,skip-on-rhel-8" >> $GITHUB_OUTPUT + echo "disabled_testtypes=$SKIP_TESTTYPES_RHEL8" >> $GITHUB_OUTPUT + echo "platform=rhel8" >> $GITHUB_OUTPUT + elif [ ${{ matrix.platform }} == "rhel9" ]; then + echo "skip_tests=skip-on-rhel,skip-on-rhel-9" >> $GITHUB_OUTPUT + echo "disabled_testtypes=$SKIP_TESTTYPES_RHEL9" >> $GITHUB_OUTPUT + echo "platform=rhel9" >> $GITHUB_OUTPUT + else + echo "Platform is not supported by kickstart tests yet!" + exit 1 + fi + + - name: Generate test selection + id: generate_query + working-directory: ./kickstart-tests + run: | + set -eux + + CHANGED_TESTS="${{steps.get_changed_tests.outputs.changed_tests }}" + DISABLED_TESTTYPES="${{ steps.get_platform_specs.outputs.disabled_testtypes }}" + LAUNCH_ARGS="${{ needs.pr-info.outputs.launch_args }}" + + if [ -n "${LAUNCH_ARGS}" ]; then + echo "description=Running tests required by the PR comment explicitly." >> $GITHUB_OUTPUT + elif [ -n "${CHANGED_TESTS}" ]; then + echo "description=Running tests affected by changes (excluding disabled)." >> $GITHUB_OUTPUT + LAUNCH_ARGS="${CHANGED_TESTS}" + if [ -n "${DISABLED_TESTTYPES}" ]; then + LAUNCH_ARGS="--skip-testtypes ${DISABLED_TESTTYPES} ${LAUNCH_ARGS}" + fi + else + echo "description=Running smoke tests (no affected tests found)." >> $GITHUB_OUTPUT + LAUNCH_ARGS="--testtype smoke" + fi + + source ./containers/runner/skip-testtypes + PERMIAN_QUERY=$(scripts/generate-permian-query.py \ + --skip-testtypes $SKIP_TESTTYPES_ANACONDA_PR \ + --skip-testtypes ${{ steps.get_platform_specs.outputs.skip_tests }} \ + ${LAUNCH_ARGS} ) + if [ $? == 0 ]; then + echo "launch_args=$LAUNCH_ARGS" >> $GITHUB_OUTPUT + echo "query=$PERMIAN_QUERY" >> $GITHUB_OUTPUT + else + echo "Parsing of the request arguments failed" + exit 1 + fi + + # we post statuses manually as this does not run from a pull_request event + # https://developer.github.com/v3/repos/statuses/#create-a-status + - name: Create in-progress status + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}' + description: '${{ steps.generate_query.outputs.description }}' + state: pending + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Fetch boot.iso and configiure its local location + - name: Fetch boot.iso if available for platform ${{ matrix.platform }} + id: boot_iso_for_platform + run: | + set -eux + BOOT_ISO_PATH="${{ github.workspace }}/${{ matrix.platform }}.boot.iso" + BOOT_ISO_URL="file://$BOOT_ISO_PATH" + if [ "${{ matrix.platform }}" == "daily-iso" ]; then + ${{ github.workspace }}/kickstart-tests/containers/runner/fetch_daily_iso.sh $GITHUB_TOKEN $BOOT_ISO_PATH + echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT + elif [ "${{ matrix.platform }}" == "rawhide" ]; then + curl -L https://download.fedoraproject.org/pub/fedora/linux/development/rawhide/Server/x86_64/os/images/boot.iso --output $BOOT_ISO_PATH + echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT + else + echo "Boot.iso for ${{ matrix.platform }} can't be fetched." + echo "boot_iso=" >> $GITHUB_OUTPUT + fi + + # Configure location of installation repositories for the platform + # Also default boot.iso is defined by the value of urls.installation_tree + # of kstestParams event structure. + - name: Set installation tree for the platform + id: set_installation_urls + run: | + set -eux + if [ "${{ matrix.platform }}" == "rhel8" ]; then + echo "installation_tree=http://download.eng.bos.redhat.com/rhel-8/nightly/RHEL-8/latest-RHEL-8.10.0/compose/BaseOS/x86_64/os" >> $GITHUB_OUTPUT + echo "modular_url=http://download.eng.bos.redhat.com/rhel-8/nightly/RHEL-8/latest-RHEL-8.10.0/compose/AppStream/x86_64/os" >> $GITHUB_OUTPUT + elif [ "${{ matrix.platform }}" == "rhel9" ]; then + echo "installation_tree=http://download.eng.bos.redhat.com/rhel-9/nightly/RHEL-9/latest-RHEL-9.4.0/compose/BaseOS/x86_64/os" >> $GITHUB_OUTPUT + echo "modular_url=http://download.eng.bos.redhat.com/rhel-9/nightly/RHEL-9/latest-RHEL-9.4.0/compose/AppStream/x86_64/os" >> $GITHUB_OUTPUT + else + echo "Installation tree location for ${{ matrix.platform }} not configured" + if [ -z "${{ steps.boot_iso_for_platform.outputs.boot_iso }}" ]; then + echo "No boot.iso source is defined" + exit 2 + fi + echo "installation_tree=" >> $GITHUB_OUTPUT + echo "modular_url=" >> $GITHUB_OUTPUT + fi + + - name: Create Permian settings file + working-directory: ./permian + run: | + cat < settings.ini + [kickstart_test] + kstest_local_repo=${{ github.workspace }}/kickstart-tests + [library] + directPath=${{ github.workspace }}/kickstart-tests/testlib + EOF + + - name: Run kickstart tests in container + working-directory: ./permian + run: | + sudo --preserve-env=TEST_JOBS \ + PYTHONPATH=${PYTHONPATH:-}:${{ github.workspace }}/tplib \ + ./run_subset --debug-log permian.log \ + --settings settings.ini \ + --override workflows.dry_run=${{ needs.pr-info.outputs.dry_run }} \ + --testcase-query '${{ steps.generate_query.outputs.query }}' \ + run_event '{ + "type":"everything", + "everything_testplan":{ + "configurations":[{"architecture":"x86_64"}], + "point_person":"rvykydal@redhat.com" + }, + ${{ steps.boot_iso_for_platform.outputs.boot_iso }} + "kstestParams":{ + "platform":"${{ steps.get_platform_specs.outputs.platform }}", + "urls":{ + "x86_64":{ + "installation_tree":"${{ steps.set_installation_urls.outputs.installation_tree }}", + "modular_url":"${{ steps.set_installation_urls.outputs.modular_url }}" + } + } + } + }' + + # Permian hides the exit code of launcher, so error out this step manually based on logs + rc=$( awk '/Runner return code: /{ print $4 }' permian.log) + if [ -n "$rc" ]; then + exit $rc + else + grep -q "All execution and reporting is done" permian.log || exit 111 + fi + + - name: Collect anaconda logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: 'logs-${{ matrix.platform }}' + # skip the /anaconda subdirectories, too large + path: | + kickstart-tests/data/logs/kstest*.log + kickstart-tests/data/logs/kstest.log.json + kickstart-tests/data/logs/kstest-*/*.log + kickstart-tests/data/logs/kstest-*/anaconda/lorax-packages.log + + - name: Collect Permian logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: 'logs-permian-${{ matrix.platform }}' + path: | + permian/permian.log + + result: + if: ${{ always() }} + name: Set result status + runs-on: ubuntu-latest + env: + STATUS_NAME: test-platforms + needs: [platform, pr-info] + steps: + - name: Show overall status + uses: octokit/request-action@v2.x + with: + route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}' + context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}' + description: 'finished' + state: ${{ needs.platform.result }} + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.rst b/README.rst index abe704a4..3a13bfcb 100644 --- a/README.rst +++ b/README.rst @@ -368,6 +368,10 @@ PR runs do *not* auto-retry test failures. This avoids introducing unstable tests, and PRs usually just run a few tests so that flakes are much less likely to ruin the result. +To test a PR on all supported platforms (including rhel) there is a +`test-platforms`_ workflow running the tests on a comment in a PR. +Running it requires admin repository permissions. + Service jobs ------------ * The `container-autoupdate`_ workflow refreshes the runner container @@ -391,5 +395,6 @@ GitHub's infrastructure and can be run manually by a developer. .. _Travis: https://travis-ci.com/ .. _.travis.yml: ./.travis.yml .. _container-autoupdate: ./.github/workflows/container-autoupdate.yml +.. _test-platforms: ./.github/workflows/test-platforms.yml .. _quay.io/rhinstaller/kstest-runner: https://quay.io/repository/rhinstaller/kstest-runner .. _daily-boot-iso: ./.github/workflows/daily-boot-iso.yml diff --git a/containers/runner/launch b/containers/runner/launch index 96fcb1c7..99805257 100755 --- a/containers/runner/launch +++ b/containers/runner/launch @@ -149,5 +149,6 @@ $CRUN run -it --rm --device=/dev/kvm --publish 127.0.0.1::16509 $PODMAN_SELINUX_ --env KSTESTS_TEST="$KSTESTS_TEST" --env TESTTYPE="${TESTTYPE:-}" --env SKIP_TESTTYPES="${SKIP_TESTTYPES:-}" \ --env KSTESTS_RUN_TIMEOUT="${TIMEOUT:-}" \ --env TEST_JOBS="$TEST_JOBS" --env PLATFORM="${PLATFORM:-}" --env TEST_RETRY="${TEST_RETRY:-}" ${UPDATES_IMG_ARGS:-} ${CONTAINER_RUN_ARGS:-} \ + --env VIRTINSTALL_OSINFO_DISABLE_REQUIRE=1 \ ${VAR_TMP:-} -v "$PWD/data:/opt/kstest/data:z" -v "$BASEDIR:/kickstart-tests:ro,z" ${DEFAULTS_SH_ARGS:-} \ $CONTAINER $RUN_COMMAND diff --git a/containers/runner/skip-testtypes b/containers/runner/skip-testtypes index b9186c51..f8a52c37 100644 --- a/containers/runner/skip-testtypes +++ b/containers/runner/skip-testtypes @@ -10,7 +10,21 @@ common_skip_array=( rawhide_skip_array=( skip-on-fedora + gh576 # clearpart-4 test is flaky on all scenarios + gh595 # proxy-cmdline failing on all scenarios + gh640 # authselect-not-set failing + gh641 # packages-multilib failing on systemd conflict + gh680 # proxy-kickstart and proxy-auth failing + gh740 # fedora-live-image-build fails on comps changes + gh769 # nodejs:16 and swig:4 missing in module-X tests + gh774 # autopart-luks-1 failing + gh777 # raid-1 failing + gh910 # stage2-from-ks test needs to be fixed for daily-iso + gh890 # default-systemd-target-vnc-graphical-provides flaking too much + gh871 # basic-ftp failing due to mirror rhbz1853668 # multipath device not constructed back after installation + gh975 # packages-default failing + gh1023 # rpm-ostree failing ) rawhide_text_skip_array=( diff --git a/dns.ks.in b/dns.ks.in index fe0525f0..676c4fa5 100644 --- a/dns.ks.in +++ b/dns.ks.in @@ -1,6 +1,8 @@ #version=DEVEL #test name: dns -# Test domain name resolution. +# Test domain name resolution: +# - origin of /etc/resolv.conf (rhbz#2018913, rhbz#1989472) +# - dns resolution in %post scripts (RHEL-26651) # Use defaults. %ksappend repos/default.ks @@ -19,6 +21,11 @@ else check_resolv_conf_is_by_resolved fi +curl fedoraproject.org +if [[ $? != 0 ]]; then + echo '*** curl fedoraproject.org failed' >> /root/RESULT +fi + # No error was written to /root/RESULT file, everything is OK if [[ ! -e /root/RESULT ]]; then echo SUCCESS > /root/RESULT diff --git a/raid-1.sh b/raid-1.sh index b7b7676e..a447fd34 100755 --- a/raid-1.sh +++ b/raid-1.sh @@ -21,6 +21,8 @@ # shellcheck disable=SC2034 TESTTYPE="raid storage coverage gh777" +# modification to trigger CI on /test-platforms comment + . ${KSTESTDIR}/functions.sh prepare_disks() { diff --git a/scripts/generate-permian-query.py b/scripts/generate-permian-query.py index 46f891b7..51156ea4 100755 --- a/scripts/generate-permian-query.py +++ b/scripts/generate-permian-query.py @@ -29,6 +29,9 @@ def parse_args(): + " or ".join(['tc.name == "{}"'.format(test) for test in args.tests]) + ")" ] + if args.skip_testtypes: + skiptypes = ','.join(itertools.chain(*args.skip_testtypes)).split(',') + conditions.extend(['"{}" not in tc.tags'.format(skiptype) for skiptype in skiptypes]) else: if args.testtype: conditions.extend(['"{}" in tc.tags'.format(args.testtype), '"knownfailure" not in tc.tags'])