diff --git a/.github/workflows/ci_pr.yml b/.github/workflows/ci_pr.yml index a91c8efd6..b00219bfe 100644 --- a/.github/workflows/ci_pr.yml +++ b/.github/workflows/ci_pr.yml @@ -8,92 +8,23 @@ on: workflow_dispatch: jobs: - test-python-2_6-and-3_4-versions: - - strategy: - fail-fast: false - matrix: - include: - - python-version: "2.6" - - python-version: "3.4" - + execute-tests: name: "Python ${{ matrix.python-version }} Unit Tests" runs-on: ubuntu-20.04 - container: - image: ubuntu:24.04 - volumes: - - /home/waagent:/home/waagent - defaults: - run: - shell: bash -l {0} - - env: - NOSEOPTS: "--verbose" - - steps: - - uses: actions/checkout@v3 - - - name: Install Python ${{ matrix.python-version }} Virtual Environment - run: | - apt-get update - apt-get install -y curl bzip2 sudo - curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2 - sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory / - # - # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now. - # - if [[ "${{ matrix.python-version }}" == "2.6" ]]; then - apt-get -y install python3 - fi - # - # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments. - # - if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then - ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}" - fi - - - name: Execute Tests - run: | - source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate - ./ci/nosetests.sh - exit $? - - test-python-2_7: - - strategy: - fail-fast: false - - name: "Python 2.7 Unit Tests" - runs-on: ubuntu-20.04 - defaults: - run: - shell: bash -l {0} - - env: - NOSEOPTS: "--verbose" - - steps: - - uses: actions/checkout@v3 - - - name: Install Python 2.7 - run: | - apt-get update - apt-get install -y curl bzip2 sudo - curl https://dcrdata.blob.core.windows.net/python/python-2.7.tar.bz2 -o python-2.7.tar.bz2 - sudo tar xjvf python-2.7.tar.bz2 --directory / - - - name: Test with nosetests - run: | - source /home/waagent/virtualenv/python2.7.16/bin/activate - ./ci/nosetests.sh - exit $? - - test-current-python-versions: - strategy: fail-fast: false matrix: include: + # + # Some of the Python versions we test are not supported by the setup-python Github Action. For those versions, we use a + # pre-built virtual environment. + # + - python-version: "2.6" + use_virtual_environment: true + - python-version: "2.7" + use_virtual_environment: true + - python-version: "3.4" + use_virtual_environment: true - python-version: "3.5" # workaround found in https://github.com/actions/setup-python/issues/866 # for issue "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:728)" on Python 3.5 @@ -106,18 +37,15 @@ jobs: - python-version: "3.10" - python-version: "3.11" - name: "Python ${{ matrix.python-version }} Unit Tests" - runs-on: ubuntu-20.04 - - env: - NOSEOPTS: "--with-timer ${{ matrix.additional-nose-opts }}" - steps: - - - name: Checkout WALinuxAgent repo + - name: Checkout WALinuxAgent uses: actions/checkout@v3 - + # + # We either install Python and the test dependencies, or download a pre-built virtual environment, depending on the + # use_virtual_environment flag. + # - name: Setup Python ${{ matrix.python-version }} + if: (!matrix.use_virtual_environment) uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -125,6 +53,7 @@ jobs: PIP_TRUSTED_HOST: ${{ matrix.pip_trusted_host }} - name: Install dependencies + if: (!matrix.use_virtual_environment) id: install-dependencies run: | sudo env "PATH=$PATH" python -m pip install --upgrade pip @@ -132,7 +61,45 @@ jobs: sudo env "PATH=$PATH" pip install -r test-requirements.txt sudo env "PATH=$PATH" pip install --upgrade pylint + - name: Setup Python ${{ matrix.python-version }} Virtual Environment + if: matrix.use_virtual_environment + id: install-venv + run: | + sudo apt-get update + sudo apt-get install -y curl bzip2 sudo + curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2 + sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory / + # + # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments. + # + if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then + sudo ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}" + fi + + # + # Execute the tests + # + - name: Execute Unit Tests + run: | + if [[ "${{ matrix.python-version }}" =~ ^3\.[1-9][0-9]+$ ]]; then + ./ci/pytest.sh + else + if [[ "${{ matrix.use_virtual_environment}}" == "true" ]]; then + export NOSEOPTS="--verbose ${{ matrix.additional-nose-opts }}" # the pytest version on the venvs does not support the --with-timer option. + source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate + else + export NOSEOPTS="--verbose --with-timer ${{ matrix.additional-nose-opts }}" + fi + ./ci/nosetests.sh + fi + + # + # Execute pylint even when the tests fail (but only if the dependencies were installed successfully) + # + # The virtual environments do not include pylint, so we skip those Python versions. + # - name: Run pylint + if: (!matrix.use_virtual_environment && (success() || (failure() && steps.install-dependencies.outcome == 'success'))) run: | # # List of files/directories to be checked by pylint. @@ -153,7 +120,7 @@ jobs: # * 'no-self-use' ("R0201: Method could be a function") was moved to an optional extension on 3.8 and is no longer used by default. It needs # to be suppressed for previous versions (3.0-3.7), though. # * 'contextmanager-generator-missing-cleanup' are false positives if yield is used inside an if-else block for contextmanager generator functions. - # (https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html). + # (https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html). # This is not implemented on versions (3.0-3.7) Bad option value 'contextmanager-generator-missing-cleanup' (bad-option-value) # * 3.9-3.11 will produce "too-many-positional-arguments" for several methods that are having more than 5 args, so we suppress that warning. # (R0917: Too many positional arguments (8/5) (too-many-positional-arguments)) @@ -173,16 +140,10 @@ jobs: pylint $PYLINT_OPTIONS $PYLINT_FILES - - name: Execute Unit Tests - if: success() || (failure() && steps.install-dependencies.outcome == 'success') - run: | - if [[ "${{ matrix.python-version }}" =~ ^3\.[1-9][0-9]+$ ]]; then - ./ci/pytest.sh - else - ./ci/nosetests.sh - fi - - - name: Compile Coverage + # + # Lastly, compile code coverage + # + - name: Compile Code Coverage if: matrix.python-version == '3.9' run: | echo looking for coverage files : @@ -191,7 +152,7 @@ jobs: sudo env "PATH=$PATH" coverage xml sudo env "PATH=$PATH" coverage report - - name: Upload Coverage + - name: Upload Code Coverage if: matrix.python-version == '3.9' uses: codecov/codecov-action@v3 with: diff --git a/tests/python_eol/Dockerfile b/tests/python_eol/Dockerfile index 83cc7380e..14226b995 100644 --- a/tests/python_eol/Dockerfile +++ b/tests/python_eol/Dockerfile @@ -58,14 +58,14 @@ function run-sudo-tests { # # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now. # - if [[ "${PYTHON_VERSION}" == "2.6" ]]; then + if [[ "${PYTHON_VERSION}" =~ ^2\.6|2\.7$ ]]; then apt-get -y install python3 fi # - # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments. + # The virtual environments for 2.6, 2.7 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments. # - if [[ "${PYTHON_VERSION}" =~ ^2\.6|3\.4$ ]]; then + if [[ "${PYTHON_VERSION}" =~ ^2\.6|2\.7|3\.4$ ]]; then /tmp/patch_python_venv.sh "${PYTHON_VERSION}" fi .. diff --git a/tests/python_eol/patch_python_venv.sh b/tests/python_eol/patch_python_venv.sh index 16a585645..582f4cf12 100755 --- a/tests/python_eol/patch_python_venv.sh +++ b/tests/python_eol/patch_python_venv.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# The python 2.6 and 3.4 virtual environments have hard dependencies on some of the shared libraries in Open SSL 1.0 (e.g libssl.so.1.0.0), which is not available beyond Ubuntu 16. +# The python 2.6, 2.7, and 3.4 virtual environments have hard dependencies on some of the shared libraries in Open SSL 1.0 (e.g libssl.so.1.0.0), which is not available beyond Ubuntu 16. # Modules like hashlib and ssl will fail to import on more recent versions of Ubuntu. The Agent uses classes HTTPSConnection and HTTPS, which depend on the ssl module. Those classes # are added conditionally on the import of ssl on httplib.py and http/client.py with code similar to: # @@ -18,15 +18,20 @@ # set -euo pipefail -if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|3\.4$ ]]; then - echo "Usage: patch_python_venv.sh 2.6|3.4" +if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|2\.7|3\.4$ ]]; then + echo "Usage: patch_python_venv.sh 2.6|2.7|3.4" exit 1 fi PYTHON_VERSION=$1 -if [[ "${PYTHON_VERSION}" == "2.6" ]]; then - cat >> /opt/python/2.6.9/lib/python2.6/httplib.py << ... +if [[ "${PYTHON_VERSION}" =~ ^2\.6|2\.7$ ]]; then + if [[ "${PYTHON_VERSION}" == "2.6" ]]; then + file_to_patch="/opt/python/2.6.9/lib/python2.6/httplib.py" + else + file_to_patch="/opt/python/2.7.16/lib/python2.7/httplib.py" + fi + cat >> "$file_to_patch" << ... # Added by WALinuxAgent dev team to work around the lack of OpenSSL 1.0 shared libraries class HTTPSConnection(HTTPConnection):