Cleanup Github Actions workflow for Unit Tests (Azure#3300)

Co-authored-by: narrieta@microsoft <narrieta>

.github/workflows/ci_pr.yml

@@ -8,92 +8,23 @@ on:
   workflow_dispatch:
 
 jobs:
-  test-python-2_6-and-3_4-versions:
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - python-version: "2.6"
-          - python-version: "3.4"
-
+  execute-tests:
     name: "Python ${{ matrix.python-version }} Unit Tests"
     runs-on: ubuntu-20.04
-    container:
-      image: ubuntu:24.04
-      volumes:
-        - /home/waagent:/home/waagent
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    env:
-      NOSEOPTS: "--verbose"
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Install Python ${{ matrix.python-version }} Virtual Environment
-      run: |
-        apt-get update
-        apt-get install -y curl bzip2 sudo
-        curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2
-        sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory /
-        #
-        # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
-        #
-        if [[ "${{ matrix.python-version }}" == "2.6" ]]; then
-            apt-get -y install python3
-        fi
-        #
-        # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
-        #
-        if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then
-          ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}"
-        fi
-
-    - name: Execute Tests
-      run: |
-        source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate
-        ./ci/nosetests.sh
-        exit $?
-
-  test-python-2_7:
-
-    strategy:
-      fail-fast: false
-
-    name: "Python 2.7 Unit Tests"
-    runs-on: ubuntu-20.04
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    env:
-      NOSEOPTS: "--verbose"
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Install Python 2.7
-      run: |
-        apt-get update
-        apt-get install -y curl bzip2 sudo
-        curl https://dcrdata.blob.core.windows.net/python/python-2.7.tar.bz2 -o python-2.7.tar.bz2
-        sudo tar xjvf python-2.7.tar.bz2 --directory /
-
-    - name: Test with nosetests
-      run: |
-        source /home/waagent/virtualenv/python2.7.16/bin/activate
-        ./ci/nosetests.sh
-        exit $?
-
-  test-current-python-versions:
-
     strategy:
       fail-fast: false
       matrix:
         include:
+          #
+          # Some of the Python versions we test are not supported by the setup-python Github Action. For those versions, we use a
+          # pre-built virtual environment.
+          #
+          - python-version: "2.6"
+            use_virtual_environment: true
+          - python-version: "2.7"
+            use_virtual_environment: true
+          - python-version: "3.4"
+            use_virtual_environment: true
           - python-version: "3.5"
             # workaround found in https://github.com/actions/setup-python/issues/866
             # for issue "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:728)" on Python 3.5
@@ -106,33 +37,69 @@ jobs:
           - python-version: "3.10"
           - python-version: "3.11"
 
-    name: "Python ${{ matrix.python-version }} Unit Tests"
-    runs-on: ubuntu-20.04
-
-    env:
-      NOSEOPTS: "--with-timer ${{ matrix.additional-nose-opts }}"
-
     steps:
-
-    - name: Checkout WALinuxAgent repo
+    - name: Checkout WALinuxAgent
       uses: actions/checkout@v3
-
+    #
+    # We either install Python and the test dependencies, or download a pre-built virtual environment, depending on the
+    # use_virtual_environment flag.
+    #
     - name: Setup Python ${{ matrix.python-version }}
+      if: (!matrix.use_virtual_environment)
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
       env:
         PIP_TRUSTED_HOST: ${{ matrix.pip_trusted_host }}
 
     - name: Install dependencies
+      if: (!matrix.use_virtual_environment)
       id: install-dependencies
       run: |
         sudo env "PATH=$PATH" python -m pip install --upgrade pip
         sudo env "PATH=$PATH" pip install -r requirements.txt
         sudo env "PATH=$PATH" pip install -r test-requirements.txt
         sudo env "PATH=$PATH" pip install --upgrade pylint
 
+    - name: Setup Python ${{ matrix.python-version }} Virtual Environment
+      if: matrix.use_virtual_environment
+      id: install-venv
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y curl bzip2 sudo
+        curl -sSf --retry 5 -o /tmp/python-${{ matrix.python-version }}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2
+        sudo tar xjf /tmp/python-${{ matrix.python-version }}.tar.bz2 --directory /
+        #
+        # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
+        #
+        if [[ "${{ matrix.python-version }}" =~ ^2\.6|3\.4$ ]]; then
+          sudo ./tests/python_eol/patch_python_venv.sh "${{ matrix.python-version }}"
+        fi
+
+    #
+    # Execute the tests
+    #
+    - name: Execute Unit Tests
+      run: |
+        if [[ "${{ matrix.python-version }}" =~ ^3\.[1-9][0-9]+$ ]]; then
+          ./ci/pytest.sh
+        else
+          if [[ "${{ matrix.use_virtual_environment}}" == "true"  ]]; then
+            export NOSEOPTS="--verbose ${{ matrix.additional-nose-opts }}"   # the pytest version on the venvs does not support the --with-timer option. 
+            source /home/waagent/virtualenv/python${{ matrix.python-version }}/bin/activate
+          else
+            export NOSEOPTS="--verbose --with-timer ${{ matrix.additional-nose-opts }}"
+          fi
+          ./ci/nosetests.sh
+        fi
+
+    #
+    # Execute pylint even when the tests fail (but only if the dependencies were installed successfully)
+    #
+    # The virtual environments do not include pylint, so we skip those Python versions.
+    #
     - name: Run pylint
+      if: (!matrix.use_virtual_environment && (success() || (failure() && steps.install-dependencies.outcome == 'success')))
       run: |
         #
         # List of files/directories to be checked by pylint.
@@ -153,7 +120,7 @@ jobs:
         # * 'no-self-use' ("R0201: Method could be a function") was moved to an optional extension on 3.8 and is no longer used by default. It needs
         #    to be suppressed for previous versions (3.0-3.7), though.
         # * 'contextmanager-generator-missing-cleanup' are false positives if yield is used inside an if-else block for contextmanager generator functions.
-        #   (https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html). 
+        #   (https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html).
         #   This is not implemented on versions (3.0-3.7) Bad option value 'contextmanager-generator-missing-cleanup' (bad-option-value)
         # * 3.9-3.11 will produce "too-many-positional-arguments" for several methods that are having more than 5 args, so we suppress that warning.
         #  (R0917: Too many positional arguments (8/5) (too-many-positional-arguments))
@@ -173,16 +140,10 @@ jobs:
 
         pylint $PYLINT_OPTIONS $PYLINT_FILES
 
-    - name: Execute Unit Tests
-      if: success() || (failure() && steps.install-dependencies.outcome == 'success')
-      run: |
-        if [[ "${{ matrix.python-version }}" =~ ^3\.[1-9][0-9]+$ ]]; then
-          ./ci/pytest.sh
-        else
-          ./ci/nosetests.sh
-        fi
-
-    - name: Compile Coverage
+    #
+    # Lastly, compile code coverage
+    #
+    - name: Compile Code Coverage
       if: matrix.python-version == '3.9'
       run: |
         echo looking for coverage files :
@@ -191,7 +152,7 @@ jobs:
         sudo env "PATH=$PATH" coverage xml
         sudo env "PATH=$PATH" coverage report
 
-    - name: Upload Coverage
+    - name: Upload Code Coverage
       if: matrix.python-version ==  '3.9'
       uses: codecov/codecov-action@v3
       with:

tests/python_eol/Dockerfile

@@ -58,14 +58,14 @@ function run-sudo-tests {
     #
     # TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now.
     #
-    if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
+    if [[ "${PYTHON_VERSION}" =~ ^2\.6|2\.7$ ]]; then
         apt-get -y install python3
     fi
 
     #
-    # The virtual environments for 2.6 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
+    # The virtual environments for 2.6, 2.7 and 3.4 have dependencies on OpenSSL 1.0, which is not available beyond Ubuntu 16. We use this script to patch the environments.
     #
-    if [[ "${PYTHON_VERSION}" =~ ^2\.6|3\.4$ ]]; then
+    if [[ "${PYTHON_VERSION}" =~ ^2\.6|2\.7|3\.4$ ]]; then
       /tmp/patch_python_venv.sh "${PYTHON_VERSION}"
     fi
 ..

tests/python_eol/patch_python_venv.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# The python 2.6 and 3.4 virtual environments have hard dependencies on some of the shared libraries in Open SSL 1.0 (e.g libssl.so.1.0.0), which is not available beyond Ubuntu 16.
+# The python 2.6, 2.7, and 3.4 virtual environments have hard dependencies on some of the shared libraries in Open SSL 1.0 (e.g libssl.so.1.0.0), which is not available beyond Ubuntu 16.
 # Modules like hashlib and ssl will fail to import on more recent versions of Ubuntu. The Agent uses classes HTTPSConnection and HTTPS, which depend on the ssl module. Those classes
 # are added conditionally on the import of ssl on httplib.py and http/client.py with code similar to:
 #
@@ -18,15 +18,20 @@
 #
 set -euo pipefail
 
-if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|3\.4$ ]]; then
-  echo "Usage: patch_python_venv.sh 2.6|3.4"
+if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|2\.7|3\.4$ ]]; then
+  echo "Usage: patch_python_venv.sh 2.6|2.7|3.4"
   exit 1
 fi
 
 PYTHON_VERSION=$1
 
-if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
-  cat >> /opt/python/2.6.9/lib/python2.6/httplib.py << ...
+if [[ "${PYTHON_VERSION}" =~ ^2\.6|2\.7$ ]]; then
+  if [[ "${PYTHON_VERSION}" == "2.6" ]]; then
+    file_to_patch="/opt/python/2.6.9/lib/python2.6/httplib.py"
+  else
+    file_to_patch="/opt/python/2.7.16/lib/python2.7/httplib.py"
+  fi
+  cat >> "$file_to_patch" << ...
 
 # Added by WALinuxAgent dev team to work around the lack of OpenSSL 1.0 shared libraries
 class HTTPSConnection(HTTPConnection):