From aa580e126a4d758953de711c408112008c29c801 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Wed, 8 Jan 2025 16:45:44 -0800 Subject: [PATCH] Create Azure Pipeline for Python 2.6 & 3.4 Unit Tests (#3284) --------- Co-authored-by: narrieta@microsoft (cherry picked from commit 029e5b9e689e6174daa6d4cc7b6087681afbb4b8) --- .github/workflows/ci_pr.yml | 43 ---------------- tests/data/ext/sample_ext-1.3.0/python.sh | 2 +- tests/ga/test_cgroupapi.py | 6 +-- tests/ga/test_cgroupconfigurator.py | 20 +++---- tests/python_eol/Dockerfile | 60 +++++++++++++++++++++ tests/python_eol/execute_tests.sh | 62 ++++++++++++++++++++++ tests/python_eol/pipeline.yml | 63 +++++++++++++++++++++++ 7 files changed, 199 insertions(+), 57 deletions(-) create mode 100644 tests/python_eol/Dockerfile create mode 100755 tests/python_eol/execute_tests.sh create mode 100644 tests/python_eol/pipeline.yml diff --git a/.github/workflows/ci_pr.yml b/.github/workflows/ci_pr.yml index 9ae44ec8c..6be27f67e 100644 --- a/.github/workflows/ci_pr.yml +++ b/.github/workflows/ci_pr.yml @@ -8,49 +8,6 @@ on: workflow_dispatch: jobs: - test-python-2_6-and-3_4-versions: - - strategy: - fail-fast: false - matrix: - include: - - python-version: "2.6" - - python-version: "3.4" - - name: "Python ${{ matrix.python-version }} Unit Tests" - runs-on: ubuntu-20.04 - container: - image: ubuntu:16.04 - volumes: - - /home/waagent:/home/waagent - defaults: - run: - shell: bash -l {0} - - env: - NOSEOPTS: "--verbose" - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - - steps: - - uses: actions/checkout@v3 - - - name: Install Python ${{ matrix.python-version }} - run: | - apt-get update - apt-get install -y curl bzip2 sudo python3 - curl https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2 -o python-${{ matrix.python-version }}.tar.bz2 - sudo tar xjvf python-${{ matrix.python-version }}.tar.bz2 --directory / - - - name: Test with nosetests - run: | - if [[ ${{ matrix.python-version }} == "2.6" ]]; then - source /home/waagent/virtualenv/python2.6.9/bin/activate - else - source /home/waagent/virtualenv/python3.4.8/bin/activate - fi - ./ci/nosetests.sh - exit $? - test-python-2_7: strategy: diff --git a/tests/data/ext/sample_ext-1.3.0/python.sh b/tests/data/ext/sample_ext-1.3.0/python.sh index dfa6afcd6..53adc0bb6 100755 --- a/tests/data/ext/sample_ext-1.3.0/python.sh +++ b/tests/data/ext/sample_ext-1.3.0/python.sh @@ -4,7 +4,7 @@ # python=$(command -v python 2> /dev/null) -if [ -z "$PYTHON" ]; then +if [ -z "$python" ]; then python=$(command -v python3) fi diff --git a/tests/ga/test_cgroupapi.py b/tests/ga/test_cgroupapi.py index 3e43d83bf..61ec6a43b 100644 --- a/tests/ga/test_cgroupapi.py +++ b/tests/ga/test_cgroupapi.py @@ -301,7 +301,7 @@ def mock_popen(command, *args, **kwargs): shell=True, timeout=300, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=output_file, stderr=output_file) @@ -317,7 +317,7 @@ def test_start_extension_cgroups_v1_command_should_execute_the_command_in_a_cgro shell=False, timeout=300, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -344,7 +344,7 @@ def test_start_extension_cgroups_v1_command_should_use_systemd_to_execute_the_co timeout=300, shell=True, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/tests/ga/test_cgroupconfigurator.py b/tests/ga/test_cgroupconfigurator.py index 646ed2006..0805d52e5 100644 --- a/tests/ga/test_cgroupconfigurator.py +++ b/tests/ga/test_cgroupconfigurator.py @@ -354,7 +354,7 @@ def test_start_extension_command_should_not_use_systemd_when_cgroups_are_not_ena timeout=300, shell=False, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -374,7 +374,7 @@ def test_start_extension_command_should_use_systemd_run_when_cgroups_v1_are_enab timeout=300, shell=False, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -395,7 +395,7 @@ def test_start_extension_command_should_start_tracking_the_extension_cgroups(sel timeout=300, shell=False, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -426,7 +426,7 @@ def mock_popen(command_arg, *args, **kwargs): timeout=300, shell=False, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -446,7 +446,7 @@ def test_start_extension_command_should_not_use_systemd_when_cgroup_v2_enabled(s timeout=300, shell=False, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -480,7 +480,7 @@ def test_start_extension_command_should_disable_cgroups_and_invoke_the_command_d timeout=300, shell=True, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=output_file, stderr=output_file) @@ -526,7 +526,7 @@ def test_start_extension_command_should_disable_cgroups_and_invoke_the_command_d timeout=300, shell=True, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=stdout, stderr=stderr) @@ -571,7 +571,7 @@ def mock_popen(command, *args, **kwargs): timeout=300, shell=True, cwd=self.tmp_dir, - env={}, + env={}.update(os.environ), stdout=stdout, stderr=stderr) @@ -856,7 +856,7 @@ def mock_popen(command, *args, **kwargs): # For the agent's processes, we use the current process and its parent (in the actual agent these would be the daemon and the extension # handler), and the commands started by the agent. # - # For other processes, we use process 1, a process that already completed, and an extension. Note that extensions are started using + # For other processes, we use a process that already completed, and an extension process. Note that extensions are started using # systemd-run and the process for that commands belongs to the agent's cgroup but the processes for the extension should be in a # different cgroup # @@ -868,7 +868,7 @@ def get_completed_process(): return completed agent_processes = [os.getppid(), os.getpid()] + agent_command_processes + [start_extension.systemd_run_pid] - other_processes = [1, get_completed_process()] + extension_processes + other_processes = [get_completed_process()] + extension_processes with patch("azurelinuxagent.ga.cgroupapi.CgroupV1.get_processes", return_value=agent_processes + other_processes): with self.assertRaises(CGroupsException) as context_manager: diff --git a/tests/python_eol/Dockerfile b/tests/python_eol/Dockerfile new file mode 100644 index 000000000..79adf8ab8 --- /dev/null +++ b/tests/python_eol/Dockerfile @@ -0,0 +1,60 @@ +# +# Environment to execute the WALinuxAgent unit tests for some versions of Python that have reached EOL and are no longer available +# in the official repositories. +# +# To build the image, set the PYTHON_VERSION argument to 2.6 or 3.4: +# +# * docker build -t python2.6 --build-arg PYTHON_VERSION=2.6 . +# * docker build -t python3.4 --build-arg PYTHON_VERSION=3.4 . +# +# We add a couple of convenience functions to execute the unit tests to the profiles of waagent and root; these can be useful in interactive sessions. Note +# that these functions assume the root of the source code has been mounted at /home/waagent/WALinuxAgent. +# +# Also, we precede "mesg n" with "tty -s" in root's profile to avoid the "standard input is not a tty" message when not running the container interactively. +# +# Sample commands: +# +# * Start an interactive session: docker run --rm -it -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login +# * Run unit tests: docker run --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-tests +# * Run tests that require root: docker run --user root --rm -v WALinuxAgent:/home/waagent/WALinuxAgent python2.6 bash --login -c run-sudo-tests +# +FROM ubuntu:16.04 +ARG PYTHON_VERSION +LABEL description="Test environment for WALinuxAgent" + +SHELL ["/bin/bash", "-c"] + +RUN \ + apt-get update && \ + apt-get -y install curl bzip2 sudo && \ + groupadd waagent && \ + useradd --shell /bin/bash --create-home -g waagent waagent && \ + curl -sSf --retry 5 -o /tmp/python-${PYTHON_VERSION}.tar.bz2 https://dcrdata.blob.core.windows.net/python/python-${PYTHON_VERSION}.tar.bz2 && \ + tar xjf /tmp/python-${PYTHON_VERSION}.tar.bz2 --directory / && \ + rm -f /tmp/python-${PYTHON_VERSION}.tar.bz2 && \ + echo $'\ + \n\ + cd /home/waagent \n\ + source /home/waagent/virtualenv/python'${PYTHON_VERSION}/bin/activate$' \n\ + function run-tests { \n\ + nosetests --verbose --ignore-files test_cgroupconfigurator_sudo.py /home/waagent/WALinuxAgent/tests \n\ + } \n\ + function run-sudo-tests { \n\ + nosetests --verbose /home/waagent/WALinuxAgent/tests/ga/test_cgroupconfigurator_sudo.py \n\ + } \n\ + ' | tee -a /home/waagent/.profile >> ~/.profile && \ + sed -i 's/mesg n || true/tty -s \&\& mesg n/' ~/.profile && \ + : + +# +# TODO: Some unit tests create helper scripts that use 'python3' as shebang; we should probably port them to Bash, but installing Python 3 as a workaround for now. +# +RUN \ + if [[ "${PYTHON_VERSION}" == "2.6" ]]; then \ + apt-get -y install python3; \ + fi + +USER waagent:waagent + + + diff --git a/tests/python_eol/execute_tests.sh b/tests/python_eol/execute_tests.sh new file mode 100755 index 000000000..4cab4c220 --- /dev/null +++ b/tests/python_eol/execute_tests.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 1 || ! "$1" =~ ^2\.6|3\.4$ ]]; then + echo "Usage: execute_tests.sh 2.6|3.4" + exit 1 +fi + +EXIT_CODE=0 +PYTHON_VERSION=$1 +CONTAINER_IMAGE="waagenttests.azurecr.io/python$PYTHON_VERSION" +CONTAINER_LOGS_DIRECTORY="/home/waagent/logs" +CONTAINER_SOURCES_DIRECTORY="/home/waagent/WALinuxAgent" +NOSETESTS_OPTIONS="--verbose --with-xunit" + +# +# Give ownership of the logs directory to 'waagent' (UID 1000) +# +sudo chown 1000 "$LOGS_DIRECTORY" + +# +# Give the current user access to the Docker daemon +# +sudo usermod -aG docker $USER +newgrp docker < /dev/null + +# +# Pull the container image and execute the tests +# +az acr login --name waagenttests --username "$CR_USER" --password "$CR_SECRET" + +docker pull "$CONTAINER_IMAGE" + +printf "\n***************************************** Running tests for Python $PYTHON_VERSION *****************************************\n\n" + +TEST_SUITE_OPTIONS="--xunit-testsuite-name='Python $PYTHON_VERSION' --xunit-file=$CONTAINER_LOGS_DIRECTORY/waagent-$PYTHON_VERSION.junit.xml" + +set -x +docker run --rm \ + --volume "$BUILD_SOURCESDIRECTORY":"$CONTAINER_SOURCES_DIRECTORY" \ + --volume "$LOGS_DIRECTORY":"$CONTAINER_LOGS_DIRECTORY" \ + "$CONTAINER_IMAGE" \ + bash --login -c "nosetests $NOSETESTS_OPTIONS $TEST_SUITE_OPTIONS --ignore-files test_cgroupconfigurator_sudo.py $CONTAINER_SOURCES_DIRECTORY/tests" \ +|| EXIT_CODE=$(($EXIT_CODE || $?)) +set +x + +printf "\n************************************** Running tests for Python $PYTHON_VERSION [sudo] **************************************\n\n" + +TEST_SUITE_OPTIONS="--xunit-testsuite-name='Python $PYTHON_VERSION [sudo]' --xunit-file=$CONTAINER_LOGS_DIRECTORY/waagent-sudo-$PYTHON_VERSION.junit.xml" + +set -x +docker run --rm \ + --user root \ + --volume "$BUILD_SOURCESDIRECTORY":"$CONTAINER_SOURCES_DIRECTORY" \ + --volume "$LOGS_DIRECTORY":"$CONTAINER_LOGS_DIRECTORY" \ + "$CONTAINER_IMAGE" \ + bash --login -c "nosetests $NOSETESTS_OPTIONS $TEST_SUITE_OPTIONS $CONTAINER_SOURCES_DIRECTORY/tests/ga/test_cgroupconfigurator_sudo.py"\ +|| EXIT_CODE=$(($EXIT_CODE || $?)) +set +x + +exit "$EXIT_CODE" diff --git a/tests/python_eol/pipeline.yml b/tests/python_eol/pipeline.yml new file mode 100644 index 000000000..efd2c9cf8 --- /dev/null +++ b/tests/python_eol/pipeline.yml @@ -0,0 +1,63 @@ +parameters: + - name: python_2_6 + displayName: Python 2.6 + type: boolean + default: true + - name: python_3_4 + displayName: Python 3.4 + type: boolean + default: true + +pool: + name: waagent-pool + +jobs: + - job: SelectPythonVersions + displayName: "Select Python versions" + steps: + - bash: | + # Create the test matrix, which is a JSON object with the selected Python versions, e.g. { "Python_2_6":{"VERSION":"2.6"}, "Python_3_4":{"VERSION":"3.4"} } + declare -a PYTHON_VERSIONS=() + if [ ${{ parameters.python_2_6 }} == "True" ]; then + PYTHON_VERSIONS+=('"Python_2_6":{"VERSION":"2.6"}') + fi + if [ ${{ parameters.python_3_4 }} == "True" ]; then + PYTHON_VERSIONS+=('"Python_3_4": {"VERSION":"3.4"}') + fi + PYTHON_VERSIONS=$(echo ${PYTHON_VERSIONS[@]} | sed 's/ /, /' | sed 's/.*/{ \0 }/') + echo "Python versions: $PYTHON_VERSIONS" + echo "##vso[task.setvariable variable=PYTHON_VERSIONS;isOutput=true]$PYTHON_VERSIONS" + name: "SetPythonVersions" + + - job: "ExecuteTests" + displayName: "Execute tests" + dependsOn: SelectPythonVersions + timeoutInMinutes: 15 + strategy: + matrix: $[ dependencies.SelectPythonVersions.outputs['SetPythonVersions.PYTHON_VERSIONS'] ] + steps: + - task: AzureKeyVault@2 + displayName: "Fetch connection info" + inputs: + azureSubscription: $(connection_info) + KeyVaultName: 'waagenttests' + SecretsFilter: 'CR-USER, CR-SECRET' + + - bash: | + mkdir $(Agent.TempDirectory)/logs + $(Build.SourcesDirectory)/tests/python_eol/execute_tests.sh $(VERSION) + displayName: "Execute tests" + continueOnError: true + env: + CR_USER: $(CR-USER) + CR_SECRET: $(CR-SECRET) + LOGS_DIRECTORY: $(Agent.TempDirectory)/logs + + - task: PublishTestResults@2 + displayName: 'Publish test results' + condition: always() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'waagent*.junit.xml' + searchFolder: $(Agent.TempDirectory)/logs + failTaskOnFailedTests: true