From f657199f379b62f4f4a97c4317890a3b5a385cc5 Mon Sep 17 00:00:00 2001 From: narrieta Date: Wed, 26 Jul 2023 07:59:53 -0700 Subject: [PATCH 1/2] Install assertpy on test VMs --- tests_e2e/orchestrator/docker/Dockerfile | 6 +- .../orchestrator/lib/agent_test_suite.py | 4 +- tests_e2e/orchestrator/scripts/prepare-pypy | 56 +++++++++++++++++++ 3 files changed, 61 insertions(+), 5 deletions(-) create mode 100755 tests_e2e/orchestrator/scripts/prepare-pypy diff --git a/tests_e2e/orchestrator/docker/Dockerfile b/tests_e2e/orchestrator/docker/Dockerfile index 2d709c791..956200521 100644 --- a/tests_e2e/orchestrator/docker/Dockerfile +++ b/tests_e2e/orchestrator/docker/Dockerfile @@ -73,6 +73,7 @@ RUN \ cd $HOME && \ git clone https://github.com/microsoft/lisa.git && \ cd lisa && \ + git checkout 05aaf490df0ce24018c6ab4e714bcc9ca8765c76 && \ \ python3 -m pip install --upgrade pip && \ python3 -m pip install --editable .[azure,libvirt] --config-settings editable_mode=compat && \ @@ -86,9 +87,8 @@ RUN \ # \ # Download Pypy to a known location, from which it will be installed to the test VMs. \ # \ - mkdir $HOME/bin && \ - wget https://downloads.python.org/pypy/pypy3.7-v7.3.5-linux64.tar.bz2 -O /tmp/pypy3.7-x64.tar.bz2 && \ - wget https://downloads.python.org/pypy/pypy3.7-v7.3.5-aarch64.tar.bz2 -O /tmp/pypy3.7-arm64.tar.bz2 && \ + wget https://dcrdata.blob.core.windows.net/python/pypy3.7-x64.tar.bz2 -O /tmp/pypy3.7-x64.tar.bz2 && \ + wget https://dcrdata.blob.core.windows.net/python/pypy3.7-arm64.tar.bz2 -O /tmp/pypy3.7-arm64.tar.bz2 && \ \ # \ # The setup for the tests depends on a few paths; add those to the profile \ diff --git a/tests_e2e/orchestrator/lib/agent_test_suite.py b/tests_e2e/orchestrator/lib/agent_test_suite.py index 06a5acde4..6f9eda020 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite.py @@ -294,10 +294,10 @@ def _setup_node(self, install_test_agent: bool) -> None: # if self.context.ssh_client.get_architecture() == "aarch64": pypy_path = Path("/tmp/pypy3.7-arm64.tar.bz2") - pypy_download = "https://downloads.python.org/pypy/pypy3.7-v7.3.5-aarch64.tar.bz2" + pypy_download = "https://dcrdata.blob.core.windows.net/python/pypy3.7-arm64.tar.bz2" else: pypy_path = Path("/tmp/pypy3.7-x64.tar.bz2") - pypy_download = "https://downloads.python.org/pypy/pypy3.7-v7.3.5-linux64.tar.bz2" + pypy_download = "https://dcrdata.blob.core.windows.net/python/pypy3.7-x64.tar.bz2" if pypy_path.exists(): log.info("Found Pypy at %s", pypy_path) else: diff --git a/tests_e2e/orchestrator/scripts/prepare-pypy b/tests_e2e/orchestrator/scripts/prepare-pypy new file mode 100755 index 000000000..fe469c914 --- /dev/null +++ b/tests_e2e/orchestrator/scripts/prepare-pypy @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This script is used to prepare a tarball containing Pypy with the assert-py module pre-installed. +# It needs to be run on x64 and arm64 VMs and the resulting tarballs need to be uploaded to storage, +# from where they are downloaded and installed to the test VMs (see wiki for detail). +# + +set -euo pipefail + +cd /tmp +rm -rf pypy3.7-* + +arch=$(uname -m) +printf "Preparing Pypy for architecture %s...\n" $arch + +printf "\n*** Downloading Pypy...\n" +if [[ $arch == "aarch64" ]]; then + tarball="pypy3.7-arm64.tar.bz2" + wget https://downloads.python.org/pypy/pypy3.7-v7.3.5-aarch64.tar.bz2 -O $tarball +else + tarball="pypy3.7-x64.tar.bz2" + wget https://downloads.python.org/pypy/pypy3.7-v7.3.5-linux64.tar.bz2 -O $tarball +fi + +printf "\n*** Installing assertpy...\n" +tar xf $tarball +./pypy3.7-v7.3.5-*/bin/pypy -m ensurepip +./pypy3.7-v7.3.5-*/bin/pypy -mpip install assertpy + +printf "\n*** Creating new tarball for Pypy...\n" +# remove the cache files created when Pypy, and set the owner to 0/0, in order to match the original tarball +find pypy3.7-v7.3.5-* -name '*.pyc' -exec rm {} \; +mv -v $tarball "$tarball.original" +tar cf $tarball --bzip2 --owner 0:0 --group 0:0 pypy3.7-v7.3.5-* +rm -rf pypy3.7-v7.3.5-* + +printf "\nPypy is ready at %s\n" "$(pwd)/$tarball" + From 395682d0e420a1f68fed348b6d45a6d299d0a1cb Mon Sep 17 00:00:00 2001 From: narrieta Date: Wed, 26 Jul 2023 15:05:12 -0700 Subject: [PATCH 2/2] set versions --- tests_e2e/orchestrator/docker/Dockerfile | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests_e2e/orchestrator/docker/Dockerfile b/tests_e2e/orchestrator/docker/Dockerfile index 956200521..bbd460e6e 100644 --- a/tests_e2e/orchestrator/docker/Dockerfile +++ b/tests_e2e/orchestrator/docker/Dockerfile @@ -68,12 +68,16 @@ RUN \ export PATH="$HOME/.local/bin:$PATH" && \ \ # \ - # Install LISA \ + # Install LISA. \ + # \ + # (note that we use a specific commit, which is the version of LISA that has been verified to work with our \ + # tests; when taking a new LISA version, make sure to verify that the tests work OK before pushing the \ + # Docker image to our registry) \ # \ cd $HOME && \ git clone https://github.com/microsoft/lisa.git && \ cd lisa && \ - git checkout 05aaf490df0ce24018c6ab4e714bcc9ca8765c76 && \ + git checkout a030c5e6a0695db77dbf5bd52a45d07cbbf00087 && \ \ python3 -m pip install --upgrade pip && \ python3 -m pip install --editable .[azure,libvirt] --config-settings editable_mode=compat && \ @@ -81,8 +85,12 @@ RUN \ # \ # Install additional test dependencies \ # \ + # (note that we update azure-mgmt-compute to 29.1.0 - LISA installs 26.1; this is needed in order to access \ + # osProfile.linuxConfiguration.enableVMAgentPlatformUpdates in the VM model - that property is used by some \ + # tests, such as Agent versioning) \ + # \ python3 -m pip install distro msrestazure pytz && \ - python3 -m pip install azure-mgmt-compute --upgrade && \ + python3 -m pip install azure-mgmt-compute==29.1.0 --upgrade && \ \ # \ # Download Pypy to a known location, from which it will be installed to the test VMs. \ @@ -91,6 +99,11 @@ RUN \ wget https://dcrdata.blob.core.windows.net/python/pypy3.7-arm64.tar.bz2 -O /tmp/pypy3.7-arm64.tar.bz2 && \ \ # \ + # Install pudb, which can be useful to debug issues in the image \ + # \ + python3 -m pip install pudb && \ + \ + # \ # The setup for the tests depends on a few paths; add those to the profile \ # \ echo 'export PYTHONPATH="$HOME/WALinuxAgent"' >> $HOME/.bash_profile && \