agent publish refactor (Azure#3091)

* agent publish refactor

* support arm 64vm

* convert dict to str

* address comments

* pylint

* new comments

* updated comment

tests_e2e/orchestrator/lib/agent_test_suite.py

@@ -149,6 +149,8 @@ def __init__(self, metadata: TestSuiteMetadata) -> None:
 
         self._test_suites: List[AgentTestSuite]  # Test suites to execute in the environment
 
+        self._test_args: Dict[str, str]  # Additional arguments pass to the test suite
+
         self._cloud: str  # Azure cloud where test VMs are located
         self._subscription_id: str  # Azure subscription where test VMs are located
         self._location: str  # Azure location (region) where test VMs are located
@@ -209,6 +211,7 @@ def _initialize(self, environment: Environment, variables: Dict[str, Any], lisa_
         self._environment_name = variables["c_env_name"]
 
         self._test_suites = variables["c_test_suites"]
+        self._test_args = self._get_test_args(variables["test_args"])
 
         self._cloud = variables["cloud"]
         self._subscription_id = variables["subscription_id"]
@@ -812,12 +815,15 @@ def _create_test_context(self,) -> AgentTestContext:
                 subscription=self._subscription_id,
                 resource_group=self._resource_group_name,
                 name=self._vm_name)
-            return AgentVmTestContext(
+            vm_test_context = AgentVmTestContext(
                 working_directory=self._working_directory,
                 vm=vm,
                 ip_address=self._vm_ip_address,
                 username=self._user,
                 identity_file=self._identity_file)
+            for key in self._test_args:
+                setattr(vm_test_context, key, self._test_args[key])
+            return vm_test_context
         else:
             log.info("Creating test context for scale set")
             if self._create_scale_set:
@@ -836,11 +842,27 @@ def _create_test_context(self,) -> AgentTestContext:
             if self._create_scale_set:
                 self._test_nodes = [_TestNode(name=i.instance_name, ip_address=i.ip_address) for i in scale_set.get_instances_ip_address()]
 
-            return AgentVmssTestContext(
+            vmss_test_context = AgentVmssTestContext(
                 working_directory=self._working_directory,
                 vmss=scale_set,
                 username=self._user,
                 identity_file=self._identity_file)
+            for key in self._test_args:
+                setattr(vmss_test_context, key, self._test_args[key])
+            return vmss_test_context
+
+    @staticmethod
+    def _get_test_args(arg_str) -> Dict[str, str]:
+        """
+        Returns the arguments to be passed to the test classes
+        """
+        test_args: Dict[str, str] = {}
+        if arg_str == "":
+            return test_args
+        for arg in arg_str.split(','):
+            key, value = map(str.strip, arg.split('='))
+            test_args[key] = value
+        return test_args
 
     @staticmethod
     def _mark_log_as_failed():

tests_e2e/orchestrator/runbook.yml

@@ -32,6 +32,12 @@ variable:
     value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline, ext_sequencing, agent_persist_firewall, publish_hostname, agent_update, recover_network_interface"
 
   #
+  # Additional arguments pass to the test suites
+  #
+  - name: test_args
+    value: ""
+    is_case_visible: true
+
   # Parameters used to create test VMs
   #
   - name: subscription_id

tests_e2e/pipeline/pipeline.yml

@@ -19,6 +19,11 @@ parameters:
     type: string
     default: "-"
 
+  - name: test_args
+    displayName: Test Args (additional arguments pass to the test suites. Comma-separated list of key=value pairs)
+    type: string
+    default: "-"
+
   - name: image
     displayName: Image (image/image set name, URN, or VHD)
     type: string
@@ -121,6 +126,7 @@ jobs:
           KEEP_ENVIRONMENT: ${{ parameters.keep_environment }}
           LOCATION: ${{ parameters.location }}
           TEST_SUITES: ${{ parameters.test_suites }}
+          TEST_ARGS: ${{ parameters.test_args }}
           VM_SIZE: ${{ parameters.vm_size }}
 
       - bash: $(Build.SourcesDirectory)/tests_e2e/pipeline/scripts/collect_artifacts.sh

tests_e2e/pipeline/scripts/execute_tests.sh

@@ -54,6 +54,9 @@ if [[ $TEST_SUITES == "-" ]]; then
 else
     TEST_SUITES="-v test_suites:\"$TEST_SUITES\""
 fi
+if [[ $TEST_ARGS == "-" ]]; then
+    TEST_ARGS=""
+fi
 if [[ $IMAGE == "-" ]]; then
     IMAGE=""
 fi
@@ -92,4 +95,5 @@ docker run --rm \
           -v location:\"$LOCATION\" \
           -v vm_size:\"$VM_SIZE\" \
           -v allow_ssh:\"$IP_ADDRESS\" \
+          -v test_args:\"$TEST_ARGS\" \
           $TEST_SUITES"

tests_e2e/tests/agent_publish/agent_publish.py

@@ -18,10 +18,13 @@
 #
 import uuid
 from datetime import datetime
-from typing import Any, Dict, List
+
+from assertpy import fail
 
 from tests_e2e.tests.lib.agent_test import AgentVmTest
 from tests_e2e.tests.lib.agent_test_context import AgentVmTestContext
+from tests_e2e.tests.lib.agent_update_helpers import request_rsm_update
+from tests_e2e.tests.lib.retry import retry_if_false
 from tests_e2e.tests.lib.vm_extension_identifier import VmExtensionIds, VmExtensionIdentifier
 from tests_e2e.tests.lib.logging import log
 from tests_e2e.tests.lib.ssh_client import SshClient
@@ -36,40 +39,126 @@ class AgentPublishTest(AgentVmTest):
     def __init__(self, context: AgentVmTestContext):
         super().__init__(context)
         self._ssh_client: SshClient = self._context.create_ssh_client()
+        self._published_version = self._get_published_version()
 
     def run(self):
         """
         we run the scenario in the following steps:
             1. Print the current agent version before the update
             2. Prepare the agent for the update
-            3. Check for agent update from the log
-            4. Print the agent version after the update
-            5. Ensure CSE is working
+            3. Check for agent update from the log and waagent version
+            4. Ensure CSE is working
         """
         self._get_agent_info()
-        self._prepare_agent()
-        self._check_update()
-        self._get_agent_info()
+
+        log.info("Testing rsm update flow....")
+        self._prepare_agent_for_rsm_update()
+        self._check_update_from_log()
+        self._verify_current_agent_version()
+        self._check_cse()
+
+        log.info("Testing self update flow....")
+        self._prepare_agent_for_self_update()
+        self._check_update_from_log()
+        self._verify_current_agent_version()
+
         self._check_cse()
 
     def get_ignore_errors_before_timestamp(self) -> datetime:
         timestamp = self._ssh_client.run_command("agent_publish-get_agent_log_record_timestamp.py")
         return datetime.strptime(timestamp.strip(), u'%Y-%m-%d %H:%M:%S.%f')
 
+    def _get_published_version(self):
+        """
+        Gets version from test_args if provided, else use the release version from source code version.py
+        """
+        if hasattr(self._context, "published_version"):
+            return self._context.published_version
+
+        version = self._ssh_client.run_command("pypy3 -c 'from azurelinuxagent.common.version import AGENT_VERSION; print(AGENT_VERSION)'").rstrip()
+        return version
+
     def _get_agent_info(self) -> None:
         stdout: str = self._ssh_client.run_command("waagent-version", use_sudo=True)
         log.info('Agent info \n%s', stdout)
 
-    def _prepare_agent(self) -> None:
+    def _verify_agent_reported_supported_feature_flag(self):
+        """
+        RSM update rely on supported feature flag that agent sends to CRP.So, checking if GA reports feature flag from reported status
+        """
+        log.info(
+            "Executing verify_versioning_supported_feature.py remote script to verify agent reported supported feature flag, so that CRP can send RSM update request")
+        self._run_remote_test(self._ssh_client, "agent_update-verify_versioning_supported_feature.py", use_sudo=True)
+        log.info("Successfully verified that Agent reported VersioningGovernance supported feature flag")
+
+    def _check_rsm_gs(self, requested_version: str) -> None:
+        # This checks if RSM GS available to the agent after we send the rsm update request
+        log.info(
+            'Executing wait_for_rsm_gs.py remote script to verify latest GS contain requested version after rsm update requested')
+        self._run_remote_test(self._ssh_client, f"agent_update-wait_for_rsm_gs.py --version {requested_version}",
+                              use_sudo=True)
+        log.info('Verified latest GS contain requested version after rsm update requested')
+
+    def _prepare_agent_for_rsm_update(self) -> None:
+        """
+        This method prepares the agent for the RSM update
+        """
+        # First we update the agent to latest version like prod
+        # Next send RSM update request for new published test version
+        log.info(
+            'Updating agent config flags to allow and download test versions')
+        output: str = self._ssh_client.run_command(
+                              "update-waagent-conf AutoUpdate.Enabled=y AutoUpdate.UpdateToLatestVersion=y", use_sudo=True)
+        log.info('Successfully updated agent update config \n %s', output)
+
+        self._verify_agent_reported_supported_feature_flag()
+        arch_type = self._ssh_client.get_architecture()
+        request_rsm_update(self._published_version, self._context.vm, arch_type)
+        self._check_rsm_gs(self._published_version)
+
+        output: str = self._ssh_client.run_command(
+                              "update-waagent-conf Debug.EnableGAVersioning=y AutoUpdate.GAFamily=Test", use_sudo=True)
+        log.info('Successfully enabled rsm updates \n %s', output)
+
+    def _prepare_agent_for_self_update(self) -> None:
+        """
+        This method prepares the agent for the self update
+        """
         log.info("Modifying agent update related config flags and renaming the log file")
-        self._run_remote_test(self._ssh_client, "sh -c 'agent-service stop && mv /var/log/waagent.log /var/log/waagent.$(date --iso-8601=seconds).log && update-waagent-conf AutoUpdate.UpdateToLatestVersion=y AutoUpdate.GAFamily=Test AutoUpdate.Enabled=y Extensions.Enabled=y'", use_sudo=True)
-        log.info('Renamed log file and updated agent-update DownloadNewAgents GAFamily config flags')
+        setup_script = ("agent-service stop &&  mv /var/log/waagent.log /var/log/waagent.$(date --iso-8601=seconds).log && "
+                        "rm -rf /var/lib/waagent/WALinuxAgent-* && "
+                        "update-waagent-conf AutoUpdate.UpdateToLatestVersion=y AutoUpdate.GAFamily=Test AutoUpdate.Enabled=y Extensions.Enabled=y Debug.EnableGAVersioning=n")
+        self._run_remote_test(self._ssh_client, f"sh -c '{setup_script}'", use_sudo=True)
+        log.info('Renamed log file and updated self-update config flags')
 
-    def _check_update(self) -> None:
+    def _check_update_from_log(self) -> None:
         log.info("Verifying for agent update status")
-        self._run_remote_test(self._ssh_client, "agent_publish-check_update.py")
+        self._run_remote_test(self._ssh_client, f"agent_publish-check_update.py --published-version {self._published_version}")
         log.info('Successfully checked the agent update')
 
+    def _verify_current_agent_version(self) -> None:
+        """
+        Verify current agent version running on published version
+        """
+
+        def _check_agent_version(version: str) -> bool:
+            waagent_version: str = self._ssh_client.run_command("waagent-version", use_sudo=True)
+            expected_version = f"Goal state agent: {version}"
+            if expected_version in waagent_version:
+                return True
+            else:
+                return False
+
+        waagent_version: str = ""
+        log.info("Verifying agent updated to published version: {0}".format(self._published_version))
+        success: bool = retry_if_false(lambda: _check_agent_version(self._published_version))
+        if not success:
+            fail("Guest agent didn't update to published version {0} but found \n {1}. \n ".format(
+                self._published_version, waagent_version))
+        waagent_version: str = self._ssh_client.run_command("waagent-version", use_sudo=True)
+        log.info(
+            f"Successfully verified agent updated to published version. Current agent version running:\n {waagent_version}")
+
     def _check_cse(self) -> None:
         custom_script_2_1 = VirtualMachineExtensionClient(
             self._context.vm,
@@ -86,20 +175,6 @@ def _check_cse(self) -> None:
         )
         custom_script_2_1.assert_instance_view(expected_version="2.1", expected_message=message)
 
-    def get_ignore_error_rules(self) -> List[Dict[str, Any]]:
-        ignore_rules = [
-            #
-            # This is expected as latest version can be the less than test version
-            #
-            # WARNING ExtHandler ExtHandler Agent WALinuxAgent-9.9.9.9 is permanently blacklisted
-            #
-            {
-                'message': r"Agent WALinuxAgent-9.9.9.9 is permanently blacklisted"
-            }
-
-        ]
-        return ignore_rules
-
 
 if __name__ == "__main__":
     AgentPublishTest.run_from_command_line()