From 93176744626df24761dc389fae26069ffb4502fc Mon Sep 17 00:00:00 2001 From: Maddie Ford Date: Tue, 3 Oct 2023 14:36:55 -0700 Subject: [PATCH 1/8] Add cpu arch to telem and local logs --- azurelinuxagent/common/event.py | 9 ++++++++- azurelinuxagent/ga/update.py | 6 +++++- resource_groups_to_delete.txt | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 resource_groups_to_delete.txt diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index 467960806..c561cb41a 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -382,6 +382,10 @@ def __init__(self): self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.VMId, "VMId_UNINITIALIZED")) self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.ImageOrigin, 0)) + @staticmethod + def _get_vm_arch(self): + return platform.machine() + @staticmethod def _get_os_version(): return "{0}:{1}-{2}-{3}:{4}".format(platform.system(), DISTRO_NAME, DISTRO_VERSION, DISTRO_CODE_NAME, platform.release()) @@ -591,13 +595,16 @@ def add_common_event_parameters(self, event, event_timestamp): This method is called for all events and ensures all telemetry fields are added before the event is sent out. Note that the event timestamp is saved in the OpcodeName field. """ + keyword_name = { + "CpuArchitecture": self._get_vm_arch() + } common_params = [TelemetryEventParam(CommonTelemetryEventSchema.GAVersion, CURRENT_AGENT), TelemetryEventParam(CommonTelemetryEventSchema.ContainerId, AgentGlobals.get_container_id()), TelemetryEventParam(CommonTelemetryEventSchema.OpcodeName, event_timestamp.strftime(logger.Logger.LogTimeFormatInUTC)), TelemetryEventParam(CommonTelemetryEventSchema.EventTid, threading.current_thread().ident), TelemetryEventParam(CommonTelemetryEventSchema.EventPid, os.getpid()), TelemetryEventParam(CommonTelemetryEventSchema.TaskName, threading.current_thread().getName()), - TelemetryEventParam(CommonTelemetryEventSchema.KeywordName, '')] + TelemetryEventParam(CommonTelemetryEventSchema.KeywordName, json.dumps(keyword_name))] if event.eventId == TELEMETRY_EVENT_EVENT_ID and event.providerId == TELEMETRY_EVENT_PROVIDER_ID: # Currently only the GuestAgentExtensionEvents has these columns, the other tables dont have them so skipping diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index 8065194d7..c6a33b5a2 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -315,9 +315,13 @@ def run(self, debug=False): logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION) logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO) + vm_arch = self._get_vm_arch() + logger.info("CPU Arch: {0}", vm_arch) + os_info_msg = u"Distro: {dist_name}-{dist_ver}; "\ u"OSUtil: {util_name}; AgentService: {service_name}; "\ u"Python: {py_major}.{py_minor}.{py_micro}; "\ + u"Arch: {vm_arch}; "\ u"systemd: {systemd}; "\ u"LISDrivers: {lis_ver}; "\ u"logrotate: {has_logrotate};".format( @@ -325,7 +329,7 @@ def run(self, debug=False): util_name=type(self.osutil).__name__, service_name=self.osutil.service_name, py_major=PY_VERSION_MAJOR, py_minor=PY_VERSION_MINOR, - py_micro=PY_VERSION_MICRO, systemd=systemd.is_systemd(), + py_micro=PY_VERSION_MICRO, vm_arch=vm_arch, systemd=systemd.is_systemd(), lis_ver=get_lis_version(), has_logrotate=has_logrotate() ) logger.info(os_info_msg) diff --git a/resource_groups_to_delete.txt b/resource_groups_to_delete.txt new file mode 100644 index 000000000..3755c468b --- /dev/null +++ b/resource_groups_to_delete.txt @@ -0,0 +1 @@ +lisa-WALinuxAgent-extseq-20231003-111343-e0 From fb8b08e233ca7117590dbe0d6dbc22d6eed82d9d Mon Sep 17 00:00:00 2001 From: Maddie Ford Date: Wed, 4 Oct 2023 18:00:02 -0700 Subject: [PATCH 2/8] Change get_vm_arch to static method --- azurelinuxagent/common/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index c561cb41a..bfe612937 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -383,7 +383,7 @@ def __init__(self): self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.ImageOrigin, 0)) @staticmethod - def _get_vm_arch(self): + def _get_vm_arch(): return platform.machine() @staticmethod From dc98535cf1460abce417fbcfc347394fb067ad82 Mon Sep 17 00:00:00 2001 From: Maddie Ford Date: Wed, 4 Oct 2023 18:51:33 -0700 Subject: [PATCH 3/8] update unit tests --- tests/common/test_event.py | 3 ++- tests/ga/test_send_telemetry_events.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/common/test_event.py b/tests/common/test_event.py index 28f2e3860..435ac2e80 100644 --- a/tests/common/test_event.py +++ b/tests/common/test_event.py @@ -20,6 +20,7 @@ import json import os +import platform import re import shutil import threading @@ -70,7 +71,7 @@ def setUp(self): CommonTelemetryEventSchema.EventTid: threading.current_thread().ident, CommonTelemetryEventSchema.EventPid: os.getpid(), CommonTelemetryEventSchema.TaskName: threading.current_thread().getName(), - CommonTelemetryEventSchema.KeywordName: '', + CommonTelemetryEventSchema.KeywordName: json.dumps({"CpuArchitecture": platform.machine()}), # common parameters computed from the OS platform CommonTelemetryEventSchema.OSVersion: EventLoggerTools.get_expected_os_version(), CommonTelemetryEventSchema.ExecutionMode: AGENT_EXECUTION_MODE, diff --git a/tests/ga/test_send_telemetry_events.py b/tests/ga/test_send_telemetry_events.py index c9e04a38c..dfee29ff1 100644 --- a/tests/ga/test_send_telemetry_events.py +++ b/tests/ga/test_send_telemetry_events.py @@ -368,13 +368,13 @@ def test_it_should_enqueue_and_send_events_properly(self, mock_lib_dir, *_): '' \ '' \ '' \ - '' \ + '' \ '' \ '' \ - '' \ + '' \ '' \ - '' \ - '' \ + '' \ + '' \ '' \ '' \ '' \ @@ -384,7 +384,7 @@ def test_it_should_enqueue_and_send_events_properly(self, mock_lib_dir, *_): '' \ '' \ ']]>'.format(AGENT_VERSION, TestSendTelemetryEventsHandler._TEST_EVENT_OPERATION, CURRENT_AGENT, test_opcodename, test_eventtid, - test_eventpid, test_taskname, osversion, int(osutil.get_total_mem()), + test_eventpid, test_taskname, json.dumps({"CpuArchitecture": platform.machine()}), osversion, int(osutil.get_total_mem()), osutil.get_processor_cores()).encode('utf-8') self.assertIn(sample_message, collected_event) From b80a078b57a26d86dbcf70ffd78b573984330a05 Mon Sep 17 00:00:00 2001 From: Maddie Ford Date: Thu, 5 Oct 2023 10:45:13 -0700 Subject: [PATCH 4/8] Remove e2e pipeline file --- resource_groups_to_delete.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 resource_groups_to_delete.txt diff --git a/resource_groups_to_delete.txt b/resource_groups_to_delete.txt deleted file mode 100644 index 3755c468b..000000000 --- a/resource_groups_to_delete.txt +++ /dev/null @@ -1 +0,0 @@ -lisa-WALinuxAgent-extseq-20231003-111343-e0 From 28b2286988801b326fca820c99a01584ea8efded Mon Sep 17 00:00:00 2001 From: Maddie Ford Date: Thu, 5 Oct 2023 11:16:22 -0700 Subject: [PATCH 5/8] Remove arch from heartbeat --- azurelinuxagent/ga/update.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index c6a33b5a2..c56884ddf 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -992,13 +992,10 @@ def _send_heartbeat_telemetry(self, protocol): if datetime.utcnow() >= (self._last_telemetry_heartbeat + UpdateHandler.TELEMETRY_HEARTBEAT_PERIOD): dropped_packets = self.osutil.get_firewall_dropped_packets(protocol.get_endpoint()) auto_update_enabled = 1 if conf.get_autoupdate_enabled() else 0 - # Include vm architecture in the heartbeat message because the kusto table does not have - # a separate column for it. - vmarch = self._get_vm_arch() - telemetry_msg = "{0};{1};{2};{3};{4};{5}".format(self._heartbeat_counter, self._heartbeat_id, dropped_packets, + telemetry_msg = "{0};{1};{2};{3};{4}".format(self._heartbeat_counter, self._heartbeat_id, dropped_packets, self._heartbeat_update_goal_state_error_count, - auto_update_enabled, vmarch) + auto_update_enabled) debug_log_msg = "[DEBUG HeartbeatCounter: {0};HeartbeatId: {1};DroppedPackets: {2};" \ "UpdateGSErrors: {3};AutoUpdate: {4}]".format(self._heartbeat_counter, self._heartbeat_id, dropped_packets, From 8772a9dbd8cf36713e4f85035221b972e6ffc3eb Mon Sep 17 00:00:00 2001 From: Maddie Ford Date: Fri, 6 Oct 2023 14:17:10 -0700 Subject: [PATCH 6/8] Move get_vm_arch to osutil --- azurelinuxagent/common/event.py | 14 +++++--------- azurelinuxagent/common/osutil/default.py | 8 ++++++++ azurelinuxagent/ga/update.py | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py index ef6ea423f..514c727ff 100644 --- a/azurelinuxagent/common/event.py +++ b/azurelinuxagent/common/event.py @@ -366,10 +366,14 @@ def __init__(self): # Parameters from OS osutil = get_osutil() + keyword_name = { + "CpuArchitecture": osutil.get_vm_arch() + } self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.OSVersion, EventLogger._get_os_version())) self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.ExecutionMode, AGENT_EXECUTION_MODE)) self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.RAM, int(EventLogger._get_ram(osutil)))) self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.Processors, int(EventLogger._get_processors(osutil)))) + self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.KeywordName, json.dumps(keyword_name))) # Parameters from goal state self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.TenantName, "TenantName_UNINITIALIZED")) @@ -383,10 +387,6 @@ def __init__(self): self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.VMId, "VMId_UNINITIALIZED")) self._common_parameters.append(TelemetryEventParam(CommonTelemetryEventSchema.ImageOrigin, 0)) - @staticmethod - def _get_vm_arch(): - return platform.machine() - @staticmethod def _get_os_version(): return "{0}:{1}-{2}-{3}:{4}".format(platform.system(), DISTRO_NAME, DISTRO_VERSION, DISTRO_CODE_NAME, platform.release()) @@ -596,16 +596,12 @@ def add_common_event_parameters(self, event, event_timestamp): This method is called for all events and ensures all telemetry fields are added before the event is sent out. Note that the event timestamp is saved in the OpcodeName field. """ - keyword_name = { - "CpuArchitecture": self._get_vm_arch() - } common_params = [TelemetryEventParam(CommonTelemetryEventSchema.GAVersion, CURRENT_AGENT), TelemetryEventParam(CommonTelemetryEventSchema.ContainerId, AgentGlobals.get_container_id()), TelemetryEventParam(CommonTelemetryEventSchema.OpcodeName, event_timestamp.strftime(logger.Logger.LogTimeFormatInUTC)), TelemetryEventParam(CommonTelemetryEventSchema.EventTid, threading.current_thread().ident), TelemetryEventParam(CommonTelemetryEventSchema.EventPid, os.getpid()), - TelemetryEventParam(CommonTelemetryEventSchema.TaskName, threading.current_thread().getName()), - TelemetryEventParam(CommonTelemetryEventSchema.KeywordName, json.dumps(keyword_name))] + TelemetryEventParam(CommonTelemetryEventSchema.TaskName, threading.current_thread().getName())] if event.eventId == TELEMETRY_EVENT_EVENT_ID and event.providerId == TELEMETRY_EVENT_PROVIDER_ID: # Currently only the GuestAgentExtensionEvents has these columns, the other tables dont have them so skipping diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py index 6430f83ec..05fba1a5c 100644 --- a/azurelinuxagent/common/osutil/default.py +++ b/azurelinuxagent/common/osutil/default.py @@ -149,6 +149,14 @@ def get_systemd_unit_file_install_path(): def get_agent_bin_path(): return "/usr/sbin" + @staticmethod + def get_vm_arch() -> str: + try: + return platform.machine() + except Exception as e: + logger.warn("Unable to determine cpu architecture: {0}", ustr(e)) + return "unknown" + def get_firewall_dropped_packets(self, dst_ip=None): # If a previous attempt failed, do not retry global _enable_firewall # pylint: disable=W0603 diff --git a/azurelinuxagent/ga/update.py b/azurelinuxagent/ga/update.py index a4af705e8..147402709 100644 --- a/azurelinuxagent/ga/update.py +++ b/azurelinuxagent/ga/update.py @@ -315,7 +315,7 @@ def run(self, debug=False): logger.info("OS: {0} {1}", DISTRO_NAME, DISTRO_VERSION) logger.info("Python: {0}.{1}.{2}", PY_VERSION_MAJOR, PY_VERSION_MINOR, PY_VERSION_MICRO) - vm_arch = self._get_vm_arch() + vm_arch = self.osutil.get_vm_arch() logger.info("CPU Arch: {0}", vm_arch) os_info_msg = u"Distro: {dist_name}-{dist_ver}; "\ From e776787fed066312c1158ce522569980015dac75 Mon Sep 17 00:00:00 2001 From: Maddie Ford Date: Fri, 6 Oct 2023 14:19:32 -0700 Subject: [PATCH 7/8] fix syntax issue --- azurelinuxagent/common/osutil/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py index 05fba1a5c..69e20bea7 100644 --- a/azurelinuxagent/common/osutil/default.py +++ b/azurelinuxagent/common/osutil/default.py @@ -150,7 +150,7 @@ def get_agent_bin_path(): return "/usr/sbin" @staticmethod - def get_vm_arch() -> str: + def get_vm_arch(): try: return platform.machine() except Exception as e: From 41f5c4a1e59695ec645e626ce2ec4e462f532cbb Mon Sep 17 00:00:00 2001 From: Maddie Ford Date: Fri, 6 Oct 2023 14:31:38 -0700 Subject: [PATCH 8/8] Fix unit test --- tests/ga/test_send_telemetry_events.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/ga/test_send_telemetry_events.py b/tests/ga/test_send_telemetry_events.py index dfee29ff1..a9c87dde9 100644 --- a/tests/ga/test_send_telemetry_events.py +++ b/tests/ga/test_send_telemetry_events.py @@ -368,13 +368,13 @@ def test_it_should_enqueue_and_send_events_properly(self, mock_lib_dir, *_): '' \ '' \ '' \ - '' \ '' \ '' \ - '' \ + '' \ '' \ - '' \ - '' \ + '' \ + '' \ + '' \ '' \ '' \ '' \ @@ -384,8 +384,8 @@ def test_it_should_enqueue_and_send_events_properly(self, mock_lib_dir, *_): '' \ '' \ ']]>'.format(AGENT_VERSION, TestSendTelemetryEventsHandler._TEST_EVENT_OPERATION, CURRENT_AGENT, test_opcodename, test_eventtid, - test_eventpid, test_taskname, json.dumps({"CpuArchitecture": platform.machine()}), osversion, int(osutil.get_total_mem()), - osutil.get_processor_cores()).encode('utf-8') + test_eventpid, test_taskname, osversion, int(osutil.get_total_mem()), + osutil.get_processor_cores(), json.dumps({"CpuArchitecture": platform.machine()})).encode('utf-8') self.assertIn(sample_message, collected_event)