Extract text from screenshot at test failure and errors from VM logs

Signed-off-by: Qi Zhang <[email protected]>
Signed-off-by: Qi Zhang <[email protected]>

README.md

@@ -4,15 +4,16 @@
 
 ### Prerequisites
 1. Install Ansible on your control machine, please refer to [Installing Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html)
-2. Install required Python libraries in requirements.txt
+2. Install Tesseract Open Source OCR Engine for extracting text from screenshots, please refer to [Installing Tesseract](https://github.com/tesseract-ocr/tesseract?tab=readme-ov-file#installing-tesseract)
+3. Install required Python libraries in requirements.txt
 ```
 $ pip install -r requirements.txt
 ```
-3. Install required Ansible collections with latest version in requirements.yml
+4. Install required Ansible collections with latest version in requirements.yml
 ```
 $ ansible-galaxy install -r requirements.yml
 ```
-4. Log in to local control machine as root or a user in sudoers, which must enable NOPASSWD for all commands
+5. Log in to local control machine as root or a user in sudoers, which must enable NOPASSWD for all commands
 
 ### Steps to Launch Testing
 1. Git clone project from github to your workspace on control machine.

autoinstall/Ubuntu/Desktop/Ubiquity/ubuntu.seed

@@ -63,7 +63,7 @@ d-i grub-installer/only_debian boolean true
 # preseeding is read.
 d-i preseed/early_command string \
     echo "Executing early command" >/dev/ttyS0; \
-    cp -a /cdrom/preseed/{{ pre_install_script_file }} /root/{{ pre_install_script_file }}; \
+    cp -a /root/cdrom/preseed/{{ pre_install_script_file }} /root/{{ pre_install_script_file }}; \
     /bin/sh /root/{{ pre_install_script_file }} >/dev/ttyS0;
 
 # Execute post install script on success

common/README.md

@@ -191,3 +191,7 @@
 * compose_vm_cdroms.yml: Generate VM CDROM device info list for creating new VM
 * download_iso_and_transfer.yml: Download ISO file and transfer to ESXi datastore
 * get_iso_file_list.yml: Generate and validate OS installation ISO file list
+* extract_errors_from_log.yml: Extract error messages in a log file downloaded from guest
+* extract_text_from_screenshot.yml: Extract text from an image file
+* collect_serial_port_log.yml: Collect VM serial port log from datastore
+* wait_vm_deploy_checkpoint.yml: Wait for a checkpoint at deploy_vm

common/collect_serial_port_log.yml

@@ -0,0 +1,49 @@
+# Copyright 2023 VMware, Inc.
+# SPDX-License-Identifier: BSD-2-Clause
+---
+# Collect VM serial port log file
+# Parameters:
+#   vm_dir_name: VM directory name, which is defined in vm_get_vm_info.yml
+#   vm_serial_file_name: VM serial port file name, which is defined in vm_add_serial_port.yml
+#   vm_serial_file_local_name: the local file name of downloaded serial port log file
+# Return:
+#   vm_serial_file_local_path: the local path to downloaded serial port log file   
+#
+- name: "Initialize facts of downloaded serial port log file name and path"
+  ansible.builtin.set_fact:
+    vm_serial_file_local_path: ""
+
+- name: "Collect VM serial port log"
+  when:
+    - vm_dir_name is defined
+    - vm_dir_name
+    - vm_serial_file_name is defined
+    - vm_serial_file_name
+  block:
+    - name: "Download VM serial port log file from datastore"
+      include_tasks: ../../common/esxi_download_datastore_file.yml
+      vars:
+        src_datastore: "{{ datastore }}"
+        src_file_path: "{{ vm_dir_name }}/{{ vm_serial_file_name }}"
+        dest_file_path: "{{ current_test_log_folder }}/{{ vm_serial_file_name }}"
+        download_file_fail_ignore: true
+
+    - name: "Get the local path to downloaded serial port log"
+      when:
+        - file_in_datastore_result is defined
+        - file_in_datastore_result == 'Success'
+        - datastore_file_download_result.changed is defined
+        - datastore_file_download_result.changed
+      block:
+        - name: "Check downloaded VM serial log at local"
+          ansible.builtin.stat:
+            path: "{{ current_test_log_folder }}/{{ vm_serial_file_name }}"
+          register: local_file_stat_result
+          ignore_errors: True
+
+        - name: "Set fact of downloaded serial log path at local"
+          ansible.builtin.set_fact:
+            vm_serial_file_local_path: "{{ current_test_log_folder }}/{{ vm_serial_file_name }}"
+          when:
+            - local_file_stat_result.stat.exists is defined
+            - local_file_stat_result.stat.exists

common/extract_errors_from_log.yml

@@ -0,0 +1,32 @@
+# Copyright 2023 VMware, Inc.
+# SPDX-License-Identifier: BSD-2-Clause
+---
+# Extract call trace or other error messages in a log file downloaded from guest
+# Parameters:
+#   local_log_path: The local log file path to extract call trace and errors
+# Return:
+#   errors_in_log: The call trace or error messages extracted from the log file
+#
+- name: "Check the local log path is an absolute path"
+  ansible.builtin.assert:
+    that:
+      - local_log_path is defined
+      - local_log_path
+      - local_log_path is ansible.builtin.abs
+    fail_msg: "Parameter 'local_log_path' must be set with a local absolute path"
+
+- name: "Initialize the facts of extracted errors from log"
+  ansible.builtin.set_fact:
+    errors_in_log: []
+
+- name: "Extract error messages from log file"
+  ansible.builtin.script: "../tools/extractor.py -t error -f {{ local_log_path }}"
+  ignore_errors: True
+  register: extract_error_result
+
+- name: "Set fact of extracted errors from log file"
+  ansible.builtin.set_fact:
+    errors_in_log: "{{ extract_error_result.stdout_lines | select }}"
+  when:
+    - extract_error_result.stdout_lines is defined
+    - extract_error_result.stdout_lines | length > 0

common/extract_text_from_screenshot.yml

@@ -0,0 +1,32 @@
+# Copyright 2023 VMware, Inc.
+# SPDX-License-Identifier: BSD-2-Clause
+---
+# Extract text from a local screenshot file
+# Parameters:
+#   local_screenshot_path: The local screenshot file path
+# Return:
+#   text_in_screenshot: The text extracted from the screenshot file
+#
+- name: "Check local screenshot file path is an absolute path"
+  ansible.builtin.assert:
+    that:
+      - local_screenshot_path is defined
+      - local_screenshot_path
+      - local_screenshot_path is ansible.builtin.abs
+    fail_msg: "Parameter 'local_screenshot_path' must be set with a local absolute path"
+
+- name: "Initialize the fact of extracted screenshot text"
+  ansible.builtin.set_fact:
+    text_in_screenshot: []
+
+- name: "Extract text from local screenshot file"
+  ansible.builtin.script: "../tools/extractor.py -t text -f {{ local_screenshot_path }}"
+  ignore_errors: True
+  register: extract_text_result
+
+- name: "Set fact of extracted screenshot text"
+  ansible.builtin.set_fact:
+    text_in_screenshot: "{{ extract_text_result.stdout_lines | select }}"
+  when:
+    - extract_text_result.stdout_lines is defined
+    - extract_text_result.stdout_lines | length > 0

common/test_rescue.yml

@@ -43,7 +43,26 @@
     - name: "Take a screenshot at VM current state"
       include_tasks: vm_take_screenshot.yml
       vars:
-        vm_take_screenshot_local_path: "{{ current_test_log_folder }}"
+        vm_screenshot_local_dir: "{{ current_test_log_folder }}"
+        vm_screenshot_local_name: "screenshot_at_{{ current_testcase_index }}_{{ ansible_play_name }}.png"
+        vm_screen_active: "{{ gosv_test_suite == 'linux' }}"
+
+    - name: "Extract text from VM screenshot at failed test case {{ current_testcase_name }}"
+      when:
+        - vm_screenshot_local_path
+        - gosv_test_suite == 'windows' or ansible_play_name != 'deploy_vm'
+      block:
+        - name: "Extract text from screenshot file"
+          include_tasks: extract_text_from_screenshot.yml
+          vars:
+            local_screenshot_path: "{{ vm_screenshot_local_path }}"
+
+        - name: "Display extracted text from screenshot"
+          ansible.builtin.debug:
+            msg: "{{ text_in_screenshot }}"
+          tags:
+            - fail_message
+          when: text_in_screenshot | length > 0
 
     - name: "Download VM's vmware.log"
       include_tasks: esxi_download_datastore_file.yml

common/vm_add_serial_port.yml

@@ -3,16 +3,16 @@
 ---
 # Add a serial port to VM by using output file
 # Parameter:
-#   vm_serial_port_file_path: The serial port output file on datastore.
+#   vm_serial_file_ds_path: The VM's serial port output file on datastore.
 # Return:
-#   vm_serial_port_output: The absolute path of the serial port output file
+#   vm_serial_file_abs_path: The absolute path of VM serial port output file
 #
-- name: "Set default serial port output file"
+- name: "Set default serial port output file for VM"
   ansible.builtin.set_fact:
-    vm_serial_port_file_path: "{{ vm_files_path_ds.strip('\\/') }}/serial-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}.log"
-  when: vm_serial_port_file_path is undefined or not vm_serial_port_file_path
+    vm_serial_file_ds_path: "{{ vm_files_path_ds.strip('\\/') }}/serial-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}.log"
+  when: vm_serial_file_ds_path is undefined or not vm_serial_file_ds_path
 
-- name: "Add a serial port using output file"
+- name: "Add a serial port using output file to VM"
   community.vmware.vmware_guest_serial_port:
     hostname: "{{ vsphere_host_name }}"
     username: "{{ vsphere_host_user }}"
@@ -21,12 +21,17 @@
     name: "{{ vm_name }}"
     backings:
       - type: "file"
-        file_path: "{{ vm_serial_port_file_path }}"
+        file_path: "{{ vm_serial_file_ds_path }}"
         yield_on_poll: true
 
-- name: "Set the absolute path of the serial port output file"
+- name: "Set facts of VM serial port output file's name and absolute path"
   ansible.builtin.set_fact:
-    vm_serial_port_output_file: "{{ vm_serial_port_file_path | replace('[', '/vmfs/volumes/') | replace('] ', '/') }}"
+    vm_serial_file_name: "{{ vm_serial_file_ds_path | basename }}"
+    vm_serial_file_abs_path: "{{ vm_serial_file_ds_path | replace('[', '/vmfs/volumes/') | replace('] ', '/') }}"
 
-- ansible.builtin.debug:
-    msg: "The VM serial port is using output file: {{ vm_serial_port_output_file }}"
+- name: "Display the VM serial port output file info"
+  ansible.builtin.debug:
+    msg:
+      - "The VM serial port output file's name: {{ vm_serial_file_name }}"
+      - "The VM serial port output file's absolute path: {{ vm_serial_file_abs_path }}"
+      - "The VM serial port output file's datastore path: {{ vm_serial_file_ds_path }}"

common/vm_get_power_state.yml

@@ -2,14 +2,20 @@
 # SPDX-License-Identifier: BSD-2-Clause
 ---
 # Get VM power state from guest facts
-- include_tasks: vm_get_config.yml
+#
+- name: "Initialize fact of VM power state"
+  ansible.builtin.set_fact:
+    vm_power_state_get: ""
+
+- name: "Get VM power state"
+  include_tasks: vm_get_config.yml
   vars:
     property_list: ['runtime.powerState']
 
-- name: Set fact of VM power state
+- name: "Set fact of VM power state"
   ansible.builtin.set_fact:
     vm_power_state_get: "{{ vm_config.runtime.powerState }}"
 
-- name: Display VM power state
+- name: "Display VM power state"
   ansible.builtin.debug:
     msg: "Get VM power state: {{ vm_power_state_get }}"

common/vm_guest_send_key.yml

@@ -2,15 +2,17 @@
 # SPDX-License-Identifier: BSD-2-Clause
 ---
 # Parameters:
-## keys_send: a list of keys to be sent to guest
-## string_send: a string to be sent to guest
-- name: Assert keys or string to be sent is defined
+#   keys_send: a list of keys to be sent to guest OS
+#   string_send: a string to be sent to guest OS
+#   vm_send_key_ignore_errors: Whether to ignore erros. Default is false.
+#
+- name: "Assert keys or string to be sent is defined"
   ansible.builtin.assert:
     that:
       - keys_send is defined or string_send is defined
     fail_msg: "Either keys_send or string_send shall be defined"
 
-- name: "Send keys to {{ vm_name }}"
+- name: "Send keys or string to {{ vm_name }}"
   community.vmware.vmware_guest_sendkey:
     hostname: "{{ vsphere_host_name }}"
     username: "{{ vsphere_host_user }}"
@@ -22,12 +24,19 @@
     keys_send: "{{ keys_send }}"
     string_send: "{{ string_send | default('') }}"
   register: vm_sendkey
+  ignore_errors: true
 
-- ansible.builtin.debug: var=vm_sendkey
-  when: enable_debug is defined and enable_debug
+- name: "Display the result of sending keys or string to guest OS"
+  ansible.builtin.debug: var=vm_sendkey
+  when: enable_debug
 
-- name: "Verify send keys to VM {{ vm_name }} operation succeed"
+- name: "Check sending keys or string to guest OS succeeds"
   ansible.builtin.assert:
     that:
-      - "not vm_sendkey.failed"
-      - "vm_sendkey.changed"
+      - not vm_sendkey.failed
+      - vm_sendkey.changed
+    fail_msg: "Failed to send keys or string to guest OS"
+    success_msg: "Successfully send keys or string to guest OS"
+  when: >-
+    (vm_send_key_ignore_errors is undefined or
+     not vm_send_key_ignore_errors)

common/vm_remove_serial_port.yml

@@ -1,15 +1,15 @@
 # Copyright 2021-2023 VMware, Inc.
 # SPDX-License-Identifier: BSD-2-Clause
 ---
-# Remove a serial port to VM by using output file
+# Remove a serial port using output file from VM
 # Parameter:
-#   vm_serial_port_file_path: The serial port output file on datastore.
+#   vm_serial_file_ds_path: The serial port output file's datastore path.
 #
 - name: "Remove the serial port output file before removing serial port"
   include_tasks: esxi_check_delete_datastore_file.yml
   vars:
     file_in_datastore: "{{ datastore }}"
-    file_in_datastore_path: "{{ vm_serial_port_file_path.split(']')[-1].strip(' ') }}"
+    file_in_datastore_path: "{{ vm_serial_file_ds_path.split(']')[-1].strip(' ') }}"
     file_in_datastore_ops: "absent"
     file_in_datastore_failed_ignore: true
 
@@ -22,7 +22,7 @@
     name: "{{ vm_name }}"
     backings:
       - type: "file"
-        file_path: "{{ vm_serial_port_file_path }}"
+        file_path: "{{ vm_serial_file_ds_path }}"
         state: absent
   register: remove_serial_port
 
@@ -36,6 +36,8 @@
       - remove_serial_port.changed
     fail_msg: "Failed to remove serial port from VM"
 
-- name: "Clean serial port output file path on datastore"
+- name: "Clean facts of VM serial port output file"
   ansible.builtin.set_fact:
-    vm_serial_port_file_path: ""
+    vm_serial_file_ds_path: ""
+    vm_serial_file_abs_path: ""
+    vm_serial_file_name: ""

common/vm_set_power_state.yml

@@ -8,7 +8,7 @@
 #   vm_set_power_state_sleep (optional): wait time in seconds when VM power state
 #     changed. Default value is 10.
 #
-- name: Set fact of valid power state list
+- name: "Set fact of valid power state list"
   ansible.builtin.set_fact:
     valid_expected_power_state:
       - powered-off
@@ -18,15 +18,18 @@
       - shutdown-guest
       - suspended
       - present
-- name: Check required parameter
+
+- name: "Check required parameter"
   ansible.builtin.assert:
     that:
       - vm_power_state_set is defined
       - vm_power_state_set | lower in valid_expected_power_state
-    fail_msg: "vm_power_state_set is undefined or invalid, valid value is in: {{ valid_expected_power_state }}"
+    fail_msg: >-
+      vm_power_state_set is undefined or invalid, valid value is in: {{ valid_expected_power_state }}
 
 # Make sure VMware tools is installed and running before shutdown VM
-- include_tasks: vm_wait_vmtools_status.yml
+- name: "Wait for VMware Tools running"
+  include_tasks: vm_wait_vmtools_status.yml
   vars:
     vm_wait_vmtools_running: true
   when: vm_power_state_set | lower == 'shutdown-guest' or vm_power_state_set | lower == 'reboot-guest'
@@ -46,7 +49,8 @@
 
 - name: "Replace serial port output file at power on"
   block:
-    - include_tasks: vm_answer_question.yml
+    - name: "Answer question about overwritting serial port file"
+      include_tasks: vm_answer_question.yml
       vars:
         vm_question: "{{ vm_change_power_state.instance.guest_question }}"
         vm_question_response: "button.serial.file.overwrite"
@@ -69,13 +73,13 @@
 - name: "Change VM power state failure"
   ansible.builtin.fail:
     msg: >
-      {% if vm_change_power_state is defined and vm_change_power_state.msg is defined %}{{ vm_change_power_state.msg }}
-      {% else %}Failed to set VM power state to {{ vm_power_state_set | lower }}{% endif %}
+      Failed to set VM power state to {{ vm_power_state_set | lower }} because of:
+      {{ vm_change_power_state.msg | default('') }}
   when: >
     vm_change_power_state is undefined or
     vm_change_power_state.failed is defined and vm_change_power_state.failed | bool
 
-- name: Display the result of changing VM power state
+- name: "Display the result of changing VM power state"
   ansible.builtin.debug: var=vm_change_power_state
 
 - name: "Wait for VM enter into the expected state"