Extract text from screenshot at test failure and errors from VM logs

README.md

@@ -4,15 +4,16 @@
 
 ### Prerequisites
 1. Install Ansible on your control machine, please refer to [Installing Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html)
-2. Install required Python libraries in requirements.txt
+2. Install Tesseract Open Source OCR Engine for extracting text from screenshots, please refer to [Installing Tesseract](https://github.com/tesseract-ocr/tesseract?tab=readme-ov-file#installing-tesseract)
+3. Install required Python libraries in requirements.txt
 ```
 $ pip install -r requirements.txt
 ```
-3. Install required Ansible collections with latest version in requirements.yml
+4. Install required Ansible collections with latest version in requirements.yml
 ```
 $ ansible-galaxy install -r requirements.yml
 ```
-4. Log in to local control machine as root or a user in sudoers, which must enable NOPASSWD for all commands
+5. Log in to local control machine as root or a user in sudoers, which must enable NOPASSWD for all commands
 
 ### Steps to Launch Testing
 1. Git clone project from github to your workspace on control machine.

autoinstall/Ubuntu/Desktop/Ubiquity/ubuntu.seed

@@ -63,7 +63,7 @@ d-i grub-installer/only_debian boolean true
 # preseeding is read.
 d-i preseed/early_command string \
     echo "Executing early command" >/dev/ttyS0; \
-    cp -a /cdrom/preseed/{{ pre_install_script_file }} /root/{{ pre_install_script_file }}; \
+    cp -a /root/cdrom/preseed/{{ pre_install_script_file }} /root/{{ pre_install_script_file }}; \
     /bin/sh /root/{{ pre_install_script_file }} >/dev/ttyS0;
 
 # Execute post install script on success

common/README.md

@@ -191,3 +191,5 @@
 * compose_vm_cdroms.yml: Generate VM CDROM device info list for creating new VM
 * download_iso_and_transfer.yml: Download ISO file and transfer to ESXi datastore
 * get_iso_file_list.yml: Generate and validate OS installation ISO file list
+* extract_errors_from_log.yml: Extract error messages in a log file downloaded from guest OS
+* extract_text_from_screenshot.yml: Extract text from an image file

common/esxi_download_datastore_file.yml

@@ -9,14 +9,22 @@
 #   download_file_timeout: timeout in seconds for downloading datastore file. Default is 300s
 #   download_file_fail_ignore: whether ignore errors or not in this task, default is false
 #
-- include_tasks: esxi_check_delete_datastore_file.yml
+- name: "Initialize the fact of downloading datastore file success status"
+  ansible.builtin.set_fact:
+    datastore_file_download_success: false
+
+- name: "Check file exists in ESXi datastore"
+  include_tasks: esxi_check_delete_datastore_file.yml
   vars:
     file_in_datastore_ops: file
     file_in_datastore: "{{ src_datastore }}"
     file_in_datastore_path: "{{ src_file_path }}"
     file_in_datastore_failed_ignore: "{{ download_file_fail_ignore | default(false) }}"
 
-- name: "File in datastore"
+- name: "File exists in ESXi datastore"
+  when:
+    - file_in_datastore_result is defined
+    - file_in_datastore_result == 'Success'
   block:
     - name: "VM datastore file URL get failure"
       ansible.builtin.fail:
@@ -28,11 +36,15 @@
            (ds_file_result.url is undefined) or
            (not ds_file_result.url))
 
-    - name: "Download File from URL"
+    - name: "Download ESXi datastore file from URL"
+      when:
+        - ds_file_result.url is defined
+        - ds_file_result.url
       block:
         - name: "Set fact of downloading URL for the src datastore file"
           ansible.builtin.set_fact:
             datastore_file_url: "{{ ds_file_result.url }}"
+
         - name: "Download datastore file"
           ansible.builtin.get_url:
             url: "{{ datastore_file_url }}"
@@ -44,12 +56,14 @@
             timeout: "{{ download_file_timeout | default(300) }}"
           ignore_errors: "{{ download_file_fail_ignore | default(false) }}"
           register: datastore_file_download_result
+
         - name: "Print datastore file download result"
           ansible.builtin.debug: var=datastore_file_download_result
           when: enable_debug
-      when:
-        - ds_file_result.url is defined
-        - ds_file_result.url
-  when:
-    - file_in_datastore_result is defined
-    - file_in_datastore_result == 'Success'
+
+        - name: "Set fact of downloading datastore file success status"
+          ansible.builtin.set_fact:
+            datastore_file_download_success: true
+          when:
+            - datastore_file_download_result.status_code is defined
+            - datastore_file_download_result.status_code | int == 200

common/extract_errors_from_log.yml

@@ -0,0 +1,37 @@
+# Copyright 2023 VMware, Inc.
+# SPDX-License-Identifier: BSD-2-Clause
+---
+# Extract call trace or other error messages in a log file downloaded from guest OS
+# Parameters:
+#   extract_log_file_path: The local log file path to extract call trace and errors
+# Return:
+#   errors_in_log: The call trace or error messages extracted from the log file
+#
+- name: "Check the local log path is an absolute path"
+  ansible.builtin.assert:
+    that:
+      - extract_log_file_path is defined
+      - extract_log_file_path
+      - extract_log_file_path is ansible.builtin.abs
+    fail_msg: "Parameter 'extract_log_file_path' must be set with a local absolute path"
+
+- name: "Initialize the fact of extracted errors from log"
+  ansible.builtin.set_fact:
+    errors_in_log: []
+
+- name: "Extract error messages from log file"
+  ansible.builtin.script: "../tools/extractor.py -t error -f {{ extract_log_file_path }}"
+  ignore_errors: true
+  register: extract_error_result
+
+- name: "Set fact of extracted errors from log file"
+  ansible.builtin.set_fact:
+    errors_in_log: "{{ extract_error_result.stdout_lines | select }}"
+  when:
+    - not extract_error_result.failed
+    - extract_error_result.stdout_lines is defined
+    - extract_error_result.stdout_lines | length > 0
+
+- name: "Display extracted errors from log file"
+  ansible.builtin.debug:
+    msg: "{{ errors_in_log }}"

common/extract_text_from_screenshot.yml

@@ -0,0 +1,37 @@
+# Copyright 2023 VMware, Inc.
+# SPDX-License-Identifier: BSD-2-Clause
+---
+# Extract text from a local screenshot file
+# Parameters:
+#   local_screenshot_path: The local screenshot file path
+# Return:
+#   text_in_screenshot: The text extracted from the screenshot file
+#
+- name: "Check local screenshot file path is an absolute path"
+  ansible.builtin.assert:
+    that:
+      - local_screenshot_path is defined
+      - local_screenshot_path
+      - local_screenshot_path is ansible.builtin.abs
+    fail_msg: "Parameter 'local_screenshot_path' must be set with a local absolute path"
+
+- name: "Initialize the fact of extracted screenshot text"
+  ansible.builtin.set_fact:
+    text_in_screenshot: []
+
+- name: "Extract text from local screenshot file"
+  ansible.builtin.script: "../tools/extractor.py -t text -f {{ local_screenshot_path }}"
+  ignore_errors: true
+  register: extract_text_result
+
+- name: "Set fact of extracted screenshot text"
+  ansible.builtin.set_fact:
+    text_in_screenshot: "{{ extract_text_result.stdout_lines | select }}"
+  when:
+    - not extract_text_result.failed
+    - extract_text_result.stdout_lines is defined
+    - extract_text_result.stdout_lines | length > 0
+
+- name: "Display extracted text from the screenshot"
+  ansible.builtin.debug:
+    msg: "{{ text_in_screenshot }}"

common/test_rescue.yml

@@ -43,7 +43,26 @@
     - name: "Take a screenshot at VM current state"
       include_tasks: vm_take_screenshot.yml
       vars:
-        vm_take_screenshot_local_path: "{{ current_test_log_folder }}"
+        vm_screenshot_local_dir: "{{ current_test_log_folder }}"
+        vm_screenshot_local_name: "screenshot_at_{{ current_testcase_index }}_{{ ansible_play_name }}.png"
+        vm_screen_active: "{{ gosv_test_suite == 'linux' }}"
+
+    - name: "Extract text from VM screenshot at failed test case {{ current_testcase_name }}"
+      when:
+        - tesseract_is_installed
+        - vm_screenshot_local_path
+      block:
+        - name: "Extract text from screenshot file"
+          include_tasks: extract_text_from_screenshot.yml
+          vars:
+            local_screenshot_path: "{{ vm_screenshot_local_path }}"
+
+        - name: "Display extracted text from screenshot"
+          ansible.builtin.debug:
+            msg: "{{ text_in_screenshot }}"
+          tags:
+            - fail_message
+          when: text_in_screenshot | length > 0
 
     - name: "Download VM's vmware.log"
       include_tasks: esxi_download_datastore_file.yml
@@ -54,6 +73,22 @@
         download_file_fail_ignore: true
       when: vm_dir_name is defined and vm_dir_name
 
+    - name: "Check Windows guest BSOD in vmware.log"
+      when:
+        - gosv_test_suite == 'windows'
+        - datastore_file_download_success
+      block:
+        - name: "Look up Windows guest BSOD in vmware.log"
+          ansible.builtin.set_fact:
+            winbsod_in_vmware_log: "{{ lookup('file', datastore_file_download_result.dest) | regex_findall('.*WinBSOD:.*') }}"
+
+        - name: "Detected Windows guest BSOD in vmware.log"
+          ansible.builtin.debug:
+            msg: "{{ winbsod_in_vmware_log }}"
+          tags:
+            - fail_message
+          when: winbsod_in_vmware_log | length > 0
+
     - name: "Take a snapshot at VM current state"
       include_tasks: vm_take_snapshot.yml
       vars:

common/vm_add_serial_port.yml

@@ -3,16 +3,16 @@
 ---
 # Add a serial port to VM by using output file
 # Parameter:
-#   vm_serial_port_file_path: The serial port output file on datastore.
+#   vm_serial_file_ds_path: The VM's serial port output file on datastore.
 # Return:
-#   vm_serial_port_output: The absolute path of the serial port output file
+#   vm_serial_file_abs_path: The absolute path of VM serial port output file
 #
-- name: "Set default serial port output file"
+- name: "Set default serial port output file for VM"
   ansible.builtin.set_fact:
-    vm_serial_port_file_path: "{{ vm_files_path_ds.strip('\\/') }}/serial-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}.log"
-  when: vm_serial_port_file_path is undefined or not vm_serial_port_file_path
+    vm_serial_file_ds_path: "{{ vm_files_path_ds.strip('\\/') }}/serial-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}.log"
+  when: vm_serial_file_ds_path is undefined or not vm_serial_file_ds_path
 
-- name: "Add a serial port using output file"
+- name: "Add a serial port using output file to VM"
   community.vmware.vmware_guest_serial_port:
     hostname: "{{ vsphere_host_name }}"
     username: "{{ vsphere_host_user }}"
@@ -21,12 +21,17 @@
     name: "{{ vm_name }}"
     backings:
       - type: "file"
-        file_path: "{{ vm_serial_port_file_path }}"
+        file_path: "{{ vm_serial_file_ds_path }}"
         yield_on_poll: true
 
-- name: "Set the absolute path of the serial port output file"
+- name: "Set facts of VM serial port output file's name and absolute path"
   ansible.builtin.set_fact:
-    vm_serial_port_output_file: "{{ vm_serial_port_file_path | replace('[', '/vmfs/volumes/') | replace('] ', '/') }}"
+    vm_serial_file_name: "{{ vm_serial_file_ds_path | basename }}"
+    vm_serial_file_abs_path: "{{ vm_serial_file_ds_path | replace('[', '/vmfs/volumes/') | replace('] ', '/') }}"
 
-- ansible.builtin.debug:
-    msg: "The VM serial port is using output file: {{ vm_serial_port_output_file }}"
+- name: "Display the VM serial port output file info"
+  ansible.builtin.debug:
+    msg:
+      - "The VM serial port output file's name: {{ vm_serial_file_name }}"
+      - "The VM serial port output file's absolute path: {{ vm_serial_file_abs_path }}"
+      - "The VM serial port output file's datastore path: {{ vm_serial_file_ds_path }}"

common/vm_get_power_state.yml

@@ -2,14 +2,20 @@
 # SPDX-License-Identifier: BSD-2-Clause
 ---
 # Get VM power state from guest facts
-- include_tasks: vm_get_config.yml
+#
+- name: "Initialize fact of VM power state"
+  ansible.builtin.set_fact:
+    vm_power_state_get: ""
+
+- name: "Get VM power state"
+  include_tasks: vm_get_config.yml
   vars:
     property_list: ['runtime.powerState']
 
-- name: Set fact of VM power state
+- name: "Set fact of VM power state"
   ansible.builtin.set_fact:
     vm_power_state_get: "{{ vm_config.runtime.powerState }}"
 
-- name: Display VM power state
+- name: "Display VM power state"
   ansible.builtin.debug:
     msg: "Get VM power state: {{ vm_power_state_get }}"

common/vm_guest_send_key.yml

@@ -2,15 +2,18 @@
 # SPDX-License-Identifier: BSD-2-Clause
 ---
 # Parameters:
-## keys_send: a list of keys to be sent to guest
-## string_send: a string to be sent to guest
-- name: Assert keys or string to be sent is defined
+#   keys_send: a list of keys to be sent to guest OS
+#   string_send: a string to be sent to guest OS
+#   vm_send_key_ignore_errors: whether to ignore errors when sending keys.
+#     Default is false.
+#
+- name: "Assert keys or string to be sent is defined"
   ansible.builtin.assert:
     that:
       - keys_send is defined or string_send is defined
     fail_msg: "Either keys_send or string_send shall be defined"
 
-- name: "Send keys to {{ vm_name }}"
+- name: "Send keys or string to {{ vm_name }}"
   community.vmware.vmware_guest_sendkey:
     hostname: "{{ vsphere_host_name }}"
     username: "{{ vsphere_host_user }}"
@@ -22,12 +25,19 @@
     keys_send: "{{ keys_send }}"
     string_send: "{{ string_send | default('') }}"
   register: vm_sendkey
+  ignore_errors: true
 
-- ansible.builtin.debug: var=vm_sendkey
-  when: enable_debug is defined and enable_debug
+- name: "Display the result of sending keys or string to guest OS"
+  ansible.builtin.debug: var=vm_sendkey
+  when: enable_debug
 
-- name: "Verify send keys to VM {{ vm_name }} operation succeed"
+- name: "Check sending keys or string to guest OS succeeds"
   ansible.builtin.assert:
     that:
-      - "not vm_sendkey.failed"
-      - "vm_sendkey.changed"
+      - not vm_sendkey.failed
+      - vm_sendkey.changed
+    fail_msg: "Failed to send keys or string to guest OS"
+    success_msg: "Successfully send keys or string to guest OS"
+  when: >-
+    (vm_send_key_ignore_errors is undefined or
+     not vm_send_key_ignore_errors)

common/vm_remove_serial_port.yml

@@ -1,15 +1,15 @@
 # Copyright 2021-2023 VMware, Inc.
 # SPDX-License-Identifier: BSD-2-Clause
 ---
-# Remove a serial port to VM by using output file
+# Remove a serial port using output file from VM
 # Parameter:
-#   vm_serial_port_file_path: The serial port output file on datastore.
+#   vm_serial_file_ds_path: The serial port output file's datastore path.
 #
 - name: "Remove the serial port output file before removing serial port"
   include_tasks: esxi_check_delete_datastore_file.yml
   vars:
     file_in_datastore: "{{ datastore }}"
-    file_in_datastore_path: "{{ vm_serial_port_file_path.split(']')[-1].strip(' ') }}"
+    file_in_datastore_path: "{{ vm_serial_file_ds_path.split(']')[-1].strip(' ') }}"
     file_in_datastore_ops: "absent"
     file_in_datastore_failed_ignore: true
 
@@ -22,7 +22,7 @@
     name: "{{ vm_name }}"
     backings:
       - type: "file"
-        file_path: "{{ vm_serial_port_file_path }}"
+        file_path: "{{ vm_serial_file_ds_path }}"
         state: absent
   register: remove_serial_port
 
@@ -36,6 +36,8 @@
       - remove_serial_port.changed
     fail_msg: "Failed to remove serial port from VM"
 
-- name: "Clean serial port output file path on datastore"
+- name: "Clean facts of VM serial port output file"
   ansible.builtin.set_fact:
-    vm_serial_port_file_path: ""
+    vm_serial_file_ds_path: ""
+    vm_serial_file_abs_path: ""
+    vm_serial_file_name: ""