From 0e7b81c7684d6a1f4502a2b9ee8b2fe67a91a01a Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Wed, 17 Jul 2024 17:34:56 +0200 Subject: [PATCH 1/7] Use --progress json for Compose 2.29.0+. --- plugins/module_utils/compose_v2.py | 108 ++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 11 deletions(-) diff --git a/plugins/module_utils/compose_v2.py b/plugins/module_utils/compose_v2.py index 0c7e79b03..f1da8f5ba 100644 --- a/plugins/module_utils/compose_v2.py +++ b/plugins/module_utils/compose_v2.py @@ -7,6 +7,7 @@ __metaclass__ = type +import json import os import re import shutil @@ -348,6 +349,69 @@ def _concat_event_msg(event, append_msg): ) +def parse_json_events(stderr, warn_function=None): + events = [] + stderr_lines = stderr.splitlines() + if stderr_lines and stderr_lines[-1] == b'': + del stderr_lines[-1] + for line in stderr_lines: + line = line.strip() + if not line.startswith(b'{') or not line.endswith(b'}'): + continue + try: + line_data = json.loads(line) + except Exception as exc: + if warn_function: + warn_function( + 'Cannot parse event from line: {0!r}: {1}. Please report this at ' + 'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md' + .format(line, exc) + ) + continue + if line_data.get('error'): + resource_type = ResourceType.UNKNOWN + event = Event( + resource_type, + line_data.get('id'), + 'Error', + line_data.get('message'), + ) + else: + resource_type = ResourceType.UNKNOWN + resource_id = line_data.get('id') + status = line_data.get('status') + text = line_data.get('text') + if isinstance(resource_id, str) and ' ' in resource_id: + resource_type_str, resource_id = resource_id.split(' ', 1) + try: + resource_type = ResourceType.from_docker_compose_event(resource_type_str) + except KeyError: + if warn_function: + warn_function( + 'Unknown resource type {0!r}. Please report this at ' + 'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md' + .format(resource_type_str) + ) + resource_type = ResourceType.UNKNOWN + elif text in DOCKER_STATUS_PULL: + resource_type = ResourceType.IMAGE + status, text = text, None + elif text in DOCKER_PULL_PROGRESS_DONE or line_data.get('text') in DOCKER_PULL_PROGRESS_WORKING: + resource_type = ResourceType.IMAGE_LAYER + status, text = text, None + if status not in DOCKER_STATUS and text in DOCKER_STATUS: + status, text = text, status + event = Event( + resource_type, + resource_id, + status, + text, + ) + + events.append(event) + return events + + def parse_events(stderr, dry_run=False, warn_function=None): events = [] error_event = None @@ -476,11 +540,15 @@ def update_failed(result, events, args, stdout, stderr, rc, cli): errors = [] for event in events: if event.status in DOCKER_STATUS_ERROR: - msg = 'Error when processing {resource_type} {resource_id}: ' - if event.resource_type == 'unknown': - msg = 'Error when processing {resource_id}: ' - if event.resource_id == '': - msg = 'General error: ' + if event.resource_id is None: + if event.resource_type == 'unknown': + msg = 'General error: ' if event.resource_type == 'unknown' else 'Error when processing {resource_type}: ' + else: + msg = 'Error when processing {resource_type} {resource_id}: ' + if event.resource_type == 'unknown': + msg = 'Error when processing {resource_id}: ' + if event.resource_id == '': + msg = 'General error: ' msg += '{status}' if event.msg is None else '{msg}' errors.append(msg.format( resource_type=event.resource_type, @@ -598,13 +666,19 @@ def __init__(self, client, min_version=MINIMUM_COMPOSE_VERSION): filenames = ', '.join(DOCKER_COMPOSE_FILES[:-1]) self.fail('"{0}" does not contain {1}, or {2}'.format(self.project_src, filenames, DOCKER_COMPOSE_FILES[-1])) + # Support for JSON output was added in Compose 2.29.0 (https://github.com/docker/compose/releases/tag/v2.29.0); + # more precisely in https://github.com/docker/compose/pull/11478 + self.use_json_events = self.compose_version >= LooseVersion('2.29.0') + def fail(self, msg, **kwargs): self.cleanup() self.client.fail(msg, **kwargs) def get_base_args(self): args = ['compose', '--ansi', 'never'] - if self.compose_version >= LooseVersion('2.19.0'): + if self.use_json_events: + args.extend(['--progress', 'json']) + elif self.compose_version >= LooseVersion('2.19.0'): # https://github.com/docker/compose/pull/10690 args.extend(['--progress', 'plain']) args.extend(['--project-directory', self.project_src]) @@ -618,17 +692,25 @@ def get_base_args(self): args.extend(['--profile', profile]) return args + def _handle_failed_cli_call(self, args, rc, stdout, stderr): + events = parse_json_events(stderr, warn_function=self.client.warn) + result = {} + self.update_failed(result, events, args, stdout, stderr, rc) + self.client.module.exit_json(**result) + def list_containers_raw(self): args = self.get_base_args() + ['ps', '--format', 'json', '--all'] if self.compose_version >= LooseVersion('2.23.0'): # https://github.com/docker/compose/pull/11038 args.append('--no-trunc') - kwargs = dict(cwd=self.project_src, check_rc=True) + kwargs = dict(cwd=self.project_src, check_rc=not self.use_json_events) if self.compose_version >= LooseVersion('2.21.0'): # Breaking change in 2.21.0: https://github.com/docker/compose/pull/10918 - dummy, containers, dummy = self.client.call_cli_json_stream(*args, **kwargs) + rc, containers, stderr = self.client.call_cli_json_stream(*args, **kwargs) else: - dummy, containers, dummy = self.client.call_cli_json(*args, **kwargs) + rc, containers, stderr = self.client.call_cli_json(*args, **kwargs) + if self.use_json_events and rc != 0: + self._handle_failed_cli_call(args, rc, containers, stderr) return containers def list_containers(self): @@ -648,11 +730,15 @@ def list_containers(self): def list_images(self): args = self.get_base_args() + ['images', '--format', 'json'] - kwargs = dict(cwd=self.project_src, check_rc=True) - dummy, images, dummy = self.client.call_cli_json(*args, **kwargs) + kwargs = dict(cwd=self.project_src, check_rc=not self.use_json_events) + rc, images, stderr = self.client.call_cli_json(*args, **kwargs) + if self.use_json_events and rc != 0: + self._handle_failed_cli_call(args, rc, images, stderr) return images def parse_events(self, stderr, dry_run=False): + if self.use_json_events: + return parse_json_events(stderr, warn_function=self.client.warn) return parse_events(stderr, dry_run=dry_run, warn_function=self.client.warn) def emit_warnings(self, events): From af1cb0eb9e4d642589e7647c8129cd631c5ee38d Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Wed, 17 Jul 2024 20:28:53 +0200 Subject: [PATCH 2/7] Add changelog fragment. --- changelogs/fragments/931-compose-2.29.0-json.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/931-compose-2.29.0-json.yml diff --git a/changelogs/fragments/931-compose-2.29.0-json.yml b/changelogs/fragments/931-compose-2.29.0-json.yml new file mode 100644 index 000000000..248b5cea0 --- /dev/null +++ b/changelogs/fragments/931-compose-2.29.0-json.yml @@ -0,0 +1,2 @@ +minor_changes: + - "docker_compose_v2* modules - support Docker Compose 2.29.0's ``json`` progress writer to avoid having to parse text output (https://github.com/ansible-collections/community.docker/pull/931)." From ec36988c29555dbac041a88786d31550b24926c5 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 20 Jul 2024 15:54:20 +0200 Subject: [PATCH 3/7] Fix/improve handling of warnings. --- plugins/module_utils/compose_v2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/module_utils/compose_v2.py b/plugins/module_utils/compose_v2.py index f1da8f5ba..28d44e5db 100644 --- a/plugins/module_utils/compose_v2.py +++ b/plugins/module_utils/compose_v2.py @@ -85,6 +85,7 @@ 'Waiting', )) DOCKER_STATUS = frozenset(DOCKER_STATUS_DONE | DOCKER_STATUS_WORKING | DOCKER_STATUS_PULL | DOCKER_STATUS_ERROR | DOCKER_STATUS_WAITING) +DOCKER_STATUS_AND_WARNING = frozenset(DOCKER_STATUS | DOCKER_STATUS_WARNING) DOCKER_PULL_PROGRESS_DONE = frozenset(( 'Already exists', @@ -399,7 +400,7 @@ def parse_json_events(stderr, warn_function=None): elif text in DOCKER_PULL_PROGRESS_DONE or line_data.get('text') in DOCKER_PULL_PROGRESS_WORKING: resource_type = ResourceType.IMAGE_LAYER status, text = text, None - if status not in DOCKER_STATUS and text in DOCKER_STATUS: + if status not in DOCKER_STATUS_AND_WARNING and text in DOCKER_STATUS_AND_WARNING: status, text = text, status event = Event( resource_type, @@ -520,7 +521,7 @@ def extract_actions(events): def emit_warnings(events, warn_function): for event in events: # If a message is present, assume it is a warning - if event.status is None and event.msg is not None: + if (event.status is None and event.msg is not None) or event.status in DOCKER_STATUS_WARNING: warn_function('Docker compose: {resource_type} {resource_id}: {msg}'.format( resource_type=event.resource_type, resource_id=event.resource_id, From 1da01f37ac7ac116916449f24e049480574f42f9 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 20 Jul 2024 16:45:19 +0200 Subject: [PATCH 4/7] Improve parsing of warnings and some one-off messages. --- plugins/module_utils/compose_v2.py | 32 ++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/plugins/module_utils/compose_v2.py b/plugins/module_utils/compose_v2.py index 28d44e5db..5afde4c84 100644 --- a/plugins/module_utils/compose_v2.py +++ b/plugins/module_utils/compose_v2.py @@ -17,6 +17,7 @@ from ansible.module_utils.basic import missing_required_lib from ansible.module_utils.common.text.converters import to_native +from ansible.module_utils.six import string_types from ansible.module_utils.six.moves import shlex_quote from ansible_collections.community.docker.plugins.module_utils.util import DockerBaseClass @@ -350,6 +351,12 @@ def _concat_event_msg(event, append_msg): ) +_JSON_LEVEL_TO_STATUS_MAP = { + 'warning': 'Warning', + 'error': 'Error', +} + + def parse_json_events(stderr, warn_function=None): events = [] stderr_lines = stderr.splitlines() @@ -358,6 +365,22 @@ def parse_json_events(stderr, warn_function=None): for line in stderr_lines: line = line.strip() if not line.startswith(b'{') or not line.endswith(b'}'): + if line.startswith(b'Warning: '): + # This is a bug in Compose that will get fixed by https://github.com/docker/compose/pull/11996 + event = Event( + ResourceType.UNKNOWN, + None, + 'Warning', + to_native(line[len(b'Warning: '):]), + ) + events.append(event) + continue + if warn_function: + warn_function( + 'Found non-JSON line: {0!r}. Please report this at ' + 'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md' + .format(line) + ) continue try: line_data = json.loads(line) @@ -396,10 +419,15 @@ def parse_json_events(stderr, warn_function=None): resource_type = ResourceType.UNKNOWN elif text in DOCKER_STATUS_PULL: resource_type = ResourceType.IMAGE - status, text = text, None + status, text = text, status elif text in DOCKER_PULL_PROGRESS_DONE or line_data.get('text') in DOCKER_PULL_PROGRESS_WORKING: resource_type = ResourceType.IMAGE_LAYER - status, text = text, None + status, text = text, status + elif status is None and isinstance(text, string_types) and text.startswith('Skipped - '): + status, text = text.split(' - ', 1) + elif line_data.get('level') in _JSON_LEVEL_TO_STATUS_MAP and 'msg' in line_data: + status = _JSON_LEVEL_TO_STATUS_MAP[line_data['level']] + text = line_data['msg'] if status not in DOCKER_STATUS_AND_WARNING and text in DOCKER_STATUS_AND_WARNING: status, text = text, status event = Event( From 969e0a0d9a77be7fc9f2f2e445bae43f3eb9319b Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 20 Jul 2024 16:51:50 +0200 Subject: [PATCH 5/7] Improve warnings. --- plugins/module_utils/compose_v2.py | 6 +-- .../tasks/tests/definition.yml | 48 +++++++++---------- .../docker_compose_v2/tasks/tests/pull.yml | 28 +++++------ .../tasks/tests/start-stop.yml | 48 +++++++++---------- .../tasks/tests/pull.yml | 28 +++++------ 5 files changed, 79 insertions(+), 79 deletions(-) diff --git a/plugins/module_utils/compose_v2.py b/plugins/module_utils/compose_v2.py index 5afde4c84..bcb27637e 100644 --- a/plugins/module_utils/compose_v2.py +++ b/plugins/module_utils/compose_v2.py @@ -377,7 +377,7 @@ def parse_json_events(stderr, warn_function=None): continue if warn_function: warn_function( - 'Found non-JSON line: {0!r}. Please report this at ' + 'Cannot parse event from non-JSON line: {0!r}. Please report this at ' 'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md' .format(line) ) @@ -412,9 +412,9 @@ def parse_json_events(stderr, warn_function=None): except KeyError: if warn_function: warn_function( - 'Unknown resource type {0!r}. Please report this at ' + 'Unknown resource type {0!r} in line {1!r}. Please report this at ' 'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md' - .format(resource_type_str) + .format(resource_type_str, line) ) resource_type = ResourceType.UNKNOWN elif text in DOCKER_STATUS_PULL: diff --git a/tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml b/tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml index 872e6a976..62c85684d 100644 --- a/tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml +++ b/tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml @@ -77,7 +77,7 @@ - assert: that: - present_1_check is changed - - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_1 is changed - present_1.containers | length == 1 - present_1.containers[0].Name == pname ~ '-' ~ cname ~ '-1' @@ -86,15 +86,15 @@ - present_1.images[0].ContainerName == pname ~ '-' ~ cname ~ '-1' - present_1.images[0].Repository == (docker_test_image_alpine | split(':') | first) - present_1.images[0].Tag == (docker_test_image_alpine | split(':') | last) - - present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2_check is not changed - - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2 is not changed - - present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3_check is changed - - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3 is changed - - present_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 #################################################################### ## Absent ########################################################## @@ -133,13 +133,13 @@ - assert: that: - absent_1_check is changed - - absent_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - absent_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - absent_1 is changed - - absent_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - absent_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - absent_2_check is not changed - - absent_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - absent_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - absent_2 is not changed - - absent_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - absent_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 #################################################################### ## Stopping and starting ########################################### @@ -259,30 +259,30 @@ - assert: that: - present_1_check is changed - - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_1 is changed - - present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2_check is not changed - - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2 is not changed - - present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3_check is changed - - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3 is changed - - present_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_4_check is not changed - - present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_4 is not changed - - present_4.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_4.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_5_check is changed - - present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_5 is changed - - present_5.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_5.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_6_check is changed - - present_6_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_6_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_6 is changed - - present_6.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_6.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_7_check is changed - - present_7_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_7_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_7 is changed - - present_7.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_7.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 diff --git a/tests/integration/targets/docker_compose_v2/tasks/tests/pull.yml b/tests/integration/targets/docker_compose_v2/tasks/tests/pull.yml index 3403db4f2..b8bbf86e7 100644 --- a/tests/integration/targets/docker_compose_v2/tasks/tests/pull.yml +++ b/tests/integration/targets/docker_compose_v2/tasks/tests/pull.yml @@ -84,16 +84,16 @@ that: - present_1_check is failed or present_1_check is changed - present_1_check is changed or present_1_check.msg.startswith('General error:') - - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_1 is failed - present_1.msg.startswith('General error:') - - present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2_check is failed - present_2_check.msg.startswith('Error when processing ' ~ cname ~ ':') - - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2 is failed - present_2.msg.startswith('Error when processing ' ~ cname ~ ':') - - present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 #################################################################### ## Regular image ################################################### @@ -194,32 +194,32 @@ - present_1_check is changed - present_1_check.actions | selectattr('status', 'eq', 'Pulling') | first - present_1_check.actions | selectattr('status', 'eq', 'Creating') | first - - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_1 is changed - present_1.actions | selectattr('status', 'eq', 'Pulling') | first - present_1.actions | selectattr('status', 'eq', 'Creating') | first - - present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2_check is not changed - - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2 is not changed - - present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3_check is changed - present_3_check.actions | selectattr('status', 'eq', 'Pulling') | first - present_3_check.actions | selectattr('status', 'eq', 'Creating') | length == 0 - present_3_check.actions | selectattr('status', 'eq', 'Recreating') | length == 0 - - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3 is not changed - present_3.actions | selectattr('status', 'eq', 'Pulling') | first - present_3.actions | selectattr('status', 'eq', 'Creating') | length == 0 - present_3.actions | selectattr('status', 'eq', 'Recreating') | length == 0 - - present_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_4_check is changed - present_4_check.actions | selectattr('status', 'eq', 'Pulling') | length == 0 - - present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_4 is changed - present_4.actions | selectattr('status', 'eq', 'Pulling') | length == 0 - - present_4.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_4.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_5_check is not changed - - present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_5 is not changed - - present_5.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_5.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 diff --git a/tests/integration/targets/docker_compose_v2/tasks/tests/start-stop.yml b/tests/integration/targets/docker_compose_v2/tasks/tests/start-stop.yml index f8b029acd..8b3d3093a 100644 --- a/tests/integration/targets/docker_compose_v2/tasks/tests/start-stop.yml +++ b/tests/integration/targets/docker_compose_v2/tasks/tests/start-stop.yml @@ -89,7 +89,7 @@ - assert: that: - present_1_check is changed - - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_1 is changed - present_1.containers | length == 1 - present_1.containers[0].Name == pname ~ '-' ~ cname ~ '-1' @@ -98,15 +98,15 @@ - present_1.images[0].ContainerName == pname ~ '-' ~ cname ~ '-1' - present_1.images[0].Repository == (docker_test_image_alpine | split(':') | first) - present_1.images[0].Tag == (docker_test_image_alpine | split(':') | last) - - present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2_check is not changed - - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2 is not changed - - present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3_check is changed - - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3 is changed - - present_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 #################################################################### ## Absent ########################################################## @@ -141,13 +141,13 @@ - assert: that: - absent_1_check is changed - - absent_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - absent_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - absent_1 is changed - - absent_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - absent_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - absent_2_check is not changed - - absent_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - absent_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - absent_2 is not changed - - absent_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - absent_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 #################################################################### ## Stopping and starting ########################################### @@ -257,30 +257,30 @@ - assert: that: - present_1_check is changed - - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_1 is changed - - present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2_check is not changed - - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_2 is not changed - - present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3_check is changed - - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_3 is changed - - present_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_4_check is not changed - - present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_4 is not changed - - present_4.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_4.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_5_check is changed - - present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_5 is changed - - present_5.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_5.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_6_check is changed - - present_6_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_6_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_6 is changed - - present_6.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_6.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_7_check is changed - - present_7_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_7_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - present_7 is changed - - present_7.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - present_7.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 diff --git a/tests/integration/targets/docker_compose_v2_pull/tasks/tests/pull.yml b/tests/integration/targets/docker_compose_v2_pull/tasks/tests/pull.yml index ed38de67e..b0bb8bc9a 100644 --- a/tests/integration/targets/docker_compose_v2_pull/tasks/tests/pull.yml +++ b/tests/integration/targets/docker_compose_v2_pull/tasks/tests/pull.yml @@ -65,10 +65,10 @@ that: - pull_1_check is failed or pull_1_check is changed - pull_1_check is changed or pull_1_check.msg.startswith('Error when processing ') - - pull_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_1 is failed - pull_1.msg.startswith('Error when processing ') - - pull_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 #################################################################### ## Regular image ################################################### @@ -141,26 +141,26 @@ that: - pull_1_check is changed - pull_1_check.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_1 is changed - pull_1.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_2_check is not changed - - pull_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_2 is not changed - - pull_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_3_check is changed - pull_3_check.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_3 is changed - pull_3.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_4_check is changed - pull_4_check.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_4_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_4_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_4 is not changed - pull_4.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_4.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_4.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - when: docker_compose_version is version('2.22.0', '<') block: @@ -194,13 +194,13 @@ that: - pull_1_check is changed - pull_1_check.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_1 is changed - pull_1.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_2_check is changed - pull_2_check.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 - pull_2 is not changed - pull_2.actions | selectattr('status', 'eq', 'Pulling') | first - - pull_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0 + - pull_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0 From f50c06c5d4e80e42a11a9686b55ef93c9fde3dc0 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Thu, 25 Jul 2024 18:12:53 +0200 Subject: [PATCH 6/7] Handle tail messages. --- plugins/module_utils/compose_v2.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/plugins/module_utils/compose_v2.py b/plugins/module_utils/compose_v2.py index bcb27637e..38ba65a92 100644 --- a/plugins/module_utils/compose_v2.py +++ b/plugins/module_utils/compose_v2.py @@ -392,7 +392,22 @@ def parse_json_events(stderr, warn_function=None): .format(line, exc) ) continue - if line_data.get('error'): + if line_data.get('tail'): + resource_type = ResourceType.UNKNOWN + msg = line_data.get('text') + status = 'Error' + if isinstance(msg, str) and msg.lower().startswith('warning:'): + # For some reason, Writer.TailMsgf() is always used for errors *except* in one place, + # where its message is prepended with 'WARNING: ' (in pkg/compose/pull.go). + status = 'Warning' + msg = msg[len('warning:'):].lstrip() + event = Event( + resource_type, + None, + status, + msg, + ) + elif line_data.get('error'): resource_type = ResourceType.UNKNOWN event = Event( resource_type, From b81b30fa4da132cc2d2ebdebd6fa240653978a32 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Thu, 25 Jul 2024 18:13:02 +0200 Subject: [PATCH 7/7] Fix bug in regular event parsing. --- plugins/module_utils/compose_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/module_utils/compose_v2.py b/plugins/module_utils/compose_v2.py index 38ba65a92..4768c11d6 100644 --- a/plugins/module_utils/compose_v2.py +++ b/plugins/module_utils/compose_v2.py @@ -493,7 +493,7 @@ def parse_events(stderr, dry_run=False, warn_function=None): index_event = _find_last_event_for(events, match.group('resource_id')) if index_event is not None: index, event = index_event - events[-1] = _concat_event_msg(event, match.group('msg')) + events[index] = _concat_event_msg(event, match.group('msg')) event, parsed = _extract_logfmt_event(line, warn_function=warn_function) if event is not None: events.append(event)