docker_compose_v2* modules: use --progress json for Compose 2.29.0+

changelogs/fragments/931-compose-2.29.0-json.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - "docker_compose_v2* modules - support Docker Compose 2.29.0's ``json`` progress writer to avoid having to parse text output (https://github.com/ansible-collections/community.docker/pull/931)."

plugins/module_utils/compose_v2.py

@@ -7,6 +7,7 @@
 __metaclass__ = type
 
 
+import json
 import os
 import re
 import shutil
@@ -16,6 +17,7 @@
 
 from ansible.module_utils.basic import missing_required_lib
 from ansible.module_utils.common.text.converters import to_native
+from ansible.module_utils.six import string_types
 from ansible.module_utils.six.moves import shlex_quote
 
 from ansible_collections.community.docker.plugins.module_utils.util import DockerBaseClass
@@ -84,6 +86,7 @@
     'Waiting',
 ))
 DOCKER_STATUS = frozenset(DOCKER_STATUS_DONE | DOCKER_STATUS_WORKING | DOCKER_STATUS_PULL | DOCKER_STATUS_ERROR | DOCKER_STATUS_WAITING)
+DOCKER_STATUS_AND_WARNING = frozenset(DOCKER_STATUS | DOCKER_STATUS_WARNING)
 
 DOCKER_PULL_PROGRESS_DONE = frozenset((
     'Already exists',
@@ -348,6 +351,111 @@ def _concat_event_msg(event, append_msg):
     )
 
 
+_JSON_LEVEL_TO_STATUS_MAP = {
+    'warning': 'Warning',
+    'error': 'Error',
+}
+
+
+def parse_json_events(stderr, warn_function=None):
+    events = []
+    stderr_lines = stderr.splitlines()
+    if stderr_lines and stderr_lines[-1] == b'':
+        del stderr_lines[-1]
+    for line in stderr_lines:
+        line = line.strip()
+        if not line.startswith(b'{') or not line.endswith(b'}'):
+            if line.startswith(b'Warning: '):
+                # This is a bug in Compose that will get fixed by https://github.com/docker/compose/pull/11996
+                event = Event(
+                    ResourceType.UNKNOWN,
+                    None,
+                    'Warning',
+                    to_native(line[len(b'Warning: '):]),
+                )
+                events.append(event)
+                continue
+            if warn_function:
+                warn_function(
+                    'Cannot parse event from non-JSON line: {0!r}. Please report this at '
+                    'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
+                    .format(line)
+                )
+            continue
+        try:
+            line_data = json.loads(line)
+        except Exception as exc:
+            if warn_function:
+                warn_function(
+                    'Cannot parse event from line: {0!r}: {1}. Please report this at '
+                    'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
+                    .format(line, exc)
+                )
+            continue
+        if line_data.get('tail'):
+            resource_type = ResourceType.UNKNOWN
+            msg = line_data.get('text')
+            status = 'Error'
+            if isinstance(msg, str) and msg.lower().startswith('warning:'):
+                # For some reason, Writer.TailMsgf() is always used for errors *except* in one place,
+                # where its message is prepended with 'WARNING: ' (in pkg/compose/pull.go).
+                status = 'Warning'
+                msg = msg[len('warning:'):].lstrip()
+            event = Event(
+                resource_type,
+                None,
+                status,
+                msg,
+            )
+        elif line_data.get('error'):
+            resource_type = ResourceType.UNKNOWN
+            event = Event(
+                resource_type,
+                line_data.get('id'),
+                'Error',
+                line_data.get('message'),
+            )
+        else:
+            resource_type = ResourceType.UNKNOWN
+            resource_id = line_data.get('id')
+            status = line_data.get('status')
+            text = line_data.get('text')
+            if isinstance(resource_id, str) and ' ' in resource_id:
+                resource_type_str, resource_id = resource_id.split(' ', 1)
+                try:
+                    resource_type = ResourceType.from_docker_compose_event(resource_type_str)
+                except KeyError:
+                    if warn_function:
+                        warn_function(
+                            'Unknown resource type {0!r} in line {1!r}. Please report this at '
+                            'https://github.com/ansible-collections/community.docker/issues/new?assignees=&labels=&projects=&template=bug_report.md'
+                            .format(resource_type_str, line)
+                        )
+                    resource_type = ResourceType.UNKNOWN
+            elif text in DOCKER_STATUS_PULL:
+                resource_type = ResourceType.IMAGE
+                status, text = text, status
+            elif text in DOCKER_PULL_PROGRESS_DONE or line_data.get('text') in DOCKER_PULL_PROGRESS_WORKING:
+                resource_type = ResourceType.IMAGE_LAYER
+                status, text = text, status
+            elif status is None and isinstance(text, string_types) and text.startswith('Skipped - '):
+                status, text = text.split(' - ', 1)
+            elif line_data.get('level') in _JSON_LEVEL_TO_STATUS_MAP and 'msg' in line_data:
+                status = _JSON_LEVEL_TO_STATUS_MAP[line_data['level']]
+                text = line_data['msg']
+            if status not in DOCKER_STATUS_AND_WARNING and text in DOCKER_STATUS_AND_WARNING:
+                status, text = text, status
+            event = Event(
+                resource_type,
+                resource_id,
+                status,
+                text,
+            )
+
+        events.append(event)
+    return events
+
+
 def parse_events(stderr, dry_run=False, warn_function=None):
     events = []
     error_event = None
@@ -385,7 +493,7 @@ def parse_events(stderr, dry_run=False, warn_function=None):
             index_event = _find_last_event_for(events, match.group('resource_id'))
             if index_event is not None:
                 index, event = index_event
-                events[-1] = _concat_event_msg(event, match.group('msg'))
+                events[index] = _concat_event_msg(event, match.group('msg'))
         event, parsed = _extract_logfmt_event(line, warn_function=warn_function)
         if event is not None:
             events.append(event)
@@ -456,7 +564,7 @@ def extract_actions(events):
 def emit_warnings(events, warn_function):
     for event in events:
         # If a message is present, assume it is a warning
-        if event.status is None and event.msg is not None:
+        if (event.status is None and event.msg is not None) or event.status in DOCKER_STATUS_WARNING:
             warn_function('Docker compose: {resource_type} {resource_id}: {msg}'.format(
                 resource_type=event.resource_type,
                 resource_id=event.resource_id,
@@ -476,11 +584,15 @@ def update_failed(result, events, args, stdout, stderr, rc, cli):
     errors = []
     for event in events:
         if event.status in DOCKER_STATUS_ERROR:
-            msg = 'Error when processing {resource_type} {resource_id}: '
-            if event.resource_type == 'unknown':
-                msg = 'Error when processing {resource_id}: '
-                if event.resource_id == '':
-                    msg = 'General error: '
+            if event.resource_id is None:
+                if event.resource_type == 'unknown':
+                    msg = 'General error: ' if event.resource_type == 'unknown' else 'Error when processing {resource_type}: '
+            else:
+                msg = 'Error when processing {resource_type} {resource_id}: '
+                if event.resource_type == 'unknown':
+                    msg = 'Error when processing {resource_id}: '
+                    if event.resource_id == '':
+                        msg = 'General error: '
             msg += '{status}' if event.msg is None else '{msg}'
             errors.append(msg.format(
                 resource_type=event.resource_type,
@@ -598,13 +710,19 @@ def __init__(self, client, min_version=MINIMUM_COMPOSE_VERSION):
             filenames = ', '.join(DOCKER_COMPOSE_FILES[:-1])
             self.fail('"{0}" does not contain {1}, or {2}'.format(self.project_src, filenames, DOCKER_COMPOSE_FILES[-1]))
 
+        # Support for JSON output was added in Compose 2.29.0 (https://github.com/docker/compose/releases/tag/v2.29.0);
+        # more precisely in https://github.com/docker/compose/pull/11478
+        self.use_json_events = self.compose_version >= LooseVersion('2.29.0')
+
     def fail(self, msg, **kwargs):
         self.cleanup()
         self.client.fail(msg, **kwargs)
 
     def get_base_args(self):
         args = ['compose', '--ansi', 'never']
-        if self.compose_version >= LooseVersion('2.19.0'):
+        if self.use_json_events:
+            args.extend(['--progress', 'json'])
+        elif self.compose_version >= LooseVersion('2.19.0'):
             # https://github.com/docker/compose/pull/10690
             args.extend(['--progress', 'plain'])
         args.extend(['--project-directory', self.project_src])
@@ -618,17 +736,25 @@ def get_base_args(self):
             args.extend(['--profile', profile])
         return args
 
+    def _handle_failed_cli_call(self, args, rc, stdout, stderr):
+        events = parse_json_events(stderr, warn_function=self.client.warn)
+        result = {}
+        self.update_failed(result, events, args, stdout, stderr, rc)
+        self.client.module.exit_json(**result)
+
     def list_containers_raw(self):
         args = self.get_base_args() + ['ps', '--format', 'json', '--all']
         if self.compose_version >= LooseVersion('2.23.0'):
             # https://github.com/docker/compose/pull/11038
             args.append('--no-trunc')
-        kwargs = dict(cwd=self.project_src, check_rc=True)
+        kwargs = dict(cwd=self.project_src, check_rc=not self.use_json_events)
         if self.compose_version >= LooseVersion('2.21.0'):
             # Breaking change in 2.21.0: https://github.com/docker/compose/pull/10918
-            dummy, containers, dummy = self.client.call_cli_json_stream(*args, **kwargs)
+            rc, containers, stderr = self.client.call_cli_json_stream(*args, **kwargs)
         else:
-            dummy, containers, dummy = self.client.call_cli_json(*args, **kwargs)
+            rc, containers, stderr = self.client.call_cli_json(*args, **kwargs)
+        if self.use_json_events and rc != 0:
+            self._handle_failed_cli_call(args, rc, containers, stderr)
         return containers
 
     def list_containers(self):
@@ -648,11 +774,15 @@ def list_containers(self):
 
     def list_images(self):
         args = self.get_base_args() + ['images', '--format', 'json']
-        kwargs = dict(cwd=self.project_src, check_rc=True)
-        dummy, images, dummy = self.client.call_cli_json(*args, **kwargs)
+        kwargs = dict(cwd=self.project_src, check_rc=not self.use_json_events)
+        rc, images, stderr = self.client.call_cli_json(*args, **kwargs)
+        if self.use_json_events and rc != 0:
+            self._handle_failed_cli_call(args, rc, images, stderr)
         return images
 
     def parse_events(self, stderr, dry_run=False):
+        if self.use_json_events:
+            return parse_json_events(stderr, warn_function=self.client.warn)
         return parse_events(stderr, dry_run=dry_run, warn_function=self.client.warn)
 
     def emit_warnings(self, events):

tests/integration/targets/docker_compose_v2/tasks/tests/definition.yml

@@ -77,7 +77,7 @@
     - assert:
         that:
           - present_1_check is changed
-          - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_1 is changed
           - present_1.containers | length == 1
           - present_1.containers[0].Name == pname ~ '-' ~ cname ~ '-1'
@@ -86,15 +86,15 @@
           - present_1.images[0].ContainerName == pname ~ '-' ~ cname ~ '-1'
           - present_1.images[0].Repository == (docker_test_image_alpine | split(':') | first)
           - present_1.images[0].Tag == (docker_test_image_alpine | split(':') | last)
-          - present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_2_check is not changed
-          - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_2 is not changed
-          - present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_3_check is changed
-          - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_3 is changed
-          - present_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
 
 ####################################################################
 ## Absent ##########################################################
@@ -133,13 +133,13 @@
     - assert:
         that:
           - absent_1_check is changed
-          - absent_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - absent_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - absent_1 is changed
-          - absent_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - absent_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - absent_2_check is not changed
-          - absent_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - absent_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - absent_2 is not changed
-          - absent_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - absent_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
 
 ####################################################################
 ## Stopping and starting ###########################################
@@ -259,30 +259,30 @@
     - assert:
         that:
           - present_1_check is changed
-          - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_1_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_1 is changed
-          - present_1.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_1.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_2_check is not changed
-          - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_2_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_2 is not changed
-          - present_2.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_2.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_3_check is changed
-          - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_3_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_3 is changed
-          - present_3.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_3.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_4_check is not changed
-          - present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_4_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_4 is not changed
-          - present_4.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_4.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_5_check is changed
-          - present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_5_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_5 is changed
-          - present_5.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_5.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_6_check is changed
-          - present_6_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_6_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_6 is changed
-          - present_6.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_6.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_7_check is changed
-          - present_7_check.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_7_check.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0
           - present_7 is changed
-          - present_7.warnings | default([]) | select('regex', 'Cannot parse event from line:') | length == 0
+          - present_7.warnings | default([]) | select('regex', 'Cannot parse event from ') | length == 0