Solve issue Object diff for roles removing most roles (#562)

* added singulars to be treated as well

* new attributes for roles

* new attributes for roles

* added changelog fragment

* fix on map_item function

* removed extra empty line

* fixes on object_diff inputs

* removed debug information. added ORGANIZATIONLESS to credentials and users without an organization

* fix lintering issues

* fix lintering issues

* fix lintering issues

* tests fixes. multiple list from redhat-cop/infra.aap_configuration#647 fixed. Test ping URL fixed

plugins/lookup/controller_object_diff.py

@@ -89,11 +89,11 @@
   returned: on successful differential
 """
 
-from ansible.plugins.lookup import LookupBase
+import copy
 from ansible.errors import AnsibleError, AnsibleLookupError
 from ansible.module_utils._text import to_native
+from ansible.plugins.lookup import LookupBase
 from ansible.utils.display import Display
-import copy
 
 
 class LookupModule(LookupBase):
@@ -114,6 +114,13 @@ def create_present_list(self, compare_list):
 
         return compare_list
 
+    def map_item(self, item, new_attribute_name, attribute_value, dupitems):
+        new_item = copy.deepcopy(item)
+        new_item.update({new_attribute_name: attribute_value})
+        for dupitem in [dupitem for dupitem in dupitems if dupitem in new_item]:
+            new_item.pop(dupitem)
+        return new_item
+
     def run(self, terms, variables=None, **kwargs):
         self.set_options(direct=kwargs)
 
@@ -179,7 +186,12 @@ def run(self, terms, variables=None, **kwargs):
             api_list_reduced = copy.deepcopy(api_list)
         elif api_list[0]["type"] == "instance_group":
             compare_list_reduced = [{key: item[key] for key in keys_to_keep} for item in compare_list]
-            api_list_reduced = [{key: item[key] for key in api_keys_to_keep} for item in api_list if item["summary_fields"]["user_capabilities"]["delete"]]
+            api_list_reduced = [
+                {key: item[key] for key in api_keys_to_keep}
+                for item in api_list
+                if (item["summary_fields"] and item["summary_fields"]["user_capabilities"]["delete"])
+            ]
+
         else:
             compare_list_reduced = [{key: item[key] for key in keys_to_keep} for item in compare_list]
             api_list_reduced = [{key: item[key] for key in api_keys_to_keep} for item in api_list]
@@ -196,7 +208,7 @@ def run(self, terms, variables=None, **kwargs):
                 item.pop("summary_fields")
         elif api_list[0]["type"] == "credential":
             for item in api_list_reduced:
-                item.update({"organization": item["summary_fields"]["organization"]["name"]})
+                item.update({"organization": item["summary_fields"]["organization"]["name"] if item["summary_fields"]["organization"] else ""})
                 item.update({"credential_type": item["summary_fields"]["credential_type"]["name"]})
                 item.pop("summary_fields")
         elif api_list[0]["type"] == "workflow_job_template_node":
@@ -225,43 +237,57 @@ def run(self, terms, variables=None, **kwargs):
             list_to_extend = []
             list_to_remove = []
             for item in compare_list_reduced:
-                target_teams_expanded = False
-                job_templates_expanded = False
-                workflows_expanded = False
+                expanded = False
+                dupitems = [
+                    "target_team",
+                    "target_teams",
+                    "job_template",
+                    "job_templates",
+                    "workflow",
+                    "workflows",
+                    "inventory",
+                    "inventories",
+                    "project",
+                    "projects",
+                    "credential",
+                    "credentials",
+                ]
+                if "target_team" in item:
+                    list_to_extend.append(self.map_item(item, "team", item["target_team"], dupitems))
+                    expanded = True
                 if "target_teams" in item:
                     for team in item["target_teams"]:
-                        new_item = copy.deepcopy(item)
-                        new_item.update({"team": team})
-                        new_item.pop("target_teams")
-                        if "job_templates" in new_item:
-                            new_item.pop("job_templates")
-                        if "workflows" in new_item:
-                            new_item.pop("workflows")
-                        list_to_extend.append(new_item)
-                    target_teams_expanded = True
+                        list_to_extend.append(self.map_item(item, "team", team, dupitems))
+                    expanded = True
+                if "job_template" in item:
+                    list_to_extend.append(self.map_item(item, "job_template", item["job_template"], dupitems))
+                    expanded = True
                 if "job_templates" in item:
                     for job_template in item["job_templates"]:
-                        new_item = copy.deepcopy(item)
-                        new_item.update({"job_template": job_template})
-                        new_item.pop("job_templates")
-                        if "target_teams" in new_item:
-                            new_item.pop("target_teams")
-                        if "workflows" in new_item:
-                            new_item.pop("workflows")
-                        list_to_extend.append(new_item)
-                    job_templates_expanded = True
+                        list_to_extend.append(self.map_item(item, "job_template", job_template, dupitems))
+                    expanded = True
+                if "workflow" in item:
+                    list_to_extend.append(self.map_item(item, "workflow_job_template", item["workflow"], dupitems))
+                    expanded = True
                 if "workflows" in item:
                     for workflow in item["workflows"]:
-                        new_item = copy.deepcopy(item)
-                        new_item.update({"workflow_job_template": workflow})
-                        new_item.pop("workflows")
-                        if "target_teams" in new_item:
-                            new_item.pop("target_teams")
-                        if "job_templates" in new_item:
-                            new_item.pop("job_templates")
-                        list_to_extend.append(new_item)
-                    workflows_expanded = True
-                if target_teams_expanded or job_templates_expanded or workflows_expanded:
+                        list_to_extend.append(self.map_item(item, "workflow_job_template", workflow, dupitems))
+                    expanded = True
+                if "inventory" in item:
+                    list_to_extend.append(self.map_item(item, "inventory", item["inventory"], dupitems))
+                    expanded = True
+                if "inventories" in item:
+                    for inventory in item["inventories"]:
+                        list_to_extend.append(self.map_item(item, "inventory", inventory, dupitems))
+                    expanded = True
+                if "project" in item:
+                    list_to_extend.append(self.map_item(item, "project", item["project"], dupitems))
+                    expanded = True
+                if "projects" in item:
+                    for project in item["projects"]:
+                        list_to_extend.append(self.map_item(item, "project", project, dupitems))
+                    expanded = True
+                if expanded:
                     list_to_remove.append(item)
             for item in list_to_remove:
                 compare_list_reduced.remove(item)
@@ -295,7 +321,8 @@ def run(self, terms, variables=None, **kwargs):
                 item.update({"state": "absent"})
         # Combine Lists
         if self.get_option("with_present"):
-            compare_list = self.create_present_list(compare_list)
+            for item in compare_list_reduced:
+                item.update({"state": "present"})
             compare_list.extend(difference)
             # Return Compare list with difference attached
             difference = compare_list
@@ -308,4 +335,4 @@ def run(self, terms, variables=None, **kwargs):
             for item in difference_to_remove:
                 difference.remove(item)
 
-        return difference
+        return [difference]

roles/filetree_create/tasks/all.yml

@@ -25,7 +25,7 @@
         msg: "The organization {{ organization_filter }} has the ID {{ organization_id }}"
 
 - name: Include tasks (block)
-  when: "['all', 'labels', 'applications', 'instance_groups', 'settings', 'inventory', 'credentials', 'credential_types', 'notification_templates', 'users', 'teams', 'organizations', 'projects', 'execution_environments', 'job_templates', 'workflow_job_templates', 'workflow_job_template_nodes', 'schedules'] | intersect(input_tag) | length > 0"
+  when: "['all', 'labels', 'applications', 'instance_groups', 'settings', 'inventory', 'credentials', 'credential_types', 'notification_templates', 'users', 'teams', 'roles', 'organizations', 'projects', 'execution_environments', 'job_templates', 'workflow_job_templates', 'workflow_job_template_nodes', 'schedules'] | intersect(input_tag) | length > 0"
   block:
     - name: "Export Inventories and related Groups and Hosts"
       ansible.builtin.include_tasks: "inventory.yml"
@@ -41,10 +41,10 @@
       when: "'notification_templates' in input_tag or 'all' in input_tag"
     - name: "Export Users"
       ansible.builtin.include_tasks: "users.yml"
-      when: "'users' in input_tag or 'all' in input_tag"
+      when: "'users' in input_tag or 'roles' in input_tag or 'all' in input_tag"
     - name: "Export Teams"
       ansible.builtin.include_tasks: "teams.yml"
-      when: "'teams' in input_tag or 'all' in input_tag"
+      when: "'teams' in input_tag or 'roles' in input_tag or 'all' in input_tag"
     - name: "Export Organizations"
       ansible.builtin.include_tasks: "organizations.yml"
       when: "'organizations' in input_tag or 'all' in input_tag"

roles/filetree_create/tasks/users.yml

@@ -9,7 +9,7 @@
 
 - name: "Add the users the Organizations information"  # noqa jinja[spacing]
   ansible.builtin.set_fact:
-    current_users: "{{ (current_users | default([])) + [user_lookvar_item | combine({'organizations': user_lookvar_item_organizations})] }}"
+    current_users: "{{ (current_users | default([])) + [user_lookvar_item | combine({'organizations': user_lookvar_item_organizations if (user_lookvar_item_organizations | length > 1) else ['ORGANIZATIONLESS']})] }}"
   vars:
     user_lookvar_item_organizations: "{{ query(controller_api_plugin, user_lookvar_item.related.organizations,
                                                host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs,

roles/filetree_create/templates/current_credentials.j2

@@ -5,6 +5,8 @@ controller_credentials:
     credential_type: "{{ current_credentials_asset_value.summary_fields.credential_type.name }}"
 {% if current_credentials_asset_value.organization is defined and current_credentials_asset_value.organization is not none %}
     organization: "{{ current_credentials_asset_value.summary_fields.organization.name }}"
+{% else %}
+    organization: "ORGANIZATIONLESS"
 {% endif %}
     inputs:
 {{ current_credentials_asset_value.inputs | to_nice_yaml(indent=2) | indent(width=6, first=True) | replace("$encrypted$", "\'\'") }}

roles/object_diff/tasks/applications.yml

@@ -9,17 +9,17 @@
 - name: "Get the API list of all Applications in Organization {{ orgs }}"
   ansible.builtin.set_fact:
     __controller_api_applications: "{{ query(controller_api_plugin, 'applications',
-                                          query_params={'organization': __controller_organization_id.id},
-                                          host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs,
-                                          return_all=true, max_objects=query_controller_api_max_objects)
-                                }}"
+                                             query_params={'organization': __controller_organization_id.id},
+                                             host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs,
+                                             return_all=true, max_objects=query_controller_api_max_objects)
+                                    }}"
 
 - name: "Find the difference of Application between what is on the Controller versus CasC on SCM"
   ansible.builtin.set_fact:
     __applications_difference: "{{ query(controller_role_plugin,
-                                      api_list=__controller_api_applications, compare_list=controller_applications,
-                                      with_present=include_present_state, set_absent=true)
-                            }}"
+                                         api_list=__controller_api_applications, compare_list=controller_applications,
+                                         with_present=include_present_state, set_absent=true) | flatten
+                                }}"
 
 - name: "Set application's list to be configured"
   ansible.builtin.set_fact:

roles/object_diff/tasks/credential_types.yml

@@ -23,8 +23,9 @@
 - name: "Find the difference of Credential Types between what is on the Controller versus CasC on SCM"
   ansible.builtin.set_fact:
     __credential_types_difference: "{{ query(controller_role_plugin,
-      api_list=__controller_api_credential_types, compare_list=controller_credential_types,
-      with_present=include_present_state, set_absent=true) }}"
+                                             api_list=__controller_api_credential_types, compare_list=controller_credential_types,
+                                             with_present=include_present_state, set_absent=true) | flatten
+                                    }}"
 
 - name: "Set credential's list to be configured"
   ansible.builtin.set_fact:

roles/object_diff/tasks/credentials.yml

@@ -20,8 +20,8 @@
 - name: "Find the difference of Credentials between what is on the Controller versus CasC on SCM"
   ansible.builtin.set_fact:
     __credentials_difference: "{{ query(controller_role_plugin,
-                                          api_list=__controller_api_credentials, compare_list=controller_credentials,
-                                          with_present=include_present_state, set_absent=true)
+                                        api_list=__controller_api_credentials, compare_list=controller_credentials,
+                                        with_present=include_present_state, set_absent=true) | flatten
                                }}"
 
 - name: "Set credential's list to be configured"

roles/object_diff/tasks/execution_environments.yml

@@ -1,27 +1,30 @@
 ---
-- name: Get the organization ID
-  ansible.builtin.set_fact:
-    __controller_organization_id: "{{ lookup(controller_api_plugin, 'organizations',
-                                              query_params={'name': orgs},
-                                              host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs)
-                                   }}"
+- name: "Block to be executed only when connected against an AAP instance (not Tower)"
+  when: is_aap
+  block:
+    - name: Get the organization ID
+      ansible.builtin.set_fact:
+        __controller_organization_id: "{{ lookup(controller_api_plugin, 'organizations',
+                                                 query_params={'name': orgs},
+                                                 host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs)
+                                       }}"
 
-- name: "Get the API list of all Execution Environments in Organization {{ orgs }}"
-  ansible.builtin.set_fact:
-    __controller_api_execution_environments: "{{ query(controller_api_plugin, 'execution_environments',
-                                          query_params={'organization': __controller_organization_id.id},
-                                          host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs,
-                                          return_all=true, max_objects=query_controller_api_max_objects)
-                                }}"
+    - name: "Get the API list of all Execution Environments in Organization {{ orgs }}"
+      ansible.builtin.set_fact:
+        __controller_api_execution_environments: "{{ query(controller_api_plugin, 'execution_environments',
+                                                           query_params={'organization': __controller_organization_id.id},
+                                                           host=controller_hostname, oauth_token=controller_oauthtoken, verify_ssl=controller_validate_certs,
+                                                           return_all=true, max_objects=query_controller_api_max_objects)
+                                                  }}"
 
-- name: "Find the difference of Execution Environment between what is on the Controller versus CasC on SCM"
-  ansible.builtin.set_fact:
-    __execution_environments_difference: "{{ query(controller_role_plugin,
-                                      api_list=__controller_api_execution_environments, compare_list=controller_execution_environments,
-                                      with_present=include_present_state, set_absent=true)
-                            }}"
+    - name: "Find the difference of Execution Environment between what is on the Controller versus CasC on SCM"
+      ansible.builtin.set_fact:
+        __execution_environments_difference: "{{ query(controller_role_plugin,
+                                                       api_list=__controller_api_execution_environments, compare_list=controller_execution_environments,
+                                                       with_present=include_present_state, set_absent=true) | flatten
+                                              }}"
 
-- name: "Set execution_environment's list to be configured"
-  ansible.builtin.set_fact:
-    controller_execution_environments: "{{ __execution_environments_difference }}"
+    - name: "Set execution_environment's list to be configured"
+      ansible.builtin.set_fact:
+        controller_execution_environments: "{{ __execution_environments_difference }}"
 ...

roles/object_diff/tasks/groups.yml

@@ -39,7 +39,7 @@
         __groups_difference: "{{ query(controller_role_plugin,
                                         query_params={'summary_fields.inventory.organization_id': controller_organization_id.id},
                                         api_list=__controller_api_groups, compare_list=controller_groups,
-                                        with_present=include_present_state, set_absent=true)
+                                        with_present=include_present_state, set_absent=true) | flatten
                               }}"
 
     - name: "Set the inventory key at the correct place"

roles/object_diff/tasks/hosts.yml

@@ -39,7 +39,7 @@
         __hosts_difference: "{{ query(controller_role_plugin,
                                         query_params={'summary_fields.inventory.organization_id': controller_organization_id.id},
                                         api_list=__controller_api_hosts, compare_list=controller_hosts,
-                                        with_present=include_present_state, set_absent=true)
+                                        with_present=include_present_state, set_absent=true) | flatten
                              }}"
 
     - name: "Set the inventory key at the correct place"