filetree_create generate unified files

.github/workflows/release.yml

@@ -17,7 +17,6 @@ jobs:
           - devel
           - 22.4.0
           - 22.3.0
-          - 21.13.0
           - 21.11.0
     uses: "./.github/workflows/ci_standalone_versioned.yml"
     with:

changelogs/fragments/filtree_create_flatten_output.yml

@@ -0,0 +1,4 @@
+---
+minor_changes:
+  - The role 'filetree_create' will now allow to export all the objects of one kind into a single file, so it can be loaded by both ansible `group_vars` syntax and `filetree_read` tool.
+...

playbooks/flatten_filetree_create_output.yaml

@@ -0,0 +1,94 @@
+---
+- name: "Flatten the filetree_create output into unique files per each object type"
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    filetree_controller_settings: &filetree_create_output_dir "{{ filetree_create_output_dir }}"
+    filetree_controller_organizations: *filetree_create_output_dir
+    filetree_controller_labels: *filetree_create_output_dir
+    filetree_controller_user_accounts: *filetree_create_output_dir
+    filetree_controller_teams: *filetree_create_output_dir
+    filetree_controller_credential_types: *filetree_create_output_dir
+    filetree_controller_credentials: *filetree_create_output_dir
+    filetree_controller_credential_input_sources: *filetree_create_output_dir
+    filetree_controller_notifications: *filetree_create_output_dir
+    filetree_controller_projects: *filetree_create_output_dir
+    filetree_controller_execution_environments: *filetree_create_output_dir
+    filetree_controller_applications: *filetree_create_output_dir
+    filetree_controller_inventories: *filetree_create_output_dir
+    filetree_controller_inventory_sources: *filetree_create_output_dir
+    filetree_controller_instance_groups: *filetree_create_output_dir
+    filetree_controller_hosts: *filetree_create_output_dir
+    filetree_controller_groups: *filetree_create_output_dir
+    filetree_controller_templates: *filetree_create_output_dir
+    filetree_controller_workflow_job_templates: *filetree_create_output_dir
+    filetree_controller_schedules: *filetree_create_output_dir
+    filetree_controller_roles: *filetree_create_output_dir
+  roles:
+    - infra.controller_configuration.filetree_read
+  post_tasks:
+    - name: "Create the output flatten dir"
+      ansible.builtin.file:
+        path: "{{ filetree_create_output_dir }}_flatten"
+        state: directory
+        mode: "0755"
+
+    - name: "Write all the objects to the corresponding file"
+      ansible.builtin.copy:
+        dest: "{{ filetree_create_output_dir }}_flatten/{{ object_type.name }}.yaml"
+        mode: "0644"
+        content: |
+          ---
+          {{ object_type.value | to_nice_yaml(indent=2) }}
+          ...
+      loop_control:
+        loop_var: object_type
+      loop:
+        - name: controller_settings
+          value: "{{ controller_settings }}"
+        - name: controller_organizations
+          value: "{{ controller_organizations }}"
+        - name: controller_labels
+          value: "{{ controller_labels }}"
+        - name: controller_user_accounts
+          value: "{{ controller_user_accounts }}"
+        - name: controller_teams
+          value: "{{ controller_teams }}"
+        - name: controller_credential_types
+          value: "{{ controller_credential_types }}"
+        - name: controller_credentials
+          value: "{{ controller_credentials }}"
+        - name: controller_credential_input_sources
+          value: "{{ controller_credential_input_sources }}"
+        - name: controller_notifications
+          value: "{{ controller_notifications }}"
+        - name: controller_projects
+          value: "{{ controller_projects }}"
+        - name: controller_execution_environments
+          value: "{{ controller_execution_environments }}"
+        - name: controller_applications
+          value: "{{ controller_applications }}"
+        - name: controller_inventories
+          value: "{{ controller_inventories }}"
+        - name: controller_inventory_sources
+          value: "{{ controller_inventory_sources }}"
+        - name: controller_instance_groups
+          value: "{{ controller_instance_groups }}"
+        - name: controller_hosts
+          value: "{{ controller_hosts }}"
+        - name: controller_groups
+          value: "{{ controller_groups }}"
+        - name: controller_templates
+          value: "{{ controller_templates }}"
+        - name: controller_workflow_job_templates
+          value: "{{ controller_workflow_job_templates | default([]) }}"
+        - name: controller_schedules
+          value: "{{ controller_schedules }}"
+        - name: controller_roles
+          value: "{{ controller_roles }}"
+...
+# Sample usage:
+#
+# ansible-playbook infra.controller_configuration.flatten_filetree_create_output.yaml -e '{filetree_create_output_dir: /tmp/filetree_output}'
+#

roles/filetree_create/README.md

@@ -19,6 +19,7 @@ The following variables are required for that role to work properly:
 | `organization_id` | N/A | no | int | Alternative to `organization_filter`, but specifiying the current organization's ID to filter by. Exports only the objects belonging to the specified organization (applies to all the objects that can be assigned to an organization). |
 | `output_path` | `/tmp/filetree_output` | yes | str | The path to the output directory where all the generated `yaml` files with the corresponding Objects as code will be written to. |
 | `input_tag` | `['all']` | no | List of Strings | The tags which are applied to the 'sub-roles'. If 'all' is in the list (the default value) then all roles will be called. |
+| `flatten_output` | N/A | no | bool | Whether to flatten the output in single files per each object type instead of the normal exportation structure |
 
 ## Dependencies
 
@@ -78,6 +79,122 @@ A list of other roles hosted on Galaxy should go here, plus any details in regar
 ...
 ```
 
+This role can generate output files in two different ways:
+
+- **Structured output**:
+
+  The output files are distributed in separate directories, by organization first, and then by object type. Into each of these directories, one file per object is generated. This way allows to organize the files using different criteria, for example, by funcionalities or applications.
+
+  The expotation can be triggered with the following command:
+
+  ```console
+  ansible-playbook -i localhost, filetree_create.yml -e '{controller_validate_certs: false, controller_hostname: localhost:8443, controller_username: admin, controller_password: password}'
+  ```
+
+  One example of this approach follows:
+
+  ```console
+  /tmp/filetree_output_distributted
+  ├── current_credential_types.yaml
+  ├── current_execution_environments.yaml
+  ├── current_instance_groups.yaml
+  ├── current_settings.yaml
+  ├── Default
+  │   ├── applications
+  │   │   ├── 23_controller_application-app2.yaml
+  │   │   └── 24_controller_application-app3.yaml
+  │   ├── credentials
+  │   │   ├── 82_Demo Credential.yaml
+  │   │   └── 84_Demo Custom Credential.yaml
+  │   ├── current_organization.yaml
+  │   ├── inventories
+  │   │   ├── Demo Inventory
+  │   │   │   └── 81_Demo Inventory.yaml
+  │   │   └── Test Inventory - Smart
+  │   │       ├── 78_Test Inventory - Smart.yaml
+  │   │       └── current_hosts.yaml
+  │   ├── job_templates
+  │   │   ├── 177_test-template-1.yaml
+  │   │   └── 190_Demo Job Template.yaml
+  │   ├── labels
+  │   │   ├── 52_Prod.yaml
+  │   │   ├── 53_differential.yaml
+  │   ├── notification_templates
+  │   │   ├── Email notification differential.yaml
+  │   │   └── Email notification.yaml
+  │   ├── projects
+  │   │   ├── 169_Test Project.yaml
+  │   │   ├── 170_Demo Project.yaml
+  │   ├── teams
+  │   │   ├── 28_satellite-qe.yaml
+  │   │   └── 29_tower-team.yaml
+  │   └── workflow_job_templates
+  │       ├── 191_Simple workflow schema.yaml
+  │       └── 200_Complicated workflow schema.yaml
+  ├── ORGANIZATIONLESS
+  │   ├── credentials
+  │   │   ├── 2_Ansible Galaxy.yaml
+  │   │   └── 3_Default Execution Environment Registry Credential.yaml
+  │   └── users
+  │       ├── admin.yaml
+  │       ├── controller_user.yaml
+  ├── schedules
+  │   ├── 1_Cleanup Job Schedule.yaml
+  │   ├── 2_Cleanup Activity Schedule.yaml
+  │   ├── 4_Cleanup Expired Sessions.yaml
+  │   ├── 52_Demo Schedule.yaml
+  │   ├── 53_Demo Schedule 2.yaml
+  │   └── 5_Cleanup Expired OAuth 2 Tokens.yaml
+  ├── team_roles
+  │   ├── current_roles_satellite-qe.yaml
+  │   └── current_roles_tower-team.yaml
+  └── user_roles
+      └── current_roles_controller_user.yaml
+  ```
+
+- **Flatten files**:
+
+  The output files are all located in the same directory. Each file contains a YAML list with all the objects belonging to the same object type. This output format allows to load all the objects both from the standard Ansible `group_vars` and from the `infra.controller_configuration.filetree_read` role.
+
+  The expotation can be triggered with the following command:
+
+  ```console
+  ansible-playbook -i localhost, filetree_create.yml -e '{controller_validate_certs: false, controller_hostname: localhost:8443, controller_username: admin, controller_password: password, flatten_output: true}'
+  ```
+
+  One example of this approach follows:
+
+  ```console
+  /tmp/filetree_output_flatten
+  ├── applications.yaml
+  ├── credentials.yaml
+  ├── current_credential_types.yaml
+  ├── current_execution_environments.yaml
+  ├── current_instance_groups.yaml
+  ├── current_settings.yaml
+  ├── groups.yaml
+  ├── hosts.yaml
+  ├── inventories.yaml
+  ├── inventory_sources.yaml
+  ├── job_templates.yaml
+  ├── labels.yaml
+  ├── notification_templates.yaml
+  ├── organizations.yaml
+  ├── projects.yaml
+  ├── schedules.yaml
+  ├── team_roles.yaml
+  ├── teams.yaml
+  ├── user_roles.yaml
+  ├── users.yaml
+  └── workflow_job_templates.yaml
+  ```
+
+A playbook to convert from the structured output to the flattened one is provided, and can be executed with the following command:
+
+```console
+ansible-playbook infra.controller_configuration.flatten_filetree_create_output.yaml -e '{filetree_create_output_dir: /tmp/filetree_output}'
+```
+
 ## License
 
 GPLv3+

roles/filetree_create/tasks/applications.yml

@@ -11,32 +11,67 @@
       order_by: 'organization,id'
   no_log: "{{ controller_configuration_filetree_create_secure_logging }}"
 
-- name: "Create the <ORGANIZATION_NAME>/applications output directory for applications in {{ output_path }}"
-  ansible.builtin.file:
-    path: "{{ __path }}"
-    state: directory
-    mode: '0755'
+- name: "Block for to generate flatten output"
+  when:
+    - flatten_output is defined
+    - flatten_output | bool
   vars:
-    __path: "{{ output_path }}/{{ needed_path | regex_replace('/', '_') }}/applications"
-  loop: "{{ (applications_lookvar | map(attribute='summary_fields') | selectattr('organization', 'defined') | map(attribute='organization') | map(attribute='name') | list | flatten | unique)
-            + (['ORGANIZATIONLESS'] if ((applications_lookvar | map(attribute='summary_fields') | selectattr('organization', 'undefined') | list | flatten) | length > 0) else [])
-         }}"
-  loop_control:
-    loop_var: needed_path
-    label: "{{ __path }}"
+    __dest: "{{ output_path }}/applications.yaml"
+  block:
+    - name: "Add current applications to the applications flat file"
+      ansible.builtin.blockinfile:
+        create: true
+        mode: "0644"
+        insertafter: EOF
+        path: "{{ __dest }}"
+        marker: ""
+        block: "{{ lookup('template', 'templates/current_applications.j2') }}"
+      vars:
+        application_organization: "{{ current_applications_asset_value.summary_fields.organization.name | default('ORGANIZATIONLESS', true) }}"
+        application_id: "{{ current_applications_asset_value.id }}"
+        application_name: "{{ current_applications_asset_value.name | regex_replace('/', '_') }}"
+        last_application: "{{ current_application_index == ((applications_lookvar | length) - 1) }}"
+      loop: "{{ applications_lookvar }}"
+      loop_control:
+        index_var: current_application_index
+        loop_var: current_applications_asset_value
+        label: "{{ __dest }}"
 
-- name: "Add current applications to the <ORGANIZATION_NAME>/applications output file in {{ output_path }}"
-  ansible.builtin.template:
-    src: "templates/current_applications.j2"
-    dest: "{{ __dest }}"
-    mode: '0644'
-  vars:
-    application_organization: "{{ current_applications_asset_value.summary_fields.organization.name | default('ORGANIZATIONLESS', true) }}"
-    application_id: "{{ current_applications_asset_value.id }}"
-    application_name: "{{ current_applications_asset_value.name | regex_replace('/', '_') }}"
-    __dest: "{{ output_path }}/{{ application_organization | regex_replace('/', '_') }}/applications/{{ application_id }}_{{ application_name | regex_replace('/', '_') }}.yaml"
-  loop: "{{ applications_lookvar }}"
-  loop_control:
-    loop_var: current_applications_asset_value
-    label: "{{ __dest }}"
+    - name: "Remove all the blank lines introduced by the last task"
+      ansible.builtin.lineinfile:
+        path: "{{ __dest }}"
+        line: ''
+        state: absent
+
+- name: "Block for to generate the filetre_create normal output"
+  when: flatten_output is not defined or not (flatten_output | bool)
+  block:
+    - name: "Create the <ORGANIZATION_NAME>/applications output directory for applications in {{ output_path }}"
+      ansible.builtin.file:
+        path: "{{ __path }}"
+        state: directory
+        mode: '0755'
+      vars:
+        __path: "{{ output_path }}/{{ needed_path | regex_replace('/', '_') }}/applications"
+      loop: "{{ (applications_lookvar | map(attribute='summary_fields') | selectattr('organization', 'defined') | map(attribute='organization') | map(attribute='name') | list | flatten | unique)
+                + (['ORGANIZATIONLESS'] if ((applications_lookvar | map(attribute='summary_fields') | selectattr('organization', 'undefined') | list | flatten) | length > 0) else [])
+             }}"
+      loop_control:
+        loop_var: needed_path
+        label: "{{ __path }}"
+
+    - name: "Add current applications to the <ORGANIZATION_NAME>/applications output file in {{ output_path }}"
+      ansible.builtin.template:
+        src: "templates/current_applications.j2"
+        dest: "{{ __dest }}"
+        mode: '0644'
+      vars:
+        application_organization: "{{ current_applications_asset_value.summary_fields.organization.name | default('ORGANIZATIONLESS', true) }}"
+        application_id: "{{ current_applications_asset_value.id }}"
+        application_name: "{{ current_applications_asset_value.name | regex_replace('/', '_') }}"
+        __dest: "{{ output_path }}/{{ application_organization | regex_replace('/', '_') }}/applications/{{ application_id }}_{{ application_name | regex_replace('/', '_') }}.yaml"
+      loop: "{{ applications_lookvar }}"
+      loop_control:
+        loop_var: current_applications_asset_value
+        label: "{{ __dest }}"
 ...