diff --git a/.github/workflows/antsibull-build.yml b/.github/workflows/antsibull-build.yml index f2fe846a..7e90c9a9 100644 --- a/.github/workflows/antsibull-build.yml +++ b/.github/workflows/antsibull-build.yml @@ -17,7 +17,7 @@ jobs: matrix: include: - name: Ansible 5 with default settings - options: '-e antsibull_ansible_version=5.99.0' + options: '-e antsibull_ansible_version=5.99.0 -e antsibull_run_sanity=true' - name: Ansible 6 with ansible-core devel options: '-e antsibull_ansible_version=6.99.0 -e antsibull_build_file=ansible-6.build -e antsibull_data_dir="{{ antsibull_data_git_dir }}/6" -e antsibull_ansible_git_version=devel' diff --git a/playbooks/nested-ansible-tests.yaml b/playbooks/nested-ansible-tests.yaml index ba60d3b5..b8cec5e9 100644 --- a/playbooks/nested-ansible-tests.yaml +++ b/playbooks/nested-ansible-tests.yaml @@ -4,7 +4,7 @@ hosts: localhost gather_facts: no vars: - # These can be supplied as extra-vars but are expected to come from roles/build-release/tasks/tests.yaml + # These can be supplied as extra-vars but are expected to come from roles/build-release/tasks/tests/nested.yaml antsibull_sdist_dir: "{{ playbook_dir | dirname }}/build" antsibull_ansible_venv: "{{ antsibull_sdist_dir }}/venv" antsibull_ansible_git_dir: "{{ antsibull_sdist_dir }}/ansible" diff --git a/roles/build-release/defaults/main.yaml b/roles/build-release/defaults/main.yaml index ecf2cd3e..5a5009b6 100644 --- a/roles/build-release/defaults/main.yaml +++ b/roles/build-release/defaults/main.yaml @@ -46,6 +46,10 @@ antsibull_ansible_venv: "{{ antsibull_sdist_dir }}/venv" # Whether or not to start from scratch with a new venv if one exists antsibull_venv_cleanup: true +# Whether to run "ansible-test sanity --docker" with podman on packaged collections +# This is disabled by default since it is time consuming to run. +antsibull_run_sanity: false + # TODO: # --dest-data-dir (Directory to write .build and .deps files to, as well as changelog and porting guide if applicable. Defaults to --data-dir) # --collection-cache (Directory of cached collection tarballs. Will be used if a collection tarball to be downloaded exists in here, and will be populated when downloading new tarballs.) diff --git a/roles/build-release/tasks/tests.yaml b/roles/build-release/tasks/tests.yaml index 804a9d5c..ccb8544d 100644 --- a/roles/build-release/tasks/tests.yaml +++ b/roles/build-release/tasks/tests.yaml @@ -20,108 +20,12 @@ virtualenv: "{{ antsibull_ansible_venv }}" virtualenv_command: "{{ ansible_python.executable }} -m venv" -# Note: the version of ansible-core doesn't necessarily match the deps file since the version requirement is >= -- block: - # Note: the value is either _ansible_base_version or _ansible_core_version depending on the version - # ex: https://github.com/ansible-community/ansible-build-data/blob/main/4/ansible-4.4.0.deps - - name: Retrieve the expected minimum version of ansible-core - shell: >- - grep -E "_ansible_(base|core)_version" {{ antsibull_data_dir }}/{{ _deps_file }} | awk '{print $2}' - changed_when: false - register: _expected_ansible_core +- name: Test and validate that we have the expected versions of collections, ansible-core and ansible + include_tasks: tests/versions.yaml - - name: Retrieve the installed version of ansible-core - shell: >- - {{ antsibull_ansible_venv }}/bin/pip show ansible-core | awk '/Version/ {print $2}' - changed_when: false - register: _installed_ansible_core +- name: Install the built ansible in venv and run integration tests with it + include_tasks: tests/nested.yaml - - name: Validate the version of ansible-core - ansible.builtin.assert: - that: - - _installed_ansible_core.stdout is version(_expected_ansible_core.stdout, '>=') - success_msg: "ansible-core {{ _installed_ansible_core.stdout }} matches (or exceeds) {{ _deps_file }}" - fail_msg: "ansible-core {{ _installed_ansible_core.stdout }} does not match {{ _deps_file }}" - -- block: - - name: Retrieve expected versions of Ansible and collections - command: cat {{ antsibull_data_dir }}/{{ _deps_file }} - changed_when: false - register: _expected_versions - - - name: Retrieve collections that should be included in the package - shell: cat {{ antsibull_data_dir }}/ansible.in | egrep -v "^#" - changed_when: false - register: _included_collections - - - name: Retrieve the installed version of ansible with pip - shell: >- - {{ antsibull_ansible_venv }}/bin/pip show ansible | awk '/Version/ {print $2}' - changed_when: false - register: _ansible_version_pypi - - - name: Retrieve the builtin reported version of ansible - command: >- - {{ antsibull_ansible_venv }}/bin/python3 -c 'from ansible_collections.ansible_release import ansible_version; print(ansible_version)' - changed_when: false - register: _ansible_version_builtin - - - name: Validate the version of ansible - ansible.builtin.assert: - that: - - "'_ansible_version: {{ _ansible_version_pypi.stdout }}' in _expected_versions.stdout" - - _ansible_version_pypi.stdout == _ansible_version_builtin.stdout - success_msg: "ansible {{ _ansible_version_pypi.stdout }} matches {{ _deps_file }} as well as 'ansible_collections.ansible_release'" - fail_msg: "ansible {{ _ansible_version_pypi.stdout }} does not match {{ _deps_file }} or 'ansible_collections.ansible_release'" - - - name: Retrieve installed collections - environment: - # In case we happen to be testing with devel, don't print a warning about it - ANSIBLE_DEVEL_WARNING: false - # Until https://github.com/ansible/ansible/pull/70173 is backported and released - ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_ansible_venv }}/lib/{{ _python_version }}/site-packages/ansible_collections" - # List collections, remove empty lines, headers, file paths and format the results in the same way as the deps file - shell: >- - {{ antsibull_ansible_venv }}/bin/ansible-galaxy collection list | egrep -v '^$|^#|---|Collection.*Version' | awk '{ print $1 ": " $2 }' - changed_when: false - register: _installed_collections - - - name: Validate that the installed collections are the expected ones - ansible.builtin.assert: - that: - - item in _expected_versions.stdout - success_msg: "{{ item }} matches {{ _deps_file }}" - fail_msg: "{{ item }} does not match {{ _deps_file }}" - loop: "{{ _installed_collections.stdout_lines }}" - - - name: Validate that included collections are packaged - ansible.builtin.assert: - that: - - item in _installed_collections.stdout - success_msg: "{{ item }} is in ansible.in and inside the package" - fail_msg: "{{ item }} is in ansible.in but not inside the package. Maybe run 'antsibull-build new-ansible --data-dir={{ antsibull_data_dir }}' to update ansible.in and then rebuild ?" - loop: "{{ _included_collections.stdout_lines }}" - -- block: - - name: Create a temporary COLLECTIONS_PATH - file: - path: "{{ antsibull_sdist_dir }}/ansible_collections" - state: directory - - - name: Install community.general for tests using 'a_module' and 'collection_version' - environment: - ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections" - command: >- - {{ antsibull_ansible_venv }}/bin/ansible-galaxy collection install community.general - - - name: Run nested Ansible tests with the Ansible we just built - environment: - ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections" - command: >- - {{ antsibull_ansible_venv }}/bin/ansible-playbook -i 'localhost,' --connection=local - -e antsibull_sdist_dir="{{ antsibull_sdist_dir }}" - -e antsibull_ansible_venv="{{ antsibull_ansible_venv }}" - -e antsibull_ansible_git_dir="{{ antsibull_ansible_git_dir }}" - -e antsibull_ansible_git_version="{{ antsibull_ansible_git_version }}" - -e _python_version="{{ _python_version }}" - {{ playbook_dir }}/nested-ansible-tests.yaml +- name: Run ansible-test sanity --docker on installed collections with podman + include_tasks: tests/sanity.yaml + when: antsibull_run_sanity | bool diff --git a/roles/build-release/tasks/tests/nested.yaml b/roles/build-release/tasks/tests/nested.yaml new file mode 100644 index 00000000..f282b821 --- /dev/null +++ b/roles/build-release/tasks/tests/nested.yaml @@ -0,0 +1,23 @@ +- block: + - name: Create a temporary COLLECTIONS_PATH + file: + path: "{{ antsibull_sdist_dir }}/ansible_collections" + state: directory + + - name: Install community.general for tests using 'a_module' and 'collection_version' + environment: + ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections" + command: >- + {{ antsibull_ansible_venv }}/bin/ansible-galaxy collection install community.general + + - name: Run nested Ansible tests with the Ansible we just built + environment: + ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_sdist_dir }}/ansible_collections" + command: >- + {{ antsibull_ansible_venv }}/bin/ansible-playbook -i 'localhost,' --connection=local + -e antsibull_sdist_dir="{{ antsibull_sdist_dir }}" + -e antsibull_ansible_venv="{{ antsibull_ansible_venv }}" + -e antsibull_ansible_git_dir="{{ antsibull_ansible_git_dir }}" + -e antsibull_ansible_git_version="{{ antsibull_ansible_git_version }}" + -e _python_version="{{ _python_version }}" + {{ playbook_dir }}/nested-ansible-tests.yaml diff --git a/roles/build-release/tasks/tests/sanity.yaml b/roles/build-release/tasks/tests/sanity.yaml new file mode 100644 index 00000000..6a0e865f --- /dev/null +++ b/roles/build-release/tasks/tests/sanity.yaml @@ -0,0 +1,55 @@ +- name: Get list of installed packages to verify if podman is installed + package_facts: + manager: "auto" + # This is noisy (and potentially sensitive) to print on stdout + no_log: true + +# This is so we can otherwise run unprivileged if podman is already installed +- when: "'podman' not in ansible_facts['packages'].keys()" + become: true + block: + - name: Install podman + package: + name: podman + state: present + rescue: + - name: Could not install podman + fail: + msg: | + Failed to elevate privileges and install podman. + Install podman manually or run ansible-playbook with elevated privileges. + +# The tests are no longer installed on ansible>=6.0, use the ones from the release tarball +- name: Unzip the release tarball + ansible.builtin.unarchive: + src: "{{ _release_archive }}" + dest: "{{ antsibull_sdist_dir }}" + remote_src: yes + +# ansible.builtin.find doesn't have mindepth +# https://github.com/ansible/ansible/issues/36369 +- name: Find collection directories + vars: + _extracted_release: "{{ antsibull_sdist_dir }}/ansible-{{ antsibull_ansible_version }}" + command: find {{ _extracted_release }}/ansible_collections -mindepth 2 -maxdepth 2 -type d + register: _collection_directories + +# Note: unless we run a git init first, ansible-test won't be able to pick up tests because it's under +# a .gitignore directory (antsibull/build) +- name: Run sanity tests and collect results at the end + shell: "git init . -b main; {{ antsibull_ansible_venv }}/bin/ansible-test sanity --skip-test pylint --docker" + ignore_errors: yes + args: + chdir: "{{ item }}" + loop: "{{ _collection_directories.stdout_lines }}" + register: _sanity_tests + +- name: Print sanity test failures + debug: + msg: | + {{ _result.item }} + {{ _result.stderr }} + when: _result.rc == 1 + loop: "{{ _sanity_tests.results }}" + loop_control: + loop_var: _result diff --git a/roles/build-release/tasks/tests/versions.yaml b/roles/build-release/tasks/tests/versions.yaml new file mode 100644 index 00000000..a281aad0 --- /dev/null +++ b/roles/build-release/tasks/tests/versions.yaml @@ -0,0 +1,84 @@ +--- +# Validate that we have the expected versions for the Ansible collections as well ansible-core and ansible + +# Note: the version of ansible-core doesn't necessarily match the deps file since the version requirement is >= +- block: + # Note: the value is either _ansible_base_version or _ansible_core_version depending on the version + # ex: https://github.com/ansible-community/ansible-build-data/blob/main/4/ansible-4.4.0.deps + - name: Retrieve the expected minimum version of ansible-core + shell: >- + grep -E "_ansible_(base|core)_version" {{ antsibull_data_dir }}/{{ _deps_file }} | awk '{print $2}' + changed_when: false + register: _expected_ansible_core + + - name: Retrieve the installed version of ansible-core + shell: >- + {{ antsibull_ansible_venv }}/bin/pip show ansible-core | awk '/Version/ {print $2}' + changed_when: false + register: _installed_ansible_core + + - name: Validate the version of ansible-core + ansible.builtin.assert: + that: + - _installed_ansible_core.stdout is version(_expected_ansible_core.stdout, '>=') + success_msg: "ansible-core {{ _installed_ansible_core.stdout }} matches (or exceeds) {{ _deps_file }}" + fail_msg: "ansible-core {{ _installed_ansible_core.stdout }} does not match {{ _deps_file }}" + +- block: + - name: Retrieve expected versions of Ansible and collections + command: cat {{ antsibull_data_dir }}/{{ _deps_file }} + changed_when: false + register: _expected_versions + + - name: Retrieve collections that should be included in the package + shell: cat {{ antsibull_data_dir }}/ansible.in | egrep -v "^#" + changed_when: false + register: _included_collections + + - name: Retrieve the installed version of ansible with pip + shell: >- + {{ antsibull_ansible_venv }}/bin/pip show ansible | awk '/Version/ {print $2}' + changed_when: false + register: _ansible_version_pypi + + - name: Retrieve the builtin reported version of ansible + command: >- + {{ antsibull_ansible_venv }}/bin/python3 -c 'from ansible_collections.ansible_release import ansible_version; print(ansible_version)' + changed_when: false + register: _ansible_version_builtin + + - name: Validate the version of ansible + ansible.builtin.assert: + that: + - "'_ansible_version: {{ _ansible_version_pypi.stdout }}' in _expected_versions.stdout" + - _ansible_version_pypi.stdout == _ansible_version_builtin.stdout + success_msg: "ansible {{ _ansible_version_pypi.stdout }} matches {{ _deps_file }} as well as 'ansible_collections.ansible_release'" + fail_msg: "ansible {{ _ansible_version_pypi.stdout }} does not match {{ _deps_file }} or 'ansible_collections.ansible_release'" + + - name: Retrieve installed collections + environment: + # In case we happen to be testing with devel, don't print a warning about it + ANSIBLE_DEVEL_WARNING: false + # Until https://github.com/ansible/ansible/pull/70173 is backported and released + ANSIBLE_COLLECTIONS_PATH: "{{ antsibull_ansible_venv }}/lib/{{ _python_version }}/site-packages/ansible_collections" + # List collections, remove empty lines, headers, file paths and format the results in the same way as the deps file + shell: >- + {{ antsibull_ansible_venv }}/bin/ansible-galaxy collection list | egrep -v '^$|^#|---|Collection.*Version' | awk '{ print $1 ": " $2 }' + changed_when: false + register: _installed_collections + + - name: Validate that the installed collections are the expected ones + ansible.builtin.assert: + that: + - item in _expected_versions.stdout + success_msg: "{{ item }} matches {{ _deps_file }}" + fail_msg: "{{ item }} does not match {{ _deps_file }}" + loop: "{{ _installed_collections.stdout_lines }}" + + - name: Validate that included collections are packaged + ansible.builtin.assert: + that: + - item in _installed_collections.stdout + success_msg: "{{ item }} is in ansible.in and inside the package" + fail_msg: "{{ item }} is in ansible.in but not inside the package. Maybe run 'antsibull-build new-ansible --data-dir={{ antsibull_data_dir }}' to update ansible.in and then rebuild ?" + loop: "{{ _included_collections.stdout_lines }}" \ No newline at end of file