[Ansible 6] Improve setup.py to exclude some unnecessary files (#342)

* Exclude certain unnecessary files from the bdist.

* Collect ignore list in code (while looking at the real file list), not in template.

* Add changelog fragment.

* Adjust to ansible-community/community-topics#76.

changelogs/fragments/342-setup.py-ignore.yml

@@ -0,0 +1,2 @@
+major_changes:
+  - From Ansible 6 on, improve ``setup.py`` to exclude unnecessary files in the Python distribution (https://github.com/ansible-community/antsibull/pull/342).

src/antsibull/build_ansible_commands.py

@@ -129,6 +129,7 @@ def write_release_py(ansible_version: PypiVer, ansible_collections_dir: str) ->
 
 def write_setup(ansible_version: PypiVer,
                 ansible_core_version: PypiVer,
+                collection_exclude_paths: t.List[str],
                 collection_deps: str,
                 package_dir: str) -> None:
     setup_filename = os.path.join(package_dir, 'setup.py')
@@ -138,6 +139,7 @@ def write_setup(ansible_version: PypiVer,
         version=ansible_version,
         ansible_core_package_name=get_ansible_core_package_name(ansible_core_version),
         ansible_core_version=ansible_core_version,
+        collection_exclude_paths=collection_exclude_paths,
         collection_deps=collection_deps)
 
     with open(setup_filename, 'w', encoding='utf-8') as f:
@@ -146,13 +148,16 @@ def write_setup(ansible_version: PypiVer,
 
 def write_python_build_files(ansible_version: PypiVer,
                              ansible_core_version: PypiVer,
+                             collection_exclude_paths: t.List[str],
                              collection_deps: str,
                              package_dir: str,
                              release_notes: t.Optional[ReleaseNotes] = None,
                              debian: bool = False) -> None:
     copy_boilerplate_files(package_dir)
     write_manifest(package_dir, release_notes, debian)
-    write_setup(ansible_version, ansible_core_version, collection_deps, package_dir)
+    write_setup(
+        ansible_version, ansible_core_version, collection_exclude_paths, collection_deps,
+        package_dir)
 
 
 def write_debian_directory(ansible_version: PypiVer,
@@ -315,6 +320,47 @@ def prepare_command() -> int:
     return 0
 
 
+def compile_collection_exclude_paths(collection_names: t.Collection[str],
+                                     collection_root: str) -> t.Tuple[t.List[str], t.List[str]]:
+    result = set()
+    ignored_files = set()
+    all_files = []
+    for collection_name in collection_names:
+        namespace, name = collection_name.split('.', 1)
+        prefix = f"{namespace}/{name}/"
+
+        # Check files
+        collection_dir = os.path.join(collection_root, namespace, name)
+        all_files.clear()
+        for directory, _, files in os.walk(collection_dir):
+            directory = os.path.relpath(directory, collection_dir)
+            for file in files:
+                all_files.append(os.path.normpath(os.path.join(directory, file)))
+
+        def ignore_file(prefix: str, filename: str):  # pylint: disable=unused-variable
+            if filename in all_files:
+                result.add(prefix + filename)
+                ignored_files.add(prefix + filename)
+
+        def ignore_start(prefix: str, start: str):
+            matching_files = [file for file in all_files if file.startswith(start)]
+            if matching_files:
+                result.add(prefix + start + '*')
+                ignored_files.update([prefix + file for file in matching_files])
+
+        def ignore_directory(prefix: str, directory: str):
+            directory = directory.rstrip('/') + '/'
+            matching_files = [file for file in all_files if file.startswith(directory)]
+            if matching_files:
+                result.add(prefix + directory + '*')
+                ignored_files.update([prefix + file for file in matching_files])
+
+        ignore_start(prefix, '.')
+        ignore_directory(prefix, 'docs')
+        ignore_directory(prefix, 'tests')
+    return sorted(result), sorted(ignored_files)
+
+
 def rebuild_single_command() -> int:
     app_ctx = app_context.app_ctx.get()
 
@@ -379,10 +425,17 @@ def rebuild_single_command() -> int:
         release_notes.write_changelog_to(app_ctx.extra['dest_data_dir'])
         release_notes.write_porting_guide_to(app_ctx.extra['dest_data_dir'])
 
+        # pylint:disable-next=unused-variable
+        collection_exclude_paths, collection_ignored_files = compile_collection_exclude_paths(
+            dependency_data.deps, ansible_collections_dir)
+
+        # TODO: do something with collection_ignored_files
+
         # Write build scripts and files
         write_build_script(app_ctx.extra['ansible_version'], ansible_core_version, package_dir)
-        write_python_build_files(app_ctx.extra['ansible_version'], ansible_core_version, '',
-                                 package_dir, release_notes, app_ctx.extra['debian'])
+        write_python_build_files(app_ctx.extra['ansible_version'], ansible_core_version,
+                                 collection_exclude_paths, '', package_dir, release_notes,
+                                 app_ctx.extra['debian'])
         if app_ctx.extra['debian']:
             write_debian_directory(app_ctx.extra['ansible_version'], ansible_core_version,
                                    package_dir)
@@ -511,7 +564,7 @@ def build_multiple_command() -> int:
         collection_deps = '\n' + ',\n'.join(collection_deps)
         write_build_script(app_ctx.extra['ansible_version'], ansible_core_version, package_dir)
         write_python_build_files(app_ctx.extra['ansible_version'], ansible_core_version,
-                                 collection_deps, package_dir)
+                                 [], collection_deps, package_dir)
 
         make_dist(package_dir, app_ctx.extra['sdist_dir'])
 

src/antsibull/data/ansible-setup_py.j2

@@ -176,6 +176,15 @@ setup(
     python_requires='>=3.8',
 {%- endif %}
     packages=['ansible_collections'],
+{% if version.major >= 6 and collection_exclude_paths %}
+    exclude_package_data={
+        'ansible_collections': [
+{%   for path in collection_exclude_paths %}
+            '{{ path }}',
+{%   endfor %}
+        ],
+    },
+{% endif %}
     include_package_data=True,
     install_requires=[
         '{{ ansible_core_package_name }} ~= {{ ansible_core_version }}',{{ collection_deps }}