Add additional logging to custom import task

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-40-gb1730d5
+2021.08.26-44-g30d780c

.github/workflows/scripts/install.sh

@@ -99,7 +99,7 @@ VARSYAML
 fi
 
 cat >> vars/main.yaml << VARSYAML
-pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "galaxy_api_default_distribution_base_path": "published", "galaxy_require_content_approval": false, "rh_entitlement_required": "insights"}
+pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "galaxy_api_default_distribution_base_path": "published", "galaxy_enable_api_access_log": true, "galaxy_require_content_approval": false, "rh_entitlement_required": "insights"}
 pulp_scheme: https
 
 pulp_container_tag: https

galaxy_ng/app/tasks/publishing.py

@@ -3,10 +3,9 @@
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.utils.translation import gettext_lazy as _
-
-from pulpcore.plugin.models import Task
 from pulp_ansible.app.models import AnsibleDistribution, AnsibleRepository, CollectionVersion
 from pulp_ansible.app.tasks.collections import import_collection
+from pulpcore.plugin.models import Task
 
 from .promotion import call_copy_task, call_remove_task
 
@@ -61,6 +60,13 @@ def import_and_move_to_staging(temp_file_pk, **kwargs):
         call_copy_task(collection_version, inbound_repo, staging_repo)
         call_remove_task(collection_version, inbound_repo)
 
+        if settings.GALAXY_ENABLE_API_ACCESS_LOG:
+            _log_collection_upload(
+                kwargs["expected_namespace"],
+                kwargs["expected_name"],
+                kwargs["expected_version"]
+            )
+
 
 def import_and_auto_approve(temp_file_pk, **kwargs):
     """Import collection version and automatically approve.
@@ -94,3 +100,15 @@ def import_and_auto_approve(temp_file_pk, **kwargs):
         log.info('Imported and auto approved collection artifact %s to repository %s',
                  collection_version.relative_path,
                  golden_repo.latest_version())
+
+        if settings.GALAXY_ENABLE_API_ACCESS_LOG:
+            _log_collection_upload(
+                kwargs["expected_namespace"],
+                kwargs["expected_name"],
+                kwargs["expected_version"]
+            )
+
+
+def _log_collection_upload(namespace, name, version):
+    api_access_log = logging.getLogger("automated_logging")
+    api_access_log.info("Collection uploaded: %s-%s-%s", namespace, name, version,)

galaxy_ng/tests/functional/cli/test_collection_upload.py

@@ -1,7 +1,8 @@
-"""Tests that Collections can be uploaded to  Pulp with the ansible-galaxy CLI."""
+"""Tests that Collections can be uploaded to Pulp with the ansible-galaxy CLI."""
 
 import subprocess
 import tempfile
+from unittest.case import skip
 
 from pulp_smash.pulp3.bindings import delete_orphans
 from pulp_smash.utils import http_get
@@ -44,3 +45,54 @@ def test_upload_collection(self):
         self.assertEqual(collection.namespace, "pulp")
         self.assertEqual(collection.name, "squeezer")
         self.assertEqual(collection.highest_version["version"], "0.0.9")
+
+        # Cleanup
+        self.delete_collection(collection.namespace, collection.name)
+        self.delete_namespace(collection.namespace)
+
+    def test_uploaded_collection_logged(self):
+        """Test whether a Collection uploaded via ansible-galaxy is logged correctly in API Access Log."""
+        delete_orphans()
+
+        # Create namespace if it doesn't exist
+        data = str(self.namespace_api.list().data)
+        if "pulp" not in data:
+            self.namespace_api.create(namespace={"name": "pulp", "groups": []})
+
+        # Preapare ansible.cfg for ansible-galaxy CLI
+        self.update_ansible_cfg("inbound-pulp")
+
+        # In a temp dir, publish a collection using ansible-galaxy CLI
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            content = http_get("https://galaxy.ansible.com/download/pulp-squeezer-0.0.9.tar.gz")
+            collection_path = f"{tmp_dir}/pulp-squeezer-0.0.9.tar.gz"
+            with open(collection_path, "wb") as f:
+                f.write(content)
+
+            cmd = "ansible-galaxy collection publish -vvv -c {}".format(collection_path)
+
+            subprocess.run(cmd.split())
+
+            cmd = f"docker cp pulp:/var/log/galaxy_api_access.log {tmp_dir}"
+
+            subprocess.run(cmd.split())
+
+            with open(f"{tmp_dir}/galaxy_api_access.log") as f:
+                log_contents = f.readlines()
+                print(log_contents)
+
+        # Verify that the collection was published
+        collections = self.collections_api.list("published")
+        self.assertEqual(collections.meta.count, 1)
+
+        # Verify that the colletion publishing was logged in the api access log
+        collection = self.collections_api.read(path="published", namespace="pulp", name="squeezer")
+        for line in log_contents:
+            if "INFO: Collection uploaded: " in line and 'pulp' in line:
+                self.assertIn(collection.namespace, line)
+                self.assertIn(collection.name, line)
+                self.assertIn(collection.highest_version["version"], line)
+
+        # Cleanup
+        self.delete_collection(collection.namespace, collection.name)
+        self.delete_namespace(collection.namespace)

galaxy_ng/tests/functional/utils.py

@@ -1,6 +1,7 @@
 """Utilities for tests for the galaxy plugin."""
 import os
 from functools import partial
+import random
 import requests
 from unittest import SkipTest
 from tempfile import NamedTemporaryFile
@@ -212,7 +213,7 @@ def update_ansible_cfg(self, base_path):
             "server_list = community_repo\n"
             "\n"
             "[galaxy_server.community_repo]\n"
-            f"url={self.cfg.get_content_host_base_url()}"
+            f"url={self.cfg.get_base_url()}"
             f"{self.galaxy_api_prefix}/content/{base_path}/\n"
             f"token={token}"
         )
@@ -236,6 +237,20 @@ def sync_repo(self, requirements_file, **kwargs):
         monitor_task(f"/pulp/api/v3/tasks/{response.task}/")
 
 
+    def delete_namespace(self, namespace_name):
+        """Delete a Namespace"""
+        # namespace_api does not support delete, so we can use the smash_client directly
+        self.smash_client.delete(
+            f"{self.galaxy_api_prefix}/v3/namespaces/{namespace_name}"
+        )
+
+    def delete_collection(self, collection_namespace, collection_name):
+        """Delete a Collection"""
+        self.smash_client.delete(
+            f"{self.galaxy_api_prefix}/v3/collections/{collection_namespace}/{collection_name}/"
+        )
+
+
 class ContainerTestCaseUsingBindings(TestCaseUsingBindings):
     @classmethod
     def setUpClass(cls):

galaxy_ng/tests/unit/app/test_tasks.py

@@ -12,6 +12,7 @@
 )
 from pulp_ansible.app.tasks.copy import copy_content
 
+from galaxy_ng.app.tasks.publishing import _log_collection_upload
 from galaxy_ng.app.tasks import import_and_auto_approve, import_and_move_to_staging
 from galaxy_ng.app.tasks.promotion import _remove_content_from_repository
 
@@ -171,3 +172,12 @@ def test_import_and_move_to_staging(self, mocked_dispatch, mocked_import, mocked
                 expected_name='',
                 expected_version='',
             )
+
+    def test_log_collection_upload(self):
+        with self.assertLogs(logger='automated_logging', level='INFO') as lm:
+            _log_collection_upload('namespace', 'name', '0.0.1')
+
+            self.assertIn(
+                'INFO:automated_logging:Collection uploaded: namespace-name-0.0.1',
+                lm.output
+            )

template_config.yml

@@ -65,6 +65,7 @@ pulp_settings:
   allowed_import_paths: /tmp
   galaxy_api_default_distribution_base_path: published
   galaxy_require_content_approval: false
+  galaxy_enable_api_access_log: true
   rh_entitlement_required: insights
 pulpcore_branch: 3.15.2
 pulpcore_pip_version_specifier: null