Add support for filtering refs during synchronization

closes #163

CHANGES/163.feature

@@ -0,0 +1,2 @@
+Added support for filtering refs during synchronization. Users can now use the flags
+``include_refs`` or ``exclude_refs`` to include or exclude refs from a remote repository.

pulp_ostree/app/migrations/0004_add_include_exclude_refs.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.12 on 2022-03-22 15:38
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ostree', '0003_create_many_to_many_objs_commits'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='ostreeremote',
+            name='exclude_refs',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255, null=True), null=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='ostreeremote',
+            name='include_refs',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255, null=True), null=True, size=None),
+        ),
+    ]

pulp_ostree/app/models.py

@@ -1,6 +1,7 @@
 from logging import getLogger
 
 from django.db import models
+from django.contrib.postgres.fields import ArrayField
 
 from pulpcore.plugin.models import (
     Content,
@@ -113,6 +114,8 @@ class OstreeRemote(Remote):
     TYPE = "ostree"
 
     depth = models.IntegerField(default=0)
+    include_refs = ArrayField(models.CharField(max_length=255, null=True), null=True)
+    exclude_refs = ArrayField(models.CharField(max_length=255, null=True), null=True)
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"

pulp_ostree/app/serializers.py

@@ -148,9 +148,33 @@ class OstreeRemoteSerializer(platform.RemoteSerializer):
         choices=[Remote.IMMEDIATE, Remote.ON_DEMAND],
         default=Remote.IMMEDIATE,
     )
+    include_refs = serializers.ListField(
+        child=serializers.CharField(max_length=255),
+        allow_null=True,
+        required=False,
+        help_text=_(
+            """
+            A list of refs to include during a sync.
+            The wildcards *, ? are recognized.
+            'include_refs' is evaluated before 'exclude_refs'.
+            """
+        ),
+    )
+    exclude_refs = serializers.ListField(
+        child=serializers.CharField(max_length=255),
+        allow_null=True,
+        required=False,
+        help_text=_(
+            """
+            A list of tags to exclude during a sync.
+            The wildcards *, ? are recognized.
+            'exclude_refs' is evaluated after 'include_refs'.
+            """
+        ),
+    )
 
     class Meta:
-        fields = platform.RemoteSerializer.Meta.fields + ("depth",)
+        fields = platform.RemoteSerializer.Meta.fields + ("depth", "include_refs", "exclude_refs")
         model = models.OstreeRemote
 
 

pulp_ostree/app/tasks/synchronizing.py

@@ -1,6 +1,7 @@
 import os
 import logging
 
+from fnmatch import fnmatch
 from gettext import gettext as _
 from pathlib import Path
 from urllib.parse import urljoin
@@ -93,13 +94,15 @@ async def run(self):
             await self.submit_metafiles()
 
             _, refs = self.repo.remote_list_refs(self.repo_name)
-            for name, _ in refs.items():
+            for name in self.filter_refs(refs.keys()):
+                ref_commit_checksum = refs[name]
+
                 ref_relative_path = os.path.join("refs/heads/", name)
-                await self.download_remote_object(ref_relative_path)
                 local_ref_path = os.path.join(self.repo_path, ref_relative_path)
 
-                with open(local_ref_path, "r") as f:
-                    ref_commit_checksum = f.read().strip()
+                with open(local_ref_path, "w") as f:
+                    f.write(ref_commit_checksum)
+                    f.flush()
 
                 relative_path = get_checksum_filepath(
                     ref_commit_checksum, OstreeObjectType.OSTREE_OBJECT_TYPE_COMMIT
@@ -164,6 +167,22 @@ async def run(self):
 
         await self.submit_ref_objects()
 
+    def filter_refs(self, refs):
+        """Filter refs by the list of include/exclude patterns."""
+
+        def _pattern_matches(ref, patterns):
+            return any(fnmatch(ref, pattern) for pattern in patterns)
+
+        include_refs = self.remote.include_refs
+        if include_refs:
+            refs = [ref for ref in refs if _pattern_matches(ref, include_refs)]
+
+        exclude_refs = self.remote.exclude_refs
+        if exclude_refs:
+            refs = [ref for ref in refs if not _pattern_matches(ref, exclude_refs)]
+
+        return refs
+
     def init_repository(self):
         """Initialize a new OSTree repository object."""
         self.repo_path = os.path.join(os.getcwd(), "repo/")

pulp_ostree/tests/functional/api/test_sync.py

@@ -18,6 +18,7 @@
     RepositoriesOstreeVersionsApi,
     RepositorySyncURL,
     RemotesOstreeApi,
+    ContentRefsApi,
 )
 
 
@@ -32,6 +33,7 @@ def setUpClass(cls):
         cls.versions_api = RepositoriesOstreeVersionsApi(client_api)
         cls.remotes_api = RemotesOstreeApi(client_api)
         cls.distributions_api = DistributionsOstreeApi(client_api)
+        cls.refs_api = ContentRefsApi(client_api)
 
     @classmethod
     def tearDownClass(cls):
@@ -97,3 +99,48 @@ def sync(self, policy):
         self.addCleanup(shutil.rmtree, remote.name)
         validate_repo_integrity(remote.name, f"{remote_name}:rawhide")
         validate_repo_integrity(remote.name, f"{remote_name}:stable")
+
+    def test_filter_rawhide_ref_sync(self):
+        """Synchronize content from a remote repository considering only a specific ref."""
+        repo = self.repositories_api.create(gen_repo())
+        self.addCleanup(self.repositories_api.delete, repo.pulp_href)
+
+        body = gen_ostree_remote(depth=0, include_refs=["rawhide"], exclude_refs=["stable"])
+        remote = self.remotes_api.create(body)
+        self.addCleanup(self.remotes_api.delete, remote.pulp_href)
+
+        self.assertEqual(remote.include_refs, ["rawhide"])
+        self.assertEqual(remote.exclude_refs, ["stable"])
+
+        repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
+        response = self.repositories_api.sync(repo.pulp_href, repository_sync_data)
+        repo_version = monitor_task(response.task).created_resources[0]
+
+        refs = self.refs_api.list(repository_version_added=repo_version).results
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0].name, "rawhide")
+
+    def test_exclude_all_refs_sync(self):
+        """Synchronize content from a remote repository when a user excludes all refs."""
+        repo = self.repositories_api.create(gen_repo())
+        self.addCleanup(self.repositories_api.delete, repo.pulp_href)
+
+        body = gen_ostree_remote(depth=0, exclude_refs=["*"])
+        remote = self.remotes_api.create(body)
+        self.addCleanup(self.remotes_api.delete, remote.pulp_href)
+
+        self.assertEqual(remote.include_refs, None)
+        self.assertEqual(remote.exclude_refs, ["*"])
+
+        repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
+        response = self.repositories_api.sync(repo.pulp_href, repository_sync_data)
+        repo_version = monitor_task(response.task).created_resources[0]
+
+        repository_version = self.versions_api.read(repo_version)
+        added_content = repository_version.content_summary.added
+
+        self.assertEqual(added_content["ostree.config"]["count"], 1)
+        self.assertEqual(added_content["ostree.summary"]["count"], 1)
+        self.assertRaises(KeyError, lambda: added_content["ostree.refs"])
+        self.assertRaises(KeyError, lambda: added_content["ostree.commit"])
+        self.assertRaises(KeyError, lambda: added_content["ostree.object"])