Add support for filtering refs during synchronization

In this commit, there was also altered a database model for Ref objects. The only information that is stored in a file is the checksum of a referenced commit. This can be saved directly to a string instead of holding reference to an artifact.

closes pulp#163

CHANGES/163.feature

@@ -0,0 +1,2 @@
+Added support for filtering refs during synchronization. Users can now use the flags
+``include_refs`` or ``exclude_refs`` to include or exclude refs from a remote repository.

pulp_ostree/app/migrations/0004_include_exclude_refs_no_artifact.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.12 on 2022-03-22 14:35
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ostree', '0003_create_many_to_many_objs_commits'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='ostreeremote',
+            name='exclude_refs',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255, null=True), null=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='ostreeremote',
+            name='include_refs',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255, null=True), null=True, size=None),
+        ),
+        migrations.AlterUniqueTogether(
+            name='ostreeref',
+            unique_together={('name', 'commit')},
+        ),
+        migrations.RemoveField(
+            model_name='ostreeref',
+            name='relative_path',
+        ),
+    ]

pulp_ostree/app/models.py

@@ -1,6 +1,7 @@
 from logging import getLogger
 
 from django.db import models
+from django.contrib.postgres.fields import ArrayField
 
 from pulpcore.plugin.models import (
     Content,
@@ -64,11 +65,10 @@ class OstreeRef(Content):
         OstreeCommit, related_name="refs_commit", null=True, on_delete=models.CASCADE
     )
     name = models.CharField(max_length=255, db_index=True)
-    relative_path = models.TextField(null=False)
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
-        unique_together = [["name", "commit", "relative_path"]]
+        unique_together = [["name", "commit"]]
 
 
 class OstreeCommitObject(models.Model):
@@ -113,6 +113,8 @@ class OstreeRemote(Remote):
     TYPE = "ostree"
 
     depth = models.IntegerField(default=0)
+    include_refs = ArrayField(models.CharField(max_length=255, null=True), null=True)
+    exclude_refs = ArrayField(models.CharField(max_length=255, null=True), null=True)
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"

pulp_ostree/app/serializers.py

@@ -73,7 +73,7 @@ class Meta:
         model = models.OstreeCommit
 
 
-class OstreeRefSerializer(platform.SingleArtifactContentSerializer):
+class OstreeRefSerializer(platform.NoArtifactContentSerializer):
     """A Serializer class for OSTree head commits."""
 
     commit = platform.DetailRelatedField(
@@ -85,7 +85,7 @@ class OstreeRefSerializer(platform.SingleArtifactContentSerializer):
     checksum = serializers.CharField(source="commit.checksum", read_only=True)
 
     class Meta:
-        fields = platform.SingleArtifactContentSerializer.Meta.fields + (
+        fields = platform.NoArtifactContentSerializer.Meta.fields + (
             "commit",
             "checksum",
             "name",
@@ -148,9 +148,33 @@ class OstreeRemoteSerializer(platform.RemoteSerializer):
         choices=[Remote.IMMEDIATE, Remote.ON_DEMAND],
         default=Remote.IMMEDIATE,
     )
+    include_refs = serializers.ListField(
+        child=serializers.CharField(max_length=255),
+        allow_null=True,
+        required=False,
+        help_text=_(
+            """
+            A list of refs to include during a sync.
+            The wildcards *, ? are recognized.
+            'include_refs' is evaluated before 'exclude_refs'.
+            """
+        ),
+    )
+    exclude_refs = serializers.ListField(
+        child=serializers.CharField(max_length=255),
+        allow_null=True,
+        required=False,
+        help_text=_(
+            """
+            A list of tags to exclude during a sync.
+            The wildcards *, ? are recognized.
+            'exclude_refs' is evaluated after 'include_refs'.
+            """
+        ),
+    )
 
     class Meta:
-        fields = platform.RemoteSerializer.Meta.fields + ("depth",)
+        fields = platform.RemoteSerializer.Meta.fields + ("depth", "include_refs", "exclude_refs")
         model = models.OstreeRemote
 
 

pulp_ostree/app/tasks/importing.py

@@ -63,7 +63,6 @@ async def parse_ref(self, name, ref_commit_checksum, has_referenced_parent=False
         relative_path = get_checksum_filepath(
             ref_commit_checksum, OstreeObjectType.OSTREE_OBJECT_TYPE_COMMIT
         )
-        ref_path = os.path.join("refs/", "heads/", name)
 
         parent_checksum = OSTree.commit_get_parent(ref_commit)
         if not parent_checksum:
@@ -74,7 +73,7 @@ async def parse_ref(self, name, ref_commit_checksum, has_referenced_parent=False
 
             await self.submit_related_objects(commit_dc)
 
-            self.init_ref_object(name, ref_path, commit_dc)
+            self.init_ref_object(name, commit_dc)
 
             return
 
@@ -83,7 +82,7 @@ async def parse_ref(self, name, ref_commit_checksum, has_referenced_parent=False
         ref_commit_dc = self.create_dc(relative_path, ref_commit)
         self.commit_dcs.append(ref_commit_dc)
 
-        self.init_ref_object(name, ref_path, ref_commit_dc)
+        self.init_ref_object(name, ref_commit_dc)
 
         try:
             _, parent_commit, _ = self.repo.load_commit(parent_checksum)

pulp_ostree/app/tasks/stages.py

@@ -30,10 +30,9 @@ async def submit_related_objects(self, commit_dc):
             object_dc.extra_data["commit_relation"] = await commit_dc.resolution()
             await self.put(object_dc)
 
-    def init_ref_object(self, name, relative_path, commit_dc):
+    def init_ref_object(self, name, commit_dc):
         """Initialize a DeclarativeContent object for a ref object."""
-        ref = OstreeRef(name=name)
-        ref_dc = self.create_dc(relative_path, ref)
+        ref_dc = DeclarativeContent(OstreeRef(name=name))
         ref_dc.extra_data["ref_commit"] = commit_dc
         self.refs_dcs.append(ref_dc)
 

pulp_ostree/app/tasks/synchronizing.py

@@ -1,6 +1,7 @@
 import os
 import logging
 
+from fnmatch import fnmatch
 from gettext import gettext as _
 from pathlib import Path
 from urllib.parse import urljoin
@@ -93,14 +94,8 @@ async def run(self):
             await self.submit_metafiles()
 
             _, refs = self.repo.remote_list_refs(self.repo_name)
-            for name, _ in refs.items():
-                ref_relative_path = os.path.join("refs/heads/", name)
-                await self.download_remote_object(ref_relative_path)
-                local_ref_path = os.path.join(self.repo_path, ref_relative_path)
-
-                with open(local_ref_path, "r") as f:
-                    ref_commit_checksum = f.read().strip()
-
+            for name in self.filter_refs(refs.keys()):
+                ref_commit_checksum = refs[name]
                 relative_path = get_checksum_filepath(
                     ref_commit_checksum, OstreeObjectType.OSTREE_OBJECT_TYPE_COMMIT
                 )
@@ -116,7 +111,7 @@ async def run(self):
 
                     await self.submit_related_objects(commit_dc)
 
-                    self.init_ref_object(name, ref_relative_path, commit_dc)
+                    self.init_ref_object(name, commit_dc)
 
                     continue
 
@@ -158,12 +153,27 @@ async def run(self):
 
                 await self.submit_previous_commits_and_related_objects()
 
-                self.init_ref_object(name, ref_relative_path, ref_commit_dc)
+                self.init_ref_object(name, ref_commit_dc)
 
             await pb.aincrement()
 
         await self.submit_ref_objects()
 
+    def filter_refs(self, refs):
+        """Filter refs by the list of include/exclude patterns."""
+        def _pattern_matches(ref, patterns):
+            return any(fnmatch(ref, pattern) for pattern in patterns)
+
+        include_refs = self.remote.include_refs
+        if include_refs:
+            refs = [ref for ref in refs if _pattern_matches(ref, include_refs)]
+
+        exclude_refs = self.remote.exclude_refs
+        if exclude_refs:
+            refs = [ref for ref in refs if not _pattern_matches(ref, exclude_refs)]
+
+        return refs
+
     def init_repository(self):
         """Initialize a new OSTree repository object."""
         self.repo_path = os.path.join(os.getcwd(), "repo/")

pulp_ostree/tests/functional/api/test_sync.py

@@ -18,6 +18,7 @@
     RepositoriesOstreeVersionsApi,
     RepositorySyncURL,
     RemotesOstreeApi,
+    ContentRefsApi,
 )
 
 
@@ -32,6 +33,7 @@ def setUpClass(cls):
         cls.versions_api = RepositoriesOstreeVersionsApi(client_api)
         cls.remotes_api = RemotesOstreeApi(client_api)
         cls.distributions_api = DistributionsOstreeApi(client_api)
+        cls.refs_api = ContentRefsApi(client_api)
 
     @classmethod
     def tearDownClass(cls):
@@ -97,3 +99,48 @@ def sync(self, policy):
         self.addCleanup(shutil.rmtree, remote.name)
         validate_repo_integrity(remote.name, f"{remote_name}:rawhide")
         validate_repo_integrity(remote.name, f"{remote_name}:stable")
+
+    def test_filter_rawhide_ref_sync(self):
+        """Synchronize content from a remote repository considering only a specific ref."""
+        repo = self.repositories_api.create(gen_repo())
+        self.addCleanup(self.repositories_api.delete, repo.pulp_href)
+
+        body = gen_ostree_remote(depth=0, include_refs=["rawhide"], exclude_refs=["stable"])
+        remote = self.remotes_api.create(body)
+        self.addCleanup(self.remotes_api.delete, remote.pulp_href)
+
+        self.assertEqual(remote.include_refs, ["rawhide"])
+        self.assertEqual(remote.exclude_refs, ["stable"])
+
+        repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
+        response = self.repositories_api.sync(repo.pulp_href, repository_sync_data)
+        repo_version = monitor_task(response.task).created_resources[0]
+
+        refs = self.refs_api.list(repository_version_added=repo_version).results
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0].name, "rawhide")
+
+    def test_exclude_all_refs_sync(self):
+        """Synchronize content from a remote repository when a user excludes all refs."""
+        repo = self.repositories_api.create(gen_repo())
+        self.addCleanup(self.repositories_api.delete, repo.pulp_href)
+
+        body = gen_ostree_remote(depth=0, exclude_refs=["rawhide", "stable"])
+        remote = self.remotes_api.create(body)
+        self.addCleanup(self.remotes_api.delete, remote.pulp_href)
+
+        self.assertEqual(remote.include_refs, None)
+        self.assertEqual(set(remote.exclude_refs), {"rawhide", "stable"})
+
+        repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
+        response = self.repositories_api.sync(repo.pulp_href, repository_sync_data)
+        repo_version = monitor_task(response.task).created_resources
+
+        repository_version = self.versions_api.read(repo_version)
+        added_content = repository_version.content_summary.added
+
+        self.assertEqual(added_content["ostree.config"]["count"], 1)
+        self.assertEqual(added_content["ostree.summary"]["count"], 1)
+        self.assertIsNone(added_content["ostree.refs"])
+        self.assertIsNone(added_content["ostree.commit"])
+        self.assertIsNone(added_content["ostree.object"])