Add Flatpak support

Add a new content type for flatpaks, which is inferred from the Koji metadata.
Flatpaks are mashed exactly the same as containers - the content type is
different mostly to get a non-confusing display of content type in the user
interface and to allow future enhancements like Flatpak-specific testing
instructions.

Signed-off-by: Owen W. Taylor <[email protected]>

bodhi/server/buildsys.py

@@ -261,6 +261,16 @@ def getBuild(self, build='TurboGears-1.0.2.2-2.fc17', other=False, testing=False
                 }
                 break
 
+            if token.endswith("flatpak"):
+                tag = "f%s-updates-testing" % token.replace("fc", "")
+                data['extra'] = {
+                    'container_koji_task_id': 21098765,
+                    'image': {
+                        'flatpak': True
+                    },
+                }
+                break
+
             # Hardcoding for modules in the dev buildsys
             if token.startswith("2017"):
                 tag = "f27M-updates-testing"
@@ -374,6 +384,15 @@ def listTags(self, build, *args, **kw):
                 {'arches': 'x86_64', 'id': 17, 'locked': True,
                  'name': 'f28C'},
             ]
+        elif 'flatpak' in build:
+            result = [
+                {'arches': 'x86_64', 'id': 15, 'locked': True,
+                 'name': 'f28F-updates-candidate'},
+                {'arches': 'x86_64', 'id': 16, 'locked': True,
+                 'name': 'f28F-updates-testing'},
+                {'arches': 'x86_64', 'id': 17, 'locked': True,
+                 'name': 'f28F'},
+            ]
         else:
             release = build.split('.')[-1].replace('fc', 'f')
             result = [

bodhi/server/consumers/masher.py

@@ -281,7 +281,8 @@ def get_masher(content_type):
         ComposerThread or None: Either a ContainerComposerThread, RPMComposerThread, or a
             ModuleComposerThread, as appropriate, or None if no masher is found.
     """
-    mashers = [ContainerComposerThread, RPMComposerThread, ModuleComposerThread]
+    mashers = [ContainerComposerThread, FlatpakComposerThread,
+               RPMComposerThread, ModuleComposerThread]
     for possible in mashers:
         if possible.ctype is content_type:
             return possible
@@ -868,6 +869,12 @@ def _compose_updates(self):
                     copy_container(build, destination_tag=dtag)
 
 
+class FlatpakComposerThread(ContainerComposerThread):
+    """Use skopeo to copy and tag flatpak images."""
+
+    ctype = ContentType.flatpak
+
+
 class PungiComposerThread(ComposerThread):
     """Compose update with Pungi."""
 

bodhi/server/migrations/versions/385acbb51075_add_flatpak_enum.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# Copyright © 2018 Red Hat, Inc.
+#
+# This file is part of Bodhi.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+add flatpak enum.
+
+Revision ID: 385acbb51075
+Revises: ebaab70b2cda
+Create Date: 2018-07-17 15:34:38.793304
+"""
+from alembic import op
+from sqlalchemy import exc
+
+
+# revision identifiers, used by Alembic.
+revision = '385acbb51075'
+down_revision = 'ebaab70b2cda'
+
+
+def upgrade():
+    """Add a 'flatpak' content type enum value in ck_content_type."""
+    op.execute('COMMIT')  # See https://bitbucket.org/zzzeek/alembic/issue/123
+    try:
+        # This will raise a ProgrammingError if the DB server doesn't use BDR.
+        op.execute('SHOW bdr.permit_ddl_locking')
+        # This server uses BDR, so let's ask for a DDL lock.
+        op.execute('SET LOCAL bdr.permit_ddl_locking = true')
+    except exc.ProgrammingError:
+        # This server doesn't use BDR, so no problem.
+        pass
+    op.execute("ALTER TYPE ck_content_type ADD VALUE 'flatpak' AFTER 'container'")
+
+
+def downgrade():
+    """
+    Alert user that we cannot downgrade this migration.
+
+    PostgreSQL does not allow enum values to be removed.
+    """
+    raise NotImplementedError("Downgrading this migration is not supported.")

bodhi/server/models.py

@@ -559,12 +559,14 @@ class ContentType(DeclEnum):
         rpm (EnumSymbol): Used to represent RPM related objects.
         module (EnumSymbol): Used to represent Module related objects.
         container (EnumSymbol): Used to represent Container related objects.
+        flatpak (EnumSymbol): Used to represent Flatpak related objects.
     """
 
     base = 'base', 'Base'
     rpm = 'rpm', 'RPM'
     module = 'module', 'Module'
     container = 'container', 'Container'
+    flatpak = 'flatpak', 'Flatpak'
 
     @classmethod
     def infer_content_class(cls, base, build):
@@ -589,7 +591,10 @@ def infer_content_class(cls, base, build):
         if 'module' in extra.get('typeinfo', {}):
             identity = cls.module
         elif 'container_koji_task_id' in extra:
-            identity = cls.container
+            if 'flatpak' in extra['image']:
+                identity = cls.flatpak
+            else:
+                identity = cls.container
 
         return base.find_polymorphic_child(identity)
 
@@ -1077,9 +1082,11 @@ def get_pkg_committers_from_pagure(self):
         """
         pagure_url = config.get('pagure_url')
         # Pagure uses plural names for its namespaces such as "rpms" except for
-        # container
+        # container. Flatpaks build directly from the 'modules' namespace
         if self.type.name == 'container':
             namespace = self.type.name
+        elif self.type.name == 'flatpak':
+            namespace = 'modules'
         else:
             namespace = self.type.name + 's'
         package_pagure_url = '{0}/api/0/{1}/{2}?expand_group=1'.format(
@@ -1249,6 +1256,14 @@ class ContainerPackage(Package):
     }
 
 
+class FlatpakPackage(Package):
+    """Represents a Flatpak package."""
+
+    __mapper_args__ = {
+        'polymorphic_identity': ContentType.flatpak,
+    }
+
+
 class ModulePackage(Package):
     """Represents a Module package."""
 
@@ -1462,6 +1477,18 @@ class ContainerBuild(Build):
     }
 
 
+class FlatpakBuild(Build):
+    """
+    Represents a Flatpak build.
+
+    Note that this model uses single-table inheritance with its Build superclass.
+    """
+
+    __mapper_args__ = {
+        'polymorphic_identity': ContentType.flatpak,
+    }
+
+
 class ModuleBuild(Build):
     """
     Represents a Module build.

bodhi/tests/server/consumers/test_masher.py

@@ -36,13 +36,13 @@
 from bodhi.server import buildsys, exceptions, log, push
 from bodhi.server.config import config
 from bodhi.server.consumers.masher import (
-    checkpoint, Masher, ComposerThread, ContainerComposerThread, RPMComposerThread,
-    ModuleComposerThread, PungiComposerThread)
+    checkpoint, Masher, ComposerThread, ContainerComposerThread, FlatpakComposerThread,
+    RPMComposerThread, ModuleComposerThread, PungiComposerThread)
 from bodhi.server.exceptions import LockedUpdateException
 from bodhi.server.models import (
-    Build, BuildrootOverride, Compose, ComposeState, ContainerBuild, Release, ReleaseState,
-    RpmBuild, TestGatingStatus, Update, UpdateRequest, UpdateStatus, UpdateType, User, ModuleBuild,
-    ContentType, Package)
+    Build, BuildrootOverride, Compose, ComposeState, ContainerBuild, FlatpakBuild,
+    Release, ReleaseState, RpmBuild, TestGatingStatus, Update, UpdateRequest, UpdateStatus,
+    UpdateType, User, ModuleBuild, ContentType, Package)
 from bodhi.server.util import mkmetadatadir
 from bodhi.tests.server import base
 
@@ -2251,6 +2251,73 @@ def test_mash_dir_dne(self):
         self.assertTrue(os.path.exists(mash_dir))
 
 
+class TestFlatpakComposerThread__compose_updates(ComposerThreadBaseTestCase):
+    """Test FlatpakComposerThread._compose_update()."""
+
+    def setUp(self):
+        super(TestFlatpakComposerThread__compose_updates, self).setUp()
+
+        user = self.db.query(User).first()
+        release = self.create_release('28F')
+        release.branch = 'f28'
+        package1 = Package(name=u'testflatpak1',
+                           type=ContentType.flatpak)
+        self.db.add(package1)
+        package2 = Package(name=u'testflatpak2',
+                           type=ContentType.flatpak)
+        self.db.add(package2)
+        build1 = FlatpakBuild(nvr=u'testflatpak1-2.0.1-71.fc28', release=release, signed=True,
+                              package=package1)
+        self.db.add(build1)
+        build2 = FlatpakBuild(nvr=u'testflatpak2-1.0.1-1.fc28', release=release, signed=True,
+                              package=package2)
+        self.db.add(build2)
+        update = Update(
+            title=u'testflatpak1-2.0.1-71.fc28, testflatpak2-1.0.1-1.fc28',
+            builds=[build1, build2], user=user,
+            status=UpdateStatus.pending,
+            request=UpdateRequest.testing,
+            notes=u'Neat I can compose flatpaks now', release=release,
+            test_gating_status=TestGatingStatus.passed)
+        update.type = UpdateType.bugfix
+        self.db.add(update)
+        # Wipe out the tag cache so it picks up our new release
+        Release._tag_cache = None
+        self.db.flush()
+
+    @mock.patch('bodhi.server.consumers.masher.subprocess.Popen')
+    def test_flatpak_compose(self, Popen):
+        """
+        Basic test that FlatpakComposerThread does the expected thing.
+
+        We don't need extensive sets of tests since FlatpakComposerThread inherits
+        all code from ContainerComposerThread.
+        """
+        Popen.return_value.communicate.return_value = ('out', 'err')
+        Popen.return_value.returncode = 0
+        msg = self._make_msg(['--releases', 'F28F'])
+        t = FlatpakComposerThread(self.semmock, msg['body']['msg']['composes'][0],
+                                  'otaylor', log, self.Session, self.tempdir)
+        t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0])
+
+        t._compose_updates()
+
+        # Popen should have been called three times per build, once for each of the destination
+        # tags. With two builds that is a total of 6 calls to Popen.
+        expected_mock_calls = []
+        for source in ('testflatpak1:2.0.1-71.fc28', 'testflatpak2:1.0.1-1.fc28'):
+            for dtag in [source.split(':')[1], source.split(':')[1].split('-')[0], 'testing']:
+                mock_call = mock.call(
+                    [config['skopeo.cmd'], 'copy',
+                     'docker://{}/f28/{}'.format(config['container.source_registry'], source),
+                     'docker://{}/f28/{}:{}'.format(config['container.destination_registry'],
+                                                    source.split(':')[0], dtag)],
+                    shell=False, stderr=-1, stdout=-1, cwd=None)
+                expected_mock_calls.append(mock_call)
+                expected_mock_calls.append(mock.call().communicate())
+        self.assertEqual(Popen.mock_calls, expected_mock_calls)
+
+
 class TestPungiComposerThread__get_master_repomd_url(ComposerThreadBaseTestCase):
     """This class contains tests for the PungiComposerThread._get_master_repomd_url() method."""
     @mock.patch.dict(

bodhi/tests/server/services/test_updates.py

@@ -464,6 +464,38 @@ def test_new_container_update(self, publish, *args):
         publish.assert_called_once_with(
             topic='update.request.testing', msg=mock.ANY)
 
+    @mock.patch(**mock_uuid4_version1)
+    @mock.patch(**mock_valid_requirements)
+    @mock.patch('bodhi.server.notifications.publish')
+    def test_new_flatpak_update(self, publish, *args):
+        self.create_release(u'28F')
+        data = self.get_update('mariadb-10.1-10.f28flatpak')
+
+        r = self.app.post_json('/updates/', data, status=200)
+
+        up = r.json_body
+        self.assertEquals(up['title'], u'mariadb-10.1-10.f28flatpak')
+        self.assertEquals(up['status'], u'pending')
+        self.assertEquals(up['request'], u'testing')
+        self.assertEquals(up['user']['name'], u'guest')
+        self.assertEquals(up['release']['name'], u'F28F')
+        self.assertEquals(up['type'], u'bugfix')
+        self.assertEquals(up['content_type'], u'flatpak')
+        self.assertEquals(up['severity'], u'unspecified')
+        self.assertEquals(up['suggest'], u'unspecified')
+        self.assertEquals(up['close_bugs'], True)
+        self.assertEquals(up['notes'], u'this is a test update')
+        self.assertIsNotNone(up['date_submitted'])
+        self.assertEquals(up['date_modified'], None)
+        self.assertEquals(up['date_approved'], None)
+        self.assertEquals(up['date_pushed'], None)
+        self.assertEquals(up['locked'], False)
+        self.assertEquals(up['alias'], u'FEDORA-%s-033713b73b' % YEAR)
+        self.assertEquals(up['karma'], 0)
+        self.assertEquals(up['requirements'], 'rpmlint')
+        publish.assert_called_once_with(
+            topic='update.request.testing', msg=mock.ANY)
+
     @mock.patch.dict('bodhi.server.validators.config', {'acl_system': u'dummy'})
     @mock.patch(**mock_valid_requirements)
     @mock.patch('bodhi.server.notifications.publish')

bodhi/tests/server/test_models.py

@@ -1033,6 +1033,57 @@ def test_get_pkg_committers_from_pagure(self, http_session):
             timeout=60)
 
 
+class TestFlatpakPackage(ModelTest, unittest.TestCase):
+    klass = model.FlatpakPackage
+    attrs = dict(name=u"flatpak-runtime")
+
+    @mock.patch('bodhi.server.util.http_session')
+    def test_get_pkg_committers_from_pagure(self, http_session):
+        """Ensure correct return value from get_pkg_committers_from_pagure()."""
+        json_output = {
+            "access_groups": {
+                "admin": [],
+                "commit": [],
+                "ticket": []
+            },
+            "access_users": {
+                "admin": [],
+                "commit": [],
+                "owner": [
+                    "otaylor"
+                ],
+                "ticket": []
+            },
+            "close_status": [],
+            "custom_keys": [],
+            "date_created": "1494947106",
+            "description": "Flatpak Runtime",
+            "fullname": "modules/flatpak-runtime",
+            "group_details": {},
+            "id": 2,
+            "milestones": {},
+            "name": "python",
+            "namespace": "rpms",
+            "parent": None,
+            "priorities": {},
+            "tags": [],
+            "user": {
+                "fullname": "Owen Taylor",
+                "name": "otaylor"
+            }
+        }
+        http_session.get.return_value.json.return_value = json_output
+        http_session.get.return_value.status_code = 200
+
+        rv = self.obj.get_pkg_committers_from_pagure()
+
+        self.assertEqual(rv, (['otaylor'], []))
+        http_session.get.assert_called_once_with(
+            ('https://src.fedoraproject.org/pagure/api/0/modules/flatpak-runtime'
+             '?expand_group=1'),
+            timeout=60)
+
+
 class TestRpmPackage(ModelTest, unittest.TestCase):
     """Unit test case for the ``RpmPackage`` model."""
     klass = model.RpmPackage