Adds existing arch/os information to searches and index operations

This code is a first step in making the current registry aware of
images which are non-Intel architecture.  While os==linux today, if
we are adding architecture we might as well add the pair of arch and
os for future use by Microsoft where os=windows but arch=amd64.

Signed-off-by: Pradipta Kr. Banerjee <[email protected]>
Signed-off-by: Phil Estes <[email protected]>

docker_registry/index.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+import re
+
 import flask
 
 from docker_registry.core import compat
@@ -13,6 +15,7 @@
 
 from .app import app  # noqa
 
+RE_USER_AGENT = re.compile('([^\s/]+)/([^\s/]+)')
 
 store = storage.load()
 
@@ -51,7 +54,7 @@ def put_username(username):
     return toolkit.response('', 204)
 
 
-def update_index_images(namespace, repository, data_arg):
+def update_index_images(namespace, repository, data_arg, arch, os):
     path = store.index_images_path(namespace, repository)
     sender = flask.current_app._get_current_object()
     try:
@@ -70,13 +73,20 @@ def update_index_images(namespace, repository, data_arg):
         data = images.values()
         # Note(dmp): unicode patch
         store.put_json(path, data)
+
+        # Get image arch and os from the json property
+        img_data = store.get_content(store.image_json_path(data[0]['id']))
+        arch = json.loads(img_data)['architecture']
+        os = json.loads(img_data)['os']
+
         signals.repository_updated.send(
-            sender, namespace=namespace, repository=repository, value=data)
+            sender, namespace=namespace, repository=repository,
+            value=data, arch=arch, os=os)
     except exceptions.FileNotFoundError:
         signals.repository_created.send(
             sender, namespace=namespace, repository=repository,
             # Note(dmp): unicode patch
-            value=json.loads(data_arg.decode('utf8')))
+            value=json.loads(data_arg.decode('utf8')), arch=arch, os=os)
         store.put_content(path, data_arg)
 
 
@@ -88,14 +98,25 @@ def update_index_images(namespace, repository, data_arg):
 @toolkit.requires_auth
 def put_repository(namespace, repository, images=False):
     data = None
+    # Default arch/os are amd64/linux
+    arch = 'amd64'
+    os = 'linux'
+    # If caller is docker host, retrieve arch/os from user agent
+    user_agent = flask.request.headers.get('user-agent', '')
+    ua = dict(RE_USER_AGENT.findall(user_agent))
+    if 'arch' in ua:
+        arch = ua['arch']
+    if 'os' in ua:
+        os = ua['os']
+
     try:
         # Note(dmp): unicode patch
         data = json.loads(flask.request.data.decode('utf8'))
     except ValueError:
         return toolkit.api_error('Error Decoding JSON', 400)
     if not isinstance(data, list):
         return toolkit.api_error('Invalid data')
-    update_index_images(namespace, repository, flask.request.data)
+    update_index_images(namespace, repository, flask.request.data, arch, os)
     headers = generate_headers(namespace, repository, 'write')
     code = 204 if images is True else 200
     return toolkit.response('', code, headers)
@@ -107,9 +128,37 @@ def put_repository(namespace, repository, images=False):
 @mirroring.source_lookup(index_route=True)
 def get_repository_images(namespace, repository):
     data = None
+    # Default arch/os are amd64/linux
+    arch = 'amd64'
+    os = 'linux'
+    # If caller is docker host, retrieve arch/os from user agent
+    user_agent = flask.request.headers.get('user-agent', '')
+    ua = dict(RE_USER_AGENT.findall(user_agent))
+    if 'arch' in ua:
+        arch = ua['arch']
+    if 'os' in ua:
+        os = ua['os']
+
     try:
         path = store.index_images_path(namespace, repository)
-        data = store.get_content(path)
+        json_data = store.get_json(path)
+        # we may not have image data (mocked up tests, etc.) so try/except
+        # on parsing arch and os--and ignore/continue if this is an image
+        # with no data
+        try:
+            img_data = store.get_content(store.image_json_path(
+                                         json_data[0]['id']))
+            # Get image arch and os from the json property
+            img_arch = json.loads(img_data)['architecture']
+            img_os = json.loads(img_data)['os']
+            if arch != img_arch or os != img_os:
+                return toolkit.api_error('images not found for arch/os pair',
+                                         404)
+            else:
+                data = store.get_content(path)
+        except exceptions.FileNotFoundError:
+            # simply return the data if image data does not exist
+            data = store.get_content(path)
     except exceptions.FileNotFoundError:
         return toolkit.api_error('images not found', 404)
     headers = generate_headers(namespace, repository, 'read')

docker_registry/lib/index/__init__.py

@@ -52,17 +52,17 @@ def _walk_storage(self, store):
                 yield({'name': name, 'description': description})
 
     def _handle_repository_created(
-            self, sender, namespace, repository, value):
+            self, sender, namespace, repository, value, arch, os):
         pass
 
     def _handle_repository_updated(
-            self, sender, namespace, repository, value):
+            self, sender, namespace, repository, value, arch, os):
         pass
 
     def _handle_repository_deleted(self, sender, namespace, repository):
         pass
 
-    def results(self, search_term=None):
+    def results(self, search_term=None, arch=None, os=None):
         """Return a list of results matching search_term
 
         The list elements should be dictionaries:

docker_registry/lib/index/db.py

@@ -42,10 +42,16 @@ class Repository (Base):
         nullable=False, unique=True)
     description = sqlalchemy.Column(
         sqlalchemy.String(length=100))
+    arch = sqlalchemy.Column(
+        sqlalchemy.String(length=10))
+    os = sqlalchemy.Column(
+        sqlalchemy.String(length=10))
 
     def __repr__(self):
-        return "<{0}(name='{1}', description='{2}')>".format(
-            type(self).__name__, self.name, self.description)
+        rep_str = ("<{0}(name='{1}', description='{2}', arch='{3}', "
+                   "os='{4}')>")
+        return rep_str.format(type(self).__name__, self.name,
+                              self.description, self.arch, self.os)
 
 
 def retry(f):
@@ -117,24 +123,25 @@ def _generate_index(self, session):
 
     @retry
     def _handle_repository_created(
-            self, sender, namespace, repository, value):
+            self, sender, namespace, repository, value, arch, os):
         name = '{0}/{1}'.format(namespace, repository)
         description = ''  # TODO(wking): store descriptions
         session = self._session()
-        session.add(Repository(name=name, description=description))
+        session.add(Repository(name=name, description=description, arch=arch,
+                    os=os))
         session.commit()
         session.close()
 
     @retry
     def _handle_repository_updated(
-            self, sender, namespace, repository, value):
+            self, sender, namespace, repository, value, arch, os):
         name = '{0}/{1}'.format(namespace, repository)
         description = ''  # TODO(wking): store descriptions
         session = self._session()
         session.query(Repository).filter(
             Repository.name == name
         ).update(
-            values={'description': description},
+            values={'description': description, 'arch': arch, 'os': os},
             synchronize_session=False
         )
         session.commit()
@@ -149,19 +156,25 @@ def _handle_repository_deleted(self, sender, namespace, repository):
         session.close()
 
     @retry
-    def results(self, search_term=None):
+    def results(self, search_term=None, arch=None, os=None):
         session = self._session()
         repositories = session.query(Repository)
         if search_term:
             like_term = '%%%s%%' % search_term
             repositories = repositories.filter(
                 sqlalchemy.sql.or_(
                     Repository.name.like(like_term),
-                    Repository.description.like(like_term)))
+                    Repository.description.like(like_term)),
+                sqlalchemy.sql.and_(
+                    Repository.arch.like(arch)),
+                sqlalchemy.sql.and_(
+                    Repository.os.like(os)))
         results = [
             {
                 'name': repo.name,
                 'description': repo.description,
+                'arch': repo.arch,
+                'os': repo.os,
             }
             for repo in repositories]
         session.close()

docker_registry/search.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import re
 
 from . import toolkit
 from .app import app
@@ -7,18 +8,30 @@
 from .lib import mirroring
 import flask
 
-
 cfg = config.load()
 
 # Enable the search index
 INDEX = index.load(cfg.search_backend.lower())
 
+RE_USER_AGENT = re.compile('([^\s/]+)/([^\s/]+)')
+
 
 @app.route('/v1/search', methods=['GET'])
 @mirroring.source_lookup(index_route=True, merge_results=True)
 def get_search():
+    # default to standard 64-bit linux, then check UA for
+    # specific arch/os (if coming from a docker host)
+    arch = 'amd64'
+    os = 'linux'
+    user_agent = flask.request.headers.get('user-agent', '')
+    ua = dict(RE_USER_AGENT.findall(user_agent))
+    if 'arch' in ua:
+        arch = ua['arch']
+    if 'os' in ua:
+        os = ua['os']
+
     search_term = flask.request.args.get('q', '')
-    results = INDEX.results(search_term=search_term)
+    results = INDEX.results(search_term=search_term, arch=arch, os=os)
     return toolkit.response({
         'query': search_term,
         'num_results': len(results),

tests/lib/index/test__init__.py

@@ -11,8 +11,10 @@ def setUp(self):
         self.index = index.Index()
 
     def test_cover_passed_methods(self):
-        self.index._handle_repository_created(None, None, None, None)
-        self.index._handle_repository_updated(None, None, None, None)
+        self.index._handle_repository_created(None, None, None, None, None,
+                                              None)
+        self.index._handle_repository_updated(None, None, None, None, None,
+                                              None)
         self.index._handle_repository_deleted(None, None, None)
 
     def test_results(self):

tests/test_index.py

@@ -28,12 +28,15 @@ def test_users(self):
 
     def test_repository_images(self):
         repo = 'test/{0}'.format(self.gen_random_string())
+        # repo2 used for PUT no Images test below so name clash is avoided
+        repo2 = 'test/{0}'.format(self.gen_random_string())
         images = [{'id': self.gen_random_string()},
                   {'id': self.gen_random_string()}]
-        # PUT
-        resp = self.http_client.put('/v1/repositories/{0}/'.format(repo),
+        # PUT no Images
+        resp = self.http_client.put('/v1/repositories/{0}/'.format(repo2),
                                     data=json.dumps(images))
         self.assertEqual(resp.status_code, 200, resp.data)
+        # PUT
         resp = self.http_client.put('/v1/repositories/{0}/images'.format(repo),
                                     data=json.dumps(images))
         self.assertEqual(resp.status_code, 204, resp.data)
@@ -44,10 +47,14 @@ def test_repository_images(self):
         data = json.loads(resp.data)
         self.assertEqual(len(data), 2)
         self.assertTrue('id' in data[0])
+
         # DELETE
         resp = self.http_client.delete('/v1/repositories/{0}/images'.format(
             repo))
         self.assertEqual(resp.status_code, 204, resp.data)
+        resp = self.http_client.delete('/v1/repositories/{0}/images'.format(
+            repo2))
+        self.assertEqual(resp.status_code, 204, resp.data)
 
     def test_auth(self):
         repo = 'test/{0}'.format(self.gen_random_string())