From b875ea6222809ea4434d0c3b3f397249a19656aa Mon Sep 17 00:00:00 2001 From: Phil Estes Date: Fri, 12 Dec 2014 16:26:30 -0500 Subject: [PATCH] Adds existing arch/os information to searches and index operations This code is a first step in making the current registry aware of images which are non-Intel architecture. While os==linux today, if we are adding architecture we might as well add the pair of arch and os for future use by Microsoft where os=windows but arch=amd64. Signed-off-by: Pradipta Kr. Banerjee Signed-off-by: Phil Estes --- docker_registry/index.py | 59 ++++++++++++++++++++++++--- docker_registry/lib/index/__init__.py | 6 +-- docker_registry/lib/index/db.py | 29 +++++++++---- docker_registry/search.py | 17 +++++++- tests/lib/index/test__init__.py | 6 ++- tests/test_index.py | 11 ++++- 6 files changed, 106 insertions(+), 22 deletions(-) diff --git a/docker_registry/index.py b/docker_registry/index.py index 902a629ca..c73339492 100644 --- a/docker_registry/index.py +++ b/docker_registry/index.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import re + import flask from docker_registry.core import compat @@ -13,6 +15,7 @@ from .app import app # noqa +RE_USER_AGENT = re.compile('([^\s/]+)/([^\s/]+)') store = storage.load() @@ -51,7 +54,7 @@ def put_username(username): return toolkit.response('', 204) -def update_index_images(namespace, repository, data_arg): +def update_index_images(namespace, repository, data_arg, arch, os): path = store.index_images_path(namespace, repository) sender = flask.current_app._get_current_object() try: @@ -70,13 +73,20 @@ def update_index_images(namespace, repository, data_arg): data = images.values() # Note(dmp): unicode patch store.put_json(path, data) + + # Get image arch and os from the json property + img_data = store.get_content(store.image_json_path(data[0]['id'])) + arch = json.loads(img_data)['architecture'] + os = json.loads(img_data)['os'] + signals.repository_updated.send( - sender, namespace=namespace, repository=repository, value=data) + sender, namespace=namespace, repository=repository, + value=data, arch=arch, os=os) except exceptions.FileNotFoundError: signals.repository_created.send( sender, namespace=namespace, repository=repository, # Note(dmp): unicode patch - value=json.loads(data_arg.decode('utf8'))) + value=json.loads(data_arg.decode('utf8')), arch=arch, os=os) store.put_content(path, data_arg) @@ -88,6 +98,17 @@ def update_index_images(namespace, repository, data_arg): @toolkit.requires_auth def put_repository(namespace, repository, images=False): data = None + # Default arch/os are amd64/linux + arch = 'amd64' + os = 'linux' + # If caller is docker host, retrieve arch/os from user agent + user_agent = flask.request.headers.get('user-agent', '') + ua = dict(RE_USER_AGENT.findall(user_agent)) + if 'arch' in ua: + arch = ua['arch'] + if 'os' in ua: + os = ua['os'] + try: # Note(dmp): unicode patch data = json.loads(flask.request.data.decode('utf8')) @@ -95,7 +116,7 @@ def put_repository(namespace, repository, images=False): return toolkit.api_error('Error Decoding JSON', 400) if not isinstance(data, list): return toolkit.api_error('Invalid data') - update_index_images(namespace, repository, flask.request.data) + update_index_images(namespace, repository, flask.request.data, arch, os) headers = generate_headers(namespace, repository, 'write') code = 204 if images is True else 200 return toolkit.response('', code, headers) @@ -107,9 +128,37 @@ def put_repository(namespace, repository, images=False): @mirroring.source_lookup(index_route=True) def get_repository_images(namespace, repository): data = None + # Default arch/os are amd64/linux + arch = 'amd64' + os = 'linux' + # If caller is docker host, retrieve arch/os from user agent + user_agent = flask.request.headers.get('user-agent', '') + ua = dict(RE_USER_AGENT.findall(user_agent)) + if 'arch' in ua: + arch = ua['arch'] + if 'os' in ua: + os = ua['os'] + try: path = store.index_images_path(namespace, repository) - data = store.get_content(path) + json_data = store.get_json(path) + # we may not have image data (mocked up tests, etc.) so try/except + # on parsing arch and os--and ignore/continue if this is an image + # with no data + try: + img_data = store.get_content(store.image_json_path( + json_data[0]['id'])) + # Get image arch and os from the json property + img_arch = json.loads(img_data)['architecture'] + img_os = json.loads(img_data)['os'] + if arch != img_arch or os != img_os: + return toolkit.api_error('images not found for arch/os pair', + 404) + else: + data = store.get_content(path) + except exceptions.FileNotFoundError: + # simply return the data if image data does not exist + data = store.get_content(path) except exceptions.FileNotFoundError: return toolkit.api_error('images not found', 404) headers = generate_headers(namespace, repository, 'read') diff --git a/docker_registry/lib/index/__init__.py b/docker_registry/lib/index/__init__.py index 901a1277e..b8560ef4c 100644 --- a/docker_registry/lib/index/__init__.py +++ b/docker_registry/lib/index/__init__.py @@ -52,17 +52,17 @@ def _walk_storage(self, store): yield({'name': name, 'description': description}) def _handle_repository_created( - self, sender, namespace, repository, value): + self, sender, namespace, repository, value, arch, os): pass def _handle_repository_updated( - self, sender, namespace, repository, value): + self, sender, namespace, repository, value, arch, os): pass def _handle_repository_deleted(self, sender, namespace, repository): pass - def results(self, search_term=None): + def results(self, search_term=None, arch=None, os=None): """Return a list of results matching search_term The list elements should be dictionaries: diff --git a/docker_registry/lib/index/db.py b/docker_registry/lib/index/db.py index e9d4c68d2..93fda13c8 100644 --- a/docker_registry/lib/index/db.py +++ b/docker_registry/lib/index/db.py @@ -42,10 +42,16 @@ class Repository (Base): nullable=False, unique=True) description = sqlalchemy.Column( sqlalchemy.String(length=100)) + arch = sqlalchemy.Column( + sqlalchemy.String(length=10)) + os = sqlalchemy.Column( + sqlalchemy.String(length=10)) def __repr__(self): - return "<{0}(name='{1}', description='{2}')>".format( - type(self).__name__, self.name, self.description) + rep_str = ("<{0}(name='{1}', description='{2}', arch='{3}', " + "os='{4}')>") + return rep_str.format(type(self).__name__, self.name, + self.description, self.arch, self.os) def retry(f): @@ -117,24 +123,25 @@ def _generate_index(self, session): @retry def _handle_repository_created( - self, sender, namespace, repository, value): + self, sender, namespace, repository, value, arch, os): name = '{0}/{1}'.format(namespace, repository) description = '' # TODO(wking): store descriptions session = self._session() - session.add(Repository(name=name, description=description)) + session.add(Repository(name=name, description=description, arch=arch, + os=os)) session.commit() session.close() @retry def _handle_repository_updated( - self, sender, namespace, repository, value): + self, sender, namespace, repository, value, arch, os): name = '{0}/{1}'.format(namespace, repository) description = '' # TODO(wking): store descriptions session = self._session() session.query(Repository).filter( Repository.name == name ).update( - values={'description': description}, + values={'description': description, 'arch': arch, 'os': os}, synchronize_session=False ) session.commit() @@ -149,7 +156,7 @@ def _handle_repository_deleted(self, sender, namespace, repository): session.close() @retry - def results(self, search_term=None): + def results(self, search_term=None, arch=None, os=None): session = self._session() repositories = session.query(Repository) if search_term: @@ -157,11 +164,17 @@ def results(self, search_term=None): repositories = repositories.filter( sqlalchemy.sql.or_( Repository.name.like(like_term), - Repository.description.like(like_term))) + Repository.description.like(like_term)), + sqlalchemy.sql.and_( + Repository.arch.like(arch)), + sqlalchemy.sql.and_( + Repository.os.like(os))) results = [ { 'name': repo.name, 'description': repo.description, + 'arch': repo.arch, + 'os': repo.os, } for repo in repositories] session.close() diff --git a/docker_registry/search.py b/docker_registry/search.py index 218a62acf..638ca34da 100644 --- a/docker_registry/search.py +++ b/docker_registry/search.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import re from . import toolkit from .app import app @@ -7,18 +8,30 @@ from .lib import mirroring import flask - cfg = config.load() # Enable the search index INDEX = index.load(cfg.search_backend.lower()) +RE_USER_AGENT = re.compile('([^\s/]+)/([^\s/]+)') + @app.route('/v1/search', methods=['GET']) @mirroring.source_lookup(index_route=True, merge_results=True) def get_search(): + # default to standard 64-bit linux, then check UA for + # specific arch/os (if coming from a docker host) + arch = 'amd64' + os = 'linux' + user_agent = flask.request.headers.get('user-agent', '') + ua = dict(RE_USER_AGENT.findall(user_agent)) + if 'arch' in ua: + arch = ua['arch'] + if 'os' in ua: + os = ua['os'] + search_term = flask.request.args.get('q', '') - results = INDEX.results(search_term=search_term) + results = INDEX.results(search_term=search_term, arch=arch, os=os) return toolkit.response({ 'query': search_term, 'num_results': len(results), diff --git a/tests/lib/index/test__init__.py b/tests/lib/index/test__init__.py index b76077195..04fdcf685 100644 --- a/tests/lib/index/test__init__.py +++ b/tests/lib/index/test__init__.py @@ -11,8 +11,10 @@ def setUp(self): self.index = index.Index() def test_cover_passed_methods(self): - self.index._handle_repository_created(None, None, None, None) - self.index._handle_repository_updated(None, None, None, None) + self.index._handle_repository_created(None, None, None, None, None, + None) + self.index._handle_repository_updated(None, None, None, None, None, + None) self.index._handle_repository_deleted(None, None, None) def test_results(self): diff --git a/tests/test_index.py b/tests/test_index.py index 511666e5a..d8e598637 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -28,12 +28,15 @@ def test_users(self): def test_repository_images(self): repo = 'test/{0}'.format(self.gen_random_string()) + # repo2 used for PUT no Images test below so name clash is avoided + repo2 = 'test/{0}'.format(self.gen_random_string()) images = [{'id': self.gen_random_string()}, {'id': self.gen_random_string()}] - # PUT - resp = self.http_client.put('/v1/repositories/{0}/'.format(repo), + # PUT no Images + resp = self.http_client.put('/v1/repositories/{0}/'.format(repo2), data=json.dumps(images)) self.assertEqual(resp.status_code, 200, resp.data) + # PUT resp = self.http_client.put('/v1/repositories/{0}/images'.format(repo), data=json.dumps(images)) self.assertEqual(resp.status_code, 204, resp.data) @@ -44,10 +47,14 @@ def test_repository_images(self): data = json.loads(resp.data) self.assertEqual(len(data), 2) self.assertTrue('id' in data[0]) + # DELETE resp = self.http_client.delete('/v1/repositories/{0}/images'.format( repo)) self.assertEqual(resp.status_code, 204, resp.data) + resp = self.http_client.delete('/v1/repositories/{0}/images'.format( + repo2)) + self.assertEqual(resp.status_code, 204, resp.data) def test_auth(self): repo = 'test/{0}'.format(self.gen_random_string())