Atomic/install.py: Record installs for later use

When installing an image, we now write a small bit of json to /var/lib/atomic/install.json. The json format is: { <image_name>: { id: <image_id>, install_date: <install_date_in_utc } } This will be used in update, run, etc to ensure that any image with an INSTALL label is first installed.
projectatomic · Mar 24, 2017 · 2ffaeb5 · 2ffaeb5
1 parent 6e1fe7a
commit 2ffaeb5
Show file tree

Hide file tree

Showing 10 changed files with 210 additions and 7 deletions.
diff --git a/Atomic/backends/_docker.py b/Atomic/backends/_docker.py
@@ -324,7 +324,9 @@ def delete_image(self, image, force=False):
         assert(image is not None)
         try:
             return self.d.remove_image(image, force=force)
-        except errors.NotFound:
+        except errors.APIError as e:
+            raise ValueError(str(e))
+        except errors.NotFound: # pylint: disable=bad-except-order
             pass
         except HTTPError:
             pass
@@ -364,6 +366,7 @@ def install(self, image, name, **kwargs):
 
     def uninstall(self, iobject, name=None, **kwargs):
         atomic = kwargs.get('atomic')
+        ignore = kwargs.get('ignore')
         assert(isinstance(atomic, Atomic))
         args = atomic.args
         con_obj = None if not name else self.has_container(name)
@@ -395,8 +398,12 @@ def uninstall(self, iobject, name=None, **kwargs):
             return 0
         if cmd:
             return util.check_call(cmd, env=atomic.cmd_env())
+
+        # Delete the entry in the install data
+        util.InstallData.delete_by_id(iobject.id, ignore=ignore)
         return self.delete_image(iobject.image, force=args.force)
 
+
     def validate_layer(self, layer):
         pass
 
@@ -450,6 +457,10 @@ def add_string_or_list_to_list(list_item, value):
             else:
                 return self._start(iobject, args, atomic)
 
+        if iobject.get_label('INSTALL') and not args.ignore and not util.InstallData.image_installed(iobject):
+            raise ValueError("The image '{}' appears to have not been installed and has an INSTALL label.  You "
+                             "should install this image first.  Re-run with --ignore to bypass this "
+                             "error.".format(iobject.name or iobject.image))
         # The object is an image
         command = []
         if iobject.run_command:

diff --git a/Atomic/install.py b/Atomic/install.py
@@ -4,6 +4,8 @@
 from .util import add_opt
 from .syscontainers import OSTREE_PRESENT
 from Atomic.backendutils import BackendUtils
+from Atomic.discovery import RegistryInspectError
+from time import gmtime, strftime
 
 try:
     from . import Atomic
@@ -129,6 +131,15 @@ def install(self):
         self.display(cmd)
 
         if not self.args.display:
+            install_data = util.InstallData.read_install_data()
+            try:
+                name = img_obj.fq_name
+            except RegistryInspectError:
+                name = img_obj.input_name
+            install_data[name] = {'id': img_obj.id,
+                                  'install_date': strftime("%Y-%m-%d %H:%M:%S", gmtime())
+                                  }
+            util.InstallData.write_install_data(install_data)
             return util.check_call(cmd)
 
     @staticmethod

diff --git a/Atomic/uninstall.py b/Atomic/uninstall.py
@@ -59,7 +59,7 @@ def uninstall(self):
                 if not img_obj:
                     raise ValueError(e)
                 be = ost
-        be.uninstall(img_obj, name=self.args.name, atomic=self)
+        be.uninstall(img_obj, name=self.args.name, atomic=self, ignore=self.args.ignore)
         return 0
 
 

diff --git a/Atomic/util.py b/Atomic/util.py
@@ -21,6 +21,9 @@
 import ipaddress
 import socket
 from Atomic.backends._docker_errors import NoDockerDaemon
+import fcntl
+import time
+
 # Atomic Utility Module
 
 ReturnTuple = collections.namedtuple('ReturnTuple',
@@ -29,6 +32,7 @@
 ATOMIC_CONFD = os.environ.get('ATOMIC_CONFD', '/etc/atomic.d/')
 ATOMIC_LIBEXEC = os.environ.get('ATOMIC_LIBEXEC', '/usr/libexec/atomic')
 ATOMIC_VAR_LIB = os.environ.get('ATOMIC_VAR_LIB', '/var/lib/atomic')
+ATOMIC_INSTALL_JSON = os.environ.get('ATOMIC_INSTALL_JSON', os.path.join(ATOMIC_VAR_LIB, 'install.json'))
 
 GOMTREE_PATH = "/usr/bin/gomtree"
 BWRAP_OCI_PATH = "/usr/bin/bwrap-oci"
@@ -794,6 +798,67 @@ def load_scan_result_file(file_name):
     """
     return json.loads(open(os.path.join(file_name), "r").read())
 
+
+class InstallData(object):
+
+    @staticmethod
+    def read_install_data():
+        if os.path.exists(ATOMIC_INSTALL_JSON):
+            return json.loads(open(os.path.join(ATOMIC_INSTALL_JSON), "r").read())
+        return {}
+
+    @staticmethod
+    def write_install_data(install_data):
+        with open(ATOMIC_INSTALL_JSON, 'w') as write_data:
+            time_out = 0
+            f_lock = False
+            while time_out < 10.5: # Ten second attempt to get a lock
+                try:
+                    fcntl.flock(write_data, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    f_lock = True
+                    break
+                except IOError:
+                    time.sleep(.5)
+                    time_out += .5
+            if not f_lock:
+                raise ValueError("Unable to get file lock for {}".format(ATOMIC_INSTALL_JSON))
+            json.dump(install_data, write_data)
+            fcntl.flock(write_data, fcntl.LOCK_UN)
+
+    @classmethod
+    def get_install_name_by_id(cls, iid, install_data=None):
+        if not install_data:
+            install_data = cls.read_install_data()
+        for installed_image in install_data:
+            if install_data[installed_image]['id'] == iid:
+                return installed_image
+        raise ValueError("Unable to find {} in installed image data ({}). Re-run command with -i to ignore".format(id, ATOMIC_INSTALL_JSON))
+
+    @classmethod
+    def delete_by_id(cls, iid, ignore=False):
+        install_data = cls.read_install_data()
+        try:
+            id_key = InstallData.get_install_name_by_id(iid, install_data=install_data)
+        except ValueError as e:
+            if not ignore:
+                raise ValueError(str(e))
+            return
+        del install_data[id_key]
+        return InstallData.write_install_data(install_data)
+
+    @classmethod
+    def image_installed(cls, img_object):
+        install_data = cls.read_install_data()
+        if install_data.get(img_object.id, None):
+            return True
+        if install_data.get(img_object.input_name, None):
+            return True
+        if install_data.get(img_object.name, None):
+            return True
+        if install_data.get(img_object.image, None):
+            return True
+        return False
+
 class Decompose(object):
     """
     Class for decomposing an input string in its respective parts like registry,

diff --git a/atomic b/atomic
@@ -124,6 +124,8 @@ def create_parser(help_text):
                         help=_("show atomic version and exit"))
     parser.add_argument('--debug', default=False, action='store_true',
                         help=_("show debug messages"))
+    parser.add_argument('-i', '--ignore', default=False, action='store_true',
+                        help=_("allows you to ignore certain errors"))
     parser.add_argument('-y', '--assumeyes', default=False, action='store_true',
                         help=_("automatically answer yes for all questions"))
     subparser = parser.add_subparsers(help=_("commands"))

diff --git a/atomic_dbus.py b/atomic_dbus.py
@@ -65,6 +65,7 @@ def __init__(self):
             self.graph = None
             self.heading = False
             self.hotfix = False
+            self.ignore = False
             self.image = None
             self.images = []
             self.import_location = None
@@ -392,15 +393,16 @@ def UnmountImage(self, mountpoint):
     # The Run method will run the specified image
     @slip.dbus.polkit.require_auth("org.atomic.readwrite")
     # Return a 0 or 1 for success.  Errors result in exceptions.
-    @dbus.service.method("org.atomic", in_signature='ssbbas', out_signature='i')
-    def Run(self, image, name='', spc=False, detach=False, command=''):
+    @dbus.service.method("org.atomic", in_signature='ssbbbas', out_signature='i')
+    def Run(self, image, name='', spc=False, detach=False, ignore=False, command=''):
         r = Run()
         args = self.Args()
         args.image = image
         args.name = name if name is not '' else None
         args.spc = spc
         args.detach = detach
         args.command = command if command is not '' else []
+        args.ignore = ignore
         r.set_args(args)
         try:
             return r.run()

diff --git a/bash/atomic b/bash/atomic
@@ -43,7 +43,7 @@ __atomic_scanners() {
 
 _atomic_atomic() {
 	local boolean_options="
-		--help -h --version -v --debug --assumeyes -y
+		--help -h --version -v --debug --assumeyes -y -i --ignore
 	"
 	case "$prev" in
 		$main_options_with_args_glob )

diff --git a/test.sh b/test.sh
@@ -142,6 +142,11 @@ if [ ! -n "${PYTHON+ }" ]; then
     fi
 fi
 
+# Add images with INSTALL labels to /var/lib/atomic/install.json
+INSTALL_DATA_FILE="$(pwd)/install.json"
+INSTALL_DATA=`docker images --no-trunc | awk '/atomic-test-/ {printf "\"%s\": {\"install_id\": \"%s\"},\n", $1, $3}' | sed 's/sha256://g' | sed '$ s/,$//'`
+echo "{$INSTALL_DATA}" > ${INSTALL_DATA_FILE}
+export ATOMIC_INSTALL_JSON=$INSTALL_DATA_FILE
 
 echo "UNIT TESTS:"
 

diff --git a/tests/integration/test_dbus.py b/tests/integration/test_dbus.py
@@ -105,11 +105,12 @@ def test_pull_already_present(self):
 
     @integration_test
     def test_run(self):
-        self.dbus_object.Run('atomic-test-3', name='atomic-dbus-3')
+        self.dbus_object.Run('atomic-test-3', 'atomic-dbus-3', False, False, True)
         self.cid = TestDBus.run_cmd('docker ps -aq -l').decode('utf-8').rstrip()
         TestDBus.add_cleanup_cmd('docker rm {}'.format(self.cid))
         container_inspect = json.loads(TestDBus.run_cmd('docker inspect {}'.format(self.cid)).decode('utf-8'))[0]
-        assert(container_inspect['Name'] == '/atomic-test-3')
+        print(container_inspect)
+        assert(container_inspect['Name'] == '/atomic-dbus-3')
 
     @integration_test
     def test_container_delete(self):

diff --git a/tests/unit/test_util.py b/tests/unit/test_util.py
@@ -1,7 +1,21 @@
+#pylint: skip-file
 import unittest
 import selinux
 import sys
 from Atomic import util
+from Atomic.backends._docker import DockerBackend
+
+no_mock = True
+try:
+    from unittest.mock import MagicMock, patch
+    no_mock = False
+except ImportError:
+    try:
+        from mock import MagicMock, patch
+        no_mock = False
+    except ImportError:
+        # Mock is already set to False
+        pass
 
 def _new_enough():
     py_version = sys.version_info
@@ -102,5 +116,97 @@ def test_valid_uri(self):
                 exception_raised = True
             self.assertTrue(exception_raised)
 
+
+class MockIO(object):
+    original_data = {"install_test": {"install_date": "2017-03-22 17:19:41", "id": "49779293ca711789a77bbdc35547a6b9ecb193a51b4e360fea95c4d206605d18"}}
+    new_data_fq = {"install_date": "2017-04-22 17:19:41","id": "16e9fdecc1febc87fb1ca09271009cf5f28eb8d4aec5515922ef298c145a6726"}
+    new_data_name= {"install_date": "2017-04-22 17:19:41","id": "16e9fdecc1febc87fb1ca09271009cf5f28eb8d4aec5515922ef298c145a6726"}
+    install_data = original_data
+
+    @classmethod
+    def read_mock(cls):
+        return cls.install_data
+
+    @classmethod
+    def write_mock(cls, val):
+        cls.install_data = val
+
+    @classmethod
+    def reset_data(cls):
+        cls.install_data = {}
+        cls.install_data = cls.original_data
+
+    @classmethod
+    def grow_data(cls, var_name, name):
+        cls.install_data[name] = getattr(cls, var_name)
+
+local_centos_inspect = {'Id': '16e9fdecc1febc87fb1ca09271009cf5f28eb8d4aec5515922ef298c145a6726', 'RepoDigests': ['docker.io/centos@sha256:7793b39617b28c6cd35774c00383b89a1265f3abf6efcaf0b8f4aafe4e0662d2'], 'Parent': '', 'GraphDriver': {'Name': 'devicemapper', 'Data': {'DeviceSize': '10737418240', 'DeviceName': 'docker-253:2-5900125-3fb5b406e6a53142129237c9e2c3a1ce8b6cf269b5f8071fcd62107c41544cd2', 'DeviceId': '779'}}, 'Created': '2016-08-30T18:20:19.39890162Z', 'Comment': '', 'DockerVersion': '1.12.1', 'VirtualSize': 210208812, 'Author': 'The CentOS Project <[email protected]> - ami_creator', 'Os': 'linux', 'RootFS': {'Type': 'layers', 'Layers': ['5fa0fa02637842ab1ddc8b3a17b86691c87c87d20800e6a95a113343f6ffd84c']}, 'Container': 'a5b0819aa82c224095e1a18e9df0776a7b38d32bacca073f054723b65fb54f0e', 'Architecture': 'amd64', 'RepoTags': ['docker.io/centos:centos7.0.1406'], 'Config': {'Labels': {}, 'Entrypoint': None, 'StdinOnce': False, 'OnBuild': None, 'Env': ['PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'], 'Volumes': None, 'Cmd': None, 'User': '', 'AttachStdin': False, 'AttachStderr': False, 'AttachStdout': False, 'WorkingDir': '', 'Tty': False, 'Image': '20ae10d641a0af6f25ceaa75fdcf591d171e3c521a54a3f3a2868b602d735e11', 'Hostname': 'a5b0819aa82c', 'Domainname': '', 'OpenStdin': False}, 'Size': 210208812, 'ContainerConfig': {'Labels': {}, 'Entrypoint': None, 'StdinOnce': False, 'OnBuild': None, 'Env': ['PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'], 'Volumes': None, 'Cmd': ['/bin/sh', '-c', '#(nop) ADD file:6a409eac27f0c7e04393da096dbeff01b929405e79b15222a0dc06a2084d3df3 in / '], 'User': '', 'AttachStdin': False, 'AttachStderr': False, 'AttachStdout': False, 'WorkingDir': '', 'Tty': False, 'Image': '20ae10d641a0af6f25ceaa75fdcf591d171e3c521a54a3f3a2868b602d735e11', 'Hostname': 'a5b0819aa82c', 'Domainname': '', 'OpenStdin': False}}
+rhel_docker_inspect = {u'Comment': u'', u'Container': u'', u'DockerVersion': u'1.9.1', u'Parent': u'', u'Created': u'2016-10-26T12:02:33.368772Z', u'Config': {u'Tty': False, u'Cmd': [u'/bin/bash'], u'Volumes': None, u'Domainname': u'', u'WorkingDir': u'', u'Image': u'f6f6121b053b2312688c87d3a1d32d06a984dc01d2ea7738508a50581cddb6b4', u'Hostname': u'', u'StdinOnce': False, u'Labels': {u'com.redhat.component': u'rhel-server-docker', u'authoritative-source-url': u'registry.access.redhat.com', u'distribution-scope': u'public', u'Vendor': u'Red Hat, Inc.', u'Name': u'rhel7/rhel', u'Build_Host': u'rcm-img01.build.eng.bos.redhat.com', u'vcs-type': u'git', u'name': u'rhel7/rhel', u'vcs-ref': u'7eeaf203cf909c2c056fba7066db9c1073a28d97', u'release': u'45', u'Version': u'7.3', u'Architecture': u'x86_64', u'version': u'7.3', u'Release': u'45', u'vendor': u'Red Hat, Inc.', u'BZComponent': u'rhel-server-docker', u'build-date': u'2016-10-26T07:54:17.037911Z', u'com.redhat.build-host': u'ip-10-29-120-48.ec2.internal', u'architecture': u'x86_64'}, u'AttachStdin': False, u'User': u'', u'Env': [u'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', u'container=docker'], u'Entrypoint': None, u'OnBuild': [], u'AttachStderr': False, u'AttachStdout': False, u'OpenStdin': False}, u'Author': u'Red Hat, Inc.', u'GraphDriver': {u'Data': {u'DeviceName': u'docker-253:2-5900125-a2bce97a4fd7ea12dce9865caa461ead8d1caf51ef452aba2f1b9d98efdf968f', u'DeviceSize': u'10737418240', u'DeviceId': u'623'}, u'Name': u'devicemapper'}, u'VirtualSize': 192508958, u'Os': u'linux', u'Architecture': u'amd64', u'RootFS': {u'Layers': [u'34d3e0e77091d9d51c6f70a7a7a4f7536aab214a55e02a8923af8f80cbe60d30', u'ccd6fc81ec49bd45f04db699401eb149b1945bb7292476b390ebdcdd7d975697'], u'Type': u'layers'}, u'ContainerConfig': {u'Tty': False, u'Cmd': None, u'Volumes': None, u'Domainname': u'', u'WorkingDir': u'', u'Image': u'', u'Hostname': u'', u'StdinOnce': False, u'Labels': None, u'AttachStdin': False, u'User': u'', u'Env': None, u'Entrypoint': None, u'OnBuild': None, u'AttachStderr': False, u'AttachStdout': False, u'OpenStdin': False}, u'Size': 192508958, u'RepoDigests': [u'registry.access.redhat.com/rhel7@sha256:da8a3e9297da7ccd1948366103d13c45b7e77489382351a777a7326004b63a21'], u'Id': u'f98706e16e41e56c4beaeea9fa77cd00fe35693635ed274f128876713afc0a1e', u'RepoTags': [u'registry.access.redhat.com/rhel7:latest']}
+
+
+@unittest.skipIf(no_mock, "Mock not found")
+@patch('Atomic.util.InstallData.read_install_data', new=MockIO.read_mock)
+@patch('Atomic.util.InstallData.write_install_data', new=MockIO.write_mock)
+class InstallData(unittest.TestCase):
+
+    class Args():
+        def __init__(self):
+            self.storage = None
+            self.debug = False
+            self.name = None
+            self.image = None
+
+    def test_read(self):
+        MockIO.reset_data()
+        self.assertEqual(util.InstallData.read_install_data(), MockIO.install_data)
+
+    def test_write(self):
+        MockIO.reset_data()
+        install_data = util.InstallData.read_install_data()
+        install_data['docker.io/library/centos:latest'] = MockIO.new_data_fq
+        util.InstallData.write_install_data(install_data)
+        self.assertTrue('docker.io/library/centos:latest' in util.InstallData.read_install_data())
+
+    def test_get_install_name_by_id(self):
+        MockIO.reset_data()
+        MockIO.grow_data('new_data_fq', 'docker.io/library/centos:latest')
+        self.assertEqual(util.InstallData.get_install_name_by_id('16e9fdecc1febc87fb1ca09271009cf5f28eb8d4aec5515922ef298c145a6726'), 'docker.io/library/centos:latest')
+
+    def test_fail_get_install_name_by_id(self):
+        MockIO.reset_data()
+        self.assertRaises(ValueError, util.InstallData.get_install_name_by_id, 1)
+
+    def test_image_installed_name(self):
+        MockIO.reset_data()
+        MockIO.grow_data('new_data_fq', 'docker.io/library/centos:latest')
+        args = self.Args()
+        args.storage = 'docker'
+        args.image = 'docker.io/library/centos:latest'
+        db = DockerBackend()
+        db._inspect_image = MagicMock(return_value=local_centos_inspect)
+        local_image_object = db.inspect_image(args.image)
+        self.assertTrue(util.InstallData.image_installed(local_image_object))
+
+    def test_image_installed_id(self):
+        MockIO.reset_data()
+        MockIO.grow_data('new_data_fq', '16e9fdecc1febc87fb1ca09271009cf5f28eb8d4aec5515922ef298c145a6726')
+        args = self.Args()
+        args.storage = 'docker'
+        args.image = 'docker.io/library/centos:latest'
+        db = DockerBackend()
+        db._inspect_image = MagicMock(return_value=local_centos_inspect)
+        local_image_object = db.inspect_image(args.image)
+        self.assertTrue(util.InstallData.image_installed(local_image_object))
+
+    def test_image_not_installed(self):
+        MockIO.reset_data()
+        args = self.Args()
+        args.storage = 'docker'
+        args.image = 'registry.access.redhat.com/rhel7'
+        db = DockerBackend()
+        db._inspect_image = MagicMock(return_value=rhel_docker_inspect)
+        local_image_object = db.inspect_image(args.image)
+        self.assertFalse(util.InstallData.image_installed(local_image_object))
+
 if __name__ == '__main__':
     unittest.main()