Enable analysis for OCI images

tern/__main__.py

@@ -13,8 +13,8 @@
 import logging
 import os
 import sys
-
-from tern.analyze.docker import run
+from tern.analyze.oci import run as oci_run
+from tern.analyze.docker import run as docker_run
 from tern.utils import cache
 from tern.utils import constants
 from tern.utils import general
@@ -24,7 +24,7 @@
 
 # global logger
 from tern.utils.general import check_image_string
-
+from tern.utils.general import check_oci_image_string
 logger = logging.getLogger(constants.logger_name)
 logger.setLevel(logging.DEBUG)
 
@@ -69,6 +69,33 @@ def create_top_dir(working_dir=None):
         os.makedirs(top_dir)
 
 
+def execute_image(args):
+    ''' Executes container images using the given inputs '''
+    if args.type == "docker":
+        # Check if the image is of image:tag
+        # or image@digest_type:digest format
+        if not check_image_string(args.image):
+            sys.stderr.write('Error running Tern\n'
+                             'Please provide docker image '
+                             'string in image:tag or '
+                             'image@digest_type:digest format\n')
+            sys.exit(1)
+        if general.check_tar(args.image):
+            logger.error("%s", errors.incorrect_raw_option)
+        else:
+            docker_run.execute_docker_image(args)
+            logger.debug('Report completed.')
+    elif args.type == "oci":
+        # Check if the image is of oci://image-location:tag
+        if not check_oci_image_string(args.image):
+            sys.stderr.write('Error running Tern\n'
+                             'Please provide oci image '
+                             'oci://image-location:tag format\n')
+            sys.exit(1)
+        oci_run.execute_oci_image(args)
+        logger.debug('Report completed.')
+
+
 def do_main(args):
     '''Execute according to subcommands'''
     # set bind mount location if working in a container
@@ -84,33 +111,21 @@ def do_main(args):
     if args.clear_cache:
         logger.debug('Clearing cache...')
         cache.clear()
-    if hasattr(args, 'name') and (args.name == 'report' or
-                                  args.name == 'lock'):
+    if hasattr(args, 'name') and \
+            (args.name == 'report' or args.name == 'lock'):
         if args.name == 'lock':
-            run.execute_dockerfile(args)
+            docker_run.execute_dockerfile(args)
         elif args.dockerfile:
-            run.execute_dockerfile(args)
-        elif args.docker_image:
-            # Check if the image is of image:tag
-            # or image@digest_type:digest format
-            if not check_image_string(args.docker_image):
-                sys.stderr.write('Error running Tern\n'
-                                 'Please provide docker image '
-                                 'string in image:tag or '
-                                 'image@digest_type:digest format\n')
-                sys.exit(1)
-            if general.check_tar(args.docker_image):
-                logger.error("%s", errors.incorrect_raw_option)
-            else:
-                run.execute_docker_image(args)
-                logger.debug('Report completed.')
+            docker_run.execute_dockerfile(args)
+        elif args.image:
+            execute_image(args)
         if args.name == 'report':
             if args.raw_image:
                 if not general.check_tar(args.raw_image):
                     logger.error("%s", errors.invalid_raw_image.format(
                         image=args.raw_image))
                 else:
-                    run.execute_docker_image(args)
+                    docker_run.execute_docker_image(args)
                     logger.debug('Report completed.')
     logger.debug('Finished')
 
@@ -153,12 +168,18 @@ def main():
     parser_report.add_argument('-d', '--dockerfile', type=check_file_existence,
                                help="Dockerfile used to build the Docker"
                                " image")
-    parser_report.add_argument('-i', '--docker-image',
-                               help="Docker image that exists locally -"
+    parser_report.add_argument('-i', '--image',
+                               help="Image that exists locally -"
+                               "either can be a docker image with format"
                                " image:tag"
+                               " or an OCI image with format"
+                               " oci://<image-location>:<image-tag>"
                                " The option can be used to pull docker"
                                " images by digest as well -"
                                " <repo>@<digest-type>:<digest>")
+    parser_report.add_argument('-t', '--type',
+                               help="type of image -"
+                               " possible values could be an oci or docker")
     parser_report.add_argument('-w', '--raw-image', metavar='FILE',
                                help="Raw container image that exists locally "
                                "in the form of a tar archive.")

tern/analyze/__init__.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause

tern/analyze/docker/run.py

@@ -72,13 +72,13 @@ def analyze(image_obj, args, dfile_lock=False, dfobj=None):
 def execute_docker_image(args):
     '''Execution path if given a Docker image'''
     logger.debug('Setting up...')
-    image_string = args.docker_image
+    image_string = args.image
     if not args.raw_image:
         # don't check docker daemon for raw images
         container.check_docker_setup()
     else:
         image_string = args.raw_image
-    report.setup(image_tag_string=image_string)
+    report.setup(image_tag_string=image_string, image_type=args.type)
     # attempt to get built image metadata
     full_image = report.load_full_image(image_string)
     if full_image.origins.is_empty():

tern/analyze/oci/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause

tern/analyze/oci/analyze.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019-2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""
+Analyze a OCI format image
+"""
+
+import sys
+import logging
+from tern.utils import rootfs
+from tern.report import errors
+from tern.analyze import common
+from tern.utils import constants
+from tern.analyze.oci import helpers
+from tern.classes.notice import Notice
+from tern.command_lib import command_lib
+
+
+# global logger
+logger = logging.getLogger(constants.logger_name)
+
+
+def analyze_oci_image(image_obj, redo=False):
+    '''Given a OCIImage object, for each layer, retrieve the packages, first
+    looking up in cache and if not there then looking up in the command
+    library. For looking up in command library first mount the filesystem
+    and then look up the command library for commands to run in chroot.'''
+
+    # set up empty master list of packages
+    master_list = []
+    prepare_for_analysis(image_obj)
+    # Analyze the first layer and get the shell
+    shell = analyze_first_layer(image_obj, master_list, redo)
+    # Analyze the remaining layers
+    analyze_subsequent_layers(image_obj, shell, master_list, redo)
+    common.save_to_cache(image_obj)
+
+
+def prepare_for_analysis(image_obj):
+    # add notices for each layer if it is imported
+    image_setup(image_obj)
+    # set up the mount points
+    rootfs.set_up()
+
+
+def abort_analysis():
+    '''Abort due to some external event'''
+    rootfs.recover()
+    sys.exit(1)
+
+
+def analyze_first_layer(image_obj, master_list, redo):
+    # set up a notice origin for the first layer
+    origin_first_layer = 'Layer: ' + image_obj.layers[0].fs_hash[:10]
+    # find the shell from the first layer
+    shell = common.get_shell(image_obj.layers[0])
+    if not shell:
+        logger.warning(errors.no_shell)
+        image_obj.layers[0].origins.add_notice_to_origins(
+            origin_first_layer, Notice(errors.no_shell, 'warning'))
+    # find the binary from the first layer
+    binary = common.get_base_bin(image_obj.layers[0])
+    if not binary:
+        logger.warning(errors.no_package_manager)
+        image_obj.layers[0].origins.add_notice_to_origins(
+            origin_first_layer, Notice(errors.no_package_manager, 'warning'))
+    # try to load packages from cache
+    if not common.load_from_cache(image_obj.layers[0], redo):
+        # set a possible OS
+        common.get_os_style(image_obj.layers[0], binary)
+        # if there is a binary, extract packages
+        if shell and binary:
+            execute_base_layer(image_obj.layers[0], binary, shell)
+    # populate the master list with all packages found in the first layer
+    for p in image_obj.layers[0].packages:
+        master_list.append(p)
+    return shell
+
+
+def execute_base_layer(base_layer, binary, shell):
+    '''Execute retrieving base layer packages'''
+    try:
+        target = rootfs.mount_base_layer(base_layer.tar_file)
+        rootfs.prep_rootfs(target)
+        common.add_base_packages(base_layer, binary, shell)
+    except KeyboardInterrupt:
+        logger.critical(errors.keyboard_interrupt)
+        abort_analysis()
+    finally:
+        # unmount proc, sys and dev
+        rootfs.undo_mount()
+        rootfs.unmount_rootfs()
+
+
+def analyze_subsequent_layers(image_obj, shell, master_list, redo):  # noqa: R0912,R0913
+    # get packages for subsequent layers
+    curr_layer = 1
+    # pylint:disable=too-many-nested-blocks
+    while curr_layer < len(image_obj.layers):
+        # if there is no shell, try to see if it exists in the current layer
+        if not shell:
+            shell = common.get_shell(image_obj.layers[curr_layer])
+        if not common.load_from_cache(image_obj.layers[curr_layer], redo):
+            # get commands that created the layer
+            # for docker images this is retrieved from the image history
+            command_list = helpers.get_commands_from_history(
+                image_obj.layers[curr_layer])
+            if command_list:
+                # mount diff layers from 0 till the current layer
+                target = mount_overlay_fs(image_obj, curr_layer)
+                # mount dev, sys and proc after mounting diff layers
+                rootfs.prep_rootfs(target)
+            # for each command look up the snippet library
+            for command in command_list:
+                pkg_listing = command_lib.get_package_listing(command.name)
+                if isinstance(pkg_listing, str):
+                    try:
+                        common.add_base_packages(
+                            image_obj.layers[curr_layer], pkg_listing, shell)
+                    except KeyboardInterrupt:
+                        logger.critical(errors.keyboard_interrupt)
+                        abort_analysis()
+                else:
+                    try:
+                        common.add_snippet_packages(
+                            image_obj.layers[curr_layer], command, pkg_listing,
+                            shell)
+                    except KeyboardInterrupt:
+                        logger.critical(errors.keyboard_interrupt)
+                        abort_analysis()
+            if command_list:
+                rootfs.undo_mount()
+                rootfs.unmount_rootfs()
+        # update the master list
+        common.update_master_list(master_list, image_obj.layers[curr_layer])
+        curr_layer = curr_layer + 1
+
+
+def image_setup(image_obj):
+    '''Add notices for each layer'''
+    for layer in image_obj.layers:
+        origin_str = 'Layer: ' + layer.fs_hash[:10]
+        layer.origins.add_notice_origin(origin_str)
+
+
+def mount_overlay_fs(image_obj, top_layer):
+    '''Given the image object and the top most layer, mount all the layers
+    until the top layer using overlayfs'''
+    tar_layers = []
+    for index in range(0, top_layer + 1):
+        tar_layers.append(image_obj.layers[index].tar_file)
+    target = rootfs.mount_diff_layers(tar_layers)
+    return target