Use dockerfile_parse to get Dockerfile info

This is work towards #522

We add initial functionality for parsing dockerfiles using
dockerfile_parse. We also add some tests for the functions therein
and some extra functions to parse various pieces of the Dockerfile
we will need.

Most of the work is in tern/analyze/docker/dockerfile.py.
We add a class called Dockerfile which will contain the information
parsed using the function get_dockerfile_obj. The typical workflow
is to create a Dockerfile object using get_dockerfile_obj from an
existing Dockerfile file. Then we can use the other functions to
return the information we want

- replace_env will do a key-value replacement of any piece of the
Dockerfile object's structure property with any key-value dict. The
typical use for this is to replace ENVs with their values in any
Dockerfile line.
- expand_vars will do the replacement wholesale for the Dockerfile
content.
- parse_from_image will get a dictionary containing tokens in the
image string for each FROM line in the Dockerfile. In order for
this to work, we also add a function called parse_image_string
to tern/utils/general.py which will do the parsing of the image
string. This helps us use this parsing for image names passed via
command line using the -i flag.
- Added tests for these functions in test_analyze_docker_dockerfile.py
For these tests, we also add some dockerfiles.

Other changes include:
- Add dockerfile-parse to the list of requirements.
- Add tests for dockerfile and general to the CI tests.

Signed-off-by: Nisha K <[email protected]>

ci/test_files_touched.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+# Copyright (c) 2019-2020 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause
 
 from git import Repo
@@ -72,6 +72,7 @@
         'tern -l report -i centos:7'],
     # tern/analyze/docker
     re.compile('tern/analyze/docker'): [
+        'python tests/test_analyze_docker_dockerfile.py',
         'tern -l report -i golang:alpine',
         'tern -l report -d samples/alpine_python/Dockerfile'],
     # tern/report
@@ -86,6 +87,7 @@
     ['tern -l report -i golang:alpine'],
     # tern/utils
     re.compile('tern/utils'): [
+        'python tests/test_util_general.py',
         'tern -l report -i golang:alpine',
         'tern -l report -d samples/alpine_python/Dockerfile'],
     # tests
@@ -106,8 +108,7 @@
     re.compile('tests/test_class_package.py'):
         ['python tests/test_class_package.py'],
     re.compile('tests/test_class_template.py'):
-        ['python tests/test_class_template.py']
-        }
+        ['python tests/test_class_template.py']}
 
 alltests = []
 for change in changes:

requirements.txt

@@ -8,6 +8,7 @@
 
 PyYAML>=5.2
 docker~=4.1
+dockerfile-parse~=0.0
 requests~=2.22
 stevedore>=1.31
 pbr>=5.4

tern/analyze/docker/dockerfile.py

@@ -1,15 +1,16 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (c) 2017-2019 VMware, Inc. All Rights Reserved.
+# Copyright (c) 2017-2020 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause
 
 """
-Dockerfile parser and information retrieval
+Dockerfile information retrieval and modification
 """
 
+from dockerfile_parse import DockerfileParser
 import re
 
-from tern.utils.general import clean_command
+from tern.utils import general
 
 directives = ['FROM',
               'ARG',
@@ -35,6 +36,88 @@
 tag_separator = ':'
 
 
+class Dockerfile():
+    ''' This class is used as a wrapper to store dockerfile information
+    retrieved from the parser.'''
+    def __init__(self):
+        self.structure = None
+        self.envs = None
+        self.prev_env = None
+        self.filepath = ""
+        self.parent_images = []
+
+    def is_none(self):
+        """Check if the object is empty."""
+        is_none = True
+        if (self.structure or
+                self.envs or
+                self.prev_env or
+                self.filepath):
+            is_none = False
+        return is_none
+
+
+def get_dockerfile_obj(dockerfile_name, prev_env=None):
+    '''Given a Dockerfile, create a Dockerfile parser object to be used later.
+    dockerfile_name: This is the path to the Dockerfile including the
+                            file name
+    prev_env: These are environment variables that may have been used in
+    previous stages in a multistage docker build. Should be a python dictionary
+    of the form {'ENV': 'value',...}'''
+    dfobj = Dockerfile()
+    with open(dockerfile_name) as f:
+        parser = DockerfileParser(parent_env=prev_env, fileobj=f)
+        dfobj.filepath = dockerfile_name
+        dfobj.structure = parser.structure
+        dfobj.envs = parser.envs
+        dfobj.prev_env = prev_env
+        dfobj.parent_images = parser.parent_images
+    return dfobj
+
+
+def replace_env(key_value_dict, df_structure_dict):
+    '''Replace the environment variables in the key_value_dict dictionary
+    with its corresponding value in the df_line_dict dictionary
+    key_value_dict: a dictionary of key-value pairs like envs in the dockerfile
+                    object
+    df_structure_dict: a dictionary from the dockerfile object's structure'''
+    for key, val in key_value_dict.items():
+        envvar1 = '$' + key
+        envvar2 = '${' + key + '}'
+        df_structure_dict['content'] = df_structure_dict['content'].replace(
+            envvar1, val)
+        df_structure_dict['content'] = df_structure_dict['content'].replace(
+            envvar2, val)
+        df_structure_dict['value'] = df_structure_dict['value'].replace(
+            envvar1, val)
+        df_structure_dict['value'] = df_structure_dict['value'].replace(
+            envvar2, val)
+
+
+def expand_vars(dfobj):
+    '''Replace the environment variables with their values if known
+    dfobj: the Dockerfile object created using get_dockerfile_obj'''
+    if dfobj.envs:
+        for obj in dfobj.structure:
+            replace_env(dfobj.envs, obj)
+    if dfobj.prev_env:
+        for obj in dfobj.structure:
+            replace_env(dfobj.prev_env, obj)
+
+
+def parse_from_image(dfobj):
+    '''Get a list of dictionaries from the FROM instruction. The dictionary
+    should be of the form:
+        [{'name': <image name used (either from dockerhub or full name)>,
+          'tag': <image tag>,
+          'digest_type': <the hashing algorithm used>
+          'digest': <image digest>}..]'''
+    image_list = []
+    for image_string in dfobj.parent_images:
+        image_list.append(general.parse_image_string(image_string))
+    return image_list
+
+
 def get_command_list(dockerfile_name):
     '''Given a Dockerfile, return a list of Docker commands'''
     with open(dockerfile_name) as f:
@@ -80,7 +163,7 @@ def get_directive_list(command_list):
     i.e. FROM, ADD, COPY etc and the object to be acted upon'''
     directive_list = []
     for command in command_list:
-        directive_list.append(get_directive(clean_command(command)))
+        directive_list.append(get_directive(general.clean_command(command)))
     return directive_list
 
 

tern/utils/general.py

@@ -148,3 +148,33 @@ def check_image_string(image_str: str):
     if re.match(tag_format, image_str) or re.match(digest_format, image_str):
         return True
     return False
+
+
+def parse_image_string(image_string):
+    '''From the image string used to reference an image, return a dictionary
+    of the form:
+        {'name': <image name used (either from dockerhub or full name)>,
+         'tag': <image tag>,
+         'digest_type': <the hashing algorithm used>,
+         'digest': <image digest>}
+    per Docker's convention, an image can be referenced either as
+    image OR image:tag OR image@hash:digest
+    we choose ':' and '@' as separators
+    Currently OCI also uses this convention'''
+    tokens = re.split(r'[@:]', image_string)
+    if len(tokens) == 1:
+        return {'name': tokens[0],
+                'tag': '',
+                'digest_type': '',
+                'digest': ''}
+    if len(tokens) == 2:
+        return {'name': tokens[0],
+                'tag': tokens[1],
+                'digest_type': '',
+                'digest': ''}
+    if len(tokens) == 3:
+        return {'name': tokens[0],
+                'tag': '',
+                'digest_type': tokens[1],
+                'digest': tokens[2]}
+    return None

tests/dockerfiles/buildpack_deps_jessie_curl

@@ -0,0 +1,18 @@
+FROM debian:jessie
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		ca-certificates \
+		curl \
+		netbase \
+		wget \
+	&& rm -rf /var/lib/apt/lists/*
+
+RUN set -ex; \
+	if ! command -v gpg > /dev/null; then \
+		apt-get update; \
+		apt-get install -y --no-install-recommends \
+			gnupg \
+			dirmngr \
+		; \
+		rm -rf /var/lib/apt/lists/*; \
+	fi

tests/dockerfiles/buildpack_deps_jessie_pinned

@@ -0,0 +1 @@
+FROM debian@sha256:e25703ee6ab5b2fac31510323d959cdae31eebdf48e88891c549e55b25ad7e94

tests/dockerfiles/golang_1.13_stretch

@@ -0,0 +1,50 @@
+FROM buildpack-deps:stretch-scm
+
+# gcc for cgo
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		g++ \
+		gcc \
+		libc6-dev \
+		make \
+		pkg-config \
+	&& rm -rf /var/lib/apt/lists/*
+
+ENV GOLANG_VERSION 1.13.6
+
+RUN set -eux; \
+	\
+# this "case" statement is generated via "update.sh"
+	dpkgArch="$(dpkg --print-architecture)"; \
+	case "${dpkgArch##*-}" in \
+		amd64) goRelArch='linux-amd64'; goRelSha256='a1bc06deb070155c4f67c579f896a45eeda5a8fa54f35ba233304074c4abbbbd' ;; \
+		armhf) goRelArch='linux-armv6l'; goRelSha256='37a1a83e363dcf146a67fa839d170fd1afb13009585fdd493d0a3370fbe6f785' ;; \
+		arm64) goRelArch='linux-arm64'; goRelSha256='0a18125c4ed80f9c3045cf92384670907c4796b43ed63c4307210fe93e5bbca5' ;; \
+		i386) goRelArch='linux-386'; goRelSha256='27feb013106da784f09e560720aa41ab395c67f7eed4c4a0fce04bc6e3d01c7d' ;; \
+		ppc64el) goRelArch='linux-ppc64le'; goRelSha256='26a977a8af5dc50a562f0a57b58dded5fa3bacfe77722cf8a84ea54ca54728dd' ;; \
+		s390x) goRelArch='linux-s390x'; goRelSha256='5cd9900a1fa0f0cac657930b648381cad9b8c5e2bbc77caf86a6fb5cedad0017' ;; \
+		*) goRelArch='src'; goRelSha256='aae5be954bdc40bcf8006eb77e8d8a5dde412722bc8effcdaf9772620d06420c'; \
+			echo >&2; echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; echo >&2 ;; \
+	esac; \
+	\
+	url="https://golang.org/dl/go${GOLANG_VERSION}.${goRelArch}.tar.gz"; \
+	wget -O go.tgz "$url"; \
+	echo "${goRelSha256} *go.tgz" | sha256sum -c -; \
+	tar -C /usr/local -xzf go.tgz; \
+	rm go.tgz; \
+	\
+	if [ "$goRelArch" = 'src' ]; then \
+		echo >&2; \
+		echo >&2 'error: UNIMPLEMENTED'; \
+		echo >&2 'TODO install golang-any from jessie-backports for GOROOT_BOOTSTRAP (and uninstall after build)'; \
+		echo >&2; \
+		exit 1; \
+	fi; \
+	\
+	export PATH="/usr/local/go/bin:$PATH"; \
+	go version
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+WORKDIR $GOPATH

tests/test_analyze_docker_dockerfile.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2020 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+import unittest
+
+from tern.analyze.docker import dockerfile
+
+
+class TestAnalyzeDockerDockerfile(unittest.TestCase):
+
+    def setUp(self):
+        self.buildpack = 'tests/dockerfiles/buildpack_deps_jessie_curl'
+        self.buildpackpinned = 'tests/dockerfiles/buildpack_deps_jessie_pinned'
+        self.golang = 'tests/dockerfiles/golang_1.13_stretch'
+
+    def tearDown(self):
+        del self.buildpack
+        del self.golang
+
+    def testDockerfileObject(self):
+        dfobj = dockerfile.Dockerfile()
+        self.assertTrue(dfobj.is_none())
+
+    def testDockerfileParserWithoutEnv(self):
+        dfobj = dockerfile.get_dockerfile_obj(self.buildpack)
+        self.assertFalse(dfobj.is_none())
+        self.assertEqual(dfobj.parent_images, ['debian:jessie'])
+        structure = [{'instruction': 'FROM',
+                      'startline': 0,
+                      'endline': 0,
+                      'content': 'FROM debian:jessie\n',
+                      'value': 'debian:jessie'},
+                     {'instruction': 'RUN',
+                      'startline': 2,
+                      'endline': 7,
+                      'content': ('RUN apt-get update && apt-get install -y --'
+                                  'no-install-recommends \\\n\t\tca-certific'
+                                  'ates \\\n\t\tcurl \\\n\t\tnetbase \\\n\t\tw'
+                                  'get \\\n\t&& rm -rf /var/lib/apt/lists/*'
+                                  '\n'),
+                      'value': ('apt-get update && apt-get install -y --no-in'
+                                'stall-recommends \t\tca-certificates \t\tcur'
+                                'l \t\tnetbase \t\twget \t&& rm -rf /var/lib/'
+                                'apt/lists/*')},
+                     {'instruction': 'RUN',
+                      'startline': 9,
+                      'endline': 17,
+                      'content': ('RUN set -ex; \\\n\tif ! command -v gpg > /'
+                                  'dev/null; then \\\n\t\tapt-get update; \\'
+                                  '\n\t\tapt-get install -y --no-install-reco'
+                                  'mmends \\\n\t\t\tgnupg \\\n\t\t\tdirmngr \\'
+                                  '\n\t\t; \\\n\t\trm -rf /var/lib/apt/lists/'
+                                  '*; \\\n\tfi\n'),
+                      'value': ('set -ex; \tif ! command -v gpg > /dev/null; t'
+                                'hen \t\tapt-get update; \t\tapt-get install -'
+                                'y --no-install-recommends \t\t\tgnupg \t\t\td'
+                                'irmngr \t\t; \t\trm -rf /var/lib/apt/lists/*'
+                                '; \tfi')}]
+        self.assertEqual(dfobj.structure, structure)
+        self.assertFalse(dfobj.envs)
+
+    def testDockerfileParserWithEnv(self):
+        dfobj = dockerfile.get_dockerfile_obj(self.buildpack,
+                                              {'buildno': '123abc'})
+        self.assertFalse(dfobj.is_none())
+        self.assertEqual(dfobj.prev_env, {'buildno': '123abc'})
+
+    def testReplaceEnv(self):
+        dfobj = dockerfile.get_dockerfile_obj(self.golang)
+        envs = {'GOLANG_VERSION': '1.13.6',
+                'GOPATH': '/go',
+                'PATH': '/go/bin:/usr/local/go/bin:'}
+        self.assertEqual(dfobj.envs, envs)
+        struct = dfobj.structure[9]
+        dockerfile.replace_env(dfobj.envs, struct)
+        self.assertEqual(struct['content'], 'WORKDIR /go\n')
+        self.assertEqual(struct['value'], '/go')
+        replace_content = ('\n\turl="https://golang.org/dl/go1.13.6.'
+                           '${goRelArch}.tar.gz"; ')
+        replace_value = (' \t\turl="https://golang.org/dl/go1.13.6'
+                         '.${goRelArch}.tar.gz"')
+        struct = dfobj.structure[5]
+        dockerfile.replace_env(dfobj.envs, struct)
+        self.assertEqual(struct['content'].split('\\')[14], replace_content)
+        self.assertEqual(struct['value'].split(';')[28], replace_value)
+
+    def testParseFromImage(self):
+        dfobj = dockerfile.get_dockerfile_obj(self.buildpack)
+        image_list = dockerfile.parse_from_image(dfobj)
+        self.assertEqual(image_list, [{'name': 'debian',
+                                       'tag': 'jessie',
+                                       'digest_type': '',
+                                       'digest': ''}])
+        dfobj = dockerfile.get_dockerfile_obj(self.buildpackpinned)
+        image_list = dockerfile.parse_from_image(dfobj)
+        debian_digest = ('e25703ee6ab5b2fac31510323d959cdae31eebdf48e88891c54'
+                         '9e55b25ad7e94')
+        self.assertEqual(image_list, [{'name': 'debian',
+                                       'tag': '',
+                                       'digest_type': 'sha256',
+                                       'digest': debian_digest}])
+
+
+if __name__ == '__main__':
+    unittest.main()