Load FileData info from cache

This PR adds the functionality to load FileData
information from the cache. A new function get_files
has been added in cache.py.
fill method has been added in FileData class.
Appropriate tests have been added

Closes tern-tools#574

Signed-off-by: PrajwalM2212 <[email protected]>

fix

tern/classes/docker_image.py

@@ -7,7 +7,9 @@
 import os
 import subprocess  # nosec
 
+from tern.classes.file_data import FileData
 from tern.utils import rootfs
+from tern.utils.cache import get_files
 from tern.utils.general import pushd
 from tern.utils.constants import manifest_file
 from tern.analyze.docker.container import extract_image_metadata
@@ -159,7 +161,16 @@ def load_image(self):
             while layer_diffs and layer_paths:
                 layer = ImageLayer(layer_diffs.pop(0), layer_paths.pop(0))
                 layer.gen_fs_hash()
-                layer.add_files()
+                raw_file_list = get_files(layer.fs_hash)
+                # Fetch file info from cache if exists
+                # else extract and store file info
+                if raw_file_list:
+                    for file_dict in raw_file_list:
+                        file = FileData(file_dict['name'], file_dict['path'])
+                        file.fill(file_dict)
+                        layer.add_file(file)
+                else:
+                    layer.add_files()
                 self._layers.append(layer)
             self.set_layer_created_by()
         except NameError:  # pylint: disable=try-except-raise

tern/classes/file_data.py

@@ -5,6 +5,7 @@
 
 import datetime
 
+from tern.classes.notice import Notice
 from tern.classes.origins import Origins
 from tern.utils.general import prop_names
 
@@ -153,3 +154,43 @@ def to_dict(self, template=None):
                 file_dict.update({prop: self.__dict__[key]})
             file_dict.update({'origins': self.origins.to_dict()})
         return file_dict
+
+    def __fill_properties(self, file_dict):
+        '''Check to see if the dictionary keys have all the properties
+        listed. If not then put a Notice object in the list of Origins'''
+        for key, prop in prop_names(self):
+            if prop not in ('name', 'origins', 'path'):
+                if prop not in file_dict.keys():
+                    self.origins.add_notice_to_origins(
+                        self.name, Notice(
+                            "No metadata for key: {}".format(prop), 'warning'))
+                else:
+                    self.__dict__[key] = file_dict[prop]
+
+    def fill(self, file_dict):
+        '''The file dict looks like this:
+            name: <name>
+            path: <path to file>
+            date: <date>
+            file_type: <file_type>
+            checksum: <checksum>
+            checksum_type: <checksum_type>
+            version_control: <version_control>
+            version: <version>
+            extattrs: <extattrs>
+            licenses: <licenses>
+            license_expressions: <license_expressions>
+            copyrights: <copyrights>
+            authors: <authors>
+            packages: <packages>
+            urls: <urls>
+        the way to use this method is to instantiate the class with the
+        name and path and then give it a file_data dictionary to fill
+        in the rest return true if package name is the same as the one
+        used to instantiate the object, false if not'''
+        success = True
+        if self.name == file_dict['name'] and self.path == file_dict['path']:
+            self.__fill_properties(file_dict)
+        else:
+            success = False
+        return success

tern/utils/cache.py

@@ -45,6 +45,14 @@ def get_packages(layer_hash):
     return []
 
 
+def get_files(layer_hash):
+    '''Given the layer hash, retrieve file layers cache record.
+    If none return an empty list'''
+    if layer_hash in cache.keys():
+        return cache[layer_hash]['files']
+    return []
+
+
 def get_layers():
     '''Return a list of layer shas'''
     return cache.keys()

tests/test_class_file_data.py

@@ -94,6 +94,33 @@ def testToDictTemplate(self):
         self.assertEqual(dict1['file.licenses'], ['MIT', 'GPL'])
         self.assertFalse(dict2['notes'])
 
+    def testFill(self):
+        file_dict = {
+            'name': 'zconf.h',
+            'path': '/usr/include/zconf.h',
+            'checksum_type': 'sha256',
+            'checksum': '77304005ceb5f0d03ad4c37eb8386a10866e'
+                        '4ceeb204f7c3b6599834c7319541',
+            'extattrs': '-rw-r--r-- 1 1000 1000 16262 Nov 13 17:57'
+                        ' /usr/include/zconf.h'
+        }
+        f = FileData('zconf.h', '/usr/include/zconf.h')
+        f.fill(file_dict)
+        self.assertEqual(f.name, 'zconf.h')
+        self.assertEqual(f.path, '/usr/include/zconf.h')
+        self.assertEqual(f.checksum_type, 'sha256')
+        self.assertEqual(f.checksum, '77304005ceb5f0d03ad4c37eb838'
+                                     '6a10866e4ceeb204f7c3b6599834c7319541')
+        self.assertEqual(f.extattrs, '-rw-r--r-- 1 1000 1000 '
+                                     '16262 Nov 13 17:57 /usr/include/zconf.h')
+        self.assertEqual(f.origins.origins[0].notices[1].message,
+                         'No metadata for key: file_type')
+        self.assertEqual(f.origins.origins[0].notices[1].level, 'warning')
+        self.assertEqual(f.origins.origins[0].notices[0].message,
+                         'No metadata for key: date')
+        self.assertEqual(f.origins.origins[0].notices[2].message,
+                         'No metadata for key: version_control')
+
 
 if __name__ == '__main__':
     unittest.main()