Merge pull request #1750 from nexB/1748-handle-ignored-directories

Don't visit files and subdir of ignored dirs

src/scancode/resource.py

@@ -437,6 +437,10 @@ def create_resources(_seq, _top, _parent, _is_file):
         # walk proper
         for top, dirs, files in os_walk(root.location, topdown=True, onerror=err):
             if skip_ignored(top):
+                # We clear out `dirs` and `files` to prevent `os_walk` from visiting
+                # the files and subdirectories of directories we are ignoring
+                dirs[:] = []
+                files[:] = []
                 continue
             # the parent reference is needed only once in a top-down walk, hence
             # the pop
@@ -706,7 +710,7 @@ def _load_resource(self, rid):
 
         if TRACE:
             logger_debug(
-                '    Codebase._load_resource: exists:', exists(cache_location), 
+                '    Codebase._load_resource: exists:', exists(cache_location),
                 'cache_location:', cache_location)
 
         if not exists(cache_location):

tests/scancode/test_resource.py

@@ -563,6 +563,17 @@ def test_distance(self):
         assert 'MANIFEST.MF' == res.name
         assert 3 == res.distance(codebase)
 
+    def test_skip_files_and_subdirs_of_ignored_dirs(self):
+        test_dir = self.get_test_loc('resource/ignore')
+        codebase = Codebase(test_dir)
+        # The `cvs` directory should not be visited
+        expected = [
+            'ignore',
+            'ignore/file1'
+        ]
+        result = [r.path for r in codebase.walk(topdown=True)]
+        self.assertEqual(expected, result)
+
 
 class TestCodebaseCache(FileBasedTesting):
     test_data_dir = join(dirname(__file__), 'data')