Merge pull request #424 from Shopify/ignore-dirs

Add a way to skip direcftories during load path scanning
Shopify · Nov 9, 2022 · fd54ddf · fd54ddf
2 parents fbb7146 + 4b7c16a
commit fd54ddf
Show file tree

Hide file tree

Showing 7 changed files with 45 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Unreleased
 
+* Add a way to skip direcftories during load path scanning.
+  If you have large non-ruby directories in the middle of your load path, it can severly slow down scanning.
+  Typically this is a problem with `node_modules`. See #277.
 * Fix `Bootsnap.unload_cache!`, it simply wouldn't work at all becaue of a merge mistake. See #421.
 
 # 1.13.0

diff --git a/README.md b/README.md
@@ -52,10 +52,11 @@ require 'bootsnap'
 env = ENV['RAILS_ENV'] || "development"
 Bootsnap.setup(
   cache_dir:            'tmp/cache',          # Path to your cache
+  ignored_directories   ['node_modules'],     # Directory names to skip.
   development_mode:     env == 'development', # Current working environment, e.g. RACK_ENV, RAILS_ENV, etc
   load_path_cache:      true,                 # Optimize the LOAD_PATH with a cache
   compile_cache_iseq:   true,                 # Compile Ruby code into ISeq cache, breaks coverage reporting.
-  compile_cache_yaml:   true                  # Compile YAML into a cache
+  compile_cache_yaml:   true,                 # Compile YAML into a cache
 )
 ```
 
@@ -77,6 +78,9 @@ well together, and are both included in a newly-generated Rails applications by
 - `DISABLE_BOOTSNAP_LOAD_PATH_CACHE` allows to disable load path caching.
 - `DISABLE_BOOTSNAP_COMPILE_CACHE` allows to disable ISeq and YAML caches.
 - `BOOTSNAP_LOG` configure bootsnap to log all caches misses to STDERR.
+- `BOOTSNAP_IGNORE_DIRECTORIES` a comma separated list of directories that shouldn't be scanned.
+  Useful when you have large directories of non-ruby files inside `$LOAD_PATH`.
+  It default to ignore any directory named `node_modules`.
 
 ### Environments
 

diff --git a/lib/bootsnap.rb b/lib/bootsnap.rb
@@ -41,6 +41,7 @@ def setup(
       load_path_cache: true,
       autoload_paths_cache: nil,
       disable_trace: nil,
+      ignore_directories: nil,
       compile_cache_iseq: true,
       compile_cache_yaml: true,
       compile_cache_json: true
@@ -65,6 +66,7 @@ def setup(
         Bootsnap::LoadPathCache.setup(
           cache_path: "#{cache_dir}/bootsnap/load-path-cache",
           development_mode: development_mode,
+          ignore_directories: ignore_directories,
         )
       end
 
@@ -113,13 +115,18 @@ def default_setup
           cache_dir = File.join(app_root, "tmp", "cache")
         end
 
+        ignore_directories = if ENV.key?("BOOTSNAP_IGNORE_DIRECTORIES")
+          ENV["BOOTSNAP_IGNORE_DIRECTORIES"].split(",")
+        end
+
         setup(
           cache_dir: cache_dir,
           development_mode: development_mode,
           load_path_cache: !ENV["DISABLE_BOOTSNAP_LOAD_PATH_CACHE"],
           compile_cache_iseq: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"] && iseq_cache_supported?,
           compile_cache_yaml: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"],
           compile_cache_json: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"],
+          ignore_directories: ignore_directories,
         )
 
         if ENV["BOOTSNAP_LOG"]

diff --git a/lib/bootsnap/load_path_cache.rb b/lib/bootsnap/load_path_cache.rb
@@ -28,7 +28,7 @@ class << self
       alias_method :enabled?, :enabled
       remove_method(:enabled)
 
-      def setup(cache_path:, development_mode:)
+      def setup(cache_path:, development_mode:, ignore_directories:)
         unless supported?
           warn("[bootsnap/setup] Load path caching is not supported on this implementation of Ruby") if $VERBOSE
           return
@@ -39,6 +39,7 @@ def setup(cache_path:, development_mode:)
         @loaded_features_index = LoadedFeaturesIndex.new
 
         @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode)
+        PathScanner.ignored_directories = ignore_directories if ignore_directories
         @enabled = true
         require_relative("load_path_cache/core_ext/kernel_require")
         require_relative("load_path_cache/core_ext/loaded_features")

diff --git a/lib/bootsnap/load_path_cache/path_scanner.rb b/lib/bootsnap/load_path_cache/path_scanner.rb
@@ -15,7 +15,11 @@ module PathScanner
         ""
       end
 
+      @ignored_directories = %w(node_modules)
+
       class << self
+        attr_accessor :ignored_directories
+
         def call(path)
           path = File.expand_path(path.to_s).freeze
           return [[], []] unless File.directory?(path)
@@ -50,6 +54,8 @@ def walk(absolute_dir_path, relative_dir_path, &block)
 
             absolute_path = "#{absolute_dir_path}/#{name}"
             if File.directory?(absolute_path)
+              next if ignored_directories.include?(name)
+
               if yield relative_path, absolute_path, true
                 walk(absolute_path, relative_path, &block)
               end

diff --git a/test/load_path_cache/core_ext/kernel_require_test.rb b/test/load_path_cache/core_ext/kernel_require_test.rb
@@ -10,7 +10,7 @@ def test_uses_the_same_duck_type_as_require
         assert_nil LoadPathCache.load_path_cache
         cache = Tempfile.new("cache")
         pid = Process.fork do
-          LoadPathCache.setup(cache_path: cache, development_mode: true)
+          LoadPathCache.setup(cache_path: cache, development_mode: true, ignore_directories: nil)
           dir = File.realpath(Dir.mktmpdir)
           $LOAD_PATH.push(dir)
           FileUtils.touch("#{dir}/a.rb")

diff --git a/test/setup_test.rb b/test/setup_test.rb
@@ -22,6 +22,7 @@ def test_default_setup
         compile_cache_iseq: Bootsnap.iseq_cache_supported?,
         compile_cache_yaml: true,
         compile_cache_json: true,
+        ignore_directories: nil,
       )
 
       Bootsnap.default_setup
@@ -37,6 +38,7 @@ def test_default_setup_with_ENV_not_dev
         compile_cache_iseq: Bootsnap.iseq_cache_supported?,
         compile_cache_yaml: true,
         compile_cache_json: true,
+        ignore_directories: nil,
       )
 
       Bootsnap.default_setup
@@ -52,6 +54,7 @@ def test_default_setup_with_DISABLE_BOOTSNAP_LOAD_PATH_CACHE
         compile_cache_iseq: Bootsnap.iseq_cache_supported?,
         compile_cache_yaml: true,
         compile_cache_json: true,
+        ignore_directories: nil,
       )
 
       Bootsnap.default_setup
@@ -67,6 +70,7 @@ def test_default_setup_with_DISABLE_BOOTSNAP_COMPILE_CACHE
         compile_cache_iseq: false,
         compile_cache_yaml: false,
         compile_cache_json: false,
+        ignore_directories: nil,
       )
 
       Bootsnap.default_setup
@@ -89,12 +93,29 @@ def test_default_setup_with_BOOTSNAP_LOG
         compile_cache_iseq: Bootsnap.iseq_cache_supported?,
         compile_cache_yaml: true,
         compile_cache_json: true,
+        ignore_directories: nil,
       )
       Bootsnap.expects(:logger=).with($stderr.method(:puts))
 
       Bootsnap.default_setup
     end
 
+    def test_default_setup_with_BOOTSNAP_IGNORE_DIRECTORIES
+      ENV["BOOTSNAP_IGNORE_DIRECTORIES"] = "foo,bar"
+
+      Bootsnap.expects(:setup).with(
+        cache_dir: @tmp_dir,
+        development_mode: true,
+        load_path_cache: true,
+        compile_cache_iseq: Bootsnap.iseq_cache_supported?,
+        compile_cache_yaml: true,
+        compile_cache_json: true,
+        ignore_directories: %w[foo bar],
+      )
+
+      Bootsnap.default_setup
+    end
+
     def test_unload_cache
       Bootsnap.unload_cache!
     end