Compress source maps

- :map metadata attribute complies with the source map v3 spec - Supports index maps - Reduced memory overhead - Improved performance
rails · Nov 11, 2016 · 386ab2b · 386ab2b
1 parent a62c025
commit 386ab2b
Show file tree

Hide file tree

Showing 29 changed files with 684 additions and 293 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,9 @@ Get upgrade notes from Sprockets 3.x to 4.x at https://github.com/rails/sprocket
 
 ## Master
 
+- Source map metadata uses compressed form specified by the [source map v3 spec](https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k). [#402] **[BREAKING]**
+- Generate [index maps](https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit#heading=h.535es3xeprgt) when decoding source maps isn't necessary. [#402]
+- Remove fingerprints from source map files. [#402]
 
 ## 4.0.0.beta4
 

diff --git a/guides/extending_sprockets.md b/guides/extending_sprockets.md
@@ -219,9 +219,9 @@ Example:
 # See metadata section for more info
 {
   dependencies: [].to_set
-  map: [
+  map: {
     # ...
-  ]
+  }
 }
 ```
 
@@ -244,12 +244,7 @@ CoffeeScript file, Sprockets will generate a JavaScript file which is what the b
 this javascript file it helps if you know where the in your original CoffeeScript file the generated JavaScript code
 came from. The source map tells the browser how to map from a generated file to an original.
 
-Sprockets expects an array of hashes for this map. Each hash must have a `:source` key, the name of the original file
-from which generated content came.
-
-```ruby
-return {data: data, map: [{ source: "original.coffee", # ... }]}
-```
+Sprockets expects this map to follow the [source map spec](https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k).
 
 - charset: This key contains the mime charset for an asset.
 

diff --git a/lib/sprockets.rb b/lib/sprockets.rb
@@ -129,7 +129,7 @@ module Sprockets
   register_bundle_metadata_reducer 'application/javascript', :data, proc { String.new("") }, Utils.method(:concat_javascript_sources)
   register_bundle_metadata_reducer '*/*', :links, :+
   register_bundle_metadata_reducer '*/*', :sources, proc { [] }, :+
-  register_bundle_metadata_reducer '*/*', :map, SourceMapUtils.method(:concat_source_maps)
+  register_bundle_metadata_reducer '*/*', :map, proc { |input| { "version" => 3, "file" => PathUtils.split_subpath(input[:load_path], input[:filename]), "sections" => [] } }, SourceMapUtils.method(:concat_source_maps)
 
   require 'sprockets/closure_compressor'
   require 'sprockets/sass_compressor'

diff --git a/lib/sprockets/babel_processor.rb b/lib/sprockets/babel_processor.rb
@@ -42,11 +42,11 @@ def call(input)
 
       result = input[:cache].fetch(@cache_key + [input[:filename]] + [data]) do
         opts = {
-          'sourceRoot' => input[:load_path],
           'moduleRoot' => nil,
           'filename' => input[:filename],
           'filenameRelative' => PathUtils.split_subpath(input[:load_path], input[:filename]),
-          'sourceFileName' => input[:source_path]
+          'sourceFileName' => File.basename(input[:filename]),
+          'sourceMapTarget' => input[:filename]
         }.merge(@options)
 
         if opts['moduleIds'] && opts['moduleRoot']
@@ -57,8 +57,8 @@ def call(input)
         Autoload::Babel::Transpiler.transform(data, opts)
       end
 
-      map = SourceMapUtils.decode_json_source_map(JSON.generate(result['map']))
-      map = SourceMapUtils.combine_source_maps(input[:metadata][:map], map["mappings"])
+      map = SourceMapUtils.format_source_map(result["map"], input)
+      map = SourceMapUtils.combine_source_maps(input[:metadata][:map], map)
 
       { data: result['code'], map: map }
     end

diff --git a/lib/sprockets/bundle.rb b/lib/sprockets/bundle.rb
@@ -32,21 +32,22 @@ def self.call(input)
       end
 
       reducers = Hash[env.match_mime_type_keys(env.config[:bundle_reducers], type).flat_map(&:to_a)]
-      process_bundle_reducers(assets, reducers).merge(dependencies: dependencies, included: assets.map(&:uri))
+      process_bundle_reducers(input, assets, reducers).merge(dependencies: dependencies, included: assets.map(&:uri))
     end
 
     # Internal: Run bundle reducers on set of Assets producing a reduced
     # metadata Hash.
     #
+    # filename - String bundle filename
     # assets - Array of Assets
     # reducers - Array of [initial, reducer_proc] pairs
     #
     # Returns reduced asset metadata Hash.
-    def self.process_bundle_reducers(assets, reducers)
+    def self.process_bundle_reducers(input, assets, reducers)
       initial = {}
       reducers.each do |k, (v, _)|
         if v.respond_to?(:call)
-          initial[k] = v.call
+          initial[k] = v.call(input)
         elsif !v.nil?
           initial[k] = v
         end

diff --git a/lib/sprockets/coffee_script_processor.rb b/lib/sprockets/coffee_script_processor.rb
@@ -21,11 +21,18 @@ def self.call(input)
       data = input[:data]
 
       js, map = input[:cache].fetch([self.cache_key, data]) do
-        result = Autoload::CoffeeScript.compile(data, sourceMap: true, sourceFiles: [input[:source_path]])
-        [result['js'], SourceMapUtils.decode_json_source_map(result['v3SourceMap'])['mappings']]
+        result = Autoload::CoffeeScript.compile(
+          data,
+          sourceMap: true,
+          sourceFiles: [File.basename(input[:filename])],
+          generatedFile: input[:filename]
+        )
+        [result['js'], JSON.parse(result['v3SourceMap'])]
       end
 
+      map = SourceMapUtils.format_source_map(map, input)
       map = SourceMapUtils.combine_source_maps(input[:metadata][:map], map)
+
       { data: js, map: map }
     end
   end

diff --git a/lib/sprockets/loader.rb b/lib/sprockets/loader.rb
@@ -155,8 +155,7 @@ def load_from_unloaded(unloaded)
             name: name,
             content_type: type,
             metadata: {
-              dependencies: dependencies,
-              map: []
+              dependencies: dependencies
             }
           })
           validate_processor_result!(result)

diff --git a/lib/sprockets/path_utils.rb b/lib/sprockets/path_utils.rb
@@ -128,6 +128,30 @@ def join(base, path)
       (Pathname.new(base) + path).to_s
     end
 
+    # Public: Sets pipeline for path
+    #
+    # path       - String path
+    # extensions - List of file extensions
+    # pipeline   - Pipeline
+    #
+    # Examples
+    #
+    #     set_pipeline('path/file.js.erb', config[:mime_exts], config[:pipeline_exts], :source)
+    #     # => 'path/file.source.js.erb'
+    #
+    #     set_pipeline('path/some.file.source.js.erb', config[:mime_exts], config[:pipeline_exts], :debug)
+    #     # => 'path/some.file.debug.js.erb'
+    #
+    # Returns string path with pipeline parsed in
+    def set_pipeline(path, mime_exts, pipeline_exts, pipeline)
+      extension, _ = match_path_extname(path, mime_exts)
+      path.chomp!(extension)
+      pipeline_old, _ = match_path_extname(path, pipeline_exts)
+      path.chomp!(pipeline_old)
+
+      "#{path}.#{pipeline}#{extension}"
+    end
+
     # Internal: Get relative path for root path and subpath.
     #
     # path    - String path

diff --git a/lib/sprockets/preprocessors/default_source_map.rb b/lib/sprockets/preprocessors/default_source_map.rb
@@ -9,17 +9,37 @@ module Preprocessors
     # available.
     class DefaultSourceMap
       def call(input)
-        result = { data: input[:data] }
-        map    = input[:metadata][:map]
+        result        = { data: input[:data] }
+        map           = input[:metadata][:map]
+        filename      = input[:filename]
+        load_path     = input[:load_path]
+        lines         = input[:data].lines.count
+        basename      = File.basename(filename)
+        mime_exts     = input[:environment].config[:mime_exts]
+        pipeline_exts = input[:environment].config[:pipeline_exts]
         if map.nil? || map.empty?
-          result[:map] ||= []
-          input[:data].each_line.with_index do |_, index|
-            line = index + 1
-            result[:map] << { source: input[:source_path], generated: [line , 0], original: [line, 0] }
-          end
+          result[:map] = {
+            "version"   => 3,
+            "file"      => PathUtils.split_subpath(load_path, filename),
+            "mappings"  => default_mappings(lines),
+            "sources"   => [PathUtils.set_pipeline(basename, mime_exts, pipeline_exts, :source)],
+            "names"     => []
+          }
         end
         return result
       end
+
+      private
+
+      def default_mappings(lines)
+        if (lines == 0)
+          ""
+        elsif (lines == 1)
+          "AAAA"
+        else
+          "AAAA;" + "AACA;"*(lines - 2) + "AACA"
+        end
+      end
     end
   end
 end
diff --git a/lib/sprockets/sass_compressor.rb b/lib/sprockets/sass_compressor.rb
@@ -49,15 +49,13 @@ def initialize(options = {})
     def call(input)
       css, map = Autoload::Sass::Engine.new(
         input[:data],
-        @options.merge(filename: 'filename')
+        @options.merge(filename: input[:filename])
       ).render_with_sourcemap('')
 
       css = css.sub("/*# sourceMappingURL= */\n", '')
 
-      map = SourceMapUtils.combine_source_maps(
-        input[:metadata][:map],
-        SourceMapUtils.decode_json_source_map(map.to_json(css_uri: 'uri'))["mappings"]
-      )
+      map = SourceMapUtils.format_source_map(JSON.parse(map.to_json(css_uri: '')), input)
+      map = SourceMapUtils.combine_source_maps(input[:metadata][:map], map)
 
       { data: css, map: map }
     end

diff --git a/lib/sprockets/sass_processor.rb b/lib/sprockets/sass_processor.rb
@@ -80,13 +80,8 @@ def call(input)
 
       css = css.sub("\n/*# sourceMappingURL= */\n", '')
 
-      map = SourceMapUtils.combine_source_maps(
-        input[:metadata][:map],
-        expand_map_sources(
-          SourceMapUtils.decode_json_source_map(map.to_json(css_uri: '', type: :inline))["mappings"],
-          input[:environment]
-        )
-      )
+      map = SourceMapUtils.format_source_map(JSON.parse(map.to_json(css_uri: '')), input)
+      map = SourceMapUtils.combine_source_maps(input[:metadata][:map], map)
 
       # Track all imported files
       sass_dependencies = Set.new([input[:filename]])
@@ -100,17 +95,6 @@ def call(input)
 
     private
 
-    def expand_source(source, env)
-      uri, _ = env.resolve!(source, pipeline: :source)
-      env.load(uri).digest_path
-    end
-
-    def expand_map_sources(mapping, env)
-      mapping.each do |map|
-        map[:source] = expand_source(map[:source], env)
-      end
-    end
-
     # Public: Build the cache store to be used by the Sass engine.
     #
     # input - the input hash.

diff --git a/lib/sprockets/sassc_compressor.rb b/lib/sprockets/sassc_compressor.rb
@@ -9,23 +9,24 @@ def initialize(options = {})
       @options = {
         syntax: :scss,
         style: :compressed,
-        source_map_embed: true,
-        source_map_file: '.'
+        source_map_contents: false,
+        omit_source_map_url: true,
       }.merge(options).freeze
     end
 
     def call(input)
       # SassC requires the template to be modifiable
       input_data = input[:data].frozen? ? input[:data].dup : input[:data]
-      data = Autoload::SassC::Engine.new(input_data, @options.merge(filename: 'filename')).render
+      engine = Autoload::SassC::Engine.new(input_data, @options.merge(filename: input[:filename], source_map_file: "#{input[:filename]}.map")) 
+
+      css = engine.render.sub(/^\n^\/\*# sourceMappingURL=.*\*\/$/m, '')
 
-      match_data = data.match(/(.*)\n\/\*# sourceMappingURL=data:application\/json;base64,(.+) \*\//m)
-      css, map = match_data[1], Base64.decode64(match_data[2])
-
-      map = SourceMapUtils.combine_source_maps(
-        input[:metadata][:map],
-        SourceMapUtils.decode_json_source_map(map)["mappings"]
-      )
+      begin
+        map = SourceMapUtils.format_source_map(JSON.parse(engine.source_map), input)
+        map = SourceMapUtils.combine_source_maps(input[:metadata][:map], map)
+      rescue SassC::NotRenderedError
+        map = input[:metadata][:map]
+      end
 
       { data: css, map: map }
     end

diff --git a/lib/sprockets/sassc_processor.rb b/lib/sprockets/sassc_processor.rb
@@ -27,25 +27,18 @@ def call(input)
         engine.render.sub(/^\n^\/\*# sourceMappingURL=.*\*\/$/m, '')
       end
 
-      map = SourceMapUtils.decode_json_source_map(engine.source_map)
-      sources = map['sources'].map do |s|
-        expand_source(PathUtils.join(File.dirname(input[:filename]), s), input[:environment])
-      end
-
-      map = map["mappings"].each do |m|
-        m[:source] = PathUtils.join(File.dirname(input[:filename]), m[:source])
-      end
-
-      map = SourceMapUtils.combine_source_maps(
-        input[:metadata][:map],
-        expand_map_sources(map, input[:environment])
-      )
+      begin
+        map = SourceMapUtils.format_source_map(JSON.parse(engine.source_map), input)
+        map = SourceMapUtils.combine_source_maps(input[:metadata][:map], map)
 
-      engine.dependencies.each do |dependency|
-        context.metadata[:dependencies] << URIUtils.build_file_digest_uri(dependency.filename)
+        engine.dependencies.each do |dependency|
+          context.metadata[:dependencies] << URIUtils.build_file_digest_uri(dependency.filename)
+        end
+      rescue SassC::NotRenderedError
+        map = input[:metadata][:map]
       end
 
-      context.metadata.merge(data: css, map: map, sources: sources)
+      context.metadata.merge(data: css, map: map)
     end
 
     private
@@ -56,7 +49,7 @@ def engine_options(input, context)
         syntax: self.class.syntax,
         load_paths: input[:environment].paths,
         importer: @importer_class,
-        source_map_contents: true,
+        source_map_contents: false,
         source_map_file: "#{input[:filename]}.map",
         omit_source_map_url: true,
         sprockets: {

diff --git a/lib/sprockets/source_map_comment_processor.rb b/lib/sprockets/source_map_comment_processor.rb
@@ -26,7 +26,7 @@ def self.call(input)
 
       uri, params = URIUtils.parse_asset_uri(input[:uri])
       uri = env.expand_from_root(params[:index_alias]) if params[:index_alias]
-      path = PathUtils.relative_path_from(uri, map.full_digest_path)
+      path = PathUtils.relative_path_from(PathUtils.split_subpath(input[:load_path], uri), map.digest_path)
 
       asset.metadata.merge(
         data: asset.source + (comment % path),

diff --git a/lib/sprockets/source_map_processor.rb b/lib/sprockets/source_map_processor.rb
@@ -17,30 +17,26 @@ def self.call(input)
 
       env = input[:environment]
 
-      uri, _  = env.resolve!(input[:filename], accept: accept)
-      asset   = env.load(uri)
-      map     = asset.metadata[:map] || []
-      sources = asset.metadata[:sources]
+      uri, _ = env.resolve!(input[:filename], accept: accept)
+      asset  = env.load(uri)
+      map    = asset.metadata[:map]
 
       # TODO: Because of the default piplene hack we have to apply dependencies
       #       from compiled asset to the source map, otherwise the source map cache
       #       will never detect the changes from directives
       dependencies = Set.new(input[:metadata][:dependencies])
       dependencies.merge(asset.metadata[:dependencies])
 
-      map.map { |m| m[:source] }.uniq.compact.each do |source|
-        # TODO: Resolve should expect fingerprints
-        fingerprint = source[/-([0-9a-f]{7,128})\.[^.]+\z/, 1]
-        if fingerprint
-          path = source.sub("-#{fingerprint}", "")
-        else
-          path = source
-        end
-        uri, _ = env.resolve!(path)
+      map["file"] = PathUtils.split_subpath(input[:load_path], input[:filename])
+      sources = map["sections"] ? map["sections"].map { |s| s["map"]["sources"] }.flatten : map["sources"]
+
+      sources.each do |source|
+        source = PathUtils.join(File.dirname(map["file"]), source)
+        uri, _ = env.resolve!(source)
         links << uri
       end
 
-      json = env.encode_json_source_map(map, sources: sources, filename: asset.logical_path)
+      json = JSON.generate(map)
 
       { data: json, links: links, dependencies: dependencies }
     end