Refresh theme-liquid-docs on theme-check startup

lib/theme_check/language_server/handler.rb

@@ -90,6 +90,8 @@ def on_initialized(_id, _params)
 
         @configuration.fetch
         @configuration.register_did_change_capability
+
+        ShopifyLiquid::SourceManager.download_or_refresh_files
       end
 
       def on_shutdown(id, _params)

lib/theme_check/remote_asset_file.rb

@@ -31,13 +31,7 @@ def content
       return if @uri.nil?
       return @content unless @content.nil?
 
-      res = Net::HTTP.start(@uri.hostname, @uri.port, use_ssl: @uri.scheme == 'https') do |http|
-        req = Net::HTTP::Get.new(@uri)
-        req['Accept-Encoding'] = 'gzip, deflate, br'
-        http.request(req)
-      end
-
-      @content = res.body
+      @content = request(@uri)
 
     rescue OpenSSL::SSL::SSLError, Zlib::StreamError, *NET_HTTP_EXCEPTIONS
       @contents = ''
@@ -47,5 +41,17 @@ def gzipped_size
       return if @uri.nil?
       @gzipped_size ||= content.bytesize
     end
+
+    private
+
+    def request(uri)
+      res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+        req = Net::HTTP::Get.new(uri)
+        req['Accept-Encoding'] = 'gzip, deflate, br'
+        http.request(req)
+      end
+
+      res.body
+    end
   end
 end

lib/theme_check/shopify_liquid.rb

@@ -4,6 +4,19 @@
 require_relative 'shopify_liquid/documentation'
 require_relative 'shopify_liquid/filter'
 require_relative 'shopify_liquid/object'
+require_relative 'shopify_liquid/source_manager'
 require_relative 'shopify_liquid/source_index'
 require_relative 'shopify_liquid/system_translations'
 require_relative 'shopify_liquid/tag'
+
+require_relative 'shopify_liquid/source_index/base_entry'
+require_relative 'shopify_liquid/source_index/filter_entry'
+require_relative 'shopify_liquid/source_index/object_entry'
+require_relative 'shopify_liquid/source_index/parameter_entry'
+require_relative 'shopify_liquid/source_index/property_entry'
+require_relative 'shopify_liquid/source_index/return_type_entry'
+require_relative 'shopify_liquid/source_index/tag_entry'
+require_relative 'shopify_liquid/source_index/base_state'
+require_relative 'shopify_liquid/source_index/filter_state'
+require_relative 'shopify_liquid/source_index/object_state'
+require_relative 'shopify_liquid/source_index/tag_state'

lib/theme_check/shopify_liquid/source_index.rb

@@ -3,50 +3,52 @@
 require 'json'
 require 'pathname'
 
-require_relative 'source_index/base_entry'
-require_relative 'source_index/filter_entry'
-require_relative 'source_index/object_entry'
-require_relative 'source_index/parameter_entry'
-require_relative 'source_index/property_entry'
-require_relative 'source_index/return_type_entry'
-require_relative 'source_index/tag_entry'
-
-require_relative './source_manager'
-
 module ThemeCheck
   module ShopifyLiquid
     class SourceIndex
       class << self
         def filters
-          @filters ||= load_file(:filters)
-            .map { |hash| SourceIndex::FilterEntry.new(hash) }
+          @filters = nil if FilterState.outdated?
+
+          @filters ||= FilterState.mark_up_to_date &&
+            load_file(:filters)
+              .map { |hash| FilterEntry.new(hash) }
         end
 
         def objects
-          @objects ||= load_file(:objects)
-            .concat(built_in_objects)
-            .map { |hash| SourceIndex::ObjectEntry.new(hash) }
+          @objects = nil if ObjectState.outdated?
+
+          @objects ||= ObjectState.mark_up_to_date &&
+            load_file(:objects)
+              .concat(built_in_objects)
+              .map { |hash| ObjectEntry.new(hash) }
         end
 
         def tags
-          @tags ||= load_file(:tags)
-            .map { |hash| SourceIndex::TagEntry.new(hash) }
+          @tags = nil if TagState.outdated?
+
+          @tags ||= TagState.mark_up_to_date &&
+            load_file(:tags)
+              .map { |hash| TagEntry.new(hash) }
         end
 
         private
 
         def load_file(file_name)
-          read_json(SourceManager.local_path(file_name))
+          read_json(local_path!(file_name))
         end
 
-        def read_json(path)
-          SourceManager.download_or_refresh_files
+        def local_path!(file_name)
+          SourceManager.download unless SourceManager.has_required_files?
+          SourceManager.local_path(file_name)
+        end
 
+        def read_json(path)
           JSON.parse(path.read)
         end
 
         def built_in_objects
-          load_file("../built_in_liquid_objects")
+          load_file('../built_in_liquid_objects')
         end
       end
     end

lib/theme_check/shopify_liquid/source_index/base_state.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module ThemeCheck
+  module ShopifyLiquid
+    class SourceIndex
+      class BaseState
+        class << self
+          def mark_outdated
+            @up_to_date = false
+          end
+
+          def mark_up_to_date
+            @up_to_date = true
+          end
+
+          def outdated?
+            @up_to_date == false
+          end
+        end
+      end
+    end
+  end
+end

lib/theme_check/shopify_liquid/source_index/filter_state.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ThemeCheck
+  module ShopifyLiquid
+    class SourceIndex
+      class FilterState < BaseState
+        @up_to_date = false
+      end
+    end
+  end
+end

lib/theme_check/shopify_liquid/source_index/object_state.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ThemeCheck
+  module ShopifyLiquid
+    class SourceIndex
+      class ObjectState < BaseState
+        @up_to_date = false
+      end
+    end
+  end
+end

lib/theme_check/shopify_liquid/source_index/tag_state.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ThemeCheck
+  module ShopifyLiquid
+    class SourceIndex
+      class TagState < BaseState
+        @up_to_date = false
+      end
+    end
+  end
+end

lib/theme_check/shopify_liquid/source_manager.rb

@@ -1,81 +1,110 @@
 # frozen_string_literal: true
 
+require 'net/http'
 require 'pathname'
+require 'tmpdir'
 
 module ThemeCheck
   module ShopifyLiquid
-    module SourceManager
-      extend self
-
-      def download_or_refresh_files
-        if has_required_files?
-          # TODO: https://github.com/Shopify/theme-check/issues/651
-          #
-          # Refresh files if they exist locally
-        else
-          download
+    class SourceManager
+      REQUIRED_FILE_NAMES = [:filters, :objects, :tags, :latest].freeze
+
+      class << self
+        def download_or_refresh_files(destination = default_destination)
+          if has_required_files?(destination)
+            refresh(destination)
+          else
+            download(destination)
+          end
         end
-      end
 
-      def download
-        create_destination unless destination_exist?
+        def download(destination = default_destination)
+          Dir.mkdir(destination) unless destination.exist?
 
-        required_file_names.each do |file_name|
-          download_file(local_path(file_name), remote_path(file_name))
+          REQUIRED_FILE_NAMES.each do |file_name|
+            download_file(local_path(file_name, destination), remote_path(file_name))
+          end
         end
-      end
 
-      def local_path(file_name)
-        documentation_directory + "#{file_name}.json"
-      end
+        def refresh(destination = default_destination)
+          refresh_threads << Thread.new { refresh_thread(destination) }
+        end
 
-      private
+        def local_path(file_name, destination = default_destination)
+          destination + "#{file_name}.json"
+        end
 
-      DOCUMENTATION_FETCH_URL = "https://github.com/Shopify/theme-liquid-docs/raw/main/data"
-      DOCUMENTATION_DIRECTORY = Pathname.new("#{__dir__}/../../../data/shopify_liquid/documentation")
-      REQUIRED_FILE_NAMES = [:filters, :objects, :tags].freeze
+        def has_required_files?(destination = default_destination)
+          REQUIRED_FILE_NAMES.all? { |file_name| local_path(file_name, destination).exist? }
+        end
 
-      def remote_path(file_name)
-        "#{documentation_fetch_url}/#{file_name}.json"
-      end
+        def wait_downloads
+          refresh_threads.each(&:join)
+        end
 
-      def download_file(local, remote)
-        File.open(local, "wb") do |file|
-          content = open_uri(remote)
+        private
 
-          file.write(content)
+        def refresh_thread(destination)
+          return unless refresh_needed?(destination)
+
+          Dir.mktmpdir do |tmp_dir|
+            download(Pathname.new(tmp_dir))
+
+            FileUtils.cp_r("#{tmp_dir}/.", destination)
+
+            mark_all_indexes_outdated
+          end
         end
-      end
 
-      def open_uri(remote)
-        # require at usage point to not slow down theme-check on startup
-        require 'open-uri'
+        def refresh_needed?(destination)
+          local_latest_content = local_path(:latest, destination).read
+          remote_latest_content = open_uri(remote_path(:latest))
 
-        URI.parse(remote).open.read
-      end
+          revision(local_latest_content) != revision(remote_latest_content)
+        end
 
-      def destination_exist?
-        documentation_directory.exist?
-      end
+        def revision(json_content)
+          # Raise an error if revision isn't found to avoid returning nil
+          JSON.parse(json_content).fetch('revision')
+        end
 
-      def create_destination
-        Dir.mkdir(documentation_directory)
-      end
+        def remote_path(file_name)
+          "https://raw.githubusercontent.com/Shopify/theme-liquid-docs/main/data/#{file_name}.json"
+        end
 
-      def documentation_directory
-        DOCUMENTATION_DIRECTORY
-      end
+        def download_file(local_path, remote_uri)
+          File.open(local_path, "wb") do |file|
+            content = open_uri(remote_uri)
+            file.write(content)
+          end
+        end
 
-      def documentation_fetch_url
-        DOCUMENTATION_FETCH_URL
-      end
+        def mark_all_indexes_outdated
+          SourceIndex::FilterState.mark_outdated
+          SourceIndex::ObjectState.mark_outdated
+          SourceIndex::TagState.mark_outdated
+        end
 
-      def required_file_names
-        REQUIRED_FILE_NAMES
-      end
+        # State
 
-      def has_required_files?
-        required_file_names.all? { |file_name| local_path(file_name).exist? }
+        def default_destination
+          @default_destination ||= Pathname.new("#{__dir__}/../../../data/shopify_liquid/documentation")
+        end
+
+        def refresh_threads
+          @refresh_threads ||= []
+        end
+
+        def open_uri(uri_str)
+          uri = URI.parse(uri_str)
+
+          res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+            req = Net::HTTP::Get.new(uri)
+            http.request(req)
+          end
+
+          res.body
+        end
       end
     end
   end

test/data/shopify_liquid/built_in_liquid_objects.json

@@ -0,0 +1 @@
+[]

test/data/shopify_liquid/documentation/latest.json

@@ -0,0 +1 @@
+{ "revision": "59abb1dcfe7ca9f9560356353e3d23b46489a330" }

test/remote_asset_file_test.rb

@@ -3,8 +3,6 @@
 
 module ThemeCheck
   class RemoteAssetFileTest < Minitest::Test
-    FakeResponse = Struct.new(:body)
-
     def setup
       @src = 'https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js'
       @asset = RemoteAssetFile.from_src(@src)
@@ -16,10 +14,8 @@ def test_instance_caching
     end
 
     def test_network_request
-      Net::HTTP.any_instance
-        .expects(:request)
-        .with { |req| req['Accept-Encoding'] == 'gzip, deflate, br' }
-        .returns(FakeResponse.new("..."))
+      @asset.expects(:request).with(uri).returns("...")
+
       assert_equal("...", @asset.content)
       assert_equal(3, @asset.gzipped_size)
     end
@@ -35,5 +31,11 @@ def test_handles_eaddr_not_avail_errors
       assert(asset.gzipped_size == 0)
       assert(asset.content.empty?)
     end
+
+    private
+
+    def uri
+      RemoteAssetFile.uri(@src)
+    end
   end
 end