On demand indexing (erlang-ls#1260)

apps/els_core/src/els_config.erl

@@ -273,8 +273,8 @@ consult_config([Path | Paths], ReportMissingConfig) ->
       [Config] -> {Path, Config}
   catch
     Class:Error ->
-      ?LOG_WARNING( "Could not read config file: path=~p class=~p error=~p"
-                  , [Path, Class, Error]),
+      ?LOG_DEBUG( "Could not read config file: path=~p class=~p error=~p"
+                , [Path, Class, Error]),
       consult_config(Paths, ReportMissingConfig)
   end.
 

apps/els_core/src/els_utils.erl

@@ -110,7 +110,7 @@ find_header(Id) ->
       {ok, Uri};
     [] ->
       FileName = atom_to_list(Id) ++ ".hrl",
-      els_indexing:find_and_index_file(FileName)
+      els_indexing:find_and_deeply_index_file(FileName)
   end.
 
 %% @doc Look for a module in the DB
@@ -128,14 +128,14 @@ find_module(Id) ->
 find_modules(Id) ->
   {ok, Candidates} = els_dt_document_index:lookup(Id),
   case [Uri || #{kind := module, uri := Uri} <- Candidates] of
-      [] ->
-          FileName = atom_to_list(Id) ++ ".erl",
-          case els_indexing:find_and_index_file(FileName) of
-              {ok, Uri} -> {ok, [Uri]};
-              Error -> Error
-          end;
-      Uris ->
-          {ok, prioritize_uris(Uris)}
+    [] ->
+      FileName = atom_to_list(Id) ++ ".erl",
+      case els_indexing:find_and_deeply_index_file(FileName) of
+        {ok, Uri} -> {ok, [Uri]};
+        Error -> Error
+      end;
+    Uris ->
+      {ok, prioritize_uris(Uris)}
   end.
 
 %% @doc Look for a document in the DB.
@@ -150,13 +150,13 @@ lookup_document(Uri) ->
       {ok, Document};
     {ok, []} ->
       Path = els_uri:path(Uri),
-      {ok, Uri} = els_indexing:index_file(Path),
+      {ok, Uri} = els_indexing:shallow_index(Path, app),
       case els_dt_document:lookup(Uri) of
         {ok, [Document]} ->
           {ok, Document};
-        Error ->
-          ?LOG_INFO("Document lookup failed [error=~p] [uri=~p]", [Error, Uri]),
-          {error, Error}
+        {ok, []} ->
+          ?LOG_INFO("Document lookup failed [uri=~p]", [Uri]),
+          {error, document_lookup_failed}
       end
   end.
 

apps/els_lsp/src/els_background_job.erl

@@ -187,13 +187,21 @@ terminate(normal, #{ config := #{on_complete := OnComplete}
   ?LOG_DEBUG("Background job completed.", []),
   OnComplete(InternalState),
   ok;
-terminate(Reason, #{ config := #{on_error := OnError}
+terminate(Reason, #{ config := #{ on_error := OnError
+                                , title := Title
+                                }
                    , internal_state := InternalState
                    , token := Token
                    , total := Total
                    , progress_enabled := ProgressEnabled
                    }) ->
-  ?LOG_WARNING( "Background job aborted. [reason=~p]", [Reason]),
+  case Reason of
+    shutdown ->
+      ?LOG_DEBUG("Background job terminated.", []);
+    _ ->
+      ?LOG_ERROR( "Background job aborted. [reason=~p] [title=~p",
+                  [Reason, Title])
+  end,
   notify_end(Token, Total, ProgressEnabled),
   OnError(InternalState),
   ok.

apps/els_lsp/src/els_buffer_server.erl

@@ -0,0 +1,126 @@
+%%%=============================================================================
+%%% @doc Buffer edits to an open buffer to avoid re-indexing too often.
+%%% @end
+%%%=============================================================================
+-module(els_buffer_server).
+
+%%==============================================================================
+%% API
+%%==============================================================================
+-export([ new/2
+        , stop/1
+        , apply_edits/2
+        , flush/1
+        ]).
+
+-export([ start_link/2 ]).
+
+%%==============================================================================
+%% Callbacks for the gen_server behaviour
+%%==============================================================================
+-behaviour(gen_server).
+-export([ init/1
+        , handle_call/3
+        , handle_cast/2
+        , handle_info/2
+        ]).
+
+%%==============================================================================
+%% Macro Definitions
+%%==============================================================================
+-define(FLUSH_DELAY, 200). %% ms
+
+%%==============================================================================
+%% Type Definitions
+%%==============================================================================
+-type text() :: binary().
+-type state() :: #{ uri := uri()
+                  , text := text()
+                  , ref := undefined | reference()
+                  , pending := [{pid(), any()}]
+                  }.
+-type buffer() :: pid().
+
+%%==============================================================================
+%% Includes
+%%==============================================================================
+-include_lib("kernel/include/logger.hrl").
+-include("els_lsp.hrl").
+
+%%==============================================================================
+%% API
+%%==============================================================================
+-spec new(uri(), text()) -> {ok, pid()}.
+new(Uri, Text) ->
+  supervisor:start_child(els_buffer_sup, [Uri, Text]).
+
+-spec stop(buffer()) -> ok.
+stop(Buffer) ->
+  supervisor:terminate_child(els_buffer_sup, Buffer).
+
+-spec apply_edits(buffer(), [els_text:edit()]) -> ok.
+apply_edits(Buffer, Edits) ->
+  gen_server:cast(Buffer, {apply_edits, Edits}).
+
+-spec flush(buffer()) -> text().
+flush(Buffer) ->
+  gen_server:call(Buffer, {flush}).
+
+-spec start_link(uri(), text()) -> {ok, buffer()}.
+start_link(Uri, Text) ->
+  gen_server:start_link(?MODULE, {Uri, Text}, []).
+
+%%==============================================================================
+%% Callbacks for the gen_server behaviour
+%%==============================================================================
+-spec init({uri(), text()}) -> {ok, state()}.
+init({Uri, Text}) ->
+  {ok, #{ uri => Uri, text => Text, ref => undefined, pending => [] }}.
+
+-spec handle_call(any(), {pid(), any()}, state()) -> {reply, any(), state()}.
+handle_call({flush}, From, State) ->
+  #{uri := Uri, ref := Ref0, pending := Pending0} = State,
+  ?LOG_DEBUG("[~p] Flushing request [uri=~p]", [?MODULE, Uri]),
+  cancel_flush(Ref0),
+  Ref = schedule_flush(),
+  {noreply, State#{ref => Ref, pending => [From|Pending0]}};
+handle_call(Request, _From, State) ->
+  {reply, {not_implemented, Request}, State}.
+
+-spec handle_cast(any(), state()) -> {noreply, state()}.
+handle_cast({apply_edits, Edits}, #{uri := Uri} = State) ->
+  ?LOG_DEBUG("[~p] Applying edits [uri=~p] [edits=~p]", [?MODULE, Uri, Edits]),
+  #{text := Text0, ref := Ref0} = State,
+  cancel_flush(Ref0),
+  Text = els_text:apply_edits(Text0, Edits),
+  Ref = schedule_flush(),
+  {noreply, State#{text => Text, ref => Ref}}.
+
+-spec handle_info(any(), state()) -> {noreply, state()}.
+handle_info(flush, #{uri := Uri, text := Text, pending := Pending0} = State) ->
+  ?LOG_DEBUG("[~p] Flushing [uri=~p]", [?MODULE, Uri]),
+  do_flush(Uri, Text),
+  [gen_server:reply(From, Text) || From <- Pending0],
+  {noreply, State#{pending => [], ref => undefined}};
+handle_info(_Request, State) ->
+  {noreply, State}.
+
+%%==============================================================================
+%% Internal Functions
+%%==============================================================================
+
+-spec schedule_flush() -> reference().
+schedule_flush() ->
+  erlang:send_after(?FLUSH_DELAY, self(), flush).
+
+-spec cancel_flush(undefined | reference()) -> ok.
+cancel_flush(undefined) ->
+  ok;
+cancel_flush(Ref) ->
+  erlang:cancel_timer(Ref),
+  ok.
+
+-spec do_flush(uri(), text()) -> ok.
+do_flush(Uri, Text) ->
+  {ok, Document} = els_utils:lookup_document(Uri),
+  els_indexing:deep_index(Document#{text => Text}).

apps/els_lsp/src/els_buffer_sup.erl

@@ -0,0 +1,47 @@
+%%==============================================================================
+%% Supervisor for Buffers
+%%==============================================================================
+-module(els_buffer_sup).
+
+%%==============================================================================
+%% Behaviours
+%%==============================================================================
+-behaviour(supervisor).
+
+%%==============================================================================
+%% Exports
+%%==============================================================================
+
+%% API
+-export([ start_link/0 ]).
+
+%% Supervisor Callbacks
+-export([ init/1 ]).
+
+%%==============================================================================
+%% Defines
+%%==============================================================================
+-define(SERVER, ?MODULE).
+
+%%==============================================================================
+%% API
+%%==============================================================================
+-spec start_link() -> {ok, pid()}.
+start_link() ->
+  supervisor:start_link({local, ?SERVER}, ?MODULE, []).
+
+%%==============================================================================
+%% Supervisor callbacks
+%%==============================================================================
+-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}.
+init([]) ->
+  SupFlags = #{ strategy  => simple_one_for_one
+              , intensity => 5
+              , period    => 60
+              },
+  ChildSpecs = [#{ id       => els_buffer_sup
+                 , start    => {els_buffer_server, start_link, []}
+                 , restart  => temporary
+                 , shutdown => 5000
+                 }],
+  {ok, {SupFlags, ChildSpecs}}.

apps/els_lsp/src/els_completion_provider.erl

@@ -44,8 +44,18 @@ handle_request({completion, Params}, State) ->
                             }
    , <<"textDocument">> := #{<<"uri">> := Uri}
    } = Params,
-  ok = els_index_buffer:flush(Uri),
-  {ok, #{text := Text} = Document} = els_utils:lookup_document(Uri),
+  %% Ensure there are no pending changes.
+  {ok, Document} = els_utils:lookup_document(Uri),
+  #{buffer := Buffer, text := Text0} = Document,
+  Text = case Buffer of
+           undefined ->
+             %% This clause is only kept due to the current test suites,
+             %% where LSP clients can trigger a completion request
+             %% before a did_open
+             Text0;
+           _ ->
+             els_buffer_server:flush(Buffer)
+         end,
   Context = maps:get( <<"context">>
                     , Params
                     , #{ <<"triggerKind">> => ?COMPLETION_TRIGGER_KIND_INVOKED }