refactor: migration system (#180)

  1. Create the `schema` table if needed. This includes a version column.
  2. If the max version selected from the `schema` table is equal to the current version, we noop and halt.
     Else, if the max version is less than the current one (compiled into this module) or nil
     we "upgrade" the database.
  3. Unless halting, we drop the non-meta tables, and then create them from scratch
  4. Return a value to signal to the caller that re-indexing is necessary.

lib/next_ls.ex

@@ -133,7 +133,7 @@ defmodule NextLS do
             [] ->
               nil
 
-            [[_pk, _mod, file, _type, _name, line, column] | _] ->
+            [[_pk, _mod, file, _type, _name, line, column | _] | _] ->
               %Location{
                 uri: "file://#{file}",
                 range: %Range{
@@ -236,6 +236,7 @@ defmodule NextLS do
       if query == "" do
         true
       else
+        # TODO: sqlite has a regexp feature, this can be done in sql most likely
         to_string(sym) =~ query
       end
     end
@@ -596,7 +597,12 @@ defmodule NextLS do
 
     task =
       Task.Supervisor.async_nolink(lsp.assigns.task_supervisor, fn ->
-        {name, Runtime.compile(runtime_pid)}
+        {_, %{mode: mode}} =
+          dispatch(lsp.assigns.registry, :databases, fn entries ->
+            Enum.find(entries, fn {_, %{runtime: runtime}} -> runtime == name end)
+          end)
+
+        {name, Runtime.compile(runtime_pid, force: mode == :reindex)}
       end)
 
     refresh_refs = Map.put(lsp.assigns.refresh_refs, task.ref, {token, "Compiled!"})

lib/next_ls/db.ex

@@ -27,10 +27,12 @@ defmodule NextLS.DB do
     registry = Keyword.fetch!(args, :registry)
     logger = Keyword.fetch!(args, :logger)
     activity = Keyword.fetch!(args, :activity)
-    Registry.register(registry, :databases, %{})
+    runtime = Keyword.fetch!(args, :runtime)
+
     {:ok, conn} = Exqlite.Basic.open(file)
+    {:ok, mode} = NextLS.DB.Schema.init({conn, logger})
 
-    NextLS.DB.Schema.init({conn, logger})
+    Registry.register(registry, :databases, %{mode: mode, runtime: runtime})
 
     {:ok,
      %{
@@ -63,7 +65,7 @@ defmodule NextLS.DB do
       )
 
     symbols =
-      for [_pk, module, file, type, name, line, column] <- rows do
+      for [_pk, module, file, type, name, line, column | _] <- rows do
         %{
           module: module,
           file: file,

lib/next_ls/db/schema.ex

@@ -1,42 +1,109 @@
 defmodule NextLS.DB.Schema do
-  @moduledoc false
+  @moduledoc """
+  The Sqlite3 database schema.
+
+  First, you are probably asking yourself, why doesn't this use Ecto?
+
+  Well, because I didn't want to. And also because I am attempting to restrict this
+  project to as few dependencies as possible.
+
+  The Ecto migration system is meant for highly durable data, that can't and shouldn't be lost,
+  whereas the data here is more like a fast and efficient cache.
+
+  Rather than coming up with convoluted data migration strategies, we follow the following algorithm.
+
+  1. Create the `schema` table if needed. This includes a version column.
+  2. If the max version selected from the `schema` table is equal to the current version, we noop and halt.
+     Else, if the max version is less than the current one (compiled into this module) or nil
+     we "upgrade" the database.
+  3. Unless halting, we drop the non-meta tables, and then create them from scratch
+  4. Return a value to signal to the caller that re-indexing is necessary.
+  """
   import NextLS.DB.Query
 
   alias NextLS.DB
 
+  @version 2
+
   def init(conn) do
+    # FIXME: this is odd tech debt. not a big deal but is confusing
+    {_, logger} = conn
+
+    NextLS.Logger.log(logger, "Beginning DB migration...")
+
     DB.__query__(
       conn,
       ~Q"""
-      CREATE TABLE IF NOT EXISTS symbols (
+      CREATE TABLE IF NOT EXISTS schema (
           id integer PRIMARY KEY,
-          module text NOT NULL,
-          file text NOT NULL,
-          type text NOT NULL,
-          name text NOT NULL,
-          line integer NOT NULL,
-          column integer NOT NULL
+          version integer NOT NULL DEFAULT 1,
+          inserted_at text NOT NULL DEFAULT CURRENT_TIMESTAMP
       );
       """,
       []
     )
 
-    DB.__query__(
-      conn,
-      ~Q"""
-      CREATE TABLE IF NOT EXISTS 'references' (
-          id integer PRIMARY KEY,
-          identifier text NOT NULL,
-          arity integer,
-          file text NOT NULL,
-          type text NOT NULL,
-          module text NOT NULL,
-          start_line integer NOT NULL,
-          start_column integer NOT NULL,
-          end_line integer NOT NULL,
-          end_column integer NOT NULL)
-      """,
-      []
-    )
+    result =
+      case DB.__query__(conn, ~Q"SELECT MAX(version) FROM schema;", []) do
+        [[version]] when version == @version ->
+          NextLS.Logger.info(logger, "Database is on the latest version: #{@version}")
+          {:ok, :noop}
+
+        result ->
+          version = with([[version]] <- result, do: version) || 0
+
+          NextLS.Logger.info(logger, """
+          Database is being upgraded from version #{version} to #{@version}.
+
+          This will trigger a full recompilation in order to re-index your codebase.
+          """)
+
+          DB.__query__(conn, ~Q"INSERT INTO schema (version) VALUES (?);", [@version])
+
+          DB.__query__(conn, ~Q"DROP TABLE IF EXISTS symbols;", [])
+          DB.__query__(conn, ~Q"DROP TABLE IF EXISTS 'references';", [])
+
+          DB.__query__(
+            conn,
+            ~Q"""
+            CREATE TABLE IF NOT EXISTS symbols (
+                id integer PRIMARY KEY,
+                module text NOT NULL,
+                file text NOT NULL,
+                type text NOT NULL,
+                name text NOT NULL,
+                line integer NOT NULL,
+                column integer NOT NULL,
+                inserted_at text NOT NULL DEFAULT CURRENT_TIMESTAMP
+            );
+            """,
+            []
+          )
+
+          DB.__query__(
+            conn,
+            ~Q"""
+            CREATE TABLE IF NOT EXISTS 'references' (
+                id integer PRIMARY KEY,
+                identifier text NOT NULL,
+                arity integer,
+                file text NOT NULL,
+                type text NOT NULL,
+                module text NOT NULL,
+                start_line integer NOT NULL,
+                start_column integer NOT NULL,
+                end_line integer NOT NULL,
+                end_column integer NOT NULL,
+                inserted_at text NOT NULL DEFAULT CURRENT_TIMESTAMP
+            )
+            """,
+            []
+          )
+
+          {:ok, :reindex}
+      end
+
+    NextLS.Logger.log(logger, "Finished DB migration...")
+    result
   end
 end

lib/next_ls/definition.ex

@@ -5,7 +5,7 @@ defmodule NextLS.Definition do
   alias NextLS.DB
 
   def fetch(file, {line, col}, db) do
-    with [[_pk, identifier, _arity, _file, type, module, _start_l, _start_c, _end_l, _end_c]] <-
+    with [[_pk, identifier, _arity, _file, type, module, _start_l, _start_c, _end_l, _end_c | _]] <-
            DB.query(
              db,
              ~Q"""

lib/next_ls/runtime.ex

@@ -33,8 +33,8 @@ defmodule NextLS.Runtime do
     end
   end
 
-  def compile(server) do
-    GenServer.call(server, :compile, :infinity)
+  def compile(server, opts \\ []) do
+    GenServer.call(server, {:compile, opts}, :infinity)
   end
 
   @impl GenServer
@@ -164,6 +164,7 @@ defmodule NextLS.Runtime do
       {:ok,
        %{
          name: name,
+         working_dir: working_dir,
          compiler_ref: nil,
          port: port,
          task_supervisor: task_supervisor,
@@ -197,9 +198,13 @@ defmodule NextLS.Runtime do
     {:reply, {:ok, reply}, state}
   end
 
-  def handle_call(:compile, from, %{node: node} = state) do
+  def handle_call({:compile, opts}, from, %{node: node} = state) do
     task =
       Task.Supervisor.async_nolink(state.task_supervisor, fn ->
+        if opts[:force] do
+          File.rm_rf!(Path.join(state.working_dir, ".elixir-tools/_build"))
+        end
+
         case :rpc.call(node, :_next_ls_private_compiler, :compile, []) do
           {:badrpc, error} ->
             NextLS.Logger.error(state.logger, "Bad RPC call to node #{node}: #{inspect(error)}")

lib/next_ls/runtime/supervisor.ex

@@ -28,7 +28,12 @@ defmodule NextLS.Runtime.Supervisor do
       {NextLS.DB.Activity,
        logger: logger, name: db_activity, lsp: lsp, timeout: Application.get_env(:next_ls, :indexing_timeout)},
       {NextLS.DB,
-       logger: logger, file: "#{hidden_folder}/nextls.db", registry: registry, name: db_name, activity: db_activity},
+       logger: logger,
+       file: "#{hidden_folder}/nextls.db",
+       registry: registry,
+       name: db_name,
+       runtime: name,
+       activity: db_activity},
       {NextLS.Runtime, init_arg[:runtime] ++ [name: name, registry: registry, parent: sidecar_name, db: db_name]}
     ]