chore/sqlite to enginev3

package.json

@@ -111,7 +111,7 @@
     "backend:updateDendronDeps": "node bootstrap/backend/updateDendronhqDeps.js",
     "stash:unstaged": "git stash save -k 'pre-linting-stash'",
     "stash:pop": "git stash && git stash pop stash@{1} && git read-tree stash && git stash drop",
-    "test": "cross-env LOG_LEVEL=error yarn jest",
+    "test": "yarn jest",
     "test:cli": "cross-env LOG_LEVEL=error npx jest --selectProjects non-plugin-tests --forceExit",
     "test:cli:update-snapshots": "yarn test:cli -u"
   },

packages/common-server/src/logger.ts

@@ -1,7 +1,6 @@
 // import pino from "pino";
 
 import { Disposable, env } from "@dendronhq/common-all";
-import _ from "lodash";
 import pino from "pino";
 
 export type LogLvl = "debug" | "info" | "error";

packages/engine-server/src/DendronEngineV3.ts

@@ -66,6 +66,7 @@ import {
   WriteSchemaResp,
   BacklinkUtils,
   DLinkUtils,
+  TimeUtils,
 } from "@dendronhq/common-all";
 import {
   createLogger,
@@ -85,6 +86,7 @@ import { LinkUtils } from "@dendronhq/unified";
 import { NodeJSFileStore } from "./store";
 import { HookUtils, RequireHookResp } from "./topics/hooks";
 import { EngineUtils } from "./utils/engineUtils";
+import { SQLiteMetadataStore } from "./drivers";
 
 type DendronEngineOptsV3 = {
   wsRoot: string;
@@ -184,6 +186,7 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine {
    * Does not throw error but returns it
    */
   async init(): Promise<DEngineInitResp> {
+    const ctx = "DendronEngineV3:init";
     const defaultResp = {
       notes: {},
       schemas: {},
@@ -262,7 +265,7 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine {
           error = new DendronCompositeError(allErrors);
       }
       this.logger.info({
-        ctx: "init:ext",
+        ctx,
         error,
         storeError: allErrors,
         hookErrors,
@@ -942,11 +945,47 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine {
   private async initNotes(
     schemas: SchemaModuleDict
   ): Promise<RespWithOptError<NotePropsByIdDict>> {
-    const ctx = "DEngine:initNotes";
-    this.logger.info({ ctx, msg: "enter" });
+    const ctx = "DEngineV3:initNotes";
     let errors: IDendronError[] = [];
     let notesFname: NotePropsByFnameDict = {};
     const start = process.hrtime();
+    const enableSQLITE =
+      DConfig.readConfigSync(this.wsRoot).workspace.metadataStore === "sqlite";
+    this.logger.info({ ctx, msg: "enter", enableSQLITE });
+    if (enableSQLITE) {
+      // eslint-disable-next-line no-new
+      const store = new SQLiteMetadataStore({
+        wsRoot: this.wsRoot,
+        force: true,
+      });
+
+      // sleep until store is done
+      const output = await TimeUtils.awaitWithLimit(
+        { limitMs: 6e4 },
+        async () => {
+          while (store.status === "loading") {
+            this.logger.info({ ctx, msg: "downloading sql dependencies..." });
+            // eslint-disable-next-line no-await-in-loop
+            await TimeUtils.sleep(1000);
+          }
+          return;
+        }
+      );
+
+      this.logger.info({
+        ctx,
+        msg: "checking if sql is initialized...",
+        output,
+      });
+      if (!(await SQLiteMetadataStore.isDBInitialized())) {
+        this.logger.info({
+          ctx,
+          msg: "db not initialized",
+        });
+        await SQLiteMetadataStore.createAllTables();
+        await SQLiteMetadataStore.createWorkspace(this.wsRoot);
+      }
+    }
 
     const allNotesList = await Promise.all(
       this.vaults.map(async (vault) => {
@@ -996,6 +1035,13 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine {
             numEntries: _.size(notesById),
             numCacheUpdates: notesCache.numCacheMisses,
           });
+          if (enableSQLITE) {
+            if (!(await SQLiteMetadataStore.isVaultInitialized(vault))) {
+              await SQLiteMetadataStore.prisma().dVault.create({
+                data: { fsPath: vault.fsPath, wsRoot: this.wsRoot },
+              });
+            }
+          }
           return notesById;
         }
         return {};
@@ -1010,12 +1056,27 @@ export class DendronEngineV3 extends EngineV3Base implements DEngine {
       },
       notesWithLinks
     );
+
+    if (enableSQLITE) {
+      this.logger.info({ ctx, msg: "updating notes in sql" });
+      // TODO: OPTIMIZE
+      await SQLiteMetadataStore.deleteAllNotes();
+      await SQLiteMetadataStore.bulkInsertAllNotes({
+        notesIdDict: allNotes,
+      });
+
+      // const allNotes = await SQLiteetadataStore.prisma().note.findMany();
+      this.logger.info({
+        ctx,
+        msg: "post:update notes in sql",
+      });
+    }
     const duration = getDurationMilliseconds(start);
     this.logger.info({ ctx, msg: `time to init notes: "${duration}" ms` });
 
     return {
       data: allNotes,
-      error: new DendronCompositeError(errors),
+      error: errors.length > 0 ? new DendronCompositeError(errors) : undefined,
     };
   }
 

packages/engine-server/src/drivers/SQLiteMetadataStore.ts

@@ -227,7 +227,8 @@ export class SQLiteMetadataStore implements IDataStore<string, NotePropsMeta> {
     const fullQuery = sqlBegin + sqlEnd;
     // eslint-disable-next-line no-useless-catch
     try {
-      await prisma.$queryRawUnsafe(fullQuery);
+      const resp = await prisma.$executeRawUnsafe(fullQuery);
+      return { query: fullQuery, resp };
     } catch (error) {
       // uncomment to log
       // console.log("---> ERROR START");
@@ -238,7 +239,11 @@ export class SQLiteMetadataStore implements IDataStore<string, NotePropsMeta> {
       // console.log("---> ERROR END");
       throw error;
     }
-    return { query: fullQuery };
+  }
+
+  static deleteAllNotes() {
+    const raw = `DELETE FROM Note`;
+    return getPrismaClient().$queryRawUnsafe(raw);
   }
 
   static async search(

packages/engine-server/src/drivers/file/storev2.ts

@@ -110,7 +110,7 @@ export class FileStorage implements DStore {
     this.anchors = [];
     this.logger = logger;
     this.config = config;
-    const ctx = "FileStorageV2";
+    const ctx = "FileStorage:constructor";
     this.logger.info({ ctx, wsRoot, vaults, level: this.logger.level });
     this.engine = props.engine;
   }
@@ -119,14 +119,17 @@ export class FileStorage implements DStore {
     const ctx = "FileStorage:init";
     let errors: IDendronError<any>[] = [];
     try {
+      this.logger.info({ ctx, msg: "pre:initSchema" });
       const resp = await this.initSchema();
       if (resp.error) {
         errors.push(FileStorage.createMalformedSchemaError(resp));
       }
       resp.data.map((ent) => {
         this.schemas[ent.root.id] = ent;
       });
+      this.logger.info({ ctx, msg: "pre:initNotes" });
       const { errors: initErrors } = await this.initNotes();
+      this.logger.info({ ctx, msg: "post:initNotes", initErrors });
       errors = errors.concat(initErrors);
       this.logger.info({ ctx, msg: "post:initNotes", errors });
 
@@ -504,7 +507,7 @@ export class FileStorage implements DStore {
   async initNotes(): Promise<{
     errors: IDendronError[];
   }> {
-    const ctx = "initNotes";
+    const ctx = "FileStorage:initNotes";
     this.logger.info({ ctx, msg: "enter" });
 
     let notesWithLinks: NoteProps[] = [];
@@ -578,7 +581,10 @@ export class FileStorage implements DStore {
 
     this._addBacklinks({ notesWithLinks, allNotes });
     const duration = getDurationMilliseconds(start);
-    this.logger.info({ ctx, msg: `time to init notes: "${duration}" ms` });
+    this.logger.info({
+      ctx,
+      msg: `time to init notes: "${duration}" ms`,
+    });
 
     return { errors };
   }

packages/engine-test-utils/src/__tests__/engine-server/drivers/sqlite.spec.ts

@@ -1,9 +1,13 @@
-import { NoteUtils } from "@dendronhq/common-all";
+import { DEngineClient, NotePropsMeta, NoteUtils } from "@dendronhq/common-all";
 import { SQLiteMetadataStore } from "@dendronhq/engine-server";
 import fs from "fs-extra";
 import _ from "lodash";
 import sinon from "sinon";
-import { ENGINE_HOOKS, runEngineTestV5 } from "../../..";
+import {
+  createEngineV3FromEngine,
+  ENGINE_HOOKS,
+  runEngineTestV5,
+} from "../../..";
 // import os from "os";
 
 // current issue with windows test
@@ -12,6 +16,19 @@ import { ENGINE_HOOKS, runEngineTestV5 } from "../../..";
 // const describeSkipWindows =
 //   os.platform() === "win32" ? describe.skip : describe;
 
+const createEngine = createEngineV3FromEngine;
+
+async function expectNotesAreEqual(opts: {
+  engine: DEngineClient;
+  notes: NotePropsMeta[];
+}) {
+  const resp = await opts.engine.bulkGetNotes(
+    opts.notes.map((note: NotePropsMeta) => note.id)
+  );
+  const sortedResp = _.sortBy(_.pick(resp.data, "fname"), "fname");
+  expect(sortedResp).toEqual(_.sortBy(_.pick(opts.notes, "fname"), "fname"));
+}
+
 describe.skip("GIVEN sqlite store", () => {
   afterEach(async () => {
     await SQLiteMetadataStore.prisma().$disconnect();
@@ -25,11 +42,12 @@ describe.skip("GIVEN sqlite store", () => {
         expect(dirList).toMatchSnapshot();
         expect(dirList.includes("metadata.db")).toBeTruthy();
         const notes = await SQLiteMetadataStore.prisma().note.findMany();
-        const engineNotes = await engine.findNotesMeta({ excludeStub: false });
-        expect(engineNotes.length).toEqual(notes.length);
+        expect(notes.length).toEqual(6);
+        await expectNotesAreEqual({ engine, notes });
       },
       {
         expect,
+        createEngine,
         preSetupHook: ENGINE_HOOKS.setupBasic,
         modConfigCb: (config) => {
           config.workspace.metadataStore = "sqlite";
@@ -46,8 +64,7 @@ describe.skip("GIVEN sqlite store", () => {
         expect(dirList).toMatchSnapshot();
         expect(dirList.includes("metadata.db")).toBeTruthy();
         const notes = await SQLiteMetadataStore.prisma().note.findMany();
-        const engineNotes = await engine.findNotesMeta({ excludeStub: false });
-        expect(engineNotes.length).toEqual(notes.length);
+        await expectNotesAreEqual({ engine, notes });
 
         const { error } = await engine.init();
         expect(error).toBeFalsy();
@@ -56,6 +73,7 @@ describe.skip("GIVEN sqlite store", () => {
       },
       {
         expect,
+        createEngine,
         preSetupHook: ENGINE_HOOKS.setupBasic,
         modConfigCb: (config) => {
           config.workspace.metadataStore = "sqlite";
@@ -72,8 +90,7 @@ describe.skip("GIVEN sqlite store", () => {
         expect(dirList).toMatchSnapshot();
         expect(dirList.includes("metadata.db")).toBeTruthy();
         const notes = await SQLiteMetadataStore.prisma().note.findMany();
-        const engineNotes = await engine.findNotesMeta({ excludeStub: false });
-        expect(engineNotes.length).toEqual(notes.length);
+        await expectNotesAreEqual({ engine, notes });
         const newNote = NoteUtils.create({
           id: "new-note",
           fname: "new-note",
@@ -90,6 +107,7 @@ describe.skip("GIVEN sqlite store", () => {
       },
       {
         expect,
+        createEngine,
         preSetupHook: ENGINE_HOOKS.setupBasic,
         modConfigCb: (config) => {
           config.workspace.metadataStore = "sqlite";