improvement(core): switch to better-sqlite3 driver

This both improves performance and fixes issues with concurrent
connections and requests, since better-sqlite3 has built-in retries.

cli/src/build-pkg.ts

@@ -23,10 +23,8 @@ const tmpDir = resolve(repoRoot, "tmp", "pkg")
 const tmpStaticDir = resolve(tmpDir, "static")
 const pkgPath = resolve(repoRoot, "node_modules", ".bin", "pkg")
 const pkgFetchPath = resolve(repoRoot, "node_modules", ".bin", "pkg-fetch")
-const preGypPath = resolve(repoRoot, "node_modules", ".bin", "node-pre-gyp")
-const sqlitePath = resolve(repoRoot, "node_modules", "sqlite3")
 const distPath = resolve(repoRoot, "dist")
-const sqliteBinFilename = "node_sqlite3.node"
+const sqliteBinFilename = "better_sqlite3.node"
 const version = getPackageVersion()
 
 // tslint:disable: no-console
@@ -131,8 +129,6 @@ async function pkgMacos(targetName: string, sourcePath: string, pkgType: string)
     targetName,
     pkgType,
     binFilename: "garden",
-    arch: "x64",
-    platform: "darwin",
   })
 
   console.log(` - ${targetName} -> fsevents.node`)
@@ -147,8 +143,6 @@ async function pkgLinux(targetName: string, sourcePath: string, pkgType: string)
     targetName,
     pkgType,
     binFilename: "garden",
-    arch: "x64",
-    platform: "linux",
   })
   await tarball(targetName)
 }
@@ -159,8 +153,6 @@ async function pkgWindows(targetName: string, sourcePath: string, pkgType: strin
     targetName,
     pkgType,
     binFilename: "garden.exe",
-    arch: "x64",
-    platform: "win32",
   })
 
   console.log(` - ${targetName} -> zip`)
@@ -206,15 +198,11 @@ async function pkgCommon({
   targetName,
   pkgType,
   binFilename,
-  arch,
-  platform,
 }: {
   sourcePath: string
   targetName: string
   pkgType: string
   binFilename: string
-  arch: string
-  platform: string
 }) {
   const targetPath = resolve(distPath, targetName)
   await remove(targetPath)
@@ -224,9 +212,8 @@ async function pkgCommon({
   await exec(pkgPath, ["--target", pkgType, sourcePath, "--output", resolve(targetPath, binFilename)])
 
   console.log(` - ${targetName} -> ${sqliteBinFilename}`)
-  await exec(preGypPath, ["install", `--target_arch=${arch}`, `--target_platform=${platform}`], { cwd: sqlitePath })
   await copy(
-    resolve(sqlitePath, "lib", "binding", `node-v72-${platform}-${arch}`, sqliteBinFilename),
+    resolve(GARDEN_CORE_ROOT, "lib", "better-sqlite3", `${targetName}-node-v68`, sqliteBinFilename),
     resolve(targetPath, sqliteBinFilename)
   )
 

core/lib/better-sqlite3/README.md

@@ -0,0 +1,3 @@
+# better-sqlite3
+
+These are pre-built binaries for [better-sqlite3](https://github.com/JoshuaWise/better-sqlite3). Unfortunately we need this while they're not bundling these binaries. See issues [#452](https://github.com/JoshuaWise/better-sqlite3/issues/452), [#439](https://github.com/JoshuaWise/better-sqlite3/issues/439) and [#438](https://github.com/JoshuaWise/better-sqlite3/issues/438) in their repo.

core/package.json

@@ -31,6 +31,7 @@
     "async-exit-hook": "^2.0.1",
     "async-lock": "^1.2.4",
     "axios": "^0.19.2",
+    "better-sqlite3": "^7.1.0",
     "bluebird": "^3.7.2",
     "certpem": "^1.1.3",
     "chalk": "^4.0.0",
@@ -115,7 +116,6 @@
     "source-map-support": "^0.5.19",
     "split": "^1.0.1",
     "split2": "^3.1.1",
-    "sqlite3": "^4.2.0",
     "string-width": "^4.2.0",
     "strip-ansi": "^6.0.0",
     "supertest": "^4.0.2",
@@ -125,7 +125,7 @@
     "tmp-promise": "^3.0.2",
     "toposort": "^2.0.2",
     "ts-stream": "^2.0.1",
-    "typeorm": "^0.2.25",
+    "typeorm-with-better-sqlite3": "^0.2.27",
     "typescript-memoize": "^1.0.0-alpha.3",
     "uniqid": "^5.2.0",
     "unixify": "^1.0.0",

core/src/db/base-entity.ts

@@ -14,7 +14,7 @@ import {
   UpdateDateColumn,
   VersionColumn,
   PrimaryGeneratedColumn,
-} from "typeorm"
+} from "typeorm-with-better-sqlite3"
 
 import { getConnection } from "./connection"
 

core/src/db/connection.ts

@@ -7,7 +7,7 @@
  */
 
 import { join } from "path"
-import { Connection, getConnectionManager, ConnectionOptions } from "typeorm"
+import { Connection, getConnectionManager, ConnectionOptions } from "typeorm-with-better-sqlite3"
 import { gardenEnv } from "../constants"
 
 let connection: Connection
@@ -24,7 +24,7 @@ export function getConnection(): Connection {
 
     // Prepare the connection (the ormconfig.json in the static dir is only used for the typeorm CLI during dev)
     const options: ConnectionOptions = {
-      type: "sqlite",
+      type: "better-sqlite3",
       database: databasePath,
       // IMPORTANT: All entities and migrations need to be manually referenced here because of how we
       // package the garden binary

core/src/db/entities/client-auth-token.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { Entity, Column, Index } from "typeorm"
+import { Entity, Column, Index } from "typeorm-with-better-sqlite3"
 import { GardenEntity } from "../base-entity"
 
 @Entity()

core/src/db/entities/garden-process.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { Entity, Column } from "typeorm"
+import { Entity, Column } from "typeorm-with-better-sqlite3"
 import { GardenEntity } from "../base-entity"
 import { partition, find, isMatch } from "lodash"
 

core/src/db/entities/local-address.ts

@@ -7,7 +7,7 @@
  */
 
 import { toLong, fromLong } from "ip"
-import { Entity, Column, Index } from "typeorm"
+import { Entity, Column, Index } from "typeorm-with-better-sqlite3"
 import { GardenEntity } from "../base-entity"
 
 const ipRangeStart = "127.10.0.1"

core/src/db/entities/warning.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { Entity, Column, Index } from "typeorm"
+import { Entity, Column, Index } from "typeorm-with-better-sqlite3"
 import { GardenEntity } from "../base-entity"
 import { LogEntry } from "../../logger/log-entry"
 import chalk from "chalk"

support/alpine-builder.Dockerfile

@@ -26,14 +26,16 @@ ADD sdk /tmp/sdk
 # Install the CLI deps
 WORKDIR /tmp/cli
 
-RUN yarn --production && \
+RUN apk add --no-cache python make gcc g++ --virtual .build-deps && \
+  yarn --production && \
   # Fix for error in this particular package
-  rm -rf node_modules/es-get-iterator/test
+  rm -rf node_modules/es-get-iterator/test && \
+  apk del .build-deps
 
 ADD static /garden/static
 
 # Create the binary
 RUN mkdir -p /garden \
   && ../pkg/node_modules/.bin/pkg --target node12-alpine-x64 . --output /garden/garden \
-  && cp node_modules/sqlite3/lib/binding/node-v72-linux-x64/node_sqlite3.node /garden \
+  && cp node_modules/better-sqlite3/build/Release/better_sqlite3.node /garden \
   && /garden/garden version

support/homebrew-formula.rb

@@ -8,7 +8,7 @@ class GardenCli < Formula
   depends_on "rsync"
 
   def install
-    libexec.install "garden", "fsevents.node", "static", "node_sqlite3.node"
+    libexec.install "garden", "fsevents.node", "static", "better_sqlite3.node"
     bin.install_symlink libexec/"garden"
   end