chore: Migrate custom store and queue to TypeScript

package.json

@@ -6,6 +6,7 @@
     "@babel/runtime": "^7.10.3",
     "@lerna/prompt": "3.18.5",
     "@types/babel__code-frame": "^7.0.1",
+    "@types/better-queue": "^3.8.2",
     "@types/bluebird": "^3.5.30",
     "@types/cache-manager": "^2.10.2",
     "@types/common-tags": "^1.8.0",

packages/gatsby/src/query/__tests__/better-queue-custom-store.js → packages/gatsby/src/query/__tests__/better-queue-custom-store.ts

@@ -1,5 +1,5 @@
-const MemoryStoreWithPriorityBuckets = require(`../better-queue-custom-store`)
-const pify = require(`pify`)
+import { MemoryStoreWithPriorityBuckets } from "../better-queue-custom-store"
+import pify from "pify"
 
 // those are tests copied from https://github.com/diamondio/better-queue-store-test/blob/master/tester.js
 // and converted from mocha to jest + used pify to make it nicer to read than callback chain
@@ -18,6 +18,7 @@ describe(`Custom better-queue memory store`, () => {
     `releaseLock`,
   ]
   beforeEach(() => {
+    // eslint-disable-next-line new-cap
     store = MemoryStoreWithPriorityBuckets()
     functions.forEach(fnName => {
       if (store[fnName]) {
@@ -51,10 +52,8 @@ describe(`Custom better-queue memory store`, () => {
     await store.putTask(`task2`, { value: `secret 2` }, 1)
     await store.putTask(`task3`, { value: `secret 3` }, 1)
 
-    let lockId, tasks
-
-    lockId = await store.takeLastN(2)
-    tasks = await store.getLock(lockId)
+    let lockId: string = await store.takeLastN(2)
+    let tasks: any = await store.getLock(lockId)
 
     // should get the third task
     expect(tasks.task3.value).toBe(`secret 3`)
@@ -80,7 +79,8 @@ describe(`Custom better-queue memory store`, () => {
     await store.putTask(`task2`, { value: `secret 2` }, 1)
     await store.putTask(`task3`, { value: `secret 3` }, 1)
 
-    let lockId, tasks
+    let lockId
+    let tasks
 
     lockId = await store.takeFirstN(2)
     tasks = await store.getLock(lockId)
@@ -109,8 +109,8 @@ describe(`Custom better-queue memory store`, () => {
     await store.putTask(`task2`, { value: `secret 2` }, 1)
     await store.putTask(`task3`, { value: `secret 3` }, 1)
 
-    const lock1 = await store.takeFirstN(1)
-    const lock2 = await store.takeLastN(1)
+    const lock1: string = await store.takeFirstN(1)
+    const lock2: string = await store.takeLastN(1)
 
     let workers
 
@@ -151,7 +151,7 @@ describe(`Custom better-queue memory store`, () => {
     await store.deleteTask(`task2`)
 
     // take 2
-    const lockId = await store.takeFirstN(2)
+    const lockId: string = await store.takeFirstN(2)
     const tasks = await store.getLock(lockId)
 
     // should get the first task
@@ -173,9 +173,8 @@ describe(`Custom better-queue memory store`, () => {
     await store.putTask(`task4`, { value: `secret 4` }, 2)
 
     // take first 2
-    let lockId, tasks
-    lockId = await store.takeFirstN(2)
-    tasks = await store.getLock(lockId)
+    let lockId: string = await store.takeFirstN(2)
+    let tasks = await store.getLock(lockId)
 
     // should get the third task
     expect(tasks.task3.value).toBe(`secret 3`)

packages/gatsby/src/query/better-queue-custom-store.js → packages/gatsby/src/query/better-queue-custom-store.ts

@@ -1,32 +1,40 @@
-function MemoryStoreWithPriorityBuckets() {
+import { Store } from "better-queue"
+
+// getRunningTasks is an extension to the interface, and is used in the tests
+interface IGatsbyBetterStore<T> extends Store<T> {
+  getRunningTasks(cb: (error: any, runningTasks: any) => void): void
+}
+
+export function MemoryStoreWithPriorityBuckets<T>(): IGatsbyBetterStore<T> {
+  type RunningTasks = Record<string, T>
   let uuid = 0
 
   /**
    * Task ids grouped by priority
    */
-  const queueMap = new Map()
+  const queueMap = new Map<number, string[]>()
 
   /**
    * Task id to task lookup
    */
-  const tasks = new Map()
+  const tasks = new Map<string, T>()
 
   /**
    * Task id to priority lookup
    */
-  const taskIdToPriority = new Map()
+  const taskIdToPriority = new Map<string, number>()
 
   /**
    * Lock to running tasks object
    */
-  const running = {}
+  const running: Record<string, RunningTasks> = {}
 
-  let priorityKeys = []
-  const updatePriorityKeys = () => {
+  let priorityKeys: number[] = []
+  const updatePriorityKeys = (): void => {
     priorityKeys = Array.from(queueMap.keys()).sort((a, b) => b - a)
   }
 
-  const addTaskWithPriority = (taskId, priority) => {
+  const addTaskWithPriority = (taskId: string, priority: number): boolean => {
     let needToUpdatePriorityKeys = false
     let priorityTasks = queueMap.get(priority)
     if (!priorityTasks) {
@@ -41,36 +49,39 @@ function MemoryStoreWithPriorityBuckets() {
   }
 
   return {
-    connect: function (cb) {
+    connect: function (cb): void {
       cb(null, tasks.size)
     },
-    getTask: function (taskId, cb) {
+    getTask: function (taskId, cb): void {
+      // @ts-ignore
       cb(null, tasks.get(taskId))
     },
-    deleteTask: function (taskId, cb) {
+    deleteTask: function (taskId, cb): void {
       if (tasks.get(taskId)) {
         tasks.delete(taskId)
         const priority = taskIdToPriority.get(taskId)
-        const priorityTasks = queueMap.get(priority)
-        priorityTasks.splice(priorityTasks.indexOf(taskId), 1)
-        taskIdToPriority.delete(taskId)
+        if (priority) {
+          const priorityTasks = queueMap.get(priority) || []
+          priorityTasks.splice(priorityTasks.indexOf(taskId), 1)
+          taskIdToPriority.delete(taskId)
+        }
       }
       cb()
     },
-    putTask: function (taskId, task, priority = 0, cb) {
+    putTask: function (taskId, task, priority = 0, cb): void {
       const oldTask = tasks.get(taskId)
       tasks.set(taskId, task)
       let needToUpdatePriorityKeys = false
       if (oldTask) {
         const oldPriority = taskIdToPriority.get(taskId)
 
-        if (oldPriority !== priority) {
-          const oldPriorityTasks = queueMap.get(oldPriority)
+        if (oldPriority && oldPriority !== priority) {
+          const oldPriorityTasks = queueMap.get(oldPriority) || []
           oldPriorityTasks.splice(oldPriorityTasks.indexOf(taskId), 1)
 
           if (
             addTaskWithPriority(taskId, priority) ||
-            oldPriority.length === 0
+            oldPriorityTasks.length === 0
           ) {
             needToUpdatePriorityKeys = true
           }
@@ -82,29 +93,33 @@ function MemoryStoreWithPriorityBuckets() {
       if (needToUpdatePriorityKeys) {
         updatePriorityKeys()
       }
-      cb()
+      cb(null)
     },
-    takeFirstN: function (n, cb) {
-      const lockId = uuid++
+    takeFirstN: function (n, cb): void {
+      const lockId = `` + uuid++
       let remainingTasks = n
       let needToUpdatePriorityKeys = false
       let haveSomeTasks = false
-      const tasksToRun = {}
+      const tasksToRun: RunningTasks = {}
 
       for (const priority of priorityKeys) {
-        const taskWithSamePriority = queueMap.get(priority)
-        const grabbedTaskIds = taskWithSamePriority.splice(0, remainingTasks)
+        const tasksWithSamePriority = queueMap.get(priority)
+        const grabbedTaskIds =
+          tasksWithSamePriority?.splice(0, remainingTasks) ?? []
         grabbedTaskIds.forEach(taskId => {
           // add task to task that will run
           // and remove it from waiting list
-          tasksToRun[taskId] = tasks.get(taskId)
-          tasks.delete(taskId)
-          taskIdToPriority.delete(taskId)
-          haveSomeTasks = true
+          const task = tasks.get(taskId)
+          if (task) {
+            tasksToRun[taskId] = task
+            tasks.delete(taskId)
+            taskIdToPriority.delete(taskId)
+            haveSomeTasks = true
+          }
         })
 
         remainingTasks -= grabbedTaskIds.length
-        if (taskWithSamePriority.length === 0) {
+        if (tasksWithSamePriority?.length === 0) {
           queueMap.delete(priority)
           needToUpdatePriorityKeys = true
         }
@@ -123,26 +138,26 @@ function MemoryStoreWithPriorityBuckets() {
 
       cb(null, lockId)
     },
-    takeLastN: function (n, cb) {
+    takeLastN: function (n, cb): void {
       // This is not really used by Gatsby, but will be implemented for
       // completion in easiest possible way (so not very performant).
       // Mostly done so generic test suite used by other stores passes.
       // This is mostly C&P from takeFirstN, with array reversal and different
       // splice args
-      const lockId = uuid++
+      const lockId = `` + uuid++
       let remainingTasks = n
       let needToUpdatePriorityKeys = false
       let haveSomeTasks = false
       const tasksToRun = {}
 
       for (const priority of priorityKeys.reverse()) {
-        const taskWithSamePriority = queueMap.get(priority)
+        const tasksWithSamePriority = queueMap.get(priority) || []
         const deleteCount = Math.min(
           remainingTasks,
-          taskWithSamePriority.length
+          tasksWithSamePriority.length
         )
-        const grabbedTaskIds = taskWithSamePriority.splice(
-          taskWithSamePriority.length - deleteCount,
+        const grabbedTaskIds = tasksWithSamePriority.splice(
+          tasksWithSamePriority.length - deleteCount,
           deleteCount
         )
         grabbedTaskIds.forEach(taskId => {
@@ -155,7 +170,7 @@ function MemoryStoreWithPriorityBuckets() {
         })
 
         remainingTasks -= grabbedTaskIds.length
-        if (taskWithSamePriority.length === 0) {
+        if (tasksWithSamePriority.length === 0) {
           queueMap.delete(priority)
           needToUpdatePriorityKeys = true
         }
@@ -174,17 +189,15 @@ function MemoryStoreWithPriorityBuckets() {
 
       cb(null, lockId)
     },
-    getRunningTasks: function (cb) {
+    getRunningTasks: function (cb): void {
       cb(null, running)
     },
-    getLock: function (lockId, cb) {
+    getLock: function (lockId, cb): void {
       cb(null, running[lockId])
     },
-    releaseLock: function (lockId, cb) {
+    releaseLock: function (lockId, cb): void {
       delete running[lockId]
-      cb()
+      cb(null)
     },
   }
 }
-
-module.exports = MemoryStoreWithPriorityBuckets