diff --git a/docs/docs/how-to/testing/unit-testing.md b/docs/docs/how-to/testing/unit-testing.md index c5625b94ef95e..fa276fbb3907a 100644 --- a/docs/docs/how-to/testing/unit-testing.md +++ b/docs/docs/how-to/testing/unit-testing.md @@ -43,6 +43,7 @@ module.exports = { ".+\\.(css|styl|less|sass|scss)$": `identity-obj-proxy`, ".+\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": `/__mocks__/file-mock.js`, "^gatsby-page-utils/(.*)$": `gatsby-page-utils/dist/$1`, // Workaround for https://github.com/facebook/jest/issues/9771 + "^gatsby-core-utils/(.*)$": `gatsby-core-utils/dist/$1`, // Workaround for https://github.com/facebook/jest/issues/9771 }, testPathIgnorePatterns: [`node_modules`, `\\.cache`, `.*/public`], transformIgnorePatterns: [`node_modules/(?!(gatsby)/)`], diff --git a/examples/using-jest/jest.config.js b/examples/using-jest/jest.config.js index b2492077dfbd8..588b2b06ec5b1 100644 --- a/examples/using-jest/jest.config.js +++ b/examples/using-jest/jest.config.js @@ -6,6 +6,7 @@ module.exports = { ".+\\.(css|styl|less|sass|scss)$": `identity-obj-proxy`, ".+\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": `/__mocks__/file-mock.js`, "^gatsby-page-utils/(.*)$": `gatsby-page-utils/dist/$1`, // Workaround for https://github.com/facebook/jest/issues/9771 + "^gatsby-core-utils/(.*)$": `gatsby-core-utils/dist/$1`, // Workaround for https://github.com/facebook/jest/issues/9771 }, testPathIgnorePatterns: [`node_modules`, `.cache`], transformIgnorePatterns: [`node_modules/(?!(gatsby)/)`], diff --git a/jest.config.js b/jest.config.js index 7c3936da273c5..f30f5b8353b19 100644 --- a/jest.config.js +++ b/jest.config.js @@ -47,6 +47,7 @@ module.exports = { "^ordered-binary$": `/node_modules/ordered-binary/dist/index.cjs`, "^msgpackr$": `/node_modules/msgpackr/dist/node.cjs`, "^gatsby-page-utils/(.*)$": `gatsby-page-utils/dist/$1`, // Workaround for https://github.com/facebook/jest/issues/9771 + "^gatsby-core-utils/(.*)$": `gatsby-core-utils/dist/$1`, // Workaround for https://github.com/facebook/jest/issues/9771 }, snapshotSerializers: [`jest-serializer-path`], collectCoverageFrom: coverageDirs, diff --git a/packages/gatsby-core-utils/README.md b/packages/gatsby-core-utils/README.md index fb16a3b62bdad..30d512d527339 100644 --- a/packages/gatsby-core-utils/README.md +++ b/packages/gatsby-core-utils/README.md @@ -104,3 +104,20 @@ const requireUtil = createRequireFromPath("../src/utils/") requireUtil("./some-tool") // ... ``` + +### Mutex + +When working inside workers or async operations you want some kind of concurrency control that a specific work load can only concurrent one at a time. This is what a [Mutex](https://en.wikipedia.org/wiki/Mutual_exclusion) does. + +By implementing the following code, the code is only executed one at a time and the other threads/async workloads are awaited until the current one is done. This is handy when writing to the same file to disk. + +```js +const { createMutex } = require("gatsby-core-utils/mutex") + +const mutex = createMutex("my-custom-mutex-key") +await mutex.acquire() + +await fs.writeFile("pathToFile", "my custom content") + +await mutex.release() +``` diff --git a/packages/gatsby-core-utils/package.json b/packages/gatsby-core-utils/package.json index 8ee0254105e74..1127d113fc900 100644 --- a/packages/gatsby-core-utils/package.json +++ b/packages/gatsby-core-utils/package.json @@ -6,6 +6,18 @@ "gatsby", "gatsby-core-utils" ], + "exports": { + ".": "./dist/index.js", + "./*": "./dist/*.js" + }, + "typesVersions": { + "*": { + "*": [ + "dist/*.d.ts", + "dist/index.d.ts" + ] + } + }, "author": "Ward Peeters ", "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-core-utils#readme", "license": "MIT", @@ -36,9 +48,12 @@ "file-type": "^16.5.3", "fs-extra": "^10.0.0", "got": "^11.8.3", + "import-from": "^4.0.0", "lock": "^1.1.0", + "lmdb": "^2.1.7", "node-object-hash": "^2.3.10", "proper-lockfile": "^4.1.2", + "resolve-from": "^5.0.0", "tmp": "^0.2.1", "xdg-basedir": "^4.0.0" }, diff --git a/packages/gatsby-core-utils/src/__tests__/mutex.ts b/packages/gatsby-core-utils/src/__tests__/mutex.ts new file mode 100644 index 0000000000000..66b7f00acd5c4 --- /dev/null +++ b/packages/gatsby-core-utils/src/__tests__/mutex.ts @@ -0,0 +1,99 @@ +import path from "path" +import { remove, mkdirp } from "fs-extra" +import { createMutex } from "../mutex" +import * as storage from "../utils/get-storage" + +jest.spyOn(storage, `getDatabaseDir`) + +function sleep(timeout = 100): Promise { + return new Promise(resolve => setTimeout(resolve, timeout)) +} + +async function doAsync( + mutex: ReturnType, + result: Array = [], + waitTime: number, + id: string +): Promise> { + await mutex.acquire() + result.push(`start ${id}`) + await sleep(waitTime) + result.push(`stop ${id}`) + await mutex.release() + + return result +} + +describe(`mutex`, () => { + const cachePath = path.join(__dirname, `.cache`) + beforeAll(async () => { + await mkdirp(cachePath) + storage.getDatabaseDir.mockReturnValue(cachePath) + }) + + afterAll(async () => { + await storage.closeDatabase() + await remove(cachePath) + }) + + it(`should only allow one action go through at the same time`, async () => { + const mutex = createMutex(`test-key`, 300) + + const result: Array = [] + + doAsync(mutex, result, 50, `1`) + await sleep(0) + await doAsync(mutex, result, 10, `2`) + + expect(result).toMatchInlineSnapshot(` + Array [ + "start 1", + "stop 1", + "start 2", + "stop 2", + ] + `) + }) + + it(`should generate the same mutex if key are identical`, async () => { + const mutex1 = createMutex(`test-key`, 300) + const mutex2 = createMutex(`test-key`, 300) + + const result: Array = [] + + const mutexPromise = doAsync(mutex1, result, 50, `1`) + await sleep(0) + await doAsync(mutex2, result, 10, `2`) + await mutexPromise + + expect(result).toMatchInlineSnapshot(` + Array [ + "start 1", + "stop 1", + "start 2", + "stop 2", + ] + `) + }) + + it(`shouldn't wait if keys are different`, async () => { + const mutex1 = createMutex(`test-key`, 300) + const mutex2 = createMutex(`other-key`, 300) + + const result: Array = [] + + const mutexPromise = doAsync(mutex1, result, 50, `1`) + await sleep(0) + await doAsync(mutex2, result, 10, `2`) + await mutexPromise + + expect(result).toMatchInlineSnapshot(` + Array [ + "start 1", + "start 2", + "stop 2", + "stop 1", + ] + `) + }) +}) diff --git a/packages/gatsby-core-utils/src/mutex.ts b/packages/gatsby-core-utils/src/mutex.ts new file mode 100644 index 0000000000000..b983e982c6ed2 --- /dev/null +++ b/packages/gatsby-core-utils/src/mutex.ts @@ -0,0 +1,57 @@ +import { getStorage, LockStatus, getDatabaseDir } from "./utils/get-storage" + +interface IMutex { + acquire(): Promise + release(): Promise +} + +// Random number to re-check if mutex got released +const DEFAULT_MUTEX_INTERVAL = 3000 + +async function waitUntilUnlocked( + storage: ReturnType, + key: string, + timeout: number +): Promise { + const isUnlocked = await storage.mutex.ifNoExists(key, () => { + storage.mutex.put(key, LockStatus.Locked) + }) + + if (isUnlocked) { + return + } + + await new Promise(resolve => { + setTimeout(() => { + resolve(waitUntilUnlocked(storage, key, timeout)) + }, timeout) + }) +} + +/** + * Creates a mutex, make sure to call `release` when you're done with it. + * + * @param {string} key A unique key + */ +export function createMutex( + key: string, + timeout = DEFAULT_MUTEX_INTERVAL +): IMutex { + const storage = getStorage(getDatabaseDir()) + const BUILD_ID = global.__GATSBY?.buildId ?? `` + const prefixedKey = `${BUILD_ID}-${key}` + + return { + acquire: (): Promise => + waitUntilUnlocked(storage, prefixedKey, timeout), + release: async (): Promise => { + await storage.mutex.remove(prefixedKey) + }, + } +} + +export async function releaseAllMutexes(): Promise { + const storage = getStorage(getDatabaseDir()) + + await storage.mutex.clearAsync() +} diff --git a/packages/gatsby-core-utils/src/utils/get-lmdb.ts b/packages/gatsby-core-utils/src/utils/get-lmdb.ts new file mode 100644 index 0000000000000..36712050f5ad2 --- /dev/null +++ b/packages/gatsby-core-utils/src/utils/get-lmdb.ts @@ -0,0 +1,16 @@ +import path from "path" +import importFrom from "import-from" +import resolveFrom from "resolve-from" + +export function getLmdb(): typeof import("lmdb") { + const gatsbyPkgRoot = path.dirname( + resolveFrom(process.cwd(), `gatsby/package.json`) + ) + + // Try to use lmdb from gatsby if not we use our own version + try { + return importFrom(gatsbyPkgRoot, `lmdb`) as typeof import("lmdb") + } catch (err) { + return require(`lmdb`) + } +} diff --git a/packages/gatsby-core-utils/src/utils/get-storage.ts b/packages/gatsby-core-utils/src/utils/get-storage.ts new file mode 100644 index 0000000000000..63441fbbbed7e --- /dev/null +++ b/packages/gatsby-core-utils/src/utils/get-storage.ts @@ -0,0 +1,68 @@ +import path from "path" +import { getLmdb } from "./get-lmdb" +import type { RootDatabase, Database } from "lmdb" + +export enum LockStatus { + Locked = 0, + Unlocked = 1, +} + +interface ICoreUtilsDatabase { + mutex: Database +} + +let databases: ICoreUtilsDatabase | undefined +let rootDb: RootDatabase + +export function getDatabaseDir(): string { + const rootDir = global.__GATSBY?.root ?? process.cwd() + return path.join(rootDir, `.cache`, `data`, `gatsby-core-utils`) +} + +export function getStorage(fullDbPath: string): ICoreUtilsDatabase { + if (!databases) { + if (!fullDbPath) { + throw new Error(`LMDB path is not set!`) + } + + // __GATSBY_OPEN_LMDBS tracks if we already opened given db in this process + // In `gatsby serve` case we might try to open it twice - once for engines + // and second to get access to `SitePage` nodes (to power trailing slashes + // redirect middleware). This ensure there is single instance within a process. + // Using more instances seems to cause weird random errors. + if (!globalThis.__GATSBY_OPEN_LMDBS) { + globalThis.__GATSBY_OPEN_LMDBS = new Map() + } + + databases = globalThis.__GATSBY_OPEN_LMDBS.get(fullDbPath) + + if (databases) { + return databases + } + + const open = getLmdb().open + + rootDb = open({ + name: `root`, + path: fullDbPath, + compression: true, + sharedStructuresKey: Symbol.for(`structures`), + }) + + databases = { + mutex: rootDb.openDB({ + name: `mutex`, + }), + } + + globalThis.__GATSBY_OPEN_LMDBS.set(fullDbPath, databases) + } + + return databases as ICoreUtilsDatabase +} + +export async function closeDatabase(): Promise { + if (rootDb) { + await rootDb.close() + } +} diff --git a/packages/gatsby/src/services/initialize.ts b/packages/gatsby/src/services/initialize.ts index d0e3da2c6de7a..1dfa0caf5952e 100644 --- a/packages/gatsby/src/services/initialize.ts +++ b/packages/gatsby/src/services/initialize.ts @@ -1,5 +1,6 @@ import _ from "lodash" import { slash, isCI } from "gatsby-core-utils" +import { releaseAllMutexes } from "gatsby-core-utils/mutex" import fs from "fs-extra" import md5File from "md5-file" import crypto from "crypto" @@ -407,34 +408,29 @@ export async function initialize({ // } // } - if ( - process.env.GATSBY_EXPERIMENTAL_PRESERVE_FILE_DOWNLOAD_CACHE || - process.env.GATSBY_EXPERIMENTAL_PRESERVE_WEBPACK_CACHE - ) { - const deleteGlobs = [ - // By default delete all files & subdirectories - `${cacheDirectory}/**`, - `${cacheDirectory}/*/`, - ] - - if (process.env.GATSBY_EXPERIMENTAL_PRESERVE_FILE_DOWNLOAD_CACHE) { - // Stop the caches directory from being deleted, add all sub directories, - // but remove gatsby-source-filesystem - deleteGlobs.push(`!${cacheDirectory}/caches`) - deleteGlobs.push(`${cacheDirectory}/caches/*`) - deleteGlobs.push(`!${cacheDirectory}/caches/gatsby-source-filesystem`) - } + const deleteGlobs = [ + // By default delete all files & subdirectories + `${cacheDirectory}/**`, + `!${cacheDirectory}/data`, + `${cacheDirectory}/data/**`, + `!${cacheDirectory}/data/gatsby-core-utils/`, + `!${cacheDirectory}/data/gatsby-core-utils/**`, + ] + + if (process.env.GATSBY_EXPERIMENTAL_PRESERVE_FILE_DOWNLOAD_CACHE) { + // Stop the caches directory from being deleted, add all sub directories, + // but remove gatsby-source-filesystem + deleteGlobs.push(`!${cacheDirectory}/caches`) + deleteGlobs.push(`${cacheDirectory}/caches/*`) + deleteGlobs.push(`!${cacheDirectory}/caches/gatsby-source-filesystem`) + } - if (process.env.GATSBY_EXPERIMENTAL_PRESERVE_WEBPACK_CACHE) { - // Add webpack - deleteGlobs.push(`!${cacheDirectory}/webpack`) - } - await del(deleteGlobs) - } else { - // Attempt to empty dir if remove fails, - // like when directory is mount point - await fs.remove(cacheDirectory).catch(() => fs.emptyDir(cacheDirectory)) + if (process.env.GATSBY_EXPERIMENTAL_PRESERVE_WEBPACK_CACHE) { + // Add webpack + deleteGlobs.push(`!${cacheDirectory}/webpack`) } + + await del(deleteGlobs) } catch (e) { reporter.error(`Failed to remove .cache files.`, e) } @@ -445,6 +441,9 @@ export async function initialize({ cacheIsCorrupt, }) + // make sure all previous mutexes are released + await releaseAllMutexes() + // in future this should show which plugin's caches are purged // possibly should also have which plugins had caches telemetry.decorateEvent(`BUILD_END`, { diff --git a/yarn.lock b/yarn.lock index 248df0c638e88..5e5a9ee9524cb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12393,6 +12393,11 @@ import-from@3.0.0, import-from@^3.0.0: dependencies: resolve-from "^5.0.0" +import-from@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/import-from/-/import-from-4.0.0.tgz#2710b8d66817d232e16f4166e319248d3d5492e2" + integrity sha512-P9J71vT5nLlDeV8FHs5nNxaLbrpfAV5cF5srvbZfpwpcJoM/xZR3hiv+q+SAnuSmuGbXMWud063iIMx/V/EWZQ== + import-lazy@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43" @@ -14594,6 +14599,17 @@ lmdb-store@^1.6.11: optionalDependencies: msgpackr "^1.4.7" +lmdb@^2.1.7: + version "2.2.1" + resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.2.1.tgz#b7fd22ed2268ab74aa71108b793678314a7b94bb" + integrity sha512-tUlIjyJvbd4mqdotI9Xe+3PZt/jqPx70VKFDrKMYu09MtBWOT3y2PbuTajX+bJFDjbgLkQC0cTx2n6dithp/zQ== + dependencies: + msgpackr "^1.5.4" + nan "^2.14.2" + node-gyp-build "^4.2.3" + ordered-binary "^1.2.4" + weak-lru-cache "^1.2.2" + load-bmfont@^1.3.1, load-bmfont@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/load-bmfont/-/load-bmfont-1.4.0.tgz#75f17070b14a8c785fe7f5bee2e6fd4f98093b6b" @@ -16228,6 +16244,13 @@ msgpackr@^1.4.7: optionalDependencies: msgpackr-extract "^1.0.14" +msgpackr@^1.5.4: + version "1.5.4" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.5.4.tgz#2b6ea6cb7d79c0ad98fc76c68163c48eda50cf0d" + integrity sha512-Z7w5Jg+2Q9z9gJxeM68d7tSuWZZGnFIRhZnyqcZCa/1dKkhOCNvR1TUV3zzJ3+vj78vlwKRzUgVDlW4jiSOeDA== + optionalDependencies: + msgpackr-extract "^1.0.14" + msw@^0.35.0: version "0.35.0" resolved "https://registry.yarnpkg.com/msw/-/msw-0.35.0.tgz#18a4ceb6c822ef226a30421d434413bc45030d38" @@ -17125,6 +17148,11 @@ ordered-binary@^1.0.0: resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.1.3.tgz#11dbc0a4cb7f8248183b9845e031b443be82571e" integrity sha512-tDTls+KllrZKJrqRXUYJtIcWIyoQycP7cVN7kzNNnhHKF2bMKHflcAQK+pF2Eb1iVaQodHxqZQr0yv4HWLGBhQ== +ordered-binary@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.2.4.tgz#51d3a03af078a0bdba6c7bc8f4fedd1f5d45d83e" + integrity sha512-A/csN0d3n+igxBPfUrjbV5GC69LWj2pjZzAAeeHXLukQ4+fytfP4T1Lg0ju7MSPSwq7KtHkGaiwO8URZN5IpLg== + ordered-read-streams@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/ordered-read-streams/-/ordered-read-streams-1.0.1.tgz#77c0cb37c41525d64166d990ffad7ec6a0e1363e" @@ -24531,6 +24559,11 @@ weak-lru-cache@^1.0.0: resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.1.2.tgz#a909a97372aabdfbfe3eb33580af255b3b198834" integrity sha512-Bi5ae8Bev3YulgtLTafpmHmvl3vGbanRkv+qqA2AX8c3qj/MUdvSuaHq7ukDYBcMDINIaRPTPEkXSNCqqWivuA== +weak-lru-cache@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19" + integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw== + web-namespaces@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/web-namespaces/-/web-namespaces-1.1.2.tgz#c8dc267ab639505276bae19e129dbd6ae72b22b4"