Skip to content

Commit

Permalink
fix(Concurrency): Fix issue with concurrent push from different sources
Browse files Browse the repository at this point in the history 
Fixes #484
  • Loading branch information
@Belphemur
Belphemur committed Sep 9, 2022
1 parent 45501bd commit daae2bb
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 128 deletions.
86 changes: 2 additions & 84 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
},
"homepage": "https://github.com/Belphemur/node-json-db",
"dependencies": {
"async-lock": "^1.3.2",
"atomically": "^1.7.0"
},
"config": {
Expand All @@ -41,90 +42,6 @@
"README.md",
"LICENSE"
],
"release": {
"branches": [
{
"name": "master",
"prerelease": false
},
{
"name": "develop",
"prerelease": true
}
],
"plugins": [
[
"@semantic-release/commit-analyzer",
{
"preset": "conventionalcommits",
"releaseRules": [
{
"breaking": true,
"release": "major"
},
{
"revert": true,
"release": "patch"
},
{
"type": "feat",
"release": "minor"
},
{
"type": "fix",
"release": "patch"
},
{
"type": "perf",
"release": "patch"
},
{
"type": "lang",
"release": "patch"
},
{
"type": "boost",
"release": "patch"
}
]
}
],
[
"@semantic-release/release-notes-generator",
{
"preset": "conventionalcommits",
"presetConfig": {
"types": [
{
"type": "boost",
"section": "Enhancements"
},
{
"type": "lang",
"section": "Languages"
},
{
"type": "fix",
"section": "Bug Fixes"
},
{
"type": "feat",
"section": "Features"
},
{
"type": "tests",
"section": "Tests"
}
]
}
}
],
"@semantic-release/changelog",
"@semantic-release/npm",
"@semantic-release/git",
"@semantic-release/github"
]
},
"release": {
"getLastRelease": "last-release-git",
"verifyConditions": [
Expand All @@ -145,6 +62,7 @@
"devDependencies": {
"@semantic-release/changelog": "^6.0.0",
"@semantic-release/git": "^10.0.0",
"@types/async-lock": "^1.1.5",
"@types/jest": "^28.1.6",
"@types/mkdirp": "^1.0.1",
"@types/node": "^18.0.0",
Expand Down
86 changes: 48 additions & 38 deletions src/JsonDB.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,21 @@ import {DatabaseError, DataError} from './lib/Errors'
import {DBParentData} from './lib/DBParentData'
import {ArrayInfo} from './lib/ArrayInfo'
import {JsonDBConfig} from './lib/JsonDBConfig'
import * as AsyncLock from 'async-lock'

export {Config} from './lib/JsonDBConfig'

type DataPath = Array<string>

export type FindCallback = (entry: any, index: number | string) => boolean

export { Config } from './lib/JsonDBConfig'

export class JsonDB {
private loaded: boolean = false
private data: KeyValue = {}
private readonly config: JsonDBConfig
private readonly lock = new AsyncLock()
private readonly lockKey = 'jsonDb'

/**
* JSONDB Constructor
Expand All @@ -40,7 +44,7 @@ export class JsonDB {
return path
}

private async retrieveData(dataPath: DataPath, create: boolean = false) : Promise<any> {
private async retrieveData(dataPath: DataPath, create: boolean = false): Promise<any> {
await this.load()

const thisDb = this
Expand Down Expand Up @@ -282,52 +286,58 @@ export class JsonDB {
* @param override overriding or not the data, if not, it will merge them
*/
public async push(dataPath: string, data: any, override: boolean = true): Promise<void> {
const dbData = await this.getParentData(dataPath, true)
// if (!dbData) {
// throw new Error('Data not found')
// }

let toSet = data
if (!override) {
if (Array.isArray(data)) {
let storedData = dbData.getData()
if (storedData === undefined) {
storedData = []
} else if (!Array.isArray(storedData)) {
throw new DataError(
"Can't merge another type of data with an Array",
3
)
}
toSet = storedData.concat(data)
} else if (data === Object(data)) {
if (Array.isArray(dbData.getData())) {
throw new DataError("Can't merge an Array with an Object", 4)
return this.lock.acquire(this.lockKey, async () => {
const dbData = await this.getParentData(dataPath, true)
// if (!dbData) {
// throw new Error('Data not found')
// }

let toSet = data
if (!override) {
if (Array.isArray(data)) {
let storedData = dbData.getData()
if (storedData === undefined) {
storedData = []
} else if (!Array.isArray(storedData)) {
throw new DataError(
"Can't merge another type of data with an Array",
3
)
}
toSet = storedData.concat(data)
} else if (data === Object(data)) {
if (Array.isArray(dbData.getData())) {
throw new DataError("Can't merge an Array with an Object", 4)
}
toSet = merge(dbData.getData(), data)
}
toSet = merge(dbData.getData(), data)
}
}
dbData.setData(toSet)
dbData.setData(toSet)

if (this.config.saveOnPush) {
await this.save()
}
});

if (this.config.saveOnPush) {
await this.save()
}
}

/**
* Delete the data
* @param dataPath path leading to the data
*/
public async delete(dataPath: string): Promise<void> {
const dbData = await this.getParentData(dataPath, true)
// if (!dbData) {
// return
// }
dbData.delete()

if (this.config.saveOnPush) {
await this.save()
}
await this.lock.acquire(this.lockKey, async () => {
const dbData = await this.getParentData(dataPath, true)
// if (!dbData) {
// return
// }
dbData.delete()

if (this.config.saveOnPush) {
await this.save()
}
});

}

/**
Expand Down
8 changes: 2 additions & 6 deletions test/06-concurrency.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ async function addData(db: JsonDB) {
}

describe('Concurrency', () => {
const db = new JsonDB(new Config('concurrency'));
const db = new JsonDB(new Config('test-concurrent'));
db.resetData({});
describe('Multi write', () => {
test('shouldn\'t corrupt the data', async () => {
let promiseList = [];
Expand All @@ -44,9 +45,4 @@ describe('Concurrency', () => {

});
});
describe('Cleanup', () => {
test('should remove the test files', () => {
fs.unlinkSync("concurrency.json")
})
});
})
10 changes: 10 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2024,6 +2024,11 @@
resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf"
integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==

"@types/async-lock@^1.1.5":
version "1.1.5"
resolved "https://registry.yarnpkg.com/@types/async-lock/-/async-lock-1.1.5.tgz#a82f33e09aef451d6ded7bffae73f9d254723124"
integrity sha512-A9ClUfmj6wwZMLRz0NaYzb98YH1exlHdf/cdDSKBfMQJnPOdO8xlEW0Eh2QsTTntGzOFWURcEjYElkZ1IY4GCQ==

"@types/babel__core@^7.1.14":
version "7.1.16"
resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.16.tgz#bc12c74b7d65e82d29876b5d0baf5c625ac58702"
Expand Down Expand Up @@ -2320,6 +2325,11 @@ asap@^2.0.0:
resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46"
integrity sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==

async-lock@^1.3.2:
version "1.3.2"
resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.3.2.tgz#56668613f91c1c55432b4db73e65c9ced664e789"
integrity sha512-phnXdS3RP7PPcmP6NWWzWMU0sLTeyvtZCxBPpZdkYE3seGLKSQZs9FrmVO/qwypq98FUtWWUEYxziLkdGk5nnA==

at-least-node@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2"
Expand Down

0 comments on commit daae2bb

Please sign in to comment.