From 177799e8e73e756503a203d0fce259fb864cc277 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Tue, 10 Oct 2023 19:23:38 -0700 Subject: [PATCH 01/14] docs(changelog): update multiple changelogs --- docs/gitbook/bullmq-pro/changelog.md | 11 +++++++++++ docs/gitbook/changelog.md | 6 ------ docs/gitbook/python/changelog.md | 3 --- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/gitbook/bullmq-pro/changelog.md b/docs/gitbook/bullmq-pro/changelog.md index 291bd693b6..6d10d422d9 100644 --- a/docs/gitbook/bullmq-pro/changelog.md +++ b/docs/gitbook/bullmq-pro/changelog.md @@ -1,3 +1,14 @@ +# [6.6.0](https://github.com/taskforcesh/bullmq-pro/compare/v6.5.0...v6.6.0) (2023-10-06) + + +### Bug Fixes +* **delayed:** trim events when moving jobs to delayed (python) ([#2211](https://github.com/taskforcesh/bullmq/issues/2211)) ([eca8c2d](https://github.com/taskforcesh/bullmq/commit/eca8c2d4dfeafbd8ac36a49764dbd4897303628c)) + +### Features + +* expose addJobLog and updateJobProgress to the Queue instance ([#2202](https://github.com/taskforcesh/bullmq/issues/2202)) ([2056939](https://github.com/taskforcesh/bullmq/commit/205693907a4d6c2da9bd0690fb552b1d1e369c08)) + + # [6.5.0](https://github.com/taskforcesh/bullmq-pro/compare/v6.4.0...v6.5.0) (2023-09-28) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 72e6e8a134..db43aaad87 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -22,15 +22,9 @@ # [4.12.0](https://github.com/taskforcesh/bullmq/compare/v4.11.4...v4.12.0) (2023-09-29) -### Bug Fixes - -* **move-to-finished:** stringify any return value [python] ([#2198](https://github.com/taskforcesh/bullmq/issues/2198)) fixes [#2196](https://github.com/taskforcesh/bullmq/issues/2196) ([07f1335](https://github.com/taskforcesh/bullmq/commit/07f13356eb1c0136f03dfdf946d163f0ef3c4d62)) - - ### Features * expose addJobLog and updateJobProgress to the Queue instance ([#2202](https://github.com/taskforcesh/bullmq/issues/2202)) ([2056939](https://github.com/taskforcesh/bullmq/commit/205693907a4d6c2da9bd0690fb552b1d1e369c08)) -* **queue:** add clean method [python] ([#2194](https://github.com/taskforcesh/bullmq/issues/2194)) ([3b67193](https://github.com/taskforcesh/bullmq/commit/3b6719379cbec5beb1b7dfb5f06d46cbbf74010f)) ## [4.11.4](https://github.com/taskforcesh/bullmq/compare/v4.11.3...v4.11.4) (2023-09-22) diff --git a/docs/gitbook/python/changelog.md b/docs/gitbook/python/changelog.md index 66f8184e93..35f74aaec5 100644 --- a/docs/gitbook/python/changelog.md +++ b/docs/gitbook/python/changelog.md @@ -6,9 +6,6 @@ ### Fix * **delayed:** Trim events when moving jobs to delayed (python) ([#2211](https://github.com/taskforcesh/bullmq/issues/2211)) ([`eca8c2d`](https://github.com/taskforcesh/bullmq/commit/eca8c2d4dfeafbd8ac36a49764dbd4897303628c)) -### Documentation -* **changelog:** Update changes ([`158b850`](https://github.com/taskforcesh/bullmq/commit/158b850c8b89ae08e93951420340fb8ee39a8d0a)) - ## v1.15.0 (2023-09-30) ### Feature * Nothing change From 67de4fbb8b161b9f74f66ae31ef800ff3cd7fe48 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Fri, 13 Oct 2023 06:45:58 -0700 Subject: [PATCH 02/14] fix(events): do not publish removed event on non-existent jobs (#2227) --- src/commands/removeJob-1.lua | 6 +++--- tests/test_events.ts | 24 +++++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/commands/removeJob-1.lua b/src/commands/removeJob-1.lua index ac8a3cedc3..3ef586ba22 100644 --- a/src/commands/removeJob-1.lua +++ b/src/commands/removeJob-1.lua @@ -53,9 +53,9 @@ local function removeJob( prefix, jobId, parentKey, removeChildren) local prev = removeJobFromAnyState(prefix, jobId) - rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed") - - rcall("XADD", prefix .. "events", "*", "event", "removed", "jobId", jobId, "prev", prev); + if rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed") > 0 then + rcall("XADD", prefix .. "events", "*", "event", "removed", "jobId", jobId, "prev", prev) + end end local prefix = KEYS[1] diff --git a/tests/test_events.ts b/tests/test_events.ts index 8eeb3c769f..8cbeac29bb 100644 --- a/tests/test_events.ts +++ b/tests/test_events.ts @@ -505,6 +505,28 @@ describe('events', function () { await worker.close(); }); + describe('when jobs removal is attempted on non-existed records', async () => { + it('should not publish removed events', async () => { + const numRemovals = 100; + const trimmedQueue = new Queue(queueName, { + connection, + }); + + const client = await trimmedQueue.client; + + for (let i = 0; i < numRemovals; i++) { + await trimmedQueue.remove(i.toString()); + } + + const eventsLength = await client.xlen(trimmedQueue.keys.events); + + expect(eventsLength).to.be.eql(0); + + await trimmedQueue.close(); + await removeAllQueueData(new IORedis(), queueName); + }); + }); + describe('when maxLen is 0', function () { it('should trim events automatically', async () => { const trimmedQueue = new Queue(queueName, { @@ -679,7 +701,7 @@ describe('events', function () { }); }); - describe('when jobs are retried inmediately', function () { + describe('when jobs are retried immediately', function () { it('should trim events so its length is at least the threshold', async () => { const numJobs = 80; const trimmedQueue = new Queue(queueName, { From a3f31c1083c333af6e12bdc9f9539f900bd67ff3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 13 Oct 2023 13:47:25 +0000 Subject: [PATCH 03/14] chore(release): 4.12.4 [skip ci] ## [4.12.4](https://github.com/taskforcesh/bullmq/compare/v4.12.3...v4.12.4) (2023-10-13) ### Bug Fixes * **events:** do not publish removed event on non-existent jobs ([#2227](https://github.com/taskforcesh/bullmq/issues/2227)) ([c134606](https://github.com/taskforcesh/bullmq/commit/c1346064c6cd9f93c59b184f150eac11d51c91b4)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index db43aaad87..689e30c03f 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [4.12.4](https://github.com/taskforcesh/bullmq/compare/v4.12.3...v4.12.4) (2023-10-13) + + +### Bug Fixes + +* **events:** do not publish removed event on non-existent jobs ([#2227](https://github.com/taskforcesh/bullmq/issues/2227)) ([c134606](https://github.com/taskforcesh/bullmq/commit/c1346064c6cd9f93c59b184f150eac11d51c91b4)) + ## [4.12.3](https://github.com/taskforcesh/bullmq/compare/v4.12.2...v4.12.3) (2023-10-10) diff --git a/package.json b/package.json index cea12786b0..abba7ebbb8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "4.12.3", + "version": "4.12.4", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", From 47db0588f0f744e583f4652c3d391cc7533958f7 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Tue, 17 Oct 2023 19:30:59 -0700 Subject: [PATCH 04/14] perf(events): trim events when removing jobs (#2235) (python) --- src/commands/addJob-9.lua | 14 +++++++------- src/commands/removeJob-1.lua | 4 +++- tests/test_events.ts | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/commands/addJob-9.lua b/src/commands/addJob-9.lua index 1d2cc7eaf0..72d175c22f 100644 --- a/src/commands/addJob-9.lua +++ b/src/commands/addJob-9.lua @@ -61,7 +61,6 @@ local parentData --- @include "includes/addDelayMarkerIfNeeded" --- @include "includes/addJobWithPriority" --- @include "includes/getTargetQueueList" ---- @include "includes/trimEvents" --- @include "includes/getNextDelayedTimestamp" --- @include "includes/updateParentDepsIfNeeded" @@ -75,8 +74,7 @@ end local jobCounter = rcall("INCR", KEYS[4]) --- Trim events before emiting them to avoid trimming events emitted in this script -trimEvents(KEYS[3], KEYS[8]) +local maxEvents = rcall("HGET", KEYS[3], "opts.maxLenEvents") or 10000 local parentDependenciesKey = args[7] local timestamp = args[4] @@ -99,7 +97,8 @@ else end rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData) end - rcall("XADD", KEYS[8], "*", "event", "duplicated", "jobId", jobId) + rcall("XADD", KEYS[8], "MAXLEN", "~", maxEvents, "*", "event", "duplicated", + "jobId", jobId) return jobId .. "" -- convert to string end @@ -139,8 +138,8 @@ if waitChildrenKey ~= nil then elseif (delayedTimestamp ~= 0) then local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff) rcall("ZADD", KEYS[5], score, jobId) - rcall("XADD", KEYS[8], "*", "event", "delayed", "jobId", jobId, "delay", - delayedTimestamp) + rcall("XADD", KEYS[8], "MAXLEN", "~", maxEvents, "*", "event", "delayed", "jobId", jobId, + "delay", delayedTimestamp) -- If wait list is empty, and this delayed job is the next one to be processed, -- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list. local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2]) @@ -157,7 +156,8 @@ else addJobWithPriority(KEYS[1], KEYS[6], priority, paused, jobId, KEYS[9]) end -- Emit waiting event - rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId) + rcall("XADD", KEYS[8], "MAXLEN", "~", maxEvents, "*", "event", "waiting", + "jobId", jobId) end -- Check if this job is a child of another job, if so add it to the parents dependencies diff --git a/src/commands/removeJob-1.lua b/src/commands/removeJob-1.lua index 3ef586ba22..d114da7fc9 100644 --- a/src/commands/removeJob-1.lua +++ b/src/commands/removeJob-1.lua @@ -54,7 +54,9 @@ local function removeJob( prefix, jobId, parentKey, removeChildren) local prev = removeJobFromAnyState(prefix, jobId) if rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed") > 0 then - rcall("XADD", prefix .. "events", "*", "event", "removed", "jobId", jobId, "prev", prev) + local maxEvents = rcall("HGET", prefix .. "meta", "opts.maxLenEvents") or 10000 + rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed", + "jobId", jobId, "prev", prev) end end diff --git a/tests/test_events.ts b/tests/test_events.ts index 8cbeac29bb..c9336136c5 100644 --- a/tests/test_events.ts +++ b/tests/test_events.ts @@ -757,6 +757,40 @@ describe('events', function () { await removeAllQueueData(new IORedis(), queueName); }); }); + + describe('when jobs removal is attempted', async () => { + it('should trim events so its length is at least the threshold', async () => { + const numRemovals = 200; + const trimmedQueue = new Queue(queueName, { + connection, + streams: { + events: { + maxLen: 20, + }, + }, + }); + + const client = await trimmedQueue.client; + + const jobs = Array.from(Array(numRemovals).keys()).map(() => ({ + name: 'test', + data: { foo: 'bar' }, + })); + await trimmedQueue.addBulk(jobs); + + for (let i = 1; i <= numRemovals; i++) { + await trimmedQueue.remove(i.toString()); + } + + const eventsLength = await client.xlen(trimmedQueue.keys.events); + + expect(eventsLength).to.be.lte(100); + expect(eventsLength).to.be.gte(20); + + await trimmedQueue.close(); + await removeAllQueueData(new IORedis(), queueName); + }); + }); }); it('should trim events manually', async () => { From 207dcdc7bb7cd4af144e1544342b59744f7fc284 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Oct 2023 02:32:12 +0000 Subject: [PATCH 05/14] chore(release): 4.12.5 [skip ci] ## [4.12.5](https://github.com/taskforcesh/bullmq/compare/v4.12.4...v4.12.5) (2023-10-18) ### Performance Improvements * **events:** trim events when removing jobs ([#2235](https://github.com/taskforcesh/bullmq/issues/2235)) (python) ([889815c](https://github.com/taskforcesh/bullmq/commit/889815c412666e5fad8f32d2e3a2d41cf650f001)) --- docs/gitbook/changelog.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/gitbook/changelog.md b/docs/gitbook/changelog.md index 689e30c03f..8a211e1458 100644 --- a/docs/gitbook/changelog.md +++ b/docs/gitbook/changelog.md @@ -1,3 +1,10 @@ +## [4.12.5](https://github.com/taskforcesh/bullmq/compare/v4.12.4...v4.12.5) (2023-10-18) + + +### Performance Improvements + +* **events:** trim events when removing jobs ([#2235](https://github.com/taskforcesh/bullmq/issues/2235)) (python) ([889815c](https://github.com/taskforcesh/bullmq/commit/889815c412666e5fad8f32d2e3a2d41cf650f001)) + ## [4.12.4](https://github.com/taskforcesh/bullmq/compare/v4.12.3...v4.12.4) (2023-10-13) diff --git a/package.json b/package.json index abba7ebbb8..d557b1bc0b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bullmq", - "version": "4.12.4", + "version": "4.12.5", "description": "Queue for messages and jobs based on Redis", "homepage": "https://bullmq.io/", "main": "./dist/cjs/index.js", From 9d1fd997d8263daa9bdca514bf6950ae5cd0d5d5 Mon Sep 17 00:00:00 2001 From: semantic-release Date: Wed, 18 Oct 2023 02:33:11 +0000 Subject: [PATCH 06/14] 1.15.2 Automatically generated by python-semantic-release --- docs/gitbook/python/changelog.md | 12 ++++++++++++ python/bullmq/__init__.py | 2 +- python/pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/python/changelog.md b/docs/gitbook/python/changelog.md index 35f74aaec5..88a1fc01ef 100644 --- a/docs/gitbook/python/changelog.md +++ b/docs/gitbook/python/changelog.md @@ -2,6 +2,18 @@ +## v1.15.2 (2023-10-18) +### Fix +* **events:** Do not publish removed event on non-existent jobs ([#2227](https://github.com/taskforcesh/bullmq/issues/2227)) ([`c134606`](https://github.com/taskforcesh/bullmq/commit/c1346064c6cd9f93c59b184f150eac11d51c91b4)) +* **events:** Trim events when retrying a job ([#2224](https://github.com/taskforcesh/bullmq/issues/2224)) ([`1986b05`](https://github.com/taskforcesh/bullmq/commit/1986b05ac03fe4ee48861aa60caadcc9df8170a6)) +* **sandbox:** Update progress value on job instance (#2214) fixes #2213 ([`3d0f36a`](https://github.com/taskforcesh/bullmq/commit/3d0f36a134b7f5c6b6de26967c9d71bcfb346e72)) + +### Documentation +* **changelog:** Update multiple changelogs ([`d4683b3`](https://github.com/taskforcesh/bullmq/commit/d4683b324ce56a2209553e28a3d52ad08ee19695)) + +### Performance +* **events:** Trim events when removing jobs (#2235) (python) ([`889815c`](https://github.com/taskforcesh/bullmq/commit/889815c412666e5fad8f32d2e3a2d41cf650f001)) + ## v1.15.1 (2023-10-04) ### Fix * **delayed:** Trim events when moving jobs to delayed (python) ([#2211](https://github.com/taskforcesh/bullmq/issues/2211)) ([`eca8c2d`](https://github.com/taskforcesh/bullmq/commit/eca8c2d4dfeafbd8ac36a49764dbd4897303628c)) diff --git a/python/bullmq/__init__.py b/python/bullmq/__init__.py index f3283745de..dc1d293fa3 100644 --- a/python/bullmq/__init__.py +++ b/python/bullmq/__init__.py @@ -3,7 +3,7 @@ A background job processor and message queue for Python based on Redis. """ -__version__ = "1.15.1" +__version__ = "1.15.2" __author__ = 'Taskforce.sh Inc.' __credits__ = 'Taskforce.sh Inc.' diff --git a/python/pyproject.toml b/python/pyproject.toml index c09f66f50e..dff64f0a74 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "bullmq" -version = "1.15.1" +version = "1.15.2" description='BullMQ for Python' readme="README.md" authors = [ From dafddd698b2fb381aa5e23b49505ea30a59f6f64 Mon Sep 17 00:00:00 2001 From: Jono Kolnik <1164060+JonathanKolnik@users.noreply.github.com> Date: Thu, 19 Oct 2023 22:35:29 -0400 Subject: [PATCH 07/14] docs(rate-limit): fix key typo in example (#2240) --- docs/gitbook/bullmq-pro/groups/rate-limiting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gitbook/bullmq-pro/groups/rate-limiting.md b/docs/gitbook/bullmq-pro/groups/rate-limiting.md index 7a7945b457..bdaf6eabab 100644 --- a/docs/gitbook/bullmq-pro/groups/rate-limiting.md +++ b/docs/gitbook/bullmq-pro/groups/rate-limiting.md @@ -19,7 +19,7 @@ const worker = new WorkerPro('myQueue', processFn, { group: { limit: { max: 100, // Limit to 100 jobs per second per group - duration 1000, + duration: 1000, } }, connection From bf4821519fbf010f319ca2bbb65a66198bf60dce Mon Sep 17 00:00:00 2001 From: roggervalf Date: Thu, 19 Oct 2023 20:00:41 -0700 Subject: [PATCH 08/14] docs(bullmq-pro): update changelog with v6.6.1 --- docs/gitbook/bullmq-pro/changelog.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/gitbook/bullmq-pro/changelog.md b/docs/gitbook/bullmq-pro/changelog.md index 6d10d422d9..3c62d37524 100644 --- a/docs/gitbook/bullmq-pro/changelog.md +++ b/docs/gitbook/bullmq-pro/changelog.md @@ -1,3 +1,11 @@ +## [6.6.1](https://github.com/taskforcesh/bullmq-pro/compare/v6.6.0...v6.6.1) (2023-10-11) + + +### Bug Fixes + +* **events:** trim events when retrying a job ([#2224](https://github.com/taskforcesh/bullmq/issues/2224)) ([1986b05](https://github.com/taskforcesh/bullmq/commit/1986b05ac03fe4ee48861aa60caadcc9df8170a6)) +* **sandbox:** update progress value on job instance ([#2214](https://github.com/taskforcesh/bullmq/issues/2214)) fixes [#2213](https://github.com/taskforcesh/bullmq/issues/2213) ([3d0f36a](https://github.com/taskforcesh/bullmq/commit/3d0f36a134b7f5c6b6de26967c9d71bcfb346e72)) + # [6.6.0](https://github.com/taskforcesh/bullmq-pro/compare/v6.5.0...v6.6.0) (2023-10-06) @@ -6,7 +14,7 @@ ### Features -* expose addJobLog and updateJobProgress to the Queue instance ([#2202](https://github.com/taskforcesh/bullmq/issues/2202)) ([2056939](https://github.com/taskforcesh/bullmq/commit/205693907a4d6c2da9bd0690fb552b1d1e369c08)) +* **queue:** expose addJobLog and updateJobProgress ([#2202](https://github.com/taskforcesh/bullmq/issues/2202)) ([2056939](https://github.com/taskforcesh/bullmq/commit/205693907a4d6c2da9bd0690fb552b1d1e369c08)) # [6.5.0](https://github.com/taskforcesh/bullmq-pro/compare/v6.4.0...v6.5.0) (2023-09-28) @@ -31,7 +39,7 @@ ### Features * **connection:** provide skipVersionCheck option for shared connections ([#2149](https://github.com/taskforcesh/bullmq/issues/2149)) ref [#2148](https://github.com/taskforcesh/bullmq/issues/2148) ([914820f](https://github.com/taskforcesh/bullmq/commit/914820f720cbc48b49f4bd1c46d148eb2bb5b79c)) -* **sandbox:** sandbox: emulate moveToDelayed method ([#180](https://github.com/taskforcesh/bullmq-pro/issues/180)) ([d61de09](https://github.com/taskforcesh/bullmq-pro/commit/d61de095115481b688101bfaf0b126a02545cc6f)) ref [#2118](https://github.com/taskforcesh/bullmq/issues/2118) +* **sandbox:** emulate moveToDelayed method ([#180](https://github.com/taskforcesh/bullmq-pro/issues/180)) ([d61de09](https://github.com/taskforcesh/bullmq-pro/commit/d61de095115481b688101bfaf0b126a02545cc6f)) ref [#2118](https://github.com/taskforcesh/bullmq/issues/2118) ### Bug Fixes @@ -92,9 +100,11 @@ ## [6.2.2](https://github.com/taskforcesh/bullmq-pro/compare/v6.2.1...v6.2.2) (2023-07-26) -### Bug Fixes +### Features -* **deps:** upgrade bullmq to 4.6.0 ([#167](https://github.com/taskforcesh/bullmq-pro/issues/167)) ([9cf92a6](https://github.com/taskforcesh/bullmq-pro/commit/9cf92a62dc3f1e9316bd4559fde4700ff4d9b12c)) +* **queue:** add promoteJobs to promote all delayed jobs ([6074592](https://github.com/taskforcesh/bullmq/commit/6074592574256ec4b1c340126288e803e56b1a64)) +* **job:** add option for removing children in remove method (python) ([#2064](https://github.com/taskforcesh/bullmq/issues/2064)) ([841dc87](https://github.com/taskforcesh/bullmq/commit/841dc87a689897df81438ad1f43e45a4da77c388)) +* **job:** add removeDependencyOnFailure option ([#1953](https://github.com/taskforcesh/bullmq/issues/1953)) ([ffd49e2](https://github.com/taskforcesh/bullmq/commit/ffd49e289c57252487200d47b92193228ae7451f)) ## [6.2.1](https://github.com/taskforcesh/bullmq-pro/compare/v6.2.0...v6.2.1) (2023-07-25) From c7044cfdc96038f5f2cb0748c69eddefcebd8e84 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Mon, 23 Oct 2023 23:11:43 -0700 Subject: [PATCH 09/14] test: add missing prefix on test cases --- tests/test_events.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_events.ts b/tests/test_events.ts index c9336136c5..d0b2ccd54c 100644 --- a/tests/test_events.ts +++ b/tests/test_events.ts @@ -510,6 +510,7 @@ describe('events', function () { const numRemovals = 100; const trimmedQueue = new Queue(queueName, { connection, + prefix, }); const client = await trimmedQueue.client; @@ -763,6 +764,7 @@ describe('events', function () { const numRemovals = 200; const trimmedQueue = new Queue(queueName, { connection, + prefix, streams: { events: { maxLen: 20, From 591655bd4e566bff38dd48272426a3478c68e4c7 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Mon, 23 Oct 2023 23:26:12 -0700 Subject: [PATCH 10/14] chore: move prefix in describe scope --- tests/test_flow.ts | 17 ++++++++++------- tests/test_pause.ts | 4 ++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/test_flow.ts b/tests/test_flow.ts index 2a3de8dfb7..12098a8f71 100644 --- a/tests/test_flow.ts +++ b/tests/test_flow.ts @@ -1306,8 +1306,11 @@ describe('flows', () => { describe('when custom prefix is set in flow producer', async () => { it('uses default prefix to add jobs', async () => { - const prefix = '{bull}'; - const childrenQueue = new Queue(queueName, { prefix, connection }); + const customPrefix = '{bull}'; + const childrenQueue = new Queue(queueName, { + prefix: customPrefix, + connection, + }); const name = 'child-job'; const values = [{ bar: 'something' }]; @@ -1356,16 +1359,16 @@ describe('flows', () => { const parentWorker = new Worker(parentQueueName, parentProcessor, { connection, - prefix, + prefix: customPrefix, }); const childrenWorker = new Worker(queueName, childrenProcessor, { connection, - prefix, + prefix: customPrefix, }); await parentWorker.waitUntilReady(); await childrenWorker.waitUntilReady(); - const flow = new FlowProducer({ prefix: '{bull}', connection }); + const flow = new FlowProducer({ prefix: customPrefix, connection }); const tree = await flow.add({ name: 'parent-job', queueName: parentQueueName, @@ -1393,8 +1396,8 @@ describe('flows', () => { await flow.close(); await childrenQueue.close(); - await removeAllQueueData(new IORedis(), parentQueueName, prefix); - await removeAllQueueData(new IORedis(), queueName, prefix); + await removeAllQueueData(new IORedis(), parentQueueName, customPrefix); + await removeAllQueueData(new IORedis(), queueName, customPrefix); }); }); diff --git a/tests/test_pause.ts b/tests/test_pause.ts index 532e96b118..ad8688a678 100644 --- a/tests/test_pause.ts +++ b/tests/test_pause.ts @@ -5,9 +5,9 @@ import { v4 } from 'uuid'; import { Job, Queue, QueueEvents, Worker } from '../src/classes'; import { delay, removeAllQueueData } from '../src/utils'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('Pause', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; + let queue: Queue; let queueName: string; let queueEvents: QueueEvents; From 4398bb2b5acda3a7489d611c66d8496104dfc471 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Mon, 23 Oct 2023 23:36:44 -0700 Subject: [PATCH 11/14] chore: move prefix in describe scope --- tests/test_rate_limiter.ts | 3 +-- tests/test_repeat.ts | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_rate_limiter.ts b/tests/test_rate_limiter.ts index d9fa4baebb..e1143cf085 100644 --- a/tests/test_rate_limiter.ts +++ b/tests/test_rate_limiter.ts @@ -12,9 +12,8 @@ import { } from '../src/classes'; import { delay, removeAllQueueData } from '../src/utils'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('Rate Limiter', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; let queue: Queue; let queueName: string; let queueEvents: QueueEvents; diff --git a/tests/test_repeat.ts b/tests/test_repeat.ts index b6c34fd1db..90d37b2adb 100644 --- a/tests/test_repeat.ts +++ b/tests/test_repeat.ts @@ -30,9 +30,8 @@ const ONE_DAY = 24 * ONE_HOUR; const NoopProc = async (job: Job) => {}; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('repeat', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; this.timeout(10000); let repeat: Repeat; let queue: Queue; From 5a1812a0b6404cf5613f8deeda6b1ca0f1e5fb87 Mon Sep 17 00:00:00 2001 From: Rogger Valverde Date: Tue, 24 Oct 2023 07:53:58 -0600 Subject: [PATCH 12/14] test: pass prefix in describe scope in test_connection.ts --- tests/test_connection.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_connection.ts b/tests/test_connection.ts index 30c3d7d2c7..2f1248102d 100644 --- a/tests/test_connection.ts +++ b/tests/test_connection.ts @@ -4,9 +4,8 @@ import { v4 } from 'uuid'; import { Queue, Job, Worker, QueueBase } from '../src/classes'; import { removeAllQueueData } from '../src/utils'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('connection', () => { + const prefix = procesos.env.BULLMQ_TEST_PREFIX ||'bull'; let queue: Queue; let queueName: string; const connection = { host: 'localhost' }; From a50e91c58629b62d35d6a6b7d55af46e5952fd92 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Tue, 24 Oct 2023 07:00:23 -0700 Subject: [PATCH 13/14] test: pass prefix in describe scopes --- tests/test_connection.ts | 2 +- tests/test_obliterate.ts | 3 +-- tests/test_sandboxed_process.ts | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_connection.ts b/tests/test_connection.ts index 2f1248102d..2fecba643f 100644 --- a/tests/test_connection.ts +++ b/tests/test_connection.ts @@ -5,7 +5,7 @@ import { Queue, Job, Worker, QueueBase } from '../src/classes'; import { removeAllQueueData } from '../src/utils'; describe('connection', () => { - const prefix = procesos.env.BULLMQ_TEST_PREFIX ||'bull'; + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; let queue: Queue; let queueName: string; const connection = { host: 'localhost' }; diff --git a/tests/test_obliterate.ts b/tests/test_obliterate.ts index 0bfd21f86e..03fe26ab03 100644 --- a/tests/test_obliterate.ts +++ b/tests/test_obliterate.ts @@ -6,9 +6,8 @@ import { v4 } from 'uuid'; import { Queue, QueueEvents, FlowProducer, Worker, Job } from '../src/classes'; import { delay, removeAllQueueData } from '../src/utils'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('Obliterate', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; let queue: Queue; let queueEvents: QueueEvents; let queueName: string; diff --git a/tests/test_sandboxed_process.ts b/tests/test_sandboxed_process.ts index 560ce8f4cb..820d9e9c8d 100644 --- a/tests/test_sandboxed_process.ts +++ b/tests/test_sandboxed_process.ts @@ -8,8 +8,6 @@ import { delay, removeAllQueueData } from '../src/utils'; import { Child } from '../src/classes/child'; const { stdout, stderr } = require('test-console'); -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('Sandboxed process using child processes', () => { sandboxProcessTests(); }); @@ -22,6 +20,7 @@ function sandboxProcessTests( { useWorkerThreads } = { useWorkerThreads: false }, ) { describe('sandboxed process', () => { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; let queue: Queue; let queueEvents: QueueEvents; let queueName: string; From afd4d384d9ed153156bbb874152d51fd791cb8a3 Mon Sep 17 00:00:00 2001 From: roggervalf Date: Tue, 24 Oct 2023 07:15:34 -0700 Subject: [PATCH 14/14] test: pass prefix in describe scopes --- tests/test_clean.ts | 3 +-- tests/test_delay.ts | 3 +-- tests/test_getters.ts | 3 +-- tests/test_job.ts | 3 +-- tests/test_metrics.ts | 3 +-- tests/test_queue.ts | 3 +-- tests/test_stalled_jobs.ts | 3 +-- 7 files changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/test_clean.ts b/tests/test_clean.ts index 5c1e7943bc..4991c73d31 100644 --- a/tests/test_clean.ts +++ b/tests/test_clean.ts @@ -12,9 +12,8 @@ import { } from '../src/classes'; import { delay, removeAllQueueData } from '../src/utils'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('Cleaner', () => { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; let queue: Queue; let queueEvents: QueueEvents; let queueName: string; diff --git a/tests/test_delay.ts b/tests/test_delay.ts index dce37fb897..0454715c64 100644 --- a/tests/test_delay.ts +++ b/tests/test_delay.ts @@ -6,9 +6,8 @@ import { v4 } from 'uuid'; import { Queue, Job, Worker, QueueEvents } from '../src/classes'; import { removeAllQueueData, delay } from '../src/utils'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('Delayed jobs', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; this.timeout(15000); let queue: Queue; diff --git a/tests/test_getters.ts b/tests/test_getters.ts index 029959d017..896de23016 100644 --- a/tests/test_getters.ts +++ b/tests/test_getters.ts @@ -9,9 +9,8 @@ import { v4 } from 'uuid'; import { FlowProducer, Queue, QueueEvents, Worker } from '../src/classes'; import { delay, removeAllQueueData } from '../src/utils'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('Jobs getters', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; let queue: Queue; let queueName: string; const connection = { host: 'localhost' }; diff --git a/tests/test_job.ts b/tests/test_job.ts index 2fb44744b0..b990188218 100644 --- a/tests/test_job.ts +++ b/tests/test_job.ts @@ -10,9 +10,8 @@ import { Job, Queue, QueueEvents, Worker } from '../src/classes'; import { JobsOptions } from '../src/types'; import { delay, getParentKey, removeAllQueueData } from '../src/utils'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('Job', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; let queue: Queue; let queueName: string; const connection = { host: 'localhost' }; diff --git a/tests/test_metrics.ts b/tests/test_metrics.ts index b8689c9085..80800121c6 100644 --- a/tests/test_metrics.ts +++ b/tests/test_metrics.ts @@ -12,9 +12,8 @@ const ONE_SECOND = 1000; const ONE_MINUTE = 60 * ONE_SECOND; const ONE_HOUR = 60 * ONE_MINUTE; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('metrics', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; this.timeout(10000); let repeat: Repeat; let queue: Queue; diff --git a/tests/test_queue.ts b/tests/test_queue.ts index 78a4d80881..f0adcf2b57 100644 --- a/tests/test_queue.ts +++ b/tests/test_queue.ts @@ -7,9 +7,8 @@ import { v4 } from 'uuid'; import { FlowProducer, Job, Queue, Worker } from '../src/classes'; import { delay, removeAllQueueData } from '../src/utils'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('queues', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; const sandbox = sinon.createSandbox(); let queue: Queue; diff --git a/tests/test_stalled_jobs.ts b/tests/test_stalled_jobs.ts index 1b85ef9662..cad01040b2 100644 --- a/tests/test_stalled_jobs.ts +++ b/tests/test_stalled_jobs.ts @@ -6,9 +6,8 @@ import { beforeEach, describe, it } from 'mocha'; import { v4 } from 'uuid'; import { expect } from 'chai'; -const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; - describe('stalled jobs', function () { + const prefix = process.env.BULLMQ_TEST_PREFIX || 'bull'; let queue: Queue; let queueName: string;