feat: upgrade pgboss to v10

README.md

@@ -31,7 +31,7 @@ Also see the individual repo README files for additional info:
 
 **Always run scripts from the root directory.**
 
-1. Make sure you are using node 18 `node --version`.
+1. Make sure you are using node 20 `node --version`.
 2. Make sure you have yarn 1.22+ installed. You do not need to install yarn 2.4+, yarn will detect the yarn 2 binary within [.yarn](./.yarn) and that will be used.
 3. If using the designer:
    - Note that the designer requires the runner to be running with the default `NODE_ENV=development` settings (see [runner/config/development.json](https://github.com/XGovFormBuilder/digital-form-builder/tree/main/runner/config/development.json)) to enable posting and previewing of forms during design.

docs/runner/submission-queue.md

@@ -188,6 +188,8 @@ following errors will be thrown:
 
 ## Migration guide
 
+### MYSQL to PGBOSS
+
 If you are moving from MYSQL to PGBOSS, ensure you have a worker which will handle the jobs added to your queue. For "zero downtime",
 
 1. Set up any new infrastructure components if necessary (e.g. database and worker)
@@ -199,3 +201,13 @@ Any submissions that have previously failed, or were submitted during deployment
 Check the database to ensure that there are no more failed entries.
 
 You may then safely remove the submitter, and MySQL database (if it is not used for any other purpose).
+
+### PGBOSS v9 to v10
+
+If you are upgrading from v3.33.0-rc.10 to any higher version, you will need to set `QUEUE_DATABASE_SCHEMA_NAME` to a new value, such as `pgboss_v10`. This is so that pgboss can set up it's schema on the first run.
+
+You should keep an old version of your queue worker running for a few minutes, so that messages can be drained form the old schema.
+It will then be safe to
+
+1. Delete the old schema
+2. Stop the old worker

runner/package.json

@@ -73,7 +73,7 @@
     "nanoid": "^3.3.4",
     "notifications-node-client": "^7.0.4",
     "nunjucks": "^3.2.3",
-    "pg-boss": "^9.0.3",
+    "pg-boss": "^10.1.3",
     "resolve": "^1.19.0",
     "schmervice": "^1.6.0",
     "tmp": "^0.2.1",

runner/src/server/services/pgBossQueueService.ts

@@ -1,6 +1,6 @@
 import { QueueService } from "server/services/QueueService";
 type QueueResponse = [number | string, string | undefined];
-import PgBoss, { Job, JobWithMetadata } from "pg-boss";
+import PgBoss, { JobWithMetadata } from "pg-boss";
 import config from "server/config";
 
 type QueueReferenceApiResponse = {
@@ -32,12 +32,19 @@ export class PgBossQueueService extends QueueService {
     });
     this.queue = boss;
     boss.on("error", this.logger.error);
-    boss.start().catch((e) => {
-      this.logger.error(
-        `Connecting to ${config.queueDatabaseUrl} failed, exiting`
-      );
-      throw e;
-    });
+    boss
+      .start()
+      .then(() => {
+        boss.createQueue(this.queueName).then(() => {
+          this.logger.info(`${this.queueName} queue created successfully`);
+        });
+      })
+      .catch((e) => {
+        this.logger.error(
+          `Connecting to ${config.queueDatabaseUrl} failed, exiting`
+        );
+        throw e;
+      });
   }
 
   /**
@@ -52,7 +59,7 @@ export class PgBossQueueService extends QueueService {
     let job;
 
     try {
-      job = await this.queue.getJobById(jobId);
+      job = await this.queue.getJobById(this.queueName, jobId);
     } catch (e) {
       this.logger.error(
         ["PgBossQueueService", "getReturnRef"],

yarn.lock

@@ -5183,7 +5183,7 @@ __metadata:
     nodemon: ^3.0.2
     notifications-node-client: ^7.0.4
     nunjucks: ^3.2.3
-    pg-boss: ^9.0.3
+    pg-boss: ^10.1.3
     pino: 8.15.1
     prisma: ^5.1.1
     resolve: ^1.19.0
@@ -6765,13 +6765,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"buffer-writer@npm:2.0.0":
-  version: 2.0.0
-  resolution: "buffer-writer@npm:2.0.0"
-  checksum: 11736b48bb75106c52ca8ec9f025e7c1b3b25ce31875f469d7210eabd5c576c329e34f6b805d4a8d605ff3f0db1e16342328802c4c963e9c826b0e43a4e631c2
-  languageName: node
-  linkType: hard
-
 "buffer-xor@npm:^1.0.3":
   version: 1.0.3
   resolution: "buffer-xor@npm:1.0.3"
@@ -7918,7 +7911,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"cron-parser@npm:^4.0.0":
+"cron-parser@npm:^4.9.0":
   version: 4.9.0
   resolution: "cron-parser@npm:4.9.0"
   dependencies:
@@ -8478,13 +8471,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"delay@npm:^5.0.0":
-  version: 5.0.0
-  resolution: "delay@npm:5.0.0"
-  checksum: 62f151151ecfde0d9afbb8a6be37a6d103c4cb24f35a20ef3fe56f920b0d0d0bb02bc9c0a3084d0179ef669ca332b91155f2ee4d9854622cd2cdba5fc95285f9
-  languageName: node
-  linkType: hard
-
 "delayed-stream@npm:~1.0.0":
   version: 1.0.0
   resolution: "delayed-stream@npm:1.0.0"
@@ -16629,13 +16615,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"packet-reader@npm:1.0.0":
-  version: 1.0.0
-  resolution: "packet-reader@npm:1.0.0"
-  checksum: 0b7516f0cbf3e322aad591bed29ba544220088c53943145c0d9121a6f59182ad811f7fd6785a8979a34356aca69d97653689029964c5998dc02645633d88ffd7
-  languageName: node
-  linkType: hard
-
 "pako@npm:~1.0.2, pako@npm:~1.0.5":
   version: 1.0.11
   resolution: "pako@npm:1.0.11"
@@ -16924,18 +16903,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"pg-boss@npm:^9.0.3":
-  version: 9.0.3
-  resolution: "pg-boss@npm:9.0.3"
+"pg-boss@npm:^10.1.3":
+  version: 10.1.3
+  resolution: "pg-boss@npm:10.1.3"
   dependencies:
-    cron-parser: ^4.0.0
-    delay: ^5.0.0
-    lodash.debounce: ^4.0.8
-    p-map: ^4.0.0
-    pg: ^8.5.1
+    cron-parser: ^4.9.0
+    pg: ^8.12.0
     serialize-error: ^8.1.0
-    uuid: ^9.0.0
-  checksum: d720acf61c42cd9b92a77367bd63330f25a6dd7123ad61b05648661e32f14d223b717f9fb25a4172f28719d6c31fc34a5f4979895816248b4d8542cd97fd3541
+  checksum: de8e1ca2356756f5a2de645f50f83b9dd17815da3b98723e3854145d714b2e4f38de5c16c96cd16cc0ab6e69f6eae2df6010f86a70b95f6276364aa9ed300cb3
   languageName: node
   linkType: hard
 
@@ -16946,10 +16921,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"pg-connection-string@npm:^2.6.2":
-  version: 2.6.2
-  resolution: "pg-connection-string@npm:2.6.2"
-  checksum: 22265882c3b6f2320785378d0760b051294a684989163d5a1cde4009e64e84448d7bf67d9a7b9e7f69440c3ee9e2212f9aa10dd17ad6773f6143c6020cebbcb5
+"pg-connection-string@npm:^2.7.0":
+  version: 2.7.0
+  resolution: "pg-connection-string@npm:2.7.0"
+  checksum: 68015a8874b7ca5dad456445e4114af3d2602bac2fdb8069315ecad0ff9660ec93259b9af7186606529ac4f6f72a06831e6f20897a689b16cc7fda7ca0e247fd
   languageName: node
   linkType: hard
 
@@ -16960,19 +16935,19 @@ __metadata:
   languageName: node
   linkType: hard
 
-"pg-pool@npm:^3.6.1":
-  version: 3.6.1
-  resolution: "pg-pool@npm:3.6.1"
+"pg-pool@npm:^3.7.0":
+  version: 3.7.0
+  resolution: "pg-pool@npm:3.7.0"
   peerDependencies:
     pg: ">=8.0"
-  checksum: 8a6513e6f74a794708c9dd16d2ccda0debadc56435ec2582de2b2e35b01315550c5dab8a0a9a2a16f4adce45523228f5739940fb7687ec7e9c300f284eb08fd1
+  checksum: 66fc1a5ad0e17b72671b9a2cd4c7a856fb08d3cb82da7af0b322590ada23127ac591111e855740405fde4f06c9de888abe9f3aa685ed6038c3232578e1fce8cf
   languageName: node
   linkType: hard
 
-"pg-protocol@npm:^1.6.0":
-  version: 1.6.0
-  resolution: "pg-protocol@npm:1.6.0"
-  checksum: e12662d2de2011e0c3a03f6a09f435beb1025acdc860f181f18a600a5495dc38a69d753bbde1ace279c8c442536af9c1a7c11e1d0fe3fad3aa1348b28d9d2683
+"pg-protocol@npm:^1.7.0":
+  version: 1.7.0
+  resolution: "pg-protocol@npm:1.7.0"
+  checksum: 2dba740f6fc4b7f9761682c4c42d183b444292cdc7638b373f5247ec995c8199c369953343479281da3c41611fe34130a80c8668348d49a399c164f802f76be2
   languageName: node
   linkType: hard
 
@@ -16989,16 +16964,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"pg@npm:^8.5.1":
-  version: 8.11.3
-  resolution: "pg@npm:8.11.3"
+"pg@npm:^8.12.0":
+  version: 8.13.0
+  resolution: "pg@npm:8.13.0"
   dependencies:
-    buffer-writer: 2.0.0
-    packet-reader: 1.0.0
     pg-cloudflare: ^1.1.1
-    pg-connection-string: ^2.6.2
-    pg-pool: ^3.6.1
-    pg-protocol: ^1.6.0
+    pg-connection-string: ^2.7.0
+    pg-pool: ^3.7.0
+    pg-protocol: ^1.7.0
     pg-types: ^2.1.0
     pgpass: 1.x
   peerDependencies:
@@ -17009,7 +16982,7 @@ __metadata:
   peerDependenciesMeta:
     pg-native:
       optional: true
-  checksum: 8af9468b8969fa0d73a6b349216c8cbc953d938fcae5594f2d24043060e9226a072c8085fc4230172b5576fcab4c39c8563c655f271dc2a9209b6ad5370cafe5
+  checksum: 81560755ff4ee62b71bf1204dd696f66451574d1db56cbd5aa514ce91c6474030ee8078461b3cb85cce8d2f185be5846e0a7a707a818f5e2e3fb198a7ea795ea
   languageName: node
   linkType: hard
 
@@ -21198,15 +21171,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"uuid@npm:^9.0.0":
-  version: 9.0.1
-  resolution: "uuid@npm:9.0.1"
-  bin:
-    uuid: dist/bin/uuid
-  checksum: 39931f6da74e307f51c0fb463dc2462807531dc80760a9bff1e35af4316131b4fc3203d16da60ae33f07fdca5b56f3f1dd662da0c99fea9aaeab2004780cc5f4
-  languageName: node
-  linkType: hard
-
 "v8-compile-cache@npm:^2.0.3, v8-compile-cache@npm:^2.1.1":
   version: 2.4.0
   resolution: "v8-compile-cache@npm:2.4.0"