feat: allow additional reporting columns and add additional metadata …

…to webhook (#1194)

* add ReportingColumns types and tests

* feat: simplify GOV.UK Pay calls. Generate GOV.UK Pay reference number once, so payments are easier to track.

* feat: add metadata to GOV.UK Pay request

* add more pay data to webhook.metadata

* use "fieldPath" instead of "fieldValue". adding documentation

* rename fieldValue to fieldPath

* add back wildcard test files

* use custom alphabet, capitals only

* add payReferenceLength

* fix tests

* docs

docs/runner/fee-options.md

@@ -35,6 +35,24 @@
      * allowSubmissionWithoutPayment must be true for this to be shown.
      */
     showPaymentSkippedWarningPage: false,
+
+    /**
+     * Adds metadata to the GOV.UK Pay request. You may add static values, or values based on the user's answer.
+     */
+    additionalReportingColumns: [
+      {
+        columnName: "country",
+        fieldPath: "beforeYouStart.country", // the path in the state object to retrieve the value from. If the value is in a section, use the format {sectionName}.{fieldName}.
+      },
+      {
+        columnName: "post",
+        fieldPath: "post",
+      },
+      {
+        columnName: "service",
+        staticValue: "fee 11",
+      },
+    ],
   },
 }
 ```
@@ -73,3 +91,71 @@ The can choose to continue or try online payment. This page will be shown only o
   },
 }
 ```
+
+## Reporting columns
+
+[GOV.UK Pay allows additional reporting columns to be configured](https://docs.payments.service.gov.uk/api_reference/create_a_payment_reference/#json-body-parameters-for-39-create-a-payment-39).
+This is useful if you wish to filter payments in GOV.UK Pay. In FCDO's case, it is useful to filter by country selected by the user.
+
+You may add static values, or values based on the user's answer. You may only configure 10 reporting columns as per
+GOV.UK Pay's limits, and ensure that each columnName is <30 characters. The values may be 100 characters.
+
+```json5
+{
+  //..
+  additionalReportingColumns: [
+    {
+      columnName: "country",
+      fieldPath: "beforeYouStart.country", // the path in the state object to retrieve the value from. If the value is in a section, use the format {sectionName}.{fieldName}.
+    },
+    {
+      columnName: "post",
+      fieldPath: "post",
+    },
+    {
+      columnName: "service",
+      staticValue: "fee 11",
+    },
+  ],
+}
+```
+
+If the value at the fieldPath cannot be found, the column will not be added to the metadata.
+
+Given the user state looks like
+
+```.ts
+const state = {
+  beforeYouStart: {
+    country: "United Kingdom",
+  }
+}
+```
+
+This will be parsed and sent to GOV.UK Pay as:
+
+```.ts
+ const requestOptions = {
+  //.. GOV.UK Pay request
+  metadata: {
+    country: "United Kingdom",
+    // no post key since it's not present in the user's state,
+    service: "fee 11"
+  }
+ }
+```
+
+When viewing this payment in GOV.UK Pay, you can sort by these columns, and will appear as "metadata" in their interface.
+
+## Other - Reference numbers
+
+Reference numbers are generated with the alphabet "1234567890ABCDEFGHIJKLMNPQRSTUVWXYZ-_". Note that the letter O is omitted.
+
+You may configure the length of the reference number by setting the environment variable `PAY_REFERENCE_LENGTH`. The default is 10 characters. 
+Use [Nano ID Collision Calculator](https://zelark.github.io/nano-id-cc/) to determine the right length for your service. 
+Since each user will "keep" their own reference number for multiple attempts, calculate the speed at unique users per hour. 
+
+e.g. If your service expects 100,000 users per annum, you should expect ~274 users per day, and 11 users per hour. 
+Using nano-id-cc, and a reference length of 10 characters it will take 102 years, or 9 million IDs generated for a 1% chance of collision.
+
+

model/src/data-model/types.ts

@@ -146,13 +146,20 @@ export type Fee = {
   prefix?: string;
 };
 
+export type AdditionalReportingColumn = {
+  columnName: string;
+  fieldPath?: string;
+  staticValue?: string;
+};
+
 export type FeeOptions = {
   paymentReferenceFormat?: string;
   payReturnUrl?: string;
   allowSubmissionWithoutPayment: boolean;
   maxAttempts: number;
   customPayErrorMessage?: string;
   showPaymentSkippedWarningPage: boolean;
+  additionalReportingColumns?: AdditionalReportingColumn[];
 };
 
 /**

model/src/schema/schema.ts

@@ -259,6 +259,16 @@ const feeOptionSchema = joi
       then: joi.boolean().valid(true, false).default(false),
       otherwise: joi.boolean().valid(false).default(false),
     }),
+    additionalReportingColumns: joi
+      .array()
+      .items(
+        joi.object({
+          columnName: joi.string().required(),
+          fieldPath: joi.string().optional(),
+          staticValue: joi.string().optional(),
+        })
+      )
+      .optional(),
   })
   .default(({ payApiKey, paymentReferenceFormat }) => {
     return {

runner/config/custom-environment-variables.json

@@ -13,6 +13,7 @@
   "matomoUrl": "MATOMO_URL",
   "payApiUrl": "PAY_API_URL",
   "payReturnUrl": "PAY_RETURN_URL",
+  "payReferenceLength": "PAY_REFERENCE_LENGTH",
   "serviceUrl": "SERVICE_URL",
   "redisHost": "REDIS_HOST",
   "redisPort": "REDIS_PORT",

runner/config/default.js

@@ -104,6 +104,7 @@ module.exports = {
   // Control which is used. Accepts "test" | "production" | "".
   apiEnv: "",
   payApiUrl: "https://publicapi.payments.service.gov.uk/v1",
+  // payReferenceLength: "10" // The length of the string generated for GOV.UK Pay references.
   // If both the api env and node env are set to "production", the pay return url will need to be secure.
   // This is not the case if either are set to "test", or if the node env is set to "development"
   // payReturnUrl: "http://localhost:3009"

runner/package.json

@@ -17,9 +17,9 @@
     "fix-lint": "yarn bin/run eslint . --fix",
     "test": "yarn lint && yarn type-check && NODE_ENV=test yarn bin/run unit-test",
     "test-cov": "yarn run unit-test-cov",
-    "test:dev": "lab -T test/.transform.js -P test/**/*.test.* -v test --coverage-exclude",
-    "unit-test": "lab -T test/.transform.js -P test/**/*.test.* -v test -S -v -r console -o stdout -r html -o unit-test.html -I version -l",
-    "unit-test-cov": "lab -T test/.transform.js -P test/**/*.test.* -v test -t 83 -S -v -r console -o stdout -r lcov -o test-coverage/lab/lcov.info -r html -o test-coverage/lab/unit-test.html -r junit -o test-results/junit/unit-test.xml -I version -l",
+    "test:dev": "lab -T test/.transform.js -P (test|src)/**/*.test.* -v test --coverage-exclude",
+    "unit-test": "lab -T test/.transform.js -P (test|src)/**/*.test.* -v test -S -v -r console -o stdout -r html -o unit-test.html -I version -l",
+    "unit-test-cov": "lab -T test/.transform.js -P (test|src)/**/*.test.* -v test -t 83 -S -v -r console -o stdout -r lcov -o test-coverage/lab/lcov.info -r html -o test-coverage/lab/unit-test.html -r junit -o test-results/junit/unit-test.xml -I version -l",
     "a11y": "node test/audit/components && node lighthouse",
     "symlink-env": "./bin/symlink-config",
     "type-check": "tsc --noEmit",

runner/src/server/plugins/applicationStatus/index.ts

@@ -89,7 +89,7 @@ const index = {
           const { pay } = await cacheService.getState(request);
           const { meta } = pay;
           meta.attempts++;
-          const res = await payService.retryPayRequest(pay);
+          const res = await payService.payRequestFromMeta(meta);
 
           await cacheService.mergeState(request, {
             webhookData: {

runner/src/server/plugins/engine/models/submission/FeesModel.ts

@@ -1,14 +1,17 @@
 import { FormModel } from "server/plugins/engine/models";
 import { FormSubmissionState } from "server/plugins/engine/types";
 import { reach } from "hoek";
-import { Fee } from "@xgovformbuilder/model";
+import { Fee, AdditionalReportingColumn } from "@xgovformbuilder/model";
 import { FeeDetails } from "server/services/payService";
 
 export type FeesModel = {
   details: FeeDetails[];
   total: number;
   prefixes: string[];
   referenceFormat?: string;
+  reportingColumns?: {
+    [key: string]: any;
+  };
 };
 
 function feesAsFeeDetails(
@@ -46,16 +49,18 @@ export function FeesModel(
     return undefined;
   }
 
-  const details = feesAsFeeDetails(applicableFees, state);
+  const columnsConfig = model.feeOptions?.additionalReportingColumns;
+  const reportingColumns = ReportingColumns(columnsConfig, state);
 
+  const details = feesAsFeeDetails(applicableFees, state);
   return details.reduce(
-    (previous: FeesModel, fee: FeeDetails) => {
+    (acc: FeesModel, fee: FeeDetails) => {
       const { amount, multiplyBy = 1, prefix = "" } = fee;
-      return {
-        ...previous,
-        total: previous.total + amount * multiplyBy,
-        prefixes: [...previous.prefixes, prefix].filter((p) => p),
-      };
+
+      acc.total = acc.total + amount * multiplyBy;
+      acc.prefixes = [...acc.prefixes, prefix].filter((p) => p);
+
+      return acc;
     },
     {
       details,
@@ -65,6 +70,33 @@ export function FeesModel(
         model.feeOptions?.paymentReferenceFormat ??
         model.def.paymentReferenceFormat ??
         "",
+      ...(reportingColumns && { reportingColumns }),
     }
   );
 }
+
+/**
+ * Creates a GOV.UK metadata object (reporting columns) to send in the payment creation.
+ */
+export function ReportingColumns(
+  reportingColumns: FormModel["feeOptions"]["additionalReportingColumns"],
+  state: FormSubmissionState
+): FeesModel["reportingColumns"] {
+  if (!reportingColumns) {
+    return;
+  }
+
+  return reportingColumns.reduce((prev, curr) => {
+    if (curr.fieldPath) {
+      const stateValue = reach(state, curr.fieldPath);
+      if (!stateValue) {
+        return prev;
+      }
+      prev[curr.columnName] = stateValue;
+    }
+    if (curr.staticValue) {
+      prev[curr.columnName] = curr.staticValue;
+    }
+    return prev;
+  }, {});
+}

runner/src/server/plugins/engine/models/submission/__tests__/FeesModel.test.ts

@@ -1,4 +1,4 @@
-import { FeesModel } from "./../FeesModel";
+import { FeesModel, ReportingColumns } from "./../FeesModel";
 import * as Code from "@hapi/code";
 import * as Lab from "@hapi/lab";
 const { expect } = Code;
@@ -10,12 +10,12 @@ import { FormModel } from "server/plugins/engine/models";
 
 suite("FeesModel", () => {
   test("returns correct FeesModel", () => {
-    const c = {
+    const state = {
       caz: "2",
     };
 
     const form = new FormModel(json, {});
-    const model = FeesModel(form, c);
+    const model = FeesModel(form, state);
     expect(model).to.equal({
       details: [
         { description: "Bristol tax", amount: 5000, condition: "dFQTyf" },
@@ -27,7 +27,7 @@ suite("FeesModel", () => {
     });
   });
   test("returns correct payment reference format when a peyment reference is supplied in the feeOptions", () => {
-    const c = {
+    const state = {
       caz: "2",
     };
     const newJson = {
@@ -37,7 +37,7 @@ suite("FeesModel", () => {
       },
     };
     const form = new FormModel(newJson, {});
-    const model = FeesModel(form, c);
+    const model = FeesModel(form, state);
     expect(model).to.equal({
       details: [
         { description: "Bristol tax", amount: 5000, condition: "dFQTyf" },
@@ -48,4 +48,94 @@ suite("FeesModel", () => {
       referenceFormat: "FCDO2-{{DATE}}",
     });
   });
+  test("returns correct payment reference format when a peyment reference is supplied in the feeOptions", () => {
+    const newJson = {
+      ...json,
+      feeOptions: {
+        paymentReferenceFormat: "FCDO2-{{DATE}}",
+        additionalReportingColumns: [
+          {
+            columnName: "zone",
+            fieldPath: "caz",
+          },
+        ],
+      },
+    };
+    const form = new FormModel(newJson, {});
+
+    const state = {
+      caz: "2",
+    };
+
+    const model = FeesModel(form, state);
+    expect(model).to.equal({
+      details: [
+        { description: "Bristol tax", amount: 5000, condition: "dFQTyf" },
+        { description: "car tax", amount: 5000 },
+      ],
+      total: 10000,
+      prefixes: [],
+      referenceFormat: "FCDO2-{{DATE}}",
+      reportingColumns: {
+        zone: "2",
+      },
+    });
+  });
+});
+
+suite("ReportingColumns", () => {
+  const additionalReportingColumns = [
+    {
+      columnName: "country",
+      fieldPath: "beforeYouStart.country",
+    },
+    {
+      columnName: "post",
+      fieldPath: "post",
+    },
+    {
+      columnName: "service",
+      staticValue: "fee 11",
+    },
+  ];
+
+  test("Returns the correct metadata for GOV.UK Pay", () => {
+    expect(
+      ReportingColumns(additionalReportingColumns, {
+        beforeYouStart: {
+          country: "Italy",
+        },
+        post: "British Embassy Rome",
+      })
+    ).to.equal({
+      country: "Italy",
+      post: "British Embassy Rome",
+      service: "fee 11",
+    });
+  });
+
+  test("Does not add a reporting column when the state value is missing for a nested state value", () => {
+    expect(
+      ReportingColumns(additionalReportingColumns, { post: "A" })
+    ).to.equal({ post: "A", service: "fee 11" });
+  });
+
+  test("Does not add a reporting column when the state value is missing for an un-nested", () => {
+    expect(
+      ReportingColumns(additionalReportingColumns, {
+        beforeYouStart: {
+          country: "Italy",
+        },
+      })
+    ).to.equal({
+      country: "Italy",
+      service: "fee 11",
+    });
+  });
+
+  test("Adds static values", () => {
+    expect(ReportingColumns(additionalReportingColumns, {})).to.equal({
+      service: "fee 11",
+    });
+  });
 });