Per-Test status updates - basic queries and endpoints

server/controllers/AutomationController.js

@@ -1,7 +1,8 @@
 const axios = require('axios');
 const {
     getCollectionJobById,
-    updateCollectionJobById
+    updateCollectionJobById,
+    updateCollectionJobTestStatusByQuery
 } = require('../models/services/CollectionJobService');
 const {
     findOrCreateTestResult
@@ -24,7 +25,7 @@ const {
 } = require('../models/services/TestResultReadService');
 const http = require('http');
 const { NO_OUTPUT_STRING } = require('../util/constants');
-const getTests = require('../models/services/TestsService');
+const runnableTestsResolver = require('../resolvers/TestPlanReport/runnableTestsResolver');
 const getGraphQLContext = require('../graphql-context');
 const httpAgent = new http.Agent({ family: 4 });
 
@@ -99,6 +100,23 @@ const updateJobStatus = async (req, res) => {
         ...(externalLogsUrl != null && { externalLogsUrl })
     };
 
+    // When new status is 'COMPLETED' or 'ERROR' or 'CANCELLED'
+    // update any CollectionJobTestStatus children still 'QUEUED' to be 'CANCELLED'
+    if (
+        status === COLLECTION_JOB_STATUS.COMPLETED ||
+        status === COLLECTION_JOB_STATUS.CANCELLED ||
+        status === COLLECTION_JOB_STATUS.ERROR
+    ) {
+        await updateCollectionJobTestStatusByQuery({
+            where: {
+                collectionJobId: req.params.jobID,
+                status: COLLECTION_JOB_STATUS.QUEUED
+            },
+            values: { status: COLLECTION_JOB_STATUS.CANCELLED },
+            transaction: req.transaction
+        });
+    }
+
     const graphqlResponse = await updateCollectionJobById({
         id: req.params.jobID,
         values: updatePayload,
@@ -137,32 +155,29 @@ const getApprovedFinalizedTestResults = async (testPlanRun, context) => {
     return getFinalizedTestResults({ testPlanReport, context });
 };
 
-const updateOrCreateTestResultWithResponses = async ({
+const getTestByRowIdentifer = async ({
+    testPlanRun,
     testRowIdentifier,
+    context
+}) => {
+    const tests = await runnableTestsResolver(
+        testPlanRun.testPlanReport,
+        null,
+        context
+    );
+    return tests.find(
+        test => parseInt(test.rowNumber, 10) === testRowIdentifier
+    );
+};
+
+const updateOrCreateTestResultWithResponses = async ({
+    testId,
     testPlanRun,
     responses,
     atVersionId,
     browserVersionId,
     context
 }) => {
-    const allTestsForTestPlanVersion = await getTests(
-        testPlanRun.testPlanReport.testPlanVersion
-    );
-
-    const isV2 =
-        testPlanRun.testPlanReport.testPlanVersion.metadata
-            .testFormatVersion === 2;
-
-    const testId = allTestsForTestPlanVersion.find(
-        test =>
-            (!isV2 || test.at?.name === 'NVDA') &&
-            parseInt(test.rowNumber, 10) === testRowIdentifier
-    )?.id;
-
-    if (testId === undefined) {
-        throwNoTestFoundError(testRowIdentifier);
-    }
-
     const { testResult } = await findOrCreateTestResult({
         testId,
         testPlanRunId: testPlanRun.id,
@@ -243,9 +258,12 @@ const updateJobResults = async (req, res) => {
     const context = getGraphQLContext({ req });
     const { transaction } = context;
     const {
+        // Old way - testCsvRow and presentationNumber can be removed
+        // once all requests from automation are using the new URL parameter
         testCsvRow,
         presentationNumber,
         responses,
+        status,
         capabilities: {
             atName,
             atVersion: atVersionName,
@@ -263,35 +281,69 @@ const updateJobResults = async (req, res) => {
             `Job with id ${id} is not running, cannot update results`
         );
     }
-
-    /* TODO: Change this to use a better key based lookup system after gh-958 */
-    const [at] = await getAts({ search: atName, transaction });
-    const [browser] = await getBrowsers({ search: browserName, transaction });
-
-    const [atVersion, browserVersion] = await Promise.all([
-        findOrCreateAtVersion({
-            where: { atId: at.id, name: atVersionName },
-            transaction
-        }),
-        findOrCreateBrowserVersion({
-            where: { browserId: browser.id, name: browserVersionName },
-            transaction
+    if (status && !Object.values(COLLECTION_JOB_STATUS).includes(status)) {
+        throw new HttpQueryError(400, `Invalid status: ${status}`, true);
+    }
+    const { testPlanRun } = job;
+
+    // New way: testRowNumber is now a URL request param
+    // Old way: v1 tests store testCsvRow in rowNumber, v2 tests store presentationNumber in rowNumber
+    const testRowIdentifier =
+        req.params.testRowNumber ?? presentationNumber ?? testCsvRow;
+    const testId = (
+        await getTestByRowIdentifer({
+            testPlanRun,
+            testRowIdentifier,
+            context
         })
-    ]);
+    )?.id;
+
+    if (testId === undefined) {
+        throwNoTestFoundError(testRowIdentifier);
+    }
 
-    const processedResponses = convertEmptyStringsToNoOutputMessages(responses);
+    // status only update, or responses were provided (default to complete)
+    if (status || responses) {
+        await updateCollectionJobTestStatusByQuery({
+            where: { collectionJobId: id, testId },
+            // default to completed if not specified (when results are present)
+            values: { status: status ?? COLLECTION_JOB_STATUS.COMPLETED },
+            transaction: req.transaction
+        });
+    }
 
-    // v1 tests store testCsvRow in rowNumber, v2 tests store presentationNumber in rowNumber
-    const testRowIdentifier = presentationNumber ?? testCsvRow;
+    // responses were provided
+    if (responses) {
+        /* TODO: Change this to use a better key based lookup system after gh-958 */
+        const [at] = await getAts({ search: atName, transaction });
+        const [browser] = await getBrowsers({
+            search: browserName,
+            transaction
+        });
 
-    await updateOrCreateTestResultWithResponses({
-        testRowIdentifier,
-        responses: processedResponses,
-        testPlanRun: job.testPlanRun,
-        atVersionId: atVersion.id,
-        browserVersionId: browserVersion.id,
-        context
-    });
+        const [atVersion, browserVersion] = await Promise.all([
+            findOrCreateAtVersion({
+                where: { atId: at.id, name: atVersionName },
+                transaction
+            }),
+            findOrCreateBrowserVersion({
+                where: { browserId: browser.id, name: browserVersionName },
+                transaction
+            })
+        ]);
+
+        const processedResponses =
+            convertEmptyStringsToNoOutputMessages(responses);
+
+        await updateOrCreateTestResultWithResponses({
+            testId,
+            responses: processedResponses,
+            testPlanRun,
+            atVersionId: atVersion.id,
+            browserVersionId: browserVersion.id,
+            context
+        });
+    }
 
     res.json({ success: true });
 };

server/graphql-schema.js

@@ -102,6 +102,25 @@ const graphqlSchema = gql`
         The URL where the logs for the job can be found.
         """
         externalLogsUrl: String
+        """
+        An array of individual test status for every runnable test in the Job.
+        """
+        testStatus: [CollectionJobTestStatus]
+    }
+
+    """
+    A status for a specific Test on a specific CollectionJob.
+    """
+    type CollectionJobTestStatus {
+        """
+        The test this status reflects.
+        """
+        test: Test!
+        """
+        The status of the test, which can be "QUEUED", "RUNNING", "COMPLETED",
+        "ERROR", or "CANCELLED"
+        """
+        status: CollectionJobStatus!
     }
 
     type Browser {

server/migrations/20240404171101-addCollectionJobTestStatus.js

@@ -0,0 +1,58 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+    async up(queryInterface, Sequelize) {
+        return queryInterface.sequelize.transaction(async transaction => {
+            await queryInterface.createTable(
+                'CollectionJobTestStatus',
+                {
+                    id: {
+                        type: Sequelize.INTEGER,
+                        primaryKey: true,
+                        autoIncrement: true
+                    },
+                    testId: {
+                        type: Sequelize.STRING,
+                        allowNull: false
+                    },
+                    collectionJobId: {
+                        type: Sequelize.INTEGER,
+                        allowNull: false,
+                        onDelete: 'CASCADE',
+                        onUpdate: 'CASCADE',
+                        references: {
+                            model: 'CollectionJob',
+                            key: 'id'
+                        }
+                    },
+                    status: {
+                        type: Sequelize.STRING,
+                        allowNull: false,
+                        defaultValue: 'QUEUED'
+                    }
+                },
+                { transaction }
+            );
+            await queryInterface.addConstraint('CollectionJobTestStatus', {
+                type: 'unique',
+                name: 'CollectionJob_Test_unique',
+                fields: ['collectionJobId', 'testId'],
+                transaction
+            });
+        });
+    },
+
+    async down(queryInterface) {
+        return queryInterface.sequelize.transaction(async transaction => {
+            await queryInterface.removeConstraint(
+                'CollectionJobTestStatus',
+                'CollectionJob_Test_unique',
+                { transaction }
+            );
+            await queryInterface.dropTable('CollectionJobTestStatus', {
+                transaction
+            });
+        });
+    }
+};

server/models/CollectionJob.js

@@ -45,6 +45,12 @@ module.exports = function (sequelize, DataTypes) {
             sourceKey: 'testPlanRunId',
             as: 'testPlanRun'
         });
+
+        Model.hasMany(models.CollectionJobTestStatus, {
+            as: 'testStatus',
+            foreignKey: 'collectionJobId',
+            sourceKey: 'id'
+        });
     };
 
     Model.QUEUED = COLLECTION_JOB_STATUS.QUEUED;

server/models/CollectionJobTestStatus.js

@@ -0,0 +1,56 @@
+const { COLLECTION_JOB_STATUS } = require('../util/enums');
+
+const MODEL_NAME = 'CollectionJobTestStatus';
+
+module.exports = function (sequelize, DataTypes) {
+    const Model = sequelize.define(
+        MODEL_NAME,
+        {
+            id: {
+                type: DataTypes.INTEGER,
+                allowNull: false,
+                primaryKey: true,
+                autoIncrement: true
+            },
+            collectionJobId: {
+                type: DataTypes.INTEGER,
+                allowNull: false,
+                references: {
+                    model: 'CollectionJob',
+                    key: 'id'
+                }
+            },
+            testId: {
+                type: DataTypes.STRING,
+                allowNull: null
+            },
+            status: {
+                type: DataTypes.STRING,
+                allowNull: false,
+                defaultValue: COLLECTION_JOB_STATUS.QUEUED
+            }
+        },
+        {
+            timestamps: false,
+            tableName: MODEL_NAME
+        }
+    );
+
+    Model.associate = function (models) {
+        Model.belongsTo(models.CollectionJob, {
+            foreignKey: 'collectionJobId',
+            targetKey: 'id',
+            as: 'collectionJob',
+            onDelete: 'CASCADE',
+            onUpdate: 'CASCADE'
+        });
+    };
+
+    Model.QUEUED = COLLECTION_JOB_STATUS.QUEUED;
+    Model.RUNNING = COLLECTION_JOB_STATUS.RUNNING;
+    Model.COMPLETED = COLLECTION_JOB_STATUS.COMPLETED;
+    Model.CANCELLED = COLLECTION_JOB_STATUS.CANCELLED;
+    Model.ERROR = COLLECTION_JOB_STATUS.ERROR;
+
+    return Model;
+};