feat(orchestrator): support pagination for /instances and /overview (#…

…1313) * feat: support pagination for /instances and /overview * refactor: isolate pagination logics from /v1 endpoints
janus-idp · Mar 7, 2024 · 79d5988 · 79d5988
1 parent df868aa
commit 79d5988
Show file tree

Hide file tree

Showing 15 changed files with 459 additions and 67 deletions.
diff --git a/plugins/orchestrator-backend/src/helpers/queryBuilder.ts b/plugins/orchestrator-backend/src/helpers/queryBuilder.ts
@@ -0,0 +1,34 @@
+import { Pagination } from '../types/pagination';
+
+export function buildGraphQlQuery(args: {
+  type: 'ProcessDefinitions' | 'ProcessInstances' | 'Jobs';
+  queryBody: string;
+  whereClause?: string;
+  pagination?: Pagination;
+}): string {
+  let query = `{${args.type}`;
+
+  if (args.whereClause || args.pagination) {
+    query += ` (`;
+
+    if (args.whereClause) {
+      query += `where: {${args.whereClause}}`;
+      if (args.pagination) {
+        query += `, `;
+      }
+    }
+    if (args.pagination) {
+      if (args.pagination.sortField) {
+        query += `orderBy: {${
+          args.pagination.sortField
+        }: ${args.pagination.order?.toUpperCase()}}, `;
+      }
+      query += `pagination: {limit: ${args.pagination.limit} , offset: ${args.pagination.offset}}`;
+    }
+
+    query += `) `;
+  }
+  query += ` {${args.queryBody} } }`;
+
+  return query;
+}
diff --git a/plugins/orchestrator-backend/src/pagination.test.ts b/plugins/orchestrator-backend/src/pagination.test.ts
@@ -0,0 +1,58 @@
+import { buildPagination } from './types/pagination';
+
+describe('buildPagination()', () => {
+  it('should build the correct pagination obj when no query parameters are passed', () => {
+    const mockRequest: any = {
+      query: {},
+    };
+    expect(buildPagination(mockRequest)).toEqual({
+      limit: 10,
+      offset: 0,
+      order: 'ASC',
+      sortField: undefined,
+    });
+  });
+  it('should build the correct pagination obj when partial query parameters are passed', () => {
+    const mockRequest: any = {
+      query: {
+        orderBy: 'lastUpdated',
+      },
+    };
+    expect(buildPagination(mockRequest)).toEqual({
+      limit: 10,
+      offset: 0,
+      order: 'ASC',
+      sortField: 'lastUpdated',
+    });
+  });
+  it('should build the correct pagination obj when all query parameters are passed', () => {
+    const mockRequest: any = {
+      query: {
+        page: 1,
+        pageSize: 50,
+        orderBy: 'lastUpdated',
+        orderDirection: 'DESC',
+      },
+    };
+    expect(buildPagination(mockRequest)).toEqual({
+      limit: 50,
+      offset: 1,
+      order: 'DESC',
+      sortField: 'lastUpdated',
+    });
+  });
+  it('should build the correct pagination obj when non numeric value passed to number fields', () => {
+    const mockRequest: any = {
+      query: {
+        page: 'abc',
+        pageSize: 'cde',
+      },
+    };
+    expect(buildPagination(mockRequest)).toEqual({
+      limit: 10,
+      offset: 0,
+      order: 'ASC',
+      sortField: undefined,
+    });
+  });
+});
diff --git a/plugins/orchestrator-backend/src/queryBuilder.test.ts b/plugins/orchestrator-backend/src/queryBuilder.test.ts
@@ -0,0 +1,49 @@
+import { buildGraphQlQuery } from './helpers/queryBuilder';
+import { Pagination } from './types/pagination';
+
+describe('GraphQL query builder', () => {
+  it('should return properly formatted graphQL query when where clause and pagination are present', () => {
+    const expectedQuery: string =
+      '{ProcessInstances (where: {processId: {isNull: false}}, orderBy: {lastUpdate: DESC}, pagination: {limit: 5 , offset: 2})  {id, processName, processId, state, start, lastUpdate, end, nodes { id }, variables, parentProcessInstance {id, processName, businessKey} } }';
+    const pagination: Pagination = {
+      offset: 2,
+      limit: 5,
+      order: 'DESC',
+      sortField: 'lastUpdate',
+    };
+    expect(
+      buildGraphQlQuery({
+        type: 'ProcessInstances',
+        queryBody:
+          'id, processName, processId, state, start, lastUpdate, end, nodes { id }, variables, parentProcessInstance {id, processName, businessKey}',
+        whereClause: 'processId: {isNull: false}',
+        pagination,
+      }),
+    ).toEqual(expectedQuery);
+  });
+
+  it('should return properly formatted graphQL query when where clause is present', () => {
+    const expectedQuery: string =
+      '{ProcessInstances (where: {processId: {isNull: false}})  {id, processName, processId, state, start, lastUpdate, end, nodes { id }, variables, parentProcessInstance {id, processName, businessKey} } }';
+    expect(
+      buildGraphQlQuery({
+        type: 'ProcessInstances',
+        queryBody:
+          'id, processName, processId, state, start, lastUpdate, end, nodes { id }, variables, parentProcessInstance {id, processName, businessKey}',
+        whereClause: 'processId: {isNull: false}',
+      }),
+    ).toEqual(expectedQuery);
+  });
+
+  it('should return properly formatted graphQL query when where clause is NOT present', () => {
+    const expectedQuery: string =
+      '{ProcessInstances {id, processName, processId, state, start, lastUpdate, end, nodes { id }, variables, parentProcessInstance {id, processName, businessKey} } }';
+    expect(
+      buildGraphQlQuery({
+        type: 'ProcessInstances',
+        queryBody:
+          'id, processName, processId, state, start, lastUpdate, end, nodes { id }, variables, parentProcessInstance {id, processName, businessKey}',
+      }),
+    ).toEqual(expectedQuery);
+  });
+});
diff --git a/plugins/orchestrator-backend/src/service/DataIndexService.ts b/plugins/orchestrator-backend/src/service/DataIndexService.ts
@@ -12,6 +12,9 @@ import {
 } from '@janus-idp/backstage-plugin-orchestrator-common';
 
 import { ErrorBuilder } from '../helpers/errorBuilder';
+import { buildGraphQlQuery } from '../helpers/queryBuilder';
+import { Pagination } from '../types/pagination';
+import { FETCH_PROCESS_INSTANCES_SORT_FIELD } from './constants';
 
 export class DataIndexService {
   private client: Client;
@@ -89,23 +92,18 @@ export class DataIndexService {
     return processDefinitions[0];
   }
 
-  public async getWorkflowInfos(): Promise<WorkflowInfo[]> {
-    const QUERY = `
-        query ProcessDefinitions {
-            ProcessDefinitions {
-                id
-                name
-                version
-                type
-                endpoint
-                serviceUrl
-                source
-            }
-        }
-      `;
-
+  public async getWorkflowInfos(
+    pagination?: Pagination,
+  ): Promise<WorkflowInfo[]> {
     this.logger.info(`getWorkflowInfos() called: ${this.dataIndexUrl}`);
-    const result = await this.client.query(QUERY, {});
+
+    const graphQlQuery = buildGraphQlQuery({
+      type: 'ProcessDefinitions',
+      queryBody: 'id, name, version, type, endpoint, serviceUrl, source',
+      pagination,
+    });
+    this.logger.debug(`GraphQL query: ${graphQlQuery}`);
+    const result = await this.client.query(graphQlQuery, {});
 
     this.logger.debug(
       `Get workflow definitions result: ${JSON.stringify(result)}`,
@@ -121,10 +119,19 @@ export class DataIndexService {
     return result.data.ProcessDefinitions;
   }
 
-  public async fetchProcessInstances(): Promise<ProcessInstance[] | undefined> {
-    const graphQlQuery =
-      '{ ProcessInstances ( orderBy: { start: ASC }, where: {processId: {isNull: false} } ) { id, processName, processId, businessKey, state, start, end, nodes { id }, variables, parentProcessInstance {id, processName, businessKey} } }';
-
+  public async fetchProcessInstances(
+    pagination?: Pagination,
+  ): Promise<ProcessInstance[] | undefined> {
+    if (pagination) pagination.sortField ??= FETCH_PROCESS_INSTANCES_SORT_FIELD;
+
+    const graphQlQuery = buildGraphQlQuery({
+      type: 'ProcessInstances',
+      queryBody:
+        'id, processName, processId, businessKey, state, start, end, nodes { id }, variables, parentProcessInstance {id, processName, businessKey}',
+      whereClause: 'processId: {isNull: false}',
+      pagination,
+    });
+    this.logger.debug(`GraphQL query: ${graphQlQuery}`);
     const result = await this.client.query(graphQlQuery, {});
 
     this.logger.debug(
@@ -147,6 +154,24 @@ export class DataIndexService {
     return processInstances;
   }
 
+  public async getProcessInstancesTotalCount(): Promise<number> {
+    const graphQlQuery = buildGraphQlQuery({
+      type: 'ProcessInstances',
+      queryBody: 'id',
+    });
+    this.logger.debug(`GraphQL query: ${graphQlQuery}`);
+    const result = await this.client.query(graphQlQuery, {});
+
+    if (result.error) {
+      this.logger.error(`Error when fetching instances: ${result.error}`);
+      throw result.error;
+    }
+
+    const idArr = result.data.ProcessInstances as ProcessInstance[];
+
+    return Promise.resolve(idArr.length);
+  }
+
   private async getWorkflowDefinitionFromInstance(instance: ProcessInstance) {
     const workflowInfo = await this.getWorkflowDefinition(instance.processId);
     if (!workflowInfo?.source) {

diff --git a/plugins/orchestrator-backend/src/service/SonataFlowService.ts b/plugins/orchestrator-backend/src/service/SonataFlowService.ts
@@ -21,6 +21,7 @@ import {
 import { spawn } from 'child_process';
 import { join, resolve } from 'path';
 
+import { Pagination } from '../types/pagination';
 import { DataIndexService } from './DataIndexService';
 import { executeWithRetry } from './Helper';
 
@@ -143,11 +144,11 @@ export class SonataFlowService {
     return undefined;
   }
 
-  public async fetchWorkflowOverviews(): Promise<
-    WorkflowOverview[] | undefined
-  > {
+  public async fetchWorkflowOverviews(
+    pagination?: Pagination,
+  ): Promise<WorkflowOverview[] | undefined> {
     try {
-      const workflowInfos = await this.dataIndex.getWorkflowInfos();
+      const workflowInfos = await this.dataIndex.getWorkflowInfos(pagination);
       if (!workflowInfos?.length) {
         return [];
       }

diff --git a/plugins/orchestrator-backend/src/service/api/v2.test.ts b/plugins/orchestrator-backend/src/service/api/v2.test.ts
@@ -4,6 +4,7 @@ import {
   WorkflowOverviewListResultDTO,
 } from '@janus-idp/backstage-plugin-orchestrator-common';
 
+import { buildPagination } from '../../types/pagination';
 import { SonataFlowService } from '../SonataFlowService';
 import { mapToWorkflowOverviewDTO } from './mapping/V2Mappings';
 import {
@@ -48,6 +49,14 @@ describe('getWorkflowOverview', () => {
 
   it('0 items in workflow overview list', async () => {
     // Arrange
+    const mockRequest: any = {
+      query: {
+        page: 1,
+        pageSize: 50,
+        orderBy: 'lastUpdated',
+        orderDirection: 'DESC',
+      },
+    };
     const mockOverviewsV1 = {
       items: [],
     };
@@ -59,6 +68,7 @@ describe('getWorkflowOverview', () => {
     // Act
     const result: WorkflowOverviewListResultDTO = await V2.getWorkflowsOverview(
       mockSonataFlowService,
+      buildPagination(mockRequest),
     );
 
     // Assert
@@ -67,15 +77,18 @@ describe('getWorkflowOverview', () => {
         mapToWorkflowOverviewDTO(item),
       ),
       paginationInfo: {
-        limit: 0,
-        offset: 0,
+        page: 1,
+        pageSize: 50,
         totalCount: mockOverviewsV1.items.length,
       },
     });
   });
 
   it('1 item in workflow overview list', async () => {
     // Arrange
+    const mockRequest: any = {
+      query: {},
+    };
     const mockOverviewsV1 = generateTestWorkflowOverviewList(1, {});
 
     (
@@ -85,6 +98,7 @@ describe('getWorkflowOverview', () => {
     // Act
     const result: WorkflowOverviewListResultDTO = await V2.getWorkflowsOverview(
       mockSonataFlowService,
+      buildPagination(mockRequest),
     );
 
     // Assert
@@ -93,15 +107,23 @@ describe('getWorkflowOverview', () => {
         mapToWorkflowOverviewDTO(item),
       ),
       paginationInfo: {
-        limit: 0,
-        offset: 0,
+        page: 0,
+        pageSize: 10,
         totalCount: mockOverviewsV1.items.length,
       },
     });
   });
 
   it('many items in workflow overview list', async () => {
     // Arrange
+    const mockRequest: any = {
+      query: {
+        page: 1,
+        pageSize: 50,
+        orderBy: 'lastUpdated',
+        orderDirection: 'DESC',
+      },
+    };
     const mockOverviewsV1 = generateTestWorkflowOverviewList(100, {});
 
     (
@@ -111,6 +133,7 @@ describe('getWorkflowOverview', () => {
     // Act
     const result: WorkflowOverviewListResultDTO = await V2.getWorkflowsOverview(
       mockSonataFlowService,
+      buildPagination(mockRequest),
     );
 
     // Assert
@@ -119,21 +142,27 @@ describe('getWorkflowOverview', () => {
         mapToWorkflowOverviewDTO(item),
       ),
       paginationInfo: {
-        limit: 0,
-        offset: 0,
+        page: 1,
+        pageSize: 50,
         totalCount: mockOverviewsV1.items.length,
       },
     });
   });
 
   it('undefined workflow overview list', async () => {
     // Arrange
+    const mockRequest: any = {
+      query: {},
+    };
     (
       mockSonataFlowService.fetchWorkflowOverviews as jest.Mock
     ).mockRejectedValue(new Error('no workflow overview'));
 
     // Act
-    const promise = V2.getWorkflowsOverview(mockSonataFlowService);
+    const promise = V2.getWorkflowsOverview(
+      mockSonataFlowService,
+      buildPagination(mockRequest),
+    );
 
     // Assert
     await expect(promise).rejects.toThrow('no workflow overview');