feat(orchestrator): add backend endpoint for getting workflows from d…

…ata index service (janus-idp#39) * feat: add backend for getting work flows from data index service * feat: cherrypick - add backend for getting work flows from data index service * feat: cherrypick - add backend for getting work flows from data index service * feat: cherrypick - add backend for getting work flows from data index service
jkilzi · Dec 7, 2023 · cd1fdb3 · cd1fdb3
1 parent b4c0426
commit cd1fdb3
Show file tree

Hide file tree

Showing 12 changed files with 246 additions and 4 deletions.
diff --git a/plugins/orchestrator-backend/config.d.ts b/plugins/orchestrator-backend/config.d.ts
@@ -59,6 +59,13 @@ export interface Config {
         path?: string;
       };
     };
+    dataIndexService: {
+      /**
+       * URL of the Data Index service.
+       * Example: http://localhost:8080/graphql
+       */
+      url?: string;
+    };
     /**
      * Configuration for the integration with the Catalog plugin.
      */

diff --git a/plugins/orchestrator-backend/dist-dynamic/package.json b/plugins/orchestrator-backend/dist-dynamic/package.json
@@ -53,6 +53,7 @@
   "dependencies": {
     "@octokit/rest": "^19.0.3",
     "@severlessworkflow/sdk-typescript": "^3.0.3",
+    "@urql/core": "^4.1.4",
     "cloudevents": "^8.0.0",
     "express": "^4.18.2",
     "express-promise-router": "^4.1.1",

diff --git a/plugins/orchestrator-backend/dist-dynamic/yarn.lock b/plugins/orchestrator-backend/dist-dynamic/yarn.lock
@@ -2,10 +2,15 @@
 # yarn lockfile v1
 
 
+"@0no-co/graphql.web@^1.0.1":
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/@0no-co/graphql.web/-/graphql.web-1.0.4.tgz#9606eb651955499525d068ce0ad8bea596286ce2"
+  integrity sha512-W3ezhHGfO0MS1PtGloaTpg0PbaT8aZSmmaerL7idtU5F7oCI+uu25k+MsMS31BVFlp4aMkHSrNRxiD72IlK8TA==
+
 "@aws-sdk/util-utf8-browser@npm:@smithy/util-utf8@^2.0.0":
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/@smithy/util-utf8/-/util-utf8-2.0.0.tgz#b4da87566ea7757435e153799df9da717262ad42"
-  integrity sha512-rctU1VkziY84n5OXe3bPNpKR001ZCME2JCaBBFgtiM2hfKbHFudc/BkMuPab8hRbLd0j3vbnBTTZ1igBf0wgiQ==
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/@smithy/util-utf8/-/util-utf8-2.0.2.tgz#626b3e173ad137208e27ed329d6bea70f4a1a7f7"
+  integrity sha512-qOiVORSPm6Ce4/Yu6hbSgNHABLP2VMv8QOC3tTDNHHlWY19pPyc++fBTbZPtx6egPXi4HQxKDnMxVxpbtX2GoA==
   dependencies:
     "@smithy/util-buffer-from" "^2.0.0"
     tslib "^2.5.0"
@@ -163,6 +168,14 @@
   resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c"
   integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==
 
+"@urql/core@^4.1.4":
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/@urql/core/-/core-4.2.0.tgz#7715491bc07e4af8b5d5039a19ea562cd109ae2f"
+  integrity sha512-GRkZ4kECR9UohWAjiSk2UYUetco6/PqSrvyC4AH6g16tyqEShA63M232cfbE1J9XJPaGNjia14Gi+oOqzp144w==
+  dependencies:
+    "@0no-co/graphql.web" "^1.0.1"
+    wonka "^6.3.2"
+
 accepts@~1.3.8:
   version "1.3.8"
   resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e"
@@ -1046,6 +1059,11 @@ winston@^3.11.0:
     triple-beam "^1.3.0"
     winston-transport "^4.5.0"
 
+wonka@^6.3.2:
+  version "6.3.4"
+  resolved "https://registry.yarnpkg.com/wonka/-/wonka-6.3.4.tgz#76eb9316e3d67d7febf4945202b5bdb2db534594"
+  integrity sha512-CjpbqNtBGNAeyNS/9W6q3kSkKE52+FjIj7AkFlLr11s/VWGUu6a2CdYSdGxocIhIVjaW/zchesBQUKPVU69Cqg==
+
 wrappy@1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"

diff --git a/plugins/orchestrator-backend/package.json b/plugins/orchestrator-backend/package.json
@@ -77,6 +77,7 @@
     "@janus-idp/backstage-plugin-orchestrator-common": "0.0.1",
     "@octokit/rest": "^19.0.3",
     "@severlessworkflow/sdk-typescript": "^3.0.3",
+    "@urql/core": "^4.1.4",
     "cloudevents": "^8.0.0",
     "express": "^4.18.2",
     "express-promise-router": "^4.1.1",

diff --git a/plugins/orchestrator-backend/src/helpers/errorBuilder.ts b/plugins/orchestrator-backend/src/helpers/errorBuilder.ts
@@ -0,0 +1,41 @@
+export const NO_DATA_INDEX_URL = 'NO_DATA_INDEX_URL';
+export const NO_BACKEND_EXEC_CTX = 'NO_BACKEND_EXEC_CTX';
+export const NO_CLIENT_PROVIDED = 'NO_CLIENT_PROVIDED';
+export const NO_LOGGER = 'NO_LOGGER';
+export const SWF_BACKEND_NOT_INITED = 'SWF_BACKEND_NOT_INITED';
+
+export class ErrorBuilder {
+  public static NewBackendError(name: string, message: string): Error {
+    const e = new Error(message);
+    e.name = name;
+    return e;
+  }
+
+  public static GET_NO_DATA_INDEX_URL_ERR(): Error {
+    return this.NewBackendError(
+      NO_DATA_INDEX_URL,
+      'No data index url specified or found',
+    );
+  }
+
+  public static GET_NO_BACKEND_EXEC_CTX_ERR(): Error {
+    return this.NewBackendError(
+      NO_BACKEND_EXEC_CTX,
+      'No or null backend execution context provided',
+    );
+  }
+
+  public static GET_NO_CLIENT_PROVIDED_ERR(): Error {
+    return this.NewBackendError(
+      NO_CLIENT_PROVIDED,
+      'No or null graphql client',
+    );
+  }
+
+  public static GET_SWF_BACKEND_NOT_INITED(): Error {
+    return this.NewBackendError(
+      SWF_BACKEND_NOT_INITED,
+      'The SonataFlow backend is not initialized, call initialize() method before trying to get the workflows.',
+    );
+  }
+}
diff --git a/plugins/orchestrator-backend/src/service/DataIndexService.ts b/plugins/orchestrator-backend/src/service/DataIndexService.ts
@@ -0,0 +1,82 @@
+import { cacheExchange, Client, fetchExchange } from '@urql/core';
+
+import { WorkflowDefinition } from '@janus-idp/backstage-plugin-orchestrator-common';
+
+import { ErrorBuilder } from '../helpers/errorBuilder';
+import { BackendExecCtx } from '../types/backendExecCtx';
+import { DEFAULT_DATA_INDEX_URL } from '../types/constants';
+
+export class DataIndexService {
+  public static backendExecCtx: BackendExecCtx;
+  private static inited = false;
+
+  public static initialize(backendExecCtx: BackendExecCtx) {
+    if (!backendExecCtx) {
+      throw ErrorBuilder.GET_NO_BACKEND_EXEC_CTX_ERR();
+    }
+
+    if (
+      !backendExecCtx.dataIndexUrl ||
+      backendExecCtx.dataIndexUrl.length === 0
+    ) {
+      throw ErrorBuilder.GET_NO_DATA_INDEX_URL_ERR();
+    }
+
+    if (!backendExecCtx.client) {
+      throw ErrorBuilder.GET_NO_CLIENT_PROVIDED_ERR();
+    }
+
+    if (!this.inited) {
+      this.backendExecCtx = backendExecCtx;
+      this.inited = true;
+      this.backendExecCtx.logger.info('Initialized the swf backend');
+    }
+  }
+
+  public static getNewGraphQLClient(
+    dataIndexUrl = DEFAULT_DATA_INDEX_URL,
+  ): Client {
+    const diURL =
+      this.backendExecCtx && this.backendExecCtx.dataIndexUrl
+        ? this.backendExecCtx.dataIndexUrl
+        : dataIndexUrl;
+    return new Client({
+      url: diURL,
+      exchanges: [cacheExchange, fetchExchange],
+    });
+  }
+
+  private static getDeployedSwfsQuery() {
+    return `
+        query ProcessDefinitions {
+            ProcessDefinitions {
+                id
+                name
+                version
+                type
+                endpoint
+                serviceUrl
+            }
+        }
+      `;
+  }
+
+  public static async getWorkflowDefinitions(): Promise<WorkflowDefinition[]> {
+    if (!this.inited) {
+      throw ErrorBuilder.GET_SWF_BACKEND_NOT_INITED();
+    }
+    const QUERY = this.getDeployedSwfsQuery();
+    this.backendExecCtx.logger.info(
+      `getWorkflowDefinitions() called:  ${this.backendExecCtx.dataIndexUrl}`,
+    );
+    const result = await this.backendExecCtx.client.query(QUERY, {});
+
+    if (result.error) {
+      this.backendExecCtx.logger.error(
+        `Error fetching data index swf results ${result.error}`,
+      );
+      throw result.error;
+    }
+    return result.data.ProcessDefinitions;
+  }
+}
diff --git a/plugins/orchestrator-backend/src/service/router.ts b/plugins/orchestrator-backend/src/service/router.ts
@@ -1,10 +1,12 @@
 import { errorHandler } from '@backstage/backend-common';
+import { Config } from '@backstage/config';
 import { DiscoveryApi } from '@backstage/core-plugin-api';
 import { ScmIntegrations } from '@backstage/integration';
 import { JsonObject, JsonValue } from '@backstage/types';
 
 import express from 'express';
 import Router from 'express-promise-router';
+import { Logger } from 'winston';
 
 import {
   fromWorkflowSource,
@@ -16,7 +18,11 @@ import {
 } from '@janus-idp/backstage-plugin-orchestrator-common';
 
 import { RouterArgs } from '../routerWrapper';
+import { ApiResponseBuilder } from '../types/apiResponse';
+import { BackendExecCtx } from '../types/backendExecCtx';
+import { DEFAULT_DATA_INDEX_URL } from '../types/constants';
 import { CloudEventService } from './CloudEventService';
+import { DataIndexService } from './DataIndexService';
 import { DataInputSchemaService } from './DataInputSchemaService';
 import { JiraEvent, JiraService } from './JiraService';
 import { OpenApiService } from './OpenApiService';
@@ -77,6 +83,8 @@ export async function createBackendRouter(
     urlReader,
   );
 
+  initDataIndexService(logger, config);
+
   setupInternalRoutes(
     router,
     args.sonataFlowService,
@@ -98,6 +106,16 @@ export async function createBackendRouter(
   return router;
 }
 
+function initDataIndexService(logger: Logger, config: Config) {
+  const dataIndexUrl =
+    config.getOptionalString('orchestrator.dataIndexService.url') ||
+    DEFAULT_DATA_INDEX_URL;
+  const client = DataIndexService.getNewGraphQLClient(dataIndexUrl);
+  const backendExecCtx = new BackendExecCtx(logger, client, dataIndexUrl);
+
+  DataIndexService.initialize(backendExecCtx);
+}
+
 // ======================================================
 // Internal Backstage API calls to delegate to SonataFlow
 // ======================================================
@@ -109,6 +127,11 @@ function setupInternalRoutes(
   dataInputSchemaService: DataInputSchemaService,
   jiraService: JiraService,
 ) {
+  router.get('/workflows/definitions', async (_, response) => {
+    const swfs = await DataIndexService.getWorkflowDefinitions();
+    response.json(ApiResponseBuilder.SUCCESS_RESPONSE(swfs));
+  });
+
   router.get('/workflows', async (_, res) => {
     const definitions = await sonataFlowService.fetchWorkflows();
 

diff --git a/plugins/orchestrator-backend/src/types/apiResponse.ts b/plugins/orchestrator-backend/src/types/apiResponse.ts
@@ -0,0 +1,31 @@
+export interface ApiResponse {
+  message?: String;
+  result?: any;
+  backEndErrCd?: String;
+}
+
+export class ApiResponseBuilder {
+  static SUCCESS_RESPONSE(result: any, message = 'success'): ApiResponse {
+    return {
+      result: result,
+      message: message,
+    };
+  }
+
+  static VALIDATION_ERR_RESPONSE(
+    backEndErrCd = 'backend validation error code',
+    message = 'backend validation error',
+  ): ApiResponse {
+    return {
+      message: message,
+      backEndErrCd: backEndErrCd,
+    };
+  }
+
+  static HTTP_ERR_RESPONSE(message = 'Internal Server Error'): ApiResponse {
+    return {
+      result: null,
+      message: message,
+    };
+  }
+}
diff --git a/plugins/orchestrator-backend/src/types/backendExecCtx.ts b/plugins/orchestrator-backend/src/types/backendExecCtx.ts
@@ -0,0 +1,16 @@
+import { Client } from '@urql/core';
+import { Logger } from 'winston';
+
+import { DEFAULT_DATA_INDEX_URL } from './constants';
+
+export class BackendExecCtx {
+  constructor(
+    public logger: Logger,
+    public client: Client,
+    public dataIndexUrl = DEFAULT_DATA_INDEX_URL,
+  ) {
+    this.dataIndexUrl = dataIndexUrl;
+    this.logger = logger;
+    this.client = client;
+  }
+}
diff --git a/plugins/orchestrator-backend/src/types/constants.ts b/plugins/orchestrator-backend/src/types/constants.ts
@@ -0,0 +1 @@
+export const DEFAULT_DATA_INDEX_URL = 'http://localhost:8080/graphql';
diff --git a/plugins/orchestrator/README.md b/plugins/orchestrator/README.md
@@ -70,9 +70,12 @@ orchestrator:
       gitRepositoryUrl: https://github.com/tiagodolphine/backstage-orchestrator-workflows
       localPath: /tmp/orchestrator/repository
       autoPush: true
+  dataIndexService:
+    url: ${DATA_INDEX_URL}
 ```
 
-when interacting with an existing Sonataflow backend service from `baseUrl` and `port`, `autoStart` needs to be unset or set to `false`, also the section of `workflowSource` can be neglect.
+- when interacting with an existing Sonataflow backend service from `baseUrl` and `port`, `autoStart` needs to be unset or set to `false`, also the section of `workflowSource` can be neglected.
+- set the environment variable `DATA_INDEX_URL`, which points to a running data index service accessible via data index graphql interface such as http://<server:port>/graphql
 
 For more information about the configuration options, including other optional properties, see the [config.d.ts](../orchestrator-common/config.d.ts) file.
 

diff --git a/yarn.lock b/yarn.lock
@@ -2,6 +2,11 @@
 # yarn lockfile v1
 
 
+"@0no-co/graphql.web@^1.0.1":
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/@0no-co/graphql.web/-/graphql.web-1.0.4.tgz#9606eb651955499525d068ce0ad8bea596286ce2"
+  integrity sha512-W3ezhHGfO0MS1PtGloaTpg0PbaT8aZSmmaerL7idtU5F7oCI+uu25k+MsMS31BVFlp4aMkHSrNRxiD72IlK8TA==
+
 "@aashutoshrathi/word-wrap@^1.2.3":
   version "1.2.6"
   resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf"
@@ -13409,6 +13414,14 @@
     "@uiw/codemirror-extensions-basic-setup" "4.21.20"
     codemirror "^6.0.0"
 
+"@urql/core@^4.1.4":
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/@urql/core/-/core-4.2.0.tgz#7715491bc07e4af8b5d5039a19ea562cd109ae2f"
+  integrity sha512-GRkZ4kECR9UohWAjiSk2UYUetco6/PqSrvyC4AH6g16tyqEShA63M232cfbE1J9XJPaGNjia14Gi+oOqzp144w==
+  dependencies:
+    "@0no-co/graphql.web" "^1.0.1"
+    wonka "^6.3.2"
+
 "@webassemblyjs/[email protected]", "@webassemblyjs/ast@^1.11.5":
   version "1.11.6"
   resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.6.tgz#db046555d3c413f8966ca50a95176a0e2c642e24"
@@ -30898,6 +30911,11 @@ winston@^3.11.0, winston@^3.2.1:
     triple-beam "^1.3.0"
     winston-transport "^4.5.0"
 
+wonka@^6.3.2:
+  version "6.3.4"
+  resolved "https://registry.yarnpkg.com/wonka/-/wonka-6.3.4.tgz#76eb9316e3d67d7febf4945202b5bdb2db534594"
+  integrity sha512-CjpbqNtBGNAeyNS/9W6q3kSkKE52+FjIj7AkFlLr11s/VWGUu6a2CdYSdGxocIhIVjaW/zchesBQUKPVU69Cqg==
+
 word-wrap@^1.2.3, word-wrap@~1.2.3:
   version "1.2.5"
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		export const DEFAULT_DATA_INDEX_URL = 'http://localhost:8080/graphql';