From 19731d1449a9d8ffa67aec069d2214e45bfe54ff Mon Sep 17 00:00:00 2001 From: Frank Kong <50030060+Zaperex@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:51:47 -0500 Subject: [PATCH] feat: update entity provider schedulers (#827) * chore(3scale): allow app-config.yaml schedule to take precedence for entity provider * chore(ocm): allow app-config.yaml schedule for entity provider to take precedence * chore(aap): update entity provider to allow app-config scheduler to take precedence * chore(keycloak): update entity provider to allow app-config scheduler to take precedence * docs(keycloak): update keycloak docs for new schedule config priorities * docs(ocm): update OCM docs for new schedule config priorities * docs(aap): update aap docs for new schedule config priorities * docs(ocm): update OCM README * docs(keycloak): update keycloak README.md * docs(3scale): update 3scale docs for new schedule config priorities * docs(aap): update aap README.md * fix(aap): update AAP entity provider scheduler logic * fix(keycloak): update entity provider scheduler logic * fix(ocm): update entity provider scheduler logic * fix(3scale): update entity provider scheduler logic * docs: update entity provider scheduler docs * docs(3scale): update entity provider documentation * docs: update entity provider documentation * docs: update entity provider documentation * deps: update yarn.lock * docs(3scale): update 3scale README * docs(aap): update aap-backend README * docs(ocm): update ocm-backend README --- plugins/3scale-backend/README.md | 102 +++++++--- .../providers/ThreeScaleApiEntityProvider.ts | 17 +- plugins/aap-backend/README.md | 20 +- .../providers/AapResourceEntityProvider.ts | 6 +- plugins/keycloak-backend/README.md | 185 ++++++++++-------- .../KeycloakOrgEntityProvider.test.ts | 2 +- .../providers/KeycloakOrgEntityProvider.ts | 17 +- .../src/providers/ManagedClusterProvider.ts | 15 +- plugins/ocm/README.md | 142 ++++++++------ 9 files changed, 305 insertions(+), 201 deletions(-) diff --git a/plugins/3scale-backend/README.md b/plugins/3scale-backend/README.md index 4cdda8fd46..99402cf61c 100644 --- a/plugins/3scale-backend/README.md +++ b/plugins/3scale-backend/README.md @@ -16,7 +16,7 @@ yarn workspace backend add @janus-idp/backstage-plugin-3scale-backend 3scale Backstage provider allows configuration of one or multiple providers using the `app-config.yaml` configuration file of Backstage. -**Procedure** +### Procedure 1. Use a `threeScaleApiEntity` marker to start configuring the `app-config.yaml` file of Backstage: @@ -34,32 +34,80 @@ yarn workspace backend add @janus-idp/backstage-plugin-3scale-backend timeout: { minutes: 1 } ``` -2. Add the following code to `packages/backend/src/plugins/catalog.ts` file: - - ```ts title="packages/backend/src/plugins/catalog.ts" - /* highlight-add-next-line */ - import { ThreeScaleApiEntityProvider } from '@janus-idp/backstage-plugin-3scale-backend'; - - export default async function createPlugin( - env: PluginEnvironment, - ): Promise { - const builder = await CatalogBuilder.create(env); - - /* ... other processors and/or providers ... */ - /* highlight-add-start */ - builder.addEntityProvider( - ThreeScaleApiEntityProvider.fromConfig(env.config, { - logger: env.logger, - scheduler: env.scheduler, - }), - ); - /* highlight-add-end */ - - const { processingEngine, router } = await builder.build(); - await processingEngine.start(); - return router; - } - ``` +2. Configure the scheduler for the entity provider using one of the following methods: + + - **Method 1**: If the scheduler is configured inside the `app-config.yaml` using the schedule config key mentioned previously, add the following code to `packages/backend/src/plugins/catalog.ts` file: + + ```ts title="packages/backend/src/plugins/catalog.ts" + /* highlight-add-next-line */ + import { ThreeScaleApiEntityProvider } from '@janus-idp/backstage-plugin-3scale-backend'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + + /* ... other processors and/or providers ... */ + /* highlight-add-start */ + builder.addEntityProvider( + ThreeScaleApiEntityProvider.fromConfig(env.config, { + logger: env.logger, + scheduler: env.scheduler, + }), + ); + /* highlight-add-end */ + + const { processingEngine, router } = await builder.build(); + await processingEngine.start(); + return router; + } + ``` + + *** + + **NOTE** + + If you have made any changes to the schedule in the `app-config.yaml` file, then restart to apply the changes. + + *** + + - **Method 2**: Add a schedule directly inside the `packages/backend/src/plugins/catalog.ts` file as follows: + + ```ts title="packages/backend/src/plugins/catalog.ts" + /* highlight-add-next-line */ + import { ThreeScaleApiEntityProvider } from '@janus-idp/backstage-plugin-3scale-backend'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + + /* ... other processors and/or providers ... */ + /* highlight-add-start */ + builder.addEntityProvider( + ThreeScaleApiEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 1 }, + timeout: { minutes: 1 }, + }), + }), + ); + /* highlight-add-end */ + + const { processingEngine, router } = await builder.build(); + await processingEngine.start(); + return router; + } + ``` + + *** + + **NOTE** + + If both the `schedule` (hard-coded schedule) and `scheduler` (`app-config.yaml` schedule) option are provided in the `packages/backend/src/plugins/catalog.ts`, the `scheduler` option takes precedence. However, if the schedule inside the `app-config.yaml` file is not configured, then the `schedule` option is used. + + *** ### Troubleshooting diff --git a/plugins/3scale-backend/src/providers/ThreeScaleApiEntityProvider.ts b/plugins/3scale-backend/src/providers/ThreeScaleApiEntityProvider.ts index d186a2d9a2..a10345555f 100644 --- a/plugins/3scale-backend/src/providers/ThreeScaleApiEntityProvider.ts +++ b/plugins/3scale-backend/src/providers/ThreeScaleApiEntityProvider.ts @@ -60,19 +60,14 @@ export class ThreeScaleApiEntityProvider implements EntityProvider { let taskRunner; - if (options.schedule) { + if (options.scheduler && providerConfig.schedule) { + // Create a scheduled task runner using the provided scheduler and schedule configuration + taskRunner = options.scheduler.createScheduledTaskRunner( + providerConfig.schedule, + ); + } else if (options.schedule) { // Use the provided schedule directly taskRunner = options.schedule; - } else if (options.scheduler) { - if (providerConfig.schedule) { - // Create a scheduled task runner using the provided scheduler and schedule configuration - taskRunner = options.scheduler.createScheduledTaskRunner( - providerConfig.schedule, - ); - } else { - // Handle the case where providerConfig.schedule is missing - throw new Error('Provider configuration schedule is missing.'); - } } else { // Handle the case where both options.schedule and options.scheduler are missing throw new Error('Neither schedule nor scheduler is provided.'); diff --git a/plugins/aap-backend/README.md b/plugins/aap-backend/README.md index 0fe223febe..a02ba16013 100644 --- a/plugins/aap-backend/README.md +++ b/plugins/aap-backend/README.md @@ -36,7 +36,7 @@ The AAP Backstage provider plugin allows the configuration of one or multiple pr 1. Configure the scheduler using one of the following options: - - Add the following code to the `packages/backend/src/plugins/catalog.ts` file if the scheduler is configured inside the `app-config.yaml` file: + - **Method 1**: If the scheduler is configured inside the `app-config.yaml` using the schedule config key mentioned previously, add the following code to `packages/backend/src/plugins/catalog.ts` file: ```ts title="packages/backend/src/plugins/catalog.ts" /* highlight-add-next-line */ @@ -63,7 +63,15 @@ The AAP Backstage provider plugin allows the configuration of one or multiple pr } ``` - - Add a schedule directly inside the `packages/backend/src/plugins/catalog.ts` file as follows: + *** + + **NOTE** + + If you have made any changes to the schedule in the `app-config.yaml` file, then restart to apply the changes. + + *** + + - **Method 2**: Add a schedule directly inside the `packages/backend/src/plugins/catalog.ts` file as follows: ```ts title="packages/backend/src/plugins/catalog.ts" /* highlight-add-next-line */ @@ -93,6 +101,14 @@ The AAP Backstage provider plugin allows the configuration of one or multiple pr } ``` + *** + + **NOTE** + + If both the `schedule` (hard-coded schedule) and `scheduler` (`app-config.yaml` schedule) option are provided in the `packages/backend/src/plugins/catalog.ts`, the `scheduler` option takes precedence. However, if the schedule inside the `app-config.yaml` file is not configured, then the `schedule` option is used. + + *** + ### Troubleshooting When you start your Backstage application, you can see the following log lines: diff --git a/plugins/aap-backend/src/providers/AapResourceEntityProvider.ts b/plugins/aap-backend/src/providers/AapResourceEntityProvider.ts index ab781f059b..a17884bc60 100644 --- a/plugins/aap-backend/src/providers/AapResourceEntityProvider.ts +++ b/plugins/aap-backend/src/providers/AapResourceEntityProvider.ts @@ -43,12 +43,12 @@ export class AapResourceEntityProvider implements EntityProvider { return providerConfigs.map(providerConfig => { let taskRunner; - if (options.schedule) { - taskRunner = options.schedule; - } else if (options.scheduler && providerConfig.schedule) { + if (options.scheduler && providerConfig.schedule) { taskRunner = options.scheduler.createScheduledTaskRunner( providerConfig.schedule, ); + } else if (options.schedule) { + taskRunner = options.schedule; } else { throw new Error( `No schedule provided neither via code nor config for AapResourceEntityProvider:${providerConfig.id}.`, diff --git a/plugins/keycloak-backend/README.md b/plugins/keycloak-backend/README.md index 022d5d4ea9..2e0dc7cbc5 100644 --- a/plugins/keycloak-backend/README.md +++ b/plugins/keycloak-backend/README.md @@ -35,88 +35,107 @@ yarn workspace backend add @janus-idp/backstage-plugin-keycloak-backend clientSecret: ${KEYCLOAK_CLIENTSECRET} ``` -2. Register the plugin in the `packages/backend/src/plugins/catalog.ts` file. You can also configure a schedule in this step. However, there are possible ways of configuration, such as: - - - Configure a schedule inside the `app-config.yaml` file: - - ```yaml title="app-config.yaml" - catalog: - providers: - keycloakOrg: - default: - # ... - # highlight-add-start - schedule: # optional; same options as in TaskScheduleDefinition - # supports cron, ISO duration, "human duration" as used in code - frequency: { minutes: 1 } - # supports ISO duration, "human duration" as used in code - timeout: { minutes: 1 } - initialDelay: { seconds: 15 } - # highlight-add-end - ``` - - Use the configured scheduler inside the `packages/backend/src/plugins/catalog.ts` as follows: - - ```ts title="packages/backend/src/plugins/catalog.ts" - /* highlight-add-start */ - import { KeycloakOrgEntityProvider } from '@janus-idp/backstage-plugin-keycloak-backend'; - - /* highlight-add-end */ - - export default async function createPlugin( - env: PluginEnvironment, - ): Promise { - const builder = await CatalogBuilder.create(env); - - /* ... other processors and/or providers ... */ - /* highlight-add-start */ - builder.addEntityProvider( - KeycloakOrgEntityProvider.fromConfig(env.config, { - id: 'development', - logger: env.logger, - scheduler: env.scheduler, - }), - ); - /* highlight-add-end */ - - const { processingEngine, router } = await builder.build(); - await processingEngine.start(); - return router; - } - ``` - - - Add a schedule directly inside the `packages/backend/src/plugins/catalog.ts` file as follows: - - ```ts title="packages/backend/src/plugins/catalog.ts" - + import { KeycloakOrgEntityProvider } from '@janus-idp/backstage-plugin-keycloak-backend'; - - export default async function createPlugin( - env: PluginEnvironment, - ): Promise { - const builder = await CatalogBuilder.create(env); - - /* ... other processors and/or providers ... */ - builder.addEntityProvider( - KeycloakOrgEntityProvider.fromConfig(env.config, { - id: 'development', - logger: env.logger, - /* highlight-add-start */ - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 1 }, - timeout: { minutes: 1 }, - initialDelay: { seconds: 15 } - }), - /* highlight-add-end */ - }), - ) - - const { processingEngine, router } = await builder.build(); - await processingEngine.start(); - return router; - } - ``` - -3. Optional: override the default Keycloak query parameters. Configure the parameters inside the `app-config.yaml` file: +1. Register the plugin in the `packages/backend/src/plugins/catalog.ts` file. You can also configure a schedule in this step. However, there are possible ways of configuration, such as: + + 1. Configure a schedule inside the `app-config.yaml` file: + + ```yaml title="app-config.yaml" + catalog: + providers: + keycloakOrg: + default: + # ... + # highlight-add-start + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 1 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 1 } + initialDelay: { seconds: 15 } + # highlight-add-end + ``` + + Then use the configured scheduler by adding the following to the `packages/backend/src/plugins/catalog.ts`: + + ```ts title="packages/backend/src/plugins/catalog.ts" + /* highlight-add-start */ + import { KeycloakOrgEntityProvider } from '@janus-idp/backstage-plugin-keycloak-backend'; + + /* highlight-add-end */ + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + + /* ... other processors and/or providers ... */ + /* highlight-add-start */ + builder.addEntityProvider( + KeycloakOrgEntityProvider.fromConfig(env.config, { + id: 'development', + logger: env.logger, + scheduler: env.scheduler, + }), + ); + /* highlight-add-end */ + + const { processingEngine, router } = await builder.build(); + await processingEngine.start(); + return router; + } + ``` + + *** + + **NOTE** + + If you have made any changes to the schedule in the `app-config.yaml` file, then restart to apply the changes. + + *** + + 1. Add a schedule directly inside the `packages/backend/src/plugins/catalog.ts` file as follows: + + ```ts title="packages/backend/src/plugins/catalog.ts" + /* highlight-add-start */ + import { KeycloakOrgEntityProvider } from '@janus-idp/backstage-plugin-keycloak-backend'; + + /* highlight-add-end */ + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + + /* ... other processors and/or providers ... */ + builder.addEntityProvider( + KeycloakOrgEntityProvider.fromConfig(env.config, { + id: 'development', + logger: env.logger, + /* highlight-add-start */ + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 1 }, + timeout: { minutes: 1 }, + initialDelay: { seconds: 15 }, + }), + /* highlight-add-end */ + }), + ); + + const { processingEngine, router } = await builder.build(); + await processingEngine.start(); + return router; + } + ``` + + *** + + **NOTE** + + If both the `schedule` (hard-coded schedule) and `scheduler` (`app-config.yaml` schedule) option are provided in the `packages/backend/src/plugins/catalog.ts`, the `scheduler` option takes precedence. However, if the schedule inside the `app-config.yaml` file is not configured, then the `schedule` option is used. + + *** + +1. Optional: override the default Keycloak query parameters. Configure the parameters inside the `app-config.yaml` file: ```yaml title="app-config.yaml" catalog: @@ -130,7 +149,7 @@ yarn workspace backend add @janus-idp/backstage-plugin-keycloak-backend # highlight-add-end ``` -4. Optional: provide a transformer function for user/group to mutate the entity before their ingestion into catalog. Extend `packages/backend/src/plugins/catalog.ts` with custom `userTransformer` and `groupTransformer` functions: +1. Optional: provide a transformer function for user/group to mutate the entity before their ingestion into catalog. Extend `packages/backend/src/plugins/catalog.ts` with custom `userTransformer` and `groupTransformer` functions: ```ts title="packages/backend/src/plugins/catalog.ts" /* highlight-add-start */ diff --git a/plugins/keycloak-backend/src/providers/KeycloakOrgEntityProvider.test.ts b/plugins/keycloak-backend/src/providers/KeycloakOrgEntityProvider.test.ts index 6267ee72e1..2d3b3e660b 100644 --- a/plugins/keycloak-backend/src/providers/KeycloakOrgEntityProvider.test.ts +++ b/plugins/keycloak-backend/src/providers/KeycloakOrgEntityProvider.test.ts @@ -38,7 +38,7 @@ describe('KeycloakOrgEntityProvider', () => { logger: getVoidLogger(), }), ).toThrow( - 'No schedule provided neither via code nor config for MicrosoftGraphOrgEntityProvider:default.', + 'No schedule provided neither via code nor config for KeycloakOrgEntityProvider:default.', ); }); diff --git a/plugins/keycloak-backend/src/providers/KeycloakOrgEntityProvider.ts b/plugins/keycloak-backend/src/providers/KeycloakOrgEntityProvider.ts index 6a55d7a531..a01aa1d2c6 100644 --- a/plugins/keycloak-backend/src/providers/KeycloakOrgEntityProvider.ts +++ b/plugins/keycloak-backend/src/providers/KeycloakOrgEntityProvider.ts @@ -126,16 +126,21 @@ export class KeycloakOrgEntityProvider implements EntityProvider { options: KeycloakOrgEntityProviderOptions, ): KeycloakOrgEntityProvider[] { return readProviderConfigs(configRoot).map(providerConfig => { - if (!options.schedule && !providerConfig.schedule) { + let taskRunner; + if (options.scheduler && providerConfig.schedule) { + // Create a scheduled task runner using the provided scheduler and schedule configuration + taskRunner = options.scheduler.createScheduledTaskRunner( + providerConfig.schedule, + ); + } else if (options.schedule) { + // Use the provided schedule directly + taskRunner = options.schedule; + } else { throw new Error( - `No schedule provided neither via code nor config for MicrosoftGraphOrgEntityProvider:${providerConfig.id}.`, + `No schedule provided neither via code nor config for KeycloakOrgEntityProvider:${providerConfig.id}.`, ); } - const taskRunner = - options.schedule ?? - options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); - const provider = new KeycloakOrgEntityProvider({ id: providerConfig.id, provider: providerConfig, diff --git a/plugins/ocm-backend/src/providers/ManagedClusterProvider.ts b/plugins/ocm-backend/src/providers/ManagedClusterProvider.ts index 569335ffca..5830bad866 100644 --- a/plugins/ocm-backend/src/providers/ManagedClusterProvider.ts +++ b/plugins/ocm-backend/src/providers/ManagedClusterProvider.ts @@ -82,11 +82,16 @@ export class ManagedClusterProvider implements EntityProvider { ) { return readOcmConfigs(config).map(provider => { const client = hubApiClient(provider, options.logger); - const taskRunner = - options.schedule || - options.scheduler!.createScheduledTaskRunner(provider.schedule!); - - if (!options.schedule && !provider.schedule) { + let taskRunner; + if (options.scheduler && provider.schedule) { + // Create a scheduled task runner using the provided scheduler and schedule configuration + taskRunner = options.scheduler.createScheduledTaskRunner( + provider.schedule, + ); + } else if (options.schedule) { + // Use the provided schedule directly + taskRunner = options.schedule; + } else { throw new Error( `No schedule provided neither via code nor config for "${provider.id}" hub.`, ); diff --git a/plugins/ocm/README.md b/plugins/ocm/README.md index a8635720fd..28918c4021 100644 --- a/plugins/ocm/README.md +++ b/plugins/ocm/README.md @@ -166,69 +166,85 @@ If you are interested in Resource discovery and do not want any of the front-end 1. Import the cluster `Resource` entity provider into the `catalog` plugin in the `packages/backend/src/plugins/catalog.ts` file. The scheduler also needs to be configured. Two configurations are possible here: - 1. Configure the scheduler inside the `app-config.yaml`: - - ```yaml title="app-config.yaml" - catalog: - providers: - ocm: - env: - # ... - # highlight-add-start - schedule: # optional; same options as in TaskScheduleDefinition - # supports cron, ISO duration, "human duration" as used in code - frequency: { minutes: 1 } - # supports ISO duration, "human duration" as used in code - timeout: { minutes: 1 } - # highlight-add-end - ``` - - and then use the configured scheduler in the `packages/backend/src/index.ts`: - - ```ts title="packages/backend/src/index.ts" - /* highlight-add-next-line */ - import { ManagedClusterProvider } from '@janus-idp/backstage-plugin-ocm-backend'; - - export default async function createPlugin( - env: PluginEnvironment, - ): Promise { - const builder = await CatalogBuilder.create(env); - // ... - /* highlight-add-start */ - const ocm = ManagedClusterProvider.fromConfig(env.config, { - logger: env.logger, - scheduler: env.scheduler, - }); - builder.addEntityProvider(ocm); - /* highlight-add-start */ - // ... - } - ``` - - 1. Add a schedule directly inside the `packages/backend/src/plugins/catalog.ts` file: - - ```ts title="packages/backend/src/plugins/catalog.ts" - /* highlight-add-next-line */ - import { ManagedClusterProvider } from '@janus-idp/backstage-plugin-ocm-backend'; - - export default async function createPlugin( - env: PluginEnvironment, - ): Promise { - const builder = await CatalogBuilder.create(env); - // ... - /* highlight-add-start */ - const ocm = ManagedClusterProvider.fromConfig(env.config, { - logger: env.logger, - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 1 }, - timeout: { minutes: 1 }, - }), - }); - builder.addEntityProvider(ocm); - /* highlight-add-start */ - // ... - } - ``` + - **Method 1**: If the scheduler is configured inside the `app-config.yaml` using the schedule config key as follows: + + ```yaml title="app-config.yaml" + catalog: + providers: + ocm: + env: + # ... + # highlight-add-start + schedule: # optional; same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 1 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 1 } + # highlight-add-end + ``` + + Then use the configured scheduler by adding the following to the `packages/backend/src/plugins/catalog.ts`: + + ```ts title="packages/backend/src/plugins/catalog.ts" + /* highlight-add-next-line */ + import { ManagedClusterProvider } from '@janus-idp/backstage-plugin-ocm-backend'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + // ... + /* highlight-add-start */ + const ocm = ManagedClusterProvider.fromConfig(env.config, { + logger: env.logger, + scheduler: env.scheduler, + }); + builder.addEntityProvider(ocm); + /* highlight-add-start */ + // ... + } + ``` + + *** + + **NOTE** + + If you have made any changes to the schedule in the `app-config.yaml` file, then restart to apply the changes. + + *** + + - **Method 2**: Add a schedule directly inside the `packages/backend/src/plugins/catalog.ts` file: + + ```ts title="packages/backend/src/plugins/catalog.ts" + /* highlight-add-next-line */ + import { ManagedClusterProvider } from '@janus-idp/backstage-plugin-ocm-backend'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + // ... + /* highlight-add-start */ + const ocm = ManagedClusterProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 1 }, + timeout: { minutes: 1 }, + }), + }); + builder.addEntityProvider(ocm); + /* highlight-add-start */ + // ... + } + ``` + + *** + + **NOTE** + + If both the `schedule` (hard-coded schedule) and `scheduler` (`app-config.yaml` schedule) option are provided in the `packages/backend/src/plugins/catalog.ts`, the `scheduler` option takes precedence. However, if the schedule inside the `app-config.yaml` file is not configured, then the `schedule` option is used. + + *** 1. Optional: Configure the default owner for the cluster entities in the catalog for a specific environment. For example, use the following code to set `foo` as the owner for clusters from `env` in the `app-config.yaml` catalog section: