From 116c2531fde6041fce018767df2101a197b6d9af Mon Sep 17 00:00:00 2001 From: gonzalojaubert Date: Tue, 19 Sep 2023 16:37:45 +0100 Subject: [PATCH 1/8] Add Azure Event Hub --- .../cli/src/templates/project/index-ts.ts | 2 + .../test/fixtures/mock_project/src/index.ts | 2 + .../src/booster-event-dispatcher.ts | 108 +----- .../src/booster-event-processor.ts | 104 ++++++ .../src/booster-event-stream-consumer.ts | 31 ++ .../src/booster-event-stream-producer.ts | 18 + packages/framework-core/src/booster.ts | 18 + packages/framework-core/src/index.ts | 2 + .../src/services/raw-events-parser.ts | 7 +- .../test/booster-event-dispatcher.test.ts | 308 +--------------- .../test/booster-event-processor.test.ts | 330 ++++++++++++++++++ .../test/services/raw-events-parser.test.ts | 12 +- .../src/config/config.ts | 12 + .../framework-integration-tests/src/index.ts | 2 + packages/framework-provider-aws/src/setup.ts | 3 + .../src/infrastructure/application-builder.ts | 31 +- .../functions/event-handler-function.ts | 28 +- .../event-stream-consumer-function.ts | 31 ++ .../event-stream-producer-function.ts | 42 +++ .../src/infrastructure/helper/function-zip.ts | 40 ++- .../src/infrastructure/helper/utils.ts | 6 +- .../src/infrastructure/index.ts | 16 +- .../infrastructure/rockets/rocket-builder.ts | 7 +- .../infrastructure/synth/application-synth.ts | 266 +++++--------- ...orm-api-management-api-operation-policy.ts | 44 +-- .../terraform-api-management-api-operation.ts | 50 ++- .../synth/terraform-api-management-api.ts | 25 +- .../synth/terraform-api-management.ts | 28 +- .../synth/terraform-containers.ts | 77 ++-- .../synth/terraform-cosmosdb-database.ts | 31 +- .../synth/terraform-cosmosdb-sql-database.ts | 22 +- .../synth/terraform-event-hub-namespace.ts | 24 ++ .../synth/terraform-event-hub.ts | 35 ++ .../synth/terraform-function-app.ts | 70 ++-- .../infrastructure/synth/terraform-outputs.ts | 34 +- .../synth/terraform-resource-group.ts | 19 +- .../synth/terraform-service-plan.ts | 29 +- .../synth/terraform-storage-account.ts | 24 +- .../terraform-web-pub-sub-extension-key.ts | 34 +- .../synth/terraform-web-pubsub-hub.ts | 53 +-- .../synth/terraform-web-pubsub.ts | 28 +- .../types/application-synth-stack.ts | 60 ++-- .../types/functionDefinition.ts | 17 + .../functions/event-handler-functions.test.ts | 3 +- .../framework-provider-azure/package.json | 1 + .../framework-provider-azure/src/constants.ts | 7 + .../framework-provider-azure/src/index.ts | 25 ++ .../library/events-stream-consumer-adapter.ts | 65 ++++ .../library/events-stream-producer-adapter.ts | 56 +++ .../framework-provider-local/src/index.ts | 5 + .../concepts/event-stream-configuration.ts | 14 + .../framework-types/src/concepts/index.ts | 1 + packages/framework-types/src/config.ts | 9 + packages/framework-types/src/index.ts | 1 + .../src/instrumentation/trace-types.ts | 2 + packages/framework-types/src/provider.ts | 12 + packages/framework-types/src/stream-types.ts | 1 + website/docs/10_going-deeper/azure-scale.mdx | 42 +++ website/sidebars.js | 1 + 59 files changed, 1452 insertions(+), 923 deletions(-) create mode 100644 packages/framework-core/src/booster-event-processor.ts create mode 100644 packages/framework-core/src/booster-event-stream-consumer.ts create mode 100644 packages/framework-core/src/booster-event-stream-producer.ts create mode 100644 packages/framework-core/test/booster-event-processor.test.ts create mode 100644 packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-consumer-function.ts create mode 100644 packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-producer-function.ts create mode 100644 packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub-namespace.ts create mode 100644 packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub.ts create mode 100644 packages/framework-provider-azure/src/library/events-stream-consumer-adapter.ts create mode 100644 packages/framework-provider-azure/src/library/events-stream-producer-adapter.ts create mode 100644 packages/framework-types/src/concepts/event-stream-configuration.ts create mode 100644 packages/framework-types/src/stream-types.ts create mode 100644 website/docs/10_going-deeper/azure-scale.mdx diff --git a/packages/cli/src/templates/project/index-ts.ts b/packages/cli/src/templates/project/index-ts.ts index fa4757028..4e19a50f2 100644 --- a/packages/cli/src/templates/project/index-ts.ts +++ b/packages/cli/src/templates/project/index-ts.ts @@ -2,6 +2,8 @@ export const template = `import { Booster } from '@boostercloud/framework-core' export { Booster, boosterEventDispatcher, + boosterProduceEventStream, + boosterConsumeEventStream, boosterServeGraphQL, boosterNotifySubscribers, boosterHealth, diff --git a/packages/cli/test/fixtures/mock_project/src/index.ts b/packages/cli/test/fixtures/mock_project/src/index.ts index ef6a7ba31..0f1606286 100644 --- a/packages/cli/test/fixtures/mock_project/src/index.ts +++ b/packages/cli/test/fixtures/mock_project/src/index.ts @@ -2,6 +2,8 @@ import { Booster } from '@boostercloud/framework-core' export { Booster, boosterEventDispatcher, + boosterProduceEventStream, + boosterConsumeEventStream, boosterPreSignUpChecker, boosterHealth, boosterServeGraphQL, diff --git a/packages/framework-core/src/booster-event-dispatcher.ts b/packages/framework-core/src/booster-event-dispatcher.ts index 01edeb1ac..714c9e6c7 100644 --- a/packages/framework-core/src/booster-event-dispatcher.ts +++ b/packages/framework-core/src/booster-event-dispatcher.ts @@ -1,21 +1,10 @@ -import { - TraceActionTypes, - BoosterConfig, - EventEnvelope, - EventHandlerGlobalError, - EventHandlerInterface, - EventInterface, - Register, - UUID, -} from '@boostercloud/framework-types' +import { TraceActionTypes, BoosterConfig } from '@boostercloud/framework-types' import { EventStore } from './services/event-store' -import { EventsStreamingCallback, RawEventsParser } from './services/raw-events-parser' +import { RawEventsParser } from './services/raw-events-parser' import { ReadModelStore } from './services/read-model-store' -import { RegisterHandler } from './booster-register-handler' -import { BoosterGlobalErrorDispatcher } from './booster-global-error-dispatcher' -import { createInstance, getLogger, Promises } from '@boostercloud/framework-common-helpers' -import { NotificationInterface } from 'framework-types/dist' +import { getLogger } from '@boostercloud/framework-common-helpers' import { Trace } from './instrumentation' +import { BoosterEventProcessor } from './booster-event-processor' export class BoosterEventDispatcher { /** @@ -30,97 +19,14 @@ export class BoosterEventDispatcher { const readModelStore = new ReadModelStore(config) logger.debug('Event workflow started for raw events:', require('util').inspect(rawEvents, false, null, false)) try { + const eventEnvelopes = config.provider.events.rawToEnvelopes(rawEvents) await RawEventsParser.streamPerEntityEvents( config, - rawEvents, - BoosterEventDispatcher.eventProcessor(eventStore, readModelStore) + eventEnvelopes, + BoosterEventProcessor.eventProcessor(eventStore, readModelStore) ) } catch (e) { logger.error('Unhandled error while dispatching event: ', e) } } - - /** - * Builds a function that will be called once for each entity from the `RawEventsParser.streamPerEntityEvents` method - * after the page of events is grouped by entity. - */ - private static eventProcessor(eventStore: EventStore, readModelStore: ReadModelStore): EventsStreamingCallback { - return async (entityName, entityID, eventEnvelopes, config) => { - const eventEnvelopesProcessors = [ - BoosterEventDispatcher.dispatchEntityEventsToEventHandlers(eventEnvelopes, config), - ] - - // Read models are not updated for notification events (events that are not related to an entity but a topic) - if (!(entityName in config.topicToEvent)) { - eventEnvelopesProcessors.push( - BoosterEventDispatcher.snapshotAndUpdateReadModels(config, entityName, entityID, eventStore, readModelStore) - ) - } - - await Promises.allSettledAndFulfilled(eventEnvelopesProcessors) - } - } - - private static async snapshotAndUpdateReadModels( - config: BoosterConfig, - entityName: string, - entityID: UUID, - eventStore: EventStore, - readModelStore: ReadModelStore - ): Promise { - const logger = getLogger(config, 'BoosterEventDispatcher#snapshotAndUpdateReadModels') - let entitySnapshot = undefined - try { - entitySnapshot = await eventStore.fetchEntitySnapshot(entityName, entityID) - } catch (e) { - logger.error('Error while fetching or reducing entity snapshot:', e) - } - if (!entitySnapshot) { - logger.debug('No new snapshot generated, skipping read models projection') - return - } - logger.debug('Snapshot loaded and started read models projection:', entitySnapshot) - await readModelStore.project(entitySnapshot) - } - - @Trace(TraceActionTypes.EVENT_HANDLERS_PROCESS) - private static async dispatchEntityEventsToEventHandlers( - entityEventEnvelopes: Array, - config: BoosterConfig - ): Promise { - const logger = getLogger(config, 'BoosterEventDispatcher.dispatchEntityEventsToEventHandlers') - for (const eventEnvelope of entityEventEnvelopes) { - const eventHandlers = config.eventHandlers[eventEnvelope.typeName] - if (!eventHandlers || eventHandlers.length == 0) { - logger.debug(`No event-handlers found for event ${eventEnvelope.typeName}. Skipping...`) - continue - } - const eventClass = config.events[eventEnvelope.typeName] ?? config.notifications[eventEnvelope.typeName] - await Promises.allSettledAndFulfilled( - eventHandlers.map(async (eventHandler: EventHandlerInterface) => { - const eventInstance = createInstance(eventClass.class, eventEnvelope.value) - const register = new Register(eventEnvelope.requestID, {}, RegisterHandler.flush, eventEnvelope.currentUser) - logger.debug('Calling "handleEvent" method on event handler: ', eventHandler) - await this.handleEvent(eventHandler, eventInstance, register, config) - return RegisterHandler.handle(config, register) - }) - ) - } - } - - @Trace(TraceActionTypes.HANDLE_EVENT) - private static async handleEvent( - eventHandler: EventHandlerInterface, - eventInstance: EventInterface, - register: Register, - config: BoosterConfig - ) { - try { - await eventHandler.handle(eventInstance, register) - } catch (e) { - const globalErrorDispatcher = new BoosterGlobalErrorDispatcher(config) - const error = await globalErrorDispatcher.dispatch(new EventHandlerGlobalError(eventInstance, e)) - if (error) throw error - } - } } diff --git a/packages/framework-core/src/booster-event-processor.ts b/packages/framework-core/src/booster-event-processor.ts new file mode 100644 index 000000000..6b78a9e44 --- /dev/null +++ b/packages/framework-core/src/booster-event-processor.ts @@ -0,0 +1,104 @@ +import { + TraceActionTypes, + BoosterConfig, + EventEnvelope, + EventHandlerGlobalError, + EventHandlerInterface, + EventInterface, + Register, + UUID, +} from '@boostercloud/framework-types' +import { EventStore } from './services/event-store' +import { EventsStreamingCallback } from './services/raw-events-parser' +import { ReadModelStore } from './services/read-model-store' +import { RegisterHandler } from './booster-register-handler' +import { BoosterGlobalErrorDispatcher } from './booster-global-error-dispatcher' +import { createInstance, getLogger, Promises } from '@boostercloud/framework-common-helpers' +import { NotificationInterface } from 'framework-types/dist' +import { Trace } from './instrumentation' + +export class BoosterEventProcessor { + /** + * Function that will be called once for each entity from the `RawEventsParser.streamPerEntityEvents` method + * after the page of events is grouped by entity. + */ + public static eventProcessor(eventStore: EventStore, readModelStore: ReadModelStore): EventsStreamingCallback { + return async (entityName, entityID, eventEnvelopes, config) => { + const eventEnvelopesProcessors = [ + BoosterEventProcessor.dispatchEntityEventsToEventHandlers(eventEnvelopes, config), + ] + + // Read models are not updated for notification events (events that are not related to an entity but a topic) + if (!(entityName in config.topicToEvent)) { + eventEnvelopesProcessors.push( + BoosterEventProcessor.snapshotAndUpdateReadModels(config, entityName, entityID, eventStore, readModelStore) + ) + } + + await Promises.allSettledAndFulfilled(eventEnvelopesProcessors) + } + } + + private static async snapshotAndUpdateReadModels( + config: BoosterConfig, + entityName: string, + entityID: UUID, + eventStore: EventStore, + readModelStore: ReadModelStore + ): Promise { + const logger = getLogger(config, 'BoosterEventDispatcher#snapshotAndUpdateReadModels') + let entitySnapshot = undefined + try { + entitySnapshot = await eventStore.fetchEntitySnapshot(entityName, entityID) + } catch (e) { + logger.error('Error while fetching or reducing entity snapshot:', e) + } + if (!entitySnapshot) { + logger.debug('No new snapshot generated, skipping read models projection') + return + } + logger.debug('Snapshot loaded and started read models projection:', entitySnapshot) + await readModelStore.project(entitySnapshot) + } + + @Trace(TraceActionTypes.EVENT_HANDLERS_PROCESS) + private static async dispatchEntityEventsToEventHandlers( + entityEventEnvelopes: Array, + config: BoosterConfig + ): Promise { + const logger = getLogger(config, 'BoosterEventDispatcher.dispatchEntityEventsToEventHandlers') + for (const eventEnvelope of entityEventEnvelopes) { + const eventHandlers = config.eventHandlers[eventEnvelope.typeName] + if (!eventHandlers || eventHandlers.length == 0) { + logger.debug(`No event-handlers found for event ${eventEnvelope.typeName}. Skipping...`) + continue + } + const eventClass = config.events[eventEnvelope.typeName] ?? config.notifications[eventEnvelope.typeName] + await Promises.allSettledAndFulfilled( + eventHandlers.map(async (eventHandler: EventHandlerInterface) => { + const eventInstance = createInstance(eventClass.class, eventEnvelope.value) + const register = new Register(eventEnvelope.requestID, {}, RegisterHandler.flush, eventEnvelope.currentUser) + logger.debug('Calling "handleEvent" method on event handler: ', eventHandler) + await this.handleEvent(eventHandler, eventInstance, register, config) + return RegisterHandler.handle(config, register) + }) + ) + } + } + + @Trace(TraceActionTypes.HANDLE_EVENT) + private static async handleEvent( + eventHandler: EventHandlerInterface, + eventInstance: EventInterface, + register: Register, + config: BoosterConfig + ) { + try { + await eventHandler.handle(eventInstance, register) + } catch (e) { + const globalErrorDispatcher = new BoosterGlobalErrorDispatcher(config) + const error = await globalErrorDispatcher.dispatch(new EventHandlerGlobalError(eventInstance, e)) + if (error) throw error + } + } +} diff --git a/packages/framework-core/src/booster-event-stream-consumer.ts b/packages/framework-core/src/booster-event-stream-consumer.ts new file mode 100644 index 000000000..40c0a0028 --- /dev/null +++ b/packages/framework-core/src/booster-event-stream-consumer.ts @@ -0,0 +1,31 @@ +import { Trace } from './instrumentation' +import { BoosterConfig, EventStream, TraceActionTypes } from '@boostercloud/framework-types' +import { getLogger } from '@boostercloud/framework-common-helpers' +import { EventStore } from './services/event-store' +import { ReadModelStore } from './services/read-model-store' +import { RawEventsParser } from './services/raw-events-parser' +import { BoosterEventProcessor } from './booster-event-processor' + +export class BoosterEventStreamConsumer { + @Trace(TraceActionTypes.CONSUME_STREAM_EVENTS) + public static async consume(rawEvents: unknown, config: BoosterConfig): Promise { + const logger = getLogger(config, 'BoosterEventDispatcher#dispatch') + const eventStore = new EventStore(config) + const readModelStore = new ReadModelStore(config) + logger.debug( + 'Stream event workflow started for raw events:', + require('util').inspect(rawEvents, false, null, false) + ) + try { + const dedupEvents: EventStream = await config.provider.events.dedupEventStream(config, rawEvents) + const eventEnvelopes = config.provider.events.rawStreamToEnvelopes(config, rawEvents, dedupEvents) + await RawEventsParser.streamPerEntityEvents( + config, + eventEnvelopes, + BoosterEventProcessor.eventProcessor(eventStore, readModelStore) + ) + } catch (e) { + logger.error('Unhandled error while consuming event: ', e) + } + } +} diff --git a/packages/framework-core/src/booster-event-stream-producer.ts b/packages/framework-core/src/booster-event-stream-producer.ts new file mode 100644 index 000000000..44bc99456 --- /dev/null +++ b/packages/framework-core/src/booster-event-stream-producer.ts @@ -0,0 +1,18 @@ +import { Trace } from './instrumentation' +import { BoosterConfig, TraceActionTypes } from '@boostercloud/framework-types' +import { getLogger } from '@boostercloud/framework-common-helpers' +import { RawEventsParser } from './services/raw-events-parser' + +export class BoosterEventStreamProducer { + @Trace(TraceActionTypes.PRODUCE_STREAM_EVENTS) + public static async produce(request: unknown, config: BoosterConfig): Promise { + const logger = getLogger(config, 'BoosterEventStreamProducer#produce') + logger.debug('Produce event workflow started for request:', require('util').inspect(request, false, null, false)) + try { + const eventEnvelopes = config.provider.events.rawToEnvelopes(request) + await RawEventsParser.streamPerEntityEvents(config, eventEnvelopes, config.provider.events.produce) + } catch (e) { + logger.error('Unhandled error while producing events: ', e) + } + } +} diff --git a/packages/framework-core/src/booster.ts b/packages/framework-core/src/booster.ts index c8e4c6e26..47e48c06b 100644 --- a/packages/framework-core/src/booster.ts +++ b/packages/framework-core/src/booster.ts @@ -32,6 +32,8 @@ import { BoosterReadModelsReader } from './booster-read-models-reader' import { BoosterEntityTouched } from './core-concepts/touch-entity/events/booster-entity-touched' import { eventSearch } from './booster-event-search' import { BoosterHealthService } from './sensor' +import { BoosterEventStreamConsumer } from './booster-event-stream-consumer' +import { BoosterEventStreamProducer } from './booster-event-stream-producer' /** * Main class to interact with Booster and configure it. @@ -130,6 +132,14 @@ export class Booster { return entitySnapshotEnvelope ? createInstance(entityClass, entitySnapshotEnvelope.value) : undefined } + public static consumeEventStream(rawEvent: unknown): Promise { + return BoosterEventStreamConsumer.consume(rawEvent, this.config) + } + + public static produceEventStream(request: unknown): Promise { + return BoosterEventStreamProducer.produce(request, this.config) + } + /** * Dispatches event messages to your application. */ @@ -231,6 +241,14 @@ function checkAndGetCurrentEnv(): string { return env } +export async function boosterConsumeEventStream(rawEvent: unknown): Promise { + return Booster.consumeEventStream(rawEvent) +} + +export async function boosterProduceEventStream(rawEvent: unknown): Promise { + return Booster.produceEventStream(rawEvent) +} + export async function boosterEventDispatcher(rawEvent: unknown): Promise { return Booster.dispatchEvent(rawEvent) } diff --git a/packages/framework-core/src/index.ts b/packages/framework-core/src/index.ts index bd43437f2..c4f716ff4 100644 --- a/packages/framework-core/src/index.ts +++ b/packages/framework-core/src/index.ts @@ -8,6 +8,8 @@ export { BoosterDataMigrationEntity } from './core-concepts/data-migration/entit export { BoosterTouchEntityHandler } from './booster-touch-entity-handler' export { boosterEventDispatcher, + boosterProduceEventStream, + boosterConsumeEventStream, boosterServeGraphQL, boosterNotifySubscribers, boosterTriggerScheduledCommand, diff --git a/packages/framework-core/src/services/raw-events-parser.ts b/packages/framework-core/src/services/raw-events-parser.ts index 6488c9fe7..1a470351e 100644 --- a/packages/framework-core/src/services/raw-events-parser.ts +++ b/packages/framework-core/src/services/raw-events-parser.ts @@ -12,14 +12,11 @@ type EnvelopesPerEntity = Record> export class RawEventsParser { public static async streamPerEntityEvents( config: BoosterConfig, - rawEvents: unknown, + eventEnvelopes: Array, callbackFn: EventsStreamingCallback ): Promise { const logger = getLogger(config, 'RawEventsParser#streamPerEntityEvents') - const eventEnvelopesPerEntity = config.provider.events - .rawToEnvelopes(rawEvents) - .filter(isEventKind) - .reduce(groupByEntity, {}) + const eventEnvelopesPerEntity = eventEnvelopes.filter(isEventKind).reduce(groupByEntity, {}) const processes = Object.values(eventEnvelopesPerEntity).map(async (entityEnvelopes) => { // All envelopes are for the same entity type/ID, so we get the first one to get those values diff --git a/packages/framework-core/test/booster-event-dispatcher.test.ts b/packages/framework-core/test/booster-event-dispatcher.test.ts index dbacd2d87..b3ef49e5f 100644 --- a/packages/framework-core/test/booster-event-dispatcher.test.ts +++ b/packages/framework-core/test/booster-event-dispatcher.test.ts @@ -1,23 +1,11 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/explicit-function-return-type */ import { BoosterEventDispatcher } from '../src/booster-event-dispatcher' -import { fake, replace, restore, createStubInstance } from 'sinon' -import { - BoosterConfig, - EntitySnapshotEnvelope, - UUID, - EntityInterface, - ProviderLibrary, - Register, - EventInterface, - NonPersistedEventEnvelope, -} from '@boostercloud/framework-types' +import { fake, replace, restore, SinonSpy } from 'sinon' +import { BoosterConfig, UUID, ProviderLibrary } from '@boostercloud/framework-types' import { expect } from './expect' import { RawEventsParser } from '../src/services/raw-events-parser' -import { ReadModelStore } from '../src/services/read-model-store' -import { EventStore } from '../src/services/event-store' -import { RegisterHandler } from '../src/booster-register-handler' -import { random } from 'faker' +import { BoosterEventProcessor } from '../src/booster-event-processor' class SomeEvent { public constructor(readonly id: UUID) {} @@ -34,58 +22,6 @@ class SomeNotification { public constructor() {} } -class AnEventHandler { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - public static async handle(event: SomeEvent, register: Register): Promise { - event.getPrefixedId('prefix') - } -} - -const someEvent: NonPersistedEventEnvelope = { - version: 1, - kind: 'event', - superKind: 'domain', - entityID: '42', - entityTypeName: 'SomeEntity', - value: { - entityID: (): UUID => { - return '42' - }, - id: '42', - }, - requestID: '123', - typeName: SomeEvent.name, -} - -const someNotification: NonPersistedEventEnvelope = { - version: 1, - kind: 'event', - superKind: 'domain', - entityID: 'default', - entityTypeName: 'defaultTopic', - value: {}, - requestID: '123', - typeName: SomeNotification.name, -} - -const someEntity: EntityInterface = { - id: '42', -} - -const someEntitySnapshot: EntitySnapshotEnvelope = { - version: 1, - kind: 'snapshot', - superKind: 'domain', - entityID: '42', - entityTypeName: 'SomeEntity', - value: someEntity, - requestID: '234', - typeName: 'SomeEntity', - createdAt: 'an uncertain future', - persistedAt: 'a few nanoseconds later', - snapshottedEventCreatedAt: 'an uncertain future', -} - describe('BoosterEventDispatcher', () => { afterEach(() => { restore() @@ -101,19 +37,26 @@ describe('BoosterEventDispatcher', () => { debug: fake(), warn: fake(), } + const rawEvents = [{ some: 'raw event' }, { some: 'other raw event' }] + const events = [{ some: 'raw event' }, { some: 'other raw event' }] + const fakeRawToEnvelopes: SinonSpy = fake.returns(events) + config.provider = { + events: { + rawToEnvelopes: fakeRawToEnvelopes, + }, + } as unknown as ProviderLibrary context('with a configured provider', () => { describe('the `dispatch` method', () => { it('calls the raw events parser once and processes all messages', async () => { replace(RawEventsParser, 'streamPerEntityEvents', fake()) - const rawEvents = [{ some: 'raw event' }, { some: 'other raw event' }] await BoosterEventDispatcher.dispatch(rawEvents, config) expect(RawEventsParser.streamPerEntityEvents).to.have.been.calledWithMatch( config, - rawEvents, - (BoosterEventDispatcher as any).eventProcessor + events, + (BoosterEventProcessor as any).eventProcessor ) }) @@ -131,230 +74,5 @@ describe('BoosterEventDispatcher', () => { ) }) }) - - describe('the `eventProcessor` method', () => { - it('waits for the snapshot generation process and read model update process to complete', async () => { - const stubEventStore = createStubInstance(EventStore) - const stubReadModelStore = createStubInstance(ReadModelStore) - - const boosterEventDispatcher = BoosterEventDispatcher as any - replace(boosterEventDispatcher, 'snapshotAndUpdateReadModels', fake()) - replace(boosterEventDispatcher, 'dispatchEntityEventsToEventHandlers', fake()) - - const callback = boosterEventDispatcher.eventProcessor(stubEventStore, stubReadModelStore) - - await callback(someEvent.entityTypeName, someEvent.entityID, [someEvent], config) - - expect(boosterEventDispatcher.snapshotAndUpdateReadModels).to.have.been.calledOnceWith( - config, - someEvent.entityTypeName, - someEvent.entityID, - stubEventStore, - stubReadModelStore - ) - }) - - it('waits for the event to be handled by the event handlers', async () => { - const stubEventStore = createStubInstance(EventStore) - const stubReadModelStore = createStubInstance(ReadModelStore) - - const boosterEventDispatcher = BoosterEventDispatcher as any - replace(boosterEventDispatcher, 'snapshotAndUpdateReadModels', fake()) - replace(boosterEventDispatcher, 'dispatchEntityEventsToEventHandlers', fake()) - - const callback = boosterEventDispatcher.eventProcessor(stubEventStore, stubReadModelStore) - - await callback(someEvent.entityTypeName, someEvent.entityID, [someEvent], config) - - expect(boosterEventDispatcher.dispatchEntityEventsToEventHandlers).to.have.been.calledOnceWith( - [someEvent], - config - ) - }) - - it("doesn't call snapshotAndUpdateReadModels if the entity name is in config.topicToEvent", async () => { - const stubEventStore = createStubInstance(EventStore) - const stubReadModelStore = createStubInstance(ReadModelStore) - - const boosterEventDispatcher = BoosterEventDispatcher as any - replace(boosterEventDispatcher, 'snapshotAndUpdateReadModels', fake()) - replace(boosterEventDispatcher, 'dispatchEntityEventsToEventHandlers', fake()) - - const overriddenConfig = { ...config } - overriddenConfig.topicToEvent = { [someEvent.entityTypeName]: 'SomeEvent' } - - const callback = boosterEventDispatcher.eventProcessor(stubEventStore, stubReadModelStore) - - await callback(someEvent.entityTypeName, someEvent.entityID, [someEvent], overriddenConfig) - overriddenConfig.topicToEvent = {} - - expect(boosterEventDispatcher.snapshotAndUpdateReadModels).not.to.have.been.called - }) - }) - - describe('the `snapshotAndUpdateReadModels` method', () => { - it('gets the updated state for the event entity', async () => { - const boosterEventDispatcher = BoosterEventDispatcher as any - const eventStore = createStubInstance(EventStore) - const readModelStore = createStubInstance(ReadModelStore) - eventStore.fetchEntitySnapshot = fake.resolves({}) as any - - await boosterEventDispatcher.snapshotAndUpdateReadModels( - config, - someEvent.entityTypeName, - someEvent.entityID, - eventStore, - readModelStore - ) - - expect(eventStore.fetchEntitySnapshot).to.have.been.called - expect(eventStore.fetchEntitySnapshot).to.have.been.calledOnceWith(someEvent.entityTypeName, someEvent.entityID) - }) - - it('projects the entity state to the corresponding read models', async () => { - const boosterEventDispatcher = BoosterEventDispatcher as any - const eventStore = createStubInstance(EventStore) - eventStore.fetchEntitySnapshot = fake.resolves(someEntitySnapshot) as any - - const readModelStore = createStubInstance(ReadModelStore) - - await boosterEventDispatcher.snapshotAndUpdateReadModels( - config, - someEvent.entityTypeName, - someEvent.entityID, - eventStore, - readModelStore - ) - expect(readModelStore.project).to.have.been.calledOnce - expect(readModelStore.project).to.have.been.calledWith(someEntitySnapshot) - }) - - context('when the entity reduction fails', () => { - it('logs the error, does not throw it, and the projects method is not called', async () => { - const boosterEventDispatcher = BoosterEventDispatcher as any - const eventStore = createStubInstance(EventStore) - const readModelStore = createStubInstance(ReadModelStore) - const error = new Error('some error') - eventStore.fetchEntitySnapshot = fake.rejects(error) as any - - await expect( - boosterEventDispatcher.snapshotAndUpdateReadModels( - config, - someEvent.entityTypeName, - someEvent.entityID, - eventStore, - readModelStore - ) - ).to.be.eventually.fulfilled - - expect(readModelStore.project).not.to.have.been.called - expect(config.logger?.error).to.have.been.calledWith( - '[Booster]|BoosterEventDispatcher#snapshotAndUpdateReadModels: ', - 'Error while fetching or reducing entity snapshot:', - error - ) - }) - }) - }) - - describe('the `dispatchEntityEventsToEventHandlers` method', () => { - afterEach(() => { - config.eventHandlers[SomeEvent.name] = [] - }) - - it('does nothing and does not throw if there are no event handlers', async () => { - replace(RegisterHandler, 'handle', fake()) - const boosterEventDispatcher = BoosterEventDispatcher as any - // We try first with null array of event handlers - config.eventHandlers[SomeEvent.name] = null as any - await boosterEventDispatcher.dispatchEntityEventsToEventHandlers([someEvent], config) - // And now with an empty array - config.eventHandlers[SomeEvent.name] = [] - await boosterEventDispatcher.dispatchEntityEventsToEventHandlers([someEvent], config) - // It should not throw any errors - }) - - it('calls all the handlers for the current event', async () => { - const fakeHandler1 = fake() - const fakeHandler2 = fake() - config.eventHandlers[SomeEvent.name] = [{ handle: fakeHandler1 }, { handle: fakeHandler2 }] - - replace(RegisterHandler, 'handle', fake()) - - const boosterEventDispatcher = BoosterEventDispatcher as any - await boosterEventDispatcher.dispatchEntityEventsToEventHandlers([someEvent], config) - - const eventValue: any = someEvent.value - const anEventInstance = new SomeEvent(eventValue.id) - anEventInstance.entityID = eventValue.entityID - - expect(fakeHandler1).to.have.been.calledOnceWith(anEventInstance) - expect(fakeHandler2).to.have.been.calledOnceWith(anEventInstance) - }) - - it('calls all the handlers, even if the event is stored in the notifications field instead of the events one', async () => { - const fakeHandler1 = fake() - const fakeHandler2 = fake() - config.eventHandlers[SomeNotification.name] = [{ handle: fakeHandler1 }, { handle: fakeHandler2 }] - - replace(RegisterHandler, 'handle', fake()) - - const boosterEventDispatcher = BoosterEventDispatcher as any - await boosterEventDispatcher.dispatchEntityEventsToEventHandlers([someNotification], config) - - const aNotificationInstance = new SomeNotification() - - expect(fakeHandler1).to.have.been.calledOnceWith(aNotificationInstance) - expect(fakeHandler2).to.have.been.calledOnceWith(aNotificationInstance) - }) - - it('calls the register handler for all the published events', async () => { - let capturedRegister1: Register = {} as any - let capturedRegister2: Register = {} as any - const fakeHandler1 = fake((event: EventInterface, register: Register) => { - capturedRegister1 = register - }) - const fakeHandler2 = fake((event: EventInterface, register: Register) => { - capturedRegister2 = register - }) - config.eventHandlers[SomeEvent.name] = [{ handle: fakeHandler1 }, { handle: fakeHandler2 }] - - replace(RegisterHandler, 'handle', fake()) - - const boosterEventDispatcher = BoosterEventDispatcher as any - await boosterEventDispatcher.dispatchEntityEventsToEventHandlers([someEvent], config) - - expect(RegisterHandler.handle).to.have.been.calledTwice - expect(RegisterHandler.handle).to.have.been.calledWith(config, capturedRegister1) - expect(RegisterHandler.handle).to.have.been.calledWith(config, capturedRegister2) - }) - - it('waits for async event handlers to finish', async () => { - let capturedRegister: Register = new Register(random.uuid(), {} as any, RegisterHandler.flush) - const fakeHandler = fake(async (event: EventInterface, register: Register) => { - await new Promise((resolve) => setTimeout(resolve, 100)) - register.events(someEvent.value as EventInterface) - capturedRegister = register - }) - config.eventHandlers[SomeEvent.name] = [{ handle: fakeHandler }] - - replace(RegisterHandler, 'handle', fake()) - - const boosterEventDispatcher = BoosterEventDispatcher as any - await boosterEventDispatcher.dispatchEntityEventsToEventHandlers([someEvent], config) - - expect(RegisterHandler.handle).to.have.been.calledWith(config, capturedRegister) - expect(capturedRegister.eventList[0]).to.be.deep.equal(someEvent.value) - }) - }) - - it('calls an instance method in the event and it is executed without failing', async () => { - config.eventHandlers[SomeEvent.name] = [{ handle: AnEventHandler.handle }] - const boosterEventDispatcher = BoosterEventDispatcher as any - const getPrefixedIdFake = fake() - replace(SomeEvent.prototype, 'getPrefixedId', getPrefixedIdFake) - await boosterEventDispatcher.dispatchEntityEventsToEventHandlers([someEvent], config) - expect(getPrefixedIdFake).to.have.been.called - }) }) }) diff --git a/packages/framework-core/test/booster-event-processor.test.ts b/packages/framework-core/test/booster-event-processor.test.ts new file mode 100644 index 000000000..6ba658071 --- /dev/null +++ b/packages/framework-core/test/booster-event-processor.test.ts @@ -0,0 +1,330 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +import { fake, replace, restore, createStubInstance } from 'sinon' +import { + BoosterConfig, + EntitySnapshotEnvelope, + UUID, + EntityInterface, + ProviderLibrary, + Register, + EventInterface, + NonPersistedEventEnvelope, +} from '@boostercloud/framework-types' +import { expect } from './expect' +import { ReadModelStore } from '../src/services/read-model-store' +import { EventStore } from '../src/services/event-store' +import { RegisterHandler } from '../src/booster-register-handler' +import { random } from 'faker' +import { BoosterEventProcessor } from '../src/booster-event-processor' + +class SomeEvent { + public constructor(readonly id: UUID) {} + + public entityID(): UUID { + return this.id + } + public getPrefixedId(prefix: string): string { + return `${prefix}-${this.id}` + } +} + +class SomeNotification { + public constructor() {} +} + +class AnEventHandler { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public static async handle(event: SomeEvent, register: Register): Promise { + event.getPrefixedId('prefix') + } +} + +const someEvent: NonPersistedEventEnvelope = { + version: 1, + kind: 'event', + superKind: 'domain', + entityID: '42', + entityTypeName: 'SomeEntity', + value: { + entityID: (): UUID => { + return '42' + }, + id: '42', + }, + requestID: '123', + typeName: SomeEvent.name, +} + +const someNotification: NonPersistedEventEnvelope = { + version: 1, + kind: 'event', + superKind: 'domain', + entityID: 'default', + entityTypeName: 'defaultTopic', + value: {}, + requestID: '123', + typeName: SomeNotification.name, +} + +const someEntity: EntityInterface = { + id: '42', +} + +const someEntitySnapshot: EntitySnapshotEnvelope = { + version: 1, + kind: 'snapshot', + superKind: 'domain', + entityID: '42', + entityTypeName: 'SomeEntity', + value: someEntity, + requestID: '234', + typeName: 'SomeEntity', + createdAt: 'an uncertain future', + persistedAt: 'a few nanoseconds later', + snapshottedEventCreatedAt: 'an uncertain future', +} + +describe('BoosterEventProcessor', () => { + afterEach(() => { + restore() + }) + + const config = new BoosterConfig('test') + config.provider = {} as ProviderLibrary + config.events[SomeEvent.name] = { class: SomeEvent } + config.notifications[SomeNotification.name] = { class: SomeNotification } + config.logger = { + info: fake(), + error: fake(), + debug: fake(), + warn: fake(), + } + + context('with a configured provider', () => { + describe('the `eventProcessor` method', () => { + it('waits for the snapshot generation process and read model update process to complete', async () => { + const stubEventStore = createStubInstance(EventStore) + const stubReadModelStore = createStubInstance(ReadModelStore) + + const boosterEventProcessor = BoosterEventProcessor as any + replace(boosterEventProcessor, 'snapshotAndUpdateReadModels', fake()) + replace(boosterEventProcessor, 'dispatchEntityEventsToEventHandlers', fake()) + + const callback = boosterEventProcessor.eventProcessor(stubEventStore, stubReadModelStore) + + await callback(someEvent.entityTypeName, someEvent.entityID, [someEvent], config) + + expect(boosterEventProcessor.snapshotAndUpdateReadModels).to.have.been.calledOnceWith( + config, + someEvent.entityTypeName, + someEvent.entityID, + stubEventStore, + stubReadModelStore + ) + }) + + it('waits for the event to be handled by the event handlers', async () => { + const stubEventStore = createStubInstance(EventStore) + const stubReadModelStore = createStubInstance(ReadModelStore) + + const boosterEventProcessor = BoosterEventProcessor as any + replace(boosterEventProcessor, 'snapshotAndUpdateReadModels', fake()) + replace(boosterEventProcessor, 'dispatchEntityEventsToEventHandlers', fake()) + + const callback = boosterEventProcessor.eventProcessor(stubEventStore, stubReadModelStore) + + await callback(someEvent.entityTypeName, someEvent.entityID, [someEvent], config) + + expect(boosterEventProcessor.dispatchEntityEventsToEventHandlers).to.have.been.calledOnceWith( + [someEvent], + config + ) + }) + + it("doesn't call snapshotAndUpdateReadModels if the entity name is in config.topicToEvent", async () => { + const stubEventStore = createStubInstance(EventStore) + const stubReadModelStore = createStubInstance(ReadModelStore) + + const boosterEventProcessor = BoosterEventProcessor as any + replace(boosterEventProcessor, 'snapshotAndUpdateReadModels', fake()) + replace(boosterEventProcessor, 'dispatchEntityEventsToEventHandlers', fake()) + + const overriddenConfig = { ...config } + overriddenConfig.topicToEvent = { [someEvent.entityTypeName]: 'SomeEvent' } + + const callback = boosterEventProcessor.eventProcessor(stubEventStore, stubReadModelStore) + + await callback(someEvent.entityTypeName, someEvent.entityID, [someEvent], overriddenConfig) + overriddenConfig.topicToEvent = {} + + expect(boosterEventProcessor.snapshotAndUpdateReadModels).not.to.have.been.called + }) + }) + + describe('the `snapshotAndUpdateReadModels` method', () => { + it('gets the updated state for the event entity', async () => { + const boosterEventProcessor = BoosterEventProcessor as any + const eventStore = createStubInstance(EventStore) + const readModelStore = createStubInstance(ReadModelStore) + eventStore.fetchEntitySnapshot = fake.resolves({}) as any + + await boosterEventProcessor.snapshotAndUpdateReadModels( + config, + someEvent.entityTypeName, + someEvent.entityID, + eventStore, + readModelStore + ) + + expect(eventStore.fetchEntitySnapshot).to.have.been.called + expect(eventStore.fetchEntitySnapshot).to.have.been.calledOnceWith(someEvent.entityTypeName, someEvent.entityID) + }) + + it('projects the entity state to the corresponding read models', async () => { + const boosterEventProcessor = BoosterEventProcessor as any + const eventStore = createStubInstance(EventStore) + eventStore.fetchEntitySnapshot = fake.resolves(someEntitySnapshot) as any + + const readModelStore = createStubInstance(ReadModelStore) + + await boosterEventProcessor.snapshotAndUpdateReadModels( + config, + someEvent.entityTypeName, + someEvent.entityID, + eventStore, + readModelStore + ) + expect(readModelStore.project).to.have.been.calledOnce + expect(readModelStore.project).to.have.been.calledWith(someEntitySnapshot) + }) + + context('when the entity reduction fails', () => { + it('logs the error, does not throw it, and the projects method is not called', async () => { + const boosterEventProcessor = BoosterEventProcessor as any + const eventStore = createStubInstance(EventStore) + const readModelStore = createStubInstance(ReadModelStore) + const error = new Error('some error') + eventStore.fetchEntitySnapshot = fake.rejects(error) as any + + await expect( + boosterEventProcessor.snapshotAndUpdateReadModels( + config, + someEvent.entityTypeName, + someEvent.entityID, + eventStore, + readModelStore + ) + ).to.be.eventually.fulfilled + + expect(readModelStore.project).not.to.have.been.called + expect(config.logger?.error).to.have.been.calledWith( + '[Booster]|BoosterEventDispatcher#snapshotAndUpdateReadModels: ', + 'Error while fetching or reducing entity snapshot:', + error + ) + }) + }) + }) + + describe('the `dispatchEntityEventsToEventHandlers` method', () => { + afterEach(() => { + config.eventHandlers[SomeEvent.name] = [] + }) + + it('does nothing and does not throw if there are no event handlers', async () => { + replace(RegisterHandler, 'handle', fake()) + const boosterEventProcessor = BoosterEventProcessor as any + // We try first with null array of event handlers + config.eventHandlers[SomeEvent.name] = null as any + await boosterEventProcessor.dispatchEntityEventsToEventHandlers([someEvent], config) + // And now with an empty array + config.eventHandlers[SomeEvent.name] = [] + await boosterEventProcessor.dispatchEntityEventsToEventHandlers([someEvent], config) + // It should not throw any errors + }) + + it('calls all the handlers for the current event', async () => { + const fakeHandler1 = fake() + const fakeHandler2 = fake() + config.eventHandlers[SomeEvent.name] = [{ handle: fakeHandler1 }, { handle: fakeHandler2 }] + + replace(RegisterHandler, 'handle', fake()) + + const boosterEventProcessor = BoosterEventProcessor as any + await boosterEventProcessor.dispatchEntityEventsToEventHandlers([someEvent], config) + + const eventValue: any = someEvent.value + const anEventInstance = new SomeEvent(eventValue.id) + anEventInstance.entityID = eventValue.entityID + + expect(fakeHandler1).to.have.been.calledOnceWith(anEventInstance) + expect(fakeHandler2).to.have.been.calledOnceWith(anEventInstance) + }) + + it('calls all the handlers, even if the event is stored in the notifications field instead of the events one', async () => { + const fakeHandler1 = fake() + const fakeHandler2 = fake() + config.eventHandlers[SomeNotification.name] = [{ handle: fakeHandler1 }, { handle: fakeHandler2 }] + + replace(RegisterHandler, 'handle', fake()) + + const boosterEventProcessor = BoosterEventProcessor as any + await boosterEventProcessor.dispatchEntityEventsToEventHandlers([someNotification], config) + + const aNotificationInstance = new SomeNotification() + + expect(fakeHandler1).to.have.been.calledOnceWith(aNotificationInstance) + expect(fakeHandler2).to.have.been.calledOnceWith(aNotificationInstance) + }) + + it('calls the register handler for all the published events', async () => { + let capturedRegister1: Register = {} as any + let capturedRegister2: Register = {} as any + const fakeHandler1 = fake((event: EventInterface, register: Register) => { + capturedRegister1 = register + }) + const fakeHandler2 = fake((event: EventInterface, register: Register) => { + capturedRegister2 = register + }) + config.eventHandlers[SomeEvent.name] = [{ handle: fakeHandler1 }, { handle: fakeHandler2 }] + + replace(RegisterHandler, 'handle', fake()) + + const boosterEventProcessor = BoosterEventProcessor as any + await boosterEventProcessor.dispatchEntityEventsToEventHandlers([someEvent], config) + + expect(RegisterHandler.handle).to.have.been.calledTwice + expect(RegisterHandler.handle).to.have.been.calledWith(config, capturedRegister1) + expect(RegisterHandler.handle).to.have.been.calledWith(config, capturedRegister2) + }) + + it('waits for async event handlers to finish', async () => { + let capturedRegister: Register = new Register(random.uuid(), {} as any, RegisterHandler.flush) + const fakeHandler = fake(async (event: EventInterface, register: Register) => { + await new Promise((resolve) => setTimeout(resolve, 100)) + register.events(someEvent.value as EventInterface) + capturedRegister = register + }) + config.eventHandlers[SomeEvent.name] = [{ handle: fakeHandler }] + + replace(RegisterHandler, 'handle', fake()) + + const boosterEventProcessor = BoosterEventProcessor as any + await boosterEventProcessor.dispatchEntityEventsToEventHandlers([someEvent], config) + + expect(RegisterHandler.handle).to.have.been.calledWith(config, capturedRegister) + expect(capturedRegister.eventList[0]).to.be.deep.equal(someEvent.value) + }) + }) + + it('calls an instance method in the event and it is executed without failing', async () => { + config.eventHandlers[SomeEvent.name] = [{ handle: AnEventHandler.handle }] + const boosterEventProcessor = BoosterEventProcessor as any + const getPrefixedIdFake = fake() + replace(SomeEvent.prototype, 'getPrefixedId', getPrefixedIdFake) + await boosterEventProcessor.dispatchEntityEventsToEventHandlers([someEvent], config) + expect(getPrefixedIdFake).to.have.been.called + }) + }) +}) diff --git a/packages/framework-core/test/services/raw-events-parser.test.ts b/packages/framework-core/test/services/raw-events-parser.test.ts index 14e42d37d..ce390a66f 100644 --- a/packages/framework-core/test/services/raw-events-parser.test.ts +++ b/packages/framework-core/test/services/raw-events-parser.test.ts @@ -16,7 +16,7 @@ describe('RawEventsParser', () => { restore() }) - const rawEvents = {} // This value doesn't matter, because we are going to fake 'rawToEnvelopes" + const rawEvents = {} // This value doesn't matter, because we are going to fake 'rawToEnvelopes' const entityAName = 'EntityA' const entityAID = 'EntityAID' const entityBName = 'EntityB' @@ -79,13 +79,15 @@ describe('RawEventsParser', () => { describe('streamPerEntityEvents', () => { it('strips all snapshots', async () => { const callbackFunction = fake() - await RawEventsParser.streamPerEntityEvents(config, rawEvents, callbackFunction) + const events = config.provider.events.rawToEnvelopes(rawEvents) + await RawEventsParser.streamPerEntityEvents(config, events, callbackFunction) expect(callbackFunction).not.to.have.been.calledWith(snapshottedEntityName) }) it('calls the callback function with ordered groups of event envelopes per entity name and ID', async () => { const callbackFunction = fake() - await RawEventsParser.streamPerEntityEvents(config, rawEvents, callbackFunction) + const events = config.provider.events.rawToEnvelopes(rawEvents) + await RawEventsParser.streamPerEntityEvents(config, events, callbackFunction) expect(callbackFunction).to.have.been.calledTwice expect(callbackFunction).to.have.been.calledWithExactly( entityAName, @@ -117,8 +119,8 @@ describe('RawEventsParser', () => { events.push(...eventEnvelopes) } ) - - await expect(RawEventsParser.streamPerEntityEvents(config, rawEvents, callbackFunction)).to.be.eventually + const eventsEnvelopes = config.provider.events.rawToEnvelopes(rawEvents) + await expect(RawEventsParser.streamPerEntityEvents(config, eventsEnvelopes, callbackFunction)).to.be.eventually .fulfilled expect(callbackFunction).to.have.been.calledTwice diff --git a/packages/framework-integration-tests/src/config/config.ts b/packages/framework-integration-tests/src/config/config.ts index 2941eee24..69f1aa1af 100644 --- a/packages/framework-integration-tests/src/config/config.ts +++ b/packages/framework-integration-tests/src/config/config.ts @@ -31,6 +31,17 @@ function configureBoosterSensorHealth(config: BoosterConfig) { }) } +function configureEventHub(config: BoosterConfig) { + config.eventStreamConfiguration = { + enabled: true, + parameters: { + streamTopic: 'test', + partitionCount: 3, + messageRetention: 1, + }, + } +} + Booster.configure('local', (config: BoosterConfig): void => { config.appName = 'my-store' config.providerPackage = '@boostercloud/framework-provider-local' @@ -120,4 +131,5 @@ Booster.configure('azure', (config: BoosterConfig): void => { configureInvocationsHandler(config) configureLogger(config) configureBoosterSensorHealth(config) + configureEventHub(config) }) diff --git a/packages/framework-integration-tests/src/index.ts b/packages/framework-integration-tests/src/index.ts index 13a380f08..1ae1780f9 100644 --- a/packages/framework-integration-tests/src/index.ts +++ b/packages/framework-integration-tests/src/index.ts @@ -7,6 +7,8 @@ export { boosterNotifySubscribers, boosterTriggerScheduledCommand, boosterRocketDispatcher, + boosterProduceEventStream, + boosterConsumeEventStream, } from '@boostercloud/framework-core' Booster.start(__dirname) diff --git a/packages/framework-provider-aws/src/setup.ts b/packages/framework-provider-aws/src/setup.ts index a118de626..70eec7421 100644 --- a/packages/framework-provider-aws/src/setup.ts +++ b/packages/framework-provider-aws/src/setup.ts @@ -66,6 +66,9 @@ export const Provider = (rockets?: RocketDescriptor[]): ProviderLibrary => { // ProviderEventsLibrary events: { rawToEnvelopes: rawEventsToEnvelopes, + rawStreamToEnvelopes: notImplementedResult() as any, + dedupEventStream: notImplementedResult() as any, + produce: notImplementedResult() as any, forEntitySince: readEntityEventsSince.bind(null, dynamoDB), latestEntitySnapshot: readEntityLatestSnapshot.bind(null, dynamoDB), search: searchEvents.bind(null, dynamoDB), diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/application-builder.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/application-builder.ts index 408ace0ca..c0e3f7daa 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/application-builder.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/application-builder.ts @@ -2,17 +2,17 @@ import { BoosterConfig } from '@boostercloud/framework-types' import { InfrastructureRocket } from './rockets/infrastructure-rocket' import { AzureStack } from './azure-stack' import * as ckdtfTemplate from './templates/cdktf' -import { createFunctionResourceGroupName, createResourceGroupName, renderToFile } from './helper/utils' +import { renderToFile } from './helper/utils' import { getLogger, Promises } from '@boostercloud/framework-common-helpers' import { App } from 'cdktf' import { ZipResource } from './types/zip-resource' import { FunctionZip } from './helper/function-zip' -import { FunctionDefinition } from './types/functionDefinition' import { RocketBuilder, RocketZipResource } from './rockets/rocket-builder' export interface ApplicationBuild { azureStack: AzureStack zipResource: ZipResource + consumerZipResource: ZipResource rocketsZipResources?: RocketZipResource[] | undefined } @@ -29,40 +29,29 @@ export class ApplicationBuilder { await rocketBuilder.synthRocket() app.synth() - const featureDefinitions = this.mountFeatureDefinitions(azureStack) - const zipResource = await FunctionZip.copyZip(featureDefinitions, 'functionApp.zip') - + azureStack.applicationStack.functionDefinitions = FunctionZip.buildAzureFunctions(this.config) + azureStack.applicationStack.consumerFunctionDefinitions = FunctionZip.buildAzureConsumerFunctions(this.config) + const zipResource = await FunctionZip.copyZip(azureStack.applicationStack.functionDefinitions!, 'functionApp.zip') + const consumerZipResource = await FunctionZip.copyZip( + azureStack.applicationStack.consumerFunctionDefinitions!, + 'consumerFunctionApp.zip' + ) const rocketsZipResources = await rocketBuilder.mountRocketsZipResources() return { azureStack, zipResource, + consumerZipResource, rocketsZipResources, } } - public async uploadFile(zipResource: ZipResource): Promise { - const logger = getLogger(this.config, 'ApplicationBuilder#uploadFile') - logger.info('Uploading zip file') - const resourceGroupName = createResourceGroupName(this.config.appName, this.config.environmentName) - const functionAppName = createFunctionResourceGroupName(resourceGroupName) - await FunctionZip.deployZip(functionAppName, resourceGroupName, zipResource) - logger.info('Zip file uploaded') - } - private async synthApplication(app: App, destinationFile: string): Promise { const logger = getLogger(this.config, 'ApplicationBuilder#synthApplication') logger.info('Synth...') return new AzureStack(app, this.config.appName + this.config.environmentName, destinationFile) } - private mountFeatureDefinitions(azureStack: AzureStack): Array { - const logger = getLogger(this.config, 'ApplicationBuilder#mountFeatureDefinitions') - logger.info('Generating Azure functions') - azureStack.applicationStack.functionDefinitions = FunctionZip.buildAzureFunctions(this.config) - return azureStack.applicationStack.functionDefinitions - } - private async generateSynthFiles(): Promise { const logger = getLogger(this.config, 'ApplicationBuilder#generateSynthFiles') logger.info('Generating cdktf files') diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-handler-function.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-handler-function.ts index 4aec2d322..9dcdcbbc8 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-handler-function.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-handler-function.ts @@ -1,25 +1,27 @@ import { BoosterConfig } from '@boostercloud/framework-types' -import { EventHandlerFunctionDefinition } from '../types/functionDefinition' +import { EventHandlerBinding, EventHandlerFunctionDefinition } from '../types/functionDefinition' +/** + * This function will listen to CosmosDB changes + */ export class EventHandlerFunction { public constructor(readonly config: BoosterConfig) {} public getFunctionDefinition(): EventHandlerFunctionDefinition { + const eventHandlerBinding: EventHandlerBinding = { + type: 'cosmosDBTrigger', + name: 'rawEvent', + direction: 'in', + leaseContainerName: 'leases', + connection: 'COSMOSDB_CONNECTION_STRING', + databaseName: this.config.resourceNames.applicationStack, + containerName: this.config.resourceNames.eventsStore, + createLeaseContainerIfNotExists: 'true', + } return { name: 'eventHandler', config: { - bindings: [ - { - type: 'cosmosDBTrigger', - name: 'rawEvent', - direction: 'in', - leaseContainerName: 'leases', - connection: 'COSMOSDB_CONNECTION_STRING', - databaseName: this.config.resourceNames.applicationStack, - containerName: this.config.resourceNames.eventsStore, - createLeaseContainerIfNotExists: 'true', - }, - ], + bindings: [eventHandlerBinding], scriptFile: this.config.functionRelativePath, entryPoint: this.config.eventDispatcherHandler.split('.')[1], }, diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-consumer-function.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-consumer-function.ts new file mode 100644 index 000000000..223db7d7a --- /dev/null +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-consumer-function.ts @@ -0,0 +1,31 @@ +import { BoosterConfig } from '@boostercloud/framework-types' +import { EventHubInputBinding, EventStreamConsumerHandlerFunctionDefinition } from '../types/functionDefinition' +import { environmentVarNames } from '@boostercloud/framework-provider-azure' + +/** + * Function to consume EventHub events + */ +export class EventStreamConsumerFunction { + public constructor(readonly config: BoosterConfig) {} + + public getFunctionDefinition(): EventStreamConsumerHandlerFunctionDefinition { + const binding: EventHubInputBinding = { + type: 'eventHubTrigger', + name: 'eventHubMessages', + direction: 'in', + eventHubName: this.config.resourceNames.streamTopic, + connection: environmentVarNames.eventHubConnectionString, + cardinality: 'many', + consumerGroup: '$Default', + dataType: 'string', + } + return { + name: 'eventHandler', + config: { + bindings: [binding], + scriptFile: this.config.functionRelativePath, + entryPoint: this.config.eventStreamConsumer.split('.')[1], + }, + } + } +} diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-producer-function.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-producer-function.ts new file mode 100644 index 000000000..e8cf36109 --- /dev/null +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-producer-function.ts @@ -0,0 +1,42 @@ +import { BoosterConfig } from '@boostercloud/framework-types' +import { + EventHandlerBinding, + EventHubOutBinding, + EventStreamProducerHandlerFunctionDefinition, +} from '../types/functionDefinition' +import { environmentVarNames } from '@boostercloud/framework-provider-azure' + +/** + * Function to consume CosmosDB changes and produce EventHub events + */ +export class EventStreamProducerFunction { + public constructor(readonly config: BoosterConfig) {} + + public getFunctionDefinition(): EventStreamProducerHandlerFunctionDefinition { + const cosmosBinding: EventHandlerBinding = { + type: 'cosmosDBTrigger', + name: 'rawEvent', + direction: 'in', + leaseCollectionName: 'leases', + connectionStringSetting: 'COSMOSDB_CONNECTION_STRING', + databaseName: this.config.resourceNames.applicationStack, + collectionName: this.config.resourceNames.eventsStore, + createLeaseCollectionIfNotExists: 'true', + } + const eventHubBinding: EventHubOutBinding = { + name: 'eventHubMessages', + direction: 'out', + type: 'eventHub', + connection: environmentVarNames.eventHubConnectionString, + eventHubName: this.config.resourceNames.streamTopic, + } + return { + name: 'eventProducer', + config: { + bindings: [cosmosBinding, eventHubBinding], + scriptFile: this.config.functionRelativePath, + entryPoint: this.config.eventStreamProducer.split('.')[1], + }, + } + } +} diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/helper/function-zip.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/helper/function-zip.ts index be587590c..83a0db47d 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/helper/function-zip.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/helper/function-zip.ts @@ -16,13 +16,19 @@ import { WebsocketConnectFunction } from '../functions/websocket-connect-functio import { WebsocketDisconnectFunction } from '../functions/websocket-disconnect-function' import { WebsocketMessagesFunction } from '../functions/websocket-messages-function' import { SensorHealthFunction } from '../functions/sensor-health-function' +import { getLogger } from '@boostercloud/framework-common-helpers' +import { EventStreamConsumerFunction } from '../functions/event-stream-consumer-function' +import { EventStreamProducerFunction } from '../functions/event-stream-producer-function' export class FunctionZip { static async deployZip( + config: BoosterConfig, functionAppName: string, resourceGroupName: string, zipResource: ZipResource ): Promise { + const logger = getLogger(config, 'function-zip#deployZip') + logger.info('Uploading zip file') const credentials = await FunctionZip.getCredentials(resourceGroupName, functionAppName) await FunctionZip.deployFunctionPackage( zipResource.path, @@ -30,6 +36,7 @@ export class FunctionZip { credentials.publishingPassword ?? '', credentials.name ?? '' ) + logger.info('Zip file uploaded') return zipResource } @@ -77,14 +84,21 @@ export class FunctionZip { } static buildAzureFunctions(config: BoosterConfig): Array { + const logger = getLogger(config, 'function-zip#buildAzureFunctions') + logger.info('Generating Azure functions') + const graphqlFunctionDefinition = new GraphqlFunction(config).getFunctionDefinition() - const eventHandlerFunctionDefinition = new EventHandlerFunction(config).getFunctionDefinition() const sensorHealthHandlerFunctionDefinition = new SensorHealthFunction(config).getFunctionDefinition() - let featuresDefinitions = [ - graphqlFunctionDefinition, - eventHandlerFunctionDefinition, - sensorHealthHandlerFunctionDefinition, - ] + let featuresDefinitions = [graphqlFunctionDefinition, sensorHealthHandlerFunctionDefinition] + if (config.eventStreamConfiguration.enabled) { + // If event stream then we build an EventHub event trigger + const eventStreamProducerFunctionDefinition = new EventStreamProducerFunction(config).getFunctionDefinition() + featuresDefinitions.push(eventStreamProducerFunctionDefinition) + } else { + // If no event stream then we build a CosmosDB event trigger + const eventHandlerFunctionDefinition = new EventHandlerFunction(config).getFunctionDefinition() + featuresDefinitions.push(eventHandlerFunctionDefinition) + } if (config.enableSubscriptions) { const messagesFunctionDefinition = new WebsocketMessagesFunction(config).getFunctionDefinition() const disconnectFunctionDefinition = new WebsocketDisconnectFunction(config).getFunctionDefinition() @@ -101,6 +115,20 @@ export class FunctionZip { if (scheduledFunctionsDefinition) { featuresDefinitions = featuresDefinitions.concat(scheduledFunctionsDefinition) } + logger.info('Azure functions generated') + return featuresDefinitions + } + + static buildAzureConsumerFunctions(config: BoosterConfig): Array { + const featuresDefinitions = [] + if (!config.eventStreamConfiguration.enabled) { + return [] + } + const logger = getLogger(config, 'function-zip#buildAzureConsumerFunctions') + logger.info('Generating Azure Consumer functions') + const eventStreamHandlerFunctionDefinition = new EventStreamConsumerFunction(config).getFunctionDefinition() + featuresDefinitions.push(eventStreamHandlerFunctionDefinition) + logger.info('Azure Consumer functions generated') return featuresDefinitions } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/helper/utils.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/helper/utils.ts index 9b8c5b750..a35682ca2 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/helper/utils.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/helper/utils.ts @@ -97,7 +97,11 @@ export function createResourceGroupName(appName: string, environmentName: string } export function createFunctionResourceGroupName(resourceGroupName: string): string { - return `${resourceGroupName}func` + return `${resourceGroupName}fpr` +} + +export function createStreamFunctionResourceGroupName(resourceGroupName: string): string { + return `${resourceGroupName}fcs` } export function createApiManagementName(resourceGroupName: string): string { diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/index.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/index.ts index f6183113a..ca630e5c9 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/index.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/index.ts @@ -1,9 +1,15 @@ import { BoosterConfig } from '@boostercloud/framework-types' -import { azureCredentials, createResourceGroupName, createResourceManagementClient } from './helper/utils' +import { + azureCredentials, + createFunctionResourceGroupName, + createResourceGroupName, + createResourceManagementClient, createStreamFunctionResourceGroupName +} from './helper/utils' import { runCommand, getLogger } from '@boostercloud/framework-common-helpers' import { InfrastructureRocket } from './rockets/infrastructure-rocket' import { ApplicationBuilder } from './application-builder' import { RocketBuilder } from './rockets/rocket-builder' +import { FunctionZip } from './helper/function-zip' export const synth = (config: BoosterConfig, rockets?: InfrastructureRocket[]): Promise => synthApp(config, rockets) @@ -36,7 +42,13 @@ async function deployApp(config: BoosterConfig, rockets?: InfrastructureRocket[] return Promise.reject(`Deployment of application ${config.appName} failed. Check cdktf logs. \n${error.message}}`) } - await applicationBuilder.uploadFile(applicationBuild.zipResource) + const resourceGroupName = createResourceGroupName(config.appName, config.environmentName) + const functionAppName = createFunctionResourceGroupName(resourceGroupName) + await FunctionZip.deployZip(config, functionAppName, resourceGroupName, applicationBuild.zipResource) + if (config.eventStreamConfiguration.enabled) { + const streamFunctionAppName = createStreamFunctionResourceGroupName(resourceGroupName) + await FunctionZip.deployZip(config, streamFunctionAppName, resourceGroupName, applicationBuild.consumerZipResource) + } if (applicationBuild.rocketsZipResources && applicationBuild.rocketsZipResources.length > 0) { const rocketBuilder = new RocketBuilder(config, applicationBuild.azureStack.applicationStack, rockets) await rocketBuilder.uploadRocketsFiles(applicationBuild.rocketsZipResources) diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/rockets/rocket-builder.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/rockets/rocket-builder.ts index 51f91e2dc..77505b4db 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/rockets/rocket-builder.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/rockets/rocket-builder.ts @@ -45,7 +45,12 @@ export class RocketBuilder { private async uploadRocketFile(rocketZipResource: RocketZipResource, resourceGroupName: string): Promise { const logger = getLogger(this.config, 'RocketBuilder#uploadRocketFile') logger.info('Uploading rockets zip file: ', rocketZipResource.functionAppName, rocketZipResource.zip.fileName) - await FunctionZip.deployZip(rocketZipResource.functionAppName, resourceGroupName, rocketZipResource.zip) + await FunctionZip.deployZip( + this.config, + rocketZipResource.functionAppName, + resourceGroupName, + rocketZipResource.zip + ) logger.info('Rocket zip files uploaded', rocketZipResource.functionAppName, rocketZipResource.zip.fileName) } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts index e48d230aa..2f3e96e5d 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts @@ -1,10 +1,11 @@ import { BoosterConfig } from '@boostercloud/framework-types' import { buildAppPrefix, - readProjectConfig, + createApiManagementName, createFunctionResourceGroupName, createResourceGroupName, - createApiManagementName, + createStreamFunctionResourceGroupName, + readProjectConfig, } from '../helper/utils' import { TerraformStack } from 'cdktf' import { TerraformServicePlan } from './terraform-service-plan' @@ -19,131 +20,64 @@ import { TerraformApiManagementApi } from './terraform-api-management-api' import { TerraformApiManagementApiOperation } from './terraform-api-management-api-operation' import { TerraformApiManagementApiOperationPolicy } from './terraform-api-management-api-operation-policy' import { TerraformWebPubsub } from './terraform-web-pubsub' -import { ApplicationSynthStack } from '../types/application-synth-stack' +import { ApplicationSynthStack, StackNames } from '../types/application-synth-stack' import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' import { TerraformOutputs } from './terraform-outputs' import { TerraformWebPubsubHub } from './terraform-web-pubsub-hub' import { TerraformWebPubSubExtensionKey } from './terraform-web-pub-sub-extension-key' +import { TerraformEventHubNamespace } from './terraform-event-hub-namespace' +import { TerraformEventHub } from './terraform-event-hub' +import { windowsFunctionApp } from '@cdktf/provider-azurerm' import { TerraformApiManagementApiOperationSensorHealth } from './terraform-api-management-api-operation-sensor-health' export class ApplicationSynth { readonly config: BoosterConfig - readonly appPrefix: string - readonly terraformStackResource: TerraformStack + readonly stackNames: StackNames - public constructor(terraformStackResource: TerraformStack) { + public constructor(terraformStack: TerraformStack) { this.config = readProjectConfig(process.cwd()) - this.appPrefix = buildAppPrefix(this.config) - this.terraformStackResource = terraformStackResource - } - - public synth(zipFile: string): ApplicationSynthStack { + const azurermProvider = new AzurermProvider(terraformStack, 'azureFeature', { + features: {}, + }) + const appPrefix = buildAppPrefix(this.config) const resourceGroupName = createResourceGroupName(this.config.appName, this.config.environmentName) const functionAppName = createFunctionResourceGroupName(resourceGroupName) + const streamFunctionAppName = createStreamFunctionResourceGroupName(resourceGroupName) const apiManagementName = createApiManagementName(resourceGroupName) - const azurermProvider = new AzurermProvider(this.terraformStackResource, 'azureFeature', { - features: {}, - }) - const hubName = 'booster' - - const resourceGroupResource = TerraformResourceGroup.build( - azurermProvider, - this.terraformStackResource, - this.appPrefix, - resourceGroupName - ) - const servicePlanResource = TerraformServicePlan.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - this.appPrefix, - resourceGroupName - ) - const storageAccountResource = TerraformStorageAccount.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - this.appPrefix, - resourceGroupName - ) - const cosmosdbDatabaseResource = TerraformCosmosdbDatabase.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - this.appPrefix, - resourceGroupName - ) - const cosmosdbSqlDatabaseResource = TerraformCosmosdbSqlDatabase.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - this.appPrefix, - cosmosdbDatabaseResource, - this.config - ) - const containersResource = TerraformContainers.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - this.appPrefix, - cosmosdbDatabaseResource, - cosmosdbSqlDatabaseResource, - this.config - ) - - let webPubSubResource - if (this.config.enableSubscriptions) { - webPubSubResource = TerraformWebPubsub.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - this.appPrefix - ) + this.stackNames = { + appPrefix: appPrefix, + terraformStack: terraformStack, + azureProvider: azurermProvider, + resourceGroupName: resourceGroupName, + functionAppName: functionAppName, + streamFunctionAppName: streamFunctionAppName, + apiManagementName: apiManagementName, + eventHubName: this.config.resourceNames.streamTopic, + webPubSubHubName: 'booster', } + } - const functionAppResource = TerraformFunctionApp.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - servicePlanResource, - storageAccountResource, - this.appPrefix, - functionAppName, - cosmosdbDatabaseResource.name, - apiManagementName, - cosmosdbDatabaseResource.primaryKey, - this.config, - zipFile, - webPubSubResource - ) - - const apiManagementResource = TerraformApiManagement.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - apiManagementName, - this.appPrefix - ) - - const apiManagementApiResource = TerraformApiManagementApi.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - apiManagementResource, - this.appPrefix, + public synth(zipFile: string): ApplicationSynthStack { + const graphQLApiOperation = 'graphql' + const resourceGroup = TerraformResourceGroup.build(this.stackNames) + const stack: ApplicationSynthStack = { ...this.stackNames, resourceGroup: resourceGroup } + stack.cosmosdbDatabase = TerraformCosmosdbDatabase.build(stack) + stack.cosmosdbSqlDatabase = TerraformCosmosdbSqlDatabase.build(stack, this.config) + stack.containers = TerraformContainers.build(stack, this.config) + this.buildEventHub(zipFile, stack) + this.buildWebPubSub(stack) + stack.apiManagement = TerraformApiManagement.build(stack) + stack.apiManagementApi = TerraformApiManagementApi.build(stack, this.config.environmentName) + stack.graphQLApiManagementApiOperation = TerraformApiManagementApiOperation.build(stack, graphQLApiOperation) + stack.applicationServicePlan = TerraformServicePlan.build(stack, 'psp', 'Y1', 1) + stack.storageAccount = TerraformStorageAccount.build(stack, 'sp') + stack.functionApp = this.buildDefaultFunctionApp(stack, zipFile) + stack.graphQLApiManagementApiOperationPolicy = TerraformApiManagementApiOperationPolicy.build( + stack, this.config.environmentName, - resourceGroupName + graphQLApiOperation ) - - const graphQLApiManagementApiOperationResource = TerraformApiManagementApiOperation.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - apiManagementApiResource, - this.appPrefix, - 'graphql' - ) - + // TODO const sensorHealthApiManagementApiOperationResource = TerraformApiManagementApiOperationSensorHealth.build( azurermProvider, this.terraformStackResource, @@ -152,18 +86,7 @@ export class ApplicationSynth { this.appPrefix, 'sensor-health' ) - - const graphQLApiManagementApiOperationPolicyResource = TerraformApiManagementApiOperationPolicy.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - graphQLApiManagementApiOperationResource, - this.appPrefix, - this.config.environmentName, - functionAppResource, - 'graphql' - ) - + // todo const sensorHealthApiManagementApiOperationPolicyResource = TerraformApiManagementApiOperationPolicy.build( azurermProvider, this.terraformStackResource, @@ -174,63 +97,60 @@ export class ApplicationSynth { functionAppResource, 'sensor-health' ) + this.buildWebPubSubHub(stack) + TerraformOutputs.build(stack) - let webPubSubHubResource + return stack + } - if (webPubSubResource) { - const functionAppDataResource = TerraformWebPubSubExtensionKey.build( + private buildDefaultFunctionApp( + stack: ApplicationSynthStack, + zipFile: string + ): windowsFunctionApp.WindowsFunctionApp { + return TerraformFunctionApp.build( + stack, + this.config, + zipFile, + stack.applicationServicePlan!, + stack.storageAccount!, + 'func', + stack.functionAppName + ) + } + + private buildEventHub(zipFile: string, stack: ApplicationSynthStack): void { + if (this.config.eventStreamConfiguration.enabled) { + stack.eventHubNamespace = TerraformEventHubNamespace.build(stack) + stack.eventHub = TerraformEventHub.build(stack, this.config) + const instanceCount = this.config.eventStreamConfiguration.parameters?.partitionCount ?? '3' + stack.eventConsumerServicePlan = TerraformServicePlan.build(stack, 'psc', 'B1', instanceCount) + stack.eventConsumerStorageAccount = TerraformStorageAccount.build(stack, 'sc') + stack.eventConsumerFunctionApp = TerraformFunctionApp.build( + stack, this.config, - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - functionAppResource, - this.appPrefix + zipFile, + stack.eventConsumerServicePlan, + stack.eventConsumerStorageAccount, + 'fhub', + stack.streamFunctionAppName ) + if (!stack.containers) { + stack.containers = [] + } + stack.containers.push(TerraformContainers.createDedupEventsContainer(stack, this.config)) + } + } - webPubSubHubResource = TerraformWebPubsubHub.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - webPubSubResource, - this.appPrefix, - functionAppResource, - functionAppDataResource, - hubName - ) + private buildWebPubSub(stack: ApplicationSynthStack): void { + if (this.config.enableSubscriptions) { + stack.webPubSub = TerraformWebPubsub.build(stack) } - TerraformOutputs.build( - azurermProvider, - this.terraformStackResource, - this.appPrefix, - resourceGroupResource, - graphQLApiManagementApiOperationResource, - sensorHealthApiManagementApiOperationResource, - hubName, - webPubSubResource - ) + } - return { - appPrefix: this.appPrefix, - terraformStack: this.terraformStackResource, - resourceGroupName: resourceGroupName, - apiManagementName: apiManagementName, - functionAppName: functionAppName, - resourceGroup: resourceGroupResource, - applicationServicePlan: servicePlanResource, - storageAccount: storageAccountResource, - functionApp: functionAppResource, - apiManagement: apiManagementResource, - apiManagementApi: apiManagementApiResource, - graphQLApiManagementApiOperation: graphQLApiManagementApiOperationResource, - graphQLApiManagementApiOperationPolicy: graphQLApiManagementApiOperationPolicyResource, - sensorHealthApiManagementApiOperation: sensorHealthApiManagementApiOperationResource, - sensorHealthApiManagementApiOperationPolicy: sensorHealthApiManagementApiOperationPolicyResource, - cosmosdbDatabase: cosmosdbDatabaseResource, - cosmosdbSqlDatabase: cosmosdbSqlDatabaseResource, - containers: containersResource, - webPubSub: webPubSubResource, - webPubSubHub: webPubSubHubResource, - azureProvider: azurermProvider, - } as ApplicationSynthStack + private buildWebPubSubHub(stack: ApplicationSynthStack) { + if (stack.webPubSub) { + stack.dataFunctionAppHostKeys = TerraformWebPubSubExtensionKey.build(stack) + stack.webPubSubHub = TerraformWebPubsubHub.build(stack) + } } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts index 1845d279e..1b40bd840 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts @@ -1,38 +1,40 @@ -import { TerraformStack } from 'cdktf' -import { - apiManagementApiOperation, - apiManagementApiOperationPolicy, - resourceGroup, - windowsFunctionApp, -} from '@cdktf/provider-azurerm' +import { apiManagementApiOperationPolicy } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' import * as Mustache from 'mustache' import { templates } from '../templates' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformApiManagementApiOperationPolicy { static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - apiManagementApiOperationResource: apiManagementApiOperation.ApiManagementApiOperation, - appPrefix: string, + { + terraformStack, + azureProvider, + appPrefix, + resourceGroupName, + functionApp, + graphQLApiManagementApiOperation, + }: ApplicationSynthStack, environmentName: string, - functionAppResource: windowsFunctionApp.WindowsFunctionApp, name: string ): apiManagementApiOperationPolicy.ApiManagementApiOperationPolicy { + if (!functionApp) { + throw new Error('Undefined functionApp resource') + } + if (!graphQLApiManagementApiOperation) { + throw new Error('Undefined graphQLApiManagementApiOperation resource') + } const idApiManagementApiOperationPolicy = toTerraformName(appPrefix, 'amaop' + name[0]) - const policyContent = Mustache.render(templates.policy, { functionAppName: functionAppResource.name }) + const policyContent = Mustache.render(templates.policy, { functionAppName: functionApp.name }) return new apiManagementApiOperationPolicy.ApiManagementApiOperationPolicy( - terraformStackResource, + terraformStack, idApiManagementApiOperationPolicy, { - apiName: apiManagementApiOperationResource.apiName, - apiManagementName: apiManagementApiOperationResource.apiManagementName, - resourceGroupName: resourceGroupResource.name, - operationId: apiManagementApiOperationResource.operationId, + apiName: graphQLApiManagementApiOperation.apiName, + apiManagementName: graphQLApiManagementApiOperation.apiManagementName, + resourceGroupName: resourceGroupName, + operationId: graphQLApiManagementApiOperation.operationId, xmlContent: policyContent, - provider: providerResource, + provider: azureProvider, } ) } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation.ts index b7801a402..6fbb7d502 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation.ts @@ -1,37 +1,31 @@ -import { TerraformStack } from 'cdktf' -import { apiManagementApi, apiManagementApiOperation, resourceGroup } from '@cdktf/provider-azurerm' +import { apiManagementApiOperation } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformApiManagementApiOperation { static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - group: resourceGroup.ResourceGroup, - apiManagementApiResource: apiManagementApi.ApiManagementApi, - appPrefix: string, + { terraformStack, azureProvider, appPrefix, resourceGroupName, apiManagementApi }: ApplicationSynthStack, name: string ): apiManagementApiOperation.ApiManagementApiOperation { + if (!apiManagementApi) { + throw new Error('Undefined apiManagementApi resource') + } const idApiManagementApiOperation = toTerraformName(appPrefix, 'amao' + name[0]) - return new apiManagementApiOperation.ApiManagementApiOperation( - terraformStackResource, - idApiManagementApiOperation, - { - operationId: `${name}POST`, - apiName: apiManagementApiResource.name, - apiManagementName: apiManagementApiResource.apiManagementName, - resourceGroupName: group.name, - displayName: `/${name}`, - method: 'POST', - urlTemplate: `/${name}`, - description: '', - response: [ - { - statusCode: 200, - }, - ], - provider: providerResource, - } - ) + return new apiManagementApiOperation.ApiManagementApiOperation(terraformStack, idApiManagementApiOperation, { + operationId: `${name}POST`, + apiName: apiManagementApi.name, + apiManagementName: apiManagementApi.apiManagementName, + resourceGroupName: resourceGroupName, + displayName: `/${name}`, + method: 'POST', + urlTemplate: `/${name}`, + description: '', + response: [ + { + statusCode: 200, + }, + ], + provider: azureProvider, + }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api.ts index ab7a78fc1..cd3ec915e 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api.ts @@ -1,29 +1,26 @@ -import { TerraformStack } from 'cdktf' -import { apiManagement, apiManagementApi, resourceGroup } from '@cdktf/provider-azurerm' +import { apiManagementApi } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformApiManagementApi { static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - apiManagementResource: apiManagement.ApiManagement, - appPrefix: string, - environmentName: string, - resourceGroupName: string + { terraformStack, azureProvider, appPrefix, resourceGroupName, apiManagement }: ApplicationSynthStack, + environmentName: string ): apiManagementApi.ApiManagementApi { + if (!apiManagement) { + throw new Error('Undefined apiManagement resource') + } const idApiManagementApi = toTerraformName(appPrefix, 'amapi') - return new apiManagementApi.ApiManagementApi(terraformStackResource, idApiManagementApi, { + return new apiManagementApi.ApiManagementApi(terraformStack, idApiManagementApi, { name: `${resourceGroupName}rest`, - resourceGroupName: resourceGroupResource.name, - apiManagementName: apiManagementResource.name, + resourceGroupName: resourceGroupName, + apiManagementName: apiManagement.name, revision: '1', displayName: `${appPrefix}-rest-api`, path: environmentName, protocols: ['http', 'https'], subscriptionRequired: false, - provider: providerResource, + provider: azureProvider, }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management.ts index aa64c296d..d0a75e509 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management.ts @@ -1,26 +1,26 @@ -import { TerraformStack } from 'cdktf' -import { apiManagement, resourceGroup } from '@cdktf/provider-azurerm' +import { apiManagement } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' import { configuration } from '../helper/params' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformApiManagement { - static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - apiManagementName: string, - appPrefix: string - ): apiManagement.ApiManagement { + static build({ + terraformStack, + azureProvider, + appPrefix, + resourceGroup, + resourceGroupName, + apiManagementName, + }: ApplicationSynthStack): apiManagement.ApiManagement { const idApiManagement = toTerraformName(appPrefix, 'am') - return new apiManagement.ApiManagement(terraformStackResource, idApiManagement, { + return new apiManagement.ApiManagement(terraformStack, idApiManagement, { name: apiManagementName, - location: resourceGroupResource.location, - resourceGroupName: resourceGroupResource.name, + location: resourceGroup.location, + resourceGroupName: resourceGroupName, publisherName: configuration.publisherName, publisherEmail: configuration.publisherEmail, skuName: 'Consumption_0', - provider: providerResource, + provider: azureProvider, }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-containers.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-containers.ts index 4892dcdc7..28079ae94 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-containers.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-containers.ts @@ -1,61 +1,57 @@ import { TerraformStack } from 'cdktf' -import { cosmosdbAccount, cosmosdbSqlContainer, cosmosdbSqlDatabase, resourceGroup } from '@cdktf/provider-azurerm' +import { cosmosdbAccount, cosmosdbSqlContainer, cosmosdbSqlDatabase } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' import { BoosterConfig } from '@boostercloud/framework-types' import { connectionsStoreAttributes, + dedupAttributes, eventsStoreAttributes, subscriptionsStoreAttributes, } from '@boostercloud/framework-provider-azure' import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' import { MAX_CONTAINER_THROUGHPUT } from '../constants' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformContainers { static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - appPrefix: string, - cosmosdbDatabaseResource: cosmosdbAccount.CosmosdbAccount, - cosmosdbSqlDatabaseResource: cosmosdbSqlDatabase.CosmosdbSqlDatabase, + { terraformStack, azureProvider, appPrefix, cosmosdbDatabase, cosmosdbSqlDatabase }: ApplicationSynthStack, config: BoosterConfig ): Array { + if (!cosmosdbDatabase) { + throw new Error('Undefined cosmosdbDatabase resource') + } + if (!cosmosdbSqlDatabase) { + throw new Error('Undefined cosmosdbSqlDatabase resource') + } const cosmosdbSqlEventContainer = this.createEventContainer( - providerResource, + azureProvider, appPrefix, - terraformStackResource, + terraformStack, config, - cosmosdbDatabaseResource, - cosmosdbSqlDatabaseResource + cosmosdbDatabase, + cosmosdbSqlDatabase ) const readModels = Object.keys(config.readModels).map((readModel) => - this.createReadModel( - providerResource, - terraformStackResource, - config, - readModel, - cosmosdbDatabaseResource, - cosmosdbSqlDatabaseResource - ) + this.createReadModel(azureProvider, terraformStack, config, readModel, cosmosdbDatabase, cosmosdbSqlDatabase) ) if (config.enableSubscriptions) { const subscriptionsContainer = this.createSubscriptionsContainer( - providerResource, + azureProvider, appPrefix, - terraformStackResource, + terraformStack, config, - cosmosdbDatabaseResource, - cosmosdbSqlDatabaseResource + cosmosdbDatabase, + cosmosdbSqlDatabase ) const connectionsContainer = this.createConnectionsContainer( - providerResource, + azureProvider, appPrefix, - terraformStackResource, + terraformStack, config, - cosmosdbDatabaseResource, - cosmosdbSqlDatabaseResource + cosmosdbDatabase, + cosmosdbSqlDatabase ) return [cosmosdbSqlEventContainer, subscriptionsContainer, connectionsContainer].concat(readModels) } @@ -156,4 +152,31 @@ export class TerraformContainers { provider: providerResource, }) } + + static createDedupEventsContainer( + { terraformStack, azureProvider, appPrefix, cosmosdbDatabase, cosmosdbSqlDatabase }: ApplicationSynthStack, + config: BoosterConfig + ): cosmosdbSqlContainer.CosmosdbSqlContainer { + if (!cosmosdbDatabase) { + throw new Error('Undefined cosmosdbDatabase resource') + } + if (!cosmosdbSqlDatabase) { + throw new Error('Undefined cosmosdbSqlDatabase resource') + } + const id = toTerraformName(appPrefix, 'dedup-table') + return new cosmosdbSqlContainer.CosmosdbSqlContainer(terraformStack, id, { + name: config.resourceNames.eventsDedup, + resourceGroupName: cosmosdbDatabase.resourceGroupName, + accountName: cosmosdbDatabase.name, + databaseName: cosmosdbSqlDatabase.name, + partitionKeyPath: `/${dedupAttributes.partitionKey}`, + uniqueKey: [{ paths: [`/${dedupAttributes.partitionKey}`] }], + partitionKeyVersion: 2, + defaultTtl: -1, + autoscaleSettings: { + maxThroughput: MAX_CONTAINER_THROUGHPUT, + }, + provider: azureProvider, + }) + } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-cosmosdb-database.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-cosmosdb-database.ts index c5858e0f8..f22ac7699 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-cosmosdb-database.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-cosmosdb-database.ts @@ -1,21 +1,20 @@ -import { TerraformStack } from 'cdktf' -import { cosmosdbAccount, resourceGroup } from '@cdktf/provider-azurerm' +import { cosmosdbAccount } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformCosmosdbDatabase { - static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - appPrefix: string, - resourceGroupName: string - ): cosmosdbAccount.CosmosdbAccount { + static build({ + terraformStack, + azureProvider, + appPrefix, + resourceGroup, + resourceGroupName, + }: ApplicationSynthStack): cosmosdbAccount.CosmosdbAccount { const idAccount = toTerraformName(appPrefix, 'dba') - return new cosmosdbAccount.CosmosdbAccount(terraformStackResource, idAccount, { + return new cosmosdbAccount.CosmosdbAccount(terraformStack, idAccount, { name: `${resourceGroupName}cdba`, - location: resourceGroupResource.location, - resourceGroupName: resourceGroupResource.name, + location: resourceGroup.location, + resourceGroupName: resourceGroupName, offerType: 'Standard', kind: 'GlobalDocumentDB', enableMultipleWriteLocations: false, @@ -23,15 +22,15 @@ export class TerraformCosmosdbDatabase { enableAutomaticFailover: true, geoLocation: [ { - location: resourceGroupResource.location, + location: resourceGroup.location, failoverPriority: 0, }, ], consistencyPolicy: { consistencyLevel: 'Session', }, - dependsOn: [resourceGroupResource], - provider: providerResource, + dependsOn: [resourceGroup], + provider: azureProvider, }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-cosmosdb-sql-database.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-cosmosdb-sql-database.ts index bcfff66e3..488ae3603 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-cosmosdb-sql-database.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-cosmosdb-sql-database.ts @@ -1,29 +1,27 @@ -import { TerraformStack } from 'cdktf' -import { cosmosdbAccount, cosmosdbSqlDatabase, resourceGroup } from '@cdktf/provider-azurerm' +import { cosmosdbSqlDatabase } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' import { BoosterConfig } from '@boostercloud/framework-types' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' import { MAX_DATABASE_THROUGHPUT } from '../constants' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformCosmosdbSqlDatabase { static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - appPrefix: string, - cosmosdbAccountResource: cosmosdbAccount.CosmosdbAccount, + { terraformStack, azureProvider, appPrefix, resourceGroupName, cosmosdbDatabase }: ApplicationSynthStack, config: BoosterConfig ): cosmosdbSqlDatabase.CosmosdbSqlDatabase { + if (!cosmosdbDatabase) { + throw new Error('Undefined cosmosdbDatabase resource') + } const idDatabase = toTerraformName(appPrefix, 'dbd') - return new cosmosdbSqlDatabase.CosmosdbSqlDatabase(terraformStackResource, idDatabase, { + return new cosmosdbSqlDatabase.CosmosdbSqlDatabase(terraformStack, idDatabase, { name: config.resourceNames.applicationStack, - resourceGroupName: resourceGroupResource.name, - accountName: cosmosdbAccountResource.name, + resourceGroupName: resourceGroupName, + accountName: cosmosdbDatabase.name, autoscaleSettings: { maxThroughput: MAX_DATABASE_THROUGHPUT, }, - provider: providerResource, + provider: azureProvider, }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub-namespace.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub-namespace.ts new file mode 100644 index 000000000..4afb9f05c --- /dev/null +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub-namespace.ts @@ -0,0 +1,24 @@ +import { eventhubNamespace } from '@cdktf/provider-azurerm' +import { toTerraformName } from '../helper/utils' +import { ApplicationSynthStack } from '../types/application-synth-stack' + +export class TerraformEventHubNamespace { + static build({ + terraformStack, + azureProvider, + appPrefix, + resourceGroup, + resourceGroupName, + }: ApplicationSynthStack): eventhubNamespace.EventhubNamespace { + const idApiManagement = toTerraformName(appPrefix, 'ehn') + return new eventhubNamespace.EventhubNamespace(terraformStack, idApiManagement, { + name: `${resourceGroupName}ehn`, + location: resourceGroup.location, + resourceGroupName: resourceGroupName, + provider: azureProvider, + sku: 'Basic', // Basic, Standard and Premium. Maximum size of Event Hubs publication: Basic=256K, Standard=1Mb + capacity: 2, // Throughput Units for a Standard SKU namespace. Default capacity has a maximum of 2, but can be increased in blocks of 2 on a committed purchase basis. Defaults to 1. + autoInflateEnabled: false, + }) + } +} diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub.ts new file mode 100644 index 000000000..5721032d1 --- /dev/null +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub.ts @@ -0,0 +1,35 @@ +import { eventhub } from '@cdktf/provider-azurerm' +import { toTerraformName } from '../helper/utils' +import { ApplicationSynthStack } from '../types/application-synth-stack' +import { BoosterConfig } from '@boostercloud/framework-types' + +export class TerraformEventHub { + private static readonly DEFAULT_PARTITION_COUNT = 1 + + private static readonly DEFAULT_MESSAGE_RETENTION = 1 + + static build( + { + terraformStack, + azureProvider, + appPrefix, + resourceGroupName, + eventHubNamespace, + eventHubName, + }: ApplicationSynthStack, + config: BoosterConfig + ): eventhub.Eventhub { + if (!eventHubNamespace) { + throw new Error('Undefined eventHubNamespace resource') + } + const idApiManagement = toTerraformName(appPrefix, 'eh') + return new eventhub.Eventhub(terraformStack, idApiManagement, { + name: eventHubName, + resourceGroupName: resourceGroupName, + provider: azureProvider, + namespaceName: eventHubNamespace.name, + partitionCount: config.eventStreamConfiguration.parameters?.partitionCount ?? this.DEFAULT_PARTITION_COUNT, // Changing this will force-recreate the resource. Cannot be changed unless Eventhub Namespace SKU is Premium + messageRetention: config.eventStreamConfiguration.parameters?.messageRetention ?? this.DEFAULT_MESSAGE_RETENTION, // Specifies the number of days to retain the events for this Event Hub. + }) + } +} diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-function-app.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-function-app.ts index 1e7c88ec0..934adefa8 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-function-app.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-function-app.ts @@ -1,47 +1,65 @@ -import { TerraformStack } from 'cdktf' -import { resourceGroup, servicePlan, storageAccount, webPubsub, windowsFunctionApp } from '@cdktf/provider-azurerm' +import { servicePlan, storageAccount, windowsFunctionApp } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' import { BoosterConfig } from '@boostercloud/framework-types' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { ApplicationSynthStack } from '../types/application-synth-stack' +import { environmentVarNames } from '@boostercloud/framework-provider-azure' export class TerraformFunctionApp { static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - servicePlanResource: servicePlan.ServicePlan, - storageAccountResource: storageAccount.StorageAccount, - appPrefix: string, - functionAppName: string, - cosmosDatabaseName: string, - apiManagementServiceName: string, - cosmosDbConnectionString: string, + { + terraformStack, + azureProvider, + appPrefix, + resourceGroup, + resourceGroupName, + cosmosdbDatabase, + apiManagementName, + eventHubNamespace, + eventHub, + webPubSub, + }: ApplicationSynthStack, config: BoosterConfig, zipFile: string, - webPubsubResource?: webPubsub.WebPubsub + applicationServicePlan: servicePlan.ServicePlan, + storageAccount: storageAccount.StorageAccount, + suffixName: string, + functionAppName: string ): windowsFunctionApp.WindowsFunctionApp { - const id = toTerraformName(appPrefix, 'func') - return new windowsFunctionApp.WindowsFunctionApp(terraformStackResource, id, { + if (!cosmosdbDatabase) { + throw new Error('Undefined cosmosdbDatabase resource') + } + if (!applicationServicePlan) { + throw new Error('Undefined applicationServicePlan resource') + } + const id = toTerraformName(appPrefix, suffixName) + const eventHubConnectionString = + eventHubNamespace?.defaultPrimaryConnectionString && eventHub?.name + ? `${eventHubNamespace.defaultPrimaryConnectionString};EntityPath=${eventHub.name}` + : '' + return new windowsFunctionApp.WindowsFunctionApp(terraformStack, id, { name: functionAppName, - location: resourceGroupResource.location, - resourceGroupName: resourceGroupResource.name, - servicePlanId: servicePlanResource.id, + location: resourceGroup.location, + resourceGroupName: resourceGroupName, + servicePlanId: applicationServicePlan.id, appSettings: { - WebPubSubConnectionString: webPubsubResource?.primaryConnectionString || '', WEBSITE_RUN_FROM_PACKAGE: '1', WEBSITE_CONTENTSHARE: id, ...config.env, + WebPubSubConnectionString: webPubSub?.primaryConnectionString || '', BOOSTER_ENV: config.environmentName, - BOOSTER_REST_API_URL: `https://${apiManagementServiceName}.azure-api.net/${config.environmentName}`, - COSMOSDB_CONNECTION_STRING: `AccountEndpoint=https://${cosmosDatabaseName}.documents.azure.com:443/;AccountKey=${cosmosDbConnectionString};`, + BOOSTER_REST_API_URL: `https://${apiManagementName}.azure-api.net/${config.environmentName}`, + [environmentVarNames.eventHubConnectionString]: eventHubConnectionString, + [environmentVarNames.eventHubName]: config.resourceNames.streamTopic, + COSMOSDB_CONNECTION_STRING: `AccountEndpoint=https://${cosmosdbDatabase.name}.documents.azure.com:443/;AccountKey=${cosmosdbDatabase.primaryKey};`, + WEBSITE_CONTENTAZUREFILECONNECTIONSTRING: storageAccount.primaryConnectionString, // Terraform bug: https://github.com/hashicorp/terraform-provider-azurerm/issues/16650 }, - storageAccountName: storageAccountResource.name, - storageAccountAccessKey: storageAccountResource.primaryAccessKey, - dependsOn: [resourceGroupResource], + storageAccountName: storageAccount.name, + storageAccountAccessKey: storageAccount.primaryAccessKey, + dependsOn: [resourceGroup], lifecycle: { ignoreChanges: ['app_settings["WEBSITE_RUN_FROM_PACKAGE"]'], }, - provider: providerResource, + provider: azureProvider, siteConfig: { applicationStack: { nodeVersion: '~18', diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-outputs.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-outputs.ts index 05255d815..96961118d 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-outputs.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-outputs.ts @@ -1,37 +1,27 @@ -import { TerraformOutput, TerraformStack } from 'cdktf' -import { apiManagementApiOperation, resourceGroup, webPubsub } from '@cdktf/provider-azurerm' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { TerraformOutput } from 'cdktf' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformOutputs { - static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - appPrefix: string, - resourceGroupResource: resourceGroup.ResourceGroup, - graphQLApiManagementApiOperationResource: apiManagementApiOperation.ApiManagementApiOperation, - sensorHealthApiManagementApiOperationResource: apiManagementApiOperation.ApiManagementApiOperation, - hubName: string, - webPubsubResource?: webPubsub.WebPubsub - ): void { + static build(applicationSynthStack: ApplicationSynthStack): void { const environment = process.env.BOOSTER_ENV ?? 'azure' - const baseUrl = `https://${resourceGroupResource.name}apis.azure-api.net/${environment}` + const baseUrl = `https://${applicationSynthStack.resourceGroupName}apis.azure-api.net/${environment}` - new TerraformOutput(providerResource, 'httpURL', { + new TerraformOutput(applicationSynthStack.azureProvider, 'httpURL', { value: baseUrl, description: 'The base URL for all the auth endpoints', }) - new TerraformOutput(providerResource, 'graphqlURL', { - value: baseUrl + graphQLApiManagementApiOperationResource.urlTemplate, + new TerraformOutput(applicationSynthStack.azureProvider, 'graphqlURL', { + value: baseUrl + applicationSynthStack.graphQLApiManagementApiOperation?.urlTemplate, description: 'The base URL for sending GraphQL mutations and queries', }) - new TerraformOutput(providerResource, 'sensorHealthURL', { - value: baseUrl + sensorHealthApiManagementApiOperationResource.urlTemplate, + new TerraformOutput(applicationSynthStack.azureProvider, 'sensorHealthURL', { + value: baseUrl + applicationSynthStack.sensorHealthApiManagementApiOperationResource.urlTemplate, description: 'The base URL for getting health information', }) - if (webPubsubResource) { - new TerraformOutput(providerResource, 'websocketURL', { - value: `wss://${webPubsubResource.hostname}/client/hubs/${hubName}`, + if (applicationSynthStack.webPubSub) { + new TerraformOutput(applicationSynthStack.azureProvider, 'websocketURL', { + value: `wss://${applicationSynthStack.webPubSub.hostname}/client/hubs/${applicationSynthStack.eventHubName}`, description: 'The URL for the websocket communication. Used for subscriptions', }) } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-resource-group.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-resource-group.ts index fd759d767..3cbd8acba 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-resource-group.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-resource-group.ts @@ -1,20 +1,19 @@ import { resourceGroup } from '@cdktf/provider-azurerm' -import { TerraformStack } from 'cdktf' import { getDeployRegion, toTerraformName } from '../helper/utils' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { StackNames } from '../types/application-synth-stack' export class TerraformResourceGroup { - static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - appPrefix: string, - resourceGroupName: string - ): resourceGroup.ResourceGroup { + static build({ + terraformStack, + resourceGroupName, + azureProvider, + appPrefix, + }: StackNames): resourceGroup.ResourceGroup { const id = toTerraformName(appPrefix, 'rg') - return new resourceGroup.ResourceGroup(terraformStackResource, id, { + return new resourceGroup.ResourceGroup(terraformStack, id, { name: resourceGroupName, location: getDeployRegion(), - provider: providerResource, + provider: azureProvider, }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-service-plan.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-service-plan.ts index 86d88c9ff..8d9a227b4 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-service-plan.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-service-plan.ts @@ -1,24 +1,23 @@ -import { TerraformStack } from 'cdktf' -import { servicePlan, resourceGroup } from '@cdktf/provider-azurerm' +import { servicePlan } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformServicePlan { static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - group: resourceGroup.ResourceGroup, - appPrefix: string, - resourceGroupName: string + { terraformStack, azureProvider, appPrefix, resourceGroupName, resourceGroup }: ApplicationSynthStack, + suffixName: string, + skuName: string, + workerCount: number ): servicePlan.ServicePlan { - const id = toTerraformName(appPrefix, 'hpn') - return new servicePlan.ServicePlan(terraformStackResource, id, { - name: `${resourceGroupName}hpn`, - location: group.location, - resourceGroupName: group.name, + const id = toTerraformName(appPrefix, suffixName) + return new servicePlan.ServicePlan(terraformStack, id, { + name: `${resourceGroupName}${suffixName}`, + location: resourceGroup.location, + resourceGroupName: resourceGroupName, osType: 'Windows', - skuName: 'Y1', - provider: providerResource, + skuName: skuName, + workerCount: workerCount, + provider: azureProvider, }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-storage-account.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-storage-account.ts index 99d27ecef..7b7c730d1 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-storage-account.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-storage-account.ts @@ -1,24 +1,20 @@ -import { TerraformStack } from 'cdktf' -import { resourceGroup, storageAccount } from '@cdktf/provider-azurerm' +import { storageAccount } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformStorageAccount { static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - appPrefix: string, - resourceGroupName: string + { terraformStack, azureProvider, appPrefix, resourceGroupName, resourceGroup }: ApplicationSynthStack, + suffixName: string ): storageAccount.StorageAccount { - const id = toTerraformName(appPrefix, 'st') - return new storageAccount.StorageAccount(terraformStackResource, id, { - name: `${resourceGroupName}sa`, - resourceGroupName: resourceGroupResource.name, - location: resourceGroupResource.location, + const id = toTerraformName(appPrefix, suffixName) + return new storageAccount.StorageAccount(terraformStack, id, { + name: `${resourceGroupName}${suffixName}`, + resourceGroupName: resourceGroupName, + location: resourceGroup.location, accountReplicationType: 'LRS', accountTier: 'Standard', - provider: providerResource, + provider: azureProvider, }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pub-sub-extension-key.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pub-sub-extension-key.ts index c70f52914..e9af54fba 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pub-sub-extension-key.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pub-sub-extension-key.ts @@ -1,29 +1,29 @@ -import { TerraformStack } from 'cdktf' -import { dataAzurermFunctionAppHostKeys, resourceGroup, windowsFunctionApp } from '@cdktf/provider-azurerm' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' -import { BoosterConfig } from '@boostercloud/framework-types' +import { dataAzurermFunctionAppHostKeys } from '@cdktf/provider-azurerm' import { TerraformFunctionAppData } from './web-pubsub-extension-key/terraform-function-app-data' import { TerraformSleep } from './web-pubsub-extension-key/terraform-sleep' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformWebPubSubExtensionKey { - static build( - config: BoosterConfig, - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - functionAppResource: windowsFunctionApp.WindowsFunctionApp, - appPrefix: string - ): dataAzurermFunctionAppHostKeys.DataAzurermFunctionAppHostKeys { + static build({ + terraformStack, + azureProvider, + appPrefix, + resourceGroup, + functionApp, + }: ApplicationSynthStack): dataAzurermFunctionAppHostKeys.DataAzurermFunctionAppHostKeys { + if (!functionApp) { + throw new Error('Undefined functionApp resource') + } // Wait for x minutes to give time to Azure to create the webpubsub_extension System Key. // Terraform doesn't provide a way to trigger it when the system key is updated - const sleepResource = TerraformSleep.build(terraformStackResource, appPrefix, [functionAppResource]) + const sleepResource = TerraformSleep.build(terraformStack, appPrefix, [functionApp]) // Return the correct system key created by Azure return TerraformFunctionAppData.build( - providerResource, - terraformStackResource, - resourceGroupResource, - functionAppResource, + azureProvider, + terraformStack, + resourceGroup, + functionApp, appPrefix, sleepResource ) diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pubsub-hub.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pubsub-hub.ts index 83111a385..968d005f7 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pubsub-hub.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pubsub-hub.ts @@ -1,40 +1,41 @@ -import { TerraformStack } from 'cdktf' import { toTerraformName } from '../helper/utils' -import { - dataAzurermFunctionAppHostKeys, - resourceGroup, - webPubsub, - webPubsubHub, - windowsFunctionApp, -} from '@cdktf/provider-azurerm' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { webPubsubHub } from '@cdktf/provider-azurerm' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformWebPubsubHub { - static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - webPubSubResource: webPubsub.WebPubsub, - appPrefix: string, - functionApp: windowsFunctionApp.WindowsFunctionApp, - dataAzurermFunctionAppHostKeys: dataAzurermFunctionAppHostKeys.DataAzurermFunctionAppHostKeys, - hubName: string - ): webPubsubHub.WebPubsubHub { + static build({ + terraformStack, + azureProvider, + appPrefix, + webPubSub, + functionApp, + dataFunctionAppHostKeys, + webPubSubHubName, + }: ApplicationSynthStack): webPubsubHub.WebPubsubHub { + if (!webPubSub) { + throw new Error('Undefined webPubSub resource') + } + if (!functionApp) { + throw new Error('Undefined functionApp resource') + } + if (!dataFunctionAppHostKeys) { + throw new Error('Undefined dataFunctionAppHostKeys resource') + } const idApiManagement = toTerraformName(appPrefix, 'wpsh') - - return new webPubsubHub.WebPubsubHub(terraformStackResource, idApiManagement, { - name: hubName, - webPubsubId: webPubSubResource.id, + const name = webPubSubHubName + return new webPubsubHub.WebPubsubHub(terraformStack, idApiManagement, { + name: name, + webPubsubId: webPubSub.id, eventHandler: [ { - urlTemplate: `https://${functionApp.name}.azurewebsites.net/runtime/webhooks/webpubsub?code=${dataAzurermFunctionAppHostKeys.webpubsubExtensionKey}`, + urlTemplate: `https://${functionApp.name}.azurewebsites.net/runtime/webhooks/webpubsub?code=${dataFunctionAppHostKeys.webpubsubExtensionKey}`, systemEvents: ['connect', 'disconnected'], userEventPattern: '*', }, ], anonymousConnectionsEnabled: true, - dependsOn: [functionApp, dataAzurermFunctionAppHostKeys, webPubSubResource], - provider: providerResource, + dependsOn: [functionApp, dataFunctionAppHostKeys, webPubSub], + provider: azureProvider, }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pubsub.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pubsub.ts index d54c40cd2..a579e7d01 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pubsub.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-web-pubsub.ts @@ -1,24 +1,24 @@ -import { TerraformStack } from 'cdktf' import { toTerraformName } from '../helper/utils' -import { resourceGroup, webPubsub } from '@cdktf/provider-azurerm' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { webPubsub } from '@cdktf/provider-azurerm' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformWebPubsub { - static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - resourceGroupResource: resourceGroup.ResourceGroup, - appPrefix: string - ): webPubsub.WebPubsub { + static build({ + terraformStack, + azureProvider, + appPrefix, + resourceGroupName, + resourceGroup, + }: ApplicationSynthStack): webPubsub.WebPubsub { const id = toTerraformName(appPrefix, 'wps') - return new webPubsub.WebPubsub(terraformStackResource, id, { - name: `${resourceGroupResource.name}wps`, - location: resourceGroupResource.location, - resourceGroupName: resourceGroupResource.name, + return new webPubsub.WebPubsub(terraformStack, id, { + name: `${resourceGroupName}wps`, + location: resourceGroup.location, + resourceGroupName: resourceGroupName, sku: 'Free_F1', capacity: 1, - provider: providerResource, + provider: azureProvider, }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/types/application-synth-stack.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/types/application-synth-stack.ts index 62464cd87..0f9d848d5 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/types/application-synth-stack.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/types/application-synth-stack.ts @@ -6,41 +6,55 @@ import { cosmosdbAccount, cosmosdbSqlContainer, cosmosdbSqlDatabase, + dataAzurermFunctionAppHostKeys, + eventhub, + eventhubNamespace, resourceGroup, servicePlan, storageAccount, webPubsub, webPubsubHub, - windowsFunctionApp, + windowsFunctionApp } from '@cdktf/provider-azurerm' import { TerraformResource, TerraformStack } from 'cdktf' import { FunctionDefinition } from './functionDefinition' import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' -export interface ApplicationSynthStack { +export interface StackNames { appPrefix: string terraformStack: TerraformStack - resourceGroupName: string | undefined - functionAppName: string | undefined - apiManagementName: string | undefined - resourceGroup: resourceGroup.ResourceGroup | undefined - applicationServicePlan: servicePlan.ServicePlan | undefined - storageAccount: storageAccount.StorageAccount | undefined - functionApp: windowsFunctionApp.WindowsFunctionApp | undefined - apiManagement: apiManagement.ApiManagement | undefined - apiManagementApi: apiManagementApi.ApiManagementApi | undefined - graphQLApiManagementApiOperation: apiManagementApiOperation.ApiManagementApiOperation | undefined - graphQLApiManagementApiOperationPolicy: apiManagementApiOperationPolicy.ApiManagementApiOperationPolicy | undefined - sensorHealthApiManagementApiOperation: apiManagementApiOperation.ApiManagementApiOperation | undefined - sensorHealthApiManagementApiOperationPolicy: - | apiManagementApiOperationPolicy.ApiManagementApiOperationPolicy - | undefined - cosmosdbDatabase: cosmosdbAccount.CosmosdbAccount | undefined - cosmosdbSqlDatabase: cosmosdbSqlDatabase.CosmosdbSqlDatabase | undefined - containers: Array | undefined - webPubSub: webPubsub.WebPubsub | undefined - webPubSubHub: webPubsubHub.WebPubsubHub | undefined - azureProvider: AzurermProvider | undefined + azureProvider: AzurermProvider + resourceGroupName: string + functionAppName: string + streamFunctionAppName: string + apiManagementName: string + eventHubName: string + webPubSubHubName: string +} + +export interface ApplicationSynthStack extends StackNames { + resourceGroup: resourceGroup.ResourceGroup + applicationServicePlan?: servicePlan.ServicePlan + storageAccount?: storageAccount.StorageAccount + functionApp?: windowsFunctionApp.WindowsFunctionApp + eventConsumerServicePlan?: servicePlan.ServicePlan + eventConsumerStorageAccount?: storageAccount.StorageAccount + eventConsumerFunctionApp?: windowsFunctionApp.WindowsFunctionApp + dataFunctionAppHostKeys?: dataAzurermFunctionAppHostKeys.DataAzurermFunctionAppHostKeys + apiManagement?: apiManagement.ApiManagement + apiManagementApi?: apiManagementApi.ApiManagementApi + graphQLApiManagementApiOperation?: apiManagementApiOperation.ApiManagementApiOperation + graphQLApiManagementApiOperationPolicy?: apiManagementApiOperationPolicy.ApiManagementApiOperationPolicy + sensorHealthApiManagementApiOperation?: apiManagementApiOperation.ApiManagementApiOperation + sensorHealthApiManagementApiOperationPolicy?: apiManagementApiOperationPolicy.ApiManagementApiOperationPolicy + cosmosdbDatabase?: cosmosdbAccount.CosmosdbAccount + cosmosdbSqlDatabase?: cosmosdbSqlDatabase.CosmosdbSqlDatabase + containers?: Array + webPubSub?: webPubsub.WebPubsub + webPubSubHub?: webPubsubHub.WebPubsubHub functionDefinitions?: Array + consumerFunctionDefinitions?: Array + eventHubNamespace?: eventhubNamespace.EventhubNamespace + eventHub?: eventhub.Eventhub rocketStack?: Array } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/types/functionDefinition.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/types/functionDefinition.ts index 8aef49720..a03a8afa1 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/types/functionDefinition.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/types/functionDefinition.ts @@ -28,6 +28,19 @@ export type EventHandlerBinding = Binding & { [key: string]: any } +export type EventHubInputBinding = Binding & { + eventHubName: string + connection: string + cardinality: string + consumerGroup: string + dataType: string +} + +export type EventHubOutBinding = Binding & { + connection: string + eventHubName: string +} + export type SubscriptionBinding = Binding & { hub: string direction: 'out' @@ -57,6 +70,10 @@ export type HttpFunctionDefinition = FunctionDefinition export type EventHandlerFunctionDefinition = FunctionDefinition +export type EventStreamProducerHandlerFunctionDefinition = FunctionDefinition + +export type EventStreamConsumerHandlerFunctionDefinition = FunctionDefinition + export type SubscriptionsNotifierFunctionDefinition = FunctionDefinition export type SocketsFunctionDefinition = FunctionDefinition diff --git a/packages/framework-provider-azure-infrastructure/test/infrastructure/functions/event-handler-functions.test.ts b/packages/framework-provider-azure-infrastructure/test/infrastructure/functions/event-handler-functions.test.ts index ab868a983..b37c76adf 100644 --- a/packages/framework-provider-azure-infrastructure/test/infrastructure/functions/event-handler-functions.test.ts +++ b/packages/framework-provider-azure-infrastructure/test/infrastructure/functions/event-handler-functions.test.ts @@ -2,6 +2,7 @@ import { BoosterConfig } from '@boostercloud/framework-types' import { describe } from 'mocha' import { EventHandlerFunction } from '../../../src/infrastructure/functions/event-handler-function' import { expect } from '../../expect' +import { EventHandlerFunctionDefinition } from '../../../src' describe('Creating event-handler-functions', () => { const config = new BoosterConfig('test') @@ -9,7 +10,7 @@ describe('Creating event-handler-functions', () => { config.resourceNames.eventsStore = 'eventsStore' it('create the expected EventHandlerFunctionDefinition', () => { - const definition = new EventHandlerFunction(config).getFunctionDefinition() + const definition: EventHandlerFunctionDefinition = new EventHandlerFunction(config).getFunctionDefinition() expect(definition).not.to.be.null expect(definition.name).to.be.equal('eventHandler') expect(definition.config.bindings[0].type).to.be.equal('cosmosDBTrigger') diff --git a/packages/framework-provider-azure/package.json b/packages/framework-provider-azure/package.json index b8c85c9bb..100f12343 100644 --- a/packages/framework-provider-azure/package.json +++ b/packages/framework-provider-azure/package.json @@ -26,6 +26,7 @@ "@azure/cosmos": "^4.0.0", "@azure/functions": "^1.2.2", "@azure/identity": "~2.1.0", + "@azure/event-hubs": "5.11.1", "@boostercloud/framework-common-helpers": "workspace:^2.1.0", "@boostercloud/framework-types": "workspace:^2.1.0", "tslib": "^2.4.0", diff --git a/packages/framework-provider-azure/src/constants.ts b/packages/framework-provider-azure/src/constants.ts index d6ee339bb..a97bbf51f 100644 --- a/packages/framework-provider-azure/src/constants.ts +++ b/packages/framework-provider-azure/src/constants.ts @@ -19,10 +19,17 @@ export const connectionsStoreAttributes = { ttl: 'expirationTime', } as const +export const dedupAttributes = { + partitionKey: 'primaryKey', + ttl: 'expirationTime', +} as const + export const environmentVarNames = { restAPIURL: 'BOOSTER_REST_API_URL', websocketAPIURL: 'BOOSTER_WEBSOCKET_API_URL', cosmosDbConnectionString: 'COSMOSDB_CONNECTION_STRING', + eventHubConnectionString: 'EVENTHUB_CONNECTION_STRING', + eventHubName: 'EVENTHUB_NAME', } as const // Azure special error codes diff --git a/packages/framework-provider-azure/src/index.ts b/packages/framework-provider-azure/src/index.ts index 54c6ac74a..ace9334a8 100644 --- a/packages/framework-provider-azure/src/index.ts +++ b/packages/framework-provider-azure/src/index.ts @@ -33,6 +33,9 @@ import { storeConnectionData, } from './library/connections-adapter' import { rawRocketInputToEnvelope } from './library/rocket-adapter' +import { produceEventsStream } from './library/events-stream-producer-adapter' +import { EventHubProducerClient, RetryMode } from '@azure/event-hubs' +import { dedupEventStream, rawEventsStreamToEnvelopes } from './library/events-stream-consumer-adapter' import { areDatabaseReadModelsUp, databaseUrl, @@ -51,6 +54,25 @@ if (typeof process.env[environmentVarNames.cosmosDbConnectionString] === 'undefi cosmosClient = new CosmosClient(process.env[environmentVarNames.cosmosDbConnectionString] as string) } +let producer: EventHubProducerClient +const eventHubConnectionString = process.env[environmentVarNames.eventHubConnectionString] +const eventHubName = process.env[environmentVarNames.eventHubName] +if ( + typeof eventHubConnectionString === 'undefined' || + typeof eventHubName === 'undefined' || + eventHubConnectionString === '' || + eventHubName === '' +) { + producer = {} as any +} else { + producer = new EventHubProducerClient(eventHubConnectionString, eventHubName, { + retryOptions: { + maxRetries: 5, + mode: RetryMode.Exponential, + }, + }) +} + /* We load the infrastructure package dynamically here to avoid including it in the * dependencies that are deployed in the lambda functions. The infrastructure * package is only used during the deploy. @@ -63,6 +85,9 @@ export const Provider = (rockets?: RocketDescriptor[]): ProviderLibrary => ({ // ProviderEventsLibrary events: { rawToEnvelopes: rawEventsToEnvelopes, + rawStreamToEnvelopes: rawEventsStreamToEnvelopes, + dedupEventStream: dedupEventStream.bind(null, cosmosClient), + produce: produceEventsStream.bind(null, producer), store: storeEvents.bind(null, cosmosClient), storeSnapshot: storeSnapshot.bind(null, cosmosClient), forEntitySince: readEntityEventsSince.bind(null, cosmosClient), diff --git a/packages/framework-provider-azure/src/library/events-stream-consumer-adapter.ts b/packages/framework-provider-azure/src/library/events-stream-consumer-adapter.ts new file mode 100644 index 000000000..8ad8b8b0e --- /dev/null +++ b/packages/framework-provider-azure/src/library/events-stream-consumer-adapter.ts @@ -0,0 +1,65 @@ +import { BoosterConfig, EventEnvelope, EventStream } from '@boostercloud/framework-types' +import { Context } from '@azure/functions' +import { getLogger } from '@boostercloud/framework-common-helpers' +import { AZURE_CONFLICT_ERROR_CODE } from '../constants' +import { CosmosClient } from '@azure/cosmos' + +interface DedupEventStream { + primaryKey: string + createdAt: string + ttl: number +} + +export async function dedupEventStream( + cosmosDb: CosmosClient, + config: BoosterConfig, + context: Context +): Promise { + const logger = getLogger(config, 'dedup-events-stream#dedupEventsStream') + const events = (context.bindings.eventHubMessages as EventStream) || [] + logger.debug(`Dedup ${events.length} events`) + + const resources: EventStream = [] + for (const event of events) { + const rawParsed = JSON.parse(event as string) + const eventTag: DedupEventStream = { + primaryKey: rawParsed._etag, + createdAt: new Date().toISOString(), + ttl: config.eventStreamConfiguration.parameters?.dedupTtl ?? 86400, + } + try { + const { resource } = await cosmosDb + .database(config.resourceNames.applicationStack) + .container(config.resourceNames.eventsDedup) + .items.create(eventTag) + if (resource) { + resources.push(event) + } + } catch (error) { + if (error.code !== AZURE_CONFLICT_ERROR_CODE) { + throw error + } + logger.warn(`Ignoring duplicated event with etag ${eventTag}.`) + } + } + return resources +} +export function rawEventsStreamToEnvelopes( + config: BoosterConfig, + context: Context, + dedupEventStream: EventStream +): Array { + const logger = getLogger(config, 'events-adapter#rawEventsStreamToEnvelopes') + logger.debug(`Mapping ${dedupEventStream.length} events`) + const bindingData = context.bindingData + return dedupEventStream.map((message, index) => { + const rawParsed = JSON.parse(message as string) + const instance = process.env.WEBSITE_INSTANCE_ID + const partitionKeyArrayElement = bindingData.partitionKeyArray[index] + const offset = bindingData.offsetArray[index] + const sequence = bindingData.sequenceNumberArray[index] + const time = bindingData.enqueuedTimeUtcArray[index] + logger.debug(`CONSUMED_EVENT:${instance}#${partitionKeyArrayElement}=>(${index}#${offset}#${sequence}#${time})`) + return rawParsed as EventEnvelope + }) +} diff --git a/packages/framework-provider-azure/src/library/events-stream-producer-adapter.ts b/packages/framework-provider-azure/src/library/events-stream-producer-adapter.ts new file mode 100644 index 000000000..da8633782 --- /dev/null +++ b/packages/framework-provider-azure/src/library/events-stream-producer-adapter.ts @@ -0,0 +1,56 @@ +import { BoosterConfig, EventEnvelope } from '@boostercloud/framework-types' +import { CreateBatchOptions, EventHubProducerClient } from '@azure/event-hubs' +import { getLogger } from '@boostercloud/framework-common-helpers' +import { partitionKeyForEvent } from './partition-keys' + +export async function produceEventsStream( + producer: EventHubProducerClient, + entityName: string, + entityID: string, + eventEnvelopes: Array, + config: BoosterConfig +): Promise { + const logger = getLogger(config, 'events-stream-producer#produceEventsStream') + logger.debug('Producing eventEnvelopes', eventEnvelopes) + const batchOptions: CreateBatchOptions = { + partitionKey: partitionKeyForEvent(entityName, entityID), + } + + let batch = await producer.createBatch(batchOptions) + let numEventsSent = 0 + let i = 0 + while (i < eventEnvelopes.length) { + // messages can fail to be added to the batch if they exceed the maximum size configured for the EventHub. + const eventEnvelope = eventEnvelopes[i] + const isAdded = batch.tryAdd({ body: eventEnvelope }) + + if (isAdded) { + logger.info(`Added ${JSON.stringify(eventEnvelope)} to the batch`) + ++i + continue + } + + if (batch.count === 0) { + throw new Error(`Message was too large and can't be sent until it's made smaller. ${eventEnvelope}`) + } + + // We reached the batch size limit + logger.info(`Sending ${batch.count} messages`) + await producer.sendBatch(batch) + numEventsSent += batch.count + + batch = await producer.createBatch(batchOptions) + } + + if (batch.count > 0) { + logger.info(`Sending remaining ${batch.count} messages`) + await producer.sendBatch(batch) + numEventsSent += batch.count + } + + logger.info(`Sent ${numEventsSent} events`) + + if (numEventsSent !== eventEnvelopes.length) { + throw new Error(`Not all messages were sent (${numEventsSent}/${eventEnvelopes.length})`) + } +} diff --git a/packages/framework-provider-local/src/index.ts b/packages/framework-provider-local/src/index.ts index df800ca3a..e90b0a530 100644 --- a/packages/framework-provider-local/src/index.ts +++ b/packages/framework-provider-local/src/index.ts @@ -71,6 +71,9 @@ export const Provider = (rocketDescriptors?: RocketDescriptor[]): ProviderLibrar // ProviderEventsLibrary events: { rawToEnvelopes: rawEventsToEnvelopes, + rawStreamToEnvelopes: notImplemented as any, + dedupEventStream: notImplemented as any, + produce: notImplemented as any, forEntitySince: readEntityEventsSince.bind(null, eventRegistry), latestEntitySnapshot: readEntityLatestSnapshot.bind(null, eventRegistry), store: storeEvents.bind(null, userApp, eventRegistry), @@ -141,3 +144,5 @@ export const Provider = (rocketDescriptors?: RocketDescriptor[]): ProviderLibrar return infrastructure.Infrastructure(rocketDescriptors) }, }) + +function notImplemented(): void {} diff --git a/packages/framework-types/src/concepts/event-stream-configuration.ts b/packages/framework-types/src/concepts/event-stream-configuration.ts new file mode 100644 index 000000000..e2468dd64 --- /dev/null +++ b/packages/framework-types/src/concepts/event-stream-configuration.ts @@ -0,0 +1,14 @@ +export type EventStreamConfiguration = + | { + enabled: true + parameters: { + streamTopic: string + partitionCount: number + messageRetention: number // Specifies the number of days to retain the events for this Event Hub. + dedupTtl?: number // Time to live in seconds + } + } + | { + enabled: false + parameters?: never + } diff --git a/packages/framework-types/src/concepts/index.ts b/packages/framework-types/src/concepts/index.ts index 3900698cd..f4a501ec0 100644 --- a/packages/framework-types/src/concepts/index.ts +++ b/packages/framework-types/src/concepts/index.ts @@ -18,3 +18,4 @@ export * from './scheduled-command' export * from './sequence-metadata' export * from './token-verifier' export * from './uuid' +export * from './event-stream-configuration' diff --git a/packages/framework-types/src/config.ts b/packages/framework-types/src/config.ts index 56a5678ba..6e624de23 100644 --- a/packages/framework-types/src/config.ts +++ b/packages/framework-types/src/config.ts @@ -15,6 +15,7 @@ import { RoleMetadata, ScheduledCommandMetadata, SchemaMigrationMetadata, + EventStreamConfiguration, } from './concepts' import { ProviderLibrary } from './provider' import { Level } from './logger' @@ -45,6 +46,8 @@ export class BoosterConfig { private _userProjectRootPath?: string public readonly codeRelativePath: string = 'dist' public readonly eventDispatcherHandler: string = path.join(this.codeRelativePath, 'index.boosterEventDispatcher') + public readonly eventStreamConsumer: string = path.join(this.codeRelativePath, 'index.boosterConsumeEventStream') + public readonly eventStreamProducer: string = path.join(this.codeRelativePath, 'index.boosterProduceEventStream') public readonly serveGraphQLHandler: string = path.join(this.codeRelativePath, 'index.boosterServeGraphQL') public readonly sensorHealthHandler: string = path.join(this.codeRelativePath, 'index.boosterHealth') public readonly scheduledTaskHandler: string = path.join( @@ -106,6 +109,8 @@ export class BoosterConfig { onEnd: async (): Promise => {}, } + public eventStreamConfiguration: EventStreamConfiguration = { enabled: false } + /** Environment variables set at deployment time on the target lambda functions */ public readonly env: Record = {} @@ -124,8 +129,10 @@ export class BoosterConfig { return { applicationStack: applicationStackName, eventsStore: applicationStackName + '-events-store', + eventsDedup: applicationStackName + '-events-dedup', subscriptionsStore: applicationStackName + '-subscriptions-store', connectionsStore: applicationStackName + '-connections-store', + streamTopic: this.eventStreamConfiguration.parameters?.streamTopic ?? 'booster_events', forReadModel(readModelName: string): string { return applicationStackName + '-' + readModelName }, @@ -227,8 +234,10 @@ export class BoosterConfig { interface ResourceNames { applicationStack: string eventsStore: string + eventsDedup: string subscriptionsStore: string connectionsStore: string + streamTopic: string forReadModel(entityName: string): string } diff --git a/packages/framework-types/src/index.ts b/packages/framework-types/src/index.ts index 0f3dc7091..1eeacd119 100644 --- a/packages/framework-types/src/index.ts +++ b/packages/framework-types/src/index.ts @@ -17,3 +17,4 @@ export * from './super-kind' export * from './instrumentation/trace-types' export * from './sensor/health-indicator-configuration' export * from './internal-info' +export * from './stream-types' diff --git a/packages/framework-types/src/instrumentation/trace-types.ts b/packages/framework-types/src/instrumentation/trace-types.ts index 4b7dc6dc4..5121cc9a3 100644 --- a/packages/framework-types/src/instrumentation/trace-types.ts +++ b/packages/framework-types/src/instrumentation/trace-types.ts @@ -12,6 +12,8 @@ export enum TraceActionTypes { HANDLE_EVENT = 'HANDLE_EVENT', DISPATCH_ENTITY_TO_EVENT_HANDLERS = 'DISPATCH_ENTITY_TO_EVENT_HANDLERS', DISPATCH_EVENTS = 'DISPATCH_EVENTS', + CONSUME_STREAM_EVENTS = 'CONSUME_STREAM_EVENTS', + PRODUCE_STREAM_EVENTS = 'PRODUCE_STREAM_EVENTS', FETCH_ENTITY_SNAPSHOT = 'FETCH_ENTITY_SNAPSHOT', STORE_SNAPSHOT = 'STORE_SNAPSHOT', LOAD_LATEST_SNAPSHOT = 'LOAD_LATEST_SNAPSHOT', diff --git a/packages/framework-types/src/provider.ts b/packages/framework-types/src/provider.ts index 8f44b9082..52017fff4 100644 --- a/packages/framework-types/src/provider.ts +++ b/packages/framework-types/src/provider.ts @@ -20,6 +20,7 @@ import { import { FilterFor, SortFor } from './searcher' import { ReadOnlyNonEmptyArray } from './typelevel' import { RocketDescriptor, RocketEnvelope } from './rockets' +import { EventStream } from './stream-types' export interface ProviderLibrary { events: ProviderEventsLibrary @@ -57,6 +58,17 @@ export interface ProviderEventsLibrary { */ rawToEnvelopes(rawEvents: unknown): Array + rawStreamToEnvelopes(config: BoosterConfig, context: unknown, dedupEventStream: EventStream): Array + + dedupEventStream(config: BoosterConfig, rawEvents: unknown): Promise + + produce( + entityName: string, + entityID: UUID, + eventEnvelopes: Array, + config: BoosterConfig + ): Promise + /** * Retrieves events for a specific entity since a given time * diff --git a/packages/framework-types/src/stream-types.ts b/packages/framework-types/src/stream-types.ts new file mode 100644 index 000000000..aa91bc68e --- /dev/null +++ b/packages/framework-types/src/stream-types.ts @@ -0,0 +1 @@ +export type EventStream = Array diff --git a/website/docs/10_going-deeper/azure-scale.mdx b/website/docs/10_going-deeper/azure-scale.mdx new file mode 100644 index 000000000..7bf891176 --- /dev/null +++ b/website/docs/10_going-deeper/azure-scale.mdx @@ -0,0 +1,42 @@ +import DocCardList from '@theme/DocCardList' + +# Scaling Booster Azure Functions + +Booster Azure Provider relies on CosmosDB [change feed processor](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/change-feed-processor) to consume new events. +In CosmosDB, the partition keys are distributed in ranges, where each range represents a physical partition. [Unlike logical partitions, +physical partitions are an internal implementation of the system and Azure Cosmos DB entirely manages physical partitions](https://learn.microsoft.com/en-us/azure/cosmos-db/partitioning-overview#physical-partitions) + +With Booster EventStream functionality, we could define the number of physical partitions our events are split and create instances for each partition. + +To enable EventStream, set the `EventStreamConfiguration` in the configuration object: + +> **Warning**: Currently, only available for Azure provider. + +```typescript + config.eventStreamConfiguration = { + enabled: true, + parameters: { + streamTopic: 'test', + partitionCount: 3, + messageRetention: 1, + }, + } +``` + +## Parameters +* StreamTopic: Define the internal topic name Booster will use. +* PartitionCount: Number of Event Hub partitions. The number of functions app consumer instances will match the partition count +* MessageRetention: Specifies the number of days to retain the events for this Event Hub + +## Infrastructure + +Enabling `EventStreamConfiguration` will apply some changes to the infrastructure: + +* Two functions will be created + * One function with a CosmosDB consumer that will produce Event Hubs events. Also, it will include the readModels functions, schedule functions app, etc... + * One function with an Event Hub consumer function app. This function will allow you to define the number of instances to be created +* A new container to handle duplicated consumed events +* A new Event Hub will be added for event handling. + + + diff --git a/website/sidebars.js b/website/sidebars.js index d031a921b..335f534cf 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -113,6 +113,7 @@ const sidebars = { 'going-deeper/custom-templates', 'going-deeper/framework-packages', 'going-deeper/instrumentation', + 'going-deeper/azure-scale' ], }, 'frequently-asked-questions', From 8a81ee5d070ee36cfbdd4c2024eb66f7f77ac6f6 Mon Sep 17 00:00:00 2001 From: gonzalojaubert Date: Wed, 4 Oct 2023 09:37:19 +0100 Subject: [PATCH 2/8] rush change --- .../azure_event_hub_2023-10-04-08-37.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 common/changes/@boostercloud/framework-core/azure_event_hub_2023-10-04-08-37.json diff --git a/common/changes/@boostercloud/framework-core/azure_event_hub_2023-10-04-08-37.json b/common/changes/@boostercloud/framework-core/azure_event_hub_2023-10-04-08-37.json new file mode 100644 index 000000000..e98c30e7a --- /dev/null +++ b/common/changes/@boostercloud/framework-core/azure_event_hub_2023-10-04-08-37.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@boostercloud/framework-core", + "comment": "Add Azure Event Hub", + "type": "minor" + } + ], + "packageName": "@boostercloud/framework-core" +} \ No newline at end of file From cb9f16c1c829d1efe3879d1ef538a09134fec190 Mon Sep 17 00:00:00 2001 From: gonzalojaubert Date: Wed, 18 Oct 2023 13:23:54 +0100 Subject: [PATCH 3/8] fix cli integration test --- .../integration/fixtures/cart-demo/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/framework-integration-tests/integration/fixtures/cart-demo/src/index.ts b/packages/framework-integration-tests/integration/fixtures/cart-demo/src/index.ts index 8e660a0e9..d0c444a65 100644 --- a/packages/framework-integration-tests/integration/fixtures/cart-demo/src/index.ts +++ b/packages/framework-integration-tests/integration/fixtures/cart-demo/src/index.ts @@ -2,6 +2,8 @@ import { Booster } from '@boostercloud/framework-core' export { Booster, boosterEventDispatcher, + boosterProduceEventStream, + boosterConsumeEventStream, boosterServeGraphQL, boosterNotifySubscribers, boosterHealth, From 48204bf73d9a8566d9c3198a6311881c15284dfc Mon Sep 17 00:00:00 2001 From: gonzalojaubert Date: Mon, 13 Nov 2023 10:20:34 +0000 Subject: [PATCH 4/8] refactor --- .../src/infrastructure/synth/terraform-event-hub.ts | 2 +- .../src/library/events-stream-consumer-adapter.ts | 4 +++- .../src/library/events-stream-producer-adapter.ts | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub.ts index 5721032d1..6f0c80fe4 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-event-hub.ts @@ -28,7 +28,7 @@ export class TerraformEventHub { resourceGroupName: resourceGroupName, provider: azureProvider, namespaceName: eventHubNamespace.name, - partitionCount: config.eventStreamConfiguration.parameters?.partitionCount ?? this.DEFAULT_PARTITION_COUNT, // Changing this will force-recreate the resource. Cannot be changed unless Eventhub Namespace SKU is Premium + partitionCount: config.eventStreamConfiguration.parameters?.partitionCount ?? this.DEFAULT_PARTITION_COUNT, // Cannot be changed unless Eventhub Namespace SKU is Premium messageRetention: config.eventStreamConfiguration.parameters?.messageRetention ?? this.DEFAULT_MESSAGE_RETENTION, // Specifies the number of days to retain the events for this Event Hub. }) } diff --git a/packages/framework-provider-azure/src/library/events-stream-consumer-adapter.ts b/packages/framework-provider-azure/src/library/events-stream-consumer-adapter.ts index 8ad8b8b0e..65c4a9d60 100644 --- a/packages/framework-provider-azure/src/library/events-stream-consumer-adapter.ts +++ b/packages/framework-provider-azure/src/library/events-stream-consumer-adapter.ts @@ -10,6 +10,8 @@ interface DedupEventStream { ttl: number } +const DEFAULT_DEDUP_TTL = 86400 + export async function dedupEventStream( cosmosDb: CosmosClient, config: BoosterConfig, @@ -25,7 +27,7 @@ export async function dedupEventStream( const eventTag: DedupEventStream = { primaryKey: rawParsed._etag, createdAt: new Date().toISOString(), - ttl: config.eventStreamConfiguration.parameters?.dedupTtl ?? 86400, + ttl: config.eventStreamConfiguration.parameters?.dedupTtl ?? DEFAULT_DEDUP_TTL, } try { const { resource } = await cosmosDb diff --git a/packages/framework-provider-azure/src/library/events-stream-producer-adapter.ts b/packages/framework-provider-azure/src/library/events-stream-producer-adapter.ts index da8633782..1aeb826b1 100644 --- a/packages/framework-provider-azure/src/library/events-stream-producer-adapter.ts +++ b/packages/framework-provider-azure/src/library/events-stream-producer-adapter.ts @@ -31,7 +31,9 @@ export async function produceEventsStream( } if (batch.count === 0) { - throw new Error(`Message was too large and can't be sent until it's made smaller. ${eventEnvelope}`) + throw new Error( + `Message was too large and can't be sent until it's made smaller. ${JSON.stringify(eventEnvelope)}` + ) } // We reached the batch size limit From 983b862b5dd4ed894ebec8e2c5223dd4c658e930 Mon Sep 17 00:00:00 2001 From: gonzalojaubert Date: Mon, 13 Nov 2023 14:12:50 +0000 Subject: [PATCH 5/8] extract max retries and mode to configuration --- .../synth/terraform-function-app.ts | 3 +++ .../framework-provider-azure/src/constants.ts | 2 ++ packages/framework-provider-azure/src/index.ts | 16 +++++++++++++--- .../src/concepts/event-stream-configuration.ts | 2 ++ website/docs/10_going-deeper/azure-scale.mdx | 12 ++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-function-app.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-function-app.ts index 934adefa8..08cfdfac9 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-function-app.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-function-app.ts @@ -50,6 +50,9 @@ export class TerraformFunctionApp { BOOSTER_REST_API_URL: `https://${apiManagementName}.azure-api.net/${config.environmentName}`, [environmentVarNames.eventHubConnectionString]: eventHubConnectionString, [environmentVarNames.eventHubName]: config.resourceNames.streamTopic, + [environmentVarNames.eventHubMaxRetries]: + config.eventStreamConfiguration.parameters?.maxRetries?.toString() || '5', + [environmentVarNames.eventHubMode]: config.eventStreamConfiguration.parameters?.mode || 'exponential', COSMOSDB_CONNECTION_STRING: `AccountEndpoint=https://${cosmosdbDatabase.name}.documents.azure.com:443/;AccountKey=${cosmosdbDatabase.primaryKey};`, WEBSITE_CONTENTAZUREFILECONNECTIONSTRING: storageAccount.primaryConnectionString, // Terraform bug: https://github.com/hashicorp/terraform-provider-azurerm/issues/16650 }, diff --git a/packages/framework-provider-azure/src/constants.ts b/packages/framework-provider-azure/src/constants.ts index a97bbf51f..8ef6c28ca 100644 --- a/packages/framework-provider-azure/src/constants.ts +++ b/packages/framework-provider-azure/src/constants.ts @@ -30,6 +30,8 @@ export const environmentVarNames = { cosmosDbConnectionString: 'COSMOSDB_CONNECTION_STRING', eventHubConnectionString: 'EVENTHUB_CONNECTION_STRING', eventHubName: 'EVENTHUB_NAME', + eventHubMaxRetries: 'EVENTHUB_MAX_RETRIES', + eventHubMode: 'EVENTHUB_MODE', } as const // Azure special error codes diff --git a/packages/framework-provider-azure/src/index.ts b/packages/framework-provider-azure/src/index.ts index ace9334a8..fab46810e 100644 --- a/packages/framework-provider-azure/src/index.ts +++ b/packages/framework-provider-azure/src/index.ts @@ -4,9 +4,9 @@ import { requestFailed, requestSucceeded } from './library/api-adapter' import { rawGraphQLRequestToEnvelope } from './library/graphql-adapter' import { rawEventsToEnvelopes, - storeEvents, readEntityEventsSince, readEntityLatestSnapshot, + storeEvents, storeSnapshot, } from './library/events-adapter' import { CosmosClient } from '@azure/cosmos' @@ -57,6 +57,8 @@ if (typeof process.env[environmentVarNames.cosmosDbConnectionString] === 'undefi let producer: EventHubProducerClient const eventHubConnectionString = process.env[environmentVarNames.eventHubConnectionString] const eventHubName = process.env[environmentVarNames.eventHubName] +const DEFAULT_MAX_RETRY = 5 +const DEFAULT_EVENT_HUB_MODE = RetryMode.Exponential if ( typeof eventHubConnectionString === 'undefined' || typeof eventHubName === 'undefined' || @@ -65,10 +67,18 @@ if ( ) { producer = {} as any } else { + const maxRetries = process.env[environmentVarNames.eventHubMaxRetries] + ? Number(process.env[environmentVarNames.eventHubMaxRetries]) + : DEFAULT_MAX_RETRY + const mode = + process.env[environmentVarNames.eventHubMaxRetries] && + process.env[environmentVarNames.eventHubMode]?.toUpperCase() === 'FIXED' + ? RetryMode.Fixed + : DEFAULT_EVENT_HUB_MODE producer = new EventHubProducerClient(eventHubConnectionString, eventHubName, { retryOptions: { - maxRetries: 5, - mode: RetryMode.Exponential, + maxRetries: maxRetries, + mode: mode, }, }) } diff --git a/packages/framework-types/src/concepts/event-stream-configuration.ts b/packages/framework-types/src/concepts/event-stream-configuration.ts index e2468dd64..490b2ad0f 100644 --- a/packages/framework-types/src/concepts/event-stream-configuration.ts +++ b/packages/framework-types/src/concepts/event-stream-configuration.ts @@ -6,6 +6,8 @@ export type EventStreamConfiguration = partitionCount: number messageRetention: number // Specifies the number of days to retain the events for this Event Hub. dedupTtl?: number // Time to live in seconds + maxRetries?: number + mode?: 'exponential' | 'fixed' } } | { diff --git a/website/docs/10_going-deeper/azure-scale.mdx b/website/docs/10_going-deeper/azure-scale.mdx index 7bf891176..91bf72a57 100644 --- a/website/docs/10_going-deeper/azure-scale.mdx +++ b/website/docs/10_going-deeper/azure-scale.mdx @@ -19,6 +19,8 @@ To enable EventStream, set the `EventStreamConfiguration` in the configuration o streamTopic: 'test', partitionCount: 3, messageRetention: 1, + maxRetries: 5, + mode: 'exponential' }, } ``` @@ -27,6 +29,10 @@ To enable EventStream, set the `EventStreamConfiguration` in the configuration o * StreamTopic: Define the internal topic name Booster will use. * PartitionCount: Number of Event Hub partitions. The number of functions app consumer instances will match the partition count * MessageRetention: Specifies the number of days to retain the events for this Event Hub +* MaxRetries: Number of retries to consume an event +* Mode: Retry mode. It could be `fixed` or `exponential` + +Note: `maxRetries` and `mode` are configured at Function level ## Infrastructure @@ -38,5 +44,11 @@ Enabling `EventStreamConfiguration` will apply some changes to the infrastructur * A new container to handle duplicated consumed events * A new Event Hub will be added for event handling. +## Recommendations + +[From the Azure documentation](https://learn.microsoft.com/en-us/azure/event-hubs/dynamically-add-partitions#recommendations): + +Dynamically adding partitions isn't recommended. While the existing data preserves ordering, partition hashing will be broken for messages hashed after the +partition count changes due to addition of partitions. From 1892f4a0046863b94acdc3ef0af2630226e45a4b Mon Sep 17 00:00:00 2001 From: gonzalojaubert Date: Thu, 16 Nov 2023 15:42:36 +0000 Subject: [PATCH 6/8] fix rebase --- .../event-stream-producer-function.ts | 8 +-- .../infrastructure/synth/application-synth.ts | 26 +++------- ...orm-api-management-api-operation-policy.ts | 1 - ...-management-api-operation-sensor-health.ts | 50 ++++++++----------- .../infrastructure/synth/terraform-outputs.ts | 2 +- 5 files changed, 34 insertions(+), 53 deletions(-) diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-producer-function.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-producer-function.ts index e8cf36109..94cee852b 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-producer-function.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/functions/event-stream-producer-function.ts @@ -17,11 +17,11 @@ export class EventStreamProducerFunction { type: 'cosmosDBTrigger', name: 'rawEvent', direction: 'in', - leaseCollectionName: 'leases', - connectionStringSetting: 'COSMOSDB_CONNECTION_STRING', + leaseContainerName: 'leases', + connection: 'COSMOSDB_CONNECTION_STRING', databaseName: this.config.resourceNames.applicationStack, - collectionName: this.config.resourceNames.eventsStore, - createLeaseCollectionIfNotExists: 'true', + containerName: this.config.resourceNames.eventsStore, + createLeaseContainerIfNotExists: 'true', } const eventHubBinding: EventHubOutBinding = { name: 'eventHubMessages', diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts index 2f3e96e5d..0354800d5 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts @@ -59,6 +59,7 @@ export class ApplicationSynth { public synth(zipFile: string): ApplicationSynthStack { const graphQLApiOperation = 'graphql' + const sensorApiOperation = 'sensor-health' const resourceGroup = TerraformResourceGroup.build(this.stackNames) const stack: ApplicationSynthStack = { ...this.stackNames, resourceGroup: resourceGroup } stack.cosmosdbDatabase = TerraformCosmosdbDatabase.build(stack) @@ -74,28 +75,15 @@ export class ApplicationSynth { stack.functionApp = this.buildDefaultFunctionApp(stack, zipFile) stack.graphQLApiManagementApiOperationPolicy = TerraformApiManagementApiOperationPolicy.build( stack, - this.config.environmentName, graphQLApiOperation ) - // TODO - const sensorHealthApiManagementApiOperationResource = TerraformApiManagementApiOperationSensorHealth.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - apiManagementApiResource, - this.appPrefix, - 'sensor-health' + stack.sensorHealthApiManagementApiOperation = TerraformApiManagementApiOperationSensorHealth.build( + stack, + sensorApiOperation ) - // todo - const sensorHealthApiManagementApiOperationPolicyResource = TerraformApiManagementApiOperationPolicy.build( - azurermProvider, - this.terraformStackResource, - resourceGroupResource, - sensorHealthApiManagementApiOperationResource, - this.appPrefix, - this.config.environmentName, - functionAppResource, - 'sensor-health' + stack.sensorHealthApiManagementApiOperationPolicy = TerraformApiManagementApiOperationPolicy.build( + stack, + sensorApiOperation ) this.buildWebPubSubHub(stack) TerraformOutputs.build(stack) diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts index 1b40bd840..a899332c8 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts @@ -14,7 +14,6 @@ export class TerraformApiManagementApiOperationPolicy { functionApp, graphQLApiManagementApiOperation, }: ApplicationSynthStack, - environmentName: string, name: string ): apiManagementApiOperationPolicy.ApiManagementApiOperationPolicy { if (!functionApp) { diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-sensor-health.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-sensor-health.ts index f556a1f60..34c49a770 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-sensor-health.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-sensor-health.ts @@ -1,37 +1,31 @@ -import { TerraformStack } from 'cdktf' -import { apiManagementApi, apiManagementApiOperation, resourceGroup } from '@cdktf/provider-azurerm' +import { apiManagementApiOperation } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' -import { AzurermProvider } from '@cdktf/provider-azurerm/lib/provider' +import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformApiManagementApiOperationSensorHealth { static build( - providerResource: AzurermProvider, - terraformStackResource: TerraformStack, - group: resourceGroup.ResourceGroup, - apiManagementApiResource: apiManagementApi.ApiManagementApi, - appPrefix: string, + { terraformStack, azureProvider, resourceGroup, apiManagementApi, appPrefix }: ApplicationSynthStack, name: string ): apiManagementApiOperation.ApiManagementApiOperation { + if (!apiManagementApi) { + throw new Error('Undefined apiManagementApi resource') + } const idApiManagementApiOperation = toTerraformName(appPrefix, 'amash' + name[0]) - return new apiManagementApiOperation.ApiManagementApiOperation( - terraformStackResource, - idApiManagementApiOperation, - { - operationId: `${name}GET`, - apiName: apiManagementApiResource.name, - apiManagementName: apiManagementApiResource.apiManagementName, - resourceGroupName: group.name, - displayName: '/sensor/health', - method: 'GET', - urlTemplate: '/sensor/health/*', - description: '', - response: [ - { - statusCode: 200, - }, - ], - provider: providerResource, - } - ) + return new apiManagementApiOperation.ApiManagementApiOperation(terraformStack, idApiManagementApiOperation, { + operationId: `${name}GET`, + apiName: apiManagementApi.name, + apiManagementName: apiManagementApi.apiManagementName, + resourceGroupName: resourceGroup.name, + displayName: '/sensor/health', + method: 'GET', + urlTemplate: '/sensor/health/*', + description: '', + response: [ + { + statusCode: 200, + }, + ], + provider: azureProvider, + }) } } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-outputs.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-outputs.ts index 96961118d..73801e92c 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-outputs.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-outputs.ts @@ -15,7 +15,7 @@ export class TerraformOutputs { description: 'The base URL for sending GraphQL mutations and queries', }) new TerraformOutput(applicationSynthStack.azureProvider, 'sensorHealthURL', { - value: baseUrl + applicationSynthStack.sensorHealthApiManagementApiOperationResource.urlTemplate, + value: baseUrl + applicationSynthStack.sensorHealthApiManagementApiOperation?.urlTemplate, description: 'The base URL for getting health information', }) From a1f35da4f820b72d0fc14c025b722e4c44769555 Mon Sep 17 00:00:00 2001 From: gonzalojaubert Date: Fri, 17 Nov 2023 11:37:31 +0000 Subject: [PATCH 7/8] fix api rebase --- common/config/rush/pnpm-lock.yaml | 185 +++++++++++++----- .../infrastructure/synth/application-synth.ts | 19 +- ...orm-api-management-api-operation-policy.ts | 27 ++- ...-management-api-operation-sensor-health.ts | 4 +- 4 files changed, 159 insertions(+), 76 deletions(-) diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 774987bc0..6022569e3 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -8,8 +8,8 @@ importers: ../../packages/application-tester: specifiers: '@apollo/client': 3.7.13 - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/jsonwebtoken': 9.0.1 '@types/node': ^18.15.3 @@ -70,10 +70,10 @@ importers: ../../packages/cli: specifiers: - '@boostercloud/application-tester': workspace:^2.0.0 - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-core': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 + '@boostercloud/application-tester': workspace:^2.1.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-core': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@oclif/core': ^3.9.0 '@oclif/plugin-help': ^5 @@ -181,8 +181,8 @@ importers: ../../packages/framework-common-helpers: specifiers: - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/chai': 4.2.18 '@types/chai-as-promised': 7.1.4 @@ -250,10 +250,10 @@ importers: ../../packages/framework-core: specifiers: - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-common-helpers': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 - '@boostercloud/metadata-booster': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-common-helpers': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 + '@boostercloud/metadata-booster': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/chai': 4.2.18 '@types/chai-as-promised': 7.1.4 @@ -350,19 +350,19 @@ importers: ../../packages/framework-integration-tests: specifiers: '@apollo/client': 3.7.13 - '@boostercloud/application-tester': workspace:^2.0.0 - '@boostercloud/cli': workspace:^2.0.0 - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-common-helpers': workspace:^2.0.0 - '@boostercloud/framework-core': workspace:^2.0.0 - '@boostercloud/framework-provider-aws': workspace:^2.0.0 - '@boostercloud/framework-provider-aws-infrastructure': workspace:^2.0.0 - '@boostercloud/framework-provider-azure': workspace:^2.0.0 - '@boostercloud/framework-provider-azure-infrastructure': workspace:^2.0.0 - '@boostercloud/framework-provider-local': workspace:^2.0.0 - '@boostercloud/framework-provider-local-infrastructure': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 - '@boostercloud/metadata-booster': workspace:^2.0.0 + '@boostercloud/application-tester': workspace:^2.1.0 + '@boostercloud/cli': workspace:^2.1.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-common-helpers': workspace:^2.1.0 + '@boostercloud/framework-core': workspace:^2.1.0 + '@boostercloud/framework-provider-aws': workspace:^2.1.0 + '@boostercloud/framework-provider-aws-infrastructure': workspace:^2.1.0 + '@boostercloud/framework-provider-azure': workspace:^2.1.0 + '@boostercloud/framework-provider-azure-infrastructure': workspace:^2.1.0 + '@boostercloud/framework-provider-local': workspace:^2.1.0 + '@boostercloud/framework-provider-local-infrastructure': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 + '@boostercloud/metadata-booster': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/aws-lambda': 8.10.48 '@types/chai': 4.2.18 @@ -484,9 +484,9 @@ importers: ../../packages/framework-provider-aws: specifiers: - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-common-helpers': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-common-helpers': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/aws-lambda': 8.10.48 '@types/chai': 4.2.18 @@ -580,10 +580,10 @@ importers: '@aws-cdk/core': ^1.170.0 '@aws-cdk/custom-resources': ^1.170.0 '@aws-cdk/cx-api': ^1.170.0 - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-common-helpers': workspace:^2.0.0 - '@boostercloud/framework-provider-aws': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-common-helpers': workspace:^2.1.0 + '@boostercloud/framework-provider-aws': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/archiver': 5.1.0 '@types/aws-lambda': 8.10.48 @@ -693,12 +693,13 @@ importers: ../../packages/framework-provider-azure: specifiers: '@azure/cosmos': ^4.0.0 + '@azure/event-hubs': 5.11.1 '@azure/functions': ^1.2.2 '@azure/identity': ~2.1.0 '@azure/web-pubsub': ~1.1.0 - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-common-helpers': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-common-helpers': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/chai': 4.2.18 '@types/chai-as-promised': 7.1.4 @@ -728,6 +729,7 @@ importers: typescript: 4.7.4 dependencies: '@azure/cosmos': 4.0.0 + '@azure/event-hubs': 5.11.1 '@azure/functions': 1.2.3 '@azure/identity': 2.1.0 '@azure/web-pubsub': 1.1.1 @@ -769,11 +771,11 @@ importers: '@azure/arm-resources': ^5.0.1 '@azure/cosmos': ^4.0.0 '@azure/identity': ~2.1.0 - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-common-helpers': workspace:^2.0.0 - '@boostercloud/framework-core': workspace:^2.0.0 - '@boostercloud/framework-provider-azure': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-common-helpers': workspace:^2.1.0 + '@boostercloud/framework-core': workspace:^2.1.0 + '@boostercloud/framework-provider-azure': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 '@cdktf/provider-azurerm': 5.0.13 '@cdktf/provider-time': 5.0.0 '@effect-ts/core': ^0.60.4 @@ -880,9 +882,9 @@ importers: ../../packages/framework-provider-local: specifiers: - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-common-helpers': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-common-helpers': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/chai': 4.2.18 '@types/chai-as-promised': 7.1.4 @@ -961,10 +963,10 @@ importers: ../../packages/framework-provider-local-infrastructure: specifiers: - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/framework-common-helpers': workspace:^2.0.0 - '@boostercloud/framework-provider-local': workspace:^2.0.0 - '@boostercloud/framework-types': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/framework-common-helpers': workspace:^2.1.0 + '@boostercloud/framework-provider-local': workspace:^2.1.0 + '@boostercloud/framework-types': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/chai': 4.2.18 '@types/chai-as-promised': 7.1.4 @@ -1044,8 +1046,8 @@ importers: ../../packages/framework-types: specifiers: - '@boostercloud/eslint-config': workspace:^2.0.0 - '@boostercloud/metadata-booster': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 + '@boostercloud/metadata-booster': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@effect-ts/node': ~0.39.0 '@types/chai': 4.2.18 @@ -1111,7 +1113,7 @@ importers: ../../packages/metadata-booster: specifiers: - '@boostercloud/eslint-config': workspace:^2.0.0 + '@boostercloud/eslint-config': workspace:^2.1.0 '@effect-ts/core': ^0.60.4 '@types/node': ^18.15.3 '@typescript-eslint/eslint-plugin': ^5.0.0 @@ -2547,6 +2549,26 @@ packages: - supports-color dev: false + /@azure/core-amqp/3.3.0: + resolution: {integrity: sha512-RYIyC8PtGpMzZRiSokADw0ezFgNq1eUkCPV8rd7tJ85dn8CAhYDEYapzMYxAwIBLWidshu14m9UWjQS7hKYDpA==} + engines: {node: '>=14.0.0'} + dependencies: + '@azure/abort-controller': 1.1.0 + '@azure/core-auth': 1.4.0 + '@azure/core-util': 1.3.1 + '@azure/logger': 1.0.4 + buffer: 6.0.3 + events: 3.3.0 + jssha: 3.3.1 + process: 0.11.10 + rhea: 3.0.2 + rhea-promise: 3.0.1 + tslib: 2.5.0 + util: 0.12.5 + transitivePeerDependencies: + - supports-color + dev: false + /@azure/core-auth/1.4.0: resolution: {integrity: sha512-HFrcTgmuSuukRf/EdPmqBrc5l6Q5Uu+2TbuhaKbgaCpP2TfAeiNaQPAadxO+CYBRHGUzIDteMAjFspFLDLnKVQ==} engines: {node: '>=12.0.0'} @@ -2640,6 +2662,26 @@ packages: - supports-color dev: false + /@azure/event-hubs/5.11.1: + resolution: {integrity: sha512-4sgPwxO0A6CjU7oZgNvavnYGQK0oluqyoJImqxmRiaptQjFpki9yH+k2WmplAXIMnXJn4znstLWv6vGSFvGmgA==} + engines: {node: '>=14.0.0'} + dependencies: + '@azure/abort-controller': 1.1.0 + '@azure/core-amqp': 3.3.0 + '@azure/core-auth': 1.4.0 + '@azure/core-tracing': 1.0.1 + '@azure/core-util': 1.3.1 + '@azure/logger': 1.0.4 + buffer: 6.0.3 + is-buffer: 2.0.5 + jssha: 3.3.1 + process: 0.11.10 + rhea-promise: 3.0.1 + tslib: 2.5.0 + transitivePeerDependencies: + - supports-color + dev: false + /@azure/functions/1.2.3: resolution: {integrity: sha512-dZITbYPNg6ay6ngcCOjRUh1wDhlFITS0zIkqplyH5KfKEAVPooaoaye5mUFnR+WP9WdGRjlNXyl/y2tgWKHcRg==} dev: false @@ -3308,7 +3350,7 @@ packages: dev: true /@pkgjs/parseargs/0.11.0: - resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==, tarball: https://repo1.uhc.com:443/artifactory/api/npm/npm-virtual/@pkgjs/parseargs/-/parseargs-0.11.0.tgz} + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} requiresBuild: true dev: true @@ -4443,6 +4485,13 @@ packages: base64-js: 1.5.1 ieee754: 1.2.1 + /buffer/6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + dev: false + /builtin-modules/3.3.0: resolution: {integrity: sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==} engines: {node: '>=6'} @@ -6238,7 +6287,7 @@ packages: dev: true /fsevents/2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==, tarball: https://repo1.uhc.com:443/artifactory/api/npm/npm-virtual/fsevents/-/fsevents-2.3.2.tgz} + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] requiresBuild: true @@ -6835,6 +6884,11 @@ packages: resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} dev: false + /is-buffer/2.0.5: + resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} + engines: {node: '>=4'} + dev: false + /is-builtin-module/3.2.1: resolution: {integrity: sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A==} engines: {node: '>=6'} @@ -7339,6 +7393,10 @@ packages: ms: 2.1.3 semver: 7.5.0 + /jssha/3.3.1: + resolution: {integrity: sha512-VCMZj12FCFMQYcFLPRm/0lOBbLi8uM2BhXPTqw3U4YAfs4AZfiApOoBLoN8cQE60Z50m1MYMTQVCfgF/KaCVhQ==} + dev: false + /jszip/3.10.1: resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} dependencies: @@ -8588,6 +8646,11 @@ packages: type: 2.7.2 dev: true + /process/0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + dev: false + /progress/2.0.3: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} @@ -8914,6 +8977,24 @@ packages: /rfdc/1.3.0: resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==} + /rhea-promise/3.0.1: + resolution: {integrity: sha512-Fcqgml7lgoyi7fH1ClsSyFr/xwToijEN3rULFgrIcL+7EHeduxkWogFxNHjFzHf2YGScAckJDaDxS1RdlTUQYw==} + dependencies: + debug: 3.2.7 + rhea: 3.0.2 + tslib: 2.5.0 + transitivePeerDependencies: + - supports-color + dev: false + + /rhea/3.0.2: + resolution: {integrity: sha512-0G1ZNM9yWin8VLvTxyISKH6KfR6gl1TW/1+5yMKPf2r1efhkzTLze09iFtT2vpDjuWIVtSmXz8r18lk/dO8qwQ==} + dependencies: + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + dev: false + /rimraf/2.6.3: resolution: {integrity: sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==} hasBin: true diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts index 0354800d5..71404051b 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts @@ -69,11 +69,7 @@ export class ApplicationSynth { this.buildWebPubSub(stack) stack.apiManagement = TerraformApiManagement.build(stack) stack.apiManagementApi = TerraformApiManagementApi.build(stack, this.config.environmentName) - stack.graphQLApiManagementApiOperation = TerraformApiManagementApiOperation.build(stack, graphQLApiOperation) - stack.applicationServicePlan = TerraformServicePlan.build(stack, 'psp', 'Y1', 1) - stack.storageAccount = TerraformStorageAccount.build(stack, 'sp') - stack.functionApp = this.buildDefaultFunctionApp(stack, zipFile) - stack.graphQLApiManagementApiOperationPolicy = TerraformApiManagementApiOperationPolicy.build( + stack.graphQLApiManagementApiOperation = TerraformApiManagementApiOperation.build( stack, graphQLApiOperation ) @@ -81,9 +77,20 @@ export class ApplicationSynth { stack, sensorApiOperation ) + stack.applicationServicePlan = TerraformServicePlan.build(stack, 'psp', 'Y1', 1) + stack.storageAccount = TerraformStorageAccount.build(stack, 'sp') + stack.functionApp = this.buildDefaultFunctionApp(stack, zipFile) + stack.graphQLApiManagementApiOperationPolicy = TerraformApiManagementApiOperationPolicy.build( + stack, + stack.graphQLApiManagementApiOperation, + graphQLApiOperation, + 'amaop' + ) stack.sensorHealthApiManagementApiOperationPolicy = TerraformApiManagementApiOperationPolicy.build( stack, - sensorApiOperation + stack.sensorHealthApiManagementApiOperation, + sensorApiOperation, + 'amaopsh' ) this.buildWebPubSubHub(stack) TerraformOutputs.build(stack) diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts index a899332c8..0b36c0492 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-policy.ts @@ -1,4 +1,4 @@ -import { apiManagementApiOperationPolicy } from '@cdktf/provider-azurerm' +import { apiManagementApiOperation, apiManagementApiOperationPolicy } from '@cdktf/provider-azurerm' import { toTerraformName } from '../helper/utils' import * as Mustache from 'mustache' import { templates } from '../templates' @@ -6,32 +6,27 @@ import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformApiManagementApiOperationPolicy { static build( - { - terraformStack, - azureProvider, - appPrefix, - resourceGroupName, - functionApp, - graphQLApiManagementApiOperation, - }: ApplicationSynthStack, - name: string + { terraformStack, azureProvider, appPrefix, resourceGroupName, functionApp }: ApplicationSynthStack, + apiManagementApiOperation: apiManagementApiOperation.ApiManagementApiOperation, + name: string, + suffix: string ): apiManagementApiOperationPolicy.ApiManagementApiOperationPolicy { if (!functionApp) { throw new Error('Undefined functionApp resource') } - if (!graphQLApiManagementApiOperation) { - throw new Error('Undefined graphQLApiManagementApiOperation resource') + if (!apiManagementApiOperation) { + throw new Error('Undefined apiManagementApiOperation resource') } - const idApiManagementApiOperationPolicy = toTerraformName(appPrefix, 'amaop' + name[0]) + const idApiManagementApiOperationPolicy = toTerraformName(appPrefix, suffix + name[0]) const policyContent = Mustache.render(templates.policy, { functionAppName: functionApp.name }) return new apiManagementApiOperationPolicy.ApiManagementApiOperationPolicy( terraformStack, idApiManagementApiOperationPolicy, { - apiName: graphQLApiManagementApiOperation.apiName, - apiManagementName: graphQLApiManagementApiOperation.apiManagementName, + apiName: apiManagementApiOperation.apiName, + apiManagementName: apiManagementApiOperation.apiManagementName, resourceGroupName: resourceGroupName, - operationId: graphQLApiManagementApiOperation.operationId, + operationId: apiManagementApiOperation.operationId, xmlContent: policyContent, provider: azureProvider, } diff --git a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-sensor-health.ts b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-sensor-health.ts index 34c49a770..945ec8a45 100644 --- a/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-sensor-health.ts +++ b/packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-api-management-api-operation-sensor-health.ts @@ -4,7 +4,7 @@ import { ApplicationSynthStack } from '../types/application-synth-stack' export class TerraformApiManagementApiOperationSensorHealth { static build( - { terraformStack, azureProvider, resourceGroup, apiManagementApi, appPrefix }: ApplicationSynthStack, + { terraformStack, azureProvider, appPrefix, resourceGroupName, apiManagementApi }: ApplicationSynthStack, name: string ): apiManagementApiOperation.ApiManagementApiOperation { if (!apiManagementApi) { @@ -15,7 +15,7 @@ export class TerraformApiManagementApiOperationSensorHealth { operationId: `${name}GET`, apiName: apiManagementApi.name, apiManagementName: apiManagementApi.apiManagementName, - resourceGroupName: resourceGroup.name, + resourceGroupName: resourceGroupName, displayName: '/sensor/health', method: 'GET', urlTemplate: '/sensor/health/*', From a38a431b29f2d45fd03e6ae8b1210a02c3238c2b Mon Sep 17 00:00:00 2001 From: gonzalojaubert Date: Fri, 17 Nov 2023 11:51:53 +0000 Subject: [PATCH 8/8] fix aws setup --- packages/framework-provider-aws/src/setup.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/framework-provider-aws/src/setup.ts b/packages/framework-provider-aws/src/setup.ts index 70eec7421..2d6066f56 100644 --- a/packages/framework-provider-aws/src/setup.ts +++ b/packages/framework-provider-aws/src/setup.ts @@ -66,9 +66,9 @@ export const Provider = (rockets?: RocketDescriptor[]): ProviderLibrary => { // ProviderEventsLibrary events: { rawToEnvelopes: rawEventsToEnvelopes, - rawStreamToEnvelopes: notImplementedResult() as any, - dedupEventStream: notImplementedResult() as any, - produce: notImplementedResult() as any, + rawStreamToEnvelopes: notImplementedResult as any, + dedupEventStream: notImplementedResult as any, + produce: notImplementedResult as any, forEntitySince: readEntityEventsSince.bind(null, dynamoDB), latestEntitySnapshot: readEntityLatestSnapshot.bind(null, dynamoDB), search: searchEvents.bind(null, dynamoDB),