feat(search events): implement search event emitter and consumer (#666)

* feat(search events): initial scaffolding

* feat(search/snowplow): search event emitter and consumer

Emit search result events from user-list-search
for corpus searches, to the event bus.

Add consumer to shared snowplow event consumer.

[POCKET-10238]

* chore: fix snowplow tests and update snowtype

* chore: fixes for lint and typescript

Removed the unnecessary snowplow codgen -- will
need to be repeated if they are regenerated unless
the version becomes compatible with the new typescript?

infrastructure/pocket-event-bridge/src/event-rules/search-api-events/eventConfig.ts

@@ -0,0 +1,7 @@
+export const eventConfig = {
+  PocketSearch: {
+    name: 'SearchApiEvents',
+    source: 'search-api-events',
+    detailType: ['corpus_search_result'],
+  },
+};

infrastructure/pocket-event-bridge/src/event-rules/search-api-events/pocketSearchEventRules.ts

@@ -0,0 +1,132 @@
+import { Construct } from 'constructs';
+import {
+  PocketEventBridgeProps,
+  PocketEventBridgeRuleWithMultipleTargets,
+  ApplicationEventBus,
+} from '@pocket-tools/terraform-modules';
+import { config } from '../../config';
+import {
+  sqsQueue,
+  snsTopic,
+  dataAwsIamPolicyDocument,
+  snsTopicPolicy,
+  dataAwsSnsTopic,
+} from '@cdktf/provider-aws';
+import { resource } from '@cdktf/provider-null';
+import { eventConfig } from './eventConfig';
+import { createDeadLetterQueueAlarm } from '../utils';
+
+export class SearchApiEvents extends Construct {
+  public readonly snsTopic: snsTopic.SnsTopic;
+  public readonly snsTopicDlq: sqsQueue.SqsQueue;
+
+  constructor(
+    scope: Construct,
+    private name: string,
+    private sharedEventBus: ApplicationEventBus,
+    private snsAlarmTopic: dataAwsSnsTopic.DataAwsSnsTopic,
+  ) {
+    super(scope, name);
+
+    this.snsTopic = new snsTopic.SnsTopic(this, 'search-api-event-topic', {
+      name: `${config.prefix}-SearchApiEventTopic`,
+      lifecycle: {
+        preventDestroy: true,
+      },
+    });
+
+    this.snsTopicDlq = new sqsQueue.SqsQueue(this, 'sns-topic-dql', {
+      name: `${config.prefix}-SNS-${eventConfig.PocketSearch.name}-Topic-DLQ`,
+      tags: config.tags,
+    });
+
+    const searchEvent = this.createSearchApiEventRules();
+    this.createPolicyForEventBridgeToSns();
+
+    //get alerted if we get 10 messages in DLQ in 4 evaluation period of 5 minutes (for shareable-list)
+    createDeadLetterQueueAlarm(
+      this,
+      snsAlarmTopic,
+      this.snsTopicDlq.name,
+      `${eventConfig.PocketSearch.name}-Rule-dlq-alarm`,
+      true,
+      4,
+      300,
+      10,
+    );
+
+    //place-holder resource used to make sure we are not
+    //removing the event-rule or the SNS by mistake
+    //if the resources are removed, this would act as an additional check
+    //to prevent resource deletion in-addition to preventDestroy
+    //e.g removing any of the dependsOn resource and running npm build would
+    //throw error
+    new resource.Resource(this, 'null-resource', {
+      dependsOn: [searchEvent.getEventBridge().rule, this.snsTopic],
+    });
+  }
+
+  /**
+   * Rolls out event bridge rule and attaches them to sns target
+   * for searchapi-events
+   * @private
+   */
+  private createSearchApiEventRules() {
+    const searchEventRuleProps: PocketEventBridgeProps = {
+      eventRule: {
+        name: `${config.prefix}-${eventConfig.PocketSearch.name}-Rule`,
+        eventPattern: {
+          source: [eventConfig.PocketSearch.source],
+          'detail-type': eventConfig.PocketSearch.detailType,
+        },
+        eventBusName: this.sharedEventBus.bus.name,
+        preventDestroy: true,
+      },
+      targets: [
+        {
+          arn: this.snsTopic.arn,
+          deadLetterArn: this.snsTopicDlq.arn,
+          targetId: `${config.prefix}-SearchApi-Event-SNS-Target`,
+          terraformResource: this.snsTopic,
+        },
+      ],
+    };
+    return new PocketEventBridgeRuleWithMultipleTargets(
+      this,
+      `${config.prefix}-SearchApi-EventBridge-Rule`,
+      searchEventRuleProps,
+    );
+  }
+
+  private createPolicyForEventBridgeToSns() {
+    const eventBridgeSnsPolicy =
+      new dataAwsIamPolicyDocument.DataAwsIamPolicyDocument(
+        this,
+        `${config.prefix}-EventBridge-SNS-Policy`,
+        {
+          statement: [
+            {
+              effect: 'Allow',
+              actions: ['sns:Publish'],
+              resources: [this.snsTopic.arn],
+              principals: [
+                {
+                  identifiers: ['events.amazonaws.com'],
+                  type: 'Service',
+                },
+              ],
+            },
+          ],
+        },
+      ).json;
+
+    return new snsTopicPolicy.SnsTopicPolicy(
+      this,
+      'searchapi-events-sns-topic-policy',
+      {
+        arn: this.snsTopic.arn,
+        policy: eventBridgeSnsPolicy,
+      },
+    );
+  }
+}

infrastructure/pocket-event-bridge/src/main.ts

@@ -28,6 +28,7 @@ import { UserRegistrationEventSchema } from './events-schema/userRegistrationEve
 import { AllEventsRule } from './event-rules/all-events/allEventRules';
 import { ForgotPassword as ForgotPasswordRequest } from './event-rules/forgot-password-request';
 import { SharesApiEvents } from './event-rules/shares-api-events/pocketShareEventRules';
+import { SearchApiEvents } from './event-rules/search-api-events/pocketSearchEventRules';
 import { CorpusEvents } from './event-rules/corpus-events/corpusEventRules';
 
 class PocketEventBus extends TerraformStack {
@@ -138,6 +139,13 @@ class PocketEventBus extends TerraformStack {
       sharedPocketEventBus,
       alarmSnsTopic,
     );
+    // search-api events
+    new SearchApiEvents(
+      this,
+      'search-api-events',
+      sharedPocketEventBus,
+      alarmSnsTopic,
+    );
     // Corpus Events (uses the default bus, not the shared event bus)
     new CorpusEvents(this, 'corpus-events', alarmSnsTopic);
 

infrastructure/shared-snowplow-consumer/src/config/index.ts

@@ -53,6 +53,7 @@ export const config = {
     shareableListItemEventTopic: 'ShareableListItemEventTopic',
     collectionEventTopic: 'CollectionEventTopic',
     sharesApiEventTopic: 'SharesApiEventTopic',
+    searchApiEventTopic: 'SearchApiEventTopic',
   },
   envVars: {
     snowplowEndpoint: snowplowEndpoint,

infrastructure/shared-snowplow-consumer/src/main.ts

@@ -119,6 +119,15 @@ class SnowplowSharedConsumerStack extends TerraformStack {
       config.eventBridge.sharesApiEventTopic,
     );
 
+    // Consumer Queue should be able to listen to search result events from search-api
+    const searchApiEventTopicArn = `arn:aws:sns:${region.name}:${caller.accountId}:${config.eventBridge.prefix}-${config.environment}-${config.eventBridge.searchApiEventTopic}`;
+    this.subscribeSqsToSnsTopic(
+      sqsConsumeQueue,
+      snsTopicDlq,
+      searchApiEventTopicArn,
+      config.eventBridge.searchApiEventTopic,
+    );
+
     // Consumer Queue should be able to listen to collection-created and collection-updated events from collection-api.
     const collectionEventTopicArn = `arn:aws:sns:${region.name}:${caller.accountId}:${config.eventBridge.prefix}-${config.environment}-${config.eventBridge.collectionEventTopic}`;
     this.subscribeSqsToSnsTopic(
@@ -136,6 +145,7 @@ class SnowplowSharedConsumerStack extends TerraformStack {
       shareableListItemEventTopicArn,
       collectionEventTopicArn,
       sharesApiEventTopicArn,
+      searchApiEventTopicArn,
     ];
 
     // Assigns inline access policy for SQS and DLQ.

infrastructure/user-list-search/apollo_ecs.tf

@@ -51,6 +51,10 @@ module "apollo" {
       name  = "ELASTICSEARCH_INDEX"
       value = local.elastic_index
     },
+    {
+      name = "EVENT_BUS_NAME"
+      value = local.event_bus_name
+    },
     {
       name  = "CORPUS_INDEX_EN"
       value = local.corpus_index_en

infrastructure/user-list-search/locals.tf

@@ -17,6 +17,7 @@ locals {
   container_name       = "node"
   container_credential = "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:Shared/DockerHub"
   elastic_index        = "list"
+  event_bus_name       = "PocketEventBridge-${local.env}-Shared-Event-Bus"
   corpus_index_en      = "corpus_en"
   corpus_index_fr      = "corpus_fr"
   corpus_index_es      = "corpus_es"