chore: fixup aws env vars and docs

docs/dev_env_vars.md

@@ -8,6 +8,7 @@ A suggested deployment strategy is to use [envconsul](https://github.com/hashico
 
 | Env Var | Default | Secret | Description |
 | --- | --- | --- | --- |
+| APP_ENVIRONMENT |  |  | The name of the environment this instance is running in. A local environment should be blank. Required to push audit to AWS. |
 | BROKER_URL |  |  | The external URL that this instance is running on. Used to create redirect urls. |
 | HOSTNAME |  |  | The hostname of the server this instance is running on. Used in logs. The instance with a hostname ending in '-0' is the primary node. It will cause issues if there is no primary node or there are multiple primary nodes'. |
 
@@ -75,8 +76,7 @@ AWS configuration used to push the audit log to a Kinesis end point. Consuming t
 | --- | --- | --- | --- |
 | AWS_ACCESS_KEY_ID |  | Yes |  |
 | AWS_SECRET_ACCESS_KEY |  | Yes |  |
-| AWS_SESSION_TOKEN |  | Yes |  |
-| AWS_KINESIS_ROLE_ARN |  | Yes |  |
+| AWS_ROLE_ARN |  | Yes |  |
 | AWS_DEFAULT_REGION | ca-central-1 |  |  |
 
 ## Log redirection

docs/development.md

@@ -132,6 +132,8 @@ If Kinesis and AWS access is not setup then some APIs will return a 503 (service
 
 ### Local MongoDB Disconnects
 
+Note: The latest versions of Podman seems to have resolved this.
+
 The connection to MongoDB may time out if your machine goes to sleep. The easiest way to recover is to stop the backend, restart the containers and rerun the vault setup. The provided restart script will do the container and setup steps for you.
 
 ```bash
@@ -204,6 +206,10 @@ To locally setup a GitHub App syncing, set the values GITHUB_SYNC_CLIENT_ID and
 
 Broker can be setup to allow users to alias their identity in other identity providers to their account.
 
+## Setup Collection Sync from OpenSearch
+
+Broker can synchronize collections with unique names from an OpenSearch index. See: [OpenSearch Integration](./operations_opensearch.md)
+
 ### GitHub Alias
 
 GitHub user alias requires a GitHub OAuth app. It is recommended that the GitHub OAuth app be registered under a GitHub organization in production. A GitHub OAuth app registered under a personal account can be used for testing.

docs/operations_opensearch.md

@@ -10,10 +10,38 @@ In order to integrate with OpenSearch, Broker requires the environment variable
 
 ## Collection Sync
 
-The broker can synchronize collections with unique names from an index. It searches for unique names from the past 12 hours and iterates over them. For each value, a document is sampled, and an entry for the collection is upserted into the broker.
+Broker can synchronize collections with unique names from an OpenSearch index. It searches for unique names from the past 12 hours and iterates over them. For each value, a document is sampled, and an entry for the collection is upserted into the broker.
 
 This is configured by adding a `sync` field to the `collectionConfig` document for the collection you want to synchronize. You must specify the index to query, the unique name value in the source index, and a mapping for the fields from the document to the collection.
 
+Partial exerpt of server collection sync configuration:
+```json
+{
+  "sync": {
+    "index": "example-d",
+    "unique": "host.hostname",
+    "map": {
+      "host.hostname": {
+        "type": "first",
+        "dest": "name"
+      },
+      "host.architecture": {
+        "type": "first",
+        "dest": "architecture"
+      },
+      "host.name": {
+        "type": "pick",
+        "endsWith": [
+          "bcgov",
+          "dmz"
+        ],
+        "dest": "hostName"
+      }
+    }
+  }
+}
+```
+
 ## Index Patterns
 
 Environment variables and other configurations that store index names can accept a comma-separated string listing the indices to query. If an index ends with -d, appropriate indices for the time period (ending in -yyyy-mm-dd) will be generated.

src/audit/audit.service.ts

@@ -15,6 +15,7 @@ import { IntentionEntity } from '../intention/entity/intention.entity';
 import { ActionEmbeddable } from '../intention/entity/action.embeddable';
 import { ArtifactEmbeddable } from '../intention/entity/artifact.embeddable';
 import { UserEmbeddable } from '../intention/entity/user.embeddable';
+import { APP_ENVIRONMENT } from '../constants';
 
 const hostInfo = {
   host: {
@@ -890,9 +891,7 @@ export class AuditService {
     return merge(ecsObj, {
       service: {
         name: 'nr-broker-backend',
-        environment: process.env.APP_ENVIRONMENT
-          ? process.env.APP_ENVIRONMENT
-          : 'unknown',
+        environment: APP_ENVIRONMENT ?? 'unknown',
       },
     });
   }

src/aws/aws.module.ts

@@ -6,21 +6,28 @@ import { AwsService } from './aws.service';
 import { FakeOpensearchService } from './fake-opensearch.service';
 import { AwsOpensearchService } from './aws-opensearch.service';
 import { OpensearchService } from './opensearch.service';
-import { AWS_OPENSEARCH_HOST } from '../constants';
+import { APP_ENVIRONMENT, AWS_OPENSEARCH_HOST } from '../constants';
 
-function useAwsServices() {
-  return !process.env.APP_ENVIRONMENT;
+function useAwsServices(): boolean {
+  return !!(
+    process.env.AWS_ACCESS_KEY_ID &&
+    process.env.AWS_SECRET_ACCESS_KEY &&
+    process.env.AWS_ROLE_ARN
+  );
 }
 
 const kinesisServiceProvider = {
   provide: KinesisService,
-  useClass: useAwsServices() ? FakeKinesisService : AwsKinesisService,
+  useClass:
+    !useAwsServices() || APP_ENVIRONMENT === ''
+      ? FakeKinesisService
+      : AwsKinesisService,
 };
 
 const opensearchServiceProvider = {
   provide: OpensearchService,
   useClass:
-    useAwsServices() && AWS_OPENSEARCH_HOST !== ''
+    !useAwsServices() || AWS_OPENSEARCH_HOST === ''
       ? FakeOpensearchService
       : AwsOpensearchService,
 };
@@ -30,7 +37,7 @@ const opensearchServiceProvider = {
  */
 @Module({
   providers: [
-    ...(useAwsServices() ? [] : [AwsService]),
+    ...(!useAwsServices() ? [] : [AwsService]),
     kinesisServiceProvider,
     opensearchServiceProvider,
   ],

src/aws/aws.service.ts

@@ -78,7 +78,7 @@ export class AwsService {
   private async createSignedHttpRequest(httpRequest: HttpRequest) {
     const sigV4Init = {
       credentials: defaultProvider(),
-      region: process.env.AWS_DEFAULT_REGION || 'ca-central-1',
+      region: process.env.AWS_DEFAULT_REGION || AWS_REGION,
       service: 'es',
       sha256: Sha256,
     };
@@ -132,7 +132,7 @@ export class AwsService {
       region: process.env.AWS_DEFAULT_REGION || AWS_REGION,
     });
     const stsAssumeRoleCommand = new AssumeRoleCommand({
-      RoleArn: process.env.AWS_KINESIS_ROLE_ARN,
+      RoleArn: process.env.AWS_ROLE_ARN,
       RoleSessionName: 'broker',
     });
     // Send command

src/collection/account.service.ts

@@ -11,6 +11,8 @@ import { Cron, CronExpression } from '@nestjs/schedule';
 import { plainToInstance } from 'class-transformer';
 import { catchError, lastValueFrom, of, switchMap } from 'rxjs';
 import ejs from 'ejs';
+import { CreateRequestContext, MikroORM } from '@mikro-orm/core';
+
 import { CollectionRepository } from '../persistence/interfaces/collection.repository';
 import { SystemRepository } from '../persistence/interfaces/system.repository';
 import { JwtRegistryEntity } from '../persistence/entity/jwt-registry.entity';
@@ -34,7 +36,6 @@ import { CollectionNameEnum } from '../persistence/dto/collection-dto-union.type
 import { RedisService } from '../redis/redis.service';
 import { VaultService } from '../vault/vault.service';
 import { GithubSyncService } from '../github/github-sync.service';
-import { CreateRequestContext } from '@mikro-orm/core';
 import { ServiceDto } from '../persistence/dto/service.dto';
 import { ProjectDto } from '../persistence/dto/project.dto';
 
@@ -54,6 +55,8 @@ export class AccountService {
     private readonly collectionRepository: CollectionRepository,
     private readonly systemRepository: SystemRepository,
     private readonly dateUtil: DateUtil,
+    // used by: @CreateRequestContext()
+    private readonly orm: MikroORM,
   ) {}
 
   async getRegisteryJwts(accountId: string): Promise<JwtRegistryEntity[]> {
@@ -429,8 +432,8 @@ export class AccountService {
     }
   }
 
-  @CreateRequestContext()
   @Cron(CronExpression.EVERY_MINUTE)
+  @CreateRequestContext()
   async runJwtLifecycle() {
     const CURRENT_TIME_MS = Date.now();
     const CURRENT_TIME_S = Math.floor(CURRENT_TIME_MS / MILLISECONDS_IN_SECOND);
@@ -479,8 +482,8 @@ export class AccountService {
     }
   }
 
-  @CreateRequestContext()
   @Cron(CronExpression.EVERY_HOUR)
+  @CreateRequestContext()
   async runJwtExpirationNotification() {
     const CURRENT_TIME_MS = Date.now();
     const CURRENT_TIME_S = Math.floor(CURRENT_TIME_MS / MILLISECONDS_IN_SECOND);

src/constants.ts

@@ -1,5 +1,7 @@
 export const BROKER_URL = process.env.BROKER_URL ?? '';
 
+export const APP_ENVIRONMENT = process.env.APP_ENVIRONMENT ?? '';
+
 export const AWS_REGION = 'ca-central-1';
 export const AWS_KINESIS_BUFFER_TIME = 100;
 export const AWS_KINESIS_MAX_RECORDS = 10;

src/graph/graph-sync/graph-sync.service.ts

@@ -1,7 +1,9 @@
 import { BadRequestException, Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
+import { CreateRequestContext, MikroORM } from '@mikro-orm/core';
 import { first, get, mapEntries, shake } from 'radash';
 import { v4 as uuidv4 } from 'uuid';
+import { plainToInstance } from 'class-transformer';
 
 import { GraphService } from '../graph.service';
 import { OpensearchService } from '../../aws/opensearch.service';
@@ -11,7 +13,6 @@ import { CollectionRepository } from '../../persistence/interfaces/collection.re
 import { VertexInsertDto } from '../../persistence/dto/vertex.dto';
 import { DateUtil, INTERVAL_HOUR_MS } from '../../util/date.util';
 import { IS_PRIMARY_NODE } from '../../constants';
-import { CreateRequestContext } from '@mikro-orm/core';
 
 @Injectable()
 export class GraphSyncService {
@@ -21,10 +22,12 @@ export class GraphSyncService {
     private readonly opensearchService: OpensearchService,
     private readonly collectionRepository: CollectionRepository,
     private readonly dateUtil: DateUtil,
+    // used by: @CreateRequestContext()
+    private readonly orm: MikroORM,
   ) {}
 
-  @CreateRequestContext()
   @Cron(CronExpression.EVERY_DAY_AT_6AM)
+  @CreateRequestContext()
   async runCollectionSync() {
     if (!IS_PRIMARY_NODE) {
       // Nodes that are not the primary one should not run sync
@@ -96,7 +99,7 @@ export class GraphSyncService {
       }
       const doc = bucketResult.hits.hits[0]._source;
 
-      const upsert: VertexInsertDto = {
+      const upsert = plainToInstance(VertexInsertDto, {
         collection: config.collection as keyof CollectionDtoUnion,
         data: shake(
           this.initFields(
@@ -121,7 +124,7 @@ export class GraphSyncService {
             }),
           ),
         ),
-      };
+      });
 
       try {
         await this.graphService.upsertVertex(null, upsert, 'name', key);

src/graph/graph.service.ts

@@ -422,20 +422,6 @@ export class GraphService {
     }
 
     vertex = this.mapCollectionToVertex(config, vertex, collection);
-    // const vertexErrors = await validate(vertex, {
-    //   whitelist: true,
-    //   forbidNonWhitelisted: true,
-    //   forbidUnknownValues: true,
-    // });
-    // if (vertexErrors.length > 0) {
-    //   throw new BadRequestException({
-    //     statusCode: 400,
-    //     message: 'Vertex validation error',
-    //     error: this.validatorUtil.buildFirstFailedPropertyErrorMsg(
-    //       vertexErrors[0],
-    //     ),
-    //   });
-    // }
 
     return [vertex, collection, config];
   }

src/intention/intention.service.ts

@@ -69,6 +69,7 @@ import { CloudObjectEmbeddable } from './entity/cloud-object.embeddable';
 import { CloudEmbeddable } from './entity/cloud.embeddable';
 import { ValidatorUtil } from '../util/validator.util';
 import { ActionSourceEmbeddable } from './entity/action-source.embeddable';
+import { MikroORM } from '@mikro-orm/mongodb';
 
 export interface IntentionOpenResponse {
   actions: {
@@ -101,6 +102,8 @@ export class IntentionService {
     private readonly persistenceUtilService: PersistenceUtilService,
     private readonly intentionUtilService: IntentionUtilService,
     private readonly validatorUtil: ValidatorUtil,
+    // used by: @CreateRequestContext()
+    private readonly orm: MikroORM,
   ) {}
 
   /**
@@ -961,8 +964,8 @@ export class IntentionService {
     return validationResult;
   }
 
-  @CreateRequestContext()
   @Cron(CronExpression.EVERY_MINUTE)
+  @CreateRequestContext()
   async handleIntentionExpiry() {
     if (!IS_PRIMARY_NODE) {
       // Nodes that are not the primary one should not do expiry
@@ -976,8 +979,8 @@ export class IntentionService {
     }
   }
 
-  @CreateRequestContext()
   @Cron(CronExpression.EVERY_HOUR)
+  @CreateRequestContext()
   async handleTransientCleanup() {
     if (!IS_PRIMARY_NODE) {
       // Nodes that are not the primary one should not do cleanup

src/main.ts

@@ -7,6 +7,7 @@ import session from 'express-session';
 import MongoStore from 'connect-mongo';
 import { AppModule } from './app.module';
 import { getMongoDbConnectionUrl } from './persistence/mongo/mongo.util';
+import { APP_ENVIRONMENT } from './constants';
 
 async function bootstrap() {
   const app = await NestFactory.create(AppModule, {
@@ -18,7 +19,9 @@ async function bootstrap() {
   app.enableCors();
 
   // Starts listening for shutdown hooks
-  // app.enableShutdownHooks();
+  if (APP_ENVIRONMENT) {
+    app.enableShutdownHooks();
+  }
 
   // Swagger docs
   const config = new DocumentBuilder()