Merge branch 'main' of https://github.com/dydxprotocol/v4-chain into …

…wl/send_hb

.github/workflows/indexer-build-docker-image-check.yml

@@ -77,3 +77,33 @@ jobs:
           DOCKER_BUILDKIT=1 docker build \
             --platform amd64 \
             -f Dockerfile.bazooka.remote .
+
+  check-build-auxo:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./indexer
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Install pnpm
+        run:
+          npm install -g [email protected]
+
+      - name: Build and install
+        run: |
+          pnpm install --loglevel warn --frozen-lockfile
+          pnpm run build:prod:all
+
+      - name: Build docker image for auxo
+        id: build-image
+        run: |
+          DOCKER_BUILDKIT=1 docker build \
+            --platform amd64 \
+            -f Dockerfile.auxo.remote .

CODEOWNERS

@@ -0,0 +1 @@
+* @dydxprotocol/backend

indexer/Dockerfile.auxo.remote

@@ -11,7 +11,11 @@ COPY ./patches ./patches
 
 # Copy auxo and imported packages
 COPY ./packages/base/ ./packages/base/
+COPY ./packages/dev/ ./packages/dev/
+COPY ./packages/postgres/ ./packages/postgres/
 COPY ./packages/v4-protos/ ./packages/v4-protos/
+COPY ./packages/kafka/ ./packages/kafka/
+COPY ./packages/redis/ ./packages/redis/
 COPY ./services/auxo/ ./services/auxo/
 COPY ./packages/v4-proto-parser/ ./packages/v4-proto-parser/
 

indexer/Dockerfile.bazooka.remote

@@ -11,6 +11,7 @@ COPY ./patches ./patches
 
 # Copy bazooka and imported packages
 COPY ./packages/base/ ./packages/base/
+COPY ./packages/dev/ ./packages/dev/
 COPY ./packages/postgres/ ./packages/postgres/
 COPY ./packages/v4-protos/ ./packages/v4-protos/
 COPY ./packages/kafka/ ./packages/kafka/

indexer/packages/base/__tests__/instance-id.test.ts

@@ -0,0 +1,87 @@
+import { setInstanceId, getInstanceId, resetForTests } from '../src/instance-id';
+import { axiosRequest } from '../src/axios';
+import { asMock } from '@dydxprotocol-indexer/dev';
+import logger from '../src/logger';
+import config from '../src/config';
+
+jest.mock('../src/axios', () => ({
+  ...(jest.requireActual('../src/axios') as object),
+  axiosRequest: jest.fn(),
+}));
+
+describe('instance-id', () => {
+  describe('setInstanceId', () => {
+    const defaultTaskArn = 'defaultTaskArn';
+    const defaultResponse = {
+      TaskARN: defaultTaskArn,
+    };
+    const ecsUrl = config.ECS_CONTAINER_METADATA_URI_V4;
+
+    beforeEach(() => {
+      config.ECS_CONTAINER_METADATA_URI_V4 = ecsUrl;
+      resetForTests();
+      jest.resetAllMocks();
+      jest.restoreAllMocks();
+      asMock(axiosRequest).mockResolvedValue(defaultResponse);
+    });
+
+    afterAll(() => {
+      jest.clearAllMocks();
+      jest.restoreAllMocks();
+    });
+
+    it('should set instance id to task ARN in staging', async () => {
+      jest.spyOn(config, 'isStaging').mockReturnValueOnce(true);
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      await setInstanceId();
+
+      expect(getInstanceId()).toEqual(defaultTaskArn);
+    });
+
+    it('should set instance id to task ARN in production', async () => {
+      jest.spyOn(config, 'isProduction').mockReturnValueOnce(true);
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      await setInstanceId();
+
+      expect(getInstanceId()).toEqual(defaultTaskArn);
+    });
+
+    it('should not call metadata endpoint if not production or staging', async () => {
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      await setInstanceId();
+
+      expect(getInstanceId()).not.toEqual(defaultTaskArn);
+      expect(asMock(axiosRequest)).not.toHaveBeenCalled();
+    });
+
+    it('should not set instance id if already set', async () => {
+      jest.spyOn(config, 'isStaging').mockReturnValue(true);
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      await setInstanceId();
+      const instanceId = getInstanceId();
+      await setInstanceId();
+
+      expect(getInstanceId()).toEqual(instanceId);
+      expect(axiosRequest).toHaveBeenCalledTimes(1);
+    });
+
+    it('should log error and set instance id to uuid if request errors', async () => {
+      jest.spyOn(config, 'isStaging').mockReturnValue(true);
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      const loggerErrorSpy = jest.spyOn(logger, 'error');
+      const emptyInstanceId = getInstanceId();
+      asMock(axiosRequest).mockRejectedValueOnce(new Error());
+      await setInstanceId();
+
+      expect(loggerErrorSpy).toHaveBeenCalledTimes(1);
+      expect(getInstanceId()).not.toEqual(emptyInstanceId);
+    });
+
+    it('should not call metadata endpoint if url is empty', async () => {
+      jest.spyOn(config, 'isStaging').mockReturnValue(true);
+      await setInstanceId();
+
+      expect(axiosRequest).not.toHaveBeenCalled();
+    });
+  });
+});

indexer/packages/base/package.json

@@ -2,14 +2,15 @@
   "name": "@dydxprotocol-indexer/base",
   "version": "0.0.1",
   "description": "",
-  "main": "build/index.js",
+  "main": "build/src/index.js",
   "devDependencies": {
     "@dydxprotocol-indexer/dev": "workspace:^0.0.1",
     "@types/big.js": "^6.1.5",
     "@types/express": "^4.17.13",
     "@types/lodash": "^4.14.182",
     "@types/traverse": "^0.6.32",
     "express": "^4.18.1",
+    "jest": "^28.1.2",
     "typescript": "^4.7.4"
   },
   "scripts": {
@@ -18,7 +19,7 @@
     "build": "rm -rf build/ && tsc",
     "build:prod": "pnpm run build",
     "build:watch": "pnpm run build -- --watch",
-    "test": "echo \"Error: no test specified\""
+    "test": "NODE_ENV=test jest --runInBand --forceExit"
   },
   "repository": {
     "type": "git",
@@ -38,6 +39,7 @@
     "hot-shots": "^9.1.0",
     "lodash": "^4.17.21",
     "traverse": "^0.6.6",
+    "uuid": "^8.3.2",
     "winston": "^3.8.1",
     "winston-transport": "^4.5.0",
     "@bugsnag/core": "^7.18.0",

indexer/packages/base/src/config.ts

@@ -30,14 +30,15 @@ export const baseConfigSchema = {
   SEND_BUGSNAG_ERRORS: parseBoolean({
     default: true,
   }),
-  SERVICE_NAME: parseString(),
+  SERVICE_NAME: parseString({ default: '' }),
 
   // Optional environment variables.
   NODE_ENV: parseString({ default: null }),
   ENABLE_LOGS_IN_TEST: parseBoolean({ default: false }),
   STATSD_HOST: parseString({ default: 'localhost' }),
   STATSD_PORT: parseInteger({ default: 8125 }),
   LOG_LEVEL: parseString({ default: 'debug' }),
+  ECS_CONTAINER_METADATA_URI_V4: parseString({ default: '' }),
 };
 
 export default parseSchema(baseConfigSchema);

indexer/packages/base/src/index.ts

@@ -13,6 +13,7 @@ export * from './constants';
 export * from './bugsnag';
 export * from './stats-util';
 export * from './date-helpers';
+export * from './instance-id';
 
 // Do this outside logger.ts to avoid a dependency cycle with logger transports that may trigger
 // additional logging.

indexer/packages/base/src/instance-id.ts

@@ -0,0 +1,51 @@
+import { v4 as uuidv4 } from 'uuid';
+
+import { axiosRequest } from './axios';
+import config from './config';
+import logger from './logger';
+
+let INSTANCE_ID: string = '';
+
+export function getInstanceId(): string {
+  return INSTANCE_ID;
+}
+
+export async function setInstanceId(): Promise<void> {
+  if (INSTANCE_ID !== '') {
+    return;
+  }
+  if (config.ECS_CONTAINER_METADATA_URI_V4 !== '' &&
+      (
+        config.isProduction() || config.isStaging()
+      )
+  ) {
+    // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html
+    const taskUrl = `${config.ECS_CONTAINER_METADATA_URI_V4}/task`;
+    try {
+      const response = await axiosRequest({
+        method: 'GET',
+        url: taskUrl,
+      }) as { TaskARN: string };
+      INSTANCE_ID = response.TaskARN;
+    } catch (error) {
+      logger.error({
+        at: 'instance-id#setInstanceId',
+        message: 'Failed to retrieve task arn from metadata endpoint. Falling back to uuid.',
+        error,
+        taskUrl,
+      });
+      INSTANCE_ID = uuidv4();
+    }
+  } else {
+    INSTANCE_ID = uuidv4();
+
+  }
+}
+
+// Exported for tests
+export function resetForTests(): void {
+  if (!config.isTest()) {
+    throw new Error(`resetForTests() cannot be called for env: ${config.NODE_ENV}`);
+  }
+  INSTANCE_ID = '';
+}

indexer/packages/base/tsconfig.json

@@ -4,6 +4,7 @@
     "outDir": "build"
   },
   "include": [
-    "src"
+    "src",
+    "__tests__"
   ]
 }

indexer/packages/kafka/src/batch-kafka-producer.ts

@@ -1,7 +1,8 @@
-import { logger } from '@dydxprotocol-indexer/base';
+import { logger, stats } from '@dydxprotocol-indexer/base';
 import { IHeaders, Producer, RecordMetadata } from 'kafkajs';
 import _ from 'lodash';
 
+import config from './config';
 import { KafkaTopics } from './types';
 
 /**
@@ -65,6 +66,7 @@ export class BatchKafkaProducer {
   }
 
   private sendBatch(): void {
+    const startTime: number = Date.now();
     if (!_.isEmpty(this.producerMessages)) {
       this.producerPromises.push(
         this.producer.send({ topic: this.topic, messages: this.producerMessages }),
@@ -80,7 +82,10 @@ export class BatchKafkaProducer {
         0,
       ),
       topic: this.topic,
+      sendTime: Date.now() - startTime,
     });
+    stats.gauge(`${config.SERVICE_NAME}.kafka_batch_size`, this.currentSize);
+    stats.timing(`${config.SERVICE_NAME}.kafka_batch_send_time`, Date.now() - startTime);
     this.producerMessages = [];
     this.currentSize = 0;
   }