fix: meta txn limiting - synchronize db (#270)

* add meta-transaction rate limiting * prettier format * wip add config * use constant header key * refactor config, metaTxnRateLimitter, add composable rate limiter Also: - fix tests * lint * refactor config * lint * add composable rate limiter * log truncated API key when rate limited. * add ability to configure multiple rate limiters per db field * change the table fields to integers * add value rate limiter, refactor composable rate limiter - add env-cmd as a dependency * lint * refactor rate limiters * add env-cmd to devDependencies * add rate limiter config * fix test env config * resolve discussions * rename rate limiter test * fix: force a synchronize Co-authored-by: oskar <[email protected]>
0xProject · Jun 25, 2020 · a3448ab · a3448ab
1 parent a057aee
commit a3448ab
Show file tree

Hide file tree

Showing 34 changed files with 1,008 additions and 111 deletions.
diff --git a/integration-test/transaction_watcher_service_test.ts b/integration-test/transaction_watcher_service_test.ts
@@ -11,7 +11,16 @@ import {
     createSwapServiceFromOrderBookService,
     getAppAsync,
 } from '../src/app';
-import * as config from '../src/config';
+import {
+    CHAIN_ID,
+    defaultHttpServiceConfig,
+    defaultHttpServiceWithRateLimiterConfig,
+    ETHEREUM_RPC_URL,
+    META_TXN_MAX_GAS_PRICE_GWEI,
+    META_TXN_RELAY_EXPECTED_MINED_SEC,
+    META_TXN_RELAY_PRIVATE_KEYS,
+    META_TXN_SIGNING_ENABLED,
+} from '../src/config';
 import { META_TRANSACTION_PATH, SRA_PATH } from '../src/constants';
 import { getDBConnectionAsync } from '../src/db_connection';
 import { TransactionEntity } from '../src/entities';
@@ -54,17 +63,17 @@ async function _waitUntilStatusAsync(
 describe('transaction watcher service', () => {
     before(async () => {
         const providerEngine = new Web3ProviderEngine();
-        providerEngine.addProvider(new RPCSubprovider(config.ETHEREUM_RPC_URL));
+        providerEngine.addProvider(new RPCSubprovider(ETHEREUM_RPC_URL));
         providerUtils.startProviderEngine(providerEngine);
         provider = providerEngine;
         connection = await getDBConnectionAsync();
         const txWatcherConfig: TransactionWatcherSignerServiceConfig = {
             provider: providerEngine,
-            chainId: config.CHAIN_ID,
-            signerPrivateKeys: config.META_TXN_RELAY_PRIVATE_KEYS,
-            expectedMinedInSec: config.META_TXN_RELAY_EXPECTED_MINED_SEC,
-            isSigningEnabled: config.META_TXN_SIGNING_ENABLED,
-            maxGasPriceGwei: config.META_TXN_MAX_GAS_PRICE_GWEI,
+            chainId: CHAIN_ID,
+            signerPrivateKeys: META_TXN_RELAY_PRIVATE_KEYS,
+            expectedMinedInSec: META_TXN_RELAY_EXPECTED_MINED_SEC,
+            isSigningEnabled: META_TXN_SIGNING_ENABLED,
+            maxGasPriceGwei: META_TXN_MAX_GAS_PRICE_GWEI,
             minSignerEthBalance: 0.1,
             transactionPollingIntervalMs: 100,
             heartbeatIntervalMs: 1000,
@@ -79,7 +88,10 @@ describe('transaction watcher service', () => {
         const stakingDataService = new StakingDataService(connection);
         const websocketOpts = { path: SRA_PATH };
         const swapService = createSwapServiceFromOrderBookService(orderBookService, provider);
-        const meshClient = new MeshClient(config.MESH_WEBSOCKET_URI, config.MESH_HTTP_URI);
+        const meshClient = new MeshClient(
+            defaultHttpServiceConfig.meshWebsocketUri,
+            defaultHttpServiceConfig.meshHttpUri,
+        );
         const metricsService = new MetricsService();
         metaTxnUser = new TestMetaTxnUser();
         ({ app } = await getAppAsync(
@@ -94,7 +106,7 @@ describe('transaction watcher service', () => {
                 websocketOpts,
                 metricsService,
             },
-            config,
+            defaultHttpServiceWithRateLimiterConfig,
         ));
     });
     it('sends a signed zeroex transaction correctly', async () => {

diff --git a/package.json b/package.json
@@ -13,8 +13,8 @@
         "build": "tsc -p tsconfig.json",
         "integration-test": "mocha --require source-map-support/register --require make-promises-safe lib/integration-test/**/*_test.js --timeout 200000 --exit",
         "test": "yarn test:rest && yarn test:rfqt",
-        "test:rest": "ETHEREUM_RPC_URL=http://localhost:8545 CHAIN_ID=1337 RFQT_API_KEY_WHITELIST='koolApiKey1,koolApikey2' RFQT_MAKER_ASSET_OFFERINGS='{\"https://mock-rfqt1.club\": [[\"0x871dd7c2b4b25e1aa18728e9d5f2af4c4e431f5c\",\"0x0b1ba0af832d7c05fd64161e0db78e85978e8082\"]]}' META_TXN_RELAY_ADDRESS=0x9eFCa436873b55a0d6AEa260f92DE50150360dca META_TXN_RELAY_PRIVATE_KEY=82b9c3b8d45f608badd8fda250a0d95088381540e850734519b659e1e1ac3e71 mocha --exclude lib/test/rfqt_test.js --require source-map-support/register --require make-promises-safe lib/test/**/*_test.js --timeout 200000 --exit",
-        "test:rfqt": "ETHEREUM_RPC_URL=http://localhost:8545 CHAIN_ID=1337 RFQT_API_KEY_WHITELIST='koolApiKey1,koolApikey2' RFQT_MAKER_ASSET_OFFERINGS='{\"https://mock-rfqt1.club\": [[\"0x871dd7c2b4b25e1aa18728e9d5f2af4c4e431f5c\",\"0x0b1ba0af832d7c05fd64161e0db78e85978e8082\"]]}' META_TXN_RELAY_ADDRESS=0x9eFCa436873b55a0d6AEa260f92DE50150360dca META_TXN_RELAY_PRIVATE_KEY=82b9c3b8d45f608badd8fda250a0d95088381540e850734519b659e1e1ac3e71 mocha --require source-map-support/register --require make-promises-safe lib/test/rfqt_test.js --timeout 200000 --exit",
+        "test:rest": "env-cmd -f ./test/test_env mocha --exclude lib/test/rfqt_test.js --require source-map-support/register --require make-promises-safe lib/test/**/*_test.js --timeout 200000 --exit",
+        "test:rfqt": "env-cmd -f ./test/test_env mocha --require source-map-support/register --require make-promises-safe lib/test/rfqt_test.js --timeout 200000 --exit",
         "dev": "nodemon -r dotenv/config src/index.ts | pino-pretty",
         "dev:service:http": "nodemon -r dotenv/config src/runners/http_service_runner.ts | pino-pretty",
         "dev:service:sra_http": "nodemon -r dotenv/config src/runners/http_sra_service_runner.ts | pino-pretty",
@@ -109,6 +109,7 @@
         "@types/supertest": "^2.0.8",
         "@types/web3": "^1.0.19",
         "@types/ws": "^6.0.2",
+        "env-cmd": "^10.1.0",
         "make-promises-safe": "^5.1.0",
         "mocha": "^6.2.2",
         "nodemon": "^1.19.4",

diff --git a/src/app.ts b/src/app.ts
@@ -15,9 +15,22 @@ import { OrderBookService } from './services/orderbook_service';
 import { StakingDataService } from './services/staking_data_service';
 import { SwapService } from './services/swap_service';
 import { TransactionWatcherSignerService } from './services/transaction_watcher_signer_service';
-import { WebsocketSRAOpts } from './types';
+import {
+    HttpServiceConfig,
+    MetaTransactionDailyLimiterConfig,
+    MetaTransactionRollingLimiterConfig,
+    WebsocketSRAOpts,
+} from './types';
 import { MeshClient } from './utils/mesh_client';
 import { OrderStoreDbAdapter } from './utils/order_store_db_adapter';
+import {
+    AvailableRateLimiter,
+    DatabaseKeysUsedForRateLimiter,
+    MetaTransactionDailyLimiter,
+    MetaTransactionRateLimiter,
+    MetaTransactionRollingLimiter,
+} from './utils/rate-limiters';
+import { MetaTransactionComposableLimiter } from './utils/rate-limiters/meta_transaction_composable_rate_limiter';
 
 export interface AppDependencies {
     connection: Connection;
@@ -30,6 +43,7 @@ export interface AppDependencies {
     websocketOpts: Partial<WebsocketSRAOpts>;
     transactionWatcherService?: TransactionWatcherSignerService;
     metricsService?: MetricsService;
+    rateLimiter?: MetaTransactionRateLimiter;
 }
 
 /**
@@ -38,29 +52,30 @@ export interface AppDependencies {
  */
 export async function getDefaultAppDependenciesAsync(
     provider: SupportedProvider,
-    config: {
-        // hack (xianny): the Mesh client constructor has a fire-and-forget promise so we are unable
-        // to catch initialisation errors. Allow the calling function to skip Mesh initialization by
-        // not providing a websocket URI
-        MESH_WEBSOCKET_URI?: string;
-        MESH_HTTP_URI?: string;
-        ENABLE_PROMETHEUS_METRICS: boolean;
-    },
+    config: HttpServiceConfig,
 ): Promise<AppDependencies> {
     const connection = await getDBConnectionAsync();
     const stakingDataService = new StakingDataService(connection);
 
     let meshClient: MeshClient | undefined;
-    if (config.MESH_WEBSOCKET_URI !== undefined) {
-        meshClient = new MeshClient(config.MESH_WEBSOCKET_URI, config.MESH_HTTP_URI);
+    // hack (xianny): the Mesh client constructor has a fire-and-forget promise so we are unable
+    // to catch initialisation errors. Allow the calling function to skip Mesh initialization by
+    // not providing a websocket URI
+    if (config.meshWebsocketUri !== undefined) {
+        meshClient = new MeshClient(config.meshWebsocketUri, config.meshHttpUri);
     } else {
         logger.warn(`Skipping Mesh client creation because no URI provided`);
     }
     let metricsService: MetricsService | undefined;
-    if (config.ENABLE_PROMETHEUS_METRICS) {
+    if (config.enablePrometheusMetrics) {
         metricsService = new MetricsService();
     }
 
+    let rateLimiter: MetaTransactionRateLimiter | undefined;
+    if (config.metaTxnRateLimiters !== undefined) {
+        rateLimiter = createMetaTransactionRateLimiterFromConfig(connection, config);
+    }
+
     const orderBookService = new OrderBookService(connection, meshClient);
 
     let swapService: SwapService | undefined;
@@ -84,6 +99,7 @@ export async function getDefaultAppDependenciesAsync(
         provider,
         websocketOpts,
         metricsService,
+        rateLimiter,
     };
 }
 /**
@@ -96,14 +112,7 @@ export async function getDefaultAppDependenciesAsync(
  */
 export async function getAppAsync(
     dependencies: AppDependencies,
-    config: {
-        HTTP_PORT: number;
-        ETHEREUM_RPC_URL: string;
-        HTTP_KEEP_ALIVE_TIMEOUT: number;
-        HTTP_HEADERS_TIMEOUT: number;
-        ENABLE_PROMETHEUS_METRICS: boolean;
-        PROMETHEUS_PORT: number;
-    },
+    config: HttpServiceConfig,
 ): Promise<{ app: Express.Application; server: Server }> {
     const app = express();
     const { server, wsService } = await runHttpServiceAsync(dependencies, config, app);
@@ -126,6 +135,46 @@ export async function getAppAsync(
     return { app, server };
 }
 
+function createMetaTransactionRateLimiterFromConfig(
+    dbConnection: Connection,
+    config: HttpServiceConfig,
+): MetaTransactionRateLimiter {
+    const rateLimiterConfigEntries = Object.entries(config.metaTxnRateLimiters);
+    const configuredRateLimiters = rateLimiterConfigEntries
+        .map(entries => {
+            const [dbField, rateLimiters] = entries;
+
+            return Object.entries(rateLimiters).map(rateLimiterEntry => {
+                const [limiterType, value] = rateLimiterEntry;
+                switch (limiterType) {
+                    case AvailableRateLimiter.Daily: {
+                        const dailyConfig = value as MetaTransactionDailyLimiterConfig;
+                        return new MetaTransactionDailyLimiter(
+                            dbField as DatabaseKeysUsedForRateLimiter,
+                            dbConnection,
+                            dailyConfig,
+                        );
+                    }
+                    case AvailableRateLimiter.Rolling: {
+                        const rollingConfig = value as MetaTransactionRollingLimiterConfig;
+                        return new MetaTransactionRollingLimiter(
+                            dbField as DatabaseKeysUsedForRateLimiter,
+                            dbConnection,
+                            rollingConfig,
+                        );
+                    }
+                    default:
+                        throw new Error('unknown rate limiter type');
+                }
+            });
+        })
+        .reduce((prev, cur, []) => {
+            return prev.concat(...cur);
+        });
+
+    return new MetaTransactionComposableLimiter(configuredRateLimiters);
+}
+
 /**
  * Instantiates SwapService using the provided OrderBookService and ethereum RPC provider.
  */

diff --git a/src/config.ts b/src/config.ts
@@ -16,7 +16,8 @@ import {
     NULL_BYTES,
 } from './constants';
 import { TokenMetadatasForChains } from './token_metadatas_for_networks';
-import { ChainId } from './types';
+import { ChainId, HttpServiceConfig, MetaTransactionRateLimitConfig } from './types';
+import { parseUtils } from './utils/parse_utils';
 
 enum EnvVarType {
     AddressList,
@@ -35,8 +36,14 @@ enum EnvVarType {
     APIKeys,
     PrivateKeys,
     RfqtMakerAssetOfferings,
+    RateLimitConfig,
 }
 
+// Log level for pino.js
+export const LOG_LEVEL: string = _.isEmpty(process.env.LOG_LEVEL)
+    ? 'info'
+    : assertEnvVarType('LOG_LEVEL', process.env.LOG_LEVEL, EnvVarType.NonEmptyString);
+
 // Network port to listen on
 export const HTTP_PORT = _.isEmpty(process.env.HTTP_PORT)
     ? 3000
@@ -204,6 +211,16 @@ export const META_TXN_MAX_GAS_PRICE_GWEI: BigNumber = _.isEmpty(process.env.META
     ? new BigNumber(50)
     : assertEnvVarType('META_TXN_MAX_GAS_PRICE_GWEI', process.env.META_TXN_MAX_GAS_PRICE_GWEI, EnvVarType.UnitAmount);
 
+export const META_TXN_RATE_LIMITER_CONFIG: MetaTransactionRateLimitConfig | undefined = _.isEmpty(
+    process.env.META_TXN_RATE_LIMIT_TYPE,
+)
+    ? undefined
+    : assertEnvVarType(
+          'META_TXN_RATE_LIMITER_CONFIG',
+          process.env.META_TXN_RATE_LIMITER_CONFIG,
+          EnvVarType.RateLimitConfig,
+      );
+
 // Whether or not prometheus metrics should be enabled.
 // tslint:disable-next-line:boolean-naming
 export const ENABLE_PROMETHEUS_METRICS: boolean = _.isEmpty(process.env.ENABLE_PROMETHEUS_METRICS)
@@ -270,6 +287,22 @@ export const ASSET_SWAPPER_MARKET_ORDERS_OPTS: Partial<SwapQuoteRequestOpts> = {
     gasSchedule: GAS_SCHEDULE,
 };
 
+export const defaultHttpServiceConfig: HttpServiceConfig = {
+    httpPort: HTTP_PORT,
+    ethereumRpcUrl: ETHEREUM_RPC_URL,
+    httpKeepAliveTimeout: HTTP_KEEP_ALIVE_TIMEOUT,
+    httpHeadersTimeout: HTTP_HEADERS_TIMEOUT,
+    enablePrometheusMetrics: ENABLE_PROMETHEUS_METRICS,
+    prometheusPort: PROMETHEUS_PORT,
+    meshWebsocketUri: MESH_WEBSOCKET_URI,
+    meshHttpUri: MESH_HTTP_URI,
+};
+
+export const defaultHttpServiceWithRateLimiterConfig: HttpServiceConfig = {
+    ...defaultHttpServiceConfig,
+    metaTxnRateLimiters: META_TXN_RATE_LIMITER_CONFIG,
+};
+
 function assertEnvVarType(name: string, value: any, expectedType: EnvVarType): any {
     let returnValue;
     switch (expectedType) {
@@ -343,6 +376,9 @@ function assertEnvVarType(name: string, value: any, expectedType: EnvVarType): a
                 throw new Error(`${name} must be supplied`);
             }
             return value;
+        case EnvVarType.RateLimitConfig:
+            assert.isString(name, value);
+            return parseUtils.parseJsonStringForMetaTransactionRateLimitConfigOrThrow(value);
         case EnvVarType.APIKeys:
             assert.isString(name, value);
             const apiKeys = (value as string).split(',');
@@ -353,7 +389,6 @@ function assertEnvVarType(name: string, value: any, expectedType: EnvVarType): a
                 }
             });
             return apiKeys;
-
         case EnvVarType.RfqtMakerAssetOfferings:
             const offerings: RfqtMakerAssetOfferings = JSON.parse(value);
             // tslint:disable-next-line:forin

diff --git a/src/constants.ts b/src/constants.ts
@@ -48,6 +48,7 @@ export const STAKING_PATH = '/staking';
 export const SWAP_PATH = '/swap/v0';
 export const META_TRANSACTION_PATH = '/meta_transaction/v0';
 export const METRICS_PATH = '/metrics';
+export const API_KEY_HEADER = '0x-api-key';
 
 // Docs
 export const SWAP_DOCS_URL = 'https://0x.org/docs/api#swap';

diff --git a/src/entities/TransactionEntity.ts b/src/entities/TransactionEntity.ts
@@ -35,10 +35,10 @@ export class TransactionEntity {
     @Column({ name: 'expected_mined_in_sec', type: 'int' })
     public expectedMinedInSec?: number;
 
-    @Column({ name: 'gas_price', type: 'varchar', nullable: true, transformer: BigNumberTransformer })
+    @Column({ name: 'gas_price', type: 'bigint', nullable: true, transformer: BigNumberTransformer })
     public gasPrice?: BigNumber;
 
-    @Column({ name: 'value', type: 'varchar', nullable: true, transformer: BigNumberTransformer })
+    @Column({ name: 'value', type: 'bigint', nullable: true, transformer: BigNumberTransformer })
     public value?: BigNumber;
 
     @Column({ name: 'gas', type: 'int', nullable: true })
@@ -59,10 +59,13 @@ export class TransactionEntity {
     @Column({ name: 'tx_status', type: 'int', nullable: true })
     public txStatus?: number;
 
-    @CreateDateColumn({ name: 'created_at' })
+    @Column({ name: 'api_key', type: 'varchar', nullable: true })
+    public apiKey?: string;
+
+    @CreateDateColumn({ name: 'created_at', type: 'timestamptz' })
     public createdAt?: Date;
 
-    @UpdateDateColumn({ name: 'updated_at' })
+    @UpdateDateColumn({ name: 'updated_at', type: 'timestamptz' })
     public updatedAt?: Date;
 
     @Column({ name: 'expected_at', type: 'timestamptz' })
@@ -97,6 +100,7 @@ export class TransactionEntity {
             txHash: '',
             to: '',
             data: '',
+            apiKey: '',
             takerAddress: '',
             status: '',
             expectedMinedInSec: META_TXN_RELAY_EXPECTED_MINED_SEC,
@@ -114,6 +118,7 @@ export class TransactionEntity {
         this.takerAddress = opts.takerAddress;
         this.to = opts.to;
         this.data = opts.data;
+        this.apiKey = opts.apiKey;
         this.status = opts.status;
         this.expectedMinedInSec = opts.expectedMinedInSec;
         this.nonce = opts.nonce;

diff --git a/src/entities/types.ts b/src/entities/types.ts
@@ -2,6 +2,8 @@ import { BigNumber } from '@0x/utils';
 
 export interface TransactionEntityOpts {
     refHash: string;
+    apiKey?: string;
+    txHash?: string;
     takerAddress?: string;
     status: string;
     expectedMinedInSec: number;
@@ -12,7 +14,6 @@ export interface TransactionEntityOpts {
     nonce?: number;
     gasPrice?: BigNumber;
     gas?: number;
-    txHash?: string;
     gasUsed?: number;
     blockNumber?: number;
     // Ethereum tx status, 1 == success, 0 == failure