Skip to content

Commit

Permalink
Merge branch 'main' into tq/dec-2141
Browse files Browse the repository at this point in the history
  • Loading branch information
@tqin7
tqin7 committed Oct 10, 2023
2 parents f4ccb40 + f50431e commit 571a00d
Show file tree
Hide file tree
Showing 90 changed files with 764 additions and 1,369 deletions.
2 changes: 2 additions & 0 deletions indexer/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
* [#469](https://github.com/dydxprotocol/v4-chain/pull/469) Added a reason field to `/screen` endpoint to display a reason for blocking an address.

### Bug Fixes
* [#528](https://github.com/dydxprotocol/v4-chain/pull/528) Fixed bug with bulk SQL queries with nullable numeric / string / boolean values.

* [#496](https://github.com/dydxprotocol/v4-chain/pull/496) Don't geo-block requests made to `comlink` if it's from an internal ip.

* [#460](https://github.com/dydxprotocol/v4-chain/pull/460/files) Fixed track lag timing to output positive numbers.
Expand Down
26 changes: 22 additions & 4 deletions indexer/packages/postgres/__tests__/stores/compliance-data-table.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,9 +243,13 @@ describe('Compliance data store', () => {
);
expect(complianceData.length).toEqual(1);

const updatedTime: string = DateTime.fromISO(
const updatedTime1: string = DateTime.fromISO(
nonBlockedComplianceData.updatedAt!,
).plus(10).toUTC().toISO();
const updatedTime2: string = DateTime.fromISO(
nonBlockedComplianceData.updatedAt!,
).plus(20).toUTC().toISO();
const otherAddress: string = 'dydx1scu097p2sstqzupe6t687kpc2w4sv665fedctf';

await ComplianceDataTable.bulkUpsert(
[
Expand All @@ -254,7 +258,14 @@ describe('Compliance data store', () => {
...nonBlockedComplianceData,
riskScore: '30.00',
blocked: true,
updatedAt: updatedTime,
updatedAt: updatedTime1,
},
{
...nonBlockedComplianceData,
address: otherAddress,
riskScore: undefined,
blocked: false,
updatedAt: updatedTime2,
},
],
);
Expand All @@ -264,13 +275,20 @@ describe('Compliance data store', () => {
[],
{ readReplica: true },
);
expect(complianceData.length).toEqual(2);
expect(complianceData.length).toEqual(3);
expect(complianceData[0]).toEqual(blockedComplianceData);
expect(complianceData[1]).toEqual({
...nonBlockedComplianceData,
riskScore: '30.00',
blocked: true,
updatedAt: updatedTime,
updatedAt: updatedTime1,
});
expect(complianceData[2]).toEqual({
...nonBlockedComplianceData,
address: otherAddress,
riskScore: null,
blocked: false,
updatedAt: updatedTime2,
});
});
});
18 changes: 15 additions & 3 deletions indexer/packages/postgres/src/helpers/stores-helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,10 +110,10 @@ export function setBulkRowsForUpdate<T extends string>({
}): string[] {
return objectArray.map((object) => columns.map((col) => {
if (stringColumns && stringColumns.includes(col)) {
return `'${object[col]}'`;
return `'${castNull(object[col])}'`;
}
if (numericColumns && numericColumns.includes(col)) {
return `${_.get(object, col)}`;
return `${castNull(_.get(object, col))}`;
}
if (bigintColumns && bigintColumns.includes(col)) {
return castValue(object[col] as number | undefined | null, 'bigint');
Expand All @@ -128,12 +128,24 @@ export function setBulkRowsForUpdate<T extends string>({
return castBinaryValue(object[col] as Buffer | null | undefined);
}
if (booleanColumns && booleanColumns.includes(col)) {
return `${object[col]}`;
return `${castNull(object[col])}`;
}
throw new Error(`Unsupported column for bulk update: ${col}`);
}).join(', '));
}

/**
* If the value is null || undefined, return 'NULL'
*/
function castNull(
value: Buffer | string | number | boolean | null | undefined,
): string {
if (value === null || value === undefined) {
return 'NULL';
}
return `${value}`;
}

/**
* If the value is null || undefined, return 'NULL' casted with typesuffix, otherwise return
* the stringified value in quotes casted with typesuffix.
Expand Down
22 changes: 0 additions & 22 deletions indexer/packages/v4-protos/src/codegen/dydxprotocol/indexer/indexer_manager/event.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,12 +118,6 @@ export interface IndexerEventsStoreValueSDKType {
export interface IndexerTendermintEvent {
/** Subtype of the event e.g. "order_fill", "subaccount_update", etc. */
subtype: string;
/**
* Base64 encoded proto from the Tendermint event.
* TODO(DEC-1720): Change to bytes post-migration.
*/

data: string;
transactionIndex?: number;
blockEvent?: IndexerTendermintEvent_BlockEvent;
/**
Expand All @@ -149,12 +143,6 @@ export interface IndexerTendermintEvent {
export interface IndexerTendermintEventSDKType {
/** Subtype of the event e.g. "order_fill", "subaccount_update", etc. */
subtype: string;
/**
* Base64 encoded proto from the Tendermint event.
* TODO(DEC-1720): Change to bytes post-migration.
*/

data: string;
transaction_index?: number;
block_event?: IndexerTendermintEvent_BlockEventSDKType;
/**
Expand Down Expand Up @@ -301,7 +289,6 @@ export const IndexerEventsStoreValue = {
function createBaseIndexerTendermintEvent(): IndexerTendermintEvent {
return {
subtype: "",
data: "",
transactionIndex: undefined,
blockEvent: undefined,
eventIndex: 0,
Expand All @@ -316,10 +303,6 @@ export const IndexerTendermintEvent = {
writer.uint32(10).string(message.subtype);
}

if (message.data !== "") {
writer.uint32(18).string(message.data);
}

if (message.transactionIndex !== undefined) {
writer.uint32(24).uint32(message.transactionIndex);
}
Expand Down Expand Up @@ -356,10 +339,6 @@ export const IndexerTendermintEvent = {
message.subtype = reader.string();
break;

case 2:
message.data = reader.string();
break;

case 3:
message.transactionIndex = reader.uint32();
break;
Expand Down Expand Up @@ -392,7 +371,6 @@ export const IndexerTendermintEvent = {
fromPartial(object: DeepPartial<IndexerTendermintEvent>): IndexerTendermintEvent {
const message = createBaseIndexerTendermintEvent();
message.subtype = object.subtype ?? "";
message.data = object.data ?? "";
message.transactionIndex = object.transactionIndex ?? undefined;
message.blockEvent = object.blockEvent ?? undefined;
message.eventIndex = object.eventIndex ?? 0;
Expand Down
32 changes: 32 additions & 0 deletions indexer/services/comlink/__tests__/controllers/api/v4/compliance-controller.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ describe('compliance-controller#V4', () => {
expect(response.body.restricted).toEqual(false);
expect(response.reason).toBeUndefined();
expect(stats.timing).toHaveBeenCalledTimes(1);
expect(stats.increment).toHaveBeenCalledWith(
'comlink.compliance-controller.compliance_data_cache_miss',
{ provider: complianceProvider.provider },
);
expect(complianceProvider.client.getComplianceResponse).toHaveBeenCalledTimes(1);

data = await ComplianceTable.findAll({}, [], {});
Expand Down Expand Up @@ -98,6 +102,10 @@ describe('compliance-controller#V4', () => {
expect(response.body.restricted).toEqual(false);
expect(response.reason).toBeUndefined();
expect(stats.timing).toHaveBeenCalledTimes(1);
expect(stats.increment).toHaveBeenCalledWith(
'comlink.compliance-controller.compliance_data_cache_hit',
{ provider: complianceProvider.provider },
);
expect(complianceProvider.client.getComplianceResponse).toHaveBeenCalledTimes(0);

data = await ComplianceTable.findAll({}, [], {});
Expand All @@ -121,6 +129,10 @@ describe('compliance-controller#V4', () => {
expect(response.body.restricted).toEqual(true);
expect(response.body.reason).toEqual(INDEXER_COMPLIANCE_BLOCKED_PAYLOAD);
expect(stats.timing).toHaveBeenCalledTimes(1);
expect(stats.increment).toHaveBeenCalledWith(
'comlink.compliance-controller.compliance_data_cache_hit',
{ provider: complianceProvider.provider },
);
expect(complianceProvider.client.getComplianceResponse).toHaveBeenCalledTimes(0);

data = await ComplianceTable.findAll({}, [], {});
Expand Down Expand Up @@ -149,6 +161,14 @@ describe('compliance-controller#V4', () => {
expect(response.body.restricted).toEqual(false);
expect(response.body.reason).toBeUndefined();
expect(stats.timing).toHaveBeenCalledTimes(1);
expect(stats.increment).toHaveBeenCalledWith(
'comlink.compliance-controller.compliance_data_cache_hit',
{ provider: complianceProvider.provider },
);
expect(stats.increment).toHaveBeenCalledWith(
'comlink.compliance-controller.refresh_compliance_data_cache',
{ provider: complianceProvider.provider },
);
expect(complianceProvider.client.getComplianceResponse).toHaveBeenCalledTimes(1);

data = await ComplianceTable.findAll({}, [], {});
Expand Down Expand Up @@ -183,6 +203,10 @@ describe('compliance-controller#V4', () => {
expect(response.body.restricted).toEqual(true);
expect(response.body.reason).toEqual(INDEXER_COMPLIANCE_BLOCKED_PAYLOAD);
expect(stats.timing).toHaveBeenCalledTimes(1);
expect(stats.increment).toHaveBeenCalledWith(
'comlink.compliance-controller.compliance_data_cache_hit',
{ provider: complianceProvider.provider },
);
expect(complianceProvider.client.getComplianceResponse).toHaveBeenCalledTimes(0);

data = await ComplianceTable.findAll({}, [], {});
Expand All @@ -207,6 +231,10 @@ describe('compliance-controller#V4', () => {
errorMsg: 'Too many requests',
expectedStatus: 429,
});
expect(stats.increment).toHaveBeenCalledWith(
'comlink.compliance-controller.compliance_screen_rate_limited_attempts',
{ provider: complianceProvider.provider },
);
});

it('Get /screen with multiple new address globally gets rate-limited', async () => {
Expand All @@ -224,6 +252,10 @@ describe('compliance-controller#V4', () => {
errorMsg: 'Too many requests',
expectedStatus: 429,
});
expect(stats.increment).toHaveBeenCalledWith(
'comlink.compliance-controller.compliance_screen_rate_limited_attempts',
{ provider: complianceProvider.provider },
);
});
});
});
25 changes: 24 additions & 1 deletion indexer/services/comlink/src/controllers/api/v4/compliance-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ class ComplianceController extends Controller {
complianceProvider.provider,
);

if (complianceData !== undefined) {
stats.increment(
`${config.SERVICE_NAME}.${controllerName}.compliance_data_cache_hit`,
{ provider: complianceProvider.provider },
);
}

// Immediately return for blocked addresses, do not refresh
if (complianceData?.blocked) {
return {
Expand All @@ -61,7 +68,19 @@ class ComplianceController extends Controller {

if (complianceData === undefined || DateTime.fromISO(complianceData.updatedAt) < ageThreshold) {
await checkRateLimit(this.ipAddress);
// TODO(IND-369): Use Ellptic client

if (complianceData === undefined) {
stats.increment(
`${config.SERVICE_NAME}.${controllerName}.compliance_data_cache_miss`,
{ provider: complianceProvider.provider },
);
} else {
stats.increment(
`${config.SERVICE_NAME}.${controllerName}.refresh_compliance_data_cache`,
{ provider: complianceProvider.provider },
);
}

const response:
ComplianceClientResponse = await complianceProvider.client.getComplianceResponse(
address,
Expand Down Expand Up @@ -111,6 +130,10 @@ router.get(
return res.send(response);
} catch (error) {
if (error instanceof TooManyRequestsError) {
stats.increment(
`${config.SERVICE_NAME}.${controllerName}.compliance_screen_rate_limited_attempts`,
{ provider: complianceProvider.provider },
);
return create4xxResponse(
res,
'Too many requests',
Expand Down
2 changes: 0 additions & 2 deletions indexer/services/ender/__tests__/helpers/indexer-proto-helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,6 @@ export function createIndexerTendermintEvent(
// blockEvent
return {
subtype,
data: '',
dataBytes,
blockEvent: IndexerTendermintEvent_BlockEvent.BLOCK_EVENT_END_BLOCK,
eventIndex,
Expand All @@ -99,7 +98,6 @@ export function createIndexerTendermintEvent(
// transactionIndex
return {
subtype,
data: '',
dataBytes,
transactionIndex,
eventIndex,
Expand Down
1 change: 0 additions & 1 deletion indexer/services/ender/__tests__/lib/helper.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ describe('helper', () => {
...eventFields,
subtype: 'order_fill',
dataBytes: Uint8Array.from(Buffer.from('data')),
data: 'data',
eventIndex: 0,
version: 1,
};
Expand Down
43 changes: 41 additions & 2 deletions indexer/services/roundtable/__tests__/tasks/update-compliance-data.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ import { ClientAndProvider } from '../../src/helpers/compliance-clients';
import { ComplianceClientResponse } from '@dydxprotocol-indexer/compliance';
import { DateTime } from 'luxon';

interface ComplianceClientResponseWithNull extends Omit<ComplianceClientResponse, 'riskScore'> {
riskScore: string | undefined | null;
}

describe('update-compliance-data', () => {
let mockProvider: ClientAndProvider;

Expand Down Expand Up @@ -215,6 +219,41 @@ describe('update-compliance-data', () => {
expectTimingStats(mockProvider.provider);
});

it('succeeds with an active address to update, undefined risk-score', async () => {
// Seed database with compliance data older than the age threshold for active addresses
await setupComplianceData(config.MAX_ACTIVE_COMPLIANCE_DATA_AGE_SECONDS * 2);

setupMockProvider(
mockProvider,
{ [testConstants.defaultAddress]: { blocked: true, riskScore: undefined } },
);

await updateComplianceDataTask(mockProvider);

const complianceData: ComplianceDataFromDatabase[] = await ComplianceTable.findAll({}, [], {});
expect(complianceData).toHaveLength(1);
expectUpdatedCompliance(
complianceData[0],
{
address: testConstants.defaultAddress,
blocked: true,
riskScore: null,
},
mockProvider.provider,
);

expectGaugeStats({
activeAddresses: 1,
newAddresses: 0,
oldAddresses: 0,
addressesScreened: 1,
upserted: 1,
},
mockProvider.provider,
);
expectTimingStats(mockProvider.provider);
});

it('succeeds with an old address to update', async () => {
// Seed database with old compliance data, and set up subaccounts to not be active
await Promise.all([
Expand Down Expand Up @@ -529,7 +568,7 @@ async function setupSubaccounts(

function setupMockProvider(
clientAndProvider: ClientAndProvider,
expectedResponses: {[address: string]: {blocked: boolean, riskScore: string}},
expectedResponses: {[address: string]: {blocked: boolean, riskScore: string | undefined }},
): void {
// eslint-disable-next-line no-param-reassign
clientAndProvider.client.getComplianceResponse = jest.fn().mockImplementation(
Expand All @@ -549,7 +588,7 @@ function setupMockProvider(

function expectUpdatedCompliance(
complianceData: ComplianceDataFromDatabase,
complianceClientResponse: ComplianceClientResponse,
complianceClientResponse: ComplianceClientResponseWithNull,
provider: string,
): void {
expect(complianceData).toEqual(expect.objectContaining({
Expand Down
Loading

0 comments on commit 571a00d

Please sign in to comment.