From e92c788fd965b0a4c0e5af0a73acd2f42d4b12df Mon Sep 17 00:00:00 2001 From: Sergey Kintsel Date: Thu, 12 Sep 2024 13:11:08 +0100 Subject: [PATCH] Improve multisig account fetching --- packages/multisig/src/fetch.test.ts | 126 ++++++++++++++++++-------- packages/multisig/src/fetch.ts | 49 +++++++--- packages/multisig/src/helpers.test.ts | 11 ++- packages/multisig/src/types.ts | 1 + 4 files changed, 132 insertions(+), 55 deletions(-) diff --git a/packages/multisig/src/fetch.test.ts b/packages/multisig/src/fetch.test.ts index 50d74aca04..8dc751a643 100644 --- a/packages/multisig/src/fetch.test.ts +++ b/packages/multisig/src/fetch.test.ts @@ -1,6 +1,7 @@ import * as api from "@tzkt/sdk-api"; import { DefaultNetworks, GHOSTNET, mockImplicitAddress } from "@umami/tezos"; import axios from "axios"; +import range from "lodash/range"; import { CODE_HASH, @@ -13,43 +14,9 @@ import { const mockedAxios = jest.spyOn(axios, "get"); const mockedContractsGet = jest.spyOn(api, "contractsGet"); +const mockedContractsGetCount = jest.spyOn(api, "contractsGetCount"); const multisigContracts = [ - { - id: 533705, - type: "contract", - address: "KT1Mqvf7bnYe4Ty2n7ZbGkdbebCd4WoTJUUp", - kind: "smart_contract", - balance: 0, - creator: { address: "tz1LbSsDSmekew3prdDGx1nS22ie6jjBN6B3" }, - numContracts: 0, - activeTokensCount: 0, - tokensCount: 0, - tokenBalancesCount: 0, - tokenTransfersCount: 0, - numDelegations: 0, - numOriginations: 1, - numTransactions: 0, - numReveals: 0, - numMigrations: 0, - transferTicketCount: 0, - increasePaidStorageCount: 0, - eventsCount: 0, - firstActivity: 1636117, - firstActivityTime: "2022-12-09T16:49:25Z", - lastActivity: 1636117, - lastActivityTime: "2022-12-09T16:49:25Z", - storage: { - owner: "tz1LbSsDSmekew3prdDGx1nS22ie6jjBN6B3", - signers: ["tz1LbSsDSmekew3prdDGx1nS22ie6jjBN6B3", "tz1dyX3B1CFYa2DfdFLyPtiJCfQRUgPVME6E"], - metadata: 216412, - threshold: "1", - last_op_id: "0", - pending_ops: 216411, - }, - typeHash: 1963879877, - codeHash: -1890025422, - }, { id: 536908, type: "contract", @@ -124,11 +91,47 @@ const multisigContracts = [ typeHash: 1963879877, codeHash: -1890025422, }, + { + id: 533705, + type: "contract", + address: "KT1Mqvf7bnYe4Ty2n7ZbGkdbebCd4WoTJUUp", + kind: "smart_contract", + balance: 0, + creator: { address: "tz1LbSsDSmekew3prdDGx1nS22ie6jjBN6B3" }, + numContracts: 0, + activeTokensCount: 0, + tokensCount: 0, + tokenBalancesCount: 0, + tokenTransfersCount: 0, + numDelegations: 0, + numOriginations: 1, + numTransactions: 0, + numReveals: 0, + numMigrations: 0, + transferTicketCount: 0, + increasePaidStorageCount: 0, + eventsCount: 0, + firstActivity: 1636117, + firstActivityTime: "2022-12-09T16:49:25Z", + lastActivity: 1636117, + lastActivityTime: "2022-12-09T16:49:25Z", + storage: { + owner: "tz1LbSsDSmekew3prdDGx1nS22ie6jjBN6B3", + signers: ["tz1LbSsDSmekew3prdDGx1nS22ie6jjBN6B3", "tz1dyX3B1CFYa2DfdFLyPtiJCfQRUgPVME6E"], + metadata: 216412, + threshold: "1", + last_op_id: "0", + pending_ops: 216411, + }, + typeHash: 1963879877, + codeHash: -1890025422, + }, ]; describe("multisig fetch", () => { const expectedMockedMultisigContracts = [ { + id: 533705, address: "KT1Mqvf7bnYe4Ty2n7ZbGkdbebCd4WoTJUUp", storage: { pending_ops: 216411, @@ -137,6 +140,7 @@ describe("multisig fetch", () => { }, }, { + id: 536908, address: "KT1VwWbTMRN5uX4bfxCcpJnPP6iAhboqhGZr", storage: { pending_ops: 219458, @@ -149,6 +153,7 @@ describe("multisig fetch", () => { }, }, { + id: 537023, address: "KT1Vdhz4izz7LASWU4tTLu3GBsvhJ8ULSi3G", storage: { pending_ops: 219535, @@ -160,26 +165,73 @@ describe("multisig fetch", () => { describe("getAllMultiSigContracts", () => { it("fetches all multisig contracts", async () => { - mockedContractsGet.mockResolvedValue(multisigContracts as any); + mockedContractsGetCount.mockResolvedValue(multisigContracts.length); + mockedContractsGet.mockResolvedValue(multisigContracts); const result = await getAllMultisigContracts(GHOSTNET); + expect(mockedContractsGetCount).toHaveBeenCalledTimes(1); + expect(mockedContractsGetCount).toHaveBeenCalledWith( + { + kind: { eq: "smart_contract" }, + typeHash: { eq: TYPE_HASH }, + codeHash: { eq: CODE_HASH }, + }, + { baseUrl: GHOSTNET.tzktApiUrl } + ); expect(mockedContractsGet).toHaveBeenCalledTimes(1); expect(mockedContractsGet).toHaveBeenCalledWith( { + kind: { eq: "smart_contract" }, typeHash: { eq: TYPE_HASH }, codeHash: { eq: CODE_HASH }, + select: { fields: ["id,address,storage"] }, includeStorage: true, limit: 10000, + offset: { pg: 0 }, }, { baseUrl: GHOSTNET.tzktApiUrl } ); expect( - result.map(({ address, storage: { pending_ops, signers, threshold } }) => ({ + result.map(({ id, address, storage: { pending_ops, signers, threshold } }) => ({ + id, address, storage: { pending_ops, signers, threshold }, })) ).toEqual(expectedMockedMultisigContracts); }); + + it("handles pagination", async () => { + mockedContractsGetCount.mockResolvedValue(79123); + mockedContractsGet.mockResolvedValue([]); + + await getAllMultisigContracts(GHOSTNET); + + expect(mockedContractsGetCount).toHaveBeenCalledTimes(1); + expect(mockedContractsGetCount).toHaveBeenCalledWith( + { + kind: { eq: "smart_contract" }, + typeHash: { eq: TYPE_HASH }, + codeHash: { eq: CODE_HASH }, + }, + { baseUrl: GHOSTNET.tzktApiUrl } + ); + + expect(mockedContractsGet).toHaveBeenCalledTimes(8); + range(8).forEach(index => { + expect(mockedContractsGet).toHaveBeenCalledWith( + { + kind: { eq: "smart_contract" }, + typeHash: { eq: TYPE_HASH }, + codeHash: { eq: CODE_HASH }, + select: { fields: ["id,address,storage"] }, + includeStorage: true, + limit: 10000, + offset: { pg: index }, + }, + { baseUrl: GHOSTNET.tzktApiUrl } + ); + }); + }); }); describe("getExistingContracts", () => { diff --git a/packages/multisig/src/fetch.ts b/packages/multisig/src/fetch.ts index 3b2fb87b80..38d3b95e50 100644 --- a/packages/multisig/src/fetch.ts +++ b/packages/multisig/src/fetch.ts @@ -1,27 +1,46 @@ -import { contractsGet } from "@tzkt/sdk-api"; +import { contractsGet, contractsGetCount } from "@tzkt/sdk-api"; import { type Network, type RawPkh } from "@umami/tezos"; import { withRateLimit } from "@umami/tzkt"; import axios from "axios"; +import range from "lodash/range"; +import sortBy from "lodash/sortBy"; import { type RawTzktMultisigBigMap, type RawTzktMultisigContract } from "./types"; export const TYPE_HASH = 1963879877; export const CODE_HASH = -1890025422; +const MULTISIG_QUERY = { + kind: { eq: "smart_contract" as const }, + typeHash: { eq: TYPE_HASH }, + codeHash: { eq: CODE_HASH }, +}; +const LIMIT = 10000; -export const getAllMultisigContracts = async (network: Network) => - withRateLimit(() => - contractsGet( - { - typeHash: { eq: TYPE_HASH }, - codeHash: { eq: CODE_HASH }, - includeStorage: true, - limit: 10000, - }, - { - baseUrl: network.tzktApiUrl, - } +export const getAllMultisigContracts = async (network: Network) => { + const count = await withRateLimit(() => + contractsGetCount(MULTISIG_QUERY as any, { baseUrl: network.tzktApiUrl }) + ); + + const batches = await Promise.all( + range(Math.ceil(count / LIMIT)).map(index => + withRateLimit(() => + contractsGet( + { + ...MULTISIG_QUERY, + includeStorage: true, + limit: LIMIT, + offset: { pg: index }, + select: { fields: [["id", "address", "storage"].join(",")] }, + }, + { + baseUrl: network.tzktApiUrl, + } + ) + ) ) - ) as Promise; + ); + return sortBy(batches.flat(), "id") as RawTzktMultisigContract[]; +}; /** * Returns existing addresses for the given contract addresses list & the given network. @@ -36,7 +55,7 @@ export const getExistingContracts = (pkhs: RawPkh[], network: Network): Promise< { address: { in: [pkhs.join(",")] }, select: { fields: ["address"] }, - limit: Math.min(10000, pkhs.length), + limit: Math.min(LIMIT, pkhs.length), }, { baseUrl: network.tzktApiUrl, diff --git a/packages/multisig/src/helpers.test.ts b/packages/multisig/src/helpers.test.ts index 7c4e624545..bb86684c9d 100644 --- a/packages/multisig/src/helpers.test.ts +++ b/packages/multisig/src/helpers.test.ts @@ -14,18 +14,22 @@ import { getRelevantMultisigContracts, parseMultisig, } from "./helpers"; -import { type RawTzktMultisigContract } from "./types"; const mockedAxios = jest.spyOn(axios, "get"); const mockedContractsGet = jest.spyOn(api, "contractsGet"); +const mockedContractsGetCount = jest.spyOn(api, "contractsGetCount"); -const tzktGetSameMultisigsResponse: RawTzktMultisigContract[] = [ +const tzktGetSameMultisigsResponse = [ { + id: 1, + type: "contract", address: mockContractAddress(0).pkh, storage: { threshold: "2", pending_ops: 0, signers: [mockImplicitAddress(0).pkh] }, }, { + id: 2, + type: "contract", address: mockContractAddress(2).pkh, storage: { threshold: "2", pending_ops: 1, signers: [mockImplicitAddress(2).pkh] }, }, @@ -35,7 +39,8 @@ describe("multisig helpers", () => { describe.each(DefaultNetworks)("on $name", network => { describe("getRelevantMultisigContracts", () => { it("fetches multisig contracts", async () => { - mockedContractsGet.mockResolvedValue(tzktGetSameMultisigsResponse as any); + mockedContractsGetCount.mockResolvedValue(tzktGetSameMultisigsResponse.length); + mockedContractsGet.mockResolvedValue(tzktGetSameMultisigsResponse); const result = await getRelevantMultisigContracts( new Set([mockImplicitAddress(0).pkh]), diff --git a/packages/multisig/src/types.ts b/packages/multisig/src/types.ts index c6160fbba4..2b6c99f1c4 100644 --- a/packages/multisig/src/types.ts +++ b/packages/multisig/src/types.ts @@ -17,6 +17,7 @@ export type Multisig = { export type MultisigPendingOperations = Record; export type RawTzktMultisigContract = { + id: number; address: string; storage: { signers: string[];