From df442206333f6ccf7e8ce878e7e812cffbe5d95b Mon Sep 17 00:00:00 2001 From: david Date: Mon, 4 Nov 2024 09:47:45 -0500 Subject: [PATCH] improve: improve performance of loading requests Signed-off-by: david --- .../services/oraclev1/gql/factory.ts | 64 +++---- src/contexts/OracleDataContext.tsx | 161 +++++++++--------- src/data/approvedIdentifiersTable.json | 2 +- src/helpers/util.ts | 131 +++++++++++++- vercel.json | 4 +- 5 files changed, 250 insertions(+), 112 deletions(-) diff --git a/libs/src/oracle-sdk-v2/services/oraclev1/gql/factory.ts b/libs/src/oracle-sdk-v2/services/oraclev1/gql/factory.ts index b5fbd39c..f7e2ac0e 100644 --- a/libs/src/oracle-sdk-v2/services/oraclev1/gql/factory.ts +++ b/libs/src/oracle-sdk-v2/services/oraclev1/gql/factory.ts @@ -2,7 +2,7 @@ import type { ChainId, OracleType } from "@shared/types"; import { parsePriceRequestGraphEntity } from "@shared/utils"; import type { Address } from "wagmi"; import type { Handlers, Service, ServiceFactory } from "../../../types"; -import { getPriceRequests } from "./queries"; +import { getPriceRequestsIncremental } from "./queries"; export type Config = { url: string; @@ -14,40 +14,40 @@ export type Config = { export const Factory = (config: Config): ServiceFactory => (handlers: Handlers): Service => { - async function fetch({ url, chainId, address, type }: Config) { - const requests = await getPriceRequests(url, chainId, type); - handlers?.requests?.( - requests.map((request) => - parsePriceRequestGraphEntity( - request, - chainId, - address as Address, - type, - ), - ), - ); - } - // if we need to bring back incremental loading - // async function fetchIncremental({ url, chainId, address, type }: Config) { - // for await (const requests of getPriceRequestsIncremental( - // url, - // chainId, - // type, - // )) { - // handlers?.requests?.( - // requests.map((request) => - // parsePriceRequestGraphEntity( - // request, - // chainId, - // address as Address, - // type, - // ), + // if we need one shot loading back + // async function fetch({ url, chainId, address, type }: Config) { + // const requests = await getPriceRequests(url, chainId, type); + // handlers?.requests?.( + // requests.map((request) => + // parsePriceRequestGraphEntity( + // request, + // chainId, + // address as Address, + // type, // ), - // ); - // } + // ), + // ); // } + async function fetchIncremental({ url, chainId, address, type }: Config) { + for await (const requests of getPriceRequestsIncremental( + url, + chainId, + type, + )) { + handlers?.requests?.( + requests.map((request) => + parsePriceRequestGraphEntity( + request, + chainId, + address as Address, + type, + ), + ), + ); + } + } async function tick() { - await fetch(config); + await fetchIncremental(config); } return { diff --git a/src/contexts/OracleDataContext.tsx b/src/contexts/OracleDataContext.tsx index 136e47da..3cc93b3d 100644 --- a/src/contexts/OracleDataContext.tsx +++ b/src/contexts/OracleDataContext.tsx @@ -6,7 +6,8 @@ import { assertionToOracleQuery, getPageForQuery, requestToOracleQuery, - sortQueries, + compareOracleQuery, + SortedList, } from "@/helpers"; import { useErrorContext } from "@/hooks"; import type { OracleQueryUI } from "@/types"; @@ -20,17 +21,10 @@ import { skinny1Ethers, } from "@libs/oracle-sdk-v2/services"; import type { Api } from "@libs/oracle-sdk-v2/services/oraclev1/ethers"; -import type { - Assertion, - Assertions, - ChainId, - OracleType, - Request, - Requests, -} from "@shared/types"; +import type { Assertion, ChainId, OracleType, Request } from "@shared/types"; import unionWith from "lodash/unionWith"; import type { ReactNode } from "react"; -import { createContext, useEffect, useReducer, useState } from "react"; +import { createContext, useEffect, useState } from "react"; //TODO: hate this approach, will need to refactor in future, current services interface does not make it easy to define custom functions // this will be moved somewhere else in future pr. @@ -99,17 +93,6 @@ export const OracleDataContext = createContext( defaultOracleDataContextState, ); -type DispatchAction = { - type: Type; - data: Data; -}; -// replace many requests, used when querying data from the graph -type ProcessRequestsAction = DispatchAction<"requests", Requests>; -// same thing with assertions -type ProcessAssertionsAction = DispatchAction<"assertions", Assertions>; - -type DispatchActions = ProcessRequestsAction | ProcessAssertionsAction; - function mergeData( prev: OracleQueryUI | undefined, next: OracleQueryUI, @@ -126,73 +109,99 @@ function mergeData( moreInformation, }; } -function DataReducerFactory( - converter: (input: Input) => OracleQueryUI, -) { - return ( - state: OracleDataContextState, - updates: Input[], - ): OracleDataContextState => { - const { all = {} } = state; - updates.forEach((update) => { - const queryUpdate = converter(update); - all[update.id] = mergeData(all[update.id], queryUpdate); - }); - const init: { - verify: OracleQueryList; - propose: OracleQueryList; - settled: OracleQueryList; - } = { - verify: [], - propose: [], - settled: [], - }; - const queries = Object.values(all).reduce((result, query) => { - const pageForQuery = getPageForQuery(query); - result[pageForQuery].push(query); - return result; - }, init); - - return { - ...state, - all: { ...all }, - ...sortQueries(queries), - }; - }; -} - -const requestReducer = DataReducerFactory(requestToOracleQuery); -const assertionReducer = DataReducerFactory(assertionToOracleQuery); +const verifyList = new SortedList(compareOracleQuery, mergeData, (x) => x.id); +const proposeList = new SortedList(compareOracleQuery, mergeData, (x) => x.id); +const settledList = new SortedList(compareOracleQuery, mergeData, (x) => x.id); -export function oracleDataReducer( - state: OracleDataContextState, - action: DispatchActions, -): OracleDataContextState { - if (action.type === "requests") { - return requestReducer(state, action.data); - } else if (action.type === "assertions") { - return assertionReducer(state, action.data); - } - return state; -} export function OracleDataProvider({ children }: { children: ReactNode }) { const { addErrorMessage } = useErrorContext(); const oraclesServices = oracles.Factory(config.subgraphs); const serviceConfigs = [...config.subgraphs, ...config.providers]; - const [queries, dispatch] = useReducer( - oracleDataReducer, - defaultOracleDataContextState, - ); + const [queries, setQueries] = useState(defaultOracleDataContextState); const [errors, setErrors] = useState( defaultOracleDataContextState.errors, ); useEffect(() => { + let lastProposeUpdate = 0; + let lastVerifyUpdate = 0; + let lastSettledUpdate = 0; + // sorted list of queries + setInterval(() => { + const state: OracleDataContextState = { + errors: [], + all: {}, + verify: verifyList.list, + propose: proposeList.list, + settled: settledList.list, + }; + let dirty = false; + if (verifyList.updateCount > lastVerifyUpdate) { + state.verify = verifyList.toSortedArray(); + lastVerifyUpdate = verifyList.updateCount; + dirty = true; + } + if (proposeList.updateCount > lastProposeUpdate) { + state.propose = proposeList.toSortedArray(); + lastProposeUpdate = proposeList.updateCount; + dirty = true; + } + if (settledList.updateCount > lastSettledUpdate) { + state.settled = settledList.toSortedArray(); + lastSettledUpdate = settledList.updateCount; + dirty = true; + } + if (dirty) { + setQueries(state); + } + }, 5000); + // its important this client only gets initialized once Client([...oraclesServices, ...oracleEthersServices], { - requests: (requests) => dispatch({ type: "requests", data: requests }), - assertions: (assertions) => - dispatch({ type: "assertions", data: assertions }), + requests: (requests) => { + for (const req of requests) { + const query = requestToOracleQuery(req); + const prev = + proposeList.get(query.id) ?? + verifyList.get(query.id) ?? + settledList.get(query.id); + const merged = { ...prev, ...query }; + // we need to delete from previous list incase they moved from one to another changing state + if (proposeList.has(query.id)) proposeList.delete(query.id); + if (verifyList.has(query.id)) verifyList.delete(query.id); + if (settledList.has(query.id)) settledList.delete(query.id); + + const pageForQuery = getPageForQuery(merged); + if (pageForQuery === "propose") { + proposeList.set(merged); + } else if (pageForQuery === "settled") { + settledList.set(merged); + } else if (pageForQuery === "verify") { + verifyList.set(merged); + } + } + }, + assertions: (assertions) => { + for (const a of assertions) { + const query = assertionToOracleQuery(a); + const prev = + proposeList.get(query.id) ?? + verifyList.get(query.id) ?? + settledList.get(query.id); + const pageForQuery = getPageForQuery(query); + const merged = { ...prev, ...query }; + if (proposeList.has(query.id)) proposeList.delete(query.id); + if (verifyList.has(query.id)) verifyList.delete(query.id); + if (settledList.has(query.id)) settledList.delete(query.id); + if (pageForQuery === "propose") { + proposeList.upsert(merged); + } else if (pageForQuery === "settled") { + settledList.upsert(merged); + } else if (pageForQuery === "verify") { + verifyList.upsert(merged); + } + } + }, errors: setErrors, }); // eslint-disable-next-line react-hooks/exhaustive-deps diff --git a/src/data/approvedIdentifiersTable.json b/src/data/approvedIdentifiersTable.json index 87da5f85..850ef78b 100644 --- a/src/data/approvedIdentifiersTable.json +++ b/src/data/approvedIdentifiersTable.json @@ -1951,4 +1951,4 @@ "url": "" } } -} \ No newline at end of file +} diff --git a/src/helpers/util.ts b/src/helpers/util.ts index 2cd09a65..229ff255 100644 --- a/src/helpers/util.ts +++ b/src/helpers/util.ts @@ -3,7 +3,7 @@ import type { OracleQueryList } from "@/contexts"; import type { DropdownItem, OracleQueryUI } from "@/types"; import { chainsById, oracleTypes } from "@shared/constants"; import type { ChainId, OracleType } from "@shared/types"; -import { capitalize, orderBy, partition, words } from "lodash"; +import { capitalize, orderBy, partition, words, sortedIndexBy } from "lodash"; import type { ReadonlyURLSearchParams } from "next/navigation"; import { css } from "styled-components"; import type { Address } from "wagmi"; @@ -298,3 +298,132 @@ export function truncateAddress(address: Address | undefined) { return `${address.slice(0, 5)}...${address.slice(-5)}`; } } + +export type SortableOracleQuery = { + livenessEndsMilliseconds?: number | null | undefined; + timeMilliseconds?: number | null | undefined; + disputeHash?: string | null | undefined; +}; + +export function compareOracleQuery( + a: SortableOracleQuery, + b: SortableOracleQuery, +): number { + const now = Date.now(); + + const aInLiveness = + (a.livenessEndsMilliseconds ?? 0) > now && a.disputeHash === undefined; + const bInLiveness = + (b.livenessEndsMilliseconds ?? 0) > now && b.disputeHash === undefined; + + if (aInLiveness && !bInLiveness) return -1; + if (!aInLiveness && bInLiveness) return 1; + + if (aInLiveness && bInLiveness) { + // Both are in liveness, compare by livenessEndsMilliseconds + return ( + (a.livenessEndsMilliseconds ?? 0) - (b.livenessEndsMilliseconds ?? 0) + ); + } else { + if ( + (b.timeMilliseconds === null || b.timeMilliseconds === undefined) && + (a.timeMilliseconds === null || a.timeMilliseconds === undefined) + ) { + return 0; + } + if (b.timeMilliseconds === null || b.timeMilliseconds === undefined) { + return 1; + } + if (a.timeMilliseconds === null || a.timeMilliseconds === undefined) { + return -1; + } + // Neither are in liveness, compare by timeMilliseconds in descending order + return a.timeMilliseconds - b.timeMilliseconds; + } +} +type Comparator = (a: T, b: T) => number; +type MergeFunction = (existing: T, newEntry: T) => T; +type GetIdFunction = (element: T) => string; + +export class SortedList { + public list: T[] = []; + private comparator: Comparator; + private mergeFn: MergeFunction; + private getIdFn: GetIdFunction; + private idMap: Map = new Map(); + public updateCount = 0; + + constructor( + comparator: Comparator, + mergeFn: MergeFunction, + getIdFn: GetIdFunction, + ) { + this.comparator = comparator; + this.mergeFn = mergeFn; + this.getIdFn = getIdFn; + } + + upsert = (value: T): void => { + const id = this.getIdFn(value); + if (this.idMap.has(id)) { + const index = this.idMap.get(id)!; + const existingValue = this.list[index]; + const mergedValue = this.mergeFn(existingValue, value); + this.list[index] = mergedValue; + } else { + const insertIndex = sortedIndexBy(this.list, value, (item) => item); + this.list.splice(insertIndex, 0, value); + this.idMap.set(id, insertIndex); + + for (let i = insertIndex + 1; i < this.list.length; i++) { + this.idMap.set(this.getIdFn(this.list[i]), i); + } + this.updateCount++; + } + }; + set = (value: T): void => { + const id = this.getIdFn(value); + if (this.idMap.has(id)) { + throw new Error(`Element with id ${id} already exists.`); + } + const insertIndex = sortedIndexBy(this.list, value, (item) => item); + this.list.splice(insertIndex, 0, value); + this.idMap.set(id, insertIndex); + for (let i = insertIndex + 1; i < this.list.length; i++) { + this.idMap.set(this.getIdFn(this.list[i]), i); + } + this.updateCount++; + }; + + has = (id: string): boolean => { + return this.idMap.has(id); + }; + + get = (id: string): T | undefined => { + const index = this.idMap.get(id); + return index !== undefined ? this.list[index] : undefined; + }; + + delete = (id: string): boolean => { + if (!this.idMap.has(id)) { + return false; + } + const index = this.idMap.get(id)!; + this.list.splice(index, 1); + this.idMap.delete(id); + + for (let i = index; i < this.list.length; i++) { + this.idMap.set(this.getIdFn(this.list[i]), i); + } + this.updateCount++; + return true; + }; + + size(): number { + return this.list.length; + } + + toSortedArray(): T[] { + return [...this.list]; + } +} diff --git a/vercel.json b/vercel.json index 527e8621..7af6ae18 100644 --- a/vercel.json +++ b/vercel.json @@ -1,5 +1,5 @@ { -"headers": [ + "headers": [ { "source": "/(.*)", "headers": [ @@ -38,5 +38,5 @@ } ] } -] + ] }