From 11eb499748f660888492e1f9550746b9c0ad4798 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Wed, 24 Jul 2024 16:30:32 -0700 Subject: [PATCH 01/17] Migrated context object creation --- runner/src/context/context.ts | 292 ++++++++++++++++++++++++++++ runner/src/context/index.ts | 2 + runner/src/indexer/indexer.ts | 263 +------------------------ runner/src/stream-handler/worker.ts | 4 +- runner/tests/integration.test.ts | 30 +-- 5 files changed, 317 insertions(+), 274 deletions(-) create mode 100644 runner/src/context/context.ts create mode 100644 runner/src/context/index.ts diff --git a/runner/src/context/context.ts b/runner/src/context/context.ts new file mode 100644 index 00000000..9f5523da --- /dev/null +++ b/runner/src/context/context.ts @@ -0,0 +1,292 @@ +import fetch from 'node-fetch'; +import { type Response } from 'node-fetch'; +import { Parser } from 'node-sql-parser'; +import { type DmlHandlerInterface } from '../dml-handler/dml-handler'; +import { type TableDefinitionNames } from '../indexer'; +import type IndexerConfig from '../indexer-config/indexer-config'; +import { LogEntry } from '../indexer-meta'; +import { wrapSpan } from '../utility'; +import assert from 'assert'; +import logger from '../logger'; +import { trace } from '@opentelemetry/api'; + +export interface ContextObject { + graphql: (operation: string, variables?: Record) => Promise + set: (key: string, value: any) => Promise + debug: (message: string) => void + log: (message: string) => void + warn: (message: string) => void + error: (message: string) => void + fetchFromSocialApi: (path: string, options?: any) => Promise + db: Record any>> +} + +interface Dependencies { + fetch?: typeof fetch + dmlHandler: DmlHandlerInterface + parser?: Parser +} + +interface Config { + hasuraAdminSecret: string + hasuraEndpoint: string +} + +const defaultConfig: Config = { + hasuraAdminSecret: process.env.HASURA_ADMIN_SECRET ?? '', + hasuraEndpoint: process.env.HASURA_ENDPOINT ?? '', +}; + +export default class ContextBuilder { + DEFAULT_HASURA_ROLE: string = 'append'; + + tracer = trace.getTracer('queryapi-runner-context'); + private readonly logger: typeof logger; + tableDefinitions: Map; + deps: Required; + + constructor ( + private readonly indexerConfig: IndexerConfig, + deps: Dependencies, + private readonly config: Config = defaultConfig, + ) { + this.logger = logger.child({ accountId: indexerConfig.accountId, functionName: indexerConfig.functionName, service: this.constructor.name }); + + this.deps = { + fetch, + parser: new Parser(), + ...deps + }; + // TODO: Move Parsing logic to separate class + this.tableDefinitions = getTableNameToDefinitionNamesMapping(indexerConfig.schema); + } + + private sanitizeTableName (tableName: string): string { + // Convert to PascalCase + let pascalCaseTableName = tableName + // Replace special characters with underscores + .replace(/[^a-zA-Z0-9_]/g, '_') + // Makes first letter and any letters following an underscore upper case + .replace(/^([a-zA-Z])|_([a-zA-Z])/g, (match: string) => match.toUpperCase()) + // Removes all underscores + .replace(/_/g, ''); + + // Add underscore if first character is a number + if (/^[0-9]/.test(pascalCaseTableName)) { + pascalCaseTableName = '_' + pascalCaseTableName; + } + + return pascalCaseTableName; + } + + private async runGraphQLQuery (operation: string, variables: any, blockHeight: number, hasuraRoleName: string | null, logError: boolean = true): Promise { + assert(this.config.hasuraAdminSecret !== '' && this.config.hasuraEndpoint !== '', 'hasuraAdminSecret and hasuraEndpoint env variables are required'); + const response: Response = await this.deps.fetch(`${this.config.hasuraEndpoint}/v1/graphql`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Hasura-Use-Backend-Only-Permissions': 'true', + ...(hasuraRoleName && { + 'X-Hasura-Role': hasuraRoleName, + 'X-Hasura-Admin-Secret': this.config.hasuraAdminSecret, + }), + }, + body: JSON.stringify({ + query: operation, + ...(variables && { variables }), + }), + }); + + const { data, errors } = await response.json(); + + if (response.status !== 200 || errors) { + if (logError) { + const message: string = errors ? errors.map((e: any) => e.message).join(', ') : `HTTP ${response.status} error writing with graphql to indexer storage`; + const mutation: string = + `mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){ + insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) { + id + } + }`; + try { + await this.runGraphQLQuery(mutation, { function_name: this.indexerConfig.fullName(), block_height: blockHeight, message }, blockHeight, this.DEFAULT_HASURA_ROLE, false); + } catch (e) { + this.logger.error('Error writing log of graphql error', e); + } + } + throw new Error(`Failed to write graphql, http status: ${response.status}, errors: ${JSON.stringify(errors, null, 2)}`); + } + + return data; + } + + buildContext (blockHeight: number, logEntries: LogEntry[]): ContextObject { + return { + graphql: async (operation: string, variables?: Record) => { + return await wrapSpan(async () => { + return await this.runGraphQLQuery(operation, variables, blockHeight, this.indexerConfig.hasuraRoleName()); + }, this.tracer, `Call graphql ${operation.includes('mutation') ? 'mutation' : 'query'} through Hasura`); + }, + set: async (key, value) => { + const mutation = ` + mutation SetKeyValue($function_name: String!, $key: String!, $value: String!) { + insert_${this.indexerConfig.hasuraRoleName()}_${this.indexerConfig.hasuraFunctionName()}_indexer_storage_one(object: {function_name: $function_name, key_name: $key, value: $value} on_conflict: {constraint: indexer_storage_pkey, update_columns: value}) {key_name} + }`; + const variables = { + function_name: this.indexerConfig.fullName(), + key, + value: value ? JSON.stringify(value) : null + }; + return await wrapSpan(async () => { + return await this.runGraphQLQuery(mutation, variables, blockHeight, this.indexerConfig.hasuraRoleName()); + }, this.tracer, 'call insert mutation through Hasura'); + }, + debug: (...log) => { + const debugLogEntry = LogEntry.userDebug(log.join(' : '), blockHeight); + logEntries.push(debugLogEntry); + }, + log: (...log) => { + const infoLogEntry = LogEntry.userInfo(log.join(' : '), blockHeight); + logEntries.push(infoLogEntry); + }, + warn: (...log) => { + const warnLogEntry = LogEntry.userWarn(log.join(' : '), blockHeight); + logEntries.push(warnLogEntry); + }, + error: (...log) => { + const errorLogEntry = LogEntry.userError(log.join(' : '), blockHeight); + logEntries.push(errorLogEntry); + }, + fetchFromSocialApi: async (path, options) => { + return await this.deps.fetch(`https://api.near.social${path}`, options); + }, + db: this.buildDatabaseContext(blockHeight, logEntries) + }; + } + + buildDatabaseContext ( + blockHeight: number, + logEntries: LogEntry[], + ): Record any>> { + try { + const tableNames = Array.from(this.tableDefinitions.keys()); + const sanitizedTableNames = new Set(); + const dmlHandler: DmlHandlerInterface = this.deps.dmlHandler; + + // Generate and collect methods for each table name + const result = tableNames.reduce((prev, tableName) => { + // Generate sanitized table name and ensure no conflict + const sanitizedTableName = this.sanitizeTableName(tableName); + const tableDefinitionNames: TableDefinitionNames = this.tableDefinitions.get(tableName) as TableDefinitionNames; + if (sanitizedTableNames.has(sanitizedTableName)) { + throw new Error(`Table ${tableName} has the same sanitized name as another table. Special characters are removed to generate context.db methods. Please rename the table.`); + } else { + sanitizedTableNames.add(sanitizedTableName); + } + + // Generate context.db methods for table + const funcForTable = { + [`${sanitizedTableName}`]: { + insert: async (objectsToInsert: any) => { + const insertLogEntry = LogEntry.userDebug(`Inserting object ${JSON.stringify(objectsToInsert)} into table ${tableName}`, blockHeight); + logEntries.push(insertLogEntry); + + return await dmlHandler.insert(tableDefinitionNames, Array.isArray(objectsToInsert) ? objectsToInsert : [objectsToInsert]); + }, + select: async (filterObj: any, limit = null) => { + const selectLogEntry = LogEntry.userDebug(`Selecting objects in table ${tableName} with values ${JSON.stringify(filterObj)} with ${limit === null ? 'no' : limit} limit`, blockHeight); + logEntries.push(selectLogEntry); + + return await dmlHandler.select(tableDefinitionNames, filterObj, limit); + }, + update: async (filterObj: any, updateObj: any) => { + const updateLogEntry = LogEntry.userDebug(`Updating objects in table ${tableName} that match ${JSON.stringify(filterObj)} with values ${JSON.stringify(updateObj)}`, blockHeight); + logEntries.push(updateLogEntry); + + return await dmlHandler.update(tableDefinitionNames, filterObj, updateObj); + }, + upsert: async (objectsToInsert: any, conflictColumns: string[], updateColumns: string[]) => { + const upsertLogEntry = LogEntry.userDebug(`Inserting objects into table ${tableName} with values ${JSON.stringify(objectsToInsert)}. Conflict on columns ${conflictColumns.join(', ')} will update values in columns ${updateColumns.join(', ')}`, blockHeight); + logEntries.push(upsertLogEntry); + + return await dmlHandler.upsert(tableDefinitionNames, Array.isArray(objectsToInsert) ? objectsToInsert : [objectsToInsert], conflictColumns, updateColumns); + }, + delete: async (filterObj: any) => { + const deleteLogEntry = LogEntry.userDebug(`Deleting objects from table ${tableName} with values ${JSON.stringify(filterObj)}`, blockHeight); + logEntries.push(deleteLogEntry); + + return await dmlHandler.delete(tableDefinitionNames, filterObj); + } + } + }; + return { + ...prev, + ...funcForTable + }; + }, {}); + return result; + } catch (err) { + const error = err as Error; + logEntries.push(LogEntry.systemWarn(`Caught error when generating context.db methods: ${error.message}`)); + } + return {}; // Default to empty object if error + } +} + +// TODO: Migrate all below code to separate class +function getColumnDefinitionNames (columnDefs: any[]): Map { + const columnDefinitionNames = new Map(); + for (const columnDef of columnDefs) { + if (columnDef.column?.type === 'column_ref') { + const columnNameDef = columnDef.column.column.expr; + const actualColumnName = columnNameDef.type === 'double_quote_string' ? `"${columnNameDef.value as string}"` : columnNameDef.value; + columnDefinitionNames.set(columnNameDef.value, actualColumnName); + } + } + return columnDefinitionNames; +} + +function retainOriginalQuoting (schema: string, tableName: string): string { + const createTableQuotedRegex = `\\b(create|CREATE)\\s+(table|TABLE)\\s+"${tableName}"\\s*`; + + if (schema.match(new RegExp(createTableQuotedRegex, 'i'))) { + return `"${tableName}"`; + } + + return tableName; +} + +function getTableNameToDefinitionNamesMapping (schema: string): Map { + const parser = new Parser(); + let schemaSyntaxTree = parser.astify(schema, { database: 'Postgresql' }); + schemaSyntaxTree = Array.isArray(schemaSyntaxTree) ? schemaSyntaxTree : [schemaSyntaxTree]; // Ensure iterable + const tableNameToDefinitionNamesMap = new Map(); + + for (const statement of schemaSyntaxTree) { + if (statement.type === 'create' && statement.keyword === 'table' && statement.table !== undefined) { + const tableName: string = statement.table[0].table; + + if (tableNameToDefinitionNamesMap.has(tableName)) { + throw new Error(`Table ${tableName} already exists in schema. Table names must be unique. Quotes are not allowed as a differentiator between table names.`); + } + + const createDefs = statement.create_definitions ?? []; + for (const columnDef of createDefs) { + if (columnDef.column?.type === 'column_ref') { + const tableDefinitionNames: TableDefinitionNames = { + tableName, + originalTableName: retainOriginalQuoting(schema, tableName), + originalColumnNames: getColumnDefinitionNames(createDefs) + }; + tableNameToDefinitionNamesMap.set(tableName, tableDefinitionNames); + } + } + } + } + + if (tableNameToDefinitionNamesMap.size === 0) { + throw new Error('Schema does not have any tables. There should be at least one table.'); + } + + return tableNameToDefinitionNamesMap; +} diff --git a/runner/src/context/index.ts b/runner/src/context/index.ts new file mode 100644 index 00000000..b931cee5 --- /dev/null +++ b/runner/src/context/index.ts @@ -0,0 +1,2 @@ +export { default } from './context'; +export type { ContextObject } from './context'; diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index 3e1891b1..29b7ad2e 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -1,4 +1,3 @@ -import fetch, { type Response } from 'node-fetch'; import { VM } from 'vm2'; import * as lakePrimitives from '@near-lake/primitives'; import { Parser } from 'node-sql-parser'; @@ -9,48 +8,23 @@ import logger from '../logger'; import LogEntry from '../indexer-meta/log-entry'; import type IndexerConfig from '../indexer-config'; import { IndexerStatus } from '../indexer-meta'; -import { wrapSpan } from '../utility'; import { type IndexerMetaInterface } from '../indexer-meta/indexer-meta'; -import { type DmlHandlerInterface } from '../dml-handler/dml-handler'; -import assert from 'assert'; +import type ContextBuilder from '../context'; interface Dependencies { fetch?: typeof fetch - dmlHandler: DmlHandlerInterface + contextBuilder: ContextBuilder indexerMeta: IndexerMetaInterface parser?: Parser }; -interface Context { - graphql: (operation: string, variables?: Record) => Promise - set: (key: string, value: any) => Promise - debug: (message: string) => void - log: (message: string) => void - warn: (message: string) => void - error: (message: string) => void - fetchFromSocialApi: (path: string, options?: any) => Promise - db: Record any>> -} - export interface TableDefinitionNames { tableName: string originalTableName: string originalColumnNames: Map } -interface Config { - hasuraAdminSecret: string - hasuraEndpoint: string - -} - -const defaultConfig: Config = { - hasuraAdminSecret: process.env.HASURA_ADMIN_SECRET ?? '', - hasuraEndpoint: process.env.HASURA_ENDPOINT ?? '', -}; - export default class Indexer { - DEFAULT_HASURA_ROLE: string; IS_FIRST_EXECUTION: boolean = true; tracer = trace.getTracer('queryapi-runner-indexer'); @@ -61,11 +35,9 @@ export default class Indexer { constructor ( private readonly indexerConfig: IndexerConfig, deps: Dependencies, - private readonly config: Config = defaultConfig, ) { this.logger = logger.child({ accountId: indexerConfig.accountId, functionName: indexerConfig.functionName, service: this.constructor.name }); - this.DEFAULT_HASURA_ROLE = 'append'; this.deps = { fetch, parser: new Parser(), @@ -94,7 +66,7 @@ export default class Indexer { this.logger.error('Failed to set status to RUNNING', e); })); const vm = new VM({ allowAsync: true }); - const context = this.buildContext(blockHeight, logEntries); + const context = this.deps.contextBuilder.buildContext(blockHeight, logEntries); vm.freeze(block, 'block'); vm.freeze(lakePrimitives, 'primitives'); @@ -133,194 +105,6 @@ export default class Indexer { } } - buildContext (blockHeight: number, logEntries: LogEntry[]): Context { - return { - graphql: async (operation, variables) => { - return await wrapSpan(async () => { - return await this.runGraphQLQuery(operation, variables, blockHeight, this.indexerConfig.hasuraRoleName()); - }, this.tracer, `Call graphql ${operation.includes('mutation') ? 'mutation' : 'query'} through Hasura`); - }, - set: async (key, value) => { - const mutation = ` - mutation SetKeyValue($function_name: String!, $key: String!, $value: String!) { - insert_${this.indexerConfig.hasuraRoleName()}_${this.indexerConfig.hasuraFunctionName()}_indexer_storage_one(object: {function_name: $function_name, key_name: $key, value: $value} on_conflict: {constraint: indexer_storage_pkey, update_columns: value}) {key_name} - }`; - const variables = { - function_name: this.indexerConfig.fullName(), - key, - value: value ? JSON.stringify(value) : null - }; - return await wrapSpan(async () => { - return await this.runGraphQLQuery(mutation, variables, blockHeight, this.indexerConfig.hasuraRoleName()); - }, this.tracer, 'call insert mutation through Hasura'); - }, - debug: (...log) => { - const debugLogEntry = LogEntry.userDebug(log.join(' : '), blockHeight); - logEntries.push(debugLogEntry); - }, - log: (...log) => { - const infoLogEntry = LogEntry.userInfo(log.join(' : '), blockHeight); - logEntries.push(infoLogEntry); - }, - warn: (...log) => { - const warnLogEntry = LogEntry.userWarn(log.join(' : '), blockHeight); - logEntries.push(warnLogEntry); - }, - error: (...log) => { - const errorLogEntry = LogEntry.userError(log.join(' : '), blockHeight); - logEntries.push(errorLogEntry); - }, - fetchFromSocialApi: async (path, options) => { - return await this.deps.fetch(`https://api.near.social${path}`, options); - }, - db: this.buildDatabaseContext(blockHeight, logEntries) - }; - } - - private getColumnDefinitionNames (columnDefs: any[]): Map { - const columnDefinitionNames = new Map(); - for (const columnDef of columnDefs) { - if (columnDef.column?.type === 'column_ref') { - const columnNameDef = columnDef.column.column.expr; - const actualColumnName = columnNameDef.type === 'double_quote_string' ? `"${columnNameDef.value as string}"` : columnNameDef.value; - columnDefinitionNames.set(columnNameDef.value, actualColumnName); - } - } - return columnDefinitionNames; - } - - private retainOriginalQuoting (schema: string, tableName: string): string { - const createTableQuotedRegex = `\\b(create|CREATE)\\s+(table|TABLE)\\s+"${tableName}"\\s*`; - - if (schema.match(new RegExp(createTableQuotedRegex, 'i'))) { - return `"${tableName}"`; - } - - return tableName; - } - - getTableNameToDefinitionNamesMapping (schema: string): Map { - let schemaSyntaxTree = this.deps.parser.astify(schema, { database: 'Postgresql' }); - schemaSyntaxTree = Array.isArray(schemaSyntaxTree) ? schemaSyntaxTree : [schemaSyntaxTree]; // Ensure iterable - const tableNameToDefinitionNamesMap = new Map(); - - for (const statement of schemaSyntaxTree) { - if (statement.type === 'create' && statement.keyword === 'table' && statement.table !== undefined) { - const tableName: string = statement.table[0].table; - - if (tableNameToDefinitionNamesMap.has(tableName)) { - throw new Error(`Table ${tableName} already exists in schema. Table names must be unique. Quotes are not allowed as a differentiator between table names.`); - } - - const createDefs = statement.create_definitions ?? []; - for (const columnDef of createDefs) { - if (columnDef.column?.type === 'column_ref') { - const tableDefinitionNames: TableDefinitionNames = { - tableName, - originalTableName: this.retainOriginalQuoting(schema, tableName), - originalColumnNames: this.getColumnDefinitionNames(createDefs) - }; - tableNameToDefinitionNamesMap.set(tableName, tableDefinitionNames); - } - } - } - } - - if (tableNameToDefinitionNamesMap.size === 0) { - throw new Error('Schema does not have any tables. There should be at least one table.'); - } - - return tableNameToDefinitionNamesMap; - } - - sanitizeTableName (tableName: string): string { - // Convert to PascalCase - let pascalCaseTableName = tableName - // Replace special characters with underscores - .replace(/[^a-zA-Z0-9_]/g, '_') - // Makes first letter and any letters following an underscore upper case - .replace(/^([a-zA-Z])|_([a-zA-Z])/g, (match: string) => match.toUpperCase()) - // Removes all underscores - .replace(/_/g, ''); - - // Add underscore if first character is a number - if (/^[0-9]/.test(pascalCaseTableName)) { - pascalCaseTableName = '_' + pascalCaseTableName; - } - - return pascalCaseTableName; - } - - buildDatabaseContext ( - blockHeight: number, - logEntries: LogEntry[], - ): Record any>> { - try { - const tableNameToDefinitionNamesMapping = this.getTableNameToDefinitionNamesMapping(this.indexerConfig.schema); - const tableNames = Array.from(tableNameToDefinitionNamesMapping.keys()); - const sanitizedTableNames = new Set(); - const dmlHandler: DmlHandlerInterface = this.deps.dmlHandler; - - // Generate and collect methods for each table name - const result = tableNames.reduce((prev, tableName) => { - // Generate sanitized table name and ensure no conflict - const sanitizedTableName = this.sanitizeTableName(tableName); - const tableDefinitionNames: TableDefinitionNames = tableNameToDefinitionNamesMapping.get(tableName) as TableDefinitionNames; - if (sanitizedTableNames.has(sanitizedTableName)) { - throw new Error(`Table ${tableName} has the same sanitized name as another table. Special characters are removed to generate context.db methods. Please rename the table.`); - } else { - sanitizedTableNames.add(sanitizedTableName); - } - - // Generate context.db methods for table - const funcForTable = { - [`${sanitizedTableName}`]: { - insert: async (objectsToInsert: any) => { - const insertLogEntry = LogEntry.userDebug(`Inserting object ${JSON.stringify(objectsToInsert)} into table ${tableName}`, blockHeight); - logEntries.push(insertLogEntry); - - return await dmlHandler.insert(tableDefinitionNames, Array.isArray(objectsToInsert) ? objectsToInsert : [objectsToInsert]); - }, - select: async (filterObj: any, limit = null) => { - const selectLogEntry = LogEntry.userDebug(`Selecting objects in table ${tableName} with values ${JSON.stringify(filterObj)} with ${limit === null ? 'no' : limit} limit`, blockHeight); - logEntries.push(selectLogEntry); - - return await dmlHandler.select(tableDefinitionNames, filterObj, limit); - }, - update: async (filterObj: any, updateObj: any) => { - const updateLogEntry = LogEntry.userDebug(`Updating objects in table ${tableName} that match ${JSON.stringify(filterObj)} with values ${JSON.stringify(updateObj)}`, blockHeight); - logEntries.push(updateLogEntry); - - return await dmlHandler.update(tableDefinitionNames, filterObj, updateObj); - }, - upsert: async (objectsToInsert: any, conflictColumns: string[], updateColumns: string[]) => { - const upsertLogEntry = LogEntry.userDebug(`Inserting objects into table ${tableName} with values ${JSON.stringify(objectsToInsert)}. Conflict on columns ${conflictColumns.join(', ')} will update values in columns ${updateColumns.join(', ')}`, blockHeight); - logEntries.push(upsertLogEntry); - - return await dmlHandler.upsert(tableDefinitionNames, Array.isArray(objectsToInsert) ? objectsToInsert : [objectsToInsert], conflictColumns, updateColumns); - }, - delete: async (filterObj: any) => { - const deleteLogEntry = LogEntry.userDebug(`Deleting objects from table ${tableName} with values ${JSON.stringify(filterObj)}`, blockHeight); - logEntries.push(deleteLogEntry); - - return await dmlHandler.delete(tableDefinitionNames, filterObj); - } - } - }; - return { - ...prev, - ...funcForTable - }; - }, {}); - return result; - } catch (error) { - if (this.IS_FIRST_EXECUTION) { - this.logger.warn('Caught error when generating context.db methods', error); - } - } - return {}; // Default to empty object if error - } - async setStatus (status: IndexerStatus): Promise { if (this.currentStatus === status) { return; @@ -332,47 +116,6 @@ export default class Indexer { await this.deps.indexerMeta?.setStatus(status); } - async runGraphQLQuery (operation: string, variables: any, blockHeight: number, hasuraRoleName: string | null, logError: boolean = true): Promise { - assert(this.config.hasuraAdminSecret !== '' && this.config.hasuraEndpoint !== '', 'hasuraAdminSecret and hasuraEndpoint env variables are required'); - const response: Response = await this.deps.fetch(`${this.config.hasuraEndpoint}/v1/graphql`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Hasura-Use-Backend-Only-Permissions': 'true', - ...(hasuraRoleName && { - 'X-Hasura-Role': hasuraRoleName, - 'X-Hasura-Admin-Secret': this.config.hasuraAdminSecret, - }), - }, - body: JSON.stringify({ - query: operation, - ...(variables && { variables }), - }), - }); - - const { data, errors } = await response.json(); - - if (response.status !== 200 || errors) { - if (logError) { - const message: string = errors ? errors.map((e: any) => e.message).join(', ') : `HTTP ${response.status} error writing with graphql to indexer storage`; - const mutation: string = - `mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){ - insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) { - id - } - }`; - try { - await this.runGraphQLQuery(mutation, { function_name: this.indexerConfig.fullName(), block_height: blockHeight, message }, blockHeight, this.DEFAULT_HASURA_ROLE, false); - } catch (e) { - this.logger.error('Error writing log of graphql error', e); - } - } - throw new Error(`Failed to write graphql, http status: ${response.status}, errors: ${JSON.stringify(errors, null, 2)}`); - } - - return data; - } - private enableAwaitTransform (code: string): string { return ` async function f(){ diff --git a/runner/src/stream-handler/worker.ts b/runner/src/stream-handler/worker.ts index 1403d660..bd865f8f 100644 --- a/runner/src/stream-handler/worker.ts +++ b/runner/src/stream-handler/worker.ts @@ -15,6 +15,7 @@ import parentLogger from '../logger'; import { wrapSpan } from '../utility'; import { type PostgresConnectionParams } from '../pg-client'; import DmlHandler from '../dml-handler/dml-handler'; +import ContextBuilder from '../context/context'; if (isMainThread) { throw new Error('Worker should not be run on main thread'); @@ -102,8 +103,9 @@ async function blockQueueConsumer (workerContext: WorkerContext): Promise const indexerConfig: IndexerConfig = workerContext.indexerConfig; const dmlHandler: DmlHandler = new DmlHandler(workerContext.databaseConnectionParams, indexerConfig); + const contextBuilder: ContextBuilder = new ContextBuilder(indexerConfig, { dmlHandler }); const indexerMeta: IndexerMeta = new IndexerMeta(indexerConfig, workerContext.databaseConnectionParams); - const indexer = new Indexer(indexerConfig, { dmlHandler, indexerMeta }); + const indexer = new Indexer(indexerConfig, { contextBuilder, indexerMeta }); let streamMessageId = ''; let currBlockHeight = 0; diff --git a/runner/tests/integration.test.ts b/runner/tests/integration.test.ts index 7b2112b0..a3aa4925 100644 --- a/runner/tests/integration.test.ts +++ b/runner/tests/integration.test.ts @@ -15,6 +15,7 @@ import { LogLevel } from '../src/indexer-meta/log-entry'; import IndexerConfig from '../src/indexer-config'; import IndexerMeta from '../src/indexer-meta/indexer-meta'; import DmlHandler from '../src/dml-handler/dml-handler'; +import ContextBuilder from '../src/context'; describe('Indexer integration', () => { jest.setTimeout(300_000); @@ -271,27 +272,30 @@ describe('Indexer integration', () => { }); }); -async function prepareIndexer(indexerConfig: IndexerConfig, provisioner: Provisioner, hasuraContainer: StartedHasuraGraphQLContainer): Promise { +async function prepareIndexer (indexerConfig: IndexerConfig, provisioner: Provisioner, hasuraContainer: StartedHasuraGraphQLContainer): Promise { await provisioner.provisionUserApi(indexerConfig); - const db_connection_params = await provisioner.getPostgresConnectionParameters(indexerConfig.userName()); - const dmlHandler = new DmlHandler(db_connection_params, indexerConfig); - const indexerMeta = new IndexerMeta(indexerConfig, db_connection_params); + const dbConnectionParams = await provisioner.getPostgresConnectionParameters(indexerConfig.userName()); + const dmlHandler = new DmlHandler(dbConnectionParams, indexerConfig); + const contextBuilder = new ContextBuilder( + indexerConfig, + { dmlHandler }, + { + hasuraAdminSecret: hasuraContainer.getAdminSecret(), + hasuraEndpoint: hasuraContainer.getEndpoint(), + }); + const indexerMeta = new IndexerMeta(indexerConfig, dbConnectionParams); return new Indexer( indexerConfig, { - dmlHandler, + contextBuilder, indexerMeta, - }, - { - hasuraAdminSecret: hasuraContainer.getAdminSecret(), - hasuraEndpoint: hasuraContainer.getEndpoint(), } ); } -async function indexerLogsQuery(indexerSchemaName: string, graphqlClient: GraphQLClient): Promise { +async function indexerLogsQuery (indexerSchemaName: string, graphqlClient: GraphQLClient): Promise { const graphqlResult: any = await graphqlClient.request(gql` query { ${indexerSchemaName}_sys_logs { @@ -302,15 +306,15 @@ async function indexerLogsQuery(indexerSchemaName: string, graphqlClient: GraphQ return graphqlResult[`${indexerSchemaName}_sys_logs`]; } -async function indexerStatusQuery(indexerSchemaName: string, graphqlClient: GraphQLClient): Promise { +async function indexerStatusQuery (indexerSchemaName: string, graphqlClient: GraphQLClient): Promise { return await indexerMetadataQuery(indexerSchemaName, 'STATUS', graphqlClient); } -async function indexerBlockHeightQuery(indexerSchemaName: string, graphqlClient: GraphQLClient): Promise { +async function indexerBlockHeightQuery (indexerSchemaName: string, graphqlClient: GraphQLClient): Promise { return await indexerMetadataQuery(indexerSchemaName, 'LAST_PROCESSED_BLOCK_HEIGHT', graphqlClient); } -async function indexerMetadataQuery(indexerSchemaName: string, attribute: string, graphqlClient: GraphQLClient): Promise { +async function indexerMetadataQuery (indexerSchemaName: string, attribute: string, graphqlClient: GraphQLClient): Promise { const graphqlResult: any = await graphqlClient.request(gql` query { ${indexerSchemaName}_sys_metadata(where: {attribute: {_eq: "${attribute}"}}) { From fb36b839f2b9a7f07d03cf4c8d1e2531379a845c Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 25 Jul 2024 12:16:19 -0700 Subject: [PATCH 02/17] Reduce indexer tests to new scope --- runner/src/context/context.test.ts | 191 ++++++ runner/src/indexer/indexer.test.ts | 1013 ++-------------------------- runner/src/indexer/indexer.ts | 2 - runner/tsconfig.json | 4 +- 4 files changed, 262 insertions(+), 948 deletions(-) create mode 100644 runner/src/context/context.test.ts diff --git a/runner/src/context/context.test.ts b/runner/src/context/context.test.ts new file mode 100644 index 00000000..fabb9880 --- /dev/null +++ b/runner/src/context/context.test.ts @@ -0,0 +1,191 @@ +const config = { + hasuraEndpoint: 'mock-hasura-endpoint', + hasuraAdminSecret: 'mock-hasura-secret', +}; + +const SIMPLE_SCHEMA = `CREATE TABLE + "posts" ( + "id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + "content" TEXT NOT NULL, + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "accounts_liked" JSONB NOT NULL DEFAULT '[]', + "last_comment_timestamp" DECIMAL(20, 0), + CONSTRAINT "posts_pkey" PRIMARY KEY ("id") + );`; + +const SOCIAL_SCHEMA = ` + CREATE TABLE + "posts" ( + "id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + "content" TEXT NOT NULL, + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "accounts_liked" JSONB NOT NULL DEFAULT '[]', + "last_comment_timestamp" DECIMAL(20, 0), + CONSTRAINT "posts_pkey" PRIMARY KEY ("id") + ); + + CREATE TABLE + "comments" ( + "id" SERIAL NOT NULL, + "post_id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0) NOT NULL, + "content" TEXT NOT NULL, + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + CONSTRAINT "comments_pkey" PRIMARY KEY ("id") + ); + + CREATE TABLE + "post_likes" ( + "post_id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0), + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id") + );`; + +const CASE_SENSITIVE_SCHEMA = ` + CREATE TABLE + Posts ( + "id" SERIAL NOT NULL, + "AccountId" VARCHAR NOT NULL, + BlockHeight DECIMAL(58, 0) NOT NULL, + "receiptId" VARCHAR NOT NULL, + content TEXT NOT NULL, + block_Timestamp DECIMAL(20, 0) NOT NULL, + "Accounts_Liked" JSONB NOT NULL DEFAULT '[]', + "LastCommentTimestamp" DECIMAL(20, 0), + CONSTRAINT "posts_pkey" PRIMARY KEY ("id") + ); + + CREATE TABLE + "CommentsTable" ( + "id" SERIAL NOT NULL, + PostId SERIAL NOT NULL, + "accountId" VARCHAR NOT NULL, + blockHeight DECIMAL(58, 0) NOT NULL, + CONSTRAINT "comments_pkey" PRIMARY KEY ("id") + );`; + +const STRESS_TEST_SCHEMA = ` + CREATE TABLE creator_quest ( + account_id VARCHAR PRIMARY KEY, + num_components_created INTEGER NOT NULL DEFAULT 0, + completed BOOLEAN NOT NULL DEFAULT FALSE + ); + + CREATE TABLE + composer_quest ( + account_id VARCHAR PRIMARY KEY, + num_widgets_composed INTEGER NOT NULL DEFAULT 0, + completed BOOLEAN NOT NULL DEFAULT FALSE + ); + + CREATE TABLE + "contractor - quest" ( + account_id VARCHAR PRIMARY KEY, + num_contracts_deployed INTEGER NOT NULL DEFAULT 0, + completed BOOLEAN NOT NULL DEFAULT FALSE + ); + + CREATE TABLE + "posts" ( + "id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + "content" TEXT NOT NULL, + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "accounts_liked" JSONB NOT NULL DEFAULT '[]', + "last_comment_timestamp" DECIMAL(20, 0), + CONSTRAINT "posts_pkey" PRIMARY KEY ("id") + ); + + CREATE TABLE + "comments" ( + "id" SERIAL NOT NULL, + "post_id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0) NOT NULL, + "content" TEXT NOT NULL, + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + CONSTRAINT "comments_pkey" PRIMARY KEY ("id") + ); + + CREATE TABLE + "post_likes" ( + "post_id" SERIAL NOT NULL, + "account_id" VARCHAR NOT NULL, + "block_height" DECIMAL(58, 0), + "block_timestamp" DECIMAL(20, 0) NOT NULL, + "receipt_id" VARCHAR NOT NULL, + CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id") + ); + + CREATE UNIQUE INDEX "posts_account_id_block_height_key" ON "posts" ("account_id" ASC, "block_height" ASC); + + CREATE UNIQUE INDEX "comments_post_id_account_id_block_height_key" ON "comments" ( + "post_id" ASC, + "account_id" ASC, + "block_height" ASC + ); + + CREATE INDEX + "posts_last_comment_timestamp_idx" ON "posts" ("last_comment_timestamp" DESC); + + ALTER TABLE + "comments" + ADD + CONSTRAINT "comments_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + + ALTER TABLE + "post_likes" + ADD + CONSTRAINT "post_likes_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE CASCADE ON UPDATE NO ACTION; + + CREATE TABLE IF NOT EXISTS + "My Table1" (id serial PRIMARY KEY); + + CREATE TABLE + "Another-Table" (id serial PRIMARY KEY); + + CREATE TABLE + IF NOT EXISTS + "Third-Table" (id serial PRIMARY KEY); + + CREATE TABLE + yet_another_table (id serial PRIMARY KEY); + `; + +const SIMPLE_REDIS_STREAM = 'test:stream'; +const SIMPLE_ACCOUNT_ID = 'morgs.near'; +const SIMPLE_FUNCTION_NAME = 'test_indexer'; +const SIMPLE_CODE = 'const a = 1;'; +const simpleSchemaConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, SIMPLE_SCHEMA, LogLevel.INFO); +const socialSchemaConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, SOCIAL_SCHEMA, LogLevel.INFO); +const caseSensitiveConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, CASE_SENSITIVE_SCHEMA, LogLevel.INFO); +const stressTestConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, STRESS_TEST_SCHEMA, LogLevel.INFO); +const genericDbCredentials: PostgresConnectionParams = { + database: 'test_near', + host: 'postgres', + password: 'test_pass', + port: 5432, + user: 'test_near' +}; + +const genericMockFetch = jest.fn() + .mockResolvedValue({ + status: 200, + json: async () => ({ + data: 'mock', + }), + }) as unknown as typeof fetch; diff --git a/runner/src/indexer/indexer.test.ts b/runner/src/indexer/indexer.test.ts index 63e857dc..6b52834e 100644 --- a/runner/src/indexer/indexer.test.ts +++ b/runner/src/indexer/indexer.test.ts @@ -3,211 +3,47 @@ import type fetch from 'node-fetch'; import Indexer from './indexer'; import { VM } from 'vm2'; -import DmlHandler from '../dml-handler/dml-handler'; +import type DmlHandler from '../dml-handler/dml-handler'; import type PgClient from '../pg-client'; import { LogLevel } from '../indexer-meta/log-entry'; import IndexerConfig from '../indexer-config/indexer-config'; import { IndexerStatus } from '../indexer-meta'; import type IndexerMeta from '../indexer-meta'; import { type PostgresConnectionParams } from '../pg-client'; +import ContextBuilder from '../context'; +import { type ContextObject } from '../context'; +import { mock } from 'node:test'; describe('Indexer unit tests', () => { - const SIMPLE_SCHEMA = `CREATE TABLE - "posts" ( - "id" SERIAL NOT NULL, - "account_id" VARCHAR NOT NULL, - "block_height" DECIMAL(58, 0) NOT NULL, - "receipt_id" VARCHAR NOT NULL, - "content" TEXT NOT NULL, - "block_timestamp" DECIMAL(20, 0) NOT NULL, - "accounts_liked" JSONB NOT NULL DEFAULT '[]', - "last_comment_timestamp" DECIMAL(20, 0), - CONSTRAINT "posts_pkey" PRIMARY KEY ("id") - );`; - - const SOCIAL_SCHEMA = ` - CREATE TABLE - "posts" ( - "id" SERIAL NOT NULL, - "account_id" VARCHAR NOT NULL, - "block_height" DECIMAL(58, 0) NOT NULL, - "receipt_id" VARCHAR NOT NULL, - "content" TEXT NOT NULL, - "block_timestamp" DECIMAL(20, 0) NOT NULL, - "accounts_liked" JSONB NOT NULL DEFAULT '[]', - "last_comment_timestamp" DECIMAL(20, 0), - CONSTRAINT "posts_pkey" PRIMARY KEY ("id") - ); - - CREATE TABLE - "comments" ( - "id" SERIAL NOT NULL, - "post_id" SERIAL NOT NULL, - "account_id" VARCHAR NOT NULL, - "block_height" DECIMAL(58, 0) NOT NULL, - "content" TEXT NOT NULL, - "block_timestamp" DECIMAL(20, 0) NOT NULL, - "receipt_id" VARCHAR NOT NULL, - CONSTRAINT "comments_pkey" PRIMARY KEY ("id") - ); - - CREATE TABLE - "post_likes" ( - "post_id" SERIAL NOT NULL, - "account_id" VARCHAR NOT NULL, - "block_height" DECIMAL(58, 0), - "block_timestamp" DECIMAL(20, 0) NOT NULL, - "receipt_id" VARCHAR NOT NULL, - CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id") - );`; - - const CASE_SENSITIVE_SCHEMA = ` - CREATE TABLE - Posts ( - "id" SERIAL NOT NULL, - "AccountId" VARCHAR NOT NULL, - BlockHeight DECIMAL(58, 0) NOT NULL, - "receiptId" VARCHAR NOT NULL, - content TEXT NOT NULL, - block_Timestamp DECIMAL(20, 0) NOT NULL, - "Accounts_Liked" JSONB NOT NULL DEFAULT '[]', - "LastCommentTimestamp" DECIMAL(20, 0), - CONSTRAINT "posts_pkey" PRIMARY KEY ("id") - ); - - CREATE TABLE - "CommentsTable" ( - "id" SERIAL NOT NULL, - PostId SERIAL NOT NULL, - "accountId" VARCHAR NOT NULL, - blockHeight DECIMAL(58, 0) NOT NULL, - CONSTRAINT "comments_pkey" PRIMARY KEY ("id") - );`; - - const STRESS_TEST_SCHEMA = ` - CREATE TABLE creator_quest ( - account_id VARCHAR PRIMARY KEY, - num_components_created INTEGER NOT NULL DEFAULT 0, - completed BOOLEAN NOT NULL DEFAULT FALSE - ); - - CREATE TABLE - composer_quest ( - account_id VARCHAR PRIMARY KEY, - num_widgets_composed INTEGER NOT NULL DEFAULT 0, - completed BOOLEAN NOT NULL DEFAULT FALSE - ); - - CREATE TABLE - "contractor - quest" ( - account_id VARCHAR PRIMARY KEY, - num_contracts_deployed INTEGER NOT NULL DEFAULT 0, - completed BOOLEAN NOT NULL DEFAULT FALSE - ); - - CREATE TABLE - "posts" ( - "id" SERIAL NOT NULL, - "account_id" VARCHAR NOT NULL, - "block_height" DECIMAL(58, 0) NOT NULL, - "receipt_id" VARCHAR NOT NULL, - "content" TEXT NOT NULL, - "block_timestamp" DECIMAL(20, 0) NOT NULL, - "accounts_liked" JSONB NOT NULL DEFAULT '[]', - "last_comment_timestamp" DECIMAL(20, 0), - CONSTRAINT "posts_pkey" PRIMARY KEY ("id") - ); - - CREATE TABLE - "comments" ( - "id" SERIAL NOT NULL, - "post_id" SERIAL NOT NULL, - "account_id" VARCHAR NOT NULL, - "block_height" DECIMAL(58, 0) NOT NULL, - "content" TEXT NOT NULL, - "block_timestamp" DECIMAL(20, 0) NOT NULL, - "receipt_id" VARCHAR NOT NULL, - CONSTRAINT "comments_pkey" PRIMARY KEY ("id") - ); - - CREATE TABLE - "post_likes" ( - "post_id" SERIAL NOT NULL, - "account_id" VARCHAR NOT NULL, - "block_height" DECIMAL(58, 0), - "block_timestamp" DECIMAL(20, 0) NOT NULL, - "receipt_id" VARCHAR NOT NULL, - CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id") - ); - - CREATE UNIQUE INDEX "posts_account_id_block_height_key" ON "posts" ("account_id" ASC, "block_height" ASC); - - CREATE UNIQUE INDEX "comments_post_id_account_id_block_height_key" ON "comments" ( - "post_id" ASC, - "account_id" ASC, - "block_height" ASC - ); - - CREATE INDEX - "posts_last_comment_timestamp_idx" ON "posts" ("last_comment_timestamp" DESC); - - ALTER TABLE - "comments" - ADD - CONSTRAINT "comments_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION; - - ALTER TABLE - "post_likes" - ADD - CONSTRAINT "post_likes_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts" ("id") ON DELETE CASCADE ON UPDATE NO ACTION; - - CREATE TABLE IF NOT EXISTS - "My Table1" (id serial PRIMARY KEY); - - CREATE TABLE - "Another-Table" (id serial PRIMARY KEY); - - CREATE TABLE - IF NOT EXISTS - "Third-Table" (id serial PRIMARY KEY); - - CREATE TABLE - yet_another_table (id serial PRIMARY KEY); - `; - const SIMPLE_REDIS_STREAM = 'test:stream'; const SIMPLE_ACCOUNT_ID = 'morgs.near'; const SIMPLE_FUNCTION_NAME = 'test_indexer'; const SIMPLE_CODE = 'const a = 1;'; + const SIMPLE_SCHEMA = 'create table posts("id" SERIAL NOT NULL PRIMARY KEY);'; + const SIMPLE_INDEXER_CONFIG: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, SIMPLE_SCHEMA, LogLevel.INFO); + + const genericMockContextObject = { + graphql: jest.fn().mockResolvedValue({ data: 'mock' }), + set: jest.fn().mockResolvedValue({ data: 'mock' }), + debug: jest.fn().mockResolvedValue(null), + log: jest.fn().mockResolvedValue(null), + warn: jest.fn().mockResolvedValue(null), + error: jest.fn().mockResolvedValue(null), + fetchFromSocialApi: jest.fn().mockResolvedValue({ data: 'mock' }), + db: { + Posts: { + insert: jest.fn().mockResolvedValue([{ colA: 'valA' }]), + select: jest.fn().mockResolvedValue([{ colA: 'valA' }]), + update: jest.fn().mockResolvedValue([{ colA: 'valA' }]), + upsert: jest.fn().mockResolvedValue([{ colA: 'valA' }]), + delete: jest.fn().mockResolvedValue([{ colA: 'valA' }]) + } + }, + } as unknown as ContextObject; - const simpleSchemaConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, SIMPLE_SCHEMA, LogLevel.INFO); - const socialSchemaConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, SOCIAL_SCHEMA, LogLevel.INFO); - const caseSensitiveConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, CASE_SENSITIVE_SCHEMA, LogLevel.INFO); - const stressTestConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, STRESS_TEST_SCHEMA, LogLevel.INFO); - - const genericDbCredentials: PostgresConnectionParams = { - database: 'test_near', - host: 'postgres', - password: 'test_pass', - port: 5432, - user: 'test_near' - }; - - const genericMockFetch = jest.fn() - .mockResolvedValue({ - status: 200, - json: async () => ({ - data: 'mock', - }), - }) as unknown as typeof fetch; - - const genericMockDmlHandler = { - insert: jest.fn().mockReturnValue([]), - select: jest.fn().mockReturnValue([]), - update: jest.fn().mockReturnValue([]), - upsert: jest.fn().mockReturnValue([]), - delete: jest.fn().mockReturnValue([]), - } as unknown as DmlHandler; + const genericMockContextBuilder = { + buildContext: jest.fn().mockReturnValue(genericMockContextObject), + } as unknown as ContextBuilder; const genericMockIndexerMeta: any = { writeLogs: jest.fn(), @@ -215,18 +51,29 @@ describe('Indexer unit tests', () => { updateBlockHeight: jest.fn().mockResolvedValue(null), } as unknown as IndexerMeta; - const config = { - hasuraEndpoint: 'mock-hasura-endpoint', - hasuraAdminSecret: 'mock-hasura-secret', - }; + test('Indexer.execute() can call context object functions', async () => { + const mockContextObject = { + graphql: jest.fn().mockResolvedValue({ data: 'mock' }), + set: jest.fn().mockResolvedValue({ data: 'mock' }), + debug: jest.fn().mockResolvedValue(null), + log: jest.fn().mockResolvedValue(null), + warn: jest.fn().mockResolvedValue(null), + error: jest.fn().mockResolvedValue(null), + fetchFromSocialApi: jest.fn().mockResolvedValue({ data: 'mock' }), + db: { + Posts: { + insert: jest.fn().mockResolvedValue([{ colA: 'valA' }]), + select: jest.fn().mockResolvedValue([{ colA: 'valA' }]), + update: jest.fn().mockResolvedValue([{ colA: 'valA' }]), + upsert: jest.fn().mockResolvedValue([{ colA: 'valA' }]), + delete: jest.fn().mockResolvedValue([{ colA: 'valA' }]) + } + }, + } as unknown as ContextObject; + const mockContextBuilder = { + buildContext: jest.fn().mockReturnValue(mockContextObject), + } as unknown as ContextBuilder; - test('Indexer.execute() should execute all functions against the current block', async () => { - const mockFetch = jest.fn(() => ({ - status: 200, - json: async () => ({ - errors: null, - }), - })); const blockHeight = 456; const mockBlock = Block.fromStreamerMessage({ block: { @@ -240,7 +87,10 @@ describe('Indexer unit tests', () => { const code = ` const foo = 3; - block.result = context.graphql(\`mutation { set(functionName: "buildnear.testnet/test", key: "height", data: "\${block.blockHeight}")}\`); + await context.graphql('query { hello }'); + await context.log('log'); + await context.fetchFromSocialApi('query { hello }'); + await context.db.Posts.insert({ foo }); `; const indexerMeta = { writeLogs: jest.fn(), @@ -249,619 +99,20 @@ describe('Indexer unit tests', () => { } as unknown as IndexerMeta; const indexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, 'buildnear.testnet', 'test', 0, code, SIMPLE_SCHEMA, LogLevel.INFO); const indexer = new Indexer(indexerConfig, { - fetch: mockFetch as unknown as typeof fetch, - dmlHandler: genericMockDmlHandler, + contextBuilder: mockContextBuilder, indexerMeta, - }, config); + }); await indexer.execute(mockBlock); - expect(mockFetch.mock.calls).toMatchSnapshot(); + expect(mockContextObject.graphql).toHaveBeenCalledWith('query { hello }'); + expect(mockContextObject.log).toHaveBeenCalledWith('log'); + expect(mockContextObject.fetchFromSocialApi).toHaveBeenCalledWith('query { hello }'); + expect(mockContextObject.db.Posts.insert).toHaveBeenCalledWith({ foo: 3 }); expect(indexerMeta.setStatus).toHaveBeenCalledWith(IndexerStatus.RUNNING); expect(indexerMeta.updateBlockHeight).toHaveBeenCalledWith(blockHeight); }); - test('Indexer.buildContext() allows execution of arbitrary GraphQL operations', async () => { - const mockFetch = jest.fn() - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ - data: { - greet: 'hello' - } - }) - }) - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ - data: { - newGreeting: { - success: true - } - } - }) - }); - const indexer = new Indexer(simpleSchemaConfig, { - fetch: mockFetch as unknown as typeof fetch, - dmlHandler: genericMockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - - const context = indexer.buildContext(1, []); - - const query = ` - query { - greet() - } - `; - const { greet } = await context.graphql(query) as { greet: string }; - - const mutation = ` - mutation { - newGreeting(greeting: "${greet} morgan") { - success - } - } - `; - const { newGreeting: { success } } = await context.graphql(mutation); - - expect(greet).toEqual('hello'); - expect(success).toEqual(true); - expect(mockFetch.mock.calls[0]).toEqual([ - `${config.hasuraEndpoint}/v1/graphql`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Hasura-Use-Backend-Only-Permissions': 'true', - 'X-Hasura-Role': 'morgs_near', - 'X-Hasura-Admin-Secret': config.hasuraAdminSecret - }, - body: JSON.stringify({ query }) - } - ]); - expect(mockFetch.mock.calls[1]).toEqual([ - `${config.hasuraEndpoint}/v1/graphql`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Hasura-Use-Backend-Only-Permissions': 'true', - 'X-Hasura-Role': 'morgs_near', - 'X-Hasura-Admin-Secret': config.hasuraAdminSecret - }, - body: JSON.stringify({ query: mutation }) - } - ]); - }); - - test('Indexer.buildContext() can fetch from the near social api', async () => { - const mockFetch = jest.fn(); - const indexer = new Indexer(simpleSchemaConfig, { - fetch: mockFetch as unknown as typeof fetch, - dmlHandler: genericMockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - - const context = indexer.buildContext(1, []); - - await context.fetchFromSocialApi('/index', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - action: 'post', - key: 'main', - options: { - limit: 1, - order: 'desc' - } - }) - }); - - expect(mockFetch.mock.calls).toMatchSnapshot(); - }); - - test('Indexer.buildContext() throws when a GraphQL response contains errors', async () => { - const mockFetch = jest.fn() - .mockResolvedValue({ - json: async () => ({ - errors: ['boom'] - }) - }); - const indexer = new Indexer(simpleSchemaConfig, { fetch: mockFetch as unknown as typeof fetch, dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }, config); - - const context = indexer.buildContext(1, []); - - await expect(async () => await context.graphql('query { hello }')).rejects.toThrow('boom'); - }); - - test('Indexer.buildContext() handles GraphQL variables', async () => { - const mockFetch = jest.fn() - .mockResolvedValue({ - status: 200, - json: async () => ({ - data: 'mock', - }), - }); - const indexer = new Indexer(simpleSchemaConfig, { fetch: mockFetch as unknown as typeof fetch, dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }, config); - - const context = indexer.buildContext(1, []); - - const query = 'query($name: String) { hello(name: $name) }'; - const variables = { name: 'morgan' }; - await context.graphql(query, variables); - - expect(mockFetch.mock.calls[0]).toEqual([ - `${config.hasuraEndpoint}/v1/graphql`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Hasura-Use-Backend-Only-Permissions': 'true', - 'X-Hasura-Role': 'morgs_near', - 'X-Hasura-Admin-Secret': config.hasuraAdminSecret - }, - body: JSON.stringify({ - query, - variables, - }), - }, - ]); - }); - - test('GetTableNameToDefinitionNamesMapping works for a variety of input schemas', async () => { - const indexer = new Indexer(stressTestConfig, { dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }); - - const tableNameToDefinitionNamesMapping = indexer.getTableNameToDefinitionNamesMapping(STRESS_TEST_SCHEMA); - expect([...tableNameToDefinitionNamesMapping.keys()]).toStrictEqual([ - 'creator_quest', - 'composer_quest', - 'contractor - quest', - 'posts', - 'comments', - 'post_likes', - 'My Table1', - 'Another-Table', - 'Third-Table', - 'yet_another_table']); - - // Test that duplicate table names throw an error - const duplicateTableSchema = `CREATE TABLE - "posts" ( - "id" SERIAL NOT NULL - ); - CREATE TABLE posts ( - "id" SERIAL NOT NULL - );`; - expect(() => { - indexer.getTableNameToDefinitionNamesMapping(duplicateTableSchema); - }).toThrow('Table posts already exists in schema. Table names must be unique. Quotes are not allowed as a differentiator between table names.'); - - // Test that schema with no tables throws an error - expect(() => { - indexer.getTableNameToDefinitionNamesMapping(''); - }).toThrow('Schema does not have any tables. There should be at least one table.'); - }); - - test('GetTableNameToDefinitionNamesMapping works for mixed quotes schema', async () => { - const indexer = new Indexer(caseSensitiveConfig, { dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }); - - const tableNameToDefinitionNamesMapping = indexer.getTableNameToDefinitionNamesMapping(CASE_SENSITIVE_SCHEMA); - const tableNames = [...tableNameToDefinitionNamesMapping.keys()]; - const originalTableNames = tableNames.map((tableName) => tableNameToDefinitionNamesMapping.get(tableName)?.originalTableName); - expect(tableNames).toStrictEqual(['Posts', 'CommentsTable']); - expect(originalTableNames).toStrictEqual(['Posts', '"CommentsTable"']); - - // Spot check quoting for columnNames - const postsColumnNames = tableNameToDefinitionNamesMapping.get('Posts')?.originalColumnNames; - const commentsColumnNames = tableNameToDefinitionNamesMapping.get('CommentsTable')?.originalColumnNames; - expect(postsColumnNames?.get('id')).toStrictEqual('"id"'); - expect(postsColumnNames?.get('AccountId')).toStrictEqual('"AccountId"'); - expect(postsColumnNames?.get('BlockHeight')).toStrictEqual('BlockHeight'); - expect(commentsColumnNames?.get('accountId')).toStrictEqual('"accountId"'); - expect(commentsColumnNames?.get('blockHeight')).toStrictEqual('blockHeight'); - }); - - test('GetSchemaLookup works for mixed quotes schema', async () => { - const indexer = new Indexer(caseSensitiveConfig, { dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }); - - const schemaLookup = indexer.getTableNameToDefinitionNamesMapping(CASE_SENSITIVE_SCHEMA); - const tableNames = [...schemaLookup.keys()]; - const originalTableNames = tableNames.map((tableName) => schemaLookup.get(tableName)?.originalTableName); - expect(tableNames).toStrictEqual(['Posts', 'CommentsTable']); - expect(originalTableNames).toStrictEqual(['Posts', '"CommentsTable"']); - - // Spot check quoting for columnNames - expect(schemaLookup.get('Posts')?.originalColumnNames.get('id')).toStrictEqual('"id"'); - expect(schemaLookup.get('Posts')?.originalColumnNames.get('AccountId')).toStrictEqual('"AccountId"'); - expect(schemaLookup.get('Posts')?.originalColumnNames.get('BlockHeight')).toStrictEqual('BlockHeight'); - expect(schemaLookup.get('CommentsTable')?.originalColumnNames.get('accountId')).toStrictEqual('"accountId"'); - expect(schemaLookup.get('CommentsTable')?.originalColumnNames.get('blockHeight')).toStrictEqual('blockHeight'); - }); - - test('SanitizeTableName works properly on many test cases', async () => { - const indexer = new Indexer(simpleSchemaConfig, { dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }, config); - - expect(indexer.sanitizeTableName('table_name')).toStrictEqual('TableName'); - expect(indexer.sanitizeTableName('tablename')).toStrictEqual('Tablename'); // name is not capitalized - expect(indexer.sanitizeTableName('table name')).toStrictEqual('TableName'); - expect(indexer.sanitizeTableName('table!name!')).toStrictEqual('TableName'); - expect(indexer.sanitizeTableName('123TABle')).toStrictEqual('_123TABle'); // underscore at beginning - expect(indexer.sanitizeTableName('123_tABLE')).toStrictEqual('_123TABLE'); // underscore at beginning, capitalization - expect(indexer.sanitizeTableName('some-table_name')).toStrictEqual('SomeTableName'); - expect(indexer.sanitizeTableName('!@#$%^&*()table@)*&(%#')).toStrictEqual('Table'); // All special characters removed - expect(indexer.sanitizeTableName('T_name')).toStrictEqual('TName'); - expect(indexer.sanitizeTableName('_table')).toStrictEqual('Table'); // Starting underscore was removed - }); - - test('indexer fails to build context.db due to collision on sanitized table names', async () => { - const schemaWithDuplicateSanitizedTableNames = `CREATE TABLE - "test table" ( - "id" SERIAL NOT NULL - ); - CREATE TABLE "test!table" ( - "id" SERIAL NOT NULL - );`; - const indexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, 'code', schemaWithDuplicateSanitizedTableNames, LogLevel.INFO); - const indexer = new Indexer(indexerConfig, { dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }, config); - - // Does not outright throw an error but instead returns an empty object - expect(indexer.buildDatabaseContext(1, [])) - .toStrictEqual({}); - }); - - test('indexer builds context and inserts an objects into existing table', async () => { - const mockDmlHandler: any = { insert: jest.fn().mockReturnValue([{ colA: 'valA' }, { colA: 'valA' }]) }; - - const indexer = new Indexer(socialSchemaConfig, { - fetch: genericMockFetch as unknown as typeof fetch, - dmlHandler: mockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - const context = indexer.buildContext(1, []); - - const objToInsert = [{ - account_id: 'morgs_near', - block_height: 1, - receipt_id: 'abc', - content: 'test', - block_timestamp: 800, - accounts_liked: JSON.stringify(['cwpuzzles.near', 'devbose.near']) - }, - { - account_id: 'morgs_near', - block_height: 2, - receipt_id: 'abc', - content: 'test', - block_timestamp: 801, - accounts_liked: JSON.stringify(['cwpuzzles.near']) - }]; - - const result = await context.db.Posts.insert(objToInsert); - expect(result.length).toEqual(2); - }); - - test('indexer builds context and does simultaneous upserts', async () => { - const mockPgClient = { - query: jest.fn().mockReturnValue({ rows: [] }), - format: jest.fn().mockReturnValue('mock') - } as unknown as PgClient; - const mockDmlHandler: any = new DmlHandler(genericDbCredentials, socialSchemaConfig, mockPgClient); - const upsertSpy = jest.spyOn(mockDmlHandler, 'upsert'); - const indexer = new Indexer(socialSchemaConfig, { - fetch: genericMockFetch as unknown as typeof fetch, - dmlHandler: mockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - const context = indexer.buildContext(1, []); - const promises: any[] = []; - - for (let i = 1; i <= 100; i++) { - const promise = context.db.Posts.upsert( - { - account_id: 'morgs_near', - block_height: i, - receipt_id: 'abc', - content: 'test_content', - block_timestamp: 800, - accounts_liked: JSON.stringify(['cwpuzzles.near', 'devbose.near']) - }, - ['account_id', 'block_height'], - ['content', 'block_timestamp'] - ); - promises.push(promise); - } - await Promise.all(promises); - - expect(upsertSpy).toHaveBeenCalledTimes(100); - }); - - test('indexer builds context and selects objects from existing table', async () => { - const selectFn = jest.fn(); - selectFn.mockImplementation((...args) => { - // Expects limit to be last parameter - return args[args.length - 1] === null ? [{ colA: 'valA' }, { colA: 'valA' }] : [{ colA: 'valA' }]; - }); - const mockDmlHandler: any = { select: selectFn }; - - const indexer = new Indexer(socialSchemaConfig, { - fetch: genericMockFetch as unknown as typeof fetch, - dmlHandler: mockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - const context = indexer.buildContext(1, []); - - const objToSelect = { - account_id: 'morgs_near', - receipt_id: 'abc', - }; - const result = await context.db.Posts.select(objToSelect); - expect(result.length).toEqual(2); - const resultLimit = await context.db.Posts.select(objToSelect, 1); - expect(resultLimit.length).toEqual(1); - }); - - test('indexer builds context and updates multiple objects from existing table', async () => { - const mockDmlHandler: any = { - update: jest.fn().mockImplementation((_, whereObj, updateObj) => { - if (whereObj.account_id === 'morgs_near' && updateObj.content === 'test_content') { - return [{ colA: 'valA' }, { colA: 'valA' }]; - } - return [{}]; - }) - }; - - const indexer = new Indexer(socialSchemaConfig, { - fetch: genericMockFetch as unknown as typeof fetch, - dmlHandler: mockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - const context = indexer.buildContext(1, []); - - const whereObj = { - account_id: 'morgs_near', - receipt_id: 'abc', - }; - const updateObj = { - content: 'test_content', - block_timestamp: 805, - }; - const result = await context.db.Posts.update(whereObj, updateObj); - expect(result.length).toEqual(2); - }); - - test('indexer builds context and upserts on existing table', async () => { - const mockDmlHandler: any = { - upsert: jest.fn().mockImplementation((_, objects, conflict, update) => { - if (objects.length === 2 && conflict.includes('account_id') && update.includes('content')) { - return [{ colA: 'valA' }, { colA: 'valA' }]; - } else if (objects.length === 1 && conflict.includes('account_id') && update.includes('content')) { - return [{ colA: 'valA' }]; - } - return [{}]; - }) - }; - - const indexer = new Indexer(socialSchemaConfig, { - fetch: genericMockFetch as unknown as typeof fetch, - dmlHandler: mockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - const context = indexer.buildContext(1, []); - - const objToInsert = [{ - account_id: 'morgs_near', - block_height: 1, - receipt_id: 'abc', - content: 'test', - block_timestamp: 800, - accounts_liked: JSON.stringify(['cwpuzzles.near', 'devbose.near']) - }, - { - account_id: 'morgs_near', - block_height: 2, - receipt_id: 'abc', - content: 'test', - block_timestamp: 801, - accounts_liked: JSON.stringify(['cwpuzzles.near']) - }]; - - let result = await context.db.Posts.upsert(objToInsert, ['account_id', 'block_height'], ['content', 'block_timestamp']); - expect(result.length).toEqual(2); - result = await context.db.Posts.upsert(objToInsert[0], ['account_id', 'block_height'], ['content', 'block_timestamp']); - expect(result.length).toEqual(1); - }); - - test('indexer builds context and deletes objects from existing table', async () => { - const mockDmlHandler: any = { delete: jest.fn().mockReturnValue([{ colA: 'valA' }, { colA: 'valA' }]) }; - - const indexer = new Indexer(socialSchemaConfig, { - fetch: genericMockFetch as unknown as typeof fetch, - dmlHandler: mockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - const context = indexer.buildContext(1, []); - - const deleteFilter = { - account_id: 'morgs_near', - receipt_id: 'abc', - }; - const result = await context.db.Posts.delete(deleteFilter); - expect(result.length).toEqual(2); - }); - - test('indexer builds context and verifies all methods generated', async () => { - const indexer = new Indexer(stressTestConfig, { - fetch: genericMockFetch as unknown as typeof fetch, - dmlHandler: genericMockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - const context = indexer.buildContext(1, []); - - expect(Object.keys(context.db)).toStrictEqual([ - 'CreatorQuest', - 'ComposerQuest', - 'ContractorQuest', - 'Posts', - 'Comments', - 'PostLikes', - 'MyTable1', - 'AnotherTable', - 'ThirdTable', - 'YetAnotherTable']); - expect(Object.keys(context.db.CreatorQuest)).toStrictEqual([ - 'insert', - 'select', - 'update', - 'upsert', - 'delete']); - expect(Object.keys(context.db.PostLikes)).toStrictEqual([ - 'insert', - 'select', - 'update', - 'upsert', - 'delete']); - expect(Object.keys(context.db.MyTable1)).toStrictEqual([ - 'insert', - 'select', - 'update', - 'upsert', - 'delete']); - }); - - test('indexer builds context and returns empty array if failed to generate db methods', async () => { - const indexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, 'code', '', LogLevel.INFO); - const indexer = new Indexer(indexerConfig, { - fetch: genericMockFetch as unknown as typeof fetch, - dmlHandler: genericMockDmlHandler, - indexerMeta: genericMockIndexerMeta, - }, config); - const context = indexer.buildContext(1, []); - - expect(Object.keys(context.db)).toStrictEqual([]); - }); - - test('Indexer.execute() allows imperative execution of GraphQL operations', async () => { - const postId = 1; - const commentId = 2; - const blockHeight = 82699904; - const mockFetch = jest.fn() - .mockReturnValueOnce({ // "running function on ..." log - status: 200, - json: async () => ({ - data: { - indexer_log_store: [ - { - id: '12345', - }, - ], - }, - }), - }) - .mockReturnValueOnce({ // set status - status: 200, - json: async () => ({ - errors: null, - }), - }) - .mockReturnValueOnce({ // query - status: 200, - json: async () => ({ - data: { - posts: [ - { - id: postId, - }, - ], - }, - }), - }) - .mockReturnValueOnce({ // mutation - status: 200, - json: async () => ({ - data: { - insert_comments: { - returning: { - id: commentId, - }, - }, - }, - }), - }) - .mockReturnValueOnce({ - status: 200, - json: async () => ({ - errors: null, - }), - }); - - const mockBlock = Block.fromStreamerMessage({ - block: { - chunks: [0], - header: { - height: blockHeight - } - }, - shards: {} - } as unknown as StreamerMessage) as unknown as Block; - - const code = ` - const { posts } = await context.graphql(\` - query { - posts(where: { id: { _eq: 1 } }) { - id - } - } - \`); - - if (!posts || posts.length === 0) { - return; - } - - const [post] = posts; - - const { insert_comments: { returning: { id } } } = await context.graphql(\` - mutation { - insert_comments( - objects: {account_id: "morgs.near", block_height: \${block.blockHeight}, content: "cool post", post_id: \${post.id}} - ) { - returning { - id - } - } - } - \`); - - return (\`Created comment \${id} on post \${post.id}\`) - `; - const indexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, 'buildnear.testnet', 'test', 0, code, SIMPLE_SCHEMA, LogLevel.INFO); - const indexer = new Indexer(indexerConfig, { - fetch: mockFetch as unknown as typeof fetch, - dmlHandler: genericMockDmlHandler, - indexerMeta: genericMockIndexerMeta - }, config); - - await indexer.execute(mockBlock); - - expect(mockFetch.mock.calls).toMatchSnapshot(); - }); - - test('Indexer.execute() console.logs', async () => { - const logs: string[] = []; - const context = { - log: (...m: string[]) => { - logs.push(...m); - } - }; - const vm = new VM(); - vm.freeze(context, 'context'); - vm.freeze(context, 'console'); - await vm.run('console.log("hello", "brave new"); context.log("world")'); - expect(logs).toEqual(['hello', 'brave new', 'world']); - }); - test('Errors thrown in VM can be caught outside the VM', async () => { const vm = new VM(); expect(() => { @@ -870,12 +121,6 @@ describe('Indexer unit tests', () => { }); test('Indexer.execute() catches errors', async () => { - const mockFetch = jest.fn(() => ({ - status: 200, - json: async () => ({ - errors: null, - }), - })); const blockHeight = 456; const mockBlock = Block.fromStreamerMessage({ block: { @@ -896,57 +141,16 @@ describe('Indexer unit tests', () => { } as unknown as IndexerMeta; const indexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, 'buildnear.testnet', 'test', 0, code, SIMPLE_SCHEMA, LogLevel.INFO); const indexer = new Indexer(indexerConfig, { - fetch: mockFetch as unknown as typeof fetch, - dmlHandler: genericMockDmlHandler, + contextBuilder: genericMockContextBuilder, indexerMeta, - }, config); + }); await expect(indexer.execute(mockBlock)).rejects.toThrow(new Error('Execution error: boom')); - expect(mockFetch.mock.calls).toMatchSnapshot(); expect(indexerMeta.setStatus).toHaveBeenNthCalledWith(1, IndexerStatus.RUNNING); expect(indexerMeta.setStatus).toHaveBeenNthCalledWith(2, IndexerStatus.FAILING); expect(indexerMeta.updateBlockHeight).not.toHaveBeenCalled(); }); - test('Indexer.execute() supplies the required role to the GraphQL endpoint', async () => { - const blockHeight = 82699904; - const mockFetch = jest.fn(() => ({ - status: 200, - json: async () => ({ - errors: null, - }), - })); - const mockBlock = Block.fromStreamerMessage({ - block: { - chunks: [0], - header: { - height: blockHeight - } - }, - shards: {} - } as unknown as StreamerMessage) as unknown as Block; - const indexerMeta = { - writeLogs: jest.fn(), - setStatus: jest.fn(), - updateBlockHeight: jest.fn().mockResolvedValue(null) - } as unknown as IndexerMeta; - const code = ` - context.graphql(\`mutation { set(functionName: "buildnear.testnet/test", key: "height", data: "\${block.blockHeight}")}\`); - `; - const indexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, 'morgs.near', 'test', 0, code, SIMPLE_SCHEMA, LogLevel.INFO); - const indexer = new Indexer(indexerConfig, { - fetch: mockFetch as unknown as typeof fetch, - dmlHandler: genericMockDmlHandler, - indexerMeta, - }, config); - - await indexer.execute(mockBlock); - - expect(indexerMeta.setStatus).toHaveBeenNthCalledWith(1, IndexerStatus.RUNNING); - expect(mockFetch.mock.calls).toMatchSnapshot(); - expect(indexerMeta.updateBlockHeight).toHaveBeenCalledWith(blockHeight); - }); - test('Indexer passes all relevant logs to writeLogs', async () => { const mockDebugIndexerMeta = { writeLogs: jest.fn(), @@ -987,33 +191,28 @@ describe('Indexer unit tests', () => { const infoIndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, 'buildnear.testnet', 'test', 0, code, SIMPLE_SCHEMA, LogLevel.INFO); const errorIndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, 'buildnear.testnet', 'test', 0, code, SIMPLE_SCHEMA, LogLevel.ERROR); const mockDmlHandler: DmlHandler = { select: jest.fn() } as unknown as DmlHandler; + const partialMockContextBuilder: ContextBuilder = new ContextBuilder(SIMPLE_INDEXER_CONFIG, { dmlHandler: mockDmlHandler }); const indexerDebug = new Indexer( debugIndexerConfig, { - fetch: jest.fn() as unknown as typeof fetch, - dmlHandler: mockDmlHandler, + contextBuilder: partialMockContextBuilder, indexerMeta: mockDebugIndexerMeta as unknown as IndexerMeta }, - config ); const indexerInfo = new Indexer( infoIndexerConfig, { - fetch: jest.fn() as unknown as typeof fetch, - dmlHandler: mockDmlHandler, + contextBuilder: partialMockContextBuilder, indexerMeta: mockInfoIndexerMeta as unknown as IndexerMeta }, - config ); const indexerError = new Indexer( errorIndexerConfig, { - fetch: jest.fn() as unknown as typeof fetch, - dmlHandler: mockDmlHandler, + contextBuilder: partialMockContextBuilder, indexerMeta: mockErrorIndexerMeta as unknown as IndexerMeta }, - config ); await indexerDebug.execute(mockBlock); @@ -1036,49 +235,7 @@ describe('Indexer unit tests', () => { expect(mockErrorIndexerMeta.updateBlockHeight).toHaveBeenCalledWith(blockHeight); }); - test('attaches the backend only header to requests to hasura', async () => { - const mockFetch = jest.fn() - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ - data: {} - }) - }); - const indexer = new Indexer(simpleSchemaConfig, { fetch: mockFetch as unknown as typeof fetch, dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }, config); - const context = indexer.buildContext(1, []); - - const mutation = ` - mutation { - newGreeting(greeting: "howdy") { - success - } - } - `; - - await context.graphql(mutation); - - expect(mockFetch.mock.calls[0]).toEqual([ - `${config.hasuraEndpoint}/v1/graphql`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Hasura-Use-Backend-Only-Permissions': 'true', - 'X-Hasura-Role': simpleSchemaConfig.hasuraRoleName(), - 'X-Hasura-Admin-Secret': config.hasuraAdminSecret - }, - body: JSON.stringify({ query: mutation }) - } - ]); - }); - it('call writeLogs method at the end of execution with correct and all logs are present', async () => { - const mockFetchDebug = jest.fn(() => ({ - status: 200, - json: async () => ({ - errors: null, - }), - })); const blockHeight = 456; const mockBlock = Block.fromStreamerMessage({ block: { @@ -1108,52 +265,20 @@ describe('Indexer unit tests', () => { const debugIndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, 'buildnear.testnet', 'test', 0, code, SIMPLE_SCHEMA, LogLevel.DEBUG); const mockDmlHandler: DmlHandler = { select: jest.fn() } as unknown as DmlHandler; + const partialMockContextBuilder: ContextBuilder = new ContextBuilder(debugIndexerConfig, { dmlHandler: mockDmlHandler }); const indexerDebug = new Indexer( debugIndexerConfig, - { fetch: mockFetchDebug as unknown as typeof fetch, dmlHandler: mockDmlHandler, indexerMeta }, - config + { contextBuilder: partialMockContextBuilder, indexerMeta }, ); await indexerDebug.execute(mockBlock); expect(indexerMeta.writeLogs).toHaveBeenCalledTimes(1); expect(indexerMeta.writeLogs.mock.calls[0][0]).toHaveLength(5); }); - test('does not attach the hasura admin secret header when no role specified', async () => { - const mockFetch = jest.fn() - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ - data: {} - }) - }); - const indexer = new Indexer(simpleSchemaConfig, { fetch: mockFetch as unknown as typeof fetch, dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }, config); - - const mutation = ` - mutation { - newGreeting(greeting: "howdy") { - success - } - } - `; - - await indexer.runGraphQLQuery(mutation, null, 0, null); - - expect(mockFetch.mock.calls[0]).toEqual([ - `${config.hasuraEndpoint}/v1/graphql`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Hasura-Use-Backend-Only-Permissions': 'true', - }, - body: JSON.stringify({ query: mutation }) - } - ]); - }); test('transformedCode applies the correct transformations', () => { const indexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, 'console.log(\'hello\')', SIMPLE_SCHEMA, LogLevel.INFO); - const indexer = new Indexer(indexerConfig, { dmlHandler: genericMockDmlHandler, indexerMeta: genericMockIndexerMeta }, config); + const indexer = new Indexer(indexerConfig, { contextBuilder: genericMockContextBuilder, indexerMeta: genericMockIndexerMeta }); const transformedFunction = indexer.transformIndexerFunction(); expect(transformedFunction).toEqual(` diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index 29b7ad2e..71859ed0 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -12,7 +12,6 @@ import { type IndexerMetaInterface } from '../indexer-meta/indexer-meta'; import type ContextBuilder from '../context'; interface Dependencies { - fetch?: typeof fetch contextBuilder: ContextBuilder indexerMeta: IndexerMetaInterface parser?: Parser @@ -39,7 +38,6 @@ export default class Indexer { this.logger = logger.child({ accountId: indexerConfig.accountId, functionName: indexerConfig.functionName, service: this.constructor.name }); this.deps = { - fetch, parser: new Parser(), ...deps }; diff --git a/runner/tsconfig.json b/runner/tsconfig.json index d4949c0e..e67cb26c 100644 --- a/runner/tsconfig.json +++ b/runner/tsconfig.json @@ -16,8 +16,8 @@ "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ - "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ - "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + "noUnusedLocals": false, /* Enable error reporting when local variables aren't read. */ + "noUnusedParameters": false, /* Raise an error when a function parameter isn't read. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ }, "include": ["./src", "./tests", "./scripts"], From f3772d5770267a790f41d66c2b144a077a810607 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 25 Jul 2024 13:13:02 -0700 Subject: [PATCH 03/17] Implement necessary unit tests in context --- .../__snapshots__/context.test.ts.snap | 158 ++++++++ runner/src/context/context.test.ts | 342 ++++++++++++++++-- runner/src/context/context.ts | 8 +- .../__snapshots__/indexer.test.ts.snap | 72 ---- runner/src/indexer/indexer.test.ts | 4 - 5 files changed, 473 insertions(+), 111 deletions(-) create mode 100644 runner/src/context/__snapshots__/context.test.ts.snap delete mode 100644 runner/src/indexer/__snapshots__/indexer.test.ts.snap diff --git a/runner/src/context/__snapshots__/context.test.ts.snap b/runner/src/context/__snapshots__/context.test.ts.snap new file mode 100644 index 00000000..05235ed9 --- /dev/null +++ b/runner/src/context/__snapshots__/context.test.ts.snap @@ -0,0 +1,158 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ContextBuilder unit tests Context object social api can fetch from the near social api 1`] = ` +[ + [ + "https://api.near.social/index", + { + "body": "{"action":"post","key":"main","options":{"limit":1,"order":"desc"}}", + "headers": { + "Content-Type": "application/json", + }, + "method": "POST", + }, + ], +] +`; + +exports[`ContextBuilder unit tests ContextBuilder adds CRUD operations for table 1`] = ` +{ + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], +} +`; + +exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 1`] = ` +{ + "Posts": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, +} +`; + +exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 2`] = ` +{ + "Comments": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "PostLikes": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "Posts": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, +} +`; + +exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 3`] = ` +{ + "CommentsTable": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "Posts": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, +} +`; + +exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 4`] = ` +{ + "AnotherTable": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "Comments": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "ComposerQuest": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "ContractorQuest": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "CreatorQuest": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "MyTable1": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "PostLikes": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "Posts": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "ThirdTable": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "YetAnotherTable": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, +} +`; diff --git a/runner/src/context/context.test.ts b/runner/src/context/context.test.ts index fabb9880..786a5cb9 100644 --- a/runner/src/context/context.test.ts +++ b/runner/src/context/context.test.ts @@ -1,9 +1,17 @@ -const config = { - hasuraEndpoint: 'mock-hasura-endpoint', - hasuraAdminSecret: 'mock-hasura-secret', -}; +import type fetch from 'node-fetch'; -const SIMPLE_SCHEMA = `CREATE TABLE +import type DmlHandler from '../dml-handler'; +import IndexerConfig from '../indexer-config'; +import { LogLevel } from '../indexer-meta/log-entry'; +import ContextBuilder from './context'; + +describe('ContextBuilder unit tests', () => { + const MOCK_CONFIG = { + hasuraEndpoint: 'mock-hasura-endpoint', + hasuraAdminSecret: 'mock-hasura-secret', + }; + + const SIMPLE_SCHEMA = `CREATE TABLE "posts" ( "id" SERIAL NOT NULL, "account_id" VARCHAR NOT NULL, @@ -16,7 +24,7 @@ const SIMPLE_SCHEMA = `CREATE TABLE CONSTRAINT "posts_pkey" PRIMARY KEY ("id") );`; -const SOCIAL_SCHEMA = ` + const SOCIAL_SCHEMA = ` CREATE TABLE "posts" ( "id" SERIAL NOT NULL, @@ -52,7 +60,7 @@ const SOCIAL_SCHEMA = ` CONSTRAINT "post_likes_pkey" PRIMARY KEY ("post_id", "account_id") );`; -const CASE_SENSITIVE_SCHEMA = ` + const CASE_SENSITIVE_SCHEMA = ` CREATE TABLE Posts ( "id" SERIAL NOT NULL, @@ -75,7 +83,7 @@ const CASE_SENSITIVE_SCHEMA = ` CONSTRAINT "comments_pkey" PRIMARY KEY ("id") );`; -const STRESS_TEST_SCHEMA = ` + const STRESS_TEST_SCHEMA = ` CREATE TABLE creator_quest ( account_id VARCHAR PRIMARY KEY, num_components_created INTEGER NOT NULL DEFAULT 0, @@ -166,26 +174,298 @@ const STRESS_TEST_SCHEMA = ` yet_another_table (id serial PRIMARY KEY); `; -const SIMPLE_REDIS_STREAM = 'test:stream'; -const SIMPLE_ACCOUNT_ID = 'morgs.near'; -const SIMPLE_FUNCTION_NAME = 'test_indexer'; -const SIMPLE_CODE = 'const a = 1;'; -const simpleSchemaConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, SIMPLE_SCHEMA, LogLevel.INFO); -const socialSchemaConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, SOCIAL_SCHEMA, LogLevel.INFO); -const caseSensitiveConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, CASE_SENSITIVE_SCHEMA, LogLevel.INFO); -const stressTestConfig: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, STRESS_TEST_SCHEMA, LogLevel.INFO); -const genericDbCredentials: PostgresConnectionParams = { - database: 'test_near', - host: 'postgres', - password: 'test_pass', - port: 5432, - user: 'test_near' -}; - -const genericMockFetch = jest.fn() - .mockResolvedValue({ - status: 200, - json: async () => ({ - data: 'mock', - }), - }) as unknown as typeof fetch; + const SIMPLE_REDIS_STREAM = 'test:stream'; + const SIMPLE_ACCOUNT_ID = 'morgs.near'; + const SIMPLE_FUNCTION_NAME = 'test_indexer'; + const SIMPLE_CODE = 'const a = 1;'; + const SIMPLE_SCHEMA_CONFIG: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, SIMPLE_SCHEMA, LogLevel.INFO); + const SIMPLE_SOCIAL_CONFIG: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, SOCIAL_SCHEMA, LogLevel.INFO); + const CASE_SENSITIVE_CONFIG: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, CASE_SENSITIVE_SCHEMA, LogLevel.INFO); + const STRESS_TEST_CONFIG: IndexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, SIMPLE_CODE, STRESS_TEST_SCHEMA, LogLevel.INFO); + + const genericMockFetch = jest.fn() + .mockResolvedValue({ + status: 200, + json: async () => ({ + data: 'mock', + }), + }) as unknown as typeof fetch; + + const genericMockDmlHandler = { + insert: jest.fn().mockReturnValue([]), + select: jest.fn().mockReturnValue([]), + update: jest.fn().mockReturnValue([]), + upsert: jest.fn().mockReturnValue([]), + delete: jest.fn().mockReturnValue([]), + } as unknown as DmlHandler; + + test('ContextBuilder can parse various schemas', () => { + const simpleContextBuilder = new ContextBuilder( + SIMPLE_SCHEMA_CONFIG, + { + dmlHandler: genericMockDmlHandler, + fetch: genericMockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + + const socialContextBuilder = new ContextBuilder( + SIMPLE_SOCIAL_CONFIG, + { + dmlHandler: genericMockDmlHandler, + fetch: genericMockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + + const caseSensitiveContextBuilder = new ContextBuilder( + CASE_SENSITIVE_CONFIG, + { + dmlHandler: genericMockDmlHandler, + fetch: genericMockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + + const stressTestContextBuilder = new ContextBuilder( + STRESS_TEST_CONFIG, + { + dmlHandler: genericMockDmlHandler, + fetch: genericMockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + + expect(simpleContextBuilder.buildContext(1, []).db).toMatchSnapshot(); + expect(socialContextBuilder.buildContext(1, []).db).toMatchSnapshot(); + expect(caseSensitiveContextBuilder.buildContext(1, []).db).toMatchSnapshot(); + expect(stressTestContextBuilder.buildContext(1, []).db).toMatchSnapshot(); + }); + + test('ContextBuilder adds CRUD operations for table', () => { + const contextBuilder = new ContextBuilder( + SIMPLE_SCHEMA_CONFIG, + { + dmlHandler: genericMockDmlHandler, + fetch: genericMockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + const context = contextBuilder.buildContext(1, []); + context.db.Posts.insert({}); + context.db.Posts.select({}); + context.db.Posts.update({}, {}); + context.db.Posts.upsert({}, [], []); + context.db.Posts.delete({}); + + expect(genericMockDmlHandler.insert).toHaveBeenCalledTimes(1); + expect(genericMockDmlHandler.select).toHaveBeenCalledTimes(1); + expect(genericMockDmlHandler.update).toHaveBeenCalledTimes(1); + expect(genericMockDmlHandler.upsert).toHaveBeenCalledTimes(1); + expect(genericMockDmlHandler.delete).toHaveBeenCalledTimes(1); + expect(context.db.Posts).toMatchSnapshot(); + }); + + test('Context object has empty db object if schema is empty', async () => { + const indexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, 'code', '', LogLevel.INFO); + const contextBuilder = new ContextBuilder( + indexerConfig, + { + dmlHandler: genericMockDmlHandler, + fetch: genericMockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + const context = contextBuilder.buildContext(1, []); + + expect(Object.keys(context.db)).toStrictEqual([]); + }); + + test('Context object has empty db object if schema fails to parse', async () => { + const schemaWithDuplicateSanitizedTableNames = `CREATE TABLE + "test table" ( + "id" SERIAL NOT NULL + ); + CREATE TABLE "test!table" ( + "id" SERIAL NOT NULL + );`; + const indexerConfig = new IndexerConfig(SIMPLE_REDIS_STREAM, SIMPLE_ACCOUNT_ID, SIMPLE_FUNCTION_NAME, 0, 'code', schemaWithDuplicateSanitizedTableNames, LogLevel.INFO); + const contextBuilder = new ContextBuilder( + indexerConfig, + { + dmlHandler: genericMockDmlHandler, + fetch: genericMockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + + // Does not outright throw an error but instead returns an empty object + expect(contextBuilder.buildDatabaseContext(1, [])) + .toStrictEqual({}); + }); + + test('Context object allows execution of arbitrary GraphQL operations', async () => { + const mockFetch = jest.fn() + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ + data: { + greet: 'hello' + } + }) + }) + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ + data: { + newGreeting: { + success: true + } + } + }) + }); + const contextBuilder = new ContextBuilder( + SIMPLE_SCHEMA_CONFIG, + { + dmlHandler: genericMockDmlHandler, + fetch: mockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + const context = contextBuilder.buildContext(1, []); + const query = ` + query { + greet() + } + `; + const { greet } = await context.graphql(query) as { greet: string }; + + const mutation = ` + mutation { + newGreeting(greeting: "${greet} morgan") { + success + } + } + `; + const { newGreeting: { success } } = await context.graphql(mutation); + + expect(greet).toEqual('hello'); + expect(success).toEqual(true); + expect(mockFetch.mock.calls[0]).toEqual([ + `${MOCK_CONFIG.hasuraEndpoint}/v1/graphql`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Hasura-Use-Backend-Only-Permissions': 'true', + 'X-Hasura-Role': 'morgs_near', + 'X-Hasura-Admin-Secret': MOCK_CONFIG.hasuraAdminSecret + }, + body: JSON.stringify({ query }) + } + ]); + expect(mockFetch.mock.calls[1]).toEqual([ + `${MOCK_CONFIG.hasuraEndpoint}/v1/graphql`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Hasura-Use-Backend-Only-Permissions': 'true', + 'X-Hasura-Role': 'morgs_near', + 'X-Hasura-Admin-Secret': MOCK_CONFIG.hasuraAdminSecret + }, + body: JSON.stringify({ query: mutation }) + } + ]); + }); + + test('Context object social api can fetch from the near social api', async () => { + const mockFetch = jest.fn(); + const contextBuilder = new ContextBuilder( + SIMPLE_SCHEMA_CONFIG, + { + dmlHandler: genericMockDmlHandler, + fetch: mockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + const context = contextBuilder.buildContext(1, []); + + await context.fetchFromSocialApi('/index', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + action: 'post', + key: 'main', + options: { + limit: 1, + order: 'desc' + } + }) + }); + + expect(mockFetch.mock.calls).toMatchSnapshot(); + }); + + test('Context object graphql function throws when a GraphQL response contains errors', async () => { + const mockFetch = jest.fn() + .mockResolvedValue({ + json: async () => ({ + errors: ['boom'] + }) + }); + const contextBuilder = new ContextBuilder( + SIMPLE_SCHEMA_CONFIG, + { + dmlHandler: genericMockDmlHandler, + fetch: mockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + const context = contextBuilder.buildContext(1, []); + + await expect(async () => await context.graphql('query { hello }')).rejects.toThrow('boom'); + }); + + test('Context object graphl handles GraphQL variables and sets backend only permissions', async () => { + const mockFetch = jest.fn() + .mockResolvedValue({ + status: 200, + json: async () => ({ + data: 'mock', + }), + }); + const contextBuilder = new ContextBuilder( + SIMPLE_SCHEMA_CONFIG, + { + dmlHandler: genericMockDmlHandler, + fetch: mockFetch as unknown as typeof fetch, + }, + MOCK_CONFIG + ); + const context = contextBuilder.buildContext(1, []); + + const query = 'query($name: String) { hello(name: $name) }'; + const variables = { name: 'morgan' }; + await context.graphql(query, variables); + + expect(mockFetch.mock.calls[0]).toEqual([ + `${MOCK_CONFIG.hasuraEndpoint}/v1/graphql`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Hasura-Use-Backend-Only-Permissions': 'true', + 'X-Hasura-Role': 'morgs_near', + 'X-Hasura-Admin-Secret': MOCK_CONFIG.hasuraAdminSecret + }, + body: JSON.stringify({ + query, + variables, + }), + }, + ]); + }); +}); diff --git a/runner/src/context/context.ts b/runner/src/context/context.ts index 9f5523da..ca32e2f1 100644 --- a/runner/src/context/context.ts +++ b/runner/src/context/context.ts @@ -168,6 +168,10 @@ export default class ContextBuilder { blockHeight: number, logEntries: LogEntry[], ): Record any>> { + if (this.tableDefinitions.size === 0) { + logEntries.push(LogEntry.systemDebug('No tables found in schema. No context.db methods generated')); + return {}; + } try { const tableNames = Array.from(this.tableDefinitions.keys()); const sanitizedTableNames = new Set(); @@ -284,9 +288,5 @@ function getTableNameToDefinitionNamesMapping (schema: string): Map { const SIMPLE_REDIS_STREAM = 'test:stream'; From c4a48226ecae450d62ccd48684e78b00e00c8ba3 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 25 Jul 2024 15:38:06 -0700 Subject: [PATCH 04/17] Update build to succeed --- runner/src/stream-handler/worker.ts | 2 +- runner/tsconfig.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/src/stream-handler/worker.ts b/runner/src/stream-handler/worker.ts index bd865f8f..c25f2849 100644 --- a/runner/src/stream-handler/worker.ts +++ b/runner/src/stream-handler/worker.ts @@ -15,7 +15,7 @@ import parentLogger from '../logger'; import { wrapSpan } from '../utility'; import { type PostgresConnectionParams } from '../pg-client'; import DmlHandler from '../dml-handler/dml-handler'; -import ContextBuilder from '../context/context'; +import ContextBuilder from '../context'; if (isMainThread) { throw new Error('Worker should not be run on main thread'); diff --git a/runner/tsconfig.json b/runner/tsconfig.json index e67cb26c..d4949c0e 100644 --- a/runner/tsconfig.json +++ b/runner/tsconfig.json @@ -16,8 +16,8 @@ "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ - "noUnusedLocals": false, /* Enable error reporting when local variables aren't read. */ - "noUnusedParameters": false, /* Raise an error when a function parameter isn't read. */ + "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ }, "include": ["./src", "./tests", "./scripts"], From e24f3d67f8d18b743be8370d18fbfa7a7b5081f7 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 25 Jul 2024 15:48:07 -0700 Subject: [PATCH 05/17] Rename context folder --- .../context-builder.test.ts.snap} | 0 .../__snapshots__/context.test.ts.snap | 158 ++++++++++++++++++ .../context-builder.test.ts} | 2 +- .../context-builder.ts} | 0 runner/src/context-builder/index.ts | 2 + runner/src/context/index.ts | 2 - runner/src/indexer/indexer.test.ts | 4 +- runner/src/indexer/indexer.ts | 4 +- runner/src/local-indexer/local-indexer.ts | 4 +- runner/src/stream-handler/worker.ts | 2 +- runner/tests/integration.test.ts | 2 +- 11 files changed, 170 insertions(+), 10 deletions(-) rename runner/src/{context/__snapshots__/context.test.ts.snap => context-builder/__snapshots__/context-builder.test.ts.snap} (100%) create mode 100644 runner/src/context-builder/__snapshots__/context.test.ts.snap rename runner/src/{context/context.test.ts => context-builder/context-builder.test.ts} (99%) rename runner/src/{context/context.ts => context-builder/context-builder.ts} (100%) create mode 100644 runner/src/context-builder/index.ts delete mode 100644 runner/src/context/index.ts diff --git a/runner/src/context/__snapshots__/context.test.ts.snap b/runner/src/context-builder/__snapshots__/context-builder.test.ts.snap similarity index 100% rename from runner/src/context/__snapshots__/context.test.ts.snap rename to runner/src/context-builder/__snapshots__/context-builder.test.ts.snap diff --git a/runner/src/context-builder/__snapshots__/context.test.ts.snap b/runner/src/context-builder/__snapshots__/context.test.ts.snap new file mode 100644 index 00000000..05235ed9 --- /dev/null +++ b/runner/src/context-builder/__snapshots__/context.test.ts.snap @@ -0,0 +1,158 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ContextBuilder unit tests Context object social api can fetch from the near social api 1`] = ` +[ + [ + "https://api.near.social/index", + { + "body": "{"action":"post","key":"main","options":{"limit":1,"order":"desc"}}", + "headers": { + "Content-Type": "application/json", + }, + "method": "POST", + }, + ], +] +`; + +exports[`ContextBuilder unit tests ContextBuilder adds CRUD operations for table 1`] = ` +{ + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], +} +`; + +exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 1`] = ` +{ + "Posts": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, +} +`; + +exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 2`] = ` +{ + "Comments": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "PostLikes": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "Posts": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, +} +`; + +exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 3`] = ` +{ + "CommentsTable": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "Posts": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, +} +`; + +exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 4`] = ` +{ + "AnotherTable": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "Comments": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "ComposerQuest": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "ContractorQuest": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "CreatorQuest": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "MyTable1": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "PostLikes": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "Posts": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "ThirdTable": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, + "YetAnotherTable": { + "delete": [Function], + "insert": [Function], + "select": [Function], + "update": [Function], + "upsert": [Function], + }, +} +`; diff --git a/runner/src/context/context.test.ts b/runner/src/context-builder/context-builder.test.ts similarity index 99% rename from runner/src/context/context.test.ts rename to runner/src/context-builder/context-builder.test.ts index 786a5cb9..8a55b13c 100644 --- a/runner/src/context/context.test.ts +++ b/runner/src/context-builder/context-builder.test.ts @@ -3,7 +3,7 @@ import type fetch from 'node-fetch'; import type DmlHandler from '../dml-handler'; import IndexerConfig from '../indexer-config'; import { LogLevel } from '../indexer-meta/log-entry'; -import ContextBuilder from './context'; +import ContextBuilder from './context-builder'; describe('ContextBuilder unit tests', () => { const MOCK_CONFIG = { diff --git a/runner/src/context/context.ts b/runner/src/context-builder/context-builder.ts similarity index 100% rename from runner/src/context/context.ts rename to runner/src/context-builder/context-builder.ts diff --git a/runner/src/context-builder/index.ts b/runner/src/context-builder/index.ts new file mode 100644 index 00000000..c3486cba --- /dev/null +++ b/runner/src/context-builder/index.ts @@ -0,0 +1,2 @@ +export { default } from './context-builder'; +export type { ContextObject } from './context-builder'; diff --git a/runner/src/context/index.ts b/runner/src/context/index.ts deleted file mode 100644 index b931cee5..00000000 --- a/runner/src/context/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { default } from './context'; -export type { ContextObject } from './context'; diff --git a/runner/src/indexer/indexer.test.ts b/runner/src/indexer/indexer.test.ts index be57a3d7..e104f9de 100644 --- a/runner/src/indexer/indexer.test.ts +++ b/runner/src/indexer/indexer.test.ts @@ -7,8 +7,8 @@ import { LogLevel } from '../indexer-meta/log-entry'; import IndexerConfig from '../indexer-config/indexer-config'; import { IndexerStatus } from '../indexer-meta'; import type IndexerMeta from '../indexer-meta'; -import ContextBuilder from '../context'; -import { type ContextObject } from '../context'; +import ContextBuilder from '../context-builder'; +import { type ContextObject } from '../context-builder'; describe('Indexer unit tests', () => { const SIMPLE_REDIS_STREAM = 'test:stream'; diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index 71859ed0..b7e60102 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -9,7 +9,7 @@ import LogEntry from '../indexer-meta/log-entry'; import type IndexerConfig from '../indexer-config'; import { IndexerStatus } from '../indexer-meta'; import { type IndexerMetaInterface } from '../indexer-meta/indexer-meta'; -import type ContextBuilder from '../context'; +import type ContextBuilder from '../context-builder'; interface Dependencies { contextBuilder: ContextBuilder @@ -28,7 +28,7 @@ export default class Indexer { tracer = trace.getTracer('queryapi-runner-indexer'); private readonly logger: typeof logger; - private readonly deps: Required; + readonly deps: Required; private currentStatus?: string; constructor ( diff --git a/runner/src/local-indexer/local-indexer.ts b/runner/src/local-indexer/local-indexer.ts index d84dfeba..56b1ac7b 100644 --- a/runner/src/local-indexer/local-indexer.ts +++ b/runner/src/local-indexer/local-indexer.ts @@ -1,3 +1,4 @@ +import ContextBuilder from '../context-builder'; import InMemoryDmlHandler from '../dml-handler/in-memory-dml-handler'; import IndexerConfig from '../indexer-config'; import { type LocalIndexerConfig } from '../indexer-config/indexer-config'; @@ -20,8 +21,9 @@ export default class LocalIndexer { logLevel: config.logLevel, }); const dmlHandler = new InMemoryDmlHandler(config.schema); + const contextBuilder = new ContextBuilder(fullIndexerConfig, { dmlHandler }); const indexerMeta = new NoOpIndexerMeta(config); - this.indexer = new Indexer(fullIndexerConfig, { indexerMeta, dmlHandler }); + this.indexer = new Indexer(fullIndexerConfig, { indexerMeta, contextBuilder }); this.lakeClient = new LakeClient(); } diff --git a/runner/src/stream-handler/worker.ts b/runner/src/stream-handler/worker.ts index c25f2849..cbd241b1 100644 --- a/runner/src/stream-handler/worker.ts +++ b/runner/src/stream-handler/worker.ts @@ -15,7 +15,7 @@ import parentLogger from '../logger'; import { wrapSpan } from '../utility'; import { type PostgresConnectionParams } from '../pg-client'; import DmlHandler from '../dml-handler/dml-handler'; -import ContextBuilder from '../context'; +import ContextBuilder from '../context-builder'; if (isMainThread) { throw new Error('Worker should not be run on main thread'); diff --git a/runner/tests/integration.test.ts b/runner/tests/integration.test.ts index a3aa4925..0790571f 100644 --- a/runner/tests/integration.test.ts +++ b/runner/tests/integration.test.ts @@ -15,7 +15,7 @@ import { LogLevel } from '../src/indexer-meta/log-entry'; import IndexerConfig from '../src/indexer-config'; import IndexerMeta from '../src/indexer-meta/indexer-meta'; import DmlHandler from '../src/dml-handler/dml-handler'; -import ContextBuilder from '../src/context'; +import ContextBuilder from '../src/context-builder'; describe('Indexer integration', () => { jest.setTimeout(300_000); From 5dedec3aac8d7dd97e274b043f4ca559bc40454e Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Fri, 26 Jul 2024 11:34:16 -0700 Subject: [PATCH 06/17] Update bitmap indexer tests to use context object --- core-indexers/receiver-blocks/unit.test.ts | 5 ++++- runner/src/local-indexer/local-indexer.ts | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/core-indexers/receiver-blocks/unit.test.ts b/core-indexers/receiver-blocks/unit.test.ts index 1d0c103b..0ed8140a 100644 --- a/core-indexers/receiver-blocks/unit.test.ts +++ b/core-indexers/receiver-blocks/unit.test.ts @@ -15,6 +15,9 @@ describe('Receiver Blocks Indexer Tests', () => { test('Try executing on a block', async () => { const localIndexer = new LocalIndexer(indexerConfig); - await localIndexer.executeOnBlock(123621232); + const context = localIndexer.getContext(); + + await localIndexer.executeOnBlock(100000000); + const receivers = context.db.Receivers.select({}) }); }); diff --git a/runner/src/local-indexer/local-indexer.ts b/runner/src/local-indexer/local-indexer.ts index 56b1ac7b..689d5c24 100644 --- a/runner/src/local-indexer/local-indexer.ts +++ b/runner/src/local-indexer/local-indexer.ts @@ -1,4 +1,4 @@ -import ContextBuilder from '../context-builder'; +import ContextBuilder, { type ContextObject } from '../context-builder'; import InMemoryDmlHandler from '../dml-handler/in-memory-dml-handler'; import IndexerConfig from '../indexer-config'; import { type LocalIndexerConfig } from '../indexer-config/indexer-config'; @@ -27,6 +27,10 @@ export default class LocalIndexer { this.lakeClient = new LakeClient(); } + getContext (): ContextObject { + return this.indexer.deps.contextBuilder.buildContext(0, []); + } + async executeOnBlock (blockHeight: number): Promise { // TODO: Cache Block data locally const block = await this.lakeClient.fetchBlock(blockHeight); From f190d5602bd388f8d115baed7029e84463ef4d05 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Wed, 7 Aug 2024 17:24:10 -0700 Subject: [PATCH 07/17] Add simple unit test for bitmap indexer --- core-indexers/receiver-blocks/unit.test.ts | 37 +++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/core-indexers/receiver-blocks/unit.test.ts b/core-indexers/receiver-blocks/unit.test.ts index 0ed8140a..0a11bcd6 100644 --- a/core-indexers/receiver-blocks/unit.test.ts +++ b/core-indexers/receiver-blocks/unit.test.ts @@ -17,7 +17,42 @@ describe('Receiver Blocks Indexer Tests', () => { const localIndexer = new LocalIndexer(indexerConfig); const context = localIndexer.getContext(); + // Run on one block to populate receivers table and initial bitmap await localIndexer.executeOnBlock(100000000); - const receivers = context.db.Receivers.select({}) + const receivers = await context.db.Receivers.select({ + receiver: 'app.nearcrowd.near' + }); + const tokenSweatId = receivers[0].id; + + const correctBitmapOne = { + first_block_height: 100000000, + block_date: '2023-08-30', + receiver_id: tokenSweatId, + bitmap: 'wA==', + last_elias_gamma_start_bit: 1, + max_index: 0, + }; + const correctBitmapTwo = { + first_block_height: 100000000, + block_date: '2023-08-30', + receiver_id: tokenSweatId, + bitmap: 'oA==', + last_elias_gamma_start_bit: 1, + max_index: 1, + }; + + let bitmap = await context.db.Bitmaps.select({ + receiver_id: tokenSweatId + }); + expect(bitmap.length).toBe(1); + expect(bitmap[0]).toEqual(correctBitmapOne); + + // Run on second block and verify bitmap update + await localIndexer.executeOnBlock(100000001); + bitmap = await context.db.Bitmaps.select({ + receiver_id: tokenSweatId + }); + expect(bitmap.length).toBe(1); + expect(bitmap[0]).toEqual(correctBitmapTwo); }); }); From cffbe64d9368b0b9509a310edf959d70e9c01b78 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Wed, 7 Aug 2024 17:26:03 -0700 Subject: [PATCH 08/17] Remove snapshots for previous context file name --- .../__snapshots__/context.test.ts.snap | 158 ------------------ 1 file changed, 158 deletions(-) delete mode 100644 runner/src/context-builder/__snapshots__/context.test.ts.snap diff --git a/runner/src/context-builder/__snapshots__/context.test.ts.snap b/runner/src/context-builder/__snapshots__/context.test.ts.snap deleted file mode 100644 index 05235ed9..00000000 --- a/runner/src/context-builder/__snapshots__/context.test.ts.snap +++ /dev/null @@ -1,158 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`ContextBuilder unit tests Context object social api can fetch from the near social api 1`] = ` -[ - [ - "https://api.near.social/index", - { - "body": "{"action":"post","key":"main","options":{"limit":1,"order":"desc"}}", - "headers": { - "Content-Type": "application/json", - }, - "method": "POST", - }, - ], -] -`; - -exports[`ContextBuilder unit tests ContextBuilder adds CRUD operations for table 1`] = ` -{ - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], -} -`; - -exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 1`] = ` -{ - "Posts": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, -} -`; - -exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 2`] = ` -{ - "Comments": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "PostLikes": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "Posts": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, -} -`; - -exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 3`] = ` -{ - "CommentsTable": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "Posts": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, -} -`; - -exports[`ContextBuilder unit tests ContextBuilder can parse various schemas 4`] = ` -{ - "AnotherTable": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "Comments": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "ComposerQuest": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "ContractorQuest": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "CreatorQuest": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "MyTable1": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "PostLikes": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "Posts": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "ThirdTable": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, - "YetAnotherTable": { - "delete": [Function], - "insert": [Function], - "select": [Function], - "update": [Function], - "upsert": [Function], - }, -} -`; From dd9169af22209152f6e354f6383682a2bee04855 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 8 Aug 2024 14:43:02 -0700 Subject: [PATCH 09/17] Move context builder inside indexer folder --- .DS_Store | Bin 0 -> 8196 bytes runner/.DS_Store | Bin 6148 -> 6148 bytes runner/src/.DS_Store | Bin 0 -> 6148 bytes runner/src/indexer/.DS_Store | Bin 0 -> 6148 bytes .../context-builder.test.ts.snap | 0 .../context-builder/context-builder.test.ts | 6 +++--- .../context-builder/context-builder.ts | 10 +++++----- .../{ => indexer}/context-builder/index.ts | 0 runner/src/indexer/indexer.test.ts | 4 ++-- runner/src/indexer/indexer.ts | 2 +- runner/tests/integration.test.ts | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) create mode 100644 .DS_Store create mode 100644 runner/src/.DS_Store create mode 100644 runner/src/indexer/.DS_Store rename runner/src/{ => indexer}/context-builder/__snapshots__/context-builder.test.ts.snap (100%) rename runner/src/{ => indexer}/context-builder/context-builder.test.ts (98%) rename runner/src/{ => indexer}/context-builder/context-builder.ts (97%) rename runner/src/{ => indexer}/context-builder/index.ts (100%) diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..b761b25a201bc0ede621b6d501b0516c733f8d39 GIT binary patch literal 8196 zcmeHMPfyf96o13tZWahJn(T$7iB}d8C5CueS2%d^!WuoOS=(*JN@;Q1vJwK>vwjD^ zf>%F@-^G*u-kXB`10*KIkT64L-gNqY^WOa4n|7xVk?LmNeWDf-S*R=*n`q7{e4lHr zOynaBR=^YOQkxR;X-Ee(Z8KmPFbo(530|aB|vXjfPmIB3``t+c3sLGERO2cvAW_8F;F2`DG zI4KP$m0MQ%2}Ox@;A~MR)zQ+lh5^GsodG_(&yh>deL<<#MigUkD4(;(5fHGDs{R{S>H;lqK+5v{QdlR0;_#2W(0Y$Ko@>Rvc*NTOC zL9s_|L3!wXX3r?8AmN)jDXi-fq&rIXb4Gr+`(coztE)fN#N^cU%&awQHLbV35Amp% zI%$@4o#AV~dJ)H=d)jy0mw`X(FW!C{N2wD;{!jpc=R@Vqt03~?Q8&&aFA;1{->@21 zqrccXI(o3azHF~OTsvO2k2cm;m+kdO8^^~D>(1SKkGK1~;a(Jf0S_oDe^AoAzCLT2 zr%P5?;r zM^w0%iM>j`3+qLkmB%xs9oX`#c9?awP+)xl`Wz+R=bFI`ex92P9EB3LLEE&288_)U zB)qu~2NHo+2aX#DLw46_^+qIVbZlPU1`{PR>cn&(C4pJcTiVbu&8$ eKL=3lWB`mu~2NHo+6MA*v-f>nTKi8=8a5YtQ#AI88@?Y@N)oFY*u9c&ODi4 T#FB#n2pAa{m^KH9Y+(ifKAI19 diff --git a/runner/src/.DS_Store b/runner/src/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..0457c33dcef9a4635401bd7173ffe3c49ae6457b GIT binary patch literal 6148 zcmeHKOKQVF43*kI1KDJmsj*ULZ8iB4BE0DRk*o&XuF}=|f0xvI^ORCy?HZ ztmk3g!m>m}o8RtTWGNzZxS<>@OwFE~PwXW#3WVbwf3lPB?q}P74wLHZ3FE%WUcRIQ z_l!UE*9pv00V+TRr~nn90(U83y%#oL1TscDC(!<{l&{gYak;PpaPc) zyvBBE{eKO=G5=qZxT6A8;HebQS^Lp8xKj4k$>prq7Wf8kHAlD^)=ok2b`11(jE%M9 dl@~=_u{EC8#4*t6$U7a#p8?Z_Mg<PVQzlzKC_F;7?I999`TMh++c^ptonSyxhw23;+f3v z{3i_PnR)47_rq=+hw~RGD+Q#06p#W^Knnay0q?!E { diff --git a/runner/src/context-builder/context-builder.ts b/runner/src/indexer/context-builder/context-builder.ts similarity index 97% rename from runner/src/context-builder/context-builder.ts rename to runner/src/indexer/context-builder/context-builder.ts index ca32e2f1..02d78128 100644 --- a/runner/src/context-builder/context-builder.ts +++ b/runner/src/indexer/context-builder/context-builder.ts @@ -1,13 +1,13 @@ import fetch from 'node-fetch'; import { type Response } from 'node-fetch'; import { Parser } from 'node-sql-parser'; -import { type DmlHandlerInterface } from '../dml-handler/dml-handler'; +import { type DmlHandlerInterface } from '../../dml-handler/dml-handler'; import { type TableDefinitionNames } from '../indexer'; -import type IndexerConfig from '../indexer-config/indexer-config'; -import { LogEntry } from '../indexer-meta'; -import { wrapSpan } from '../utility'; +import type IndexerConfig from '../../indexer-config/indexer-config'; +import { LogEntry } from '../../indexer-meta'; +import { wrapSpan } from '../../utility'; import assert from 'assert'; -import logger from '../logger'; +import logger from '../../logger'; import { trace } from '@opentelemetry/api'; export interface ContextObject { diff --git a/runner/src/context-builder/index.ts b/runner/src/indexer/context-builder/index.ts similarity index 100% rename from runner/src/context-builder/index.ts rename to runner/src/indexer/context-builder/index.ts diff --git a/runner/src/indexer/indexer.test.ts b/runner/src/indexer/indexer.test.ts index e104f9de..4cfb1fd8 100644 --- a/runner/src/indexer/indexer.test.ts +++ b/runner/src/indexer/indexer.test.ts @@ -7,8 +7,8 @@ import { LogLevel } from '../indexer-meta/log-entry'; import IndexerConfig from '../indexer-config/indexer-config'; import { IndexerStatus } from '../indexer-meta'; import type IndexerMeta from '../indexer-meta'; -import ContextBuilder from '../context-builder'; -import { type ContextObject } from '../context-builder'; +import ContextBuilder from './context-builder'; +import { type ContextObject } from './context-builder'; describe('Indexer unit tests', () => { const SIMPLE_REDIS_STREAM = 'test:stream'; diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index b7e60102..f4f24fce 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -9,7 +9,7 @@ import LogEntry from '../indexer-meta/log-entry'; import type IndexerConfig from '../indexer-config'; import { IndexerStatus } from '../indexer-meta'; import { type IndexerMetaInterface } from '../indexer-meta/indexer-meta'; -import type ContextBuilder from '../context-builder'; +import type ContextBuilder from './context-builder'; interface Dependencies { contextBuilder: ContextBuilder diff --git a/runner/tests/integration.test.ts b/runner/tests/integration.test.ts index 0790571f..7803fa8d 100644 --- a/runner/tests/integration.test.ts +++ b/runner/tests/integration.test.ts @@ -15,7 +15,7 @@ import { LogLevel } from '../src/indexer-meta/log-entry'; import IndexerConfig from '../src/indexer-config'; import IndexerMeta from '../src/indexer-meta/indexer-meta'; import DmlHandler from '../src/dml-handler/dml-handler'; -import ContextBuilder from '../src/context-builder'; +import ContextBuilder from '../src/indexer/context-builder'; describe('Indexer integration', () => { jest.setTimeout(300_000); From 77718895c874009d76993949ba9794c693315512 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 8 Aug 2024 14:49:23 -0700 Subject: [PATCH 10/17] Move local indexer to under indexer --- .gitignore | 1 + core-indexers/receiver-blocks/unit.test.ts | 2 +- runner/src/{ => indexer}/local-indexer/index.ts | 0 .../src/{ => indexer}/local-indexer/local-indexer.ts | 12 ++++++------ 4 files changed, 8 insertions(+), 7 deletions(-) rename runner/src/{ => indexer}/local-indexer/index.ts (100%) rename runner/src/{ => indexer}/local-indexer/local-indexer.ts (75%) diff --git a/.gitignore b/.gitignore index 826b0b18..d56e830c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ redis/ node_modules/ .vscode/ runner/yarn.lock +**/.DS_Store diff --git a/core-indexers/receiver-blocks/unit.test.ts b/core-indexers/receiver-blocks/unit.test.ts index 0a11bcd6..fdbc98f7 100644 --- a/core-indexers/receiver-blocks/unit.test.ts +++ b/core-indexers/receiver-blocks/unit.test.ts @@ -1,5 +1,5 @@ import fs from 'fs'; -import LocalIndexer from 'queryapi-runner/src/local-indexer'; +import LocalIndexer from 'queryapi-runner/src/indexer/local-indexer'; import { LocalIndexerConfig } from 'queryapi-runner/src/indexer-config/indexer-config'; import { LogLevel } from 'queryapi-runner/src/indexer-meta/log-entry'; import path from 'path'; diff --git a/runner/src/local-indexer/index.ts b/runner/src/indexer/local-indexer/index.ts similarity index 100% rename from runner/src/local-indexer/index.ts rename to runner/src/indexer/local-indexer/index.ts diff --git a/runner/src/local-indexer/local-indexer.ts b/runner/src/indexer/local-indexer/local-indexer.ts similarity index 75% rename from runner/src/local-indexer/local-indexer.ts rename to runner/src/indexer/local-indexer/local-indexer.ts index 689d5c24..41fbd36d 100644 --- a/runner/src/local-indexer/local-indexer.ts +++ b/runner/src/indexer/local-indexer/local-indexer.ts @@ -1,10 +1,10 @@ import ContextBuilder, { type ContextObject } from '../context-builder'; -import InMemoryDmlHandler from '../dml-handler/in-memory-dml-handler'; -import IndexerConfig from '../indexer-config'; -import { type LocalIndexerConfig } from '../indexer-config/indexer-config'; -import NoOpIndexerMeta from '../indexer-meta/no-op-indexer-meta'; -import Indexer from '../indexer/indexer'; -import LakeClient from '../lake-client/lake-client'; +import InMemoryDmlHandler from '../../dml-handler/in-memory-dml-handler'; +import IndexerConfig from '../../indexer-config'; +import { type LocalIndexerConfig } from '../../indexer-config/indexer-config'; +import NoOpIndexerMeta from '../../indexer-meta/no-op-indexer-meta'; +import Indexer from '../../indexer/indexer'; +import LakeClient from '../../lake-client/lake-client'; export default class LocalIndexer { public readonly indexer: Indexer; From dbc32469206a1506c0862c3459c933fc38c9244e Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 8 Aug 2024 15:39:25 -0700 Subject: [PATCH 11/17] Move dml handler under context and use absolute paths --- .gitignore | 1 + runner/jest.config.js | 3 ++- runner/src/.DS_Store | Bin 6148 -> 0 bytes runner/src/dml-handler/index.ts | 1 - runner/src/indexer/.DS_Store | Bin 6148 -> 0 bytes .../context-builder/context-builder.test.ts | 4 ++-- .../context-builder/context-builder.ts | 8 ++++---- .../dml-handler/dml-handler.test.ts | 10 +++++----- .../dml-handler/dml-handler.ts | 8 ++++---- .../dml-handler/in-memory-dml-handler.test.ts | 2 +- .../dml-handler/in-memory-dml-handler.ts | 2 +- .../context-builder/dml-handler/index.ts | 3 +++ runner/src/indexer/indexer.test.ts | 2 +- .../indexer/local-indexer/local-indexer.ts | 2 +- runner/src/stream-handler/worker.ts | 4 ++-- runner/tests/integration.test.ts | 18 +++++++++--------- runner/tsconfig.json | 1 + 17 files changed, 37 insertions(+), 32 deletions(-) delete mode 100644 runner/src/.DS_Store delete mode 100644 runner/src/dml-handler/index.ts delete mode 100644 runner/src/indexer/.DS_Store rename runner/src/{ => indexer/context-builder}/dml-handler/dml-handler.test.ts (96%) rename runner/src/{ => indexer/context-builder}/dml-handler/dml-handler.ts (96%) rename runner/src/{ => indexer/context-builder}/dml-handler/in-memory-dml-handler.test.ts (99%) rename runner/src/{ => indexer/context-builder}/dml-handler/in-memory-dml-handler.ts (99%) create mode 100644 runner/src/indexer/context-builder/dml-handler/index.ts diff --git a/.gitignore b/.gitignore index d56e830c..08f084ab 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ node_modules/ .vscode/ runner/yarn.lock **/.DS_Store + diff --git a/runner/jest.config.js b/runner/jest.config.js index 1814e47c..40ad52d9 100644 --- a/runner/jest.config.js +++ b/runner/jest.config.js @@ -1,5 +1,6 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', - roots: ['./src', './tests'], + roots: ['./'], + modulePaths: [''], }; diff --git a/runner/src/.DS_Store b/runner/src/.DS_Store deleted file mode 100644 index 0457c33dcef9a4635401bd7173ffe3c49ae6457b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOKQVF43*kI1KDJmsj*ULZ8iB4BE0DRk*o&XuF}=|f0xvI^ORCy?HZ ztmk3g!m>m}o8RtTWGNzZxS<>@OwFE~PwXW#3WVbwf3lPB?q}P74wLHZ3FE%WUcRIQ z_l!UE*9pv00V+TRr~nn90(U83y%#oL1TscDC(!<{l&{gYak;PpaPc) zyvBBE{eKO=G5=qZxT6A8;HebQS^Lp8xKj4k$>prq7Wf8kHAlD^)=ok2b`11(jE%M9 dl@~=_u{EC8#4*t6$U7a#p8?Z_Mg<PVQzlzKC_F;7?I999`TMh++c^ptonSyxhw23;+f3v z{3i_PnR)47_rq=+hw~RGD+Q#06p#W^Knnay0q?!E { const getDbConnectionParameters: PostgresConnectionParams = { diff --git a/runner/src/dml-handler/dml-handler.ts b/runner/src/indexer/context-builder/dml-handler/dml-handler.ts similarity index 96% rename from runner/src/dml-handler/dml-handler.ts rename to runner/src/indexer/context-builder/dml-handler/dml-handler.ts index 29feb7d3..36ccbe9e 100644 --- a/runner/src/dml-handler/dml-handler.ts +++ b/runner/src/indexer/context-builder/dml-handler/dml-handler.ts @@ -1,7 +1,7 @@ -import { wrapError } from '../utility'; -import PgClient, { type PostgresConnectionParams } from '../pg-client'; -import { type TableDefinitionNames } from '../indexer'; -import type IndexerConfig from '../indexer-config/indexer-config'; +import { wrapError } from 'src/utility'; +import PgClient, { type PostgresConnectionParams } from 'src/pg-client'; +import { type TableDefinitionNames } from 'src/indexer'; +import type IndexerConfig from 'src/indexer-config/indexer-config'; import { type Tracer, trace, type Span } from '@opentelemetry/api'; import { type QueryResult } from 'pg'; diff --git a/runner/src/dml-handler/in-memory-dml-handler.test.ts b/runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.test.ts similarity index 99% rename from runner/src/dml-handler/in-memory-dml-handler.test.ts rename to runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.test.ts index 6fbc20c7..db631955 100644 --- a/runner/src/dml-handler/in-memory-dml-handler.test.ts +++ b/runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.test.ts @@ -1,4 +1,4 @@ -import { type TableDefinitionNames } from '../indexer'; +import { type TableDefinitionNames } from 'src/indexer'; import InMemoryDmlHandler from './in-memory-dml-handler'; const DEFAULT_ITEM_1_WITHOUT_ID = { diff --git a/runner/src/dml-handler/in-memory-dml-handler.ts b/runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.ts similarity index 99% rename from runner/src/dml-handler/in-memory-dml-handler.ts rename to runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.ts index d6fa9f1d..6a6d3509 100644 --- a/runner/src/dml-handler/in-memory-dml-handler.ts +++ b/runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.ts @@ -1,5 +1,5 @@ import { type AST, Parser } from 'node-sql-parser'; -import { type TableDefinitionNames } from '../indexer'; +import { type TableDefinitionNames } from 'src/indexer'; import { type PostgresRow, type WhereClauseMulti, type WhereClauseSingle, type DmlHandlerInterface } from './dml-handler'; // TODO: Define class to represent specification diff --git a/runner/src/indexer/context-builder/dml-handler/index.ts b/runner/src/indexer/context-builder/dml-handler/index.ts new file mode 100644 index 00000000..7bdf14d6 --- /dev/null +++ b/runner/src/indexer/context-builder/dml-handler/index.ts @@ -0,0 +1,3 @@ +export { default as DmlHandler } from './dml-handler'; +export { default as InMemoryDmlHandler } from './in-memory-dml-handler'; +export type { DmlHandlerInterface } from './dml-handler'; diff --git a/runner/src/indexer/indexer.test.ts b/runner/src/indexer/indexer.test.ts index 4cfb1fd8..053cd8c3 100644 --- a/runner/src/indexer/indexer.test.ts +++ b/runner/src/indexer/indexer.test.ts @@ -2,7 +2,7 @@ import { Block, type StreamerMessage } from '@near-lake/primitives'; import Indexer from './indexer'; import { VM } from 'vm2'; -import type DmlHandler from '../dml-handler/dml-handler'; +import type { DmlHandler } from './context-builder/dml-handler'; import { LogLevel } from '../indexer-meta/log-entry'; import IndexerConfig from '../indexer-config/indexer-config'; import { IndexerStatus } from '../indexer-meta'; diff --git a/runner/src/indexer/local-indexer/local-indexer.ts b/runner/src/indexer/local-indexer/local-indexer.ts index 41fbd36d..709587ef 100644 --- a/runner/src/indexer/local-indexer/local-indexer.ts +++ b/runner/src/indexer/local-indexer/local-indexer.ts @@ -1,5 +1,5 @@ import ContextBuilder, { type ContextObject } from '../context-builder'; -import InMemoryDmlHandler from '../../dml-handler/in-memory-dml-handler'; +import InMemoryDmlHandler from '../context-builder/dml-handler/in-memory-dml-handler'; import IndexerConfig from '../../indexer-config'; import { type LocalIndexerConfig } from '../../indexer-config/indexer-config'; import NoOpIndexerMeta from '../../indexer-meta/no-op-indexer-meta'; diff --git a/runner/src/stream-handler/worker.ts b/runner/src/stream-handler/worker.ts index cbd241b1..837b73f2 100644 --- a/runner/src/stream-handler/worker.ts +++ b/runner/src/stream-handler/worker.ts @@ -14,8 +14,8 @@ import IndexerConfig from '../indexer-config'; import parentLogger from '../logger'; import { wrapSpan } from '../utility'; import { type PostgresConnectionParams } from '../pg-client'; -import DmlHandler from '../dml-handler/dml-handler'; -import ContextBuilder from '../context-builder'; +import DmlHandler from 'src/indexer/context-builder/dml-handler/dml-handler'; +import ContextBuilder from 'src/indexer/context-builder'; if (isMainThread) { throw new Error('Worker should not be run on main thread'); diff --git a/runner/tests/integration.test.ts b/runner/tests/integration.test.ts index 7803fa8d..2c1e6a3e 100644 --- a/runner/tests/integration.test.ts +++ b/runner/tests/integration.test.ts @@ -2,20 +2,20 @@ import { Block, type StreamerMessage } from '@near-lake/primitives'; import { Network, type StartedNetwork } from 'testcontainers'; import { gql, GraphQLClient } from 'graphql-request'; -import Indexer from '../src/indexer'; -import HasuraClient from '../src/hasura-client'; -import Provisioner from '../src/provisioner'; -import PgClient from '../src/pg-client'; +import Indexer from 'src/indexer'; +import HasuraClient from 'src/hasura-client'; +import Provisioner from 'src/provisioner'; +import PgClient from 'src/pg-client'; import { HasuraGraphQLContainer, type StartedHasuraGraphQLContainer } from './testcontainers/hasura'; import { PostgreSqlContainer, type StartedPostgreSqlContainer } from './testcontainers/postgres'; import block_115185108 from './blocks/00115185108/streamer_message.json'; import block_115185109 from './blocks/00115185109/streamer_message.json'; -import { LogLevel } from '../src/indexer-meta/log-entry'; -import IndexerConfig from '../src/indexer-config'; -import IndexerMeta from '../src/indexer-meta/indexer-meta'; -import DmlHandler from '../src/dml-handler/dml-handler'; -import ContextBuilder from '../src/indexer/context-builder'; +import { LogLevel } from 'src/indexer-meta/log-entry'; +import IndexerConfig from 'src/indexer-config'; +import IndexerMeta from 'src/indexer-meta/indexer-meta'; +import DmlHandler from 'src/indexer/context-builder/dml-handler/dml-handler'; +import ContextBuilder from 'src/indexer/context-builder'; describe('Indexer integration', () => { jest.setTimeout(300_000); diff --git a/runner/tsconfig.json b/runner/tsconfig.json index d4949c0e..803301ea 100644 --- a/runner/tsconfig.json +++ b/runner/tsconfig.json @@ -3,6 +3,7 @@ "target": "es2018", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ "lib": ["es2021"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ "module": "commonjs", /* Specify what module code is generated. */ + "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ "rootDirs": ["./src", "./tests", "./scripts"], "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ "resolveJsonModule": true, /* Enable importing .json files. */ From b4c0000163099ab16006f2933519521c3320d817 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 8 Aug 2024 16:16:44 -0700 Subject: [PATCH 12/17] Support absolute import usage from core-indexers --- core-indexers/jest.config.js | 6 +++++- core-indexers/package.json | 3 --- core-indexers/tsconfig.json | 6 +++++- runner/src/indexer-config/index.ts | 1 + runner/src/indexer/local-indexer/local-indexer.ts | 10 +++++----- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/core-indexers/jest.config.js b/core-indexers/jest.config.js index 1e525917..6f85189b 100644 --- a/core-indexers/jest.config.js +++ b/core-indexers/jest.config.js @@ -2,5 +2,9 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', testMatch: ['**/*.test.ts'], - testTimeout: 10000 + testTimeout: 10000, + moduleNameMapper: { + '^runner/(.*)$': '/../runner/$1', // Ensure tests can find runner imports + '^src/(.*)$': '/../runner/src/$1' // Ensure tests can find runner absolute imports + } }; diff --git a/core-indexers/package.json b/core-indexers/package.json index c79a6426..ce9f8086 100644 --- a/core-indexers/package.json +++ b/core-indexers/package.json @@ -7,9 +7,6 @@ }, "author": "", "license": "ISC", - "dependencies": { - "queryapi-runner": "file:../runner" - }, "devDependencies": { "@eslint/eslintrc": "^3.1.0", "@eslint/js": "^9.7.0", diff --git a/core-indexers/tsconfig.json b/core-indexers/tsconfig.json index 9555e472..340d8c59 100644 --- a/core-indexers/tsconfig.json +++ b/core-indexers/tsconfig.json @@ -3,7 +3,11 @@ "target": "es2018", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ "lib": ["es2021"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ "module": "commonjs", /* Specify what module code is generated. */ - "rootDir": "..", + "rootDir": "../", + "paths": { + "@queryapi-runner/*": ["../runner/*"], /* Allow imports from runner using queryapi-runner alias */ + "src/*": ["../runner/src/*"] /* Allow absolute imports in runner */ + }, "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ "resolveJsonModule": true, /* Enable importing .json files. */ "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ diff --git a/runner/src/indexer-config/index.ts b/runner/src/indexer-config/index.ts index 1131822d..a0beb66d 100644 --- a/runner/src/indexer-config/index.ts +++ b/runner/src/indexer-config/index.ts @@ -1 +1,2 @@ export { default } from './indexer-config'; +export { ProvisioningConfig, LocalIndexerConfig } from './indexer-config'; diff --git a/runner/src/indexer/local-indexer/local-indexer.ts b/runner/src/indexer/local-indexer/local-indexer.ts index 709587ef..ddeada94 100644 --- a/runner/src/indexer/local-indexer/local-indexer.ts +++ b/runner/src/indexer/local-indexer/local-indexer.ts @@ -1,10 +1,10 @@ import ContextBuilder, { type ContextObject } from '../context-builder'; import InMemoryDmlHandler from '../context-builder/dml-handler/in-memory-dml-handler'; -import IndexerConfig from '../../indexer-config'; -import { type LocalIndexerConfig } from '../../indexer-config/indexer-config'; -import NoOpIndexerMeta from '../../indexer-meta/no-op-indexer-meta'; -import Indexer from '../../indexer/indexer'; -import LakeClient from '../../lake-client/lake-client'; +import IndexerConfig from 'src/indexer-config'; +import { type LocalIndexerConfig } from 'src/indexer-config'; +import NoOpIndexerMeta from 'src/indexer-meta/no-op-indexer-meta'; +import Indexer from 'src/indexer'; +import LakeClient from 'src/lake-client/lake-client'; export default class LocalIndexer { public readonly indexer: Indexer; From 578b3a9aedcbcd4d7527eddc71beacfcb379a145 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 8 Aug 2024 16:44:42 -0700 Subject: [PATCH 13/17] Remove absolute imports and push dml hander up one level --- core-indexers/jest.config.js | 6 +----- core-indexers/tsconfig.json | 1 - .../context-builder/context-builder.test.ts | 4 ++-- .../indexer/context-builder/context-builder.ts | 8 ++++---- .../dml-handler/dml-handler.test.ts | 10 +++++----- .../dml-handler/dml-handler.ts | 8 ++++---- .../dml-handler/in-memory-dml-handler.test.ts | 2 +- .../dml-handler/in-memory-dml-handler.ts | 2 +- .../{context-builder => }/dml-handler/index.ts | 0 runner/src/indexer/index.ts | 3 ++- runner/src/indexer/indexer.test.ts | 2 +- .../{local-indexer => }/local-indexer.ts | 14 +++++++------- runner/src/indexer/local-indexer/index.ts | 1 - runner/src/stream-handler/worker.ts | 6 +++--- runner/tests/integration.test.ts | 18 +++++++++--------- runner/tsconfig.json | 1 - 16 files changed, 40 insertions(+), 46 deletions(-) rename runner/src/indexer/{context-builder => }/dml-handler/dml-handler.test.ts (96%) rename runner/src/indexer/{context-builder => }/dml-handler/dml-handler.ts (96%) rename runner/src/indexer/{context-builder => }/dml-handler/in-memory-dml-handler.test.ts (99%) rename runner/src/indexer/{context-builder => }/dml-handler/in-memory-dml-handler.ts (99%) rename runner/src/indexer/{context-builder => }/dml-handler/index.ts (100%) rename runner/src/indexer/{local-indexer => }/local-indexer.ts (72%) delete mode 100644 runner/src/indexer/local-indexer/index.ts diff --git a/core-indexers/jest.config.js b/core-indexers/jest.config.js index 6f85189b..1e525917 100644 --- a/core-indexers/jest.config.js +++ b/core-indexers/jest.config.js @@ -2,9 +2,5 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', testMatch: ['**/*.test.ts'], - testTimeout: 10000, - moduleNameMapper: { - '^runner/(.*)$': '/../runner/$1', // Ensure tests can find runner imports - '^src/(.*)$': '/../runner/src/$1' // Ensure tests can find runner absolute imports - } + testTimeout: 10000 }; diff --git a/core-indexers/tsconfig.json b/core-indexers/tsconfig.json index 340d8c59..7cbb4fad 100644 --- a/core-indexers/tsconfig.json +++ b/core-indexers/tsconfig.json @@ -6,7 +6,6 @@ "rootDir": "../", "paths": { "@queryapi-runner/*": ["../runner/*"], /* Allow imports from runner using queryapi-runner alias */ - "src/*": ["../runner/src/*"] /* Allow absolute imports in runner */ }, "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ "resolveJsonModule": true, /* Enable importing .json files. */ diff --git a/runner/src/indexer/context-builder/context-builder.test.ts b/runner/src/indexer/context-builder/context-builder.test.ts index 33effbb8..f6809366 100644 --- a/runner/src/indexer/context-builder/context-builder.test.ts +++ b/runner/src/indexer/context-builder/context-builder.test.ts @@ -1,7 +1,7 @@ import type fetch from 'node-fetch'; -import type { DmlHandler } from './dml-handler'; -import IndexerConfig from 'src/indexer-config'; +import type { DmlHandler } from '../dml-handler'; +import IndexerConfig from '../../indexer-config'; import { LogLevel } from '../../indexer-meta/log-entry'; import ContextBuilder from './context-builder'; diff --git a/runner/src/indexer/context-builder/context-builder.ts b/runner/src/indexer/context-builder/context-builder.ts index c23094c4..0f1507f6 100644 --- a/runner/src/indexer/context-builder/context-builder.ts +++ b/runner/src/indexer/context-builder/context-builder.ts @@ -1,11 +1,11 @@ import fetch from 'node-fetch'; import { type Response } from 'node-fetch'; import { Parser } from 'node-sql-parser'; -import { type DmlHandlerInterface } from './dml-handler/dml-handler'; +import { type DmlHandlerInterface } from '../dml-handler'; import { type TableDefinitionNames } from '../indexer'; -import type IndexerConfig from 'src/indexer-config/indexer-config'; -import { LogEntry } from 'src/indexer-meta'; -import { wrapSpan } from 'src/utility'; +import type IndexerConfig from '../../indexer-config'; +import { LogEntry } from '../../indexer-meta'; +import { wrapSpan } from '../../utility'; import assert from 'assert'; import logger from '../../logger'; import { trace } from '@opentelemetry/api'; diff --git a/runner/src/indexer/context-builder/dml-handler/dml-handler.test.ts b/runner/src/indexer/dml-handler/dml-handler.test.ts similarity index 96% rename from runner/src/indexer/context-builder/dml-handler/dml-handler.test.ts rename to runner/src/indexer/dml-handler/dml-handler.test.ts index ae613636..8f52d6a1 100644 --- a/runner/src/indexer/context-builder/dml-handler/dml-handler.test.ts +++ b/runner/src/indexer/dml-handler/dml-handler.test.ts @@ -1,10 +1,10 @@ import pgFormat from 'pg-format'; import DmlHandler from './dml-handler'; -import type PgClient from 'src/pg-client'; -import { type PostgresConnectionParams } from 'src/pg-client'; -import { type TableDefinitionNames } from 'src/indexer'; -import IndexerConfig from 'src/indexer-config'; -import { LogLevel } from 'src/indexer-meta/log-entry'; +import type PgClient from '../../pg-client'; +import { type PostgresConnectionParams } from '../../pg-client'; +import { type TableDefinitionNames } from '../../indexer'; +import IndexerConfig from '../../indexer-config'; +import { LogLevel } from '../../indexer-meta/log-entry'; describe('DML Handler tests', () => { const getDbConnectionParameters: PostgresConnectionParams = { diff --git a/runner/src/indexer/context-builder/dml-handler/dml-handler.ts b/runner/src/indexer/dml-handler/dml-handler.ts similarity index 96% rename from runner/src/indexer/context-builder/dml-handler/dml-handler.ts rename to runner/src/indexer/dml-handler/dml-handler.ts index 36ccbe9e..05b36d68 100644 --- a/runner/src/indexer/context-builder/dml-handler/dml-handler.ts +++ b/runner/src/indexer/dml-handler/dml-handler.ts @@ -1,7 +1,7 @@ -import { wrapError } from 'src/utility'; -import PgClient, { type PostgresConnectionParams } from 'src/pg-client'; -import { type TableDefinitionNames } from 'src/indexer'; -import type IndexerConfig from 'src/indexer-config/indexer-config'; +import { wrapError } from '../../utility'; +import PgClient, { type PostgresConnectionParams } from '../../pg-client'; +import { type TableDefinitionNames } from '../../indexer'; +import type IndexerConfig from '../../indexer-config/indexer-config'; import { type Tracer, trace, type Span } from '@opentelemetry/api'; import { type QueryResult } from 'pg'; diff --git a/runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.test.ts b/runner/src/indexer/dml-handler/in-memory-dml-handler.test.ts similarity index 99% rename from runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.test.ts rename to runner/src/indexer/dml-handler/in-memory-dml-handler.test.ts index db631955..b8f94675 100644 --- a/runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.test.ts +++ b/runner/src/indexer/dml-handler/in-memory-dml-handler.test.ts @@ -1,4 +1,4 @@ -import { type TableDefinitionNames } from 'src/indexer'; +import { type TableDefinitionNames } from '../../indexer'; import InMemoryDmlHandler from './in-memory-dml-handler'; const DEFAULT_ITEM_1_WITHOUT_ID = { diff --git a/runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.ts b/runner/src/indexer/dml-handler/in-memory-dml-handler.ts similarity index 99% rename from runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.ts rename to runner/src/indexer/dml-handler/in-memory-dml-handler.ts index 6a6d3509..450ff010 100644 --- a/runner/src/indexer/context-builder/dml-handler/in-memory-dml-handler.ts +++ b/runner/src/indexer/dml-handler/in-memory-dml-handler.ts @@ -1,5 +1,5 @@ import { type AST, Parser } from 'node-sql-parser'; -import { type TableDefinitionNames } from 'src/indexer'; +import { type TableDefinitionNames } from '../../indexer'; import { type PostgresRow, type WhereClauseMulti, type WhereClauseSingle, type DmlHandlerInterface } from './dml-handler'; // TODO: Define class to represent specification diff --git a/runner/src/indexer/context-builder/dml-handler/index.ts b/runner/src/indexer/dml-handler/index.ts similarity index 100% rename from runner/src/indexer/context-builder/dml-handler/index.ts rename to runner/src/indexer/dml-handler/index.ts diff --git a/runner/src/indexer/index.ts b/runner/src/indexer/index.ts index f4d7192a..7c066fb5 100644 --- a/runner/src/indexer/index.ts +++ b/runner/src/indexer/index.ts @@ -1,2 +1,3 @@ -export { default } from './indexer'; +export { default as Indexer } from './indexer'; +export { default as LocalIndexer } from './local-indexer'; export type { TableDefinitionNames } from './indexer'; diff --git a/runner/src/indexer/indexer.test.ts b/runner/src/indexer/indexer.test.ts index 053cd8c3..4dd36620 100644 --- a/runner/src/indexer/indexer.test.ts +++ b/runner/src/indexer/indexer.test.ts @@ -2,7 +2,7 @@ import { Block, type StreamerMessage } from '@near-lake/primitives'; import Indexer from './indexer'; import { VM } from 'vm2'; -import type { DmlHandler } from './context-builder/dml-handler'; +import type { DmlHandler } from './dml-handler'; import { LogLevel } from '../indexer-meta/log-entry'; import IndexerConfig from '../indexer-config/indexer-config'; import { IndexerStatus } from '../indexer-meta'; diff --git a/runner/src/indexer/local-indexer/local-indexer.ts b/runner/src/indexer/local-indexer.ts similarity index 72% rename from runner/src/indexer/local-indexer/local-indexer.ts rename to runner/src/indexer/local-indexer.ts index ddeada94..9baeaa14 100644 --- a/runner/src/indexer/local-indexer/local-indexer.ts +++ b/runner/src/indexer/local-indexer.ts @@ -1,10 +1,10 @@ -import ContextBuilder, { type ContextObject } from '../context-builder'; -import InMemoryDmlHandler from '../context-builder/dml-handler/in-memory-dml-handler'; -import IndexerConfig from 'src/indexer-config'; -import { type LocalIndexerConfig } from 'src/indexer-config'; -import NoOpIndexerMeta from 'src/indexer-meta/no-op-indexer-meta'; -import Indexer from 'src/indexer'; -import LakeClient from 'src/lake-client/lake-client'; +import ContextBuilder, { type ContextObject } from './context-builder'; +import InMemoryDmlHandler from './dml-handler/in-memory-dml-handler'; +import IndexerConfig from '../indexer-config'; +import { type LocalIndexerConfig } from '../indexer-config'; +import NoOpIndexerMeta from '../indexer-meta/no-op-indexer-meta'; +import Indexer from './indexer'; +import LakeClient from '../lake-client/lake-client'; export default class LocalIndexer { public readonly indexer: Indexer; diff --git a/runner/src/indexer/local-indexer/index.ts b/runner/src/indexer/local-indexer/index.ts deleted file mode 100644 index b0546169..00000000 --- a/runner/src/indexer/local-indexer/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from './local-indexer'; diff --git a/runner/src/stream-handler/worker.ts b/runner/src/stream-handler/worker.ts index 837b73f2..3ca3e5c9 100644 --- a/runner/src/stream-handler/worker.ts +++ b/runner/src/stream-handler/worker.ts @@ -3,7 +3,7 @@ import { trace, type Span, context } from '@opentelemetry/api'; import promClient from 'prom-client'; import { Block } from '@near-lake/primitives'; -import Indexer from '../indexer'; +import { Indexer } from '../indexer'; import RedisClient from '../redis-client'; import { METRICS } from '../metrics'; import LakeClient from '../lake-client'; @@ -14,8 +14,8 @@ import IndexerConfig from '../indexer-config'; import parentLogger from '../logger'; import { wrapSpan } from '../utility'; import { type PostgresConnectionParams } from '../pg-client'; -import DmlHandler from 'src/indexer/context-builder/dml-handler/dml-handler'; -import ContextBuilder from 'src/indexer/context-builder'; +import { DmlHandler } from '../indexer/dml-handler'; +import ContextBuilder from '../indexer/context-builder'; if (isMainThread) { throw new Error('Worker should not be run on main thread'); diff --git a/runner/tests/integration.test.ts b/runner/tests/integration.test.ts index 2c1e6a3e..1fd71d9c 100644 --- a/runner/tests/integration.test.ts +++ b/runner/tests/integration.test.ts @@ -2,20 +2,20 @@ import { Block, type StreamerMessage } from '@near-lake/primitives'; import { Network, type StartedNetwork } from 'testcontainers'; import { gql, GraphQLClient } from 'graphql-request'; -import Indexer from 'src/indexer'; -import HasuraClient from 'src/hasura-client'; -import Provisioner from 'src/provisioner'; -import PgClient from 'src/pg-client'; +import { Indexer } from '../src/indexer'; +import HasuraClient from '../src/hasura-client'; +import Provisioner from '../src/provisioner'; +import PgClient from '../src/pg-client'; import { HasuraGraphQLContainer, type StartedHasuraGraphQLContainer } from './testcontainers/hasura'; import { PostgreSqlContainer, type StartedPostgreSqlContainer } from './testcontainers/postgres'; import block_115185108 from './blocks/00115185108/streamer_message.json'; import block_115185109 from './blocks/00115185109/streamer_message.json'; -import { LogLevel } from 'src/indexer-meta/log-entry'; -import IndexerConfig from 'src/indexer-config'; -import IndexerMeta from 'src/indexer-meta/indexer-meta'; -import DmlHandler from 'src/indexer/context-builder/dml-handler/dml-handler'; -import ContextBuilder from 'src/indexer/context-builder'; +import { LogLevel } from '../src/indexer-meta/log-entry'; +import IndexerConfig from '../src/indexer-config'; +import IndexerMeta from '../src/indexer-meta/indexer-meta'; +import { DmlHandler } from '../src/indexer/dml-handler'; +import ContextBuilder from '../src/indexer/context-builder'; describe('Indexer integration', () => { jest.setTimeout(300_000); diff --git a/runner/tsconfig.json b/runner/tsconfig.json index 803301ea..d4949c0e 100644 --- a/runner/tsconfig.json +++ b/runner/tsconfig.json @@ -3,7 +3,6 @@ "target": "es2018", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ "lib": ["es2021"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ "module": "commonjs", /* Specify what module code is generated. */ - "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ "rootDirs": ["./src", "./tests", "./scripts"], "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ "resolveJsonModule": true, /* Enable importing .json files. */ From 20e841105614e3f7538f661d51ce3431ca723c44 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 8 Aug 2024 16:47:47 -0700 Subject: [PATCH 14/17] Undo changes to jest --- runner/jest.config.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runner/jest.config.js b/runner/jest.config.js index 40ad52d9..1814e47c 100644 --- a/runner/jest.config.js +++ b/runner/jest.config.js @@ -1,6 +1,5 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', - roots: ['./'], - modulePaths: [''], + roots: ['./src', './tests'], }; From 83d15194303d9e5aa91e8ddf775323b9da4a525f Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Thu, 8 Aug 2024 16:49:32 -0700 Subject: [PATCH 15/17] Remove ds store file --- runner/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 runner/.DS_Store diff --git a/runner/.DS_Store b/runner/.DS_Store deleted file mode 100644 index 8e860eb42c5deca1e40f728ff7e6d135c3a1093f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKu};H44E3d<3S#NVm>*#1AA~A=0O}8*BoYjXE+VjpzhPzI0}$$f_!?Fgc=nmL zX+UfURkq}N7oUA`UQXvbB65q{YECpGq6Eqq9isV0c${@4ky<#;$sC@#`f|Q3ebbJ- z&F~)?;D0xzmafUu4Rq)K9X#!hH$Kmc+83}VuP<*OcMq%6u}}4@PkXh!PI&#%ggWF= zR$W;a-q%=CU#7vLitbM~Bire$9;fH;&ytQ#)%Z;!2`bfHAQ54DbdE zNsa#l&u4eTmgVF%p&N^K?<0%0hlTF0bzl-5DE;TPOlg) zgu|YyU#8dx4B_PD&6r2u?DU4>^cd$-aVM7rwAL6f28Ikw!sCqV|Jl#?|6!0l83V?^ zUNPX}e3>usNm^S+AIG&eg Date: Thu, 8 Aug 2024 16:51:40 -0700 Subject: [PATCH 16/17] Remove other ds store file --- .DS_Store | Bin 8196 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index b761b25a201bc0ede621b6d501b0516c733f8d39..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMPfyf96o13tZWahJn(T$7iB}d8C5CueS2%d^!WuoOS=(*JN@;Q1vJwK>vwjD^ zf>%F@-^G*u-kXB`10*KIkT64L-gNqY^WOa4n|7xVk?LmNeWDf-S*R=*n`q7{e4lHr zOynaBR=^YOQkxR;X-Ee(Z8KmPFbo(530|aB|vXjfPmIB3``t+c3sLGERO2cvAW_8F;F2`DG zI4KP$m0MQ%2}Ox@;A~MR)zQ+lh5^GsodG_(&yh>deL<<#MigUkD4(;(5fHGDs{R{S>H;lqK+5v{QdlR0;_#2W(0Y$Ko@>Rvc*NTOC zL9s_|L3!wXX3r?8AmN)jDXi-fq&rIXb4Gr+`(coztE)fN#N^cU%&awQHLbV35Amp% zI%$@4o#AV~dJ)H=d)jy0mw`X(FW!C{N2wD;{!jpc=R@Vqt03~?Q8&&aFA;1{->@21 zqrccXI(o3azHF~OTsvO2k2cm;m+kdO8^^~D>(1SKkGK1~;a(Jf0S_oDe^AoAzCLT2 zr%P5?;r zM^w0%iM>j`3+qLkmB%xs9oX`#c9?awP+)xl`Wz+R=bFI`ex92P9EB3LLEE&288_)U z Date: Thu, 8 Aug 2024 16:52:23 -0700 Subject: [PATCH 17/17] Dont change root --- core-indexers/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-indexers/tsconfig.json b/core-indexers/tsconfig.json index 7cbb4fad..0cb90e9d 100644 --- a/core-indexers/tsconfig.json +++ b/core-indexers/tsconfig.json @@ -3,7 +3,7 @@ "target": "es2018", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ "lib": ["es2021"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ "module": "commonjs", /* Specify what module code is generated. */ - "rootDir": "../", + "rootDir": "..", "paths": { "@queryapi-runner/*": ["../runner/*"], /* Allow imports from runner using queryapi-runner alias */ },