From c3f24c139a540760c938274124bfc9e991aa4c7d Mon Sep 17 00:00:00 2001 From: Tyler Hall Date: Tue, 15 Oct 2024 19:59:18 +0000 Subject: [PATCH] feat(cu): add flag to disable/enable wasm metering and apply #1040 --- servers/cu/src/bootstrap.js | 2 ++ servers/cu/src/config.js | 2 ++ servers/cu/src/domain/model.js | 4 ++++ servers/cu/src/effects/wasm.js | 11 +++++++---- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/servers/cu/src/bootstrap.js b/servers/cu/src/bootstrap.js index 360a4d669..799936877 100644 --- a/servers/cu/src/bootstrap.js +++ b/servers/cu/src/bootstrap.js @@ -201,6 +201,7 @@ export const createApis = async (ctx) => { ctx.logger('Ignoring Arweave Checkpoints for processes [ %s ]', ctx.PROCESS_IGNORE_ARWEAVE_CHECKPOINTS.join(', ')) ctx.logger('Ignoring Arweave Checkpoints [ %s ]', ctx.IGNORE_ARWEAVE_CHECKPOINTS.join(', ')) ctx.logger('Trusting Arweave Checkpoint Owners [ %s ]', ctx.PROCESS_CHECKPOINT_TRUSTED_OWNERS.join(', ')) + ctx.logger('Process metering applied set to "%s"', ctx.PROCESS_WASM_APPLY_METERING) ctx.logger('Allowing only process owners [ %s ]', ctx.ALLOW_OWNERS.join(', ')) ctx.logger('Restricting processes [ %s ]', ctx.RESTRICT_PROCESSES.join(', ')) ctx.logger('Allowing only processes [ %s ]', ctx.ALLOW_PROCESSES.join(', ')) @@ -241,6 +242,7 @@ export const createApis = async (ctx) => { fetch: ctx.fetch, ARWEAVE_URL: ctx.ARWEAVE_URL, WASM_BINARY_FILE_DIRECTORY: ctx.WASM_BINARY_FILE_DIRECTORY, + PROCESS_WASM_APPLY_METERING: ctx.PROCESS_WASM_APPLY_METERING, logger: ctx.logger, cache: WasmClient.createWasmModuleCache({ MAX_SIZE: ctx.WASM_MODULE_CACHE_MAX_SIZE }) }) diff --git a/servers/cu/src/config.js b/servers/cu/src/config.js index 510e9e063..ccb65c11c 100644 --- a/servers/cu/src/config.js +++ b/servers/cu/src/config.js @@ -121,6 +121,7 @@ const CONFIG_ENVS = { PROCESS_WASM_COMPUTE_MAX_LIMIT: process.env.PROCESS_WASM_COMPUTE_MAX_LIMIT || 9_000_000_000_000, // 9t PROCESS_WASM_SUPPORTED_FORMATS: process.env.PROCESS_WASM_SUPPORTED_FORMATS || DEFAULT_PROCESS_WASM_MODULE_FORMATS, PROCESS_WASM_SUPPORTED_EXTENSIONS: process.env.PROCESS_WASM_SUPPORTED_EXTENSIONS || [], + PROCESS_WASM_APPLY_METERING: process.env.PROCESS_WASM_APPLY_METERING !== 'false', WASM_EVALUATION_MAX_WORKERS: process.env.WASM_EVALUATION_MAX_WORKERS || Math.max(cpus().length - 1, 1), WASM_EVALUATION_PRIMARY_WORKERS_PERCENTAGE: process.env.WASM_EVALUATION_PRIMARY_WORKERS_PERCENTAGE || 70, // 70% of worker threads allocated to primary workloads WASM_EVALUATION_WORKERS_DRY_RUN_MAX_QUEUE: process.env.WASM_EVALUATION_WORKERS_DRY_RUN_MAX_QUEUE || 3000, @@ -171,6 +172,7 @@ const CONFIG_ENVS = { PROCESS_WASM_COMPUTE_MAX_LIMIT: process.env.PROCESS_WASM_COMPUTE_MAX_LIMIT || 9_000_000_000_000, // 9t PROCESS_WASM_SUPPORTED_FORMATS: process.env.PROCESS_WASM_SUPPORTED_FORMATS || DEFAULT_PROCESS_WASM_MODULE_FORMATS, PROCESS_WASM_SUPPORTED_EXTENSIONS: process.env.PROCESS_WASM_SUPPORTED_EXTENSIONS || [], + PROCESS_WASM_APPLY_METERING: process.env.PROCESS_WASM_APPLY_METERING !== 'false', WASM_EVALUATION_MAX_WORKERS: process.env.WASM_EVALUATION_MAX_WORKERS || Math.max(cpus().length - 1, 1), WASM_EVALUATION_PRIMARY_WORKERS_PERCENTAGE: process.env.WASM_EVALUATION_PRIMARY_WORKERS_PERCENTAGE || 70, // 70% of worker threads allocated to primary workloads WASM_EVALUATION_WORKERS_DRY_RUN_MAX_QUEUE: process.env.WASM_EVALUATION_WORKERS_DRY_RUN_MAX_QUEUE || 3000, diff --git a/servers/cu/src/domain/model.js b/servers/cu/src/domain/model.js index 2e3854ecb..60c43d28a 100644 --- a/servers/cu/src/domain/model.js +++ b/servers/cu/src/domain/model.js @@ -42,6 +42,10 @@ export const domainConfigSchema = z.object({ * The wasm extensions that this CU supports */ PROCESS_WASM_SUPPORTED_EXTENSIONS: commaDelimitedArraySchema, + /** + * Whether or not to apply metering to the wasm execution + */ + PROCESS_WASM_APPLY_METERING: z.preprocess((val) => !!val, z.boolean()), /** * The url for the graphql server to be used by the CU * to query for metadata from an Arweave Gateway diff --git a/servers/cu/src/effects/wasm.js b/servers/cu/src/effects/wasm.js index 4cf68ddf6..e1adf3ae9 100644 --- a/servers/cu/src/effects/wasm.js +++ b/servers/cu/src/effects/wasm.js @@ -128,12 +128,15 @@ export function bootstrapWasmInstanceWith () { } } -export function loadWasmModuleWith ({ fetch, ARWEAVE_URL, WASM_BINARY_FILE_DIRECTORY, logger, cache }) { +export function loadWasmModuleWith ({ fetch, ARWEAVE_URL, WASM_BINARY_FILE_DIRECTORY, PROCESS_WASM_APPLY_METERING, logger, cache }) { const streamTransactionData = fromPromise(streamTransactionDataWith({ fetch, ARWEAVE_URL, logger })) const readWasmFile = fromPromise(readWasmFileWith({ DIR: WASM_BINARY_FILE_DIRECTORY })) const writeWasmFile = writeWasmFileWith({ DIR: WASM_BINARY_FILE_DIRECTORY }) - const toWasmResponse = (moduleOptions) => fromPromise((stream) => WebAssembly.compileStreaming(wasmResponse(Readable.toWeb(stream), moduleOptions))) + const compileStreaming = (stream, moduleOptions) => WebAssembly.compileStreaming( + wasmResponse(stream), + { ...moduleOptions, applyMetering: PROCESS_WASM_APPLY_METERING } + ) function maybeCachedModule (args) { const { moduleId } = args @@ -152,7 +155,7 @@ export function loadWasmModuleWith ({ fetch, ARWEAVE_URL, WASM_BINARY_FILE_DIREC return of(moduleId) .chain(readWasmFile) - .chain(toWasmResponse(moduleOptions)) + .chain(fromPromise((nStream) => compileStreaming(Readable.toWeb(nStream), moduleOptions))) .bimap(always(args), identity) } @@ -169,7 +172,7 @@ export function loadWasmModuleWith ({ fetch, ARWEAVE_URL, WASM_BINARY_FILE_DIREC .chain(fromPromise(([s1, s2]) => Promise.all([ writeWasmFile(moduleId, Readable.fromWeb(s1)), - WebAssembly.compileStreaming(wasmResponse(s2), moduleOptions) + compileStreaming(s2, moduleOptions) ]) )) .map(([, res]) => res)