From b4674a0d9c8e206a1d9979293b96b73513d40ac3 Mon Sep 17 00:00:00 2001 From: Sergey Timoshin Date: Sat, 14 Dec 2024 09:52:34 +0000 Subject: [PATCH 1/3] feat: add support for `--sbf-program` flag in test-validator This allows adding SBF programs to the genesis configuration with upgrades disabled. The flag accepts pairs of arguments (address and path) to specify programs, and validates input to ensure correct usage. --- cli/src/commands/test-validator/index.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/cli/src/commands/test-validator/index.ts b/cli/src/commands/test-validator/index.ts index 554e2ba6c..a5cf3a751 100644 --- a/cli/src/commands/test-validator/index.ts +++ b/cli/src/commands/test-validator/index.ts @@ -119,6 +119,12 @@ class SetupCommand extends Command { required: false, exclusive: ["geyser-config"], }), + "sbf-program": Flags.string({ + description: + "Add a SBF program to the genesis configuration with upgrades disabled. If the ledger already exists then this parameter is silently ignored. First argument can be a pubkey string or path to a keypair", + required: false, + multiple: true, + }), }; async run() { @@ -139,7 +145,22 @@ class SetupCommand extends Command { }); this.log("\nTest validator stopped successfully \x1b[32m✔\x1b[0m"); } else { + const rawValues = flags["sbf-program"] || []; + + if (rawValues.length % 2 !== 0) { + this.error("Each --sbf-program flag must have exactly two arguments"); + } + + const programs: { address: string; path: string }[] = []; + for (let i = 0; i < rawValues.length; i += 2) { + programs.push({ + address: rawValues[i], + path: rawValues[i + 1], + }); + } + await initTestEnv({ + additionalPrograms: programs, checkPhotonVersion: !flags["relax-indexer-version-constraint"], indexer: !flags["skip-indexer"], limitLedgerSize: flags["limit-ledger-size"], From 948b56f364136f9a3b23600ce72a92c5beca9e4d Mon Sep 17 00:00:00 2001 From: Sergey Timoshin Date: Sun, 15 Dec 2024 16:13:52 +0000 Subject: [PATCH 2/3] feat: enhance test-validator with program validation and system program checks + add tests --- cli/package.json | 3 +- cli/src/commands/test-validator/index.ts | 54 ++- cli/src/utils/initTestEnv.ts | 58 +-- cli/src/utils/processPhotonIndexer.ts | 16 +- .../commands/test-validator/index.test.ts | 347 ++++++++++++++++++ 5 files changed, 438 insertions(+), 40 deletions(-) create mode 100644 cli/test/commands/test-validator/index.test.ts diff --git a/cli/package.json b/cli/package.json index 1eae65781..353370cc4 100644 --- a/cli/package.json +++ b/cli/package.json @@ -121,9 +121,10 @@ "test-decompress-sol": "mocha ./test/commands/decompress-sol/index.test.ts -t 10000000 --exit", "test-compress-spl": "mocha ./test/commands/compress-spl/index.test.ts -t 10000000 --exit", "test-decompress-spl": "mocha ./test/commands/decompress-spl/index.test.ts -t 10000000 --exit", + "test-test-validator": "mocha ./test/commands/test-validator/index.test.ts -t 10000000 --exit", "kill": "killall solana-test-validator || true && killall solana-test-val || true && sleep 1", "test-cli": "pnpm test-config && pnpm kill", - "test": "pnpm kill && pnpm test-cli && pnpm test-utils && pnpm test-create-mint && pnpm test-mint-to && pnpm test-transfer && pnpm test-merge-token-accounts && pnpm test-create-token-pool && pnpm test-compress-spl && pnpm test-decompress-spl && pnpm test-balance && pnpm test-compress-sol && pnpm test-decompress-sol && pnpm test-approve-and-mint-to", + "test": "pnpm kill && pnpm test-cli && pnpm test-utils && pnpm test-create-mint && pnpm test-mint-to && pnpm test-transfer && pnpm test-merge-token-accounts && pnpm test-create-token-pool && pnpm test-compress-spl && pnpm test-decompress-spl && pnpm test-balance && pnpm test-compress-sol && pnpm test-decompress-sol && pnpm test-approve-and-mint-to && pnpm test-test-validator", "install-local": "pnpm build && pnpm global remove @lightprotocol/zk-compression-cli || true && pnpm global add $PWD", "version": "oclif readme && git add README.md" }, diff --git a/cli/src/commands/test-validator/index.ts b/cli/src/commands/test-validator/index.ts index a5cf3a751..be7f9c81d 100644 --- a/cli/src/commands/test-validator/index.ts +++ b/cli/src/commands/test-validator/index.ts @@ -1,5 +1,9 @@ import { Command, Flags } from "@oclif/core"; -import { initTestEnv, stopTestEnv } from "../../utils/initTestEnv"; +import { + initTestEnv, + stopTestEnv, + SYSTEM_PROGRAMS, +} from "../../utils/initTestEnv"; import { CustomLoader } from "../../utils/index"; import path from "path"; import fs from "fs"; @@ -15,7 +19,10 @@ class SetupCommand extends Command { '$ light test-validator --validator-args "--limit-ledger-size 50000000"', ]; - protected finally(_: Error | undefined): Promise { + protected finally(err: Error | undefined): Promise { + if (err) { + console.error(err); + } process.exit(); } @@ -124,9 +131,50 @@ class SetupCommand extends Command { "Add a SBF program to the genesis configuration with upgrades disabled. If the ledger already exists then this parameter is silently ignored. First argument can be a pubkey string or path to a keypair", required: false, multiple: true, + summary: "Usage: --sbf-program
", }), }; + validatePrograms(programs: { address: string; path: string }[]): void { + // Check for duplicate addresses among provided programs + const addresses = new Set(); + for (const program of programs) { + if (addresses.has(program.address)) { + this.error(`Duplicate program address detected: ${program.address}`); + } + addresses.add(program.address); + + // Get the program filename from the path + const programFileName = path.basename(program.path); + + // Check for collisions with system programs (both address and filename) + const systemProgramCollision = SYSTEM_PROGRAMS.find( + (sysProg) => + sysProg.id === program.address || + (sysProg.name && programFileName === sysProg.name), + ); + + if (systemProgramCollision) { + const collisionType = + systemProgramCollision.id === program.address + ? `address (${program.address})` + : `filename (${programFileName})`; + + this.error( + `Program ${collisionType} collides with system program ` + + `"${systemProgramCollision.name || systemProgramCollision.id}". ` + + `System programs cannot be overwritten.`, + ); + } + + // Validate program file exists + const programPath = path.resolve(program.path); + if (!fs.existsSync(programPath)) { + this.error(`Program file not found: ${programPath}`); + } + } + } + async run() { const { flags } = await this.parse(SetupCommand); const loader = new CustomLoader("Performing setup tasks...\n"); @@ -159,6 +207,8 @@ class SetupCommand extends Command { }); } + this.validatePrograms(programs); + await initTestEnv({ additionalPrograms: programs, checkPhotonVersion: !flags["relax-indexer-version-constraint"], diff --git a/cli/src/utils/initTestEnv.ts b/cli/src/utils/initTestEnv.ts index 51fd387bf..635f5b4b0 100644 --- a/cli/src/utils/initTestEnv.ts +++ b/cli/src/utils/initTestEnv.ts @@ -20,6 +20,35 @@ import { import { killProver, startProver } from "./processProverServer"; import { killIndexer, startIndexer } from "./processPhotonIndexer"; +type Program = { id: string; name?: string; tag?: string; path?: string }; +export const SYSTEM_PROGRAMS: Program[] = [ + { + id: "noopb9bkMVfRPU8AsbpTUg8AQkHtKwMYZiFUjNRtMmV", + name: "spl_noop.so", + tag: SPL_NOOP_PROGRAM_TAG, + }, + { + id: "SySTEM1eSU2p4BGQfQpimFEWWSC1XDFeun3Nqzz3rT7", + name: "light_system_program.so", + tag: LIGHT_SYSTEM_PROGRAM_TAG, + }, + { + id: "cTokenmWW8bLPjZEBAUgYy3zKxQZW6VKi7bqNFEVv3m", + name: "light_compressed_token.so", + tag: LIGHT_COMPRESSED_TOKEN_TAG, + }, + { + id: "compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq", + name: "account_compression.so", + tag: LIGHT_ACCOUNT_COMPRESSION_TAG, + }, + { + id: "Lighton6oQpVkeewmo2mcPTQQp7kYHr4fWpAgJyEmDX", + name: "light_registry.so", + tag: LIGHT_REGISTRY_TAG, + }, +]; + export async function stopTestEnv(options: { indexer: boolean; prover: boolean; @@ -208,35 +237,8 @@ export async function getSolanaArgs({ gossipHost?: string; downloadBinaries?: boolean; }): Promise> { - type Program = { id: string; name?: string; tag?: string; path?: string }; // TODO: adjust program tags - const programs: Program[] = [ - { - id: "noopb9bkMVfRPU8AsbpTUg8AQkHtKwMYZiFUjNRtMmV", - name: "spl_noop.so", - tag: SPL_NOOP_PROGRAM_TAG, - }, - { - id: "SySTEM1eSU2p4BGQfQpimFEWWSC1XDFeun3Nqzz3rT7", - name: "light_system_program.so", - tag: LIGHT_SYSTEM_PROGRAM_TAG, - }, - { - id: "cTokenmWW8bLPjZEBAUgYy3zKxQZW6VKi7bqNFEVv3m", - name: "light_compressed_token.so", - tag: LIGHT_COMPRESSED_TOKEN_TAG, - }, - { - id: "compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq", - name: "account_compression.so", - tag: LIGHT_ACCOUNT_COMPRESSION_TAG, - }, - { - id: "Lighton6oQpVkeewmo2mcPTQQp7kYHr4fWpAgJyEmDX", - name: "light_registry.so", - tag: LIGHT_REGISTRY_TAG, - }, - ]; + const programs: Program[] = [...SYSTEM_PROGRAMS]; if (additionalPrograms) additionalPrograms.forEach((program) => { programs.push({ id: program.address, path: program.path }); diff --git a/cli/src/utils/processPhotonIndexer.ts b/cli/src/utils/processPhotonIndexer.ts index a06455274..8a875a0a1 100644 --- a/cli/src/utils/processPhotonIndexer.ts +++ b/cli/src/utils/processPhotonIndexer.ts @@ -21,16 +21,14 @@ export async function startIndexer( throw new Error(message); } else { console.log("Starting indexer..."); - let args: string[] = []; + const args: string[] = [ + "--port", + indexerPort.toString(), + "--rpc-url", + rpcUrl, + ]; if (photonDatabaseUrl) { - args = [ - "--db-url", - photonDatabaseUrl, - "--port", - indexerPort.toString(), - "--rpc-url", - rpcUrl, - ]; + args.push("--db-url", photonDatabaseUrl); } spawnBinary(INDEXER_PROCESS_NAME, args); await waitForServers([{ port: indexerPort, path: "/getIndexerHealth" }]); diff --git a/cli/test/commands/test-validator/index.test.ts b/cli/test/commands/test-validator/index.test.ts new file mode 100644 index 000000000..8bdf21bd4 --- /dev/null +++ b/cli/test/commands/test-validator/index.test.ts @@ -0,0 +1,347 @@ +import { expect } from "@oclif/test"; +import { defaultSolanaWalletKeypair, programsDirPath } from "../../../src"; +import { Connection, Keypair } from "@solana/web3.js"; +import * as path from "path"; +import * as fs from "fs"; +import { killProcess } from "../../../src/utils/process"; +import { exec as execCb } from "child_process"; +import { promisify } from "util"; +import { ExecException } from "node:child_process"; + +const exec = promisify(execCb); + +describe("test-validator command", function () { + this.timeout(120_000); + + const defaultRpcPort = 8899; + const customRpcPort = 8877; + const defaultIndexerPort = 8784; + const customIndexerPort = 8755; + const defaultProverPort = 3001; + const customProverPort = 3002; + + async function cleanupProcesses() { + console.log("Running cleanup..."); + try { + // Stop the validator using CLI command first + try { + await exec("./test_bin/dev test-validator --stop"); + console.log("Validator stopped via CLI command"); + } catch (e) { + console.log("No running validator to stop via CLI"); + } + + // Then force kill any remaining processes + await killProcess("solana-test-validator"); + await killProcess("photon"); + await killProcess("prover"); + + // Wait for processes to fully terminate + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Verify processes are actually stopped + try { + await fetch(`http://localhost:${defaultRpcPort}/health`); + throw new Error("Validator still running"); + } catch (e) { + console.log("Validator stopped"); + } + + try { + await fetch(`http://localhost:${defaultIndexerPort}/health`); + throw new Error("Indexer still running"); + } catch (e) { + console.log("Indexer stopped"); + } + + try { + await fetch(`http://localhost:${defaultProverPort}/health`); + throw new Error("Prover still running"); + } catch (e) { + console.log("Prover stopped"); + } + + console.log("Cleanup completed successfully"); + } catch (error) { + console.error("Error in cleanup:", error); + throw error; + } + } + + before(async function () { + await cleanupProcesses(); + }); + + afterEach(async function () { + await cleanupProcesses(); + }); + + it("should start validator with default settings", async function () { + console.log("Starting test-validator..."); + + const { stdout } = await exec("./test_bin/dev test-validator"); + console.log("Command output:", stdout); + + expect(stdout).to.contain("Setup tasks completed successfully"); + console.log("Stdout check passed"); + + console.log("Waiting for validator to be fully ready..."); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + console.log("Attempting to connect to validator..."); + const connection = new Connection(`http://localhost:${defaultRpcPort}`, { + commitment: "confirmed", + confirmTransactionInitialTimeout: 10000, + }); + + try { + console.log("Getting validator version..."); + const version = await connection.getVersion(); + console.log("Validator version:", version); + expect(version).to.have.property("solana-core"); + + console.log("Getting wallet balance..."); + const payer = defaultSolanaWalletKeypair(); + const balance = await connection.getBalance(payer.publicKey); + console.log("Wallet balance:", balance); + expect(balance).to.be.at.least(0); + + console.log("Test completed successfully"); + } catch (error) { + console.error("Error during validator checks:", error); + throw error; + } + }); + + it("should start validator with custom ports", async function () { + const command = [ + "./test_bin/dev test-validator", + `--rpc-port ${customRpcPort}`, + `--indexer-port ${customIndexerPort}`, + `--prover-port ${customProverPort}`, + ].join(" "); + + const { stdout } = await exec(command); + expect(stdout).to.contain("Setup tasks completed successfully"); + + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Verify validator on custom port + const connection = new Connection(`http://localhost:${customRpcPort}`); + const version = await connection.getVersion(); + expect(version).to.have.property("solana-core"); + + // Verify indexer on custom port + const indexerResponse = await fetch( + `http://localhost:${customIndexerPort}/health`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }, + ); + expect(indexerResponse.status).to.equal(200); + + // Verify prover on custom port + const proverResponse = await fetch( + `http://localhost:${customProverPort}/health`, + ); + expect(proverResponse.status).to.equal(200); + }); + + it("should start validator without indexer and prover", async function () { + const { stdout } = await exec( + "./test_bin/dev test-validator --skip-indexer --skip-prover", + ); + expect(stdout).to.contain("Setup tasks completed successfully"); + + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Verify validator is running + const connection = new Connection(`http://localhost:${defaultRpcPort}`); + const version = await connection.getVersion(); + expect(version).to.have.property("solana-core"); + + // Verify indexer is not running + try { + await fetch(`http://localhost:${defaultIndexerPort}/health`); + throw new Error("Indexer should not be running"); + } catch (error) { + expect(error).to.exist; + } + + // Verify prover is not running + try { + await fetch(`http://localhost:${defaultProverPort}/health`); + throw new Error("Prover should not be running"); + } catch (error) { + expect(error).to.exist; + } + }); + + it("should fail with invalid geyser config path", async function () { + try { + await exec( + "./test_bin/dev test-validator --geyser-config nonexistent.json", + ); + throw new Error("Should have failed"); + } catch (error) { + const execError = error as ExecException & { + stdout?: string; + stderr?: string; + }; + // Check either error message or stderr + const errorText = execError.message || execError.stderr || ""; + expect(errorText).to.contain("Geyser config file not found"); + } + }); + + it("should start and stop validator with custom arguments", async function () { + const startCommand = + './test_bin/dev test-validator --validator-args "--log-messages-bytes-limit 1000"'; + const { stdout: startOutput } = await exec(startCommand); + expect(startOutput).to.contain("Setup tasks completed successfully"); + + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Verify validator is running + const connection = new Connection(`http://localhost:${defaultRpcPort}`); + const version = await connection.getVersion(); + expect(version).to.have.property("solana-core"); + + // Stop validator + const stopCommand = "./test_bin/dev test-validator --stop"; + const { stdout: stopOutput } = await exec(stopCommand); + expect(stopOutput).to.contain("Test validator stopped successfully"); + + // Verify validator is stopped + try { + await connection.getVersion(); + throw new Error("Validator should be stopped"); + } catch (error) { + expect(error).to.exist; + } + }); + + const SYSTEM_PROGRAM_ID = "noopb9bkMVfRPU8AsbpTUg8AQkHtKwMYZiFUjNRtMmV"; + const SYSTEM_PROGRAM_NAME = "spl_noop.so"; + + it("should fail when deploying program with system program address", async function () { + const testProgramPath = path.join(programsDirPath(), "test_program.so"); + fs.writeFileSync(testProgramPath, "dummy program data"); + + try { + const command = [ + "./test_bin/dev test-validator", + "--sbf-program", + SYSTEM_PROGRAM_ID, // Use system program address + testProgramPath, + ].join(" "); + + await exec(command); + throw new Error( + "Should have failed due to system program address collision", + ); + } catch (error) { + const execError = error as ExecException & { + stdout?: string; + stderr?: string; + }; + const errorText = execError.message || execError.stderr || ""; + expect(errorText).to.contain("collides with system program"); + expect(errorText).to.contain(SYSTEM_PROGRAM_ID); + } finally { + fs.unlinkSync(testProgramPath); + } + }); + + it("should fail when deploying program with system program name", async function () { + const testProgramPath = path.join(programsDirPath(), SYSTEM_PROGRAM_NAME); + fs.writeFileSync(testProgramPath, "dummy program data"); + + try { + const testKeypair = Keypair.generate(); + const command = [ + "./test_bin/dev test-validator", + "--sbf-program", + testKeypair.publicKey.toString(), + testProgramPath, + ].join(" "); + + await exec(command); + throw new Error( + "Should have failed due to system program name collision", + ); + } catch (error) { + const execError = error as ExecException & { + stdout?: string; + stderr?: string; + }; + const errorText = execError.message || execError.stderr || ""; + expect(errorText).to.contain("collides with system program"); + expect(errorText).to.contain(SYSTEM_PROGRAM_NAME); + } finally { + fs.unlinkSync(testProgramPath); + } + }); + + it("should fail when deploying multiple programs with same address", async function () { + const testProgramPath1 = path.join(programsDirPath(), "test_program1.so"); + const testProgramPath2 = path.join(programsDirPath(), "test_program2.so"); + fs.writeFileSync(testProgramPath1, "dummy program data 1"); + fs.writeFileSync(testProgramPath2, "dummy program data 2"); + + try { + const testKeypair = Keypair.generate(); + const command = [ + "./test_bin/dev test-validator", + "--sbf-program", + testKeypair.publicKey.toString(), + testProgramPath1, + "--sbf-program", + testKeypair.publicKey.toString(), // Same address as first program + testProgramPath2, + ].join(" "); + + await exec(command); + throw new Error("Should have failed due to duplicate program address"); + } catch (error) { + const execError = error as ExecException & { + stdout?: string; + stderr?: string; + }; + const errorText = execError.message || execError.stderr || ""; + expect(errorText).to.contain("Duplicate program address detected"); + } finally { + fs.unlinkSync(testProgramPath1); + fs.unlinkSync(testProgramPath2); + } + }); + + it("should succeed with valid program deployment avoiding system collisions", async function () { + const testProgramPath = path.join(programsDirPath(), "custom_program.so"); + fs.writeFileSync(testProgramPath, "dummy program data"); + + try { + const testKeypair = Keypair.generate(); + const command = [ + "./test_bin/dev test-validator", + "--sbf-program", + testKeypair.publicKey.toString(), + testProgramPath, + ].join(" "); + + const { stdout } = await exec(command); + expect(stdout).to.contain("Setup tasks completed successfully"); + + await new Promise((resolve) => setTimeout(resolve, 5000)); + + // Verify validator is running + const connection = new Connection(`http://localhost:${defaultRpcPort}`); + const version = await connection.getVersion(); + expect(version).to.have.property("solana-core"); + } finally { + fs.unlinkSync(testProgramPath); + } + }); +}); From dd276c2f9709b01137fb3a7b1f1f3328ab965132 Mon Sep 17 00:00:00 2001 From: Sergey Timoshin Date: Sun, 15 Dec 2024 19:19:24 +0000 Subject: [PATCH 3/3] * added usage example about how to use --sbf-program flag programs in test-validator command * added test check to validate the program deployment and ensure the account is executable --- cli/src/commands/test-validator/index.ts | 1 + cli/test/commands/test-validator/index.test.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/cli/src/commands/test-validator/index.ts b/cli/src/commands/test-validator/index.ts index be7f9c81d..570dad4be 100644 --- a/cli/src/commands/test-validator/index.ts +++ b/cli/src/commands/test-validator/index.ts @@ -17,6 +17,7 @@ class SetupCommand extends Command { "$ light test-validator --skip-indexer", "$ light test-validator --geyser-config ./config.json", '$ light test-validator --validator-args "--limit-ledger-size 50000000"', + "$ light test-validator --sbf-program
", ]; protected finally(err: Error | undefined): Promise { diff --git a/cli/test/commands/test-validator/index.test.ts b/cli/test/commands/test-validator/index.test.ts index 8bdf21bd4..b2008ef79 100644 --- a/cli/test/commands/test-validator/index.test.ts +++ b/cli/test/commands/test-validator/index.test.ts @@ -340,6 +340,12 @@ describe("test-validator command", function () { const connection = new Connection(`http://localhost:${defaultRpcPort}`); const version = await connection.getVersion(); expect(version).to.have.property("solana-core"); + + let programAccountInfo = await connection.getAccountInfo( + testKeypair.publicKey, + ); + expect(programAccountInfo).to.exist; + expect(programAccountInfo!.executable).to.be.true; } finally { fs.unlinkSync(testProgramPath); }