diff --git a/.github/workflows/zxc-e2e-test.yaml b/.github/workflows/zxc-e2e-test.yaml index eb064a635..5ab5f5e70 100644 --- a/.github/workflows/zxc-e2e-test.yaml +++ b/.github/workflows/zxc-e2e-test.yaml @@ -167,7 +167,7 @@ jobs: if: ${{ runner.os == 'linux' && (inputs.npm-test-script == 'test-e2e-node-local-hedera' || inputs.npm-test-script == 'test-e2e-node-local-ptt' || inputs.npm-test-script == 'test-e2e-node-add-local') && !cancelled() && !failure() }} run: | cd .. - git clone https://github.com/hashgraph/hedera-services.git --depth 1 --branch v0.58.0 + git clone https://github.com/hashgraph/hedera-services.git --depth 1 --branch v0.58.1 cd hedera-services ls -ltr ${{ env.CG_EXEC }} ./gradlew assemble --stacktrace --info diff --git a/HelperTasks.yml b/Taskfile.helper.yml similarity index 99% rename from HelperTasks.yml rename to Taskfile.helper.yml index bacd8307d..ccc17a41d 100644 --- a/HelperTasks.yml +++ b/Taskfile.helper.yml @@ -163,7 +163,7 @@ tasks: if [[ "${SOLO_CHART_VERSION}" != "" ]]; then export SOLO_CHART_FLAG='--solo-chart-version ${SOLO_CHART_VERSION}' fi - SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- network deploy --namespace "${SOLO_NAMESPACE}" --node-aliases-unparsed {{.node_identifiers}} ${RELEASE_FLAG} ${SOLO_CHART_FLAG} ${VALUES_FLAG} ${SETTINGS_FLAG} ${LOG4J2_FLAG} ${APPLICATION_PROPERTIES_FLAG} -q + SOLO_HOME_DIR=${SOLO_HOME_DIR} npm run solo -- network deploy --namespace "${SOLO_NAMESPACE}" --node-aliases-unparsed {{.node_identifiers}} ${RELEASE_FLAG} ${SOLO_CHART_FLAG} ${VALUES_FLAG} ${SETTINGS_FLAG} ${LOG4J2_FLAG} ${APPLICATION_PROPERTIES_FLAG} ${GENESIS_THROTTLES_FLAG} -q - | if [[ "${CONSENSUS_NODE_VERSION}" != "" ]]; then export CONSENSUS_NODE_FLAG='--release-tag ${CONSENSUS_NODE_VERSION}' diff --git a/Taskfile.yml b/Taskfile.yml index e03d96a28..1f7261e03 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,7 +1,7 @@ version: 3 includes: helper: - taskfile: ./HelperTasks.yml + taskfile: ./Taskfile.helper.yml flatten: true env: SOLO_NETWORK_SIZE: 2 diff --git a/examples/ExamplesTasks.yml b/examples/Taskfile.examples.yml similarity index 94% rename from examples/ExamplesTasks.yml rename to examples/Taskfile.examples.yml index 928c0740e..38ac0d4db 100644 --- a/examples/ExamplesTasks.yml +++ b/examples/Taskfile.examples.yml @@ -1,7 +1,7 @@ version: 3 includes: helper: - taskfile: ../HelperTasks.yml + taskfile: ../Taskfile.helper.yml flatten: true tasks: diff --git a/examples/custom-network-config/Taskfile.yml b/examples/custom-network-config/Taskfile.yml index 7d888c8b5..550f3e9eb 100644 --- a/examples/custom-network-config/Taskfile.yml +++ b/examples/custom-network-config/Taskfile.yml @@ -1,7 +1,7 @@ version: 3 includes: main: - taskfile: ../ExamplesTasks.yml + taskfile: ../Taskfile.examples.yml flatten: true env: SOLO_NETWORK_SIZE: 10 diff --git a/examples/performance-tuning/Latitude/Taskfile.yml b/examples/performance-tuning/latitude/Taskfile.yml similarity index 94% rename from examples/performance-tuning/Latitude/Taskfile.yml rename to examples/performance-tuning/latitude/Taskfile.yml index dff796a7f..d664266a4 100644 --- a/examples/performance-tuning/Latitude/Taskfile.yml +++ b/examples/performance-tuning/latitude/Taskfile.yml @@ -1,7 +1,7 @@ version: 3 includes: main: - taskfile: ../../ExamplesTasks.yml + taskfile: ../../Taskfile.examples.yml flatten: true vars: solo_home_override_dir: "%HOME%/.solo" diff --git a/examples/performance-tuning/Latitude/application.properties b/examples/performance-tuning/latitude/application.properties similarity index 100% rename from examples/performance-tuning/Latitude/application.properties rename to examples/performance-tuning/latitude/application.properties diff --git a/examples/performance-tuning/Latitude/init-containers-values.yaml b/examples/performance-tuning/latitude/init-containers-values.yaml similarity index 100% rename from examples/performance-tuning/Latitude/init-containers-values.yaml rename to examples/performance-tuning/latitude/init-containers-values.yaml diff --git a/examples/performance-tuning/Latitude/log4j2.xml b/examples/performance-tuning/latitude/log4j2.xml similarity index 100% rename from examples/performance-tuning/Latitude/log4j2.xml rename to examples/performance-tuning/latitude/log4j2.xml diff --git a/examples/performance-tuning/Latitude/nlg-values.yaml b/examples/performance-tuning/latitude/nlg-values.yaml similarity index 100% rename from examples/performance-tuning/Latitude/nlg-values.yaml rename to examples/performance-tuning/latitude/nlg-values.yaml diff --git a/examples/performance-tuning/Latitude/settings.txt b/examples/performance-tuning/latitude/settings.txt similarity index 100% rename from examples/performance-tuning/Latitude/settings.txt rename to examples/performance-tuning/latitude/settings.txt diff --git a/examples/performance-tuning/solo-perf-test/Taskfile.yml b/examples/performance-tuning/solo-perf-test/Taskfile.yml index 9cdfefb03..2e3a05def 100644 --- a/examples/performance-tuning/solo-perf-test/Taskfile.yml +++ b/examples/performance-tuning/solo-perf-test/Taskfile.yml @@ -1,7 +1,7 @@ version: 3 includes: main: - taskfile: ../../ExamplesTasks.yml + taskfile: ../../Taskfile.examples.yml flatten: true env: SOLO_NETWORK_SIZE: 7 diff --git a/examples/solo-gke-test/Taskfile.yml b/examples/solo-gke-test/Taskfile.yml index 07ab69c63..e5899e35e 100644 --- a/examples/solo-gke-test/Taskfile.yml +++ b/examples/solo-gke-test/Taskfile.yml @@ -1,7 +1,7 @@ version: 3 includes: main: - taskfile: ../ExamplesTasks.yml + taskfile: ../Taskfile.examples.yml flatten: true vars: solo_home_override_dir: "/Users/user/.solo-gke-test" @@ -17,3 +17,4 @@ env: APPLICATION_PROPERTIES_FLAG: "--application-properties {{.USER_WORKING_DIR}}/application.properties" HEDERA_SERVICES_ROOT: "/Users/user/source/hedera-services" LOCAL_BUILD_FLAG: "--local-build-path {{.HEDERA_SERVICES_ROOT}}/hedera-node/data" + GENESIS_THROTTLES_FLAG: "--genesis-throttles-file {{.USER_WORKING_DIR}}/throttles.json" diff --git a/examples/solo-gke-test/application.properties b/examples/solo-gke-test/application.properties index 213d33929..afde2a4a8 100644 --- a/examples/solo-gke-test/application.properties +++ b/examples/solo-gke-test/application.properties @@ -15,3 +15,7 @@ hedera.profiles.active=TEST staking.periodMins=1 nodes.updateAccountIdAllowed=true blockStream.streamMode=RECORDS + +# Override the throttle definitions to be used during genesis. +# This override is required because release <= 0.58.x use a different default path. +bootstrap.throttleDefsJson.resource=data/config/genesis-throttles.json diff --git a/examples/solo-gke-test/throttles.json b/examples/solo-gke-test/throttles.json new file mode 100644 index 000000000..6ffc1c201 --- /dev/null +++ b/examples/solo-gke-test/throttles.json @@ -0,0 +1,204 @@ +{ + "buckets": [ + { + "burstPeriod": 0, + "burstPeriodMs": 15000, + "name": "ThroughputLimits", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 150000, + "operations": [ + "ScheduleCreate", + "CryptoCreate", + "CryptoTransfer", + "CryptoUpdate", + "CryptoDelete", + "CryptoGetInfo", + "CryptoGetAccountRecords", + "ConsensusCreateTopic", + "ConsensusSubmitMessage", + "ConsensusUpdateTopic", + "ConsensusDeleteTopic", + "ConsensusGetTopicInfo", + "TokenGetNftInfo", + "TokenGetInfo", + "ScheduleDelete", + "ScheduleGetInfo", + "FileGetContents", + "FileGetInfo", + "ContractUpdate", + "ContractDelete", + "ContractGetInfo", + "ContractGetBytecode", + "ContractGetRecords", + "ContractCallLocal", + "TransactionGetRecord", + "GetVersionInfo", + "TokenGetAccountNftInfos", + "TokenGetNftInfos", + "CryptoApproveAllowance", + "CryptoDeleteAllowance", + "UtilPrng", + "NodeCreate", + "NodeUpdate", + "NodeDelete" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 40000, + "operations": [ + "FileCreate", + "FileUpdate", + "FileAppend", + "FileDelete" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 100000, + "operations": [ + "ScheduleSign" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 125000, + "operations": [ + "TokenMint" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 35000, + "operations": [ + "ContractCall", + "ContractCreate", + "EthereumTransaction" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 100000, + "operations": [ + "TokenCreate", + "TokenDelete", + "TokenBurn", + "TokenUpdate", + "TokenFeeScheduleUpdate", + "TokenAssociateToAccount", + "TokenAccountWipe", + "TokenDissociateFromAccount", + "TokenFreezeAccount", + "TokenUnfreezeAccount", + "TokenGrantKycToAccount", + "TokenRevokeKycFromAccount", + "TokenPause", + "TokenUnpause", + "TokenUpdateNfts", + "TokenReject", + "TokenAirdrop", + "TokenClaimAirdrop", + "TokenCancelAirdrop" + ] + } + ] + }, + { + "burstPeriod": 0, + "burstPeriodMs": 1000, + "name": "OffHeapQueryLimits", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 100000, + "operations": [ + "FileGetContents", + "FileGetInfo", + "ContractGetInfo", + "ContractGetBytecode", + "ContractCallLocal" + ] + } + ] + }, + { + "burstPeriod": 0, + "burstPeriodMs": 3000, + "name": "PriorityReservations", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 40000, + "operations": [ + "FileCreate", + "FileUpdate", + "FileAppend", + "FileDelete" + ] + } + ] + }, + { + "burstPeriod": 0, + "burstPeriodMs": 15000, + "name": "CreationLimits", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 40000, + "operations": [ + "CryptoCreate", + "NodeCreate" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 5000, + "operations": [ + "ConsensusCreateTopic" + ] + }, + { + "opsPerSec": 0, + "milliOpsPerSec": 100000, + "operations": [ + "TokenCreate", + "TokenAssociateToAccount", + "ScheduleCreate", + "TokenAirdrop" + ] + } + ] + }, + { + "burstPeriod": 0, + "burstPeriodMs": 1000, + "name": "FreeQueryLimits", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 1000000000, + "operations": [ + "TransactionGetReceipt" + ] + } + ] + }, + { + "burstPeriod": 0, + "burstPeriodMs": 1000, + "name": "BalanceQueryLimits", + "throttleGroups": [ + { + "opsPerSec": 0, + "milliOpsPerSec": 1000000, + "operations": [ + "CryptoGetAccountBalance" + ] + } + ] + } + ] +} diff --git a/src/commands/flags.ts b/src/commands/flags.ts index 498640886..e73cee854 100644 --- a/src/commands/flags.ts +++ b/src/commands/flags.ts @@ -903,6 +903,17 @@ export class Flags { prompt: undefined, }; + static readonly genesisThrottlesFile: CommandFlag = { + constName: 'genesisThrottlesFile', + name: 'genesis-throttles-file', + definition: { + describe: 'throttles.json file used during network genesis', + defaultValue: '', + type: 'string', + }, + prompt: undefined, + }; + static readonly settingTxt: CommandFlag = { constName: 'settingTxt', name: 'settings-txt', @@ -1646,6 +1657,7 @@ export class Flags { Flags.generateEcdsaKey, Flags.generateGossipKeys, Flags.generateTlsKeys, + Flags.genesisThrottlesFile, Flags.gossipEndpoints, Flags.gossipPrivateKey, Flags.gossipPublicKey, diff --git a/src/commands/network.ts b/src/commands/network.ts index ea6db9bdc..b3fd759f9 100644 --- a/src/commands/network.ts +++ b/src/commands/network.ts @@ -17,14 +17,14 @@ import {ListrEnquirerPromptAdapter} from '@listr2/prompt-adapter-enquirer'; import chalk from 'chalk'; import {Listr, type ListrTask} from 'listr2'; -import {SoloError, IllegalArgumentError, MissingArgumentError} from '../core/errors.js'; +import {IllegalArgumentError, MissingArgumentError, SoloError} from '../core/errors.js'; import {BaseCommand} from './base.js'; import {Flags as flags} from './flags.js'; import * as constants from '../core/constants.js'; import {Templates} from '../core/templates.js'; import * as helpers from '../core/helpers.js'; +import {addDebugOptions, resolveValidJsonFilePath, validatePath} from '../core/helpers.js'; import path from 'path'; -import {addDebugOptions, validatePath} from '../core/helpers.js'; import fs from 'fs'; import {RemoteConfigTasks} from '../core/config/remote/remote_config_tasks.js'; import {type KeyManager} from '../core/key_manager.js'; @@ -63,6 +63,8 @@ export interface NetworkDeployConfigClass { grpcTlsKeyPath: string; grpcWebTlsKeyPath: string; genesisNetworkData: GenesisNetworkDataConstructor; + genesisThrottlesFile: string; + resolvedThrottlesFile: string; getUnusedConfigs: () => string[]; haproxyIps: string; envoyIps: string; @@ -107,6 +109,7 @@ export class NetworkCommand extends BaseCommand { flags.applicationEnv, flags.applicationProperties, flags.bootstrapProperties, + flags.genesisThrottlesFile, flags.cacheDir, flags.chainId, flags.chartDirectory, @@ -144,6 +147,7 @@ export class NetworkCommand extends BaseCommand { haproxyIpsParsed?: Record; envoyIpsParsed?: Record; genesisNetworkData: GenesisNetworkDataConstructor; + resolvedThrottlesFile: string; }) { let valuesArg = config.chartDirectory ? `-f ${path.join(config.chartDirectory, 'solo-deployment', 'values.yaml')}` @@ -192,6 +196,10 @@ export class NetworkCommand extends BaseCommand { }); } + if (config.resolvedThrottlesFile) { + valuesArg += ` --set-file "hedera.configMaps.genesisThrottlesJson=${config.resolvedThrottlesFile}"`; + } + if (config.valuesFile) { valuesArg += this.prepareValuesFiles(config.valuesFile); } @@ -211,6 +219,7 @@ export class NetworkCommand extends BaseCommand { flags.applicationEnv, flags.applicationProperties, flags.bootstrapProperties, + flags.genesisThrottlesFile, flags.cacheDir, flags.chainId, flags.chartDirectory, @@ -238,6 +247,7 @@ export class NetworkCommand extends BaseCommand { 'stagingDir', 'stagingKeysDir', 'valuesArg', + 'resolvedThrottlesFile', ]) as NetworkDeployConfigClass; config.nodeAliases = helpers.parseNodeAliases(config.nodeAliasesUnparsed); @@ -268,6 +278,11 @@ export class NetworkCommand extends BaseCommand { config.keysDir, ); + config.resolvedThrottlesFile = resolveValidJsonFilePath( + config.genesisThrottlesFile, + flags.genesisThrottlesFile.definition.defaultValue as string, + ); + config.valuesArg = await this.prepareValuesArg(config); if (!(await this.k8.hasNamespace(config.namespace))) { @@ -554,6 +569,7 @@ export class NetworkCommand extends BaseCommand { }; checkTimeout: boolean; } + let networkDestroySuccess = true; const tasks = new Listr( [ diff --git a/src/core/certificate_manager.ts b/src/core/certificate_manager.ts index 6a6408638..06757cad3 100644 --- a/src/core/certificate_manager.ts +++ b/src/core/certificate_manager.ts @@ -92,7 +92,7 @@ export class CertificateManager { const isSecretCreated = await this.k8.createSecret(name, namespace, 'Opaque', data, labels, true); if (!isSecretCreated) { - throw new SoloError(`failed to create secret for tsc certificates for node '${nodeAlias}'`); + throw new SoloError(`failed to create secret for TLS certificates for node '${nodeAlias}'`); } } catch (e: Error | any) { const errorMessage = diff --git a/src/core/helpers.ts b/src/core/helpers.ts index ca9329195..c850a4a7e 100644 --- a/src/core/helpers.ts +++ b/src/core/helpers.ts @@ -342,3 +342,45 @@ export function addFlagsToArgv( return argv; } + +export function resolveValidJsonFilePath(filePath: string, defaultPath?: string): string { + if (!filePath) { + if (defaultPath) { + return resolveValidJsonFilePath(defaultPath, null); + } + + return ''; + } + + const resolvedFilePath = fs.realpathSync(validatePath(filePath)); + + if (!fs.existsSync(resolvedFilePath)) { + if (defaultPath) { + return resolveValidJsonFilePath(defaultPath, null); + } + + throw new SoloError(`File does not exist: ${filePath}`); + } + + // If the file is empty (or size cannot be determined) then fallback on the default values + const throttleInfo = fs.statSync(resolvedFilePath); + if (throttleInfo.size === 0 && defaultPath) { + return resolveValidJsonFilePath(defaultPath, null); + } else if (!defaultPath) { + throw new SoloError(`File is empty: ${filePath}`); + } + + try { + // Ensure the file contains valid JSON data + JSON.parse(fs.readFileSync(resolvedFilePath, 'utf8')); + return resolvedFilePath; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (e: unknown) { + // Fallback to the default values if an error occurs due to invalid JSON data or unable to read the file size + if (defaultPath) { + return resolveValidJsonFilePath(defaultPath, null); + } + + throw new SoloError(`Invalid JSON data in file: ${filePath}`); + } +} diff --git a/test/test_util.ts b/test/test_util.ts index d5d427772..83a02ef47 100644 --- a/test/test_util.ts +++ b/test/test_util.ts @@ -231,7 +231,7 @@ export function e2eTestSuite( describe(`E2E Test Suite for '${testName}'`, function () { this.bail(true); // stop on first failure, nothing else will matter if network doesn't come up correctly - describe(`Bootstrap network for test [release ${argv[flags.releaseTag.name]}}]`, () => { + describe(`Bootstrap network for test [release ${argv[flags.releaseTag.name]}]`, () => { before(() => { bootstrapResp.opts.logger.showUser( `------------------------- START: bootstrap (${testName}) ----------------------------`, diff --git a/version.ts b/version.ts index fb867b700..0a3e1eef4 100644 --- a/version.ts +++ b/version.ts @@ -20,8 +20,8 @@ */ export const HELM_VERSION = 'v3.14.2'; -export const SOLO_CHART_VERSION = '0.38.2'; -export const HEDERA_PLATFORM_VERSION = 'v0.58.0'; +export const SOLO_CHART_VERSION = '0.39.0'; +export const HEDERA_PLATFORM_VERSION = 'v0.58.1'; export const MIRROR_NODE_VERSION = '0.118.1'; export const HEDERA_EXPLORER_VERSION = '0.2.1'; export const HEDERA_JSON_RPC_RELAY_VERSION = 'v0.59.0';