From fc69911a3ca06960e22b10fea081710d8f0a5e0b Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 5 Dec 2024 14:46:50 +0000 Subject: [PATCH 1/7] first draft --- src/@types/OceanNode.ts | 1 + src/components/P2P/index.ts | 6 +++++- src/utils/config.ts | 3 ++- src/utils/constants.ts | 5 +++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/@types/OceanNode.ts b/src/@types/OceanNode.ts index bc0f9d535..f221a30d8 100644 --- a/src/@types/OceanNode.ts +++ b/src/@types/OceanNode.ts @@ -94,6 +94,7 @@ export interface OceanNodeConfig { rateLimit?: number denyList?: DenyList unsafeURLs?: string[] + isBootstrap?: boolean } export interface P2PStatusResponse { diff --git a/src/components/P2P/index.ts b/src/components/P2P/index.ts index 9e9ec1c79..f8333115d 100644 --- a/src/components/P2P/index.ts +++ b/src/components/P2P/index.ts @@ -124,7 +124,11 @@ export class OceanP2P extends EventEmitter { this._protocol = '/ocean/nodes/1.0.0' // this._interval = setInterval(this._pollPeers.bind(this), this._options.pollInterval) - this._libp2p.handle(this._protocol, handleProtocolCommands.bind(this)) + + // only enable handling of commands if not bootstrap node + if (!this._config.isBootstrap) { + this._libp2p.handle(this._protocol, handleProtocolCommands.bind(this)) + } setInterval(this.republishStoredDDOS.bind(this), REPUBLISH_INTERVAL_HOURS) diff --git a/src/utils/config.ts b/src/utils/config.ts index bc659edcf..c96b4f3ff 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -616,7 +616,8 @@ async function getEnvConfig(isStartup?: boolean): Promise { ENVIRONMENT_VARIABLES.UNSAFE_URLS, isStartup, knownUnsafeURLs - ) + ), + isBootstrap: getBoolEnvValue(process.env.IS_BOOTSTRAP, false) } if (!previousConfiguration) { diff --git a/src/utils/constants.ts b/src/utils/constants.ts index b1acee704..2bf91f761 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -317,6 +317,11 @@ export const ENVIRONMENT_VARIABLES: Record = { name: 'DB_TYPE', value: process.env.DB_TYPE, required: false + }, + IS_BOOTSTRAP: { + name: 'IS_BOOTSTRAP', + value: process.env.IS_BOOTSTRAP, + required: false } } From e7360da61bcbc7c796f48fa8f5dd064e64804331 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 5 Dec 2024 14:56:56 +0000 Subject: [PATCH 2/7] add doc --- docs/env.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/env.md b/docs/env.md index a20500736..9c718177c 100644 --- a/docs/env.md +++ b/docs/env.md @@ -25,6 +25,7 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `RATE_DENY_LIST`: Blocked list of IPs and peer IDs. Example: `"{ \"peers\": [\"16Uiu2HAkuYfgjXoGcSSLSpRPD6XtUgV71t5RqmTmcqdbmrWY9MJo\"], \"ips\": [\"127.0.0.1\"] }"` - `MAX_REQ_PER_SECOND`: Number of requests per second allowed by the same client. Example: `3` - `MAX_CHECKSUM_LENGTH`: Define the maximum length for a file if checksum is required (Mb). Example: `10` +- `IS_BOOTSTRAP`: Is this node to be used as bootstrap node or not. Default is `false`. ## Logs From f8227428374d7d73e411968a64e8e87440af9095 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 5 Dec 2024 15:02:27 +0000 Subject: [PATCH 3/7] Release 0.2.1 --- CHANGELOG.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 123905f44..d85867151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,54 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [v0.2.1](https://github.com/oceanprotocol/ocean-node/compare/v0.2.0...v0.2.1) + +- have bootstrap config var [`#782`](https://github.com/oceanprotocol/ocean-node/pull/782) +- Removing unused express dependency from dashboard [`#761`](https://github.com/oceanprotocol/ocean-node/pull/761) +- check if rpcs and indexer_networks are aligned, log [`#750`](https://github.com/oceanprotocol/ocean-node/pull/750) +- remove node 16 image references [`#759`](https://github.com/oceanprotocol/ocean-node/pull/759) +- try debug and fix randomly logs failing test [`#752`](https://github.com/oceanprotocol/ocean-node/pull/752) +- fix dashboard hash out of sync [`#744`](https://github.com/oceanprotocol/ocean-node/pull/744) +- improve reindex command, optional block, also check if thread is running [`#747`](https://github.com/oceanprotocol/ocean-node/pull/747) +- Add cleaner logging when idexing and getting/updating block info from db [`#743`](https://github.com/oceanprotocol/ocean-node/pull/743) +- Add INDEXER_NETWORKS env var within CI job. [`#734`](https://github.com/oceanprotocol/ocean-node/pull/734) +- fix system tests. Running old version of node? [`#733`](https://github.com/oceanprotocol/ocean-node/pull/733) +- rm console.logs [`#731`](https://github.com/oceanprotocol/ocean-node/pull/731) +- fix wrong block for log [`#727`](https://github.com/oceanprotocol/ocean-node/pull/727) +- nonce db sql lite [`#723`](https://github.com/oceanprotocol/ocean-node/pull/723) +- Bump version axios 1.6.0 -> 1.7.4. [`#716`](https://github.com/oceanprotocol/ocean-node/pull/716) +- Bump version express 4.18.2 -> 4.21.0. [`#717`](https://github.com/oceanprotocol/ocean-node/pull/717) +- Feature/ add Elasticsearch database alternative for typesense [`#599`](https://github.com/oceanprotocol/ocean-node/pull/599) +- Database tests 2db [`#707`](https://github.com/oceanprotocol/ocean-node/pull/707) +- add credentials at service level, on schemas [`#710`](https://github.com/oceanprotocol/ocean-node/pull/710) +- Bump micromatch from 4.0.5 to 4.0.8 in /dashboard [`#649`](https://github.com/oceanprotocol/ocean-node/pull/649) +- Bump undici from 5.27.0 to 5.28.4 [`#610`](https://github.com/oceanprotocol/ocean-node/pull/610) +- testing changes [`#718`](https://github.com/oceanprotocol/ocean-node/pull/718) +- Policy Server [`#694`](https://github.com/oceanprotocol/ocean-node/pull/694) +- fix missing/invalid db_url log message, put warn at startup [`#654`](https://github.com/oceanprotocol/ocean-node/pull/654) +- move c2d engines under OceanNode class [`#702`](https://github.com/oceanprotocol/ocean-node/pull/702) +- improve error message, transfer fees and tweak node response [`#701`](https://github.com/oceanprotocol/ocean-node/pull/701) +- fix default filtered p2p ip list (non routable ips that should not be announced) [`#699`](https://github.com/oceanprotocol/ocean-node/pull/699) +- chore: add datatokenaddress in order table [`#700`](https://github.com/oceanprotocol/ocean-node/pull/700) +- fixing tests [`#703`](https://github.com/oceanprotocol/ocean-node/pull/703) +- Updating deprecated actions [`#681`](https://github.com/oceanprotocol/ocean-node/pull/681) +- add access control to service level [`#698`](https://github.com/oceanprotocol/ocean-node/pull/698) +- customize boostrap params [`#696`](https://github.com/oceanprotocol/ocean-node/pull/696) +- Issue 668 oasis decryption [`#679`](https://github.com/oceanprotocol/ocean-node/pull/679) +- Add max ram memory in node start cmd [`#697`](https://github.com/oceanprotocol/ocean-node/pull/697) +- Feature/add elasticdb search [`#695`](https://github.com/oceanprotocol/ocean-node/pull/695) +- fix: search and nonce [`#688`](https://github.com/oceanprotocol/ocean-node/pull/688) +- Issue 501 - C2D getJobStatus - did vs documentId vs agreementId [`#502`](https://github.com/oceanprotocol/ocean-node/pull/502) +- Feature/add elasticdb tvl [`#677`](https://github.com/oceanprotocol/ocean-node/pull/677) +- Renaming dashboard as Control Panel [`#674`](https://github.com/oceanprotocol/ocean-node/pull/674) +- add new schema + tests [`a648cb3`](https://github.com/oceanprotocol/ocean-node/commit/a648cb38df2dea537ca28e67ae4bf1113c9b287d) +- normalize responses [`584b6c0`](https://github.com/oceanprotocol/ocean-node/commit/584b6c09c67fb66fee7d0d01cca51693ea2a3723) +- first draft [`b3afc02`](https://github.com/oceanprotocol/ocean-node/commit/b3afc02219233af4026c43b886bf97c0a2edb323) + #### [v0.2.0](https://github.com/oceanprotocol/ocean-node/compare/v0.1.0...v0.2.0) +> 9 September 2024 + - Downgrade indexer errors [`#675`](https://github.com/oceanprotocol/ocean-node/pull/675) - Update system requirements: change iOS to macOS [`#673`](https://github.com/oceanprotocol/ocean-node/pull/673) - Fix crash on connection reset - P2P streams [`#659`](https://github.com/oceanprotocol/ocean-node/pull/659) diff --git a/package-lock.json b/package-lock.json index 98a29ee61..94f029570 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ocean-node", - "version": "0.2.0", + "version": "0.2.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ocean-node", - "version": "0.2.0", + "version": "0.2.1", "license": "Apache-2.0", "dependencies": { "@chainsafe/libp2p-gossipsub": "^13.1.0", diff --git a/package.json b/package.json index d09f7d0aa..43dd377a4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ocean-node", - "version": "0.2.0", + "version": "0.2.1", "description": "Ocean Node is used to run all core services in the Ocean stack", "author": "Ocean Protocol Foundation", "license": "Apache-2.0", From 13b4adde61a00283580e6329916513c7f7a2bece Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 5 Dec 2024 15:59:07 +0000 Subject: [PATCH 4/7] fix usage of getBoolEnvValue --- src/utils/config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/config.ts b/src/utils/config.ts index c96b4f3ff..17d4fc020 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -549,7 +549,7 @@ async function getEnvConfig(isStartup?: boolean): Promise { ), dhtMaxInboundStreams: getIntEnvValue(process.env.P2P_dhtMaxInboundStreams, 500), dhtMaxOutboundStreams: getIntEnvValue(process.env.P2P_dhtMaxOutboundStreams, 500), - enableDHTServer: getBoolEnvValue(process.env.P2P_ENABLE_DHT_SERVER, false), + enableDHTServer: getBoolEnvValue('P2P_ENABLE_DHT_SERVER', false), mDNSInterval: getIntEnvValue(process.env.P2P_mDNSInterval, 20e3), // 20 seconds connectionsMaxParallelDials: getIntEnvValue( process.env.P2P_connectionsMaxParallelDials, @@ -617,7 +617,7 @@ async function getEnvConfig(isStartup?: boolean): Promise { isStartup, knownUnsafeURLs ), - isBootstrap: getBoolEnvValue(process.env.IS_BOOTSTRAP, false) + isBootstrap: getBoolEnvValue('IS_BOOTSTRAP', false) } if (!previousConfiguration) { From b2bfc648c0b39fd9eaa21149b58b37258195a42e Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 9 Dec 2024 09:42:40 +0000 Subject: [PATCH 5/7] add missing check on republish ddos --- src/components/P2P/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/P2P/index.ts b/src/components/P2P/index.ts index f8333115d..a58b00d35 100644 --- a/src/components/P2P/index.ts +++ b/src/components/P2P/index.ts @@ -779,7 +779,7 @@ export class OceanP2P extends EventEmitter { // related: https://github.com/libp2p/go-libp2p-kad-dht/issues/323 async republishStoredDDOS() { try { - if (!this.db) { + if (!this.db || !this.db.ddo) { P2P_LOGGER.logMessage( `republishStoredDDOS() attempt aborted because there is no database!`, true From 2359529b8c26f83074216fbb2ef986bc88d5d58c Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 9 Dec 2024 14:03:52 +0000 Subject: [PATCH 6/7] recalculate hash if needed --- src/index.ts | 10 +++++++--- src/utils/config.ts | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/index.ts b/src/index.ts index 1796a87b9..a66577c9b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,6 +37,12 @@ declare global { } } +export async function recalculateCodeHash(): Promise { + const __filename = fileURLToPath(import.meta.url) + const __dirname = path.dirname(__filename) + const codeHash = await computeCodebaseHash(__dirname) + return codeHash +} // we have 5 json examples // we should have some DDO class too function loadInitialDDOS(): any[] { @@ -74,9 +80,7 @@ OCEAN_NODE_LOGGER.logMessageWithEmoji( ) const config = await getConfiguration(true, isStartup) -const __filename = fileURLToPath(import.meta.url) -const __dirname = path.dirname(__filename) -config.codeHash = await computeCodebaseHash(__dirname) +config.codeHash = await recalculateCodeHash() OCEAN_NODE_LOGGER.info(`Codebase hash: ${config.codeHash}`) if (!config) { diff --git a/src/utils/config.ts b/src/utils/config.ts index 17d4fc020..afc495c74 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -26,6 +26,7 @@ import { } from '../utils/address.js' import { CONFIG_LOGGER } from './logging/common.js' import { create256Hash } from './crypt.js' +import { recalculateCodeHash } from '..' // usefull for lazy loading and avoid boilerplate on other places let previousConfiguration: OceanNodeConfig = null @@ -475,6 +476,10 @@ export async function getConfiguration( if (!previousConfiguration || forceReload) { previousConfiguration = await getEnvConfig(isStartup) } + if (!previousConfiguration.codeHash) { + previousConfiguration.codeHash = await recalculateCodeHash() + } + return previousConfiguration } From b3c26dcaa9c710c3d6fcdac88c15d569dda6d7e6 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 9 Dec 2024 14:22:11 +0000 Subject: [PATCH 7/7] remove import function, dependency problem --- src/index.ts | 10 +++------- src/utils/config.ts | 8 ++++++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/index.ts b/src/index.ts index a66577c9b..1796a87b9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,12 +37,6 @@ declare global { } } -export async function recalculateCodeHash(): Promise { - const __filename = fileURLToPath(import.meta.url) - const __dirname = path.dirname(__filename) - const codeHash = await computeCodebaseHash(__dirname) - return codeHash -} // we have 5 json examples // we should have some DDO class too function loadInitialDDOS(): any[] { @@ -80,7 +74,9 @@ OCEAN_NODE_LOGGER.logMessageWithEmoji( ) const config = await getConfiguration(true, isStartup) -config.codeHash = await recalculateCodeHash() +const __filename = fileURLToPath(import.meta.url) +const __dirname = path.dirname(__filename) +config.codeHash = await computeCodebaseHash(__dirname) OCEAN_NODE_LOGGER.info(`Codebase hash: ${config.codeHash}`) if (!config) { diff --git a/src/utils/config.ts b/src/utils/config.ts index afc495c74..b15d63df7 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -9,6 +9,7 @@ import { C2DClusterType } from '../@types/C2D.js' import { createFromPrivKey } from '@libp2p/peer-id-factory' import { keys } from '@libp2p/crypto' import { + computeCodebaseHash, DEFAULT_RATE_LIMIT_PER_SECOND, ENVIRONMENT_VARIABLES, EnvVariable, @@ -26,7 +27,8 @@ import { } from '../utils/address.js' import { CONFIG_LOGGER } from './logging/common.js' import { create256Hash } from './crypt.js' -import { recalculateCodeHash } from '..' +import { fileURLToPath } from 'url' +import path from 'path' // usefull for lazy loading and avoid boilerplate on other places let previousConfiguration: OceanNodeConfig = null @@ -477,7 +479,9 @@ export async function getConfiguration( previousConfiguration = await getEnvConfig(isStartup) } if (!previousConfiguration.codeHash) { - previousConfiguration.codeHash = await recalculateCodeHash() + const __filename = fileURLToPath(import.meta.url) + const __dirname = path.dirname(__filename.replace('utils/', '')) + previousConfiguration.codeHash = await computeCodebaseHash(__dirname) } return previousConfiguration