From 38bd5b5835af7cae1caf05e6cbe6d432b454798f Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Mon, 25 Nov 2024 20:25:27 +0900 Subject: [PATCH 01/17] add new ccip load test, compiles --- deployment/environment.go | 1 + integration-tests/load/ccip/ccip_load.go | 170 ++++++++++++++++++ integration-tests/load/ccip/commit.go | 11 ++ .../load/ccip/destination_gun.go | 146 +++++++++++++++ integration-tests/load/ccip/helpers.go | 55 ++++++ integration-tests/testconfig/ccip/config.go | 1 + integration-tests/testconfig/ccip/load.go | 5 + 7 files changed, 389 insertions(+) create mode 100644 integration-tests/load/ccip/ccip_load.go create mode 100644 integration-tests/load/ccip/commit.go create mode 100644 integration-tests/load/ccip/destination_gun.go create mode 100644 integration-tests/load/ccip/helpers.go create mode 100644 integration-tests/testconfig/ccip/load.go diff --git a/deployment/environment.go b/deployment/environment.go index a37622dc3ac..36eea0ee1e7 100644 --- a/deployment/environment.go +++ b/deployment/environment.go @@ -36,6 +36,7 @@ type OnchainClient interface { bind.DeployBackend BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) + BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) } // OffchainClient interacts with the job-distributor diff --git a/integration-tests/load/ccip/ccip_load.go b/integration-tests/load/ccip/ccip_load.go new file mode 100644 index 00000000000..52e4bbdb039 --- /dev/null +++ b/integration-tests/load/ccip/ccip_load.go @@ -0,0 +1,170 @@ +package ccip + +import ( + "fmt" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" + "github.com/smartcontractkit/chainlink-testing-framework/wasp" + "github.com/smartcontractkit/chainlink/deployment" + ccipdeployment "github.com/smartcontractkit/chainlink/deployment/ccip" + tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" + "github.com/stretchr/testify/require" + "math/big" + "sync" + "testing" + "time" +) + +var ( + CommonTestLabels = map[string]string{ + "branch": "ccip_load_crib", + "commit": "ccip_load_crib", + } + wg sync.WaitGroup +) + +// step 1: setup +// Parse the test config, initialize CRIB with configurations defined +// step 2: load +// Use wasp to initiate load +// step 3: parse logs +// Parse all events from the simulated chains, send to Loki +// step 4: teardown +// Stop the chains, cleanup the environment +func LoadTestCCIP(t *testing.T) { + ctx := tests.Context(t) + l := logging.GetTestLogger(t) + + config, err := tc.GetConfig([]string{"Load"}, tc.CCIP) + if err != nil { + t.Fatal(err) + } + l.Info().Interface("loadedTestConfig", config).Msg("Loaded Test Config") + + l.Info().Msg("Starting ccip load test") + l.Info(). + Int("Number of Nodes", *(config.CCIP.Load.NoOfNodes)). + Interface("config", config.CCIP.Load). + Msg("Test Config") + + var env = deployment.Environment{} + // output, err := actions.SetupCCIPHomeChain(l, sethClient, config.CCIP, workerNodes) + // require.NoError(t, err) + // env, err = crib.DeployPrerequisites(output, config.CCIP) + // merge addressbooks + // env, err := crib.DeployCCIPContracts(output, config.CCIP) + + // Need to keep track of the block number for each chain so that event subscription can be done from that block. + startBlocks := make(map[uint64]*uint64) + state, err := ccipdeployment.LoadOnchainState(env) + require.NoError(t, err) + + cfgl := config.Logging.Loki + + // Parse all events from the simulated chains, send to Loki + loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken)) + if err != nil { + l.Error().Err(err).Msg("Failed to create Loki client") + return + } + defer loki.StopNow() + + // Based on the config, initiate a DestinationGun + p := wasp.NewProfile() + for selector, chain := range env.Chains { + latesthdr, err := chain.Client.HeaderByNumber(ctx, nil) + require.NoError(t, err) + block := latesthdr.Number.Uint64() + startBlocks[selector] = &block + + p.Add(wasp.NewGenerator(&wasp.Config{ + T: t, + GenName: "ccipLoad", + LoadType: wasp.RPS, + CallTimeout: 20 * time.Second, + Schedule: wasp.Plain(1, 20*time.Second), + // will need to be divided by number of chains + // this schedule is per generator + // in this example, it would be 1 request per 10seconds per generator (dest chain) + // so if there are 3 generators, it would be 3 requests per 10 seconds over the network + Gun: NewDestinationGun(l, selector, env, state.Chains[selector].Receiver.Address(), loki), + Labels: CommonTestLabels, + LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), + // use the same loki client using `NewLokiClient` with the same config for sending events + })) + } + + _, err = p.Run(true) + + lokiLabels := map[string]string{} + for chainSelector, startBlock := range startBlocks { + wg.Add(1) + go func(chainSelector uint64, startBlock *uint64) { + defer wg.Done() + filterOpts := &bind.FilterOpts{ + Start: *startBlock, + End: nil, // To the latest block + Context: ctx, + } + + offRamp := state.Chains[chainSelector].OffRamp + // Filter CommitReportAccepted events + commitIterator, err := offRamp.FilterCommitReportAccepted(filterOpts) + require.NoError(t, err) + + for commitIterator.Next() { + event := commitIterator.Event + fmt.Printf("CommitReportAccepted event: %+v\n", event) + + blockNum := commitIterator.Event.Raw.BlockNumber + block, err := env.Chains[chainSelector].Client.BlockByNumber(ctx, big.NewInt(int64(blockNum))) + require.NoError(t, err) + timestamp := time.Unix(int64(block.Time()), 0) + + for _, root := range event.MerkleRoots { + lokiLabels = setLokiLabels(root.SourceChainSelector, chainSelector) + + for i := root.MinSeqNr; i <= root.MaxSeqNr; i++ { + // todo: push loki calls to channel? + SendMetricsToLoki(l, loki, lokiLabels, &LokiMetric{ + EventType: committed, + Timestamp: timestamp, + SequenceNumber: i, + }) + } + } + } + + // Filter ExecutionStateChanged events + execIterator, err := state.Chains[chainSelector].OffRamp.FilterExecutionStateChanged(filterOpts, []uint64{chainSelector}, nil, nil) + require.NoError(t, err) + for execIterator.Next() { + event := execIterator.Event + fmt.Printf("ExecutionStateChanged event: %+v\n", event) + + blockNum := execIterator.Event.Raw.BlockNumber + block, err := env.Chains[chainSelector].Client.BlockByNumber(ctx, big.NewInt(int64(blockNum))) + require.NoError(t, err) + timestamp := time.Unix(int64(block.Time()), 0) + + // todo: push loki calls to channel? + lokiLabels = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) + SendMetricsToLoki(l, loki, lokiLabels, &LokiMetric{ + EventType: executed, + Timestamp: timestamp, + GasUsed: execIterator.Event.GasUsed.Uint64(), + SequenceNumber: execIterator.Event.SequenceNumber, + }) + + } + }(chainSelector, startBlock) + } + + wg.Wait() + + // Stop the chains, cleanup the environment + + // crib.StopChains(env) + // crib.StopNodes(env) +} diff --git a/integration-tests/load/ccip/commit.go b/integration-tests/load/ccip/commit.go new file mode 100644 index 00000000000..6c05d5980b2 --- /dev/null +++ b/integration-tests/load/ccip/commit.go @@ -0,0 +1,11 @@ +package ccip + +import ( + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/offramp" +) + +func ProcessCommitReportEvents(destCS uint64, offRamp offramp.OffRamp, filterOpts *bind.FilterOpts) error { + + return err +} diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go new file mode 100644 index 00000000000..455f6dcc186 --- /dev/null +++ b/integration-tests/load/ccip/destination_gun.go @@ -0,0 +1,146 @@ +package ccip + +import ( + "context" + "fmt" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/wasp" + "github.com/smartcontractkit/chainlink/deployment" + ccipdeployment "github.com/smartcontractkit/chainlink/deployment/ccip" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" + "sync/atomic" + "time" +) + +type ChainSelectorPair struct { + src uint64 + dst uint64 +} + +type DestinationGun struct { + l zerolog.Logger + env deployment.Environment + seqNums map[ChainSelectorPair]*atomic.Uint64 + roundNum *atomic.Int32 + chainSelector uint64 + receiver common.Address + loki *wasp.LokiClient +} + +func NewDestinationGun(l zerolog.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, loki *wasp.LokiClient) *DestinationGun { + seqNums := make(map[ChainSelectorPair]*atomic.Uint64) + for _, cs := range env.AllChainSelectorsExcluding([]uint64{chainSelector}) { + + seqNums[ChainSelectorPair{ + src: cs, + dst: chainSelector, + }] = &atomic.Uint64{} + seqNums[ChainSelectorPair{ + src: chainSelector, + dst: cs, + }].Store(1) + } + return &DestinationGun{ + l: l, + env: env, + seqNums: seqNums, + roundNum: &atomic.Int32{}, + chainSelector: chainSelector, + receiver: receiver, + loki: loki, + } +} + +func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { + m.roundNum.Add(1) + requestedRound := m.roundNum.Load() + + waspGroup := fmt.Sprintf("%d-%s", m.chainSelector, "messageOnly") + + state, err := ccipdeployment.LoadOnchainState(m.env) + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + + src, err := m.MustSourceChain() + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + + csPair := ChainSelectorPair{ + src: src, + dst: m.chainSelector, + } + m.seqNums[csPair].Add(1) + m.l.Info(). + Int32("RoundNum", requestedRound). + Uint64("Destination ChainSelector", m.chainSelector). + Uint64("Source ChainSelector", src). + Uint64("SequenceNumber", m.seqNums[csPair].Load()). + Msg("starting transmit") + + r := state.Chains[src].Router + + msg, err := m.GetMessage() + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + + fee, err := r.GetFee( + &bind.CallOpts{Context: context.Background()}, src, msg) + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + if msg.FeeToken == common.HexToAddress("0x0") { + m.env.Chains[src].DeployerKey.Value = fee + defer func() { m.env.Chains[src].DeployerKey.Value = nil }() + } + _, err = r.CcipSend( + m.env.Chains[src].DeployerKey, + m.chainSelector, + msg) + if err != nil { + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + + lokiLabels := setLokiLabels(src, m.chainSelector) + SendMetricsToLoki(m.l, m.loki, lokiLabels, &LokiMetric{ + EventType: transmitted, + Timestamp: time.Now(), + SequenceNumber: m.seqNums[csPair].Load(), + }) + + return &wasp.Response{Failed: false, Group: waspGroup} +} + +// GetCycledClient will return a valid client from a random source chain +func (m *DestinationGun) MustSourceChain() (uint64, error) { + + // TODO: check if this chain has sent a message recently, if so, switch to the next chain + otherCS := m.env.AllChainSelectorsExcluding([]uint64{m.chainSelector}) + if len(otherCS) == 0 { + return 0, fmt.Errorf("no other chains to send from") + } + index := m.roundNum.Load() % int32(len(otherCS)) + return otherCS[index], nil +} + +// GetMessage will return the message to be sent while considering expected load of different messages +// TODO: implement randomness and different types of messages +func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { + rcv, err := utils.ABIEncode(`[{"type":"address"}]`, m.receiver) + if err != nil { + m.l.Error().Err(err).Msg("Error encoding receiver address") + } + + return router.ClientEVM2AnyMessage{ + Receiver: rcv, + Data: common.Hex2Bytes("hello world"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }, nil +} diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go new file mode 100644 index 00000000000..e7afe6883e4 --- /dev/null +++ b/integration-tests/load/ccip/helpers.go @@ -0,0 +1,55 @@ +package ccip + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/wasp" + "github.com/smartcontractkit/chainlink/deployment" + "time" +) + +const ( + transmitted = iota + committed + executed + LokiLoadLabel = "ccip_load_test" + ErrLokiClient = "failed to create Loki client for monitoring" + ErrLokiPush = "failed to push metrics to Loki" +) + +// todo: Have a different struct for commit/exec? +type LokiMetric struct { + EventType int `json:"event_type"` + Timestamp time.Time `json:"timestamp"` + GasUsed uint64 `json:"gas_used"` + SequenceNumber uint64 `json:"sequence_number"` +} + +func GetAddressFromTypeAndVersion(ab deployment.AddressBook, cs uint64, tv string) (common.Address, error) { + allAddr, err := ab.AddressesForChain(cs) + if err != nil { + return common.Address{}, err + } + for addr, tv := range allAddr { + if tv.Type == tv.Type && tv.Version == tv.Version { + return common.HexToAddress(addr), nil + } + } + + return common.Address{}, fmt.Errorf("address not found for chain selector %d and typeAndVersion %s", cs, tv) +} + +func SendMetricsToLoki(l zerolog.Logger, lc *wasp.LokiClient, updatedLabels map[string]string, metrics *LokiMetric) { + if err := lc.HandleStruct(wasp.LabelsMapToModel(updatedLabels), time.Now(), metrics); err != nil { + l.Error().Err(err).Msg(ErrLokiPush) + } +} + +func setLokiLabels(src, dst uint64) map[string]string { + return map[string]string{ + "sourceSelector": fmt.Sprintf("%d", src), + "destinationSelector": fmt.Sprintf("%d", dst), + "testType": LokiLoadLabel, + } +} diff --git a/integration-tests/testconfig/ccip/config.go b/integration-tests/testconfig/ccip/config.go index 70c850fd591..374322041ec 100644 --- a/integration-tests/testconfig/ccip/config.go +++ b/integration-tests/testconfig/ccip/config.go @@ -38,6 +38,7 @@ type Config struct { HomeChainSelector *string `toml:",omitempty"` FeedChainSelector *string `toml:",omitempty"` RMNConfig RMNConfig `toml:",omitempty"` + Load *LoadConfig `toml:",omitempty"` } type RMNConfig struct { diff --git a/integration-tests/testconfig/ccip/load.go b/integration-tests/testconfig/ccip/load.go new file mode 100644 index 00000000000..af2e89db79a --- /dev/null +++ b/integration-tests/testconfig/ccip/load.go @@ -0,0 +1,5 @@ +package ccip + +type LoadConfig struct { + NoOfNodes *int +} From 7f1b1be50035d83648b46dfa72ed72ebd2a72278 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 26 Nov 2024 00:07:50 +0900 Subject: [PATCH 02/17] fix build --- deployment/environment/memory/sim.go | 4 ++++ integration-tests/load/ccip/commit.go | 11 ----------- 2 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 integration-tests/load/ccip/commit.go diff --git a/deployment/environment/memory/sim.go b/deployment/environment/memory/sim.go index c0fba87e2b3..93f77a0d29b 100644 --- a/deployment/environment/memory/sim.go +++ b/deployment/environment/memory/sim.go @@ -80,6 +80,10 @@ func (b *Backend) NonceAt(ctx context.Context, account common.Address, blockNumb return b.Sim.Client().NonceAt(ctx, account, blockNumber) } +func (b *Backend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { + return b.Sim.Client().BlockByNumber(ctx, number) +} + func NewBackend(sim *simulated.Backend) *Backend { if sim == nil { panic("simulated backend is nil") diff --git a/integration-tests/load/ccip/commit.go b/integration-tests/load/ccip/commit.go deleted file mode 100644 index 6c05d5980b2..00000000000 --- a/integration-tests/load/ccip/commit.go +++ /dev/null @@ -1,11 +0,0 @@ -package ccip - -import ( - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/offramp" -) - -func ProcessCommitReportEvents(destCS uint64, offRamp offramp.OffRamp, filterOpts *bind.FilterOpts) error { - - return err -} From 2270338b8579ed2b9ca084b16aea5bb2f442aadb Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 26 Nov 2024 00:13:49 +0900 Subject: [PATCH 03/17] make test runnable --- integration-tests/load/ccip/{ccip_load.go => ccip_test.go} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename integration-tests/load/ccip/{ccip_load.go => ccip_test.go} (99%) diff --git a/integration-tests/load/ccip/ccip_load.go b/integration-tests/load/ccip/ccip_test.go similarity index 99% rename from integration-tests/load/ccip/ccip_load.go rename to integration-tests/load/ccip/ccip_test.go index 52e4bbdb039..d9782218519 100644 --- a/integration-tests/load/ccip/ccip_load.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -32,7 +32,7 @@ var ( // Parse all events from the simulated chains, send to Loki // step 4: teardown // Stop the chains, cleanup the environment -func LoadTestCCIP(t *testing.T) { +func TestCCIP_Load(t *testing.T) { ctx := tests.Context(t) l := logging.GetTestLogger(t) From 5f0ca7454f9b43479766d7b49ad69df754f562e2 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 26 Nov 2024 17:40:26 +0900 Subject: [PATCH 04/17] rename --- integration-tests/load/ccip/ccip_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index d9782218519..3fc130b855d 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -32,7 +32,7 @@ var ( // Parse all events from the simulated chains, send to Loki // step 4: teardown // Stop the chains, cleanup the environment -func TestCCIP_Load(t *testing.T) { +func TestCCIPLoad_RPS(t *testing.T) { ctx := tests.Context(t) l := logging.GetTestLogger(t) From e8a0bb3711e923e9b7c721c76ec675be29de815d Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Wed, 27 Nov 2024 21:38:08 +0900 Subject: [PATCH 05/17] in progress --- integration-tests/load/ccip/ccip_test.go | 12 +++- integration-tests/load/ccip/helpers.go | 70 +++++++++++++++++++ .../ccip-v2-scripts-address-book.json | 1 + .../ccip-v2-scripts-nodes-details.json | 1 + 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json create mode 100644 integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 3fc130b855d..79c87106960 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -21,7 +21,9 @@ var ( "branch": "ccip_load_crib", "commit": "ccip_load_crib", } - wg sync.WaitGroup + wg sync.WaitGroup + abPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json" + nodeIdsPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-node-details.json" ) // step 1: setup @@ -48,6 +50,8 @@ func TestCCIPLoad_RPS(t *testing.T) { Interface("config", config.CCIP.Load). Msg("Test Config") + var env = generateEnvironment() + var env = deployment.Environment{} // output, err := actions.SetupCCIPHomeChain(l, sethClient, config.CCIP, workerNodes) // require.NoError(t, err) @@ -168,3 +172,9 @@ func TestCCIPLoad_RPS(t *testing.T) { // crib.StopChains(env) // crib.StopNodes(env) } + +func generateEnvironment() { + ab := readAddressBook(abPath) + nIds := readNodeIds(nodeIdsPath) + +} diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go index e7afe6883e4..0ef76126967 100644 --- a/integration-tests/load/ccip/helpers.go +++ b/integration-tests/load/ccip/helpers.go @@ -1,11 +1,15 @@ package ccip import ( + "encoding/json" "fmt" "github.com/ethereum/go-ethereum/common" "github.com/rs/zerolog" "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" + "io" + "os" "time" ) @@ -53,3 +57,69 @@ func setLokiLabels(src, dst uint64) map[string]string { "testType": LokiLoadLabel, } } + +func readFile(inputDir string, fileName string) []byte { + file, err := os.Open(fmt.Sprintf("%s/%s", inputDir, fileName)) + if err != nil { + fmt.Println("Error opening file:", err) + panic(err) + } + defer file.Close() + + // Read the file's content into a byte slice + byteValue, err := io.ReadAll(file) + if err != nil { + fmt.Println("Error reading file:", err) + panic(err) + } + return byteValue +} + +func ReadAddressBook(inputDir string) *deployment.AddressBookMap { + byteValue := readFile(inputDir, AddressBookFileName) + + var result map[uint64]map[string]deployment.TypeAndVersion + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + // Print the deserialized map + fmt.Println(result) + return deployment.NewMemoryAddressBookFromMap(result) +} + +func ReadNodesDetails(inputDir string) NodesDetails { + byteValue := readFile(inputDir, NodesDetailsFileName) + + var result NodesDetails + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + // Print the deserialized map + fmt.Println(result) + return result +} + +func NewDeployEnvironmentFromCribOutput(lggr logger.Logger, ab deployment.AddressBook, nodeIDs []string) (*deployment.Environment, error) { + chains, err := devenv.NewChains(lggr, output.Chains) + if err != nil { + return nil, err + } + return deployment.NewEnvironment( + "Crib Environment", + lggr, + ab, + chains, + nodeIDs, + nil, // todo: populate the offchain client using output.DON + ), nil +} diff --git a/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json new file mode 100644 index 00000000000..e4b2672cb5f --- /dev/null +++ b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json @@ -0,0 +1 @@ +{"12922642891491394802":{"0x05Aa229Aec102f78CE0E852A812a388F076Aa555":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x0D4ff719551E23185Aeb16FFbF2ABEbB90635942":{"Type":"TestRouter","Version":"1.2.0"},"0x0f5D1ef48f12b6f691401bfe88c2037c690a6afe":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x2dE080e97B0caE9825375D31f5D0eD5751fDf16D":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x2fc631e4B3018258759C52AF169200213e84ABab":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x5C7c905B505f0Cf40Ab6600d05e677F717916F6B":{"Type":"Router","Version":"1.2.0"},"0x63cf2Cd54fE91e3545D1379abf5bfd194545259d":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x712516e61C8B383dF4A63CFe83d7701Bce54B03e":{"Type":"LinkToken","Version":"1.0.0"},"0x71C95911E9a5D330f4D621842EC243EE1343292e":{"Type":"PriceFeed","Version":"1.0.0"},"0x73eccD6288e117cAcA738BDAD4FEC51312166C1A":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x8464135c8F25Da09e49BC8782676a84730C318bC":{"Type":"PriceFeed","Version":"1.0.0"},"0x85C5Dd61585773423e378146D4bEC6f8D149E248":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x948B3c65b89DF0B4894ABE91E6D02FE579834F8F":{"Type":"WETH9","Version":"1.0.0"},"0xAfe1b5bdEbD4ae65AF2024738bf0735fbb65d44b":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xC6bA8C3233eCF65B761049ef63466945c362EdD2":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0xbCF26943C0197d2eE0E5D05c716Be60cc2761508":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0xcA03Dc4665A8C3603cb4Fd5Ce71Af9649dC00d44":{"Type":"RBACTimelock","Version":"1.0.0"},"0xe6b98F104c1BEf218F3893ADab4160Dc73Eb8367":{"Type":"ARMProxy","Version":"1.0.0"},"0xfbAb4aa40C202E4e80390171E82379824f7372dd":{"Type":"NonceManager","Version":"1.6.0-dev"}},"3379446385462418246":{"0x09635F643e140090A9A8Dcd712eD6285858ceBef":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x0B306BF915C4d645ff596e518fAf3F9669b97016":{"Type":"LinkToken","Version":"1.0.0"},"0x1613beB3B2C4f22Ee086B2b38C1476A3cE7f78E8":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6":{"Type":"CCIPHome","Version":"1.6.0-dev"},"0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x3Aa5ebB10DC797CAC828524e59A333d0A371443c":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0x4A679253410272dd5232B3Ff7cF5dbB88f295319":{"Type":"RBACTimelock","Version":"1.0.0"},"0x59b670e9fA9D0A427751Af201D676719a970857b":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x67d269191c92Caf3cD7723F116c85e6E9bf55933":{"Type":"ARMProxy","Version":"1.0.0"},"0x7a2088a1bFc9d81c55368AE168C2C02570cB814F":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x851356ae760d987E095750cCeb3bC6014560891C":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x8A791620dd6260079BF849Dc5567aDC3F2FdC318":{"Type":"RMNHome","Version":"1.6.0-dev"},"0x9A676e781A523b5d0C0e43731313A708CB607508":{"Type":"WETH9","Version":"1.0.0"},"0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0x9E545E3C0baAB3E08CdfD552C960A1050f373042":{"Type":"NonceManager","Version":"1.6.0-dev"},"0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E":{"Type":"Router","Version":"1.2.0"},"0xa513E6E4b8f2a923D98304ec87F64353C4D5C853":{"Type":"CapabilitiesRegistry","Version":"1.0.0"},"0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690":{"Type":"TestRouter","Version":"1.2.0"}}} diff --git a/integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json new file mode 100644 index 00000000000..477ae0527b1 --- /dev/null +++ b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json @@ -0,0 +1 @@ +{"NodeIDs":["node_2URuou3RXmtZu5gLQX8qd","node_m9TTQbUxBx3WjDEjmpVDL","node_4FiKVPtuQjCTvHnS7QpES","node_A4VTgecDwMoG2YYicyjuG","node_jQFpzXDadzaADq147nThS"]} From d5b7c47bd5b2ca165f1316569f1ef2297857357f Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 17 Dec 2024 15:21:26 +0800 Subject: [PATCH 06/17] build --- deployment/environment.go | 1 - deployment/environment/memory/sim.go | 4 - integration-tests/load/ccip/ccip_test.go | 57 +++++-------- .../load/ccip/destination_gun.go | 25 +++--- integration-tests/load/ccip/helpers.go | 84 +++++++------------ 5 files changed, 63 insertions(+), 108 deletions(-) diff --git a/deployment/environment.go b/deployment/environment.go index 36eea0ee1e7..a37622dc3ac 100644 --- a/deployment/environment.go +++ b/deployment/environment.go @@ -36,7 +36,6 @@ type OnchainClient interface { bind.DeployBackend BalanceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (*big.Int, error) NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error) - BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) } // OffchainClient interacts with the job-distributor diff --git a/deployment/environment/memory/sim.go b/deployment/environment/memory/sim.go index 93f77a0d29b..c0fba87e2b3 100644 --- a/deployment/environment/memory/sim.go +++ b/deployment/environment/memory/sim.go @@ -80,10 +80,6 @@ func (b *Backend) NonceAt(ctx context.Context, account common.Address, blockNumb return b.Sim.Client().NonceAt(ctx, account, blockNumber) } -func (b *Backend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { - return b.Sim.Client().BlockByNumber(ctx, number) -} - func NewBackend(sim *simulated.Backend) *Backend { if sim == nil { panic("simulated backend is nil") diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 79c87106960..6f364252fb9 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -3,11 +3,10 @@ package ccip import ( "fmt" "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - "github.com/smartcontractkit/chainlink-testing-framework/lib/logging" "github.com/smartcontractkit/chainlink-testing-framework/wasp" - "github.com/smartcontractkit/chainlink/deployment" - ccipdeployment "github.com/smartcontractkit/chainlink/deployment/ccip" + ccipchangeset "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" "github.com/stretchr/testify/require" "math/big" @@ -21,9 +20,7 @@ var ( "branch": "ccip_load_crib", "commit": "ccip_load_crib", } - wg sync.WaitGroup - abPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json" - nodeIdsPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-node-details.json" + wg sync.WaitGroup ) // step 1: setup @@ -36,32 +33,24 @@ var ( // Stop the chains, cleanup the environment func TestCCIPLoad_RPS(t *testing.T) { ctx := tests.Context(t) - l := logging.GetTestLogger(t) + lggr := logger.Test(t) config, err := tc.GetConfig([]string{"Load"}, tc.CCIP) - if err != nil { - t.Fatal(err) - } - l.Info().Interface("loadedTestConfig", config).Msg("Loaded Test Config") - - l.Info().Msg("Starting ccip load test") - l.Info(). - Int("Number of Nodes", *(config.CCIP.Load.NoOfNodes)). - Interface("config", config.CCIP.Load). - Msg("Test Config") + require.NoError(t, err) + lggr.Infof("loaded test config: %+v", config) - var env = generateEnvironment() + lggr.Info("starting ccip load test") + lggr.Infof("Number of nodes: %d", *(config.CCIP.Load.NoOfNodes)) + lggr.Infof("config: %+v", config.CCIP.Load) + lggr.Info("Test Config") - var env = deployment.Environment{} - // output, err := actions.SetupCCIPHomeChain(l, sethClient, config.CCIP, workerNodes) - // require.NoError(t, err) - // env, err = crib.DeployPrerequisites(output, config.CCIP) - // merge addressbooks - // env, err := crib.DeployCCIPContracts(output, config.CCIP) + env, err := CreateEnvironmentFromCribOutput(t, lggr) + require.NoError(t, err) + require.NotNil(t, env) // Need to keep track of the block number for each chain so that event subscription can be done from that block. startBlocks := make(map[uint64]*uint64) - state, err := ccipdeployment.LoadOnchainState(env) + state, err := ccipchangeset.LoadOnchainState(*env) require.NoError(t, err) cfgl := config.Logging.Loki @@ -69,7 +58,7 @@ func TestCCIPLoad_RPS(t *testing.T) { // Parse all events from the simulated chains, send to Loki loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken)) if err != nil { - l.Error().Err(err).Msg("Failed to create Loki client") + env.Logger.Error("failed to create loki client") return } defer loki.StopNow() @@ -92,7 +81,7 @@ func TestCCIPLoad_RPS(t *testing.T) { // this schedule is per generator // in this example, it would be 1 request per 10seconds per generator (dest chain) // so if there are 3 generators, it would be 3 requests per 10 seconds over the network - Gun: NewDestinationGun(l, selector, env, state.Chains[selector].Receiver.Address(), loki), + Gun: NewDestinationGun(env.Logger, selector, env, state.Chains[selector].Receiver.Address(), loki), Labels: CommonTestLabels, LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), // use the same loki client using `NewLokiClient` with the same config for sending events @@ -122,9 +111,9 @@ func TestCCIPLoad_RPS(t *testing.T) { fmt.Printf("CommitReportAccepted event: %+v\n", event) blockNum := commitIterator.Event.Raw.BlockNumber - block, err := env.Chains[chainSelector].Client.BlockByNumber(ctx, big.NewInt(int64(blockNum))) + header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) require.NoError(t, err) - timestamp := time.Unix(int64(block.Time()), 0) + timestamp := time.Unix(int64(header.Time()), 0) for _, root := range event.MerkleRoots { lokiLabels = setLokiLabels(root.SourceChainSelector, chainSelector) @@ -148,9 +137,9 @@ func TestCCIPLoad_RPS(t *testing.T) { fmt.Printf("ExecutionStateChanged event: %+v\n", event) blockNum := execIterator.Event.Raw.BlockNumber - block, err := env.Chains[chainSelector].Client.BlockByNumber(ctx, big.NewInt(int64(blockNum))) + header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) require.NoError(t, err) - timestamp := time.Unix(int64(block.Time()), 0) + timestamp := time.Unix(int64(header.Time), 0) // todo: push loki calls to channel? lokiLabels = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) @@ -172,9 +161,3 @@ func TestCCIPLoad_RPS(t *testing.T) { // crib.StopChains(env) // crib.StopNodes(env) } - -func generateEnvironment() { - ab := readAddressBook(abPath) - nIds := readNodeIds(nodeIdsPath) - -} diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index 455f6dcc186..3ba9e2416ad 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -5,12 +5,13 @@ import ( "fmt" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/deployment" - ccipdeployment "github.com/smartcontractkit/chainlink/deployment/ccip" + ccipchangeset "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" + "sync/atomic" "time" ) @@ -21,7 +22,7 @@ type ChainSelectorPair struct { } type DestinationGun struct { - l zerolog.Logger + l logger.Logger env deployment.Environment seqNums map[ChainSelectorPair]*atomic.Uint64 roundNum *atomic.Int32 @@ -30,7 +31,7 @@ type DestinationGun struct { loki *wasp.LokiClient } -func NewDestinationGun(l zerolog.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, loki *wasp.LokiClient) *DestinationGun { +func NewDestinationGun(l logger.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, loki *wasp.LokiClient) *DestinationGun { seqNums := make(map[ChainSelectorPair]*atomic.Uint64) for _, cs := range env.AllChainSelectorsExcluding([]uint64{chainSelector}) { @@ -60,7 +61,7 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { waspGroup := fmt.Sprintf("%d-%s", m.chainSelector, "messageOnly") - state, err := ccipdeployment.LoadOnchainState(m.env) + state, err := ccipchangeset.LoadOnchainState(m.env) if err != nil { return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } @@ -75,12 +76,7 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { dst: m.chainSelector, } m.seqNums[csPair].Add(1) - m.l.Info(). - Int32("RoundNum", requestedRound). - Uint64("Destination ChainSelector", m.chainSelector). - Uint64("Source ChainSelector", src). - Uint64("SequenceNumber", m.seqNums[csPair].Load()). - Msg("starting transmit") + m.l.Infow("Starting transmit with ", "RoundNum", requestedRound, "Destination ChainSelector", m.chainSelector, "Source ChainSelector", src, "SequenceNumber", m.seqNums[csPair].Load()) r := state.Chains[src].Router @@ -116,10 +112,10 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { return &wasp.Response{Failed: false, Group: waspGroup} } -// GetCycledClient will return a valid client from a random source chain +// MustSourceChain will return a chain selector to send a message from func (m *DestinationGun) MustSourceChain() (uint64, error) { - // TODO: check if this chain has sent a message recently, if so, switch to the next chain + // TODO: make this smarter by checking if this chain has sent a message recently, if so, switch to the next chain otherCS := m.env.AllChainSelectorsExcluding([]uint64{m.chainSelector}) if len(otherCS) == 0 { return 0, fmt.Errorf("no other chains to send from") @@ -133,7 +129,8 @@ func (m *DestinationGun) MustSourceChain() (uint64, error) { func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { rcv, err := utils.ABIEncode(`[{"type":"address"}]`, m.receiver) if err != nil { - m.l.Error().Err(err).Msg("Error encoding receiver address") + m.l.Error("Error encoding receiver address") + return router.ClientEVM2AnyMessage{}, err } return router.ClientEVM2AnyMessage{ diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go index 0ef76126967..0df27431963 100644 --- a/integration-tests/load/ccip/helpers.go +++ b/integration-tests/load/ccip/helpers.go @@ -1,15 +1,17 @@ package ccip import ( + "context" "encoding/json" "fmt" - "github.com/ethereum/go-ethereum/common" - "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/environment/devenv" + "github.com/stretchr/testify/require" "io" "os" + "testing" "time" ) @@ -17,9 +19,12 @@ const ( transmitted = iota committed executed - LokiLoadLabel = "ccip_load_test" - ErrLokiClient = "failed to create Loki client for monitoring" - ErrLokiPush = "failed to push metrics to Loki" + LokiLoadLabel = "ccip_load_test" + ErrLokiClient = "failed to create Loki client for monitoring" + ErrLokiPush = "failed to push metrics to Loki" + abPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json" + nodeIdsPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-node-details.json" + chainConfigPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-chains-details.json" ) // todo: Have a different struct for commit/exec? @@ -30,36 +35,23 @@ type LokiMetric struct { SequenceNumber uint64 `json:"sequence_number"` } -func GetAddressFromTypeAndVersion(ab deployment.AddressBook, cs uint64, tv string) (common.Address, error) { - allAddr, err := ab.AddressesForChain(cs) - if err != nil { - return common.Address{}, err - } - for addr, tv := range allAddr { - if tv.Type == tv.Type && tv.Version == tv.Version { - return common.HexToAddress(addr), nil - } - } - - return common.Address{}, fmt.Errorf("address not found for chain selector %d and typeAndVersion %s", cs, tv) -} - -func SendMetricsToLoki(l zerolog.Logger, lc *wasp.LokiClient, updatedLabels map[string]string, metrics *LokiMetric) { +func SendMetricsToLoki(l logger.Logger, lc *wasp.LokiClient, updatedLabels map[string]string, metrics *LokiMetric) { if err := lc.HandleStruct(wasp.LabelsMapToModel(updatedLabels), time.Now(), metrics); err != nil { - l.Error().Err(err).Msg(ErrLokiPush) + l.Error(ErrLokiPush) } } func setLokiLabels(src, dst uint64) map[string]string { return map[string]string{ - "sourceSelector": fmt.Sprintf("%d", src), + "sourceEvmChainId": fmt.Sprintf("%d", src), + "destEvmChainId": fmt.Sprintf("%d", src), "destinationSelector": fmt.Sprintf("%d", dst), "testType": LokiLoadLabel, } } -func readFile(inputDir string, fileName string) []byte { - file, err := os.Open(fmt.Sprintf("%s/%s", inputDir, fileName)) +func readFile(filePath string) []byte { + file, err := os.Open(filePath) if err != nil { fmt.Println("Error opening file:", err) panic(err) @@ -75,42 +67,28 @@ func readFile(inputDir string, fileName string) []byte { return byteValue } -func ReadAddressBook(inputDir string) *deployment.AddressBookMap { - byteValue := readFile(inputDir, AddressBookFileName) - - var result map[uint64]map[string]deployment.TypeAndVersion +func readFromFile[T []string | *deployment.AddressBookMap | []devenv.ChainConfig](t *testing.T, inputDir string) T { + byteValue := readFile(inputDir) + var result T // Unmarshal the JSON into the map err := json.Unmarshal(byteValue, &result) - if err != nil { - fmt.Println("Error unmarshalling JSON:", err) - panic(err) - } + require.NoError(t, err) // Print the deserialized map fmt.Println(result) - return deployment.NewMemoryAddressBookFromMap(result) + return result } -func ReadNodesDetails(inputDir string) NodesDetails { - byteValue := readFile(inputDir, NodesDetailsFileName) +func CreateEnvironmentFromCribOutput(t *testing.T, lggr logger.Logger) (*deployment.Environment, error) { + ab := readFromFile[*deployment.AddressBookMap](t, abPath) + nodeIds := readFromFile[[]string](t, nodeIdsPath) + chainDetails := readFromFile[[]devenv.ChainConfig](t, chainConfigPath) - var result NodesDetails - - // Unmarshal the JSON into the map - err := json.Unmarshal(byteValue, &result) - if err != nil { - fmt.Println("Error unmarshalling JSON:", err) - panic(err) - } - - // Print the deserialized map - fmt.Println(result) - return result -} + // todo: make sure to call chainDetails.SetDeployerKey() for each chain + // where private keys should be stored in env vars or .toml -func NewDeployEnvironmentFromCribOutput(lggr logger.Logger, ab deployment.AddressBook, nodeIDs []string) (*deployment.Environment, error) { - chains, err := devenv.NewChains(lggr, output.Chains) + chains, err := devenv.NewChains(lggr, chainDetails) if err != nil { return nil, err } @@ -119,7 +97,9 @@ func NewDeployEnvironmentFromCribOutput(lggr logger.Logger, ab deployment.Addres lggr, ab, chains, - nodeIDs, - nil, // todo: populate the offchain client using output.DON + nodeIds, + nil, + func() context.Context { return context.Background() }, + deployment.XXXGenerateTestOCRSecrets(), ), nil } From 3b09526bd4d619c67f1056437955c50e2ce745eb Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 17 Dec 2024 19:36:18 +0800 Subject: [PATCH 07/17] load test transmitting --- deployment/ccip/changeset/state.go | 2 ++ integration-tests/load/ccip/ccip_test.go | 34 ++++++++----------- .../load/ccip/destination_gun.go | 9 ++--- integration-tests/testconfig/ccip/ccip.toml | 5 +++ integration-tests/testconfig/ccip/load.go | 3 +- 5 files changed, 26 insertions(+), 27 deletions(-) diff --git a/deployment/ccip/changeset/state.go b/deployment/ccip/changeset/state.go index aa07168a6d2..0be6a1b3ed9 100644 --- a/deployment/ccip/changeset/state.go +++ b/deployment/ccip/changeset/state.go @@ -445,6 +445,8 @@ func LoadChainState(chain deployment.Chain, addresses map[string]deployment.Type return state, err } state.OffRamp = offRamp + case deployment.NewTypeAndVersion(ARMProxy, deployment.Version1_6_0_dev).String(): + fallthrough case deployment.NewTypeAndVersion(ARMProxy, deployment.Version1_0_0).String(): armProxy, err := rmn_proxy_contract.NewRMNProxy(common.HexToAddress(address), chain.Client) if err != nil { diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 6f364252fb9..cdc98dc133a 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -7,6 +7,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink-testing-framework/wasp" ccipchangeset "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" + crib "github.com/smartcontractkit/chainlink/deployment/environment/crib" tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" "github.com/stretchr/testify/require" "math/big" @@ -23,6 +24,8 @@ var ( wg sync.WaitGroup ) +const CRIB_DIRECTORY = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" + // step 1: setup // Parse the test config, initialize CRIB with configurations defined // step 2: load @@ -37,14 +40,12 @@ func TestCCIPLoad_RPS(t *testing.T) { config, err := tc.GetConfig([]string{"Load"}, tc.CCIP) require.NoError(t, err) - lggr.Infof("loaded test config: %+v", config) + lggr.Infof("loaded ccip test config: %+v", config.CCIP.Load) - lggr.Info("starting ccip load test") - lggr.Infof("Number of nodes: %d", *(config.CCIP.Load.NoOfNodes)) - lggr.Infof("config: %+v", config.CCIP.Load) - lggr.Info("Test Config") + cribEnv := crib.NewDevspaceEnvFromStateDir(CRIB_DIRECTORY) - env, err := CreateEnvironmentFromCribOutput(t, lggr) + cribDeployOutput := cribEnv.GetConfig() + env, err := crib.NewDeployEnvironmentFromCribOutput(lggr, cribDeployOutput) require.NoError(t, err) require.NotNil(t, env) @@ -53,17 +54,12 @@ func TestCCIPLoad_RPS(t *testing.T) { state, err := ccipchangeset.LoadOnchainState(*env) require.NoError(t, err) - cfgl := config.Logging.Loki - // Parse all events from the simulated chains, send to Loki - loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken)) - if err != nil { - env.Logger.Error("failed to create loki client") - return - } + loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(config.CCIP.Load.LokiEndpoint, nil, nil, nil)) + require.NoError(t, err) defer loki.StopNow() - // Based on the config, initiate a DestinationGun + // Based on the config, initialize DestinationGun p := wasp.NewProfile() for selector, chain := range env.Chains { latesthdr, err := chain.Client.HeaderByNumber(ctx, nil) @@ -81,9 +77,9 @@ func TestCCIPLoad_RPS(t *testing.T) { // this schedule is per generator // in this example, it would be 1 request per 10seconds per generator (dest chain) // so if there are 3 generators, it would be 3 requests per 10 seconds over the network - Gun: NewDestinationGun(env.Logger, selector, env, state.Chains[selector].Receiver.Address(), loki), + Gun: NewDestinationGun(env.Logger, selector, *env, state.Chains[selector].Receiver.Address(), loki), Labels: CommonTestLabels, - LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), + LokiConfig: wasp.NewLokiConfig(config.CCIP.Load.LokiEndpoint, nil, nil, nil), // use the same loki client using `NewLokiClient` with the same config for sending events })) } @@ -113,14 +109,14 @@ func TestCCIPLoad_RPS(t *testing.T) { blockNum := commitIterator.Event.Raw.BlockNumber header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) require.NoError(t, err) - timestamp := time.Unix(int64(header.Time()), 0) + timestamp := time.Unix(int64(header.Time), 0) for _, root := range event.MerkleRoots { lokiLabels = setLokiLabels(root.SourceChainSelector, chainSelector) for i := root.MinSeqNr; i <= root.MaxSeqNr; i++ { // todo: push loki calls to channel? - SendMetricsToLoki(l, loki, lokiLabels, &LokiMetric{ + SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ EventType: committed, Timestamp: timestamp, SequenceNumber: i, @@ -143,7 +139,7 @@ func TestCCIPLoad_RPS(t *testing.T) { // todo: push loki calls to channel? lokiLabels = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) - SendMetricsToLoki(l, loki, lokiLabels, &LokiMetric{ + SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ EventType: executed, Timestamp: timestamp, GasUsed: execIterator.Event.GasUsed.Uint64(), diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index 3ba9e2416ad..acc33b93bdc 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -11,8 +11,7 @@ import ( ccipchangeset "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" - - "sync/atomic" + "go.uber.org/atomic" "time" ) @@ -38,11 +37,7 @@ func NewDestinationGun(l logger.Logger, chainSelector uint64, env deployment.Env seqNums[ChainSelectorPair{ src: cs, dst: chainSelector, - }] = &atomic.Uint64{} - seqNums[ChainSelectorPair{ - src: chainSelector, - dst: cs, - }].Store(1) + }] = atomic.NewUint64(1) } return &DestinationGun{ l: l, diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index 3f4ba43c48c..b3e3b944b77 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -240,3 +240,8 @@ addresses_to_fund = [ [Seth] # Seth specific configuration, no need for generating ephemeral addresses for ccip-tests. ephemeral_addresses_number = 0 + + +[Load.CCIP.Load] +LokiEndpoint = "http://localhost:3030/loki/api/v1/push" +NoOfNodes = 4 diff --git a/integration-tests/testconfig/ccip/load.go b/integration-tests/testconfig/ccip/load.go index af2e89db79a..5b226d6e7eb 100644 --- a/integration-tests/testconfig/ccip/load.go +++ b/integration-tests/testconfig/ccip/load.go @@ -1,5 +1,6 @@ package ccip type LoadConfig struct { - NoOfNodes *int + NoOfNodes *int + LokiEndpoint *string } From c53e8d40cb5d25c7ccaaa1f76d5b4d124ab1e1eb Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Wed, 18 Dec 2024 18:48:32 +0800 Subject: [PATCH 08/17] load test, have transaction, need to use event watcher --- deployment/environment/crib/env.go | 18 +++++++++++++-- deployment/environment/devenv/chain.go | 1 - integration-tests/load/ccip/ccip_test.go | 23 ++++++++++++++----- .../load/ccip/destination_gun.go | 12 +++++++--- integration-tests/load/ccip/helpers.go | 17 ++++++++++---- 5 files changed, 55 insertions(+), 16 deletions(-) diff --git a/deployment/environment/crib/env.go b/deployment/environment/crib/env.go index 3af1acaf754..281565480df 100644 --- a/deployment/environment/crib/env.go +++ b/deployment/environment/crib/env.go @@ -1,5 +1,7 @@ package crib +import "fmt" + const ( AddressBookFileName = "ccip-v2-scripts-address-book.json" NodesDetailsFileName = "ccip-v2-scripts-nodes-details.json" @@ -16,15 +18,27 @@ func NewDevspaceEnvFromStateDir(envStateDir string) CRIBEnv { } } -func (c CRIBEnv) GetConfig() DeployOutput { +func (c CRIBEnv) GetConfig(deployerKeys map[uint64]string) (DeployOutput, error) { reader := NewOutputReader(c.envStateDir) nodesDetails := reader.ReadNodesDetails() chainConfigs := reader.ReadChainConfigs() + for i, chain := range chainConfigs { + key, ok := deployerKeys[chain.ChainID] + if !ok { + return DeployOutput{}, fmt.Errorf("could not find deployer key for %s", key) + } + err := chain.SetDeployerKey(&key) + if err != nil { + return DeployOutput{}, err + } + chainConfigs[i] = chain + } + return DeployOutput{ AddressBook: reader.ReadAddressBook(), NodeIDs: nodesDetails.NodeIDs, Chains: chainConfigs, - } + }, nil } type RPC struct { diff --git a/deployment/environment/devenv/chain.go b/deployment/environment/devenv/chain.go index 265a6647050..19d58882e03 100644 --- a/deployment/environment/devenv/chain.go +++ b/deployment/environment/devenv/chain.go @@ -68,7 +68,6 @@ func (c *ChainConfig) SetDeployerKey(pvtKeyStr *string) error { if err != nil { return fmt.Errorf("failed to create transactor: %w", err) } - fmt.Printf("Deployer Address: %s for chain id %d\n", deployer.From.Hex(), c.ChainID) c.DeployerKey = deployer return nil } diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index cdc98dc133a..8f5233fcfda 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -21,7 +21,11 @@ var ( "branch": "ccip_load_crib", "commit": "ccip_load_crib", } - wg sync.WaitGroup + wg sync.WaitGroup + SIM_CHAIN_PRIVATE_KEYS = map[uint64]string{ + 1337: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + 2337: "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", + } ) const CRIB_DIRECTORY = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" @@ -44,7 +48,8 @@ func TestCCIPLoad_RPS(t *testing.T) { cribEnv := crib.NewDevspaceEnvFromStateDir(CRIB_DIRECTORY) - cribDeployOutput := cribEnv.GetConfig() + cribDeployOutput, err := cribEnv.GetConfig(SIM_CHAIN_PRIVATE_KEYS) + require.NoError(t, err) env, err := crib.NewDeployEnvironmentFromCribOutput(lggr, cribDeployOutput) require.NoError(t, err) require.NotNil(t, env) @@ -71,8 +76,8 @@ func TestCCIPLoad_RPS(t *testing.T) { T: t, GenName: "ccipLoad", LoadType: wasp.RPS, - CallTimeout: 20 * time.Second, - Schedule: wasp.Plain(1, 20*time.Second), + CallTimeout: 5 * time.Second, + Schedule: wasp.Plain(1, 5*time.Second), // will need to be divided by number of chains // this schedule is per generator // in this example, it would be 1 request per 10seconds per generator (dest chain) @@ -91,6 +96,8 @@ func TestCCIPLoad_RPS(t *testing.T) { wg.Add(1) go func(chainSelector uint64, startBlock *uint64) { defer wg.Done() + lggr.Infow("Starting to query for events on ", "chainSelector", chainSelector, "startblock", startBlock) + filterOpts := &bind.FilterOpts{ Start: *startBlock, End: nil, // To the latest block @@ -102,6 +109,8 @@ func TestCCIPLoad_RPS(t *testing.T) { commitIterator, err := offRamp.FilterCommitReportAccepted(filterOpts) require.NoError(t, err) + fmt.Printf("Events on commitIterator %+v", commitIterator) + for commitIterator.Next() { event := commitIterator.Event fmt.Printf("CommitReportAccepted event: %+v\n", event) @@ -112,7 +121,8 @@ func TestCCIPLoad_RPS(t *testing.T) { timestamp := time.Unix(int64(header.Time), 0) for _, root := range event.MerkleRoots { - lokiLabels = setLokiLabels(root.SourceChainSelector, chainSelector) + lokiLabels, err = setLokiLabels(root.SourceChainSelector, chainSelector) + require.NoError(t, err) for i := root.MinSeqNr; i <= root.MaxSeqNr; i++ { // todo: push loki calls to channel? @@ -138,7 +148,8 @@ func TestCCIPLoad_RPS(t *testing.T) { timestamp := time.Unix(int64(header.Time), 0) // todo: push loki calls to channel? - lokiLabels = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) + lokiLabels, err = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) + require.NoError(t, err) SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ EventType: executed, Timestamp: timestamp, diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index acc33b93bdc..c517cdd0ca2 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -81,23 +81,29 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { } fee, err := r.GetFee( - &bind.CallOpts{Context: context.Background()}, src, msg) + &bind.CallOpts{Context: context.Background()}, m.chainSelector, msg) if err != nil { + m.l.Errorw("could not get fee ", "dstChainSelector", m.chainSelector, "msg", msg, "fee", fee) return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } + m.l.Debugw("setting fee for ", "srcChain", src, "dstChain", m.chainSelector, "fee", fee, "msg", msg) if msg.FeeToken == common.HexToAddress("0x0") { m.env.Chains[src].DeployerKey.Value = fee defer func() { m.env.Chains[src].DeployerKey.Value = nil }() } - _, err = r.CcipSend( + tx, err := r.CcipSend( m.env.Chains[src].DeployerKey, m.chainSelector, msg) if err != nil { + m.l.Errorw("execution reverted from ", "sourceChain", src, "destchain", m.chainSelector, "err", err, "tx", tx) return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } - lokiLabels := setLokiLabels(src, m.chainSelector) + lokiLabels, err := setLokiLabels(src, m.chainSelector) + if err != nil { + m.l.Errorw("Failed setting loki labels", "error", err) + } SendMetricsToLoki(m.l, m.loki, lokiLabels, &LokiMetric{ EventType: transmitted, Timestamp: time.Now(), diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go index 0df27431963..faeede61e7a 100644 --- a/integration-tests/load/ccip/helpers.go +++ b/integration-tests/load/ccip/helpers.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + chainselectors "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/deployment" @@ -41,13 +42,21 @@ func SendMetricsToLoki(l logger.Logger, lc *wasp.LokiClient, updatedLabels map[s } } -func setLokiLabels(src, dst uint64) map[string]string { +func setLokiLabels(src, dst uint64) (map[string]string, error) { + srcChainId, err := chainselectors.GetChainIDFromSelector(src) + if err != nil { + return nil, err + } + dstChainId, err := chainselectors.GetChainIDFromSelector(dst) + if err != nil { + return nil, err + } return map[string]string{ - "sourceEvmChainId": fmt.Sprintf("%d", src), - "destEvmChainId": fmt.Sprintf("%d", src), + "sourceEvmChainId": fmt.Sprintf("%s", srcChainId), + "destEvmChainId": fmt.Sprintf("%s", dstChainId), "destinationSelector": fmt.Sprintf("%d", dst), "testType": LokiLoadLabel, - } + }, nil } func readFile(filePath string) []byte { From 61d9da0d2382620bccecef0d2d43dba6079faf1b Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Wed, 8 Jan 2025 20:33:00 +0800 Subject: [PATCH 09/17] progress, going to rebase --- integration-tests/load/ccip/ccip_test.go | 31 +++++++++- .../load/ccip/destination_gun.go | 59 ++++++++++++++++--- integration-tests/testconfig/ccip/ccip.toml | 2 +- 3 files changed, 82 insertions(+), 10 deletions(-) diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 8f5233fcfda..1ee1049b502 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -3,6 +3,7 @@ package ccip import ( "fmt" "github.com/ethereum/go-ethereum/accounts/abi/bind" + cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink-testing-framework/wasp" @@ -64,6 +65,8 @@ func TestCCIPLoad_RPS(t *testing.T) { require.NoError(t, err) defer loki.StopNow() + gunMap := make(map[uint64]*DestinationGun) + // Based on the config, initialize DestinationGun p := wasp.NewProfile() for selector, chain := range env.Chains { @@ -72,6 +75,11 @@ func TestCCIPLoad_RPS(t *testing.T) { block := latesthdr.Number.Uint64() startBlocks[selector] = &block + gunMap[selector] = NewDestinationGun(env.Logger, selector, *env, state.Chains[selector].Receiver.Address(), loki) + + } + + for _, gun := range gunMap { p.Add(wasp.NewGenerator(&wasp.Config{ T: t, GenName: "ccipLoad", @@ -82,7 +90,7 @@ func TestCCIPLoad_RPS(t *testing.T) { // this schedule is per generator // in this example, it would be 1 request per 10seconds per generator (dest chain) // so if there are 3 generators, it would be 3 requests per 10 seconds over the network - Gun: NewDestinationGun(env.Logger, selector, *env, state.Chains[selector].Receiver.Address(), loki), + Gun: gun, Labels: CommonTestLabels, LokiConfig: wasp.NewLokiConfig(config.CCIP.Load.LokiEndpoint, nil, nil, nil), // use the same loki client using `NewLokiClient` with the same config for sending events @@ -90,6 +98,18 @@ func TestCCIPLoad_RPS(t *testing.T) { } _, err = p.Run(true) + csPair := ChainSelectorPair{ + src: 12922642891491394802, + dst: 3379446385462418246, + } + src, dst := env.Chains[csPair.src], env.Chains[csPair.dst] + startblk := uint64(11654) + + seqNum := gunMap[csPair.dst].seqNums[csPair].End.Load() + _, err = ccipchangeset.ConfirmCommitWithExpectedSeqNumRange(t, src, dst, state.Chains[3379446385462418246].OffRamp, &startblk, cciptypes.SeqNumRange{ + cciptypes.SeqNum(seqNum), + cciptypes.SeqNum(seqNum), + }, false) lokiLabels := map[string]string{} for chainSelector, startBlock := range startBlocks { @@ -97,6 +117,9 @@ func TestCCIPLoad_RPS(t *testing.T) { go func(chainSelector uint64, startBlock *uint64) { defer wg.Done() lggr.Infow("Starting to query for events on ", "chainSelector", chainSelector, "startblock", startBlock) + latesthdr, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, nil) + require.NoError(t, err) + lggr.Infow("Current block number", "chainSelector", chainSelector, "block", latesthdr.Number.Uint64()) filterOpts := &bind.FilterOpts{ Start: *startBlock, @@ -112,6 +135,8 @@ func TestCCIPLoad_RPS(t *testing.T) { fmt.Printf("Events on commitIterator %+v", commitIterator) for commitIterator.Next() { + fmt.Printf("Events on commitIterator inside %+v", commitIterator) + event := commitIterator.Event fmt.Printf("CommitReportAccepted event: %+v\n", event) @@ -126,6 +151,8 @@ func TestCCIPLoad_RPS(t *testing.T) { for i := root.MinSeqNr; i <= root.MaxSeqNr; i++ { // todo: push loki calls to channel? + fmt.Printf("Pushed loki for seqNumber %d\n", i) + SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ EventType: committed, Timestamp: timestamp, @@ -150,6 +177,8 @@ func TestCCIPLoad_RPS(t *testing.T) { // todo: push loki calls to channel? lokiLabels, err = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) require.NoError(t, err) + + fmt.Printf("Pushed loki exec for seqNumber %d\n", execIterator.Event.SequenceNumber) SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ EventType: executed, Timestamp: timestamp, diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index c517cdd0ca2..4f94ef2b071 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -20,10 +20,15 @@ type ChainSelectorPair struct { dst uint64 } +type SeqNumRange struct { + Start *atomic.Uint64 + End *atomic.Uint64 +} + type DestinationGun struct { l logger.Logger env deployment.Environment - seqNums map[ChainSelectorPair]*atomic.Uint64 + seqNums map[ChainSelectorPair]SeqNumRange roundNum *atomic.Int32 chainSelector uint64 receiver common.Address @@ -31,13 +36,17 @@ type DestinationGun struct { } func NewDestinationGun(l logger.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, loki *wasp.LokiClient) *DestinationGun { - seqNums := make(map[ChainSelectorPair]*atomic.Uint64) + seqNums := make(map[ChainSelectorPair]SeqNumRange) for _, cs := range env.AllChainSelectorsExcluding([]uint64{chainSelector}) { + // query for the actual sequence number seqNums[ChainSelectorPair{ src: cs, dst: chainSelector, - }] = atomic.NewUint64(1) + }] = SeqNumRange{ + Start: atomic.NewUint64(0), + End: atomic.NewUint64(0), + } } return &DestinationGun{ l: l, @@ -66,12 +75,16 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } + lokiLabels, err := setLokiLabels(src, m.chainSelector) + if err != nil { + m.l.Errorw("Failed setting loki labels", "error", err) + } + csPair := ChainSelectorPair{ src: src, dst: m.chainSelector, } - m.seqNums[csPair].Add(1) - m.l.Infow("Starting transmit with ", "RoundNum", requestedRound, "Destination ChainSelector", m.chainSelector, "Source ChainSelector", src, "SequenceNumber", m.seqNums[csPair].Load()) + m.l.Infow("Starting transmit with ", "RoundNum", requestedRound, "Destination ChainSelector", m.chainSelector, "Source ChainSelector", src, "SequenceNumber", m.seqNums[csPair].End.Load()) r := state.Chains[src].Router @@ -100,16 +113,38 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } - lokiLabels, err := setLokiLabels(src, m.chainSelector) + blockNum, err := m.env.Chains[src].Confirm(tx) if err != nil { - m.l.Errorw("Failed setting loki labels", "error", err) + m.l.Errorw("could not confirm tx on source", "tx", tx, "err", err) + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} + } + + it, err := state.Chains[src].OnRamp.FilterCCIPMessageSent(&bind.FilterOpts{ + Start: blockNum, + End: &blockNum, + Context: context.Background(), + }, []uint64{m.chainSelector}, []uint64{}) + if err != nil { + m.l.Errorw("could not find sent message event on src chain", "src", src, "dst", m.chainSelector, "err", err) + return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } + if !it.Next() { + m.l.Errorw("Could not find event") + return &wasp.Response{Error: "Could not iterate", Group: waspGroup, Failed: true} + } + SendMetricsToLoki(m.l, m.loki, lokiLabels, &LokiMetric{ EventType: transmitted, Timestamp: time.Now(), - SequenceNumber: m.seqNums[csPair].Load(), + SequenceNumber: m.seqNums[csPair].End.Load(), }) + if m.seqNums[csPair].End.Load() == 0 { + m.seqNums[csPair].Start.Store(it.Event.SequenceNumber) + } + + m.seqNums[csPair].End.Store(it.Event.SequenceNumber) + return &wasp.Response{Failed: false, Group: waspGroup} } @@ -142,3 +177,11 @@ func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { ExtraArgs: nil, }, nil } + +func (m *DestinationGun) GetSequenceNumberRange(csPair ChainSelectorPair) (uint64, uint64, error) { + if r, ok := m.seqNums[csPair]; !ok { + return 0, 0, fmt.Errorf("no sequence number found for chain pair %v", csPair) + } else { + return r.Start.Load(), r.End.Load(), nil + } +} diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index b3e3b944b77..1e0afee34de 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -243,5 +243,5 @@ ephemeral_addresses_number = 0 [Load.CCIP.Load] -LokiEndpoint = "http://localhost:3030/loki/api/v1/push" +LokiEndpoint = "http://loki.localhost/loki/api/v1/push" NoOfNodes = 4 From 4314e83f3837cdffe9faa26ff343db09a95e7c38 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Thu, 9 Jan 2025 11:31:05 +0800 Subject: [PATCH 10/17] changeset applies only once --- deployment/environment/crib/ccip_deployer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment/environment/crib/ccip_deployer.go b/deployment/environment/crib/ccip_deployer.go index 639e42b4024..ff3199a81a9 100644 --- a/deployment/environment/crib/ccip_deployer.go +++ b/deployment/environment/crib/ccip_deployer.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "github.com/smartcontractkit/chainlink-ccip/chainconfig" + cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" "math/big" "github.com/ethereum/go-ethereum/common" @@ -94,7 +96,6 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de ChainSelector: chain, }) } - // set up chains chainConfigs := make(map[uint64]changeset.ChainConfig) nodeInfo, err := deployment.NodeInfo(e.NodeIDs, e.Offchain) From d57b10812e3521ed66c6cfd810b85cca14151731 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Fri, 10 Jan 2025 12:48:46 +0800 Subject: [PATCH 11/17] should nto use testrouter when adding lanes --- deployment/environment/crib/ccip_deployer.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/deployment/environment/crib/ccip_deployer.go b/deployment/environment/crib/ccip_deployer.go index ff3199a81a9..5c6863f801c 100644 --- a/deployment/environment/crib/ccip_deployer.go +++ b/deployment/environment/crib/ccip_deployer.go @@ -5,14 +5,12 @@ import ( "errors" "fmt" "github.com/smartcontractkit/chainlink-ccip/chainconfig" - cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" "math/big" "github.com/ethereum/go-ethereum/common" "github.com/smartcontractkit/ccip-owner-contracts/pkg/config" - "github.com/smartcontractkit/chainlink-ccip/chainconfig" cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" @@ -206,7 +204,7 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de to: { from: { IsEnabled: true, - TestRouter: true, + TestRouter: false, }, }, }, @@ -215,7 +213,7 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de { Changeset: commonchangeset.WrapChangeSet(changeset.UpdateRouterRamps), Config: changeset.UpdateRouterRampsConfig{ - TestRouter: true, + TestRouter: false, UpdatesByChain: map[uint64]changeset.RouterUpdates{ // onRamp update on source chain from: { From 1ea5e2f8cdc4cbf7a3b71b3cfb4c604766574317 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Fri, 10 Jan 2025 23:00:30 +0800 Subject: [PATCH 12/17] test changes --- ccip/config/evm/fallback.toml | 95 ------------------- deployment/ccip/changeset/cs_ccip_home.go | 6 +- deployment/environment/crib/ccip_deployer.go | 45 ++++----- deployment/environment/devenv/chain.go | 11 ++- integration-tests/load/ccip/ccip_test.go | 47 +++++++-- .../load/ccip/destination_gun.go | 91 +++++++++++++++--- integration-tests/load/ccip/helpers.go | 1 + integration-tests/testconfig/ccip/ccip.toml | 9 +- integration-tests/testconfig/ccip/load.go | 8 +- 9 files changed, 169 insertions(+), 144 deletions(-) delete mode 100644 ccip/config/evm/fallback.toml diff --git a/ccip/config/evm/fallback.toml b/ccip/config/evm/fallback.toml deleted file mode 100644 index c1f963a33ff..00000000000 --- a/ccip/config/evm/fallback.toml +++ /dev/null @@ -1,95 +0,0 @@ -AutoCreateKey = true -BlockBackfillDepth = 10 -BlockBackfillSkip = false -FinalityDepth = 50 -FinalityTagEnabled = false -LogBackfillBatchSize = 1000 -LogPollInterval = '15s' -LogKeepBlocksDepth = 100000 -# CCIP uses paging when removing logs to avoid pushing too much pressure on the database -LogPrunePageSize = 10000 -BackupLogPollerBlockDelay = 100 -MinContractPayment = '.00001 link' -MinIncomingConfirmations = 3 -NonceAutoSync = true -NoNewHeadsThreshold = '3m' -RPCDefaultBatchSize = 250 -RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 -NoNewFinalizedHeadsThreshold = '0' -LogBroadcasterEnabled = true - -[Transactions] -ForwardersEnabled = false -MaxInFlight = 16 -MaxQueued = 250 -ReaperInterval = '1h' -ReaperThreshold = '168h' -ResendAfterThreshold = '1m' - -[Transactions.AutoPurge] -Enabled = false - -[BalanceMonitor] -Enabled = true - -[GasEstimator] -Mode = 'BlockHistory' -PriceDefault = '20 gwei' -PriceMax = '115792089237316195423570985008687907853269984665.640564039457584007913129639935 tether' -PriceMin = '1 gwei' -LimitDefault = 8_000_000 -LimitMax = 8_000_000 -LimitMultiplier = '1' -LimitTransfer = 21_000 -BumpMin = '5 gwei' -BumpPercent = 20 -BumpThreshold = 3 -EIP1559DynamicFees = false -FeeCapDefault = '100 gwei' -TipCapDefault = '1' -TipCapMin = '1' -EstimateLimit = false - -[GasEstimator.BlockHistory] -BatchSize = 25 -BlockHistorySize = 8 -CheckInclusionBlocks = 12 -CheckInclusionPercentile = 90 -TransactionPercentile = 60 - -[GasEstimator.FeeHistory] -CacheTimeout = '10s' - -[HeadTracker] -HistoryDepth = 100 -MaxBufferSize = 3 -SamplingInterval = '1s' -FinalityTagBypass = true -MaxAllowedFinalityDepth = 10000 - -[NodePool] -PollFailureThreshold = 5 -PollInterval = '10s' -SelectionMode = 'HighestHead' -SyncThreshold = 5 -LeaseDuration = '0s' -NodeIsSyncingEnabled = false -FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' -NewHeadsPollInterval = '0s' - -[OCR] -ContractConfirmations = 4 -ContractTransmitterTransmitTimeout = '10s' -DatabaseTimeout = '10s' -DeltaCOverride = '168h' -DeltaCJitterOverride = '1h' -ObservationGracePeriod = '1s' - -[OCR2.Automation] -GasLimit = 5400000 - -[Workflow] -GasLimitDefault = 400_000 diff --git a/deployment/ccip/changeset/cs_ccip_home.go b/deployment/ccip/changeset/cs_ccip_home.go index 7d3327a31f2..a7ac1a63df2 100644 --- a/deployment/ccip/changeset/cs_ccip_home.go +++ b/deployment/ccip/changeset/cs_ccip_home.go @@ -1024,8 +1024,10 @@ func (c UpdateChainConfigConfig) Validate(e deployment.Environment) error { if !exists { return fmt.Errorf("home chain %d does not exist", c.HomeChainSelector) } - if err := commoncs.ValidateOwnership(e.GetContext(), c.MCMS != nil, e.Chains[c.HomeChainSelector].DeployerKey.From, homeChainState.Timelock.Address(), homeChainState.CCIPHome); err != nil { - return err + if c.MCMS != nil { + if err := commoncs.ValidateOwnership(e.GetContext(), c.MCMS != nil, e.Chains[c.HomeChainSelector].DeployerKey.From, homeChainState.Timelock.Address(), homeChainState.CCIPHome); err != nil { + return err + } } for _, remove := range c.RemoteChainRemoves { if err := deployment.IsValidChainSelector(remove); err != nil { diff --git a/deployment/environment/crib/ccip_deployer.go b/deployment/environment/crib/ccip_deployer.go index 5c6863f801c..10799e67f60 100644 --- a/deployment/environment/crib/ccip_deployer.go +++ b/deployment/environment/crib/ccip_deployer.go @@ -152,19 +152,18 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de return DeployCCIPOutput{}, fmt.Errorf("failed to load onchain state: %w", err) } // Add all lanes - for from := range e.Chains { - for to := range e.Chains { - if from != to { - stateChain1 := state.Chains[from] + for src := range e.Chains { + for dst := range e.Chains { + if src != dst { + stateChain1 := state.Chains[src] newEnv, err := commonchangeset.ApplyChangesets(nil, *e, nil, []commonchangeset.ChangesetApplication{ { Changeset: commonchangeset.WrapChangeSet(changeset.UpdateOnRampsDests), Config: changeset.UpdateOnRampDestsConfig{ UpdatesByChain: map[uint64]map[uint64]changeset.OnRampDestinationUpdate{ - from: { - to: { + src: { + dst: { IsEnabled: true, - TestRouter: false, AllowListEnabled: false, }, }, @@ -175,13 +174,13 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de Changeset: commonchangeset.WrapChangeSet(changeset.UpdateFeeQuoterPricesCS), Config: changeset.UpdateFeeQuoterPricesConfig{ PricesByChain: map[uint64]changeset.FeeQuoterPriceUpdatePerSource{ - from: { + src: { TokenPrices: map[common.Address]*big.Int{ stateChain1.LinkToken.Address(): changeset.DefaultLinkPrice, stateChain1.Weth9.Address(): changeset.DefaultWethPrice, }, GasPrices: map[uint64]*big.Int{ - to: changeset.DefaultGasPrice, + dst: changeset.DefaultGasPrice, }, }, }, @@ -191,8 +190,8 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de Changeset: commonchangeset.WrapChangeSet(changeset.UpdateFeeQuoterDests), Config: changeset.UpdateFeeQuoterDestsConfig{ UpdatesByChain: map[uint64]map[uint64]fee_quoter.FeeQuoterDestChainConfig{ - from: { - to: changeset.DefaultFeeQuoterDestChainConfig(), + src: { + dst: changeset.DefaultFeeQuoterDestChainConfig(), }, }, }, @@ -201,10 +200,9 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de Changeset: commonchangeset.WrapChangeSet(changeset.UpdateOffRampSources), Config: changeset.UpdateOffRampSourcesConfig{ UpdatesByChain: map[uint64]map[uint64]changeset.OffRampSourceUpdate{ - to: { - from: { - IsEnabled: true, - TestRouter: false, + dst: { + src: { + IsEnabled: true, }, }, }, @@ -213,18 +211,21 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de { Changeset: commonchangeset.WrapChangeSet(changeset.UpdateRouterRamps), Config: changeset.UpdateRouterRampsConfig{ - TestRouter: false, UpdatesByChain: map[uint64]changeset.RouterUpdates{ - // onRamp update on source chain - from: { + src: { + OffRampUpdates: map[uint64]bool{ + dst: true, + }, OnRampUpdates: map[uint64]bool{ - to: true, + dst: true, }, }, - // off - from: { + dst: { OffRampUpdates: map[uint64]bool{ - to: true, + src: true, + }, + OnRampUpdates: map[uint64]bool{ + src: true, }, }, }, diff --git a/deployment/environment/devenv/chain.go b/deployment/environment/devenv/chain.go index 19d58882e03..c53cb82a7ed 100644 --- a/deployment/environment/devenv/chain.go +++ b/deployment/environment/devenv/chain.go @@ -22,13 +22,18 @@ const ( EVMChainType = "EVM" ) +type CribRPCs struct { + Internal string + External string +} + // ChainConfig holds the configuration for a with a deployer key which can be used to send transactions to the chain. type ChainConfig struct { ChainID uint64 // chain id as per EIP-155, mainly applicable for EVM chains ChainName string // name of the chain populated from chainselector repo ChainType string // should denote the chain family. Acceptable values are EVM, COSMOS, SOLANA, STARKNET, APTOS etc - WSRPCs []string // websocket rpcs to connect to the chain - HTTPRPCs []string // http rpcs to connect to the chain + WSRPCs []CribRPCs // websocket rpcs to connect to the chain + HTTPRPCs []CribRPCs // http rpcs to connect to the chain DeployerKey *bind.TransactOpts // key to deploy and configure contracts on the chain Users []*bind.TransactOpts // map of addresses to their transact opts to interact with the chain as users } @@ -97,7 +102,7 @@ func NewChains(logger logger.Logger, configs []ChainConfig) (map[uint64]deployme // TODO : better client handling var ec *ethclient.Client for _, rpc := range chainCfg.WSRPCs { - ec, err = ethclient.Dial(rpc) + ec, err = ethclient.Dial(rpc.External) if err != nil { logger.Warnf("failed to dial ws rpc %s", rpc) continue diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 1ee1049b502..8148df444b0 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -46,6 +46,17 @@ func TestCCIPLoad_RPS(t *testing.T) { config, err := tc.GetConfig([]string{"Load"}, tc.CCIP) require.NoError(t, err) lggr.Infof("loaded ccip test config: %+v", config.CCIP.Load) + userOverrides := config.CCIP.Load + + timeout, err := time.ParseDuration(*userOverrides.TestTimeout) + if err != nil { + require.NoError(t, err) + } + t.Cleanup(func() { + time.AfterFunc(timeout, func() { + t.Fatalf("Test passed timeout after %v", timeout) + }) + }) cribEnv := crib.NewDevspaceEnvFromStateDir(CRIB_DIRECTORY) @@ -61,7 +72,7 @@ func TestCCIPLoad_RPS(t *testing.T) { require.NoError(t, err) // Parse all events from the simulated chains, send to Loki - loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(config.CCIP.Load.LokiEndpoint, nil, nil, nil)) + loki, err := wasp.NewLokiClient(wasp.NewLokiConfig(userOverrides.LokiEndpoint, nil, nil, nil)) require.NoError(t, err) defer loki.StopNow() @@ -75,8 +86,16 @@ func TestCCIPLoad_RPS(t *testing.T) { block := latesthdr.Number.Uint64() startBlocks[selector] = &block - gunMap[selector] = NewDestinationGun(env.Logger, selector, *env, state.Chains[selector].Receiver.Address(), loki) - + // Only create a destination gun if we have decided to send traffic to this chain + for _, cs := range *userOverrides.EnabledDestionationChains { + if cs == selector { + gunMap[selector], err = NewDestinationGun(env.Logger, selector, *env, state.Chains[selector].Receiver.Address(), userOverrides, loki) + if err != nil { + lggr.Errorw("Failed to initialize DestinationGun for", "chainSelector", chain, "error", err) + t.Fatal(err) + } + } + } } for _, gun := range gunMap { @@ -85,7 +104,7 @@ func TestCCIPLoad_RPS(t *testing.T) { GenName: "ccipLoad", LoadType: wasp.RPS, CallTimeout: 5 * time.Second, - Schedule: wasp.Plain(1, 5*time.Second), + Schedule: wasp.Plain(1, time.Duration(*userOverrides.SecondsPerRequestPerDestination)*time.Second), // will need to be divided by number of chains // this schedule is per generator // in this example, it would be 1 request per 10seconds per generator (dest chain) @@ -97,20 +116,34 @@ func TestCCIPLoad_RPS(t *testing.T) { })) } - _, err = p.Run(true) + // find get fee revert csPair := ChainSelectorPair{ src: 12922642891491394802, dst: 3379446385462418246, } + res, err := state.Chains[csPair.src].Router.IsChainSupported(nil, csPair.dst) + lggr.Infow("IsChainSupported", "res", res, "err", err) + + destChainConfig, err := state.Chains[csPair.src].FeeQuoter.GetDestChainConfig(nil, csPair.dst) + lggr.Infow("GetDestChainConfig", "destChainConfig", destChainConfig, "err", err) + + // find the getFee revert + _, err = p.Run(true) + //csPair := ChainSelectorPair{ + // src: 12922642891491394802, + // dst: 3379446385462418246, + //} src, dst := env.Chains[csPair.src], env.Chains[csPair.dst] startblk := uint64(11654) seqNum := gunMap[csPair.dst].seqNums[csPair].End.Load() _, err = ccipchangeset.ConfirmCommitWithExpectedSeqNumRange(t, src, dst, state.Chains[3379446385462418246].OffRamp, &startblk, cciptypes.SeqNumRange{ - cciptypes.SeqNum(seqNum), - cciptypes.SeqNum(seqNum), + cciptypes.SeqNum(seqNum - 1), + cciptypes.SeqNum(seqNum - 1), }, false) + // todo: create channels that watch for these events beforehand using WatchExecutionStateChanged and WatchCommitReportAccepted + // rather than waiting for the generator to finish lokiLabels := map[string]string{} for chainSelector, startBlock := range startBlocks { wg.Add(1) diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index 4f94ef2b071..06b91a4d727 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -9,9 +9,12 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/wasp" "github.com/smartcontractkit/chainlink/deployment" ccipchangeset "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" + "github.com/smartcontractkit/chainlink/integration-tests/testconfig/ccip" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" "go.uber.org/atomic" + "math/big" + "math/rand" "time" ) @@ -32,10 +35,11 @@ type DestinationGun struct { roundNum *atomic.Int32 chainSelector uint64 receiver common.Address + testConfig *ccip.LoadConfig loki *wasp.LokiClient } -func NewDestinationGun(l logger.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, loki *wasp.LokiClient) *DestinationGun { +func NewDestinationGun(l logger.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, overrides *ccip.LoadConfig, loki *wasp.LokiClient) (*DestinationGun, error) { seqNums := make(map[ChainSelectorPair]SeqNumRange) for _, cs := range env.AllChainSelectorsExcluding([]uint64{chainSelector}) { @@ -48,15 +52,38 @@ func NewDestinationGun(l logger.Logger, chainSelector uint64, env deployment.Env End: atomic.NewUint64(0), } } - return &DestinationGun{ + dg := DestinationGun{ l: l, env: env, seqNums: seqNums, roundNum: &atomic.Int32{}, chainSelector: chainSelector, receiver: receiver, + testConfig: overrides, loki: loki, } + + err := dg.Validate() + if err != nil { + return nil, err + } + + return &dg, nil +} + +func (m *DestinationGun) Validate() error { + if len(*m.testConfig.MessageTypeWeights) != 3 { + return fmt.Errorf( + "message type must have 3 weights corresponding to message only, token only, token with message") + } + sum := 0 + for _, weight := range *m.testConfig.MessageTypeWeights { + sum += weight + } + if sum != 100 { + return fmt.Errorf("message type weights must sum to 100") + } + return nil } func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { @@ -84,7 +111,11 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { src: src, dst: m.chainSelector, } - m.l.Infow("Starting transmit with ", "RoundNum", requestedRound, "Destination ChainSelector", m.chainSelector, "Source ChainSelector", src, "SequenceNumber", m.seqNums[csPair].End.Load()) + m.l.Infow("Starting transmit with ", + "RoundNum", requestedRound, + "Source ChainSelector", src, + "Destination ChainSelector", m.chainSelector, + "SequenceNumber", m.seqNums[csPair].End.Load()) r := state.Chains[src].Router @@ -96,7 +127,7 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { fee, err := r.GetFee( &bind.CallOpts{Context: context.Background()}, m.chainSelector, msg) if err != nil { - m.l.Errorw("could not get fee ", "dstChainSelector", m.chainSelector, "msg", msg, "fee", fee) + m.l.Errorw("could not get fee ", "dstChainSelector", m.chainSelector, "msg", msg, "fee", fee, "err", err) return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } m.l.Debugw("setting fee for ", "srcChain", src, "dstChain", m.chainSelector, "fee", fee, "msg", msg) @@ -152,6 +183,7 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { func (m *DestinationGun) MustSourceChain() (uint64, error) { // TODO: make this smarter by checking if this chain has sent a message recently, if so, switch to the next chain + // Currently performing a round robin otherCS := m.env.AllChainSelectorsExcluding([]uint64{m.chainSelector}) if len(otherCS) == 0 { return 0, fmt.Errorf("no other chains to send from") @@ -161,7 +193,6 @@ func (m *DestinationGun) MustSourceChain() (uint64, error) { } // GetMessage will return the message to be sent while considering expected load of different messages -// TODO: implement randomness and different types of messages func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { rcv, err := utils.ABIEncode(`[{"type":"address"}]`, m.receiver) if err != nil { @@ -169,13 +200,49 @@ func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { return router.ClientEVM2AnyMessage{}, err } - return router.ClientEVM2AnyMessage{ - Receiver: rcv, - Data: common.Hex2Bytes("hello world"), - TokenAmounts: nil, - FeeToken: common.HexToAddress("0x0"), - ExtraArgs: nil, - }, nil + messages := []router.ClientEVM2AnyMessage{ + { + Receiver: rcv, + Data: common.Hex2Bytes("message"), + TokenAmounts: nil, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }, + { + Receiver: rcv, + TokenAmounts: []router.ClientEVMTokenAmount{ + { + Token: common.HexToAddress("0x0"), + Amount: big.NewInt(100), + }, + }, + Data: common.Hex2Bytes("hello world"), + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }, + { + Receiver: rcv, + Data: common.Hex2Bytes("message with token"), + TokenAmounts: []router.ClientEVMTokenAmount{ + { + Token: common.HexToAddress("0x0"), + Amount: big.NewInt(100), + }, + }, + FeeToken: common.HexToAddress("0x0"), + ExtraArgs: nil, + }, + } + // Select a random message + randomValue := rand.Intn(100) + switch { + case randomValue < (*m.testConfig.MessageTypeWeights)[0]: + return messages[0], nil + case randomValue < (*m.testConfig.MessageTypeWeights)[0]+(*m.testConfig.MessageTypeWeights)[1]: + return messages[1], nil + default: + return messages[2], nil + } } func (m *DestinationGun) GetSequenceNumberRange(csPair ChainSelectorPair) (uint64, uint64, error) { diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go index faeede61e7a..8634f49edbf 100644 --- a/integration-tests/load/ccip/helpers.go +++ b/integration-tests/load/ccip/helpers.go @@ -106,6 +106,7 @@ func CreateEnvironmentFromCribOutput(t *testing.T, lggr logger.Logger) (*deploym lggr, ab, chains, + nil, nodeIds, nil, func() context.Context { return context.Background() }, diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index 1e0afee34de..d9eaeced856 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -241,7 +241,14 @@ addresses_to_fund = [ # Seth specific configuration, no need for generating ephemeral addresses for ccip-tests. ephemeral_addresses_number = 0 - [Load.CCIP.Load] +# replace this with the loki endpoint of the crib stack LokiEndpoint = "http://loki.localhost/loki/api/v1/push" NoOfNodes = 4 +TestTimeout = "1m" +# corresponds to [data only, token only, message with data] +MessageTypeWeights = [100,0,0] +# each destination chain will have 1 request per 5 seconds +SecondsPerRequestPerDestination = 5 +# destination chain selectors to send messages to +EnabledDestionationChains = [3379446385462418246] \ No newline at end of file diff --git a/integration-tests/testconfig/ccip/load.go b/integration-tests/testconfig/ccip/load.go index 5b226d6e7eb..57bbc5e37e9 100644 --- a/integration-tests/testconfig/ccip/load.go +++ b/integration-tests/testconfig/ccip/load.go @@ -1,6 +1,10 @@ package ccip type LoadConfig struct { - NoOfNodes *int - LokiEndpoint *string + TestTimeout *string + NoOfNodes *int + LokiEndpoint *string + MessageTypeWeights *[]int + SecondsPerRequestPerDestination *int + EnabledDestionationChains *[]uint64 } From 81103371fb8c89e6c397cdcca14112f00d6d0742 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Tue, 14 Jan 2025 18:06:14 +0800 Subject: [PATCH 13/17] set ocr3 configuration so the bootstrap can create a don --- deployment/ccip/changeset/cs_ccip_home.go | 7 +- deployment/ccip/changeset/cs_prerequisites.go | 1 - deployment/environment/crib/ccip_deployer.go | 66 +++++++++++++++++++ 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/deployment/ccip/changeset/cs_ccip_home.go b/deployment/ccip/changeset/cs_ccip_home.go index a7ac1a63df2..bbd7028a8f3 100644 --- a/deployment/ccip/changeset/cs_ccip_home.go +++ b/deployment/ccip/changeset/cs_ccip_home.go @@ -303,8 +303,10 @@ func (s SetCandidateConfigBase) Validate(e deployment.Environment, state CCIPOnC if !exists { return fmt.Errorf("home chain %d does not exist", s.HomeChainSelector) } - if err := commoncs.ValidateOwnership(e.GetContext(), s.MCMS != nil, e.Chains[s.HomeChainSelector].DeployerKey.From, homeChainState.Timelock.Address(), homeChainState.CapabilityRegistry); err != nil { - return err + if s.MCMS != nil { + if err := commoncs.ValidateOwnership(e.GetContext(), s.MCMS != nil, e.Chains[s.HomeChainSelector].DeployerKey.From, homeChainState.Timelock.Address(), homeChainState.CapabilityRegistry); err != nil { + return err + } } for chainSelector, params := range s.OCRConfigPerRemoteChainSelector { @@ -1105,7 +1107,6 @@ func UpdateChainConfig(e deployment.Environment, cfg UpdateChainConfigConfig) (d if err != nil { return deployment.ChangesetOutput{}, err } - e.Logger.Infof("Updated chain config on chain %d removes %v, adds %v", cfg.HomeChainSelector, cfg.RemoteChainRemoves, cfg.RemoteChainAdds) return deployment.ChangesetOutput{}, nil } diff --git a/deployment/ccip/changeset/cs_prerequisites.go b/deployment/ccip/changeset/cs_prerequisites.go index 2736ecf44bf..f09e7ca24e9 100644 --- a/deployment/ccip/changeset/cs_prerequisites.go +++ b/deployment/ccip/changeset/cs_prerequisites.go @@ -306,7 +306,6 @@ func deployPrerequisiteContracts(e deployment.Environment, ab deployment.Address e.Logger.Errorw("Failed to confirm assign registry module on token admin registry", "chain", chain.String(), "err", err) return fmt.Errorf("failed to confirm assign registry module on token admin registry: %w", err) } - e.Logger.Infow("assigned registry module on token admin registry") } if weth9Contract == nil { weth, err := deployment.DeployContract(lggr, chain, ab, diff --git a/deployment/environment/crib/ccip_deployer.go b/deployment/environment/crib/ccip_deployer.go index 10799e67f60..abd5a8ab273 100644 --- a/deployment/environment/crib/ccip_deployer.go +++ b/deployment/environment/crib/ccip_deployer.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "github.com/smartcontractkit/chainlink-ccip/chainconfig" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types" "math/big" "github.com/ethereum/go-ethereum/common" @@ -151,6 +152,71 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de if err != nil { return DeployCCIPOutput{}, fmt.Errorf("failed to load onchain state: %w", err) } + + var ocrConfigPerSelector = make(map[uint64]changeset.CCIPOCRParams) + for selector := range e.Chains { + //tokenConfig := changeset.NewTestTokenConfig(state.Chains[feedChainSel].USDFeeds) + ocrConfigPerSelector[selector] = changeset.DefaultOCRParams( + selector, + //tokenConfig.GetTokenInfo(lggr, state.Chains[selector].LinkToken, state.Chains[selector].Weth9), + nil, + nil, + ) + } + + *e, err = commonchangeset.ApplyChangesets(nil, *e, nil, []commonchangeset.ChangesetApplication{ + { + // Add the DONs and candidate commit OCR instances for the chain. + Changeset: commonchangeset.WrapChangeSet(changeset.AddDonAndSetCandidateChangeset), + Config: changeset.AddDonAndSetCandidateChangesetConfig{ + changeset.SetCandidateConfigBase{ + HomeChainSelector: homeChainSel, + FeedChainSelector: feedChainSel, + OCRConfigPerRemoteChainSelector: ocrConfigPerSelector, + PluginType: types.PluginTypeCCIPCommit, + }, + }, + }, + { + // Add the exec OCR instances for the new chains. + Changeset: commonchangeset.WrapChangeSet(changeset.SetCandidateChangeset), + Config: changeset.SetCandidateChangesetConfig{ + changeset.SetCandidateConfigBase{ + HomeChainSelector: homeChainSel, + FeedChainSelector: feedChainSel, + OCRConfigPerRemoteChainSelector: ocrConfigPerSelector, + PluginType: types.PluginTypeCCIPExec, + }, + }, + }, + { + // Promote everything + Changeset: commonchangeset.WrapChangeSet(changeset.PromoteAllCandidatesChangeset), + Config: changeset.PromoteCandidatesChangesetConfig{ + HomeChainSelector: homeChainSel, + RemoteChainSelectors: chainSelectors, + PluginType: types.PluginTypeCCIPCommit, + }, + }, + { + // Promote everything + Changeset: commonchangeset.WrapChangeSet(changeset.PromoteAllCandidatesChangeset), + Config: changeset.PromoteCandidatesChangesetConfig{ + HomeChainSelector: homeChainSel, + RemoteChainSelectors: chainSelectors, + PluginType: types.PluginTypeCCIPExec, + }, + }, + { + // Enable the OCR config on the remote chains. + Changeset: commonchangeset.WrapChangeSet(changeset.SetOCR3OffRamp), + Config: changeset.SetOCR3OffRampConfig{ + HomeChainSel: homeChainSel, + RemoteChainSels: chainSelectors, + }, + }, + }) + // Add all lanes for src := range e.Chains { for dst := range e.Chains { From 2c02914924498e7ed810e82e502cb62fbb15a17a Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Wed, 15 Jan 2025 12:45:08 +0800 Subject: [PATCH 14/17] load test WORKING --- .../ccip/changeset/cs_update_rmn_config.go | 1 + deployment/environment/crib/ccip_deployer.go | 23 ++++- deployment/environment/crib/env.go | 8 +- deployment/environment/crib/helpers.go | 88 +++++++++++++++++++ integration-tests/load/ccip/ccip_test.go | 9 +- 5 files changed, 115 insertions(+), 14 deletions(-) create mode 100644 deployment/environment/crib/helpers.go diff --git a/deployment/ccip/changeset/cs_update_rmn_config.go b/deployment/ccip/changeset/cs_update_rmn_config.go index 309f10c8311..155ad43a733 100644 --- a/deployment/ccip/changeset/cs_update_rmn_config.go +++ b/deployment/ccip/changeset/cs_update_rmn_config.go @@ -46,6 +46,7 @@ func (c SetRMNRemoteOnRMNProxyConfig) Validate(state CCIPOnChainState) error { } func SetRMNRemoteOnRMNProxy(e deployment.Environment, cfg SetRMNRemoteOnRMNProxyConfig) (deployment.ChangesetOutput, error) { + e.Logger.Infow("Setting RMNRemote on RMNProxy", "chainSelectors", cfg.ChainSelectors) state, err := LoadOnchainState(e) if err != nil { return deployment.ChangesetOutput{}, fmt.Errorf("failed to load onchain state: %w", err) diff --git a/deployment/environment/crib/ccip_deployer.go b/deployment/environment/crib/ccip_deployer.go index abd5a8ab273..ded754f924a 100644 --- a/deployment/environment/crib/ccip_deployer.go +++ b/deployment/environment/crib/ccip_deployer.go @@ -72,7 +72,7 @@ func DeployHomeChainContracts(ctx context.Context, lggr logger.Logger, envConfig // DeployCCIPAndAddLanes is the actual ccip setup once the nodes are initialized. func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig devenv.EnvironmentConfig, homeChainSel, feedChainSel uint64, ab deployment.AddressBook) (DeployCCIPOutput, error) { - e, _, err := devenv.NewEnvironment(func() context.Context { return ctx }, lggr, envConfig) + e, don, err := devenv.NewEnvironment(func() context.Context { return ctx }, lggr, envConfig) if err != nil { return DeployCCIPOutput{}, fmt.Errorf("failed to initiate new environment: %w", err) } @@ -143,6 +143,12 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de HomeChainSelector: homeChainSel, }, }, + { + Changeset: commonchangeset.WrapChangeSet(changeset.SetRMNRemoteOnRMNProxy), + Config: changeset.SetRMNRemoteOnRMNProxyConfig{ + ChainSelectors: chainSelectors, + }, + }, { Changeset: commonchangeset.WrapChangeSet(changeset.CCIPCapabilityJobspec), Config: struct{}{}, @@ -153,6 +159,15 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de return DeployCCIPOutput{}, fmt.Errorf("failed to load onchain state: %w", err) } + // find out which rmn proxy we're pointing to + for _, chain := range chainSelectors { + r, err := state.Chains[chain].RMNProxy.GetARM(nil) + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to get rmn proxy: %w", err) + } + lggr.Infow("pointed to arm for ", "chain", chain, "rmnProxy", r.String()) + } + var ocrConfigPerSelector = make(map[uint64]changeset.CCIPOCRParams) for selector := range e.Chains { //tokenConfig := changeset.NewTestTokenConfig(state.Chains[feedChainSel].USDFeeds) @@ -306,10 +321,16 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de } } + // distribute funds to transmitters + // we need to use the nodeinfo from the envConfig here, because multiAddr is not + // populated in the environment variable + distributeFunds(lggr, don.PluginNodes(), *e) + addresses, err := e.ExistingAddresses.Addresses() if err != nil { return DeployCCIPOutput{}, fmt.Errorf("failed to get convert address book to address book map: %w", err) } + return DeployCCIPOutput{ AddressBook: *deployment.NewMemoryAddressBookFromMap(addresses), NodeIDs: e.NodeIDs, diff --git a/deployment/environment/crib/env.go b/deployment/environment/crib/env.go index 281565480df..22cb4ca0aad 100644 --- a/deployment/environment/crib/env.go +++ b/deployment/environment/crib/env.go @@ -1,7 +1,5 @@ package crib -import "fmt" - const ( AddressBookFileName = "ccip-v2-scripts-address-book.json" NodesDetailsFileName = "ccip-v2-scripts-nodes-details.json" @@ -18,15 +16,11 @@ func NewDevspaceEnvFromStateDir(envStateDir string) CRIBEnv { } } -func (c CRIBEnv) GetConfig(deployerKeys map[uint64]string) (DeployOutput, error) { +func (c CRIBEnv) GetConfig(key string) (DeployOutput, error) { reader := NewOutputReader(c.envStateDir) nodesDetails := reader.ReadNodesDetails() chainConfigs := reader.ReadChainConfigs() for i, chain := range chainConfigs { - key, ok := deployerKeys[chain.ChainID] - if !ok { - return DeployOutput{}, fmt.Errorf("could not find deployer key for %s", key) - } err := chain.SetDeployerKey(&key) if err != nil { return DeployOutput{}, err diff --git a/deployment/environment/crib/helpers.go b/deployment/environment/crib/helpers.go new file mode 100644 index 00000000000..965122e8ee6 --- /dev/null +++ b/deployment/environment/crib/helpers.go @@ -0,0 +1,88 @@ +package crib + +import ( + "context" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + gethtypes "github.com/ethereum/go-ethereum/core/types" + chainsel "github.com/smartcontractkit/chain-selectors" + "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/conversions" + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" + "github.com/smartcontractkit/chainlink/v2/core/logger" +) + +func distributeFunds(lggr logger.Logger, nodeInfo []devenv.Node, env deployment.Environment) { + fmt.Println("Funding transmitters...") + transmittersStr := make([]common.Address, 0) + fundingAmount := big.NewInt(500_000_000_000_000_000) // 0.5 ETH + minThreshold := big.NewInt(50_000_000_000_000_000) // 0.05 ETH + + // todo: parallelize this using a waitgroup + for sel, chain := range env.Chains { + for _, n := range nodeInfo { + + chainId, err := chainsel.ChainIdFromSelector(sel) + if err != nil { + lggr.Errorw("could not get chain id from selector", "selector", sel, "err", err) + continue + } + addr := common.HexToAddress(n.AccountAddr[chainId]) + balance, err := chain.Client.BalanceAt(context.Background(), addr, nil) + if err != nil { + fmt.Printf("Error fetching balance for %s: %v\n", n, err) + continue + } + if balance.Cmp(minThreshold) < 0 { + fmt.Printf( + "Transmitter %s with address %s, has insufficient funds, funding with %s ETH. Current balance: %s, threshold: %s\n", + n.Name, + addr.String(), + conversions.WeiToEther(fundingAmount).String(), + conversions.WeiToEther(balance).String(), + conversions.WeiToEther(minThreshold).String(), + ) + transmittersStr = append(transmittersStr, addr) + } + } + latesthdr, err := chain.Client.HeaderByNumber(context.Background(), nil) + if err != nil { + lggr.Errorw("could not get header, skipping chain", "chain", sel, "err", err) + continue + } + block := latesthdr.Number.Uint64() + + nonce, err := chain.Client.NonceAt(context.Background(), chain.DeployerKey.From, big.NewInt(int64(block))) + if err != nil { + lggr.Warnw("could not get latest nonce for deployer key", "err", err) + continue + } + for _, transmitter := range transmittersStr { + tx := gethtypes.NewTransaction(nonce, transmitter, fundingAmount, uint64(1000000), big.NewInt(1000000), nil) + + signedTx, err := chain.DeployerKey.Signer(chain.DeployerKey.From, tx) + if err != nil { + lggr.Errorw("could not sign transaction to transmitter on ", "chain", sel, "transmitter", transmitter, "err", err) + continue + } + + lggr.Infow("sending transaction for ", "transmitter", transmitter.String(), "chain", sel) + err = chain.Client.SendTransaction(context.Background(), signedTx) + if err != nil { + lggr.Errorw("could not send transaction to transmitter on ", "chain", sel, "transmitter", transmitter, "err", err) + continue + } + + _, err = bind.WaitMined(context.Background(), chain.Client, signedTx) + if err != nil { + lggr.Errorw("could not mine transaction to transmitter on ", "chain", sel) + continue + } + nonce++ + } + } +} diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 8148df444b0..8f305ca9062 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -22,11 +22,8 @@ var ( "branch": "ccip_load_crib", "commit": "ccip_load_crib", } - wg sync.WaitGroup - SIM_CHAIN_PRIVATE_KEYS = map[uint64]string{ - 1337: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", - 2337: "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", - } + wg sync.WaitGroup + SIM_CHAIN_TEST_KEY = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" ) const CRIB_DIRECTORY = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" @@ -60,7 +57,7 @@ func TestCCIPLoad_RPS(t *testing.T) { cribEnv := crib.NewDevspaceEnvFromStateDir(CRIB_DIRECTORY) - cribDeployOutput, err := cribEnv.GetConfig(SIM_CHAIN_PRIVATE_KEYS) + cribDeployOutput, err := cribEnv.GetConfig(SIM_CHAIN_TEST_KEY) require.NoError(t, err) env, err := crib.NewDeployEnvironmentFromCribOutput(lggr, cribDeployOutput) require.NoError(t, err) From 2ce7370436a0c94965c13b056ec3e08bbfbf5b4a Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Wed, 15 Jan 2025 13:05:36 +0800 Subject: [PATCH 15/17] use changeset cspairs in the ccip tests --- integration-tests/load/ccip/ccip_test.go | 23 ++++++++++--------- .../load/ccip/destination_gun.go | 23 ++++++++----------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 8f305ca9062..2aebdbb73f3 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -114,31 +114,32 @@ func TestCCIPLoad_RPS(t *testing.T) { } // find get fee revert - csPair := ChainSelectorPair{ - src: 12922642891491394802, - dst: 3379446385462418246, + csPair := ccipchangeset.SourceDestPair{ + SourceChainSelector: 12922642891491394802, + DestChainSelector: 3379446385462418246, } - res, err := state.Chains[csPair.src].Router.IsChainSupported(nil, csPair.dst) + res, err := state.Chains[csPair.SourceChainSelector].Router.IsChainSupported(nil, csPair.DestChainSelector) lggr.Infow("IsChainSupported", "res", res, "err", err) - destChainConfig, err := state.Chains[csPair.src].FeeQuoter.GetDestChainConfig(nil, csPair.dst) + destChainConfig, err := state.Chains[csPair.SourceChainSelector].FeeQuoter.GetDestChainConfig(nil, csPair.DestChainSelector) lggr.Infow("GetDestChainConfig", "destChainConfig", destChainConfig, "err", err) // find the getFee revert _, err = p.Run(true) - //csPair := ChainSelectorPair{ - // src: 12922642891491394802, - // dst: 3379446385462418246, - //} - src, dst := env.Chains[csPair.src], env.Chains[csPair.dst] + + src, dst := env.Chains[csPair.SourceChainSelector], env.Chains[csPair.DestChainSelector] startblk := uint64(11654) - seqNum := gunMap[csPair.dst].seqNums[csPair].End.Load() + seqNum := gunMap[csPair.DestChainSelector].seqNums[csPair].End.Load() _, err = ccipchangeset.ConfirmCommitWithExpectedSeqNumRange(t, src, dst, state.Chains[3379446385462418246].OffRamp, &startblk, cciptypes.SeqNumRange{ cciptypes.SeqNum(seqNum - 1), cciptypes.SeqNum(seqNum - 1), }, false) + ccipchangeset.ConfirmExecWithSeqNrsForAll(t, *env, state, map[ccipchangeset.SourceDestPair][]uint64{ + csPair: {seqNum - 1}, + }, startBlocks) + // todo: create channels that watch for these events beforehand using WatchExecutionStateChanged and WatchCommitReportAccepted // rather than waiting for the generator to finish lokiLabels := map[string]string{} diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index 06b91a4d727..2193108701b 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -18,11 +18,6 @@ import ( "time" ) -type ChainSelectorPair struct { - src uint64 - dst uint64 -} - type SeqNumRange struct { Start *atomic.Uint64 End *atomic.Uint64 @@ -31,7 +26,7 @@ type SeqNumRange struct { type DestinationGun struct { l logger.Logger env deployment.Environment - seqNums map[ChainSelectorPair]SeqNumRange + seqNums map[ccipchangeset.SourceDestPair]SeqNumRange roundNum *atomic.Int32 chainSelector uint64 receiver common.Address @@ -40,13 +35,13 @@ type DestinationGun struct { } func NewDestinationGun(l logger.Logger, chainSelector uint64, env deployment.Environment, receiver common.Address, overrides *ccip.LoadConfig, loki *wasp.LokiClient) (*DestinationGun, error) { - seqNums := make(map[ChainSelectorPair]SeqNumRange) + seqNums := make(map[ccipchangeset.SourceDestPair]SeqNumRange) for _, cs := range env.AllChainSelectorsExcluding([]uint64{chainSelector}) { // query for the actual sequence number - seqNums[ChainSelectorPair{ - src: cs, - dst: chainSelector, + seqNums[ccipchangeset.SourceDestPair{ + SourceChainSelector: cs, + DestChainSelector: chainSelector, }] = SeqNumRange{ Start: atomic.NewUint64(0), End: atomic.NewUint64(0), @@ -107,9 +102,9 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { m.l.Errorw("Failed setting loki labels", "error", err) } - csPair := ChainSelectorPair{ - src: src, - dst: m.chainSelector, + csPair := ccipchangeset.SourceDestPair{ + SourceChainSelector: src, + DestChainSelector: m.chainSelector, } m.l.Infow("Starting transmit with ", "RoundNum", requestedRound, @@ -245,7 +240,7 @@ func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { } } -func (m *DestinationGun) GetSequenceNumberRange(csPair ChainSelectorPair) (uint64, uint64, error) { +func (m *DestinationGun) GetSequenceNumberRange(csPair ccipchangeset.SourceDestPair) (uint64, uint64, error) { if r, ok := m.seqNums[csPair]; !ok { return 0, 0, fmt.Errorf("no sequence number found for chain pair %v", csPair) } else { From 92649b9a7175988cd806d564de4955b59dcbab0b Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Thu, 16 Jan 2025 00:28:48 +0800 Subject: [PATCH 16/17] optimizations for load test --- deployment/environment/crib/ccip_deployer.go | 12 +- integration-tests/load/ccip/ccip_test.go | 160 ++++++++---------- .../load/ccip/destination_gun.go | 27 +-- integration-tests/load/ccip/helpers.go | 72 +------- integration-tests/testconfig/ccip/ccip.toml | 11 +- integration-tests/testconfig/ccip/load.go | 13 +- 6 files changed, 104 insertions(+), 191 deletions(-) diff --git a/deployment/environment/crib/ccip_deployer.go b/deployment/environment/crib/ccip_deployer.go index ded754f924a..e7668c50082 100644 --- a/deployment/environment/crib/ccip_deployer.go +++ b/deployment/environment/crib/ccip_deployer.go @@ -95,6 +95,8 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de ChainSelector: chain, }) } + + // todo: parallelize these step across all chains // set up chains chainConfigs := make(map[uint64]changeset.ChainConfig) nodeInfo, err := deployment.NodeInfo(e.NodeIDs, e.Offchain) @@ -159,15 +161,6 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de return DeployCCIPOutput{}, fmt.Errorf("failed to load onchain state: %w", err) } - // find out which rmn proxy we're pointing to - for _, chain := range chainSelectors { - r, err := state.Chains[chain].RMNProxy.GetARM(nil) - if err != nil { - return DeployCCIPOutput{}, fmt.Errorf("failed to get rmn proxy: %w", err) - } - lggr.Infow("pointed to arm for ", "chain", chain, "rmnProxy", r.String()) - } - var ocrConfigPerSelector = make(map[uint64]changeset.CCIPOCRParams) for selector := range e.Chains { //tokenConfig := changeset.NewTestTokenConfig(state.Chains[feedChainSel].USDFeeds) @@ -232,6 +225,7 @@ func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig de }, }) + // todo: parallelize // Add all lanes for src := range e.Chains { for dst := range e.Chains { diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 2aebdbb73f3..3467a01b372 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -3,7 +3,6 @@ package ccip import ( "fmt" "github.com/ethereum/go-ethereum/accounts/abi/bind" - cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" "github.com/smartcontractkit/chainlink-testing-framework/wasp" @@ -26,8 +25,6 @@ var ( SIM_CHAIN_TEST_KEY = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" ) -const CRIB_DIRECTORY = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" - // step 1: setup // Parse the test config, initialize CRIB with configurations defined // step 2: load @@ -45,7 +42,7 @@ func TestCCIPLoad_RPS(t *testing.T) { lggr.Infof("loaded ccip test config: %+v", config.CCIP.Load) userOverrides := config.CCIP.Load - timeout, err := time.ParseDuration(*userOverrides.TestTimeout) + timeout, err := time.ParseDuration(*userOverrides.LoadDuration) if err != nil { require.NoError(t, err) } @@ -55,7 +52,7 @@ func TestCCIPLoad_RPS(t *testing.T) { }) }) - cribEnv := crib.NewDevspaceEnvFromStateDir(CRIB_DIRECTORY) + cribEnv := crib.NewDevspaceEnvFromStateDir(*userOverrides.CribEnvDirectory) cribDeployOutput, err := cribEnv.GetConfig(SIM_CHAIN_TEST_KEY) require.NoError(t, err) @@ -74,24 +71,20 @@ func TestCCIPLoad_RPS(t *testing.T) { defer loki.StopNow() gunMap := make(map[uint64]*DestinationGun) - // Based on the config, initialize DestinationGun p := wasp.NewProfile() - for selector, chain := range env.Chains { - latesthdr, err := chain.Client.HeaderByNumber(ctx, nil) + + // Only create a destination gun if we have decided to send traffic to this chain + for _, cs := range *userOverrides.EnabledDestionationChains { + latesthdr, err := env.Chains[cs].Client.HeaderByNumber(ctx, nil) require.NoError(t, err) block := latesthdr.Number.Uint64() - startBlocks[selector] = &block - - // Only create a destination gun if we have decided to send traffic to this chain - for _, cs := range *userOverrides.EnabledDestionationChains { - if cs == selector { - gunMap[selector], err = NewDestinationGun(env.Logger, selector, *env, state.Chains[selector].Receiver.Address(), userOverrides, loki) - if err != nil { - lggr.Errorw("Failed to initialize DestinationGun for", "chainSelector", chain, "error", err) - t.Fatal(err) - } - } + startBlocks[cs] = &block + + gunMap[cs], err = NewDestinationGun(env.Logger, cs, *env, state.Chains[cs].Receiver.Address(), userOverrides, loki) + if err != nil { + lggr.Errorw("Failed to initialize DestinationGun for", "chainSelector", cs, "error", err) + t.Fatal(err) } } @@ -101,11 +94,11 @@ func TestCCIPLoad_RPS(t *testing.T) { GenName: "ccipLoad", LoadType: wasp.RPS, CallTimeout: 5 * time.Second, - Schedule: wasp.Plain(1, time.Duration(*userOverrides.SecondsPerRequestPerDestination)*time.Second), + Schedule: wasp.Plain(1, time.Duration(*userOverrides.RequestFrequency)*time.Second), // will need to be divided by number of chains // this schedule is per generator - // in this example, it would be 1 request per 10seconds per generator (dest chain) - // so if there are 3 generators, it would be 3 requests per 10 seconds over the network + // in this example, it would be 1 request per 5seconds per generator (dest chain) + // so if there are 3 generators, it would be 3 requests per 5 seconds over the network Gun: gun, Labels: CommonTestLabels, LokiConfig: wasp.NewLokiConfig(config.CCIP.Load.LokiEndpoint, nil, nil, nil), @@ -113,63 +106,46 @@ func TestCCIPLoad_RPS(t *testing.T) { })) } - // find get fee revert - csPair := ccipchangeset.SourceDestPair{ - SourceChainSelector: 12922642891491394802, - DestChainSelector: 3379446385462418246, - } - res, err := state.Chains[csPair.SourceChainSelector].Router.IsChainSupported(nil, csPair.DestChainSelector) - lggr.Infow("IsChainSupported", "res", res, "err", err) - - destChainConfig, err := state.Chains[csPair.SourceChainSelector].FeeQuoter.GetDestChainConfig(nil, csPair.DestChainSelector) - lggr.Infow("GetDestChainConfig", "destChainConfig", destChainConfig, "err", err) - - // find the getFee revert _, err = p.Run(true) - src, dst := env.Chains[csPair.SourceChainSelector], env.Chains[csPair.DestChainSelector] - startblk := uint64(11654) - - seqNum := gunMap[csPair.DestChainSelector].seqNums[csPair].End.Load() - _, err = ccipchangeset.ConfirmCommitWithExpectedSeqNumRange(t, src, dst, state.Chains[3379446385462418246].OffRamp, &startblk, cciptypes.SeqNumRange{ - cciptypes.SeqNum(seqNum - 1), - cciptypes.SeqNum(seqNum - 1), - }, false) - - ccipchangeset.ConfirmExecWithSeqNrsForAll(t, *env, state, map[ccipchangeset.SourceDestPair][]uint64{ - csPair: {seqNum - 1}, - }, startBlocks) + // wait for offchain to complete handling load fully + execExpectedSeqNums := make(map[ccipchangeset.SourceDestPair][]uint64) + commitExepectedSeqNums := make(map[ccipchangeset.SourceDestPair]uint64) + for _, gun := range gunMap { + for csPair := range gun.seqNums { + commitExepectedSeqNums[csPair] = gun.seqNums[csPair].End.Load() + for i := gun.seqNums[csPair].Start.Load(); i <= gun.seqNums[csPair].End.Load(); i++ { + execExpectedSeqNums[csPair] = append(execExpectedSeqNums[csPair], i) + } + } + } + ccipchangeset.ConfirmCommitForAllWithExpectedSeqNums(t, *env, state, commitExepectedSeqNums, startBlocks) + ccipchangeset.ConfirmExecWithSeqNrsForAll(t, *env, state, execExpectedSeqNums, startBlocks) - // todo: create channels that watch for these events beforehand using WatchExecutionStateChanged and WatchCommitReportAccepted - // rather than waiting for the generator to finish + // todo: create channels that subscribe to these events beforehand using WatchExecutionStateChanged and WatchCommitReportAccepted lokiLabels := map[string]string{} for chainSelector, startBlock := range startBlocks { + filterOpts := &bind.FilterOpts{ + Start: *startBlock, + End: nil, // To the latest block + Context: ctx, + } + wg.Add(1) - go func(chainSelector uint64, startBlock *uint64) { + go func(chainSelector uint64, startBlock *uint64, filterOpts *bind.FilterOpts) { defer wg.Done() lggr.Infow("Starting to query for events on ", "chainSelector", chainSelector, "startblock", startBlock) latesthdr, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, nil) require.NoError(t, err) lggr.Infow("Current block number", "chainSelector", chainSelector, "block", latesthdr.Number.Uint64()) - filterOpts := &bind.FilterOpts{ - Start: *startBlock, - End: nil, // To the latest block - Context: ctx, - } - offRamp := state.Chains[chainSelector].OffRamp // Filter CommitReportAccepted events commitIterator, err := offRamp.FilterCommitReportAccepted(filterOpts) require.NoError(t, err) - fmt.Printf("Events on commitIterator %+v", commitIterator) - for commitIterator.Next() { - fmt.Printf("Events on commitIterator inside %+v", commitIterator) - event := commitIterator.Event - fmt.Printf("CommitReportAccepted event: %+v\n", event) blockNum := commitIterator.Event.Raw.BlockNumber header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) @@ -182,49 +158,55 @@ func TestCCIPLoad_RPS(t *testing.T) { for i := root.MinSeqNr; i <= root.MaxSeqNr; i++ { // todo: push loki calls to channel? - fmt.Printf("Pushed loki for seqNumber %d\n", i) SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ EventType: committed, Timestamp: timestamp, SequenceNumber: i, }) + fmt.Printf("Pushed loki for seqNumber %d\n", i) + } } } - - // Filter ExecutionStateChanged events - execIterator, err := state.Chains[chainSelector].OffRamp.FilterExecutionStateChanged(filterOpts, []uint64{chainSelector}, nil, nil) - require.NoError(t, err) - for execIterator.Next() { - event := execIterator.Event - fmt.Printf("ExecutionStateChanged event: %+v\n", event) - - blockNum := execIterator.Event.Raw.BlockNumber - header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) + }(chainSelector, startBlock, filterOpts) + + for sourceCS := range env.Chains { + wg.Add(1) + go func(srcSelector uint64, startBlock *uint64, filterOpts *bind.FilterOpts) { + defer wg.Done() + csPair := ccipchangeset.SourceDestPair{ + SourceChainSelector: srcSelector, + DestChainSelector: chainSelector, + } + // Filter ExecutionStateChanged events + execIterator, err := state.Chains[chainSelector].OffRamp.FilterExecutionStateChanged(filterOpts, []uint64{srcSelector}, execExpectedSeqNums[csPair], nil) require.NoError(t, err) - timestamp := time.Unix(int64(header.Time), 0) - // todo: push loki calls to channel? - lokiLabels, err = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) - require.NoError(t, err) + for execIterator.Next() { + event := execIterator.Event + fmt.Printf("ExecutionStateChanged event: %+v\n", event) - fmt.Printf("Pushed loki exec for seqNumber %d\n", execIterator.Event.SequenceNumber) - SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ - EventType: executed, - Timestamp: timestamp, - GasUsed: execIterator.Event.GasUsed.Uint64(), - SequenceNumber: execIterator.Event.SequenceNumber, - }) + blockNum := execIterator.Event.Raw.BlockNumber + header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) + require.NoError(t, err) + timestamp := time.Unix(int64(header.Time), 0) - } - }(chainSelector, startBlock) + // todo: push loki calls to channel? + lokiLabels, err = setLokiLabels(execIterator.Event.SourceChainSelector, chainSelector) + require.NoError(t, err) + + SendMetricsToLoki(lggr, loki, lokiLabels, &LokiMetric{ + EventType: executed, + Timestamp: timestamp, + GasUsed: execIterator.Event.GasUsed.Uint64(), + SequenceNumber: execIterator.Event.SequenceNumber, + }) + fmt.Printf("Pushed loki exec for seqNumber %d\n", execIterator.Event.SequenceNumber) + } + }(sourceCS, startBlock, filterOpts) + } } wg.Wait() - - // Stop the chains, cleanup the environment - - // crib.StopChains(env) - // crib.StopNodes(env) } diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index 2193108701b..52714436a16 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -109,8 +109,7 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { m.l.Infow("Starting transmit with ", "RoundNum", requestedRound, "Source ChainSelector", src, - "Destination ChainSelector", m.chainSelector, - "SequenceNumber", m.seqNums[csPair].End.Load()) + "Destination ChainSelector", m.chainSelector) r := state.Chains[src].Router @@ -159,17 +158,27 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { return &wasp.Response{Error: "Could not iterate", Group: waspGroup, Failed: true} } + m.l.Infow("Transmitted message with", + "sourceChain", src, + "destChain", m.chainSelector, + "sequence number", it.Event.SequenceNumber) + SendMetricsToLoki(m.l, m.loki, lokiLabels, &LokiMetric{ EventType: transmitted, Timestamp: time.Now(), - SequenceNumber: m.seqNums[csPair].End.Load(), + SequenceNumber: it.Event.SequenceNumber, }) - if m.seqNums[csPair].End.Load() == 0 { + // if this is the first time we are sending a message, set the start sequence number + // if we ran into a concurrency issue, store the lowest sequence number + if it.Event.SequenceNumber < m.seqNums[csPair].Start.Load() || m.seqNums[csPair].End.Load() == 0 { m.seqNums[csPair].Start.Store(it.Event.SequenceNumber) } - m.seqNums[csPair].End.Store(it.Event.SequenceNumber) + // only store the greatest sequence number we have seen as the maximum + if it.Event.SequenceNumber > m.seqNums[csPair].End.Load() { + m.seqNums[csPair].End.Store(it.Event.SequenceNumber) + } return &wasp.Response{Failed: false, Group: waspGroup} } @@ -239,11 +248,3 @@ func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { return messages[2], nil } } - -func (m *DestinationGun) GetSequenceNumberRange(csPair ccipchangeset.SourceDestPair) (uint64, uint64, error) { - if r, ok := m.seqNums[csPair]; !ok { - return 0, 0, fmt.Errorf("no sequence number found for chain pair %v", csPair) - } else { - return r.Start.Load(), r.End.Load(), nil - } -} diff --git a/integration-tests/load/ccip/helpers.go b/integration-tests/load/ccip/helpers.go index 8634f49edbf..d7e1beec552 100644 --- a/integration-tests/load/ccip/helpers.go +++ b/integration-tests/load/ccip/helpers.go @@ -1,18 +1,10 @@ package ccip import ( - "context" - "encoding/json" "fmt" chainselectors "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-testing-framework/wasp" - "github.com/smartcontractkit/chainlink/deployment" - "github.com/smartcontractkit/chainlink/deployment/environment/devenv" - "github.com/stretchr/testify/require" - "io" - "os" - "testing" "time" ) @@ -20,12 +12,9 @@ const ( transmitted = iota committed executed - LokiLoadLabel = "ccip_load_test" - ErrLokiClient = "failed to create Loki client for monitoring" - ErrLokiPush = "failed to push metrics to Loki" - abPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json" - nodeIdsPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-node-details.json" - chainConfigPath = "/Users/austin.wang/ccip-core/repos/chainlink/integration-tests/load/ccip/testfiles/ccip-v2-scripts-chains-details.json" + LokiLoadLabel = "ccip_load_test" + ErrLokiClient = "failed to create Loki client for monitoring" + ErrLokiPush = "failed to push metrics to Loki" ) // todo: Have a different struct for commit/exec? @@ -58,58 +47,3 @@ func setLokiLabels(src, dst uint64) (map[string]string, error) { "testType": LokiLoadLabel, }, nil } - -func readFile(filePath string) []byte { - file, err := os.Open(filePath) - if err != nil { - fmt.Println("Error opening file:", err) - panic(err) - } - defer file.Close() - - // Read the file's content into a byte slice - byteValue, err := io.ReadAll(file) - if err != nil { - fmt.Println("Error reading file:", err) - panic(err) - } - return byteValue -} - -func readFromFile[T []string | *deployment.AddressBookMap | []devenv.ChainConfig](t *testing.T, inputDir string) T { - byteValue := readFile(inputDir) - - var result T - // Unmarshal the JSON into the map - err := json.Unmarshal(byteValue, &result) - require.NoError(t, err) - - // Print the deserialized map - fmt.Println(result) - return result -} - -func CreateEnvironmentFromCribOutput(t *testing.T, lggr logger.Logger) (*deployment.Environment, error) { - ab := readFromFile[*deployment.AddressBookMap](t, abPath) - nodeIds := readFromFile[[]string](t, nodeIdsPath) - chainDetails := readFromFile[[]devenv.ChainConfig](t, chainConfigPath) - - // todo: make sure to call chainDetails.SetDeployerKey() for each chain - // where private keys should be stored in env vars or .toml - - chains, err := devenv.NewChains(lggr, chainDetails) - if err != nil { - return nil, err - } - return deployment.NewEnvironment( - "Crib Environment", - lggr, - ab, - chains, - nil, - nodeIds, - nil, - func() context.Context { return context.Background() }, - deployment.XXXGenerateTestOCRSecrets(), - ), nil -} diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index d9eaeced856..176758582f4 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -244,11 +244,12 @@ ephemeral_addresses_number = 0 [Load.CCIP.Load] # replace this with the loki endpoint of the crib stack LokiEndpoint = "http://loki.localhost/loki/api/v1/push" -NoOfNodes = 4 -TestTimeout = "1m" +LoadDuration = "1m" # corresponds to [data only, token only, message with data] MessageTypeWeights = [100,0,0] -# each destination chain will have 1 request per 5 seconds -SecondsPerRequestPerDestination = 5 +# each destination chain will receive 1 request per RequestFrequency +RequestFrequency = 10 # destination chain selectors to send messages to -EnabledDestionationChains = [3379446385462418246] \ No newline at end of file +EnabledDestionationChains = [3379446385462418246] +# Directory where we receive environment configuration from crib +CribEnvDirectory = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" \ No newline at end of file diff --git a/integration-tests/testconfig/ccip/load.go b/integration-tests/testconfig/ccip/load.go index 57bbc5e37e9..c3b80275c88 100644 --- a/integration-tests/testconfig/ccip/load.go +++ b/integration-tests/testconfig/ccip/load.go @@ -1,10 +1,11 @@ package ccip type LoadConfig struct { - TestTimeout *string - NoOfNodes *int - LokiEndpoint *string - MessageTypeWeights *[]int - SecondsPerRequestPerDestination *int - EnabledDestionationChains *[]uint64 + LoadDuration *string + NoOfNodes *int + LokiEndpoint *string + MessageTypeWeights *[]int + RequestFrequency *int + EnabledDestionationChains *[]uint64 + CribEnvDirectory *string } From 6e439dfff25750fa76325fe69943f09aee1b8102 Mon Sep 17 00:00:00 2001 From: 0xAustinWang Date: Thu, 16 Jan 2025 02:06:40 +0800 Subject: [PATCH 17/17] cleanup --- deployment/environment/crib/helpers.go | 16 ++++------ integration-tests/load/ccip/ccip_test.go | 18 ++++++----- .../load/ccip/destination_gun.go | 31 ++++++++++++------- .../ccip-v2-scripts-address-book.json | 1 - .../ccip-v2-scripts-nodes-details.json | 1 - integration-tests/testconfig/ccip/ccip.toml | 10 +++--- integration-tests/testconfig/ccip/load.go | 2 +- 7 files changed, 43 insertions(+), 36 deletions(-) delete mode 100644 integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json delete mode 100644 integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json diff --git a/deployment/environment/crib/helpers.go b/deployment/environment/crib/helpers.go index 965122e8ee6..41d2fa7fa28 100644 --- a/deployment/environment/crib/helpers.go +++ b/deployment/environment/crib/helpers.go @@ -2,7 +2,6 @@ package crib import ( "context" - "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -17,7 +16,6 @@ import ( ) func distributeFunds(lggr logger.Logger, nodeInfo []devenv.Node, env deployment.Environment) { - fmt.Println("Funding transmitters...") transmittersStr := make([]common.Address, 0) fundingAmount := big.NewInt(500_000_000_000_000_000) // 0.5 ETH minThreshold := big.NewInt(50_000_000_000_000_000) // 0.05 ETH @@ -34,17 +32,15 @@ func distributeFunds(lggr logger.Logger, nodeInfo []devenv.Node, env deployment. addr := common.HexToAddress(n.AccountAddr[chainId]) balance, err := chain.Client.BalanceAt(context.Background(), addr, nil) if err != nil { - fmt.Printf("Error fetching balance for %s: %v\n", n, err) + lggr.Errorw("error fetching balance for %s: %v\n", n.Name, err) continue } if balance.Cmp(minThreshold) < 0 { - fmt.Printf( - "Transmitter %s with address %s, has insufficient funds, funding with %s ETH. Current balance: %s, threshold: %s\n", - n.Name, - addr.String(), - conversions.WeiToEther(fundingAmount).String(), - conversions.WeiToEther(balance).String(), - conversions.WeiToEther(minThreshold).String(), + lggr.Infow( + "sending funds to", + "node", n.Name, + "address", addr.String(), + "amount", conversions.WeiToEther(fundingAmount).String(), ) transmittersStr = append(transmittersStr, addr) } diff --git a/integration-tests/load/ccip/ccip_test.go b/integration-tests/load/ccip/ccip_test.go index 3467a01b372..9b8dc056fbe 100644 --- a/integration-tests/load/ccip/ccip_test.go +++ b/integration-tests/load/ccip/ccip_test.go @@ -1,7 +1,6 @@ package ccip import ( - "fmt" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" @@ -88,13 +87,21 @@ func TestCCIPLoad_RPS(t *testing.T) { } } + loadDuration, err := time.ParseDuration(*userOverrides.LoadDuration) + require.NoError(t, err) + requestFrequency, err := time.ParseDuration(*userOverrides.RequestFrequency) + require.NoError(t, err) + for _, gun := range gunMap { p.Add(wasp.NewGenerator(&wasp.Config{ T: t, GenName: "ccipLoad", LoadType: wasp.RPS, CallTimeout: 5 * time.Second, - Schedule: wasp.Plain(1, time.Duration(*userOverrides.RequestFrequency)*time.Second), + // 1 request per second for n seconds + Schedule: wasp.Plain(1, loadDuration), + // limit requests to 1 per duration + RateLimitUnitDuration: requestFrequency, // will need to be divided by number of chains // this schedule is per generator // in this example, it would be 1 request per 5seconds per generator (dest chain) @@ -164,7 +171,7 @@ func TestCCIPLoad_RPS(t *testing.T) { Timestamp: timestamp, SequenceNumber: i, }) - fmt.Printf("Pushed loki for seqNumber %d\n", i) + lggr.Infow("pushed loki commit event for ", "seqNumber", i, "src", root.SourceChainSelector, "dest", chainSelector) } } @@ -184,9 +191,6 @@ func TestCCIPLoad_RPS(t *testing.T) { require.NoError(t, err) for execIterator.Next() { - event := execIterator.Event - fmt.Printf("ExecutionStateChanged event: %+v\n", event) - blockNum := execIterator.Event.Raw.BlockNumber header, err := env.Chains[chainSelector].Client.HeaderByNumber(ctx, big.NewInt(int64(blockNum))) require.NoError(t, err) @@ -202,7 +206,7 @@ func TestCCIPLoad_RPS(t *testing.T) { GasUsed: execIterator.Event.GasUsed.Uint64(), SequenceNumber: execIterator.Event.SequenceNumber, }) - fmt.Printf("Pushed loki exec for seqNumber %d\n", execIterator.Event.SequenceNumber) + lggr.Infow("pushed loki exec event for ", "seqNumber", execIterator.Event.SequenceNumber, "src", execIterator.Event.SourceChainSelector, "dest", chainSelector) } }(sourceCS, startBlock, filterOpts) } diff --git a/integration-tests/load/ccip/destination_gun.go b/integration-tests/load/ccip/destination_gun.go index 52714436a16..cb9959e3b1a 100644 --- a/integration-tests/load/ccip/destination_gun.go +++ b/integration-tests/load/ccip/destination_gun.go @@ -106,10 +106,6 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { SourceChainSelector: src, DestChainSelector: m.chainSelector, } - m.l.Infow("Starting transmit with ", - "RoundNum", requestedRound, - "Source ChainSelector", src, - "Destination ChainSelector", m.chainSelector) r := state.Chains[src].Router @@ -121,29 +117,43 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { fee, err := r.GetFee( &bind.CallOpts{Context: context.Background()}, m.chainSelector, msg) if err != nil { - m.l.Errorw("could not get fee ", "dstChainSelector", m.chainSelector, "msg", msg, "fee", fee, "err", err) + m.l.Errorw("could not get fee ", + "dstChainSelector", m.chainSelector, + "msg", msg, + "fee", fee, + "err", deployment.MaybeDataErr(err)) return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } - m.l.Debugw("setting fee for ", "srcChain", src, "dstChain", m.chainSelector, "fee", fee, "msg", msg) if msg.FeeToken == common.HexToAddress("0x0") { m.env.Chains[src].DeployerKey.Value = fee defer func() { m.env.Chains[src].DeployerKey.Value = nil }() } + m.l.Debugw("sending message ", + "srcChain", src, + "dstChain", m.chainSelector, + "round", requestedRound, + "fee", fee, + "msg", msg) tx, err := r.CcipSend( m.env.Chains[src].DeployerKey, m.chainSelector, msg) if err != nil { - m.l.Errorw("execution reverted from ", "sourceChain", src, "destchain", m.chainSelector, "err", err, "tx", tx) + m.l.Errorw("execution reverted from ", + "sourceChain", src, + "destchain", m.chainSelector, + "err", deployment.MaybeDataErr(err)) return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } blockNum, err := m.env.Chains[src].Confirm(tx) if err != nil { - m.l.Errorw("could not confirm tx on source", "tx", tx, "err", err) + m.l.Errorw("could not confirm tx on source", "tx", tx, "err", deployment.MaybeDataErr(err)) return &wasp.Response{Error: err.Error(), Group: waspGroup, Failed: true} } + // todo: wasp should not manage confirming the message + // instead, we should manage the sequence number atomically (at a higher level) it, err := state.Chains[src].OnRamp.FilterCCIPMessageSent(&bind.FilterOpts{ Start: blockNum, End: &blockNum, @@ -185,7 +195,6 @@ func (m *DestinationGun) Call(_ *wasp.Generator) *wasp.Response { // MustSourceChain will return a chain selector to send a message from func (m *DestinationGun) MustSourceChain() (uint64, error) { - // TODO: make this smarter by checking if this chain has sent a message recently, if so, switch to the next chain // Currently performing a round robin otherCS := m.env.AllChainSelectorsExcluding([]uint64{m.chainSelector}) @@ -207,7 +216,7 @@ func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { messages := []router.ClientEVM2AnyMessage{ { Receiver: rcv, - Data: common.Hex2Bytes("message"), + Data: common.Hex2Bytes("0xabcdefabcdef"), TokenAmounts: nil, FeeToken: common.HexToAddress("0x0"), ExtraArgs: nil, @@ -220,7 +229,7 @@ func (m *DestinationGun) GetMessage() (router.ClientEVM2AnyMessage, error) { Amount: big.NewInt(100), }, }, - Data: common.Hex2Bytes("hello world"), + Data: common.Hex2Bytes("0xabcdefabcdef"), FeeToken: common.HexToAddress("0x0"), ExtraArgs: nil, }, diff --git a/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json deleted file mode 100644 index e4b2672cb5f..00000000000 --- a/integration-tests/load/ccip/testfiles/ccip-v2-scripts-address-book.json +++ /dev/null @@ -1 +0,0 @@ -{"12922642891491394802":{"0x05Aa229Aec102f78CE0E852A812a388F076Aa555":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x0D4ff719551E23185Aeb16FFbF2ABEbB90635942":{"Type":"TestRouter","Version":"1.2.0"},"0x0f5D1ef48f12b6f691401bfe88c2037c690a6afe":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x2dE080e97B0caE9825375D31f5D0eD5751fDf16D":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x2fc631e4B3018258759C52AF169200213e84ABab":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x5C7c905B505f0Cf40Ab6600d05e677F717916F6B":{"Type":"Router","Version":"1.2.0"},"0x63cf2Cd54fE91e3545D1379abf5bfd194545259d":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x712516e61C8B383dF4A63CFe83d7701Bce54B03e":{"Type":"LinkToken","Version":"1.0.0"},"0x71C95911E9a5D330f4D621842EC243EE1343292e":{"Type":"PriceFeed","Version":"1.0.0"},"0x73eccD6288e117cAcA738BDAD4FEC51312166C1A":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x8464135c8F25Da09e49BC8782676a84730C318bC":{"Type":"PriceFeed","Version":"1.0.0"},"0x85C5Dd61585773423e378146D4bEC6f8D149E248":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x948B3c65b89DF0B4894ABE91E6D02FE579834F8F":{"Type":"WETH9","Version":"1.0.0"},"0xAfe1b5bdEbD4ae65AF2024738bf0735fbb65d44b":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xC6bA8C3233eCF65B761049ef63466945c362EdD2":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0xbCF26943C0197d2eE0E5D05c716Be60cc2761508":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0xcA03Dc4665A8C3603cb4Fd5Ce71Af9649dC00d44":{"Type":"RBACTimelock","Version":"1.0.0"},"0xe6b98F104c1BEf218F3893ADab4160Dc73Eb8367":{"Type":"ARMProxy","Version":"1.0.0"},"0xfbAb4aa40C202E4e80390171E82379824f7372dd":{"Type":"NonceManager","Version":"1.6.0-dev"}},"3379446385462418246":{"0x09635F643e140090A9A8Dcd712eD6285858ceBef":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x0B306BF915C4d645ff596e518fAf3F9669b97016":{"Type":"LinkToken","Version":"1.0.0"},"0x1613beB3B2C4f22Ee086B2b38C1476A3cE7f78E8":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6":{"Type":"CCIPHome","Version":"1.6.0-dev"},"0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x3Aa5ebB10DC797CAC828524e59A333d0A371443c":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0x4A679253410272dd5232B3Ff7cF5dbB88f295319":{"Type":"RBACTimelock","Version":"1.0.0"},"0x59b670e9fA9D0A427751Af201D676719a970857b":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x67d269191c92Caf3cD7723F116c85e6E9bf55933":{"Type":"ARMProxy","Version":"1.0.0"},"0x7a2088a1bFc9d81c55368AE168C2C02570cB814F":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x851356ae760d987E095750cCeb3bC6014560891C":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x8A791620dd6260079BF849Dc5567aDC3F2FdC318":{"Type":"RMNHome","Version":"1.6.0-dev"},"0x9A676e781A523b5d0C0e43731313A708CB607508":{"Type":"WETH9","Version":"1.0.0"},"0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0x9E545E3C0baAB3E08CdfD552C960A1050f373042":{"Type":"NonceManager","Version":"1.6.0-dev"},"0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E":{"Type":"Router","Version":"1.2.0"},"0xa513E6E4b8f2a923D98304ec87F64353C4D5C853":{"Type":"CapabilitiesRegistry","Version":"1.0.0"},"0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690":{"Type":"TestRouter","Version":"1.2.0"}}} diff --git a/integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json b/integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json deleted file mode 100644 index 477ae0527b1..00000000000 --- a/integration-tests/load/ccip/testfiles/ccip-v2-scripts-nodes-details.json +++ /dev/null @@ -1 +0,0 @@ -{"NodeIDs":["node_2URuou3RXmtZu5gLQX8qd","node_m9TTQbUxBx3WjDEjmpVDL","node_4FiKVPtuQjCTvHnS7QpES","node_A4VTgecDwMoG2YYicyjuG","node_jQFpzXDadzaADq147nThS"]} diff --git a/integration-tests/testconfig/ccip/ccip.toml b/integration-tests/testconfig/ccip/ccip.toml index 176758582f4..174efa452a0 100644 --- a/integration-tests/testconfig/ccip/ccip.toml +++ b/integration-tests/testconfig/ccip/ccip.toml @@ -244,12 +244,12 @@ ephemeral_addresses_number = 0 [Load.CCIP.Load] # replace this with the loki endpoint of the crib stack LokiEndpoint = "http://loki.localhost/loki/api/v1/push" -LoadDuration = "1m" -# corresponds to [data only, token only, message with data] +# MessageTypeWeights corresponds with [data only, token only, message with data] MessageTypeWeights = [100,0,0] -# each destination chain will receive 1 request per RequestFrequency -RequestFrequency = 10 +# each destination chain will receive 1 incoming request per RequestFrequency for the duration of LoadDuration +LoadDuration = "1m" +RequestFrequency = "10s" # destination chain selectors to send messages to -EnabledDestionationChains = [3379446385462418246] +EnabledDestionationChains = [3379446385462418246, 909606746561742123] # Directory where we receive environment configuration from crib CribEnvDirectory = "/Users/austin.wang/ccip-core/repos/crib/deployments/ccip-v2/.tmp" \ No newline at end of file diff --git a/integration-tests/testconfig/ccip/load.go b/integration-tests/testconfig/ccip/load.go index c3b80275c88..84610132f67 100644 --- a/integration-tests/testconfig/ccip/load.go +++ b/integration-tests/testconfig/ccip/load.go @@ -5,7 +5,7 @@ type LoadConfig struct { NoOfNodes *int LokiEndpoint *string MessageTypeWeights *[]int - RequestFrequency *int + RequestFrequency *string EnabledDestionationChains *[]uint64 CribEnvDirectory *string }