From b84ff0b7fde7e0b121f9ca7793f52d88f41d95ff Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Tue, 16 Aug 2022 17:34:58 +0300 Subject: [PATCH 01/11] added fallback observers mechanism --- data/observer.go | 7 +- observer/baseNodeProvider.go | 149 ++++++++++++++++++---- observer/baseNodeProvider_test.go | 164 ++++++++++++++++++------- observer/circularQueueNodesProvider.go | 10 +- observer/simpleNodesProvider.go | 7 +- 5 files changed, 254 insertions(+), 83 deletions(-) diff --git a/data/observer.go b/data/observer.go index b43c74c5..7d3620d8 100644 --- a/data/observer.go +++ b/data/observer.go @@ -2,9 +2,10 @@ package data // NodeData holds an observer data type NodeData struct { - ShardId uint32 - Address string - IsSynced bool + ShardId uint32 + Address string + IsSynced bool + IsFallback bool } // NodesReloadResponse is a DTO that holds details about nodes reloading diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index f0684eed..df155a3b 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -12,11 +12,13 @@ import ( ) type baseNodeProvider struct { - mutNodes sync.RWMutex - nodesMap map[uint32][]*data.NodeData - configurationFilePath string - syncedNodes []*data.NodeData - outOfSyncNodes []*data.NodeData + mutNodes sync.RWMutex + nodesMap map[uint32][]*data.NodeData + configurationFilePath string + syncedNodes []*data.NodeData + outOfSyncNodes []*data.NodeData + syncedFallbackNodes []*data.NodeData + outOfSyncFallbackNodes []*data.NodeData } func (bnp *baseNodeProvider) initNodesMaps(nodes []*data.NodeData) error { @@ -32,8 +34,9 @@ func (bnp *baseNodeProvider) initNodesMaps(nodes []*data.NodeData) error { bnp.mutNodes.Lock() bnp.nodesMap = newNodes - bnp.syncedNodes = initAllNodesSlice(newNodes) + bnp.syncedNodes, bnp.syncedFallbackNodes = initAllNodesSlice(newNodes) bnp.outOfSyncNodes = make([]*data.NodeData, 0) + bnp.outOfSyncFallbackNodes = make([]*data.NodeData, 0) bnp.mutNodes.Unlock() return nil @@ -68,7 +71,8 @@ func (bnp *baseNodeProvider) UpdateNodesBasedOnSyncState(nodesWithSyncStatus []* return } - if len(bnp.syncedNodes) == len(syncedNodes) && len(outOfSyncNodes) == 0 { + totalSyncedNodes := len(bnp.syncedNodes) + len(bnp.syncedFallbackNodes) + if totalSyncedNodes == len(syncedNodes) && len(outOfSyncNodes) == 0 { bnp.printSyncedNodesInShardsUnprotected() // early exit as all the nodes are in sync return @@ -84,12 +88,17 @@ func (bnp *baseNodeProvider) UpdateNodesBasedOnSyncState(nodesWithSyncStatus []* func (bnp *baseNodeProvider) printSyncedNodesInShardsUnprotected() { for shardID, nodes := range bnp.nodesMap { inSyncAddresses := make([]string, 0) + fallbackAddresses := make([]string, 0) for _, node := range nodes { inSyncAddresses = append(inSyncAddresses, node.Address) + if node.IsFallback { + fallbackAddresses = append(fallbackAddresses, node.Address) + } } log.Info(fmt.Sprintf("shard %d active nodes", shardID), "observers count", len(nodes), - "addresses", strings.Join(inSyncAddresses, ", ")) + "addresses", strings.Join(inSyncAddresses, ", "), + "fallback addresses", strings.Join(fallbackAddresses, ", ")) } } @@ -128,7 +137,12 @@ func computeSyncedAndOutOfSyncNodes(nodes []*data.NodeData, shardIDs []uint32) ( func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*data.NodeData) { for _, node := range receivedSyncedNodes { - if bnp.isReceivedSyncedNodeExistent(node) { + if bnp.isReceivedSyncedNodeExistent(node) || bnp.isReceivedSyncedNodeExistentAsFallback(node) { + continue + } + + if node.IsFallback { + bnp.syncedFallbackNodes = append(bnp.syncedFallbackNodes, node) continue } @@ -136,6 +150,10 @@ func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*da } bnp.nodesMap = nodesSliceToShardedMap(bnp.syncedNodes) + syncedFallbackNodesMap := nodesSliceToShardedMap(bnp.syncedFallbackNodes) + for shardId := range syncedFallbackNodesMap { + bnp.nodesMap[shardId] = append(bnp.nodesMap[shardId], syncedFallbackNodesMap[shardId]...) + } } func (bnp *baseNodeProvider) isReceivedSyncedNodeExistent(receivedNode *data.NodeData) bool { @@ -148,18 +166,33 @@ func (bnp *baseNodeProvider) isReceivedSyncedNodeExistent(receivedNode *data.Nod return false } +func (bnp *baseNodeProvider) isReceivedSyncedNodeExistentAsFallback(receivedNode *data.NodeData) bool { + for _, node := range bnp.syncedFallbackNodes { + if node.Address == receivedNode.Address && node.ShardId == receivedNode.ShardId { + return true + } + } + + return false +} + func (bnp *baseNodeProvider) addToOutOfSyncUnprotected(node *data.NodeData) { - if bnp.outOfSyncNodes == nil { - bnp.outOfSyncNodes = make([]*data.NodeData, 0) + source := &bnp.outOfSyncNodes + if node.IsFallback { + source = &bnp.outOfSyncFallbackNodes + } + + if *source == nil { + *source = make([]*data.NodeData, 0) } - for _, oosNode := range bnp.outOfSyncNodes { + for _, oosNode := range *source { if oosNode.Address == node.Address && oosNode.ShardId == node.ShardId { return } } - bnp.outOfSyncNodes = append(bnp.outOfSyncNodes, node) + *source = append(*source, node) } func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( @@ -168,21 +201,41 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( ) { if len(outOfSyncNodes) == 0 { bnp.outOfSyncNodes = make([]*data.NodeData, 0) + bnp.outOfSyncFallbackNodes = make([]*data.NodeData, 0) return } + syncedNodes, syncedFallbackNodes := splitSyncedNodes(syncedNodesMap) + for _, outOfSyncNode := range outOfSyncNodes { - if len(syncedNodesMap[outOfSyncNode.ShardId]) < 1 { - log.Warn("cannot remove observer as not enough will remain in shard", - "address", outOfSyncNode.Address, - "shard", outOfSyncNode.ShardId) - continue + hasSyncedNodes := len(syncedNodes[outOfSyncNode.ShardId]) >= 1 + if !outOfSyncNode.IsFallback && !hasSyncedNodes { + hasSyncedFallbackNodes := len(syncedFallbackNodes[outOfSyncNode.ShardId]) >= 1 + if !hasSyncedFallbackNodes { + log.Warn("cannot remove observer as not enough will remain in shard", + "address", outOfSyncNode.Address, + "shard", outOfSyncNode.ShardId) + continue + } + + log.Info("not enough nodes in shard, using synced fallback nodes", "shard", outOfSyncNode.ShardId) + bnp.moveFallbackNodesToSyncedUnprotected(syncedFallbackNodes[outOfSyncNode.ShardId]) } bnp.removeNodeUnprotected(outOfSyncNode) } } +func (bnp *baseNodeProvider) moveFallbackNodesToSyncedUnprotected(syncedFallbackNodes []*data.NodeData) { + for _, node := range syncedFallbackNodes { + if bnp.isReceivedSyncedNodeExistent(node) { + continue + } + + bnp.syncedNodes = append(bnp.syncedNodes, node) + } +} + func (bnp *baseNodeProvider) removeNodeUnprotected(node *data.NodeData) { bnp.removeNodeFromShardedMapUnprotected(node) bnp.removeNodeFromSyncedNodesUnprotected(node) @@ -220,8 +273,13 @@ func (bnp *baseNodeProvider) removeNodeFromShardedMapUnprotected(node *data.Node } func (bnp *baseNodeProvider) removeNodeFromSyncedNodesUnprotected(nodeToRemove *data.NodeData) { + source := &bnp.syncedNodes + if nodeToRemove.IsFallback { + source = &bnp.syncedFallbackNodes + } + nodeIndex := -1 - for idx, node := range bnp.syncedNodes { + for idx, node := range *source { if node.Address == nodeToRemove.Address && node.ShardId == nodeToRemove.ShardId { nodeIndex = idx break @@ -232,9 +290,9 @@ func (bnp *baseNodeProvider) removeNodeFromSyncedNodesUnprotected(nodeToRemove * return } - copy(bnp.syncedNodes[nodeIndex:], bnp.syncedNodes[nodeIndex+1:]) - bnp.syncedNodes[len(bnp.syncedNodes)-1] = nil - bnp.syncedNodes = bnp.syncedNodes[:len(bnp.syncedNodes)-1] + copy((*source)[nodeIndex:], (*source)[nodeIndex+1:]) + (*source)[len(*source)-1] = nil + *source = (*source)[:len(*source)-1] } // ReloadNodes will reload the observers or the full history observers @@ -270,7 +328,7 @@ func (bnp *baseNodeProvider) ReloadNodes(nodesType data.NodeType) data.NodesRelo bnp.mutNodes.Lock() bnp.nodesMap = newNodes - bnp.syncedNodes = initAllNodesSlice(newNodes) + bnp.syncedNodes, bnp.syncedFallbackNodes = initAllNodesSlice(newNodes) bnp.mutNodes.Unlock() return data.NodesReloadResponse{ @@ -280,6 +338,22 @@ func (bnp *baseNodeProvider) ReloadNodes(nodesType data.NodeType) data.NodesRelo } } +func (bnp *baseNodeProvider) getSyncedNodesForShardUnprotected(shardId uint32) ([]*data.NodeData, error) { + nodesForShard, ok := bnp.nodesMap[shardId] + if !ok { + return nil, ErrShardNotAvailable + } + + syncedNodes := make([]*data.NodeData, 0) + for _, node := range nodesForShard { + if !node.IsFallback { + syncedNodes = append(syncedNodes, node) + } + } + + return syncedNodes, nil +} + func loadMainConfig(filepath string) (*config.Config, error) { cfg := &config.Config{} err := core.LoadTomlFile(cfg, filepath) @@ -312,8 +386,9 @@ func prepareReloadResponseMessage(newNodes map[uint32][]*data.NodeData) string { return retString } -func initAllNodesSlice(nodesOnShards map[uint32][]*data.NodeData) []*data.NodeData { +func initAllNodesSlice(nodesOnShards map[uint32][]*data.NodeData) ([]*data.NodeData, []*data.NodeData) { sliceToReturn := make([]*data.NodeData, 0) + fallbackNodes := make([]*data.NodeData, 0) shardIDs := getSortedSliceIDsSlice(nodesOnShards) finishedShards := make(map[uint32]struct{}) @@ -324,7 +399,12 @@ func initAllNodesSlice(nodesOnShards map[uint32][]*data.NodeData) []*data.NodeDa continue } - sliceToReturn = append(sliceToReturn, nodesOnShards[shardID][i]) + node := nodesOnShards[shardID][i] + if node.IsFallback { + fallbackNodes = append(fallbackNodes, node) + } else { + sliceToReturn = append(sliceToReturn, node) + } } if len(finishedShards) == len(nodesOnShards) { @@ -332,7 +412,7 @@ func initAllNodesSlice(nodesOnShards map[uint32][]*data.NodeData) []*data.NodeDa } } - return sliceToReturn + return sliceToReturn, fallbackNodes } func getSortedSliceIDsSlice(nodesOnShards map[uint32][]*data.NodeData) []uint32 { @@ -346,3 +426,20 @@ func getSortedSliceIDsSlice(nodesOnShards map[uint32][]*data.NodeData) []uint32 return shardIDs } + +func splitSyncedNodes(nodes map[uint32][]*data.NodeData) (map[uint32][]*data.NodeData, map[uint32][]*data.NodeData) { + syncedNodes := make(map[uint32][]*data.NodeData, 0) + syncedFallbackNodes := make(map[uint32][]*data.NodeData, 0) + for shardId, nodesInShard := range nodes { + for _, node := range nodesInShard { + if node.IsFallback { + syncedFallbackNodes[shardId] = append(syncedFallbackNodes[shardId], node) + continue + } + + syncedNodes[shardId] = append(syncedNodes[shardId], node) + } + } + + return syncedNodes, syncedFallbackNodes +} diff --git a/observer/baseNodeProvider_test.go b/observer/baseNodeProvider_test.go index de7e8c37..09f87706 100644 --- a/observer/baseNodeProvider_test.go +++ b/observer/baseNodeProvider_test.go @@ -82,28 +82,32 @@ func TestInitAllNodesSlice_BalancesNumObserversDistribution(t *testing.T) { {Address: "shard 0 - id 1"}, {Address: "shard 0 - id 2"}, {Address: "shard 0 - id 3"}, + {Address: "shard 0 - id 4", IsFallback: true}, }, 1: { {Address: "shard 1 - id 0"}, {Address: "shard 1 - id 1"}, {Address: "shard 1 - id 2"}, {Address: "shard 1 - id 3"}, + {Address: "shard 1 - id 4", IsFallback: true}, }, 2: { {Address: "shard 2 - id 0"}, {Address: "shard 2 - id 1"}, {Address: "shard 2 - id 2"}, {Address: "shard 2 - id 3"}, + {Address: "shard 2 - id 4", IsFallback: true}, }, core.MetachainShardId: { {Address: "shard meta - id 0"}, {Address: "shard meta - id 1"}, {Address: "shard meta - id 2"}, {Address: "shard meta - id 3"}, + {Address: "shard meta - id 4", IsFallback: true}, }, } - expectedOrder := []string{ + expectedSyncedOrder := []string{ "shard 0 - id 0", "shard 1 - id 0", "shard 2 - id 0", @@ -122,9 +126,20 @@ func TestInitAllNodesSlice_BalancesNumObserversDistribution(t *testing.T) { "shard meta - id 3", } - result := initAllNodesSlice(nodesMap) - for i, r := range result { - assert.Equal(t, expectedOrder[i], r.Address) + resultSynced, resultFallback := initAllNodesSlice(nodesMap) + for i, r := range resultSynced { + assert.Equal(t, expectedSyncedOrder[i], r.Address) + } + + expectedFallbackOrder := []string{ + "shard 0 - id 4", + "shard 1 - id 4", + "shard 2 - id 4", + "shard meta - id 4", + } + + for i, r := range resultFallback { + assert.Equal(t, expectedFallbackOrder[i], r.Address) } } @@ -152,10 +167,11 @@ func TestInitAllNodesSlice_UnbalancedNumObserversDistribution(t *testing.T) { {Address: "shard meta - id 2"}, {Address: "shard meta - id 3"}, {Address: "shard meta - id 4"}, + {Address: "shard meta - id 5", IsFallback: true}, }, } - expectedOrder := []string{ + expectedSyncedOrder := []string{ "shard 0 - id 0", "shard 1 - id 0", "shard 2 - id 0", @@ -171,9 +187,16 @@ func TestInitAllNodesSlice_UnbalancedNumObserversDistribution(t *testing.T) { "shard meta - id 4", } - result := initAllNodesSlice(nodesMap) - for i, r := range result { - assert.Equal(t, expectedOrder[i], r.Address) + resultSynced, resultFallback := initAllNodesSlice(nodesMap) + for i, r := range resultSynced { + assert.Equal(t, expectedSyncedOrder[i], r.Address) + } + + expectedFallbackOrder := []string{ + "shard meta - id 5", + } + for i, r := range resultFallback { + assert.Equal(t, expectedFallbackOrder[i], r.Address) } } @@ -201,7 +224,7 @@ func TestInitAllNodesSlice_EmptyObserversSliceForAShardShouldStillWork(t *testin "shard meta - id 1", } - result := initAllNodesSlice(nodesMap) + result, _ := initAllNodesSlice(nodesMap) for i, r := range result { assert.Equal(t, expectedOrder[i], r.Address) } @@ -220,7 +243,7 @@ func TestInitAllNodesSlice_SingleShardShouldWork(t *testing.T) { "shard 0 - id 0", } - result := initAllNodesSlice(nodesMap) + result, _ := initAllNodesSlice(nodesMap) for i, r := range result { assert.Equal(t, expectedOrder[i], r.Address) } @@ -229,30 +252,37 @@ func TestInitAllNodesSlice_SingleShardShouldWork(t *testing.T) { func TestBaseNodeProvider_UpdateNodesBasedOnSyncState(t *testing.T) { t.Parallel() - allNodes := prepareNodes(6) + allNodes := prepareNodes(8) + setFallbackNodes(allNodes, 0, 1, 4, 5) - nodesMap := nodesSliceToShardedMap(allNodes) bnp := &baseNodeProvider{ configurationFilePath: configurationPath, - nodesMap: nodesMap, - syncedNodes: allNodes, } + _ = bnp.initNodesMaps(allNodes) - setSyncedStateToNodes(allNodes, false, 1, 2, 4, 5) + setSyncedStateToNodes(allNodes, false, 1, 2, 5, 6) bnp.UpdateNodesBasedOnSyncState(allNodes) require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 1, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, }, convertSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ - {Address: "addr1", ShardId: 0, IsSynced: false}, + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr4", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + + require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: false}, - {Address: "addr4", ShardId: 1, IsSynced: false}, - {Address: "addr5", ShardId: 1, IsSynced: false}, + {Address: "addr6", ShardId: 1, IsSynced: false}, }, convertSlice(bnp.outOfSyncNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.outOfSyncFallbackNodes)) } func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldNotRemoveNodeIfNotEnoughLeft(t *testing.T) { @@ -281,52 +311,90 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldNotRemoveNodeIfNotEno }, convertSlice(bnp.outOfSyncNodes)) } +func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldMoveFallbackNode(t *testing.T) { + t.Parallel() + + allNodes := prepareNodes(4) + setFallbackNodes(allNodes, 0) + + bnp := &baseNodeProvider{ + configurationFilePath: configurationPath, + } + _ = bnp.initNodesMaps(allNodes) + + setSyncedStateToNodes(allNodes, false, 1) + + bnp.UpdateNodesBasedOnSyncState(allNodes) + + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 1, IsSynced: true}, + {Address: "addr3", ShardId: 1, IsSynced: true}, + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr1", ShardId: 0, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) + + require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) +} + func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIterations(t *testing.T) { t.Parallel() allNodes := prepareNodes(10) + setFallbackNodes(allNodes, 0, 1, 5, 6) - nodesMap := nodesSliceToShardedMap(allNodes) bnp := &baseNodeProvider{ configurationFilePath: configurationPath, - nodesMap: nodesMap, - syncedNodes: allNodes, } + _ = bnp.initNodesMaps(allNodes) setSyncedStateToNodes(allNodes, false, 1, 3, 5, 7, 9) bnp.UpdateNodesBasedOnSyncState(allNodes) require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true}, {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr6", ShardId: 1, IsSynced: true}, {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, }, convertSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ - {Address: "addr1", ShardId: 0, IsSynced: false}, + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ {Address: "addr3", ShardId: 0, IsSynced: false}, - {Address: "addr5", ShardId: 1, IsSynced: false}, {Address: "addr7", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, }, convertSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.outOfSyncFallbackNodes)) allNodes = prepareNodes(10) + setFallbackNodes(allNodes, 0, 1, 5, 6) bnp.UpdateNodesBasedOnSyncState(allNodes) require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true}, {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr6", ShardId: 1, IsSynced: true}, {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr1", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr5", ShardId: 1, IsSynced: true}, {Address: "addr7", ShardId: 1, IsSynced: true}, {Address: "addr9", ShardId: 1, IsSynced: true}, }, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) } func prepareNodes(count int) []*data.NodeData { @@ -352,6 +420,12 @@ func setSyncedStateToNodes(nodes []*data.NodeData, state bool, indices ...int) { } } +func setFallbackNodes(nodes []*data.NodeData, indices ...int) { + for _, idx := range indices { + nodes[idx].IsFallback = true + } +} + func convertSlice(nodes []*data.NodeData) []data.NodeData { newSlice := make([]data.NodeData, 0, len(nodes)) for _, node := range nodes { @@ -381,9 +455,9 @@ func testComputeSyncedAndOutOfSyncNodesAllNodesSynced(t *testing.T) { shardIDs := []uint32{0, 1} input := []*data.NodeData{ {Address: "0", ShardId: 0, IsSynced: true}, - {Address: "1", ShardId: 0, IsSynced: true}, + {Address: "1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "2", ShardId: 1, IsSynced: true}, - {Address: "3", ShardId: 1, IsSynced: true}, + {Address: "3", ShardId: 1, IsSynced: true, IsFallback: true}, } synced, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) @@ -398,18 +472,22 @@ func testComputeSyncedAndOutOfSyncNodesEnoughSyncedObservers(t *testing.T) { input := []*data.NodeData{ {Address: "0", ShardId: 0, IsSynced: true}, {Address: "1", ShardId: 0, IsSynced: false}, - {Address: "2", ShardId: 1, IsSynced: true}, - {Address: "3", ShardId: 1, IsSynced: false}, + {Address: "2", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "3", ShardId: 1, IsSynced: true}, + {Address: "4", ShardId: 1, IsSynced: false}, + {Address: "5", ShardId: 1, IsSynced: true, IsFallback: true}, } synced, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) require.Equal(t, []*data.NodeData{ {Address: "0", ShardId: 0, IsSynced: true}, - {Address: "2", ShardId: 1, IsSynced: true}, + {Address: "2", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "3", ShardId: 1, IsSynced: true}, + {Address: "5", ShardId: 1, IsSynced: true, IsFallback: true}, }, synced) require.Equal(t, []*data.NodeData{ {Address: "1", ShardId: 0, IsSynced: false}, - {Address: "3", ShardId: 1, IsSynced: false}, + {Address: "4", ShardId: 1, IsSynced: false}, }, notSynced) } @@ -419,9 +497,9 @@ func testComputeSyncedAndOutOfSyncNodesAllNodesNotSynced(t *testing.T) { shardIDs := []uint32{0, 1} input := []*data.NodeData{ {Address: "0", ShardId: 0, IsSynced: false}, - {Address: "1", ShardId: 0, IsSynced: false}, + {Address: "1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "2", ShardId: 1, IsSynced: false}, - {Address: "3", ShardId: 1, IsSynced: false}, + {Address: "3", ShardId: 1, IsSynced: false, IsFallback: true}, } synced, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) @@ -430,8 +508,8 @@ func testComputeSyncedAndOutOfSyncNodesAllNodesNotSynced(t *testing.T) { {Address: "2", ShardId: 1, IsSynced: false}, }, synced) require.Equal(t, []*data.NodeData{ - {Address: "1", ShardId: 0, IsSynced: false}, - {Address: "3", ShardId: 1, IsSynced: false}, + {Address: "1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "3", ShardId: 1, IsSynced: false, IsFallback: true}, }, notSynced) } diff --git a/observer/circularQueueNodesProvider.go b/observer/circularQueueNodesProvider.go index 5b400801..9ebbab5e 100644 --- a/observer/circularQueueNodesProvider.go +++ b/observer/circularQueueNodesProvider.go @@ -39,13 +39,13 @@ func (cqnp *circularQueueNodesProvider) GetNodesByShardId(shardId uint32) ([]*da cqnp.mutNodes.Lock() defer cqnp.mutNodes.Unlock() - nodesForShard := cqnp.nodesMap[shardId] - if len(nodesForShard) == 0 { - return nil, ErrShardNotAvailable + syncedNodesForShard, err := cqnp.getSyncedNodesForShardUnprotected(shardId) + if err != nil { + return nil, err } - position := cqnp.computeCounterForShard(shardId, uint32(len(nodesForShard))) - sliceToRet := append(nodesForShard[position:], nodesForShard[:position]...) + position := cqnp.computeCounterForShard(shardId, uint32(len(syncedNodesForShard))) + sliceToRet := append(syncedNodesForShard[position:], syncedNodesForShard[:position]...) return sliceToRet, nil } diff --git a/observer/simpleNodesProvider.go b/observer/simpleNodesProvider.go index ab49803c..80996534 100644 --- a/observer/simpleNodesProvider.go +++ b/observer/simpleNodesProvider.go @@ -31,12 +31,7 @@ func (snp *simpleNodesProvider) GetNodesByShardId(shardId uint32) ([]*data.NodeD snp.mutNodes.RLock() defer snp.mutNodes.RUnlock() - nodesForShard, ok := snp.nodesMap[shardId] - if !ok { - return nil, ErrShardNotAvailable - } - - return nodesForShard, nil + return snp.getSyncedNodesForShardUnprotected(shardId) } // GetAllNodes will return a slice containing all the nodes From 2790b8be232da25a50c99117528fa72547166a60 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Tue, 16 Aug 2022 20:01:23 +0300 Subject: [PATCH 02/11] removed isFallback check --- observer/baseNodeProvider.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index df155a3b..33b96d3b 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -346,9 +346,7 @@ func (bnp *baseNodeProvider) getSyncedNodesForShardUnprotected(shardId uint32) ( syncedNodes := make([]*data.NodeData, 0) for _, node := range nodesForShard { - if !node.IsFallback { - syncedNodes = append(syncedNodes, node) - } + syncedNodes = append(syncedNodes, node) } return syncedNodes, nil From 832ab3152835e700f7694ceb39a12c76b624f89b Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 17 Aug 2022 15:13:12 +0300 Subject: [PATCH 03/11] fixes after testing --- observer/baseNodeProvider.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index 33b96d3b..f6411563 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -54,6 +54,12 @@ func (bnp *baseNodeProvider) GetAllNodesWithSyncState() []*data.NodeData { for _, node := range bnp.outOfSyncNodes { nodesSlice = append(nodesSlice, node) } + for _, node := range bnp.syncedFallbackNodes { + nodesSlice = append(nodesSlice, node) + } + for _, node := range bnp.outOfSyncFallbackNodes { + nodesSlice = append(nodesSlice, node) + } return nodesSlice } @@ -90,10 +96,11 @@ func (bnp *baseNodeProvider) printSyncedNodesInShardsUnprotected() { inSyncAddresses := make([]string, 0) fallbackAddresses := make([]string, 0) for _, node := range nodes { - inSyncAddresses = append(inSyncAddresses, node.Address) if node.IsFallback { fallbackAddresses = append(fallbackAddresses, node.Address) + continue } + inSyncAddresses = append(inSyncAddresses, node.Address) } log.Info(fmt.Sprintf("shard %d active nodes", shardID), "observers count", len(nodes), @@ -208,8 +215,8 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( syncedNodes, syncedFallbackNodes := splitSyncedNodes(syncedNodesMap) for _, outOfSyncNode := range outOfSyncNodes { - hasSyncedNodes := len(syncedNodes[outOfSyncNode.ShardId]) >= 1 - if !outOfSyncNode.IsFallback && !hasSyncedNodes { + hasEnoughSyncedNodes := len(syncedNodes[outOfSyncNode.ShardId]) > 1 + if !outOfSyncNode.IsFallback && !hasEnoughSyncedNodes { hasSyncedFallbackNodes := len(syncedFallbackNodes[outOfSyncNode.ShardId]) >= 1 if !hasSyncedFallbackNodes { log.Warn("cannot remove observer as not enough will remain in shard", @@ -233,6 +240,7 @@ func (bnp *baseNodeProvider) moveFallbackNodesToSyncedUnprotected(syncedFallback } bnp.syncedNodes = append(bnp.syncedNodes, node) + bnp.removeNodeFromSyncedNodesUnprotected(node) } } @@ -339,14 +347,15 @@ func (bnp *baseNodeProvider) ReloadNodes(nodesType data.NodeType) data.NodesRelo } func (bnp *baseNodeProvider) getSyncedNodesForShardUnprotected(shardId uint32) ([]*data.NodeData, error) { - nodesForShard, ok := bnp.nodesMap[shardId] - if !ok { - return nil, ErrShardNotAvailable + syncedNodes := make([]*data.NodeData, 0) + for _, node := range bnp.syncedNodes { + if node.ShardId == shardId { + syncedNodes = append(syncedNodes, node) + } } - syncedNodes := make([]*data.NodeData, 0) - for _, node := range nodesForShard { - syncedNodes = append(syncedNodes, node) + if len(syncedNodes) == 0 { + return nil, ErrShardNotAvailable } return syncedNodes, nil From 3d5e4871d63a557f28dfd9255980d69ec3aac41f Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 17 Aug 2022 17:05:59 +0300 Subject: [PATCH 04/11] more fixes + refactor + fixed tests --- observer/baseNodeProvider.go | 90 ++++++++++++-------------- observer/baseNodeProvider_test.go | 103 ++++++++++++------------------ process/baseProcessor.go | 3 +- 3 files changed, 84 insertions(+), 112 deletions(-) diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index f6411563..17c79aad 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -71,23 +71,25 @@ func (bnp *baseNodeProvider) UpdateNodesBasedOnSyncState(nodesWithSyncStatus []* defer bnp.mutNodes.Unlock() shardIDs := getSortedSliceIDsSlice(bnp.nodesMap) - syncedNodes, outOfSyncNodes, err := computeSyncedAndOutOfSyncNodes(nodesWithSyncStatus, shardIDs) + syncedNodes, syncedFallbackNodes, outOfSyncNodes, err := computeSyncedAndOutOfSyncNodes(nodesWithSyncStatus, shardIDs) if err != nil { log.Error("cannot update nodes based on sync state", "error", err) return } - totalSyncedNodes := len(bnp.syncedNodes) + len(bnp.syncedFallbackNodes) - if totalSyncedNodes == len(syncedNodes) && len(outOfSyncNodes) == 0 { + sameNumOfSynced := len(bnp.syncedNodes) == len(syncedNodes) + sameNumOfSyncedFallback := len(bnp.syncedFallbackNodes) == len(syncedFallbackNodes) + if sameNumOfSynced && sameNumOfSyncedFallback && len(outOfSyncNodes) == 0 { bnp.printSyncedNodesInShardsUnprotected() // early exit as all the nodes are in sync return } syncedNodesMap := nodesSliceToShardedMap(syncedNodes) + syncedFallbackNodesMap := nodesSliceToShardedMap(syncedFallbackNodes) - bnp.removeOutOfSyncNodesUnprotected(outOfSyncNodes, syncedNodesMap) - bnp.addSyncedNodesUnprotected(syncedNodes) + bnp.removeOutOfSyncNodesUnprotected(outOfSyncNodes, syncedNodesMap, syncedFallbackNodesMap) + bnp.addSyncedNodesUnprotected(syncedNodes, syncedFallbackNodes) bnp.printSyncedNodesInShardsUnprotected() } @@ -109,13 +111,18 @@ func (bnp *baseNodeProvider) printSyncedNodesInShardsUnprotected() { } } -func computeSyncedAndOutOfSyncNodes(nodes []*data.NodeData, shardIDs []uint32) ([]*data.NodeData, []*data.NodeData, error) { +func computeSyncedAndOutOfSyncNodes(nodes []*data.NodeData, shardIDs []uint32) ([]*data.NodeData, []*data.NodeData, []*data.NodeData, error) { tempSyncedNodesMap := make(map[uint32][]*data.NodeData) + tempSyncedFallbackNodesMap := make(map[uint32][]*data.NodeData) tempNotSyncedNodesMap := make(map[uint32][]*data.NodeData) for _, node := range nodes { if node.IsSynced { - tempSyncedNodesMap[node.ShardId] = append(tempSyncedNodesMap[node.ShardId], node) + if node.IsFallback { + tempSyncedFallbackNodesMap[node.ShardId] = append(tempSyncedFallbackNodesMap[node.ShardId], node) + } else { + tempSyncedNodesMap[node.ShardId] = append(tempSyncedNodesMap[node.ShardId], node) + } continue } @@ -123,37 +130,37 @@ func computeSyncedAndOutOfSyncNodes(nodes []*data.NodeData, shardIDs []uint32) ( } syncedNodes := make([]*data.NodeData, 0) + syncedFallbackNodes := make([]*data.NodeData, 0) notSyncedNodes := make([]*data.NodeData, 0) for _, shardID := range shardIDs { - if len(tempSyncedNodesMap[shardID]) > 0 { - syncedNodes = append(syncedNodes, tempSyncedNodesMap[shardID]...) - notSyncedNodes = append(notSyncedNodes, tempNotSyncedNodesMap[shardID]...) - continue - } + syncedNodes = append(syncedNodes, tempSyncedNodesMap[shardID]...) + syncedFallbackNodes = append(syncedFallbackNodes, tempSyncedFallbackNodesMap[shardID]...) + notSyncedNodes = append(notSyncedNodes, tempNotSyncedNodesMap[shardID]...) - if len(tempNotSyncedNodesMap[shardID]) == 0 { - return nil, nil, fmt.Errorf("%w for shard %d - no synced or not synced node", ErrWrongObserversConfiguration, shardID) + totalLen := len(tempSyncedNodesMap[shardID]) + len(tempSyncedFallbackNodesMap[shardID]) + len(tempNotSyncedNodesMap[shardID]) + if totalLen == 0 { + return nil, nil, nil, fmt.Errorf("%w for shard %d - no synced or not synced node", ErrWrongObserversConfiguration, shardID) } - - syncedNodes = append(syncedNodes, tempNotSyncedNodesMap[shardID][0]) - notSyncedNodes = append(notSyncedNodes, tempNotSyncedNodesMap[shardID][1:]...) } - return syncedNodes, notSyncedNodes, nil + return syncedNodes, syncedFallbackNodes, notSyncedNodes, nil } -func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*data.NodeData) { +func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*data.NodeData, receivedSyncedFallbackNodes []*data.NodeData) { for _, node := range receivedSyncedNodes { - if bnp.isReceivedSyncedNodeExistent(node) || bnp.isReceivedSyncedNodeExistentAsFallback(node) { + if bnp.isReceivedSyncedNodeExistent(node) { continue } - if node.IsFallback { - bnp.syncedFallbackNodes = append(bnp.syncedFallbackNodes, node) + bnp.syncedNodes = append(bnp.syncedNodes, node) + } + + for _, node := range receivedSyncedFallbackNodes { + if bnp.isReceivedSyncedNodeExistentAsFallback(node) { continue } - bnp.syncedNodes = append(bnp.syncedNodes, node) + bnp.syncedFallbackNodes = append(bnp.syncedFallbackNodes, node) } bnp.nodesMap = nodesSliceToShardedMap(bnp.syncedNodes) @@ -205,6 +212,7 @@ func (bnp *baseNodeProvider) addToOutOfSyncUnprotected(node *data.NodeData) { func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( outOfSyncNodes []*data.NodeData, syncedNodesMap map[uint32][]*data.NodeData, + syncedFallbackNodesMap map[uint32][]*data.NodeData, ) { if len(outOfSyncNodes) == 0 { bnp.outOfSyncNodes = make([]*data.NodeData, 0) @@ -212,21 +220,24 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( return } - syncedNodes, syncedFallbackNodes := splitSyncedNodes(syncedNodesMap) - + recoveryNodes := make(map[uint32][]*data.NodeData, 0) for _, outOfSyncNode := range outOfSyncNodes { - hasEnoughSyncedNodes := len(syncedNodes[outOfSyncNode.ShardId]) > 1 - if !outOfSyncNode.IsFallback && !hasEnoughSyncedNodes { - hasSyncedFallbackNodes := len(syncedFallbackNodes[outOfSyncNode.ShardId]) >= 1 - if !hasSyncedFallbackNodes { + hasEnoughSyncedNodes := len(syncedNodesMap[outOfSyncNode.ShardId]) >= 1 + if !hasEnoughSyncedNodes { + hasSyncedFallbackNodes := len(syncedFallbackNodesMap[outOfSyncNode.ShardId]) >= 1 + if !hasSyncedFallbackNodes && len(recoveryNodes[outOfSyncNode.ShardId]) == 0 { log.Warn("cannot remove observer as not enough will remain in shard", "address", outOfSyncNode.Address, + "is fallback", outOfSyncNode.IsFallback, "shard", outOfSyncNode.ShardId) + recoveryNodes[outOfSyncNode.ShardId] = append(recoveryNodes[outOfSyncNode.ShardId], outOfSyncNode) continue } - log.Info("not enough nodes in shard, using synced fallback nodes", "shard", outOfSyncNode.ShardId) - bnp.moveFallbackNodesToSyncedUnprotected(syncedFallbackNodes[outOfSyncNode.ShardId]) + if !outOfSyncNode.IsFallback { + log.Info("not enough nodes in shard, using synced fallback nodes", "shard", outOfSyncNode.ShardId) + bnp.moveFallbackNodesToSyncedUnprotected(syncedFallbackNodesMap[outOfSyncNode.ShardId]) + } } bnp.removeNodeUnprotected(outOfSyncNode) @@ -433,20 +444,3 @@ func getSortedSliceIDsSlice(nodesOnShards map[uint32][]*data.NodeData) []uint32 return shardIDs } - -func splitSyncedNodes(nodes map[uint32][]*data.NodeData) (map[uint32][]*data.NodeData, map[uint32][]*data.NodeData) { - syncedNodes := make(map[uint32][]*data.NodeData, 0) - syncedFallbackNodes := make(map[uint32][]*data.NodeData, 0) - for shardId, nodesInShard := range nodes { - for _, node := range nodesInShard { - if node.IsFallback { - syncedFallbackNodes[shardId] = append(syncedFallbackNodes[shardId], node) - continue - } - - syncedNodes[shardId] = append(syncedNodes[shardId], node) - } - } - - return syncedNodes, syncedFallbackNodes -} diff --git a/observer/baseNodeProvider_test.go b/observer/baseNodeProvider_test.go index 09f87706..ba7dcbc9 100644 --- a/observer/baseNodeProvider_test.go +++ b/observer/baseNodeProvider_test.go @@ -341,6 +341,22 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldMoveFallbackNode(t *t }, convertSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) + + setSyncedStateToNodes(allNodes, false, 0) + bnp.UpdateNodesBasedOnSyncState(allNodes) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 1, IsSynced: true}, + {Address: "addr3", ShardId: 1, IsSynced: true}, + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.syncedNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr1", ShardId: 0, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) } func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIterations(t *testing.T) { @@ -441,9 +457,6 @@ func TestComputeSyncAndOutOfSyncNodes(t *testing.T) { t.Run("all nodes synced", testComputeSyncedAndOutOfSyncNodesAllNodesSynced) t.Run("enough synced nodes", testComputeSyncedAndOutOfSyncNodesEnoughSyncedObservers) t.Run("all nodes are out of sync", testComputeSyncedAndOutOfSyncNodesAllNodesNotSynced) - t.Run("all nodes are out of sync, should use only the first out of sync as synced", - testComputeSyncedAndOutOfSyncNodesNoSyncedObserversShouldOnlyGetFirstOutOfSyncObserver) - t.Run("only one out of sync node per shard", testComputeSyncedAndOutOfSyncNodesOnlyOneOutOfSyncObserverInShard) t.Run("invalid config - no node", testComputeSyncedAndOutOfSyncNodesInvalidConfigurationNoNodeAtAll) t.Run("invalid config - no node in a shard", testComputeSyncedAndOutOfSyncNodesInvalidConfigurationNoNodeInAShard) t.Run("edge case - address should not exist in both sync and not-synced lists", testEdgeCaseAddressShouldNotExistInBothLists) @@ -460,8 +473,15 @@ func testComputeSyncedAndOutOfSyncNodesAllNodesSynced(t *testing.T) { {Address: "3", ShardId: 1, IsSynced: true, IsFallback: true}, } - synced, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) - require.Equal(t, input, synced) + synced, syncedFb, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) + require.Equal(t, []*data.NodeData{ + {Address: "0", ShardId: 0, IsSynced: true}, + {Address: "2", ShardId: 1, IsSynced: true}, + }, synced) + require.Equal(t, []*data.NodeData{ + {Address: "1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "3", ShardId: 1, IsSynced: true, IsFallback: true}, + }, syncedFb) require.Empty(t, notSynced) } @@ -478,13 +498,15 @@ func testComputeSyncedAndOutOfSyncNodesEnoughSyncedObservers(t *testing.T) { {Address: "5", ShardId: 1, IsSynced: true, IsFallback: true}, } - synced, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) + synced, syncedFb, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) require.Equal(t, []*data.NodeData{ {Address: "0", ShardId: 0, IsSynced: true}, - {Address: "2", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "3", ShardId: 1, IsSynced: true}, - {Address: "5", ShardId: 1, IsSynced: true, IsFallback: true}, }, synced) + require.Equal(t, []*data.NodeData{ + {Address: "2", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "5", ShardId: 1, IsSynced: true, IsFallback: true}, + }, syncedFb) require.Equal(t, []*data.NodeData{ {Address: "1", ShardId: 0, IsSynced: false}, {Address: "4", ShardId: 1, IsSynced: false}, @@ -502,41 +524,10 @@ func testComputeSyncedAndOutOfSyncNodesAllNodesNotSynced(t *testing.T) { {Address: "3", ShardId: 1, IsSynced: false, IsFallback: true}, } - synced, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) - require.Equal(t, []*data.NodeData{ - {Address: "0", ShardId: 0, IsSynced: false}, - {Address: "2", ShardId: 1, IsSynced: false}, - }, synced) - require.Equal(t, []*data.NodeData{ - {Address: "1", ShardId: 0, IsSynced: false, IsFallback: true}, - {Address: "3", ShardId: 1, IsSynced: false, IsFallback: true}, - }, notSynced) -} - -func testComputeSyncedAndOutOfSyncNodesNoSyncedObserversShouldOnlyGetFirstOutOfSyncObserver(t *testing.T) { - t.Parallel() - - shardIDs := []uint32{0, 1} - input := []*data.NodeData{ - {Address: "0", ShardId: 0, IsSynced: false}, - {Address: "1", ShardId: 0, IsSynced: false}, - {Address: "2", ShardId: 0, IsSynced: false}, - {Address: "3", ShardId: 1, IsSynced: false}, - {Address: "4", ShardId: 1, IsSynced: false}, - {Address: "5", ShardId: 1, IsSynced: false}, - } - - synced, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) - require.Equal(t, []*data.NodeData{ - {Address: "0", ShardId: 0, IsSynced: false}, - {Address: "3", ShardId: 1, IsSynced: false}, - }, synced) - require.Equal(t, []*data.NodeData{ - {Address: "1", ShardId: 0, IsSynced: false}, - {Address: "2", ShardId: 0, IsSynced: false}, - {Address: "4", ShardId: 1, IsSynced: false}, - {Address: "5", ShardId: 1, IsSynced: false}, - }, notSynced) + synced, syncedFb, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) + require.Equal(t, []*data.NodeData{}, synced) + require.Equal(t, []*data.NodeData{}, syncedFb) + require.Equal(t, input, notSynced) } func testEdgeCaseAddressShouldNotExistInBothLists(t *testing.T) { @@ -589,31 +580,15 @@ func testEdgeCaseAddressShouldNotExistInBothLists(t *testing.T) { require.False(t, slicesHaveCommonObjects(bnp.syncedNodes, bnp.outOfSyncNodes)) } -func testComputeSyncedAndOutOfSyncNodesOnlyOneOutOfSyncObserverInShard(t *testing.T) { - t.Parallel() - - shardIDs := []uint32{0, 1} - input := []*data.NodeData{ - {Address: "0", ShardId: 0, IsSynced: false}, - {Address: "1", ShardId: 1, IsSynced: false}, - } - - synced, notSynced, _ := computeSyncedAndOutOfSyncNodes(input, shardIDs) - require.Equal(t, []*data.NodeData{ - {Address: "0", ShardId: 0, IsSynced: false}, - {Address: "1", ShardId: 1, IsSynced: false}, - }, synced) - require.Empty(t, notSynced) -} - func testComputeSyncedAndOutOfSyncNodesInvalidConfigurationNoNodeAtAll(t *testing.T) { t.Parallel() shardIDs := []uint32{0, 1} var input []*data.NodeData - synced, notSynced, err := computeSyncedAndOutOfSyncNodes(input, shardIDs) + synced, syncedFb, notSynced, err := computeSyncedAndOutOfSyncNodes(input, shardIDs) require.Error(t, err) require.Nil(t, synced) + require.Nil(t, syncedFb) require.Nil(t, notSynced) // no node in one shard @@ -623,9 +598,10 @@ func testComputeSyncedAndOutOfSyncNodesInvalidConfigurationNoNodeAtAll(t *testin Address: "0", ShardId: 0, IsSynced: true, }, } - synced, notSynced, err = computeSyncedAndOutOfSyncNodes(input, shardIDs) + synced, syncedFb, notSynced, err = computeSyncedAndOutOfSyncNodes(input, shardIDs) require.True(t, errors.Is(err, ErrWrongObserversConfiguration)) require.Nil(t, synced) + require.Nil(t, syncedFb) require.Nil(t, notSynced) } @@ -639,9 +615,10 @@ func testComputeSyncedAndOutOfSyncNodesInvalidConfigurationNoNodeInAShard(t *tes Address: "0", ShardId: 0, IsSynced: true, }, } - synced, notSynced, err := computeSyncedAndOutOfSyncNodes(input, shardIDs) + synced, syncedFb, notSynced, err := computeSyncedAndOutOfSyncNodes(input, shardIDs) require.True(t, errors.Is(err, ErrWrongObserversConfiguration)) require.Nil(t, synced) + require.Nil(t, syncedFb) require.Nil(t, notSynced) } diff --git a/process/baseProcessor.go b/process/baseProcessor.go index 54e13f1c..ff21af0e 100644 --- a/process/baseProcessor.go +++ b/process/baseProcessor.go @@ -420,7 +420,8 @@ func (bp *BaseProcessor) isNodeSynced(node *proxyData.NodeData) (bool, error) { "nonce", nonce, "probable highest nonce", probableHighestNonce, "is synced", isNodeSynced, - "is ready for VM Queries", isReadyForVMQueries) + "is ready for VM Queries", isReadyForVMQueries, + "is fallback", node.IsFallback) if !isReadyForVMQueries { isNodeSynced = false From 41b45fde8592121eccba50f0e0a72461008ce2cc Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Mon, 29 Aug 2022 20:50:30 +0300 Subject: [PATCH 05/11] fix after review + refactor + new tests --- observer/baseNodeProvider.go | 98 +++++++++++++---- observer/baseNodeProvider_test.go | 169 +++++++++++++++++++++++++++--- 2 files changed, 233 insertions(+), 34 deletions(-) diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index 17c79aad..361eff96 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -19,6 +19,7 @@ type baseNodeProvider struct { outOfSyncNodes []*data.NodeData syncedFallbackNodes []*data.NodeData outOfSyncFallbackNodes []*data.NodeData + lastSyncedNodes map[uint32]*data.NodeData } func (bnp *baseNodeProvider) initNodesMaps(nodes []*data.NodeData) error { @@ -37,6 +38,7 @@ func (bnp *baseNodeProvider) initNodesMaps(nodes []*data.NodeData) error { bnp.syncedNodes, bnp.syncedFallbackNodes = initAllNodesSlice(newNodes) bnp.outOfSyncNodes = make([]*data.NodeData, 0) bnp.outOfSyncFallbackNodes = make([]*data.NodeData, 0) + bnp.lastSyncedNodes = make(map[uint32]*data.NodeData) bnp.mutNodes.Unlock() return nil @@ -88,6 +90,7 @@ func (bnp *baseNodeProvider) UpdateNodesBasedOnSyncState(nodesWithSyncStatus []* syncedNodesMap := nodesSliceToShardedMap(syncedNodes) syncedFallbackNodesMap := nodesSliceToShardedMap(syncedFallbackNodes) + bnp.removeFallbackNodesFromSyncedUnprotected() bnp.removeOutOfSyncNodesUnprotected(outOfSyncNodes, syncedNodesMap, syncedFallbackNodesMap) bnp.addSyncedNodesUnprotected(syncedNodes, syncedFallbackNodes) bnp.printSyncedNodesInShardsUnprotected() @@ -148,6 +151,7 @@ func computeSyncedAndOutOfSyncNodes(nodes []*data.NodeData, shardIDs []uint32) ( func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*data.NodeData, receivedSyncedFallbackNodes []*data.NodeData) { for _, node := range receivedSyncedNodes { + bnp.removeFromOutOfSyncIfNeededUnprotected(node) if bnp.isReceivedSyncedNodeExistent(node) { continue } @@ -156,6 +160,7 @@ func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*da } for _, node := range receivedSyncedFallbackNodes { + bnp.removeFromOutOfSyncIfNeededUnprotected(node) if bnp.isReceivedSyncedNodeExistentAsFallback(node) { continue } @@ -170,6 +175,19 @@ func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*da } } +func (bnp *baseNodeProvider) removeFromOutOfSyncIfNeededUnprotected(node *data.NodeData) { + source := &bnp.outOfSyncNodes + if node.IsFallback { + source = &bnp.outOfSyncFallbackNodes + } + + if *source == nil { + return + } + + bnp.removeNodeFromListUnprotected(node, source) +} + func (bnp *baseNodeProvider) isReceivedSyncedNodeExistent(receivedNode *data.NodeData) bool { for _, node := range bnp.syncedNodes { if node.Address == receivedNode.Address && node.ShardId == receivedNode.ShardId { @@ -220,26 +238,45 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( return } - recoveryNodes := make(map[uint32][]*data.NodeData, 0) for _, outOfSyncNode := range outOfSyncNodes { + hasOneSyncedNode := len(syncedNodesMap[outOfSyncNode.ShardId]) >= 1 + hasEnoughSyncedFallbackNodes := len(syncedFallbackNodesMap[outOfSyncNode.ShardId]) > 1 + canDeleteFallbackNode := hasOneSyncedNode || hasEnoughSyncedFallbackNodes + if outOfSyncNode.IsFallback && canDeleteFallbackNode { + bnp.removeNodeUnprotected(outOfSyncNode) + continue + } + + // if trying to delete last fallback, use last known synced node + // if backup node does not exist, keep fallback + hasBackup := bnp.lastSyncedNodes[outOfSyncNode.ShardId] != nil + if outOfSyncNode.IsFallback && hasBackup { + bnp.removeNodeUnprotected(outOfSyncNode) + bnp.syncedNodes = append(bnp.syncedNodes, bnp.lastSyncedNodes[outOfSyncNode.ShardId]) + continue + } + hasEnoughSyncedNodes := len(syncedNodesMap[outOfSyncNode.ShardId]) >= 1 - if !hasEnoughSyncedNodes { - hasSyncedFallbackNodes := len(syncedFallbackNodesMap[outOfSyncNode.ShardId]) >= 1 - if !hasSyncedFallbackNodes && len(recoveryNodes[outOfSyncNode.ShardId]) == 0 { - log.Warn("cannot remove observer as not enough will remain in shard", - "address", outOfSyncNode.Address, - "is fallback", outOfSyncNode.IsFallback, - "shard", outOfSyncNode.ShardId) - recoveryNodes[outOfSyncNode.ShardId] = append(recoveryNodes[outOfSyncNode.ShardId], outOfSyncNode) - continue - } + if hasEnoughSyncedNodes { + bnp.removeNodeUnprotected(outOfSyncNode) + continue + } - if !outOfSyncNode.IsFallback { - log.Info("not enough nodes in shard, using synced fallback nodes", "shard", outOfSyncNode.ShardId) - bnp.moveFallbackNodesToSyncedUnprotected(syncedFallbackNodesMap[outOfSyncNode.ShardId]) - } + // trying to remove last synced node + // if fallbacks are available, save this one as backup and use fallbacks + // else, keep using this one + hasOneSyncedFallbackNode := len(syncedFallbackNodesMap[outOfSyncNode.ShardId]) >= 1 + if !hasOneSyncedFallbackNode { + log.Warn("cannot remove observer as not enough will remain in shard", + "address", outOfSyncNode.Address, + "is fallback", outOfSyncNode.IsFallback, + "shard", outOfSyncNode.ShardId) + continue } + log.Info("not enough nodes in shard, using synced fallback nodes", "shard", outOfSyncNode.ShardId) + bnp.moveFallbackNodesToSyncedUnprotected(syncedFallbackNodesMap[outOfSyncNode.ShardId]) + bnp.lastSyncedNodes[outOfSyncNode.ShardId] = outOfSyncNode bnp.removeNodeUnprotected(outOfSyncNode) } } @@ -255,6 +292,16 @@ func (bnp *baseNodeProvider) moveFallbackNodesToSyncedUnprotected(syncedFallback } } +func (bnp *baseNodeProvider) removeFallbackNodesFromSyncedUnprotected() { + syncedWithoutFallback := make([]*data.NodeData, 0) + for _, node := range bnp.syncedNodes { + if !node.IsFallback { + syncedWithoutFallback = append(syncedWithoutFallback, node) + } + } + bnp.syncedNodes = syncedWithoutFallback +} + func (bnp *baseNodeProvider) removeNodeUnprotected(node *data.NodeData) { bnp.removeNodeFromShardedMapUnprotected(node) bnp.removeNodeFromSyncedNodesUnprotected(node) @@ -297,6 +344,10 @@ func (bnp *baseNodeProvider) removeNodeFromSyncedNodesUnprotected(nodeToRemove * source = &bnp.syncedFallbackNodes } + bnp.removeNodeFromListUnprotected(nodeToRemove, source) +} + +func (bnp *baseNodeProvider) removeNodeFromListUnprotected(nodeToRemove *data.NodeData, source *[]*data.NodeData) { nodeIndex := -1 for idx, node := range *source { if node.Address == nodeToRemove.Address && node.ShardId == nodeToRemove.ShardId { @@ -348,6 +399,7 @@ func (bnp *baseNodeProvider) ReloadNodes(nodesType data.NodeType) data.NodesRelo bnp.mutNodes.Lock() bnp.nodesMap = newNodes bnp.syncedNodes, bnp.syncedFallbackNodes = initAllNodesSlice(newNodes) + bnp.lastSyncedNodes = make(map[uint32]*data.NodeData) bnp.mutNodes.Unlock() return data.NodesReloadResponse{ @@ -365,11 +417,21 @@ func (bnp *baseNodeProvider) getSyncedNodesForShardUnprotected(shardId uint32) ( } } - if len(syncedNodes) == 0 { - return nil, ErrShardNotAvailable + if len(syncedNodes) != 0 { + return syncedNodes, nil + } + + for _, node := range bnp.syncedFallbackNodes { + if node.ShardId == shardId { + syncedNodes = append(syncedNodes, node) + } + } + + if len(syncedNodes) != 0 { + return syncedNodes, nil } - return syncedNodes, nil + return nil, ErrShardNotAvailable } func loadMainConfig(filepath string) (*config.Config, error) { diff --git a/observer/baseNodeProvider_test.go b/observer/baseNodeProvider_test.go index ba7dcbc9..d81aa28a 100644 --- a/observer/baseNodeProvider_test.go +++ b/observer/baseNodeProvider_test.go @@ -260,9 +260,10 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncState(t *testing.T) { } _ = bnp.initNodesMaps(allNodes) - setSyncedStateToNodes(allNodes, false, 1, 2, 5, 6) + nodesCopy := copyNodes(allNodes) + setSyncedStateToNodes(nodesCopy, false, 1, 2, 5, 6) - bnp.UpdateNodesBasedOnSyncState(allNodes) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ {Address: "addr3", ShardId: 0, IsSynced: true}, @@ -295,19 +296,34 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldNotRemoveNodeIfNotEno configurationFilePath: configurationPath, nodesMap: nodesMap, syncedNodes: allNodes, + lastSyncedNodes: map[uint32]*data.NodeData{}, } - setSyncedStateToNodes(allNodes, false, 0, 1, 2, 3) + nodesCopy := copyNodes(allNodes) + setSyncedStateToNodes(nodesCopy, false, 0, 2) - bnp.UpdateNodesBasedOnSyncState(allNodes) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr1", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 1, IsSynced: true}, + }, convertSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false}, {Address: "addr2", ShardId: 1, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) + + setSyncedStateToNodes(nodesCopy, false, 1, 3) + + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + + require.Equal(t, []data.NodeData{ + {Address: "addr1", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 1, IsSynced: true}, }, convertSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ - {Address: "addr1", ShardId: 0, IsSynced: false}, - {Address: "addr3", ShardId: 1, IsSynced: false}, + {Address: "addr0", ShardId: 0, IsSynced: false}, + {Address: "addr2", ShardId: 1, IsSynced: false}, }, convertSlice(bnp.outOfSyncNodes)) } @@ -322,9 +338,10 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldMoveFallbackNode(t *t } _ = bnp.initNodesMaps(allNodes) - setSyncedStateToNodes(allNodes, false, 1) + nodesCopy := copyNodes(allNodes) + setSyncedStateToNodes(nodesCopy, false, 1) - bnp.UpdateNodesBasedOnSyncState(allNodes) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 1, IsSynced: true}, @@ -342,21 +359,80 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldMoveFallbackNode(t *t require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) - setSyncedStateToNodes(allNodes, false, 0) - bnp.UpdateNodesBasedOnSyncState(allNodes) + // make the fallback node inactive, so the last known regular observer will be used + setSyncedStateToNodes(nodesCopy, false, 0) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 1, IsSynced: true}, {Address: "addr3", ShardId: 1, IsSynced: true}, - {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false}, }, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr1", ShardId: 0, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.outOfSyncFallbackNodes)) + + // bring back the fallback node + setSyncedStateToNodes(nodesCopy, true, 0) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 1, IsSynced: true}, + {Address: "addr3", ShardId: 1, IsSynced: true}, + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, }, convertSlice(bnp.syncedFallbackNodes)) require.Equal(t, []data.NodeData{ {Address: "addr1", ShardId: 0, IsSynced: false}, }, convertSlice(bnp.outOfSyncNodes)) + + require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) + + // make the fallback node inactive, so the last known regular observer will be used + setSyncedStateToNodes(nodesCopy, false, 0) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 1, IsSynced: true}, + {Address: "addr3", ShardId: 1, IsSynced: true}, + {Address: "addr1", ShardId: 0, IsSynced: false}, + }, convertSlice(bnp.syncedNodes)) + + require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr1", ShardId: 0, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.outOfSyncFallbackNodes)) + + // bring back regular node synced + setSyncedStateToNodes(nodesCopy, true, 1) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 1, IsSynced: true}, + {Address: "addr3", ShardId: 1, IsSynced: true}, + {Address: "addr1", ShardId: 0, IsSynced: true}, + }, convertSlice(bnp.syncedNodes)) + + require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) + + require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.outOfSyncFallbackNodes)) + } func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIterations(t *testing.T) { @@ -370,9 +446,10 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter } _ = bnp.initNodesMaps(allNodes) - setSyncedStateToNodes(allNodes, false, 1, 3, 5, 7, 9) + nodesCopy := copyNodes(allNodes) + setSyncedStateToNodes(nodesCopy, false, 1, 3, 5, 7, 9) - bnp.UpdateNodesBasedOnSyncState(allNodes) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: true}, {Address: "addr8", ShardId: 1, IsSynced: true}, @@ -392,10 +469,10 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, }, convertSlice(bnp.outOfSyncFallbackNodes)) - allNodes = prepareNodes(10) - setFallbackNodes(allNodes, 0, 1, 5, 6) + nodesCopy = prepareNodes(10) + setFallbackNodes(nodesCopy, 0, 1, 5, 6) - bnp.UpdateNodesBasedOnSyncState(allNodes) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: true}, @@ -411,6 +488,52 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, }, convertSlice(bnp.syncedFallbackNodes)) + + // unsync all nodes + setSyncedStateToNodes(nodesCopy, false, 2, 3, 4, 7, 8, 9) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) + + // bring one node on each shard back + setSyncedStateToNodes(nodesCopy, true, 2, 7) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + }, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) } func prepareNodes(count int) []*data.NodeData { @@ -430,6 +553,20 @@ func prepareNodes(count int) []*data.NodeData { return nodes } +func copyNodes(nodes []*data.NodeData) []*data.NodeData { + nodesCopy := make([]*data.NodeData, len(nodes)) + for i, node := range nodes { + nodesCopy[i] = &data.NodeData{ + ShardId: node.ShardId, + Address: node.Address, + IsSynced: node.IsSynced, + IsFallback: node.IsFallback, + } + } + + return nodesCopy +} + func setSyncedStateToNodes(nodes []*data.NodeData, state bool, indices ...int) { for _, idx := range indices { nodes[idx].IsSynced = state From 4d9c2980f981b731c57b8367906d014505d739e1 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 8 Sep 2022 19:25:27 +0300 Subject: [PATCH 06/11] fixes after test: removed code that adds duplicates, added extra tests, added example into config.toml --- cmd/proxy/config/config.toml | 6 ++++++ observer/baseNodeProvider.go | 4 ---- observer/baseNodeProvider_test.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/cmd/proxy/config/config.toml b/cmd/proxy/config/config.toml index bfa2b403..d61623a5 100644 --- a/cmd/proxy/config/config.toml +++ b/cmd/proxy/config/config.toml @@ -64,6 +64,7 @@ # List of Observers. If you want to define a metachain observer (needed for validator statistics route) use # shard id 4294967295 +# Fallback observers which are only used when regular ones are offline should have IsFallback = true [[Observers]] ShardId = 0 Address = "http://127.0.0.1:8081" @@ -72,6 +73,11 @@ ShardId = 1 Address = "http://127.0.0.1:8082" +[[Observers]] + ShardId = 1 + Address = "http://127.0.0.1:8083" + IsFallback = true + [[FullHistoryNodes]] ShardId = 0 Address = "http://127.0.0.1:8081" diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index 361eff96..27c79cba 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -169,10 +169,6 @@ func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*da } bnp.nodesMap = nodesSliceToShardedMap(bnp.syncedNodes) - syncedFallbackNodesMap := nodesSliceToShardedMap(bnp.syncedFallbackNodes) - for shardId := range syncedFallbackNodesMap { - bnp.nodesMap[shardId] = append(bnp.nodesMap[shardId], syncedFallbackNodesMap[shardId]...) - } } func (bnp *baseNodeProvider) removeFromOutOfSyncIfNeededUnprotected(node *data.NodeData) { diff --git a/observer/baseNodeProvider_test.go b/observer/baseNodeProvider_test.go index d81aa28a..98f1909d 100644 --- a/observer/baseNodeProvider_test.go +++ b/observer/baseNodeProvider_test.go @@ -468,6 +468,13 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, }, convertSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + }, convertSlice(bnp.nodesMap[0])) + require.Equal(t, []data.NodeData{ + {Address: "addr8", ShardId: 1, IsSynced: true}, + }, convertSlice(bnp.nodesMap[1])) nodesCopy = prepareNodes(10) setFallbackNodes(nodesCopy, 0, 1, 5, 6) @@ -488,6 +495,16 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, }, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + }, convertSlice(bnp.nodesMap[0])) + require.Equal(t, []data.NodeData{ + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertSlice(bnp.nodesMap[1])) // unsync all nodes setSyncedStateToNodes(nodesCopy, false, 2, 3, 4, 7, 8, 9) @@ -513,6 +530,14 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr8", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, }, convertSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.nodesMap[0])) + require.Equal(t, []data.NodeData{ + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.nodesMap[1])) // bring one node on each shard back setSyncedStateToNodes(nodesCopy, true, 2, 7) @@ -534,6 +559,12 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr8", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, }, convertSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + }, convertSlice(bnp.nodesMap[0])) + require.Equal(t, []data.NodeData{ + {Address: "addr7", ShardId: 1, IsSynced: true}, + }, convertSlice(bnp.nodesMap[1])) } func prepareNodes(count int) []*data.NodeData { From d329223afc25366e8d48f8c762c349b2297556e4 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 14 Sep 2022 14:21:24 +0300 Subject: [PATCH 07/11] fix after test: fixed print of fallback nodes --- observer/baseNodeProvider.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index 27c79cba..1900cdaa 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -99,18 +99,25 @@ func (bnp *baseNodeProvider) UpdateNodesBasedOnSyncState(nodesWithSyncStatus []* func (bnp *baseNodeProvider) printSyncedNodesInShardsUnprotected() { for shardID, nodes := range bnp.nodesMap { inSyncAddresses := make([]string, 0) - fallbackAddresses := make([]string, 0) + inSyncFallbackAddresses := make([]string, 0) for _, node := range nodes { if node.IsFallback { - fallbackAddresses = append(fallbackAddresses, node.Address) - continue + continue // added in the next for } inSyncAddresses = append(inSyncAddresses, node.Address) } + + for _, activeFallbackNode := range bnp.syncedFallbackNodes { + if activeFallbackNode.ShardId == shardID { + inSyncFallbackAddresses = append(inSyncFallbackAddresses, activeFallbackNode.Address) + } + } + + totalNumOfActiveNodes := len(inSyncAddresses) + len(inSyncFallbackAddresses) log.Info(fmt.Sprintf("shard %d active nodes", shardID), - "observers count", len(nodes), + "observers count", totalNumOfActiveNodes, "addresses", strings.Join(inSyncAddresses, ", "), - "fallback addresses", strings.Join(fallbackAddresses, ", ")) + "fallback addresses", strings.Join(inSyncFallbackAddresses, ", ")) } } From 4f73cd74d41b0cedfda910b799cc6dfd3f5987cd Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 15 Sep 2022 15:48:29 +0300 Subject: [PATCH 08/11] fixes after tests + refactor, removed the nodesMap, added a member for shards only --- observer/baseNodeProvider.go | 149 ++++---- observer/baseNodeProvider_test.go | 471 +++++++++++++------------ observer/circularQueueNodesProvider.go | 7 +- observer/simpleNodesProvider.go | 4 +- 4 files changed, 306 insertions(+), 325 deletions(-) diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index 1900cdaa..12b7d62d 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -13,7 +13,7 @@ import ( type baseNodeProvider struct { mutNodes sync.RWMutex - nodesMap map[uint32][]*data.NodeData + shardIds []uint32 configurationFilePath string syncedNodes []*data.NodeData outOfSyncNodes []*data.NodeData @@ -22,7 +22,7 @@ type baseNodeProvider struct { lastSyncedNodes map[uint32]*data.NodeData } -func (bnp *baseNodeProvider) initNodesMaps(nodes []*data.NodeData) error { +func (bnp *baseNodeProvider) initNodes(nodes []*data.NodeData) error { if len(nodes) == 0 { return ErrEmptyObserversList } @@ -34,7 +34,7 @@ func (bnp *baseNodeProvider) initNodesMaps(nodes []*data.NodeData) error { } bnp.mutNodes.Lock() - bnp.nodesMap = newNodes + bnp.shardIds = getSortedShardIDsSlice(newNodes) bnp.syncedNodes, bnp.syncedFallbackNodes = initAllNodesSlice(newNodes) bnp.outOfSyncNodes = make([]*data.NodeData, 0) bnp.outOfSyncFallbackNodes = make([]*data.NodeData, 0) @@ -72,8 +72,7 @@ func (bnp *baseNodeProvider) UpdateNodesBasedOnSyncState(nodesWithSyncStatus []* bnp.mutNodes.Lock() defer bnp.mutNodes.Unlock() - shardIDs := getSortedSliceIDsSlice(bnp.nodesMap) - syncedNodes, syncedFallbackNodes, outOfSyncNodes, err := computeSyncedAndOutOfSyncNodes(nodesWithSyncStatus, shardIDs) + syncedNodes, syncedFallbackNodes, outOfSyncNodes, err := computeSyncedAndOutOfSyncNodes(nodesWithSyncStatus, bnp.shardIds) if err != nil { log.Error("cannot update nodes based on sync state", "error", err) return @@ -90,34 +89,34 @@ func (bnp *baseNodeProvider) UpdateNodesBasedOnSyncState(nodesWithSyncStatus []* syncedNodesMap := nodesSliceToShardedMap(syncedNodes) syncedFallbackNodesMap := nodesSliceToShardedMap(syncedFallbackNodes) - bnp.removeFallbackNodesFromSyncedUnprotected() bnp.removeOutOfSyncNodesUnprotected(outOfSyncNodes, syncedNodesMap, syncedFallbackNodesMap) bnp.addSyncedNodesUnprotected(syncedNodes, syncedFallbackNodes) bnp.printSyncedNodesInShardsUnprotected() } func (bnp *baseNodeProvider) printSyncedNodesInShardsUnprotected() { - for shardID, nodes := range bnp.nodesMap { - inSyncAddresses := make([]string, 0) - inSyncFallbackAddresses := make([]string, 0) - for _, node := range nodes { - if node.IsFallback { - continue // added in the next for - } - inSyncAddresses = append(inSyncAddresses, node.Address) - } + inSyncAddresses := make(map[uint32][]string, 0) + for _, syncedNode := range bnp.syncedNodes { + inSyncAddresses[syncedNode.ShardId] = append(inSyncAddresses[syncedNode.ShardId], syncedNode.Address) + } - for _, activeFallbackNode := range bnp.syncedFallbackNodes { - if activeFallbackNode.ShardId == shardID { - inSyncFallbackAddresses = append(inSyncFallbackAddresses, activeFallbackNode.Address) - } - } + inSyncFallbackAddresses := make(map[uint32][]string, 0) + for _, syncedFallbackNode := range bnp.syncedFallbackNodes { + inSyncFallbackAddresses[syncedFallbackNode.ShardId] = append(inSyncFallbackAddresses[syncedFallbackNode.ShardId], syncedFallbackNode.Address) + } - totalNumOfActiveNodes := len(inSyncAddresses) + len(inSyncFallbackAddresses) + for _, shardID := range bnp.shardIds { + totalNumOfActiveNodes := len(inSyncAddresses[shardID]) + len(inSyncFallbackAddresses[shardID]) + // if none of them is active, use the backup if exists + hasBackup := bnp.lastSyncedNodes[shardID] != nil + if totalNumOfActiveNodes == 0 && hasBackup { + totalNumOfActiveNodes++ + inSyncAddresses[shardID] = append(inSyncAddresses[shardID], bnp.lastSyncedNodes[shardID].Address) + } log.Info(fmt.Sprintf("shard %d active nodes", shardID), "observers count", totalNumOfActiveNodes, - "addresses", strings.Join(inSyncAddresses, ", "), - "fallback addresses", strings.Join(inSyncFallbackAddresses, ", ")) + "addresses", strings.Join(inSyncAddresses[shardID], ", "), + "fallback addresses", strings.Join(inSyncFallbackAddresses[shardID], ", ")) } } @@ -164,6 +163,7 @@ func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*da } bnp.syncedNodes = append(bnp.syncedNodes, node) + bnp.lastSyncedNodes = make(map[uint32]*data.NodeData) } for _, node := range receivedSyncedFallbackNodes { @@ -174,8 +174,6 @@ func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*da bnp.syncedFallbackNodes = append(bnp.syncedFallbackNodes, node) } - - bnp.nodesMap = nodesSliceToShardedMap(bnp.syncedNodes) } func (bnp *baseNodeProvider) removeFromOutOfSyncIfNeededUnprotected(node *data.NodeData) { @@ -255,7 +253,6 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( hasBackup := bnp.lastSyncedNodes[outOfSyncNode.ShardId] != nil if outOfSyncNode.IsFallback && hasBackup { bnp.removeNodeUnprotected(outOfSyncNode) - bnp.syncedNodes = append(bnp.syncedNodes, bnp.lastSyncedNodes[outOfSyncNode.ShardId]) continue } @@ -277,70 +274,19 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( continue } - log.Info("not enough nodes in shard, using synced fallback nodes", "shard", outOfSyncNode.ShardId) - bnp.moveFallbackNodesToSyncedUnprotected(syncedFallbackNodesMap[outOfSyncNode.ShardId]) - bnp.lastSyncedNodes[outOfSyncNode.ShardId] = outOfSyncNode - bnp.removeNodeUnprotected(outOfSyncNode) - } -} - -func (bnp *baseNodeProvider) moveFallbackNodesToSyncedUnprotected(syncedFallbackNodes []*data.NodeData) { - for _, node := range syncedFallbackNodes { - if bnp.isReceivedSyncedNodeExistent(node) { - continue - } - - bnp.syncedNodes = append(bnp.syncedNodes, node) - bnp.removeNodeFromSyncedNodesUnprotected(node) - } -} - -func (bnp *baseNodeProvider) removeFallbackNodesFromSyncedUnprotected() { - syncedWithoutFallback := make([]*data.NodeData, 0) - for _, node := range bnp.syncedNodes { - if !node.IsFallback { - syncedWithoutFallback = append(syncedWithoutFallback, node) + log.Info("not enough nodes left in shard, will use fallback nodes", "shard", outOfSyncNode.ShardId) + if !outOfSyncNode.IsFallback { + bnp.lastSyncedNodes[outOfSyncNode.ShardId] = outOfSyncNode } + bnp.removeNodeUnprotected(outOfSyncNode) } - bnp.syncedNodes = syncedWithoutFallback } func (bnp *baseNodeProvider) removeNodeUnprotected(node *data.NodeData) { - bnp.removeNodeFromShardedMapUnprotected(node) bnp.removeNodeFromSyncedNodesUnprotected(node) bnp.addToOutOfSyncUnprotected(node) } -func (bnp *baseNodeProvider) removeNodeFromShardedMapUnprotected(node *data.NodeData) { - nodeIndex := -1 - nodesInShard := bnp.nodesMap[node.ShardId] - if len(nodesInShard) == 0 { - log.Error("no observer in shard", "shard ID", node.ShardId) - return - } - - for idx, nodeInShard := range nodesInShard { - if node.ShardId == nodeInShard.ShardId && node.Address == nodeInShard.Address { - nodeIndex = idx - break - } - } - - if nodeIndex == -1 { - return - } - - copy(nodesInShard[nodeIndex:], nodesInShard[nodeIndex+1:]) - nodesInShard[len(nodesInShard)-1] = nil - nodesInShard = nodesInShard[:len(nodesInShard)-1] - - bnp.nodesMap[node.ShardId] = nodesInShard - log.Info("updated observers sharded map after removing out of sync observer", - "address", node.Address, - "shard ID", node.ShardId, - "num observers left in shard", len(nodesInShard)) -} - func (bnp *baseNodeProvider) removeNodeFromSyncedNodesUnprotected(nodeToRemove *data.NodeData) { source := &bnp.syncedNodes if nodeToRemove.IsFallback { @@ -371,7 +317,7 @@ func (bnp *baseNodeProvider) removeNodeFromListUnprotected(nodeToRemove *data.No // ReloadNodes will reload the observers or the full history observers func (bnp *baseNodeProvider) ReloadNodes(nodesType data.NodeType) data.NodesReloadResponse { bnp.mutNodes.RLock() - numOldShardsCount := len(bnp.nodesMap) + numOldShardsCount := len(bnp.shardIds) bnp.mutNodes.RUnlock() newConfig, err := loadMainConfig(bnp.configurationFilePath) @@ -400,7 +346,7 @@ func (bnp *baseNodeProvider) ReloadNodes(nodesType data.NodeType) data.NodesRelo } bnp.mutNodes.Lock() - bnp.nodesMap = newNodes + bnp.shardIds = getSortedShardIDsSlice(newNodes) bnp.syncedNodes, bnp.syncedFallbackNodes = initAllNodesSlice(newNodes) bnp.lastSyncedNodes = make(map[uint32]*data.NodeData) bnp.mutNodes.Unlock() @@ -419,7 +365,6 @@ func (bnp *baseNodeProvider) getSyncedNodesForShardUnprotected(shardId uint32) ( syncedNodes = append(syncedNodes, node) } } - if len(syncedNodes) != 0 { return syncedNodes, nil } @@ -429,14 +374,44 @@ func (bnp *baseNodeProvider) getSyncedNodesForShardUnprotected(shardId uint32) ( syncedNodes = append(syncedNodes, node) } } - if len(syncedNodes) != 0 { return syncedNodes, nil } + backupNode, hasBackup := bnp.lastSyncedNodes[shardId] + if hasBackup { + return []*data.NodeData{backupNode}, nil + } + return nil, ErrShardNotAvailable } +func (bnp *baseNodeProvider) getSyncedNodesUnprotected() ([]*data.NodeData, error) { + syncedNodes := make([]*data.NodeData, 0) + for _, node := range bnp.syncedNodes { + syncedNodes = append(syncedNodes, node) + } + if len(syncedNodes) != 0 { + return syncedNodes, nil + } + + for _, node := range bnp.syncedFallbackNodes { + syncedNodes = append(syncedNodes, node) + } + if len(syncedNodes) != 0 { + return syncedNodes, nil + } + + for _, backupNode := range bnp.lastSyncedNodes { + syncedNodes = append(syncedNodes, backupNode) + } + if len(syncedNodes) != 0 { + return syncedNodes, nil + } + + return nil, ErrEmptyObserversList +} + func loadMainConfig(filepath string) (*config.Config, error) { cfg := &config.Config{} err := core.LoadTomlFile(cfg, filepath) @@ -472,7 +447,7 @@ func prepareReloadResponseMessage(newNodes map[uint32][]*data.NodeData) string { func initAllNodesSlice(nodesOnShards map[uint32][]*data.NodeData) ([]*data.NodeData, []*data.NodeData) { sliceToReturn := make([]*data.NodeData, 0) fallbackNodes := make([]*data.NodeData, 0) - shardIDs := getSortedSliceIDsSlice(nodesOnShards) + shardIDs := getSortedShardIDsSlice(nodesOnShards) finishedShards := make(map[uint32]struct{}) for i := 0; ; i++ { @@ -498,7 +473,7 @@ func initAllNodesSlice(nodesOnShards map[uint32][]*data.NodeData) ([]*data.NodeD return sliceToReturn, fallbackNodes } -func getSortedSliceIDsSlice(nodesOnShards map[uint32][]*data.NodeData) []uint32 { +func getSortedShardIDsSlice(nodesOnShards map[uint32][]*data.NodeData) []uint32 { shardIDs := make([]uint32, 0) for shardID := range nodesOnShards { shardIDs = append(shardIDs, shardID) diff --git a/observer/baseNodeProvider_test.go b/observer/baseNodeProvider_test.go index 98f1909d..f8a86135 100644 --- a/observer/baseNodeProvider_test.go +++ b/observer/baseNodeProvider_test.go @@ -18,10 +18,7 @@ const configurationPath = "testdata/config.toml" func TestBaseNodeProvider_ReloadNodesDifferentNumberOfNewShard(t *testing.T) { bnp := &baseNodeProvider{ configurationFilePath: configurationPath, - nodesMap: map[uint32][]*data.NodeData{ - 0: {{Address: "addr1", ShardId: 0}}, - 1: {{Address: "addr2", ShardId: 1}}, - }, + shardIds: []uint32{0, 1}, } response := bnp.ReloadNodes(data.Observer) @@ -41,11 +38,7 @@ func TestBaseNodeProvider_ReloadNodesConfigurationFileNotFound(t *testing.T) { func TestBaseNodeProvider_ReloadNodesShouldWork(t *testing.T) { bnp := &baseNodeProvider{ configurationFilePath: configurationPath, - nodesMap: map[uint32][]*data.NodeData{ - 0: {{Address: "addr1", ShardId: 0}}, - 1: {{Address: "addr2", ShardId: 1}}, - 2: {{Address: "addr3", ShardId: core.MetachainShardId}}, - }, + shardIds: []uint32{0, 1, core.MetachainShardId}, } response := bnp.ReloadNodes(data.Observer) @@ -258,7 +251,7 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncState(t *testing.T) { bnp := &baseNodeProvider{ configurationFilePath: configurationPath, } - _ = bnp.initNodesMaps(allNodes) + _ = bnp.initNodes(allNodes) nodesCopy := copyNodes(allNodes) setSyncedStateToNodes(nodesCopy, false, 1, 2, 5, 6) @@ -294,7 +287,7 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldNotRemoveNodeIfNotEno nodesMap := nodesSliceToShardedMap(allNodes) bnp := &baseNodeProvider{ configurationFilePath: configurationPath, - nodesMap: nodesMap, + shardIds: getSortedShardIDsSlice(nodesMap), syncedNodes: allNodes, lastSyncedNodes: map[uint32]*data.NodeData{}, } @@ -313,6 +306,7 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldNotRemoveNodeIfNotEno {Address: "addr2", ShardId: 1, IsSynced: false}, }, convertSlice(bnp.outOfSyncNodes)) + nodesCopy = copyNodes(nodesCopy) setSyncedStateToNodes(nodesCopy, false, 1, 3) bnp.UpdateNodesBasedOnSyncState(nodesCopy) @@ -327,114 +321,6 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldNotRemoveNodeIfNotEno }, convertSlice(bnp.outOfSyncNodes)) } -func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldMoveFallbackNode(t *testing.T) { - t.Parallel() - - allNodes := prepareNodes(4) - setFallbackNodes(allNodes, 0) - - bnp := &baseNodeProvider{ - configurationFilePath: configurationPath, - } - _ = bnp.initNodesMaps(allNodes) - - nodesCopy := copyNodes(allNodes) - setSyncedStateToNodes(nodesCopy, false, 1) - - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 1, IsSynced: true}, - {Address: "addr3", ShardId: 1, IsSynced: true}, - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr1", ShardId: 0, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) - - require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) - - // make the fallback node inactive, so the last known regular observer will be used - setSyncedStateToNodes(nodesCopy, false, 0) - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 1, IsSynced: true}, - {Address: "addr3", ShardId: 1, IsSynced: true}, - {Address: "addr1", ShardId: 0, IsSynced: false}, - }, convertSlice(bnp.syncedNodes)) - - require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr1", ShardId: 0, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, - }, convertSlice(bnp.outOfSyncFallbackNodes)) - - // bring back the fallback node - setSyncedStateToNodes(nodesCopy, true, 0) - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 1, IsSynced: true}, - {Address: "addr3", ShardId: 1, IsSynced: true}, - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr1", ShardId: 0, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) - - require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) - - // make the fallback node inactive, so the last known regular observer will be used - setSyncedStateToNodes(nodesCopy, false, 0) - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 1, IsSynced: true}, - {Address: "addr3", ShardId: 1, IsSynced: true}, - {Address: "addr1", ShardId: 0, IsSynced: false}, - }, convertSlice(bnp.syncedNodes)) - - require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr1", ShardId: 0, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, - }, convertSlice(bnp.outOfSyncFallbackNodes)) - - // bring back regular node synced - setSyncedStateToNodes(nodesCopy, true, 1) - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 1, IsSynced: true}, - {Address: "addr3", ShardId: 1, IsSynced: true}, - {Address: "addr1", ShardId: 0, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) - - require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) - - require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, - }, convertSlice(bnp.outOfSyncFallbackNodes)) - -} - func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIterations(t *testing.T) { t.Parallel() @@ -444,127 +330,236 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter bnp := &baseNodeProvider{ configurationFilePath: configurationPath, } - _ = bnp.initNodesMaps(allNodes) + _ = bnp.initNodes(allNodes) + + var allNodesWithSyncState []data.NodeData + // sync all nodes, call update 3 times same state nodesCopy := copyNodes(allNodes) + setFallbackNodes(nodesCopy, 0, 1, 5, 6) + for i := 0; i < 3; i++ { + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) + + allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, allNodesWithSyncState) + } + + // unsync some nodes, call Update 3 times same state + nodesCopy = copyNodes(allNodes) setSyncedStateToNodes(nodesCopy, false, 1, 3, 5, 7, 9) - - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr3", ShardId: 0, IsSynced: false}, - {Address: "addr7", ShardId: 1, IsSynced: false}, - {Address: "addr9", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, - }, convertSlice(bnp.outOfSyncFallbackNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - }, convertSlice(bnp.nodesMap[0])) - require.Equal(t, []data.NodeData{ - {Address: "addr8", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.nodesMap[1])) - + for i := 0; i < 3; i++ { + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + }, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.outOfSyncFallbackNodes)) + + allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + }, allNodesWithSyncState) + } + + // sync all nodes, call update 3 times same state + nodesCopy = prepareNodes(10) + setFallbackNodes(nodesCopy, 0, 1, 5, 6) + for i := 0; i < 3; i++ { + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) + + allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + }, allNodesWithSyncState) + } + + // unsync all regular observers, call update 3 times same state nodesCopy = prepareNodes(10) setFallbackNodes(nodesCopy, 0, 1, 5, 6) - - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, - {Address: "addr9", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 0, IsSynced: true}, - }, convertSlice(bnp.nodesMap[0])) - require.Equal(t, []data.NodeData{ - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, - {Address: "addr9", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.nodesMap[1])) - - // unsync all nodes setSyncedStateToNodes(nodesCopy, false, 2, 3, 4, 7, 8, 9) - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: false}, - {Address: "addr3", ShardId: 0, IsSynced: false}, - {Address: "addr4", ShardId: 0, IsSynced: false}, - {Address: "addr7", ShardId: 1, IsSynced: false}, - {Address: "addr8", ShardId: 1, IsSynced: false}, - {Address: "addr9", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.nodesMap[0])) - require.Equal(t, []data.NodeData{ - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.nodesMap[1])) - - // bring one node on each shard back + for i := 0; i < 3; i++ { + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) + + allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + }, allNodesWithSyncState) + } + + // unsync fallbacks as well -> should keep the last regular ones, call update 3 times same state + nodesCopy = copyNodes(nodesCopy) + setSyncedStateToNodes(nodesCopy, false, 0, 1, 5, 6) + for i := 0; i < 3; i++ { + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, data.NodeData{Address: "addr4", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) + require.Equal(t, data.NodeData{Address: "addr9", ShardId: 1, IsSynced: false}, *bnp.lastSyncedNodes[1]) + + allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, + }, allNodesWithSyncState) + } + + // bring one node on each shard back, call update 3 times same state + nodesCopy = copyNodes(nodesCopy) setSyncedStateToNodes(nodesCopy, true, 2, 7) - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr3", ShardId: 0, IsSynced: false}, - {Address: "addr4", ShardId: 0, IsSynced: false}, - {Address: "addr8", ShardId: 1, IsSynced: false}, - {Address: "addr9", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: true}, - }, convertSlice(bnp.nodesMap[0])) - require.Equal(t, []data.NodeData{ - {Address: "addr7", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.nodesMap[1])) + for i := 0; i < 3; i++ { + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + }, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, + }, convertSlice(bnp.outOfSyncFallbackNodes)) + + allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, + }, allNodesWithSyncState) + } } func prepareNodes(count int) []*data.NodeData { @@ -706,7 +701,7 @@ func testEdgeCaseAddressShouldNotExistInBothLists(t *testing.T) { nodesMap := nodesSliceToShardedMap(allNodes) bnp := &baseNodeProvider{ configurationFilePath: configurationPath, - nodesMap: nodesMap, + shardIds: getSortedShardIDsSlice(nodesMap), syncedNodes: allNodes, } @@ -809,3 +804,11 @@ func slicesHaveCommonObjects(firstSlice []*data.NodeData, secondSlice []*data.No return false } + +func getNodesData(allNodesWithSyncState []*data.NodeData) []data.NodeData { + receivedNodes := make([]data.NodeData, len(allNodesWithSyncState)) + for idx, nodeWithState := range allNodesWithSyncState { + receivedNodes[idx] = *nodeWithState + } + return receivedNodes +} diff --git a/observer/circularQueueNodesProvider.go b/observer/circularQueueNodesProvider.go index 9ebbab5e..1bf231e5 100644 --- a/observer/circularQueueNodesProvider.go +++ b/observer/circularQueueNodesProvider.go @@ -21,7 +21,7 @@ func NewCircularQueueNodesProvider(observers []*data.NodeData, configurationFile configurationFilePath: configurationFilePath, } - err := bop.initNodesMaps(observers) + err := bop.initNodes(observers) if err != nil { return nil, err } @@ -55,7 +55,10 @@ func (cqnp *circularQueueNodesProvider) GetAllNodes() ([]*data.NodeData, error) cqnp.mutNodes.Lock() defer cqnp.mutNodes.Unlock() - allNodes := cqnp.syncedNodes + allNodes, err := cqnp.getSyncedNodesUnprotected() + if err != nil { + return nil, err + } position := cqnp.computeCounterForAllNodes(uint32(len(allNodes))) sliceToRet := append(allNodes[position:], allNodes[:position]...) diff --git a/observer/simpleNodesProvider.go b/observer/simpleNodesProvider.go index 80996534..1d87fdc7 100644 --- a/observer/simpleNodesProvider.go +++ b/observer/simpleNodesProvider.go @@ -16,7 +16,7 @@ func NewSimpleNodesProvider(observers []*data.NodeData, configurationFilePath st configurationFilePath: configurationFilePath, } - err := bop.initNodesMaps(observers) + err := bop.initNodes(observers) if err != nil { return nil, err } @@ -39,7 +39,7 @@ func (snp *simpleNodesProvider) GetAllNodes() ([]*data.NodeData, error) { snp.mutNodes.RLock() defer snp.mutNodes.RUnlock() - return snp.syncedNodes, nil + return snp.getSyncedNodesUnprotected() } // IsInterfaceNil returns true if there is no value under the interface From b6a4ca3217b0f037aa29f4c77ce31a263c28fc3c Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Thu, 15 Sep 2022 19:29:47 +0300 Subject: [PATCH 09/11] more fixes after tests + extra unittests --- observer/baseNodeProvider.go | 17 +- observer/baseNodeProvider_test.go | 299 ++++++++++++++++++++---------- 2 files changed, 208 insertions(+), 108 deletions(-) diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index 12b7d62d..f36ecabe 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -265,20 +265,25 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( // trying to remove last synced node // if fallbacks are available, save this one as backup and use fallbacks // else, keep using this one - hasOneSyncedFallbackNode := len(syncedFallbackNodesMap[outOfSyncNode.ShardId]) >= 1 - if !hasOneSyncedFallbackNode { - log.Warn("cannot remove observer as not enough will remain in shard", + // save this last regular observer as backup in case fallbacks go offline + wasSyncedAtPreviousStep := bnp.isReceivedSyncedNodeExistent(outOfSyncNode) + if !outOfSyncNode.IsFallback && wasSyncedAtPreviousStep { + log.Info("backup observer updated", "address", outOfSyncNode.Address, "is fallback", outOfSyncNode.IsFallback, "shard", outOfSyncNode.ShardId) + bnp.lastSyncedNodes[outOfSyncNode.ShardId] = outOfSyncNode + } + hasOneSyncedFallbackNode := len(syncedFallbackNodesMap[outOfSyncNode.ShardId]) >= 1 + if hasOneSyncedFallbackNode { + bnp.removeNodeUnprotected(outOfSyncNode) continue } - log.Info("not enough nodes left in shard, will use fallback nodes", "shard", outOfSyncNode.ShardId) + // safe to delete regular observer, as it is already in lastSyncedNodes map if !outOfSyncNode.IsFallback { - bnp.lastSyncedNodes[outOfSyncNode.ShardId] = outOfSyncNode + bnp.removeNodeUnprotected(outOfSyncNode) } - bnp.removeNodeUnprotected(outOfSyncNode) } } diff --git a/observer/baseNodeProvider_test.go b/observer/baseNodeProvider_test.go index f8a86135..698ea749 100644 --- a/observer/baseNodeProvider_test.go +++ b/observer/baseNodeProvider_test.go @@ -3,6 +3,7 @@ package observer import ( "errors" "fmt" + "sort" "testing" "github.com/ElrondNetwork/elrond-go-core/core" @@ -261,22 +262,22 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncState(t *testing.T) { require.Equal(t, []data.NodeData{ {Address: "addr3", ShardId: 0, IsSynced: true}, {Address: "addr7", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) + }, convertAndSortSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr4", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) + }, convertAndSortSlice(bnp.syncedFallbackNodes)) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: false}, {Address: "addr6", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) + }, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, - }, convertSlice(bnp.outOfSyncFallbackNodes)) + }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) } func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldNotRemoveNodeIfNotEnoughLeft(t *testing.T) { @@ -300,25 +301,26 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldNotRemoveNodeIfNotEno require.Equal(t, []data.NodeData{ {Address: "addr1", ShardId: 0, IsSynced: true}, {Address: "addr3", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) + }, convertAndSortSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false}, {Address: "addr2", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) + }, convertAndSortSlice(bnp.outOfSyncNodes)) nodesCopy = copyNodes(nodesCopy) setSyncedStateToNodes(nodesCopy, false, 1, 3) bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{ - {Address: "addr1", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false}, + {Address: "addr1", ShardId: 0, IsSynced: false}, {Address: "addr2", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) + {Address: "addr3", ShardId: 1, IsSynced: false}, + }, convertAndSortSlice(bnp.outOfSyncNodes)) + require.Equal(t, data.NodeData{Address: "addr1", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) + require.Equal(t, data.NodeData{Address: "addr3", ShardId: 1, IsSynced: false}, *bnp.lastSyncedNodes[1]) } func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIterations(t *testing.T) { @@ -332,8 +334,6 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter } _ = bnp.initNodes(allNodes) - var allNodesWithSyncState []data.NodeData - // sync all nodes, call update 3 times same state nodesCopy := copyNodes(allNodes) setFallbackNodes(nodesCopy, 0, 1, 5, 6) @@ -341,34 +341,33 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr9", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncNodes)) + }, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) - require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) + }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) - allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr9", ShardId: 1, IsSynced: true}, - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, allNodesWithSyncState) + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) } // unsync some nodes, call Update 3 times same state @@ -378,36 +377,35 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr4", ShardId: 0, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) + {Address: "addr8", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) + }, convertAndSortSlice(bnp.syncedFallbackNodes)) require.Equal(t, []data.NodeData{ {Address: "addr3", ShardId: 0, IsSynced: false}, {Address: "addr7", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) + }, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, - }, convertSlice(bnp.outOfSyncFallbackNodes)) + }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) - allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr9", ShardId: 1, IsSynced: false}, - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, - }, allNodesWithSyncState) + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) } // sync all nodes, call update 3 times same state @@ -417,34 +415,33 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr9", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncNodes)) + }, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) - require.Equal(t, []data.NodeData{}, convertSlice(bnp.outOfSyncFallbackNodes)) + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) - allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr9", ShardId: 1, IsSynced: true}, - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - }, allNodesWithSyncState) + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) } // unsync all regular observers, call update 3 times same state @@ -453,13 +450,13 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter setSyncedStateToNodes(nodesCopy, false, 2, 3, 4, 7, 8, 9) for i := 0; i < 3; i++ { bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertSlice(bnp.syncedFallbackNodes)) + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertAndSortSlice(bnp.syncedFallbackNodes)) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: false}, {Address: "addr3", ShardId: 0, IsSynced: false}, @@ -467,21 +464,20 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr7", ShardId: 1, IsSynced: false}, {Address: "addr8", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) + }, convertAndSortSlice(bnp.outOfSyncNodes)) - allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr2", ShardId: 0, IsSynced: false}, {Address: "addr3", ShardId: 0, IsSynced: false}, {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr7", ShardId: 1, IsSynced: false}, {Address: "addr8", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - }, allNodesWithSyncState) + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) } // unsync fallbacks as well -> should keep the last regular ones, call update 3 times same state @@ -489,8 +485,8 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter setSyncedStateToNodes(nodesCopy, false, 0, 1, 5, 6) for i := 0; i < 3; i++ { bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.syncedFallbackNodes)) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: false}, {Address: "addr3", ShardId: 0, IsSynced: false}, @@ -498,29 +494,28 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr7", ShardId: 1, IsSynced: false}, {Address: "addr8", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) + }, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, - }, convertSlice(bnp.outOfSyncFallbackNodes)) + }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) require.Equal(t, data.NodeData{Address: "addr4", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) require.Equal(t, data.NodeData{Address: "addr9", ShardId: 1, IsSynced: false}, *bnp.lastSyncedNodes[1]) - allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr2", ShardId: 0, IsSynced: false}, {Address: "addr3", ShardId: 0, IsSynced: false}, {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, {Address: "addr7", ShardId: 1, IsSynced: false}, {Address: "addr8", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, - {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, - }, allNodesWithSyncState) + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) } // bring one node on each shard back, call update 3 times same state @@ -531,35 +526,139 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: true}, {Address: "addr7", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{}, convertSlice(bnp.syncedFallbackNodes)) + }, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.syncedFallbackNodes)) require.Equal(t, []data.NodeData{ {Address: "addr3", ShardId: 0, IsSynced: false}, {Address: "addr4", ShardId: 0, IsSynced: false}, {Address: "addr8", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) + }, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, - }, convertSlice(bnp.outOfSyncFallbackNodes)) + }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) - allNodesWithSyncState = getNodesData(bnp.GetAllNodesWithSyncState()) require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, {Address: "addr3", ShardId: 0, IsSynced: false}, {Address: "addr4", ShardId: 0, IsSynced: false}, - {Address: "addr8", ShardId: 1, IsSynced: false}, - {Address: "addr9", ShardId: 1, IsSynced: false}, - {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, - }, allNodesWithSyncState) + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) } + + // sync all nodes + nodesCopy = prepareNodes(10) + setFallbackNodes(nodesCopy, 0, 1, 5, 6) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) + + // unsync fallbacks from shard 0 + nodesCopy = copyNodes(nodesCopy) + setSyncedStateToNodes(nodesCopy, false, 0, 1) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) + + // unsync all regular observers + nodesCopy = copyNodes(nodesCopy) + setSyncedStateToNodes(nodesCopy, false, 2, 3, 4, 7, 8, 9) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertAndSortSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, data.NodeData{Address: "addr4", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) } func prepareNodes(count int) []*data.NodeData { @@ -605,12 +704,16 @@ func setFallbackNodes(nodes []*data.NodeData, indices ...int) { } } -func convertSlice(nodes []*data.NodeData) []data.NodeData { +func convertAndSortSlice(nodes []*data.NodeData) []data.NodeData { newSlice := make([]data.NodeData, 0, len(nodes)) for _, node := range nodes { newSlice = append(newSlice, *node) } + sort.Slice(newSlice, func(i, j int) bool { + return newSlice[i].Address < newSlice[j].Address + }) + return newSlice } @@ -714,14 +817,14 @@ func testEdgeCaseAddressShouldNotExistInBothLists(t *testing.T) { {Address: "addr4", ShardId: 0, IsSynced: true}, {Address: "addr6", ShardId: 1, IsSynced: true}, {Address: "addr8", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) + }, convertAndSortSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr1", ShardId: 0, IsSynced: false}, {Address: "addr3", ShardId: 0, IsSynced: false}, {Address: "addr5", ShardId: 1, IsSynced: false}, {Address: "addr7", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, - }, convertSlice(bnp.outOfSyncNodes)) + }, convertAndSortSlice(bnp.outOfSyncNodes)) require.False(t, slicesHaveCommonObjects(bnp.syncedNodes, bnp.outOfSyncNodes)) allNodes = prepareNodes(10) @@ -730,16 +833,16 @@ func testEdgeCaseAddressShouldNotExistInBothLists(t *testing.T) { require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: true}, - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr6", ShardId: 1, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr1", ShardId: 0, IsSynced: true}, + {Address: "addr2", ShardId: 0, IsSynced: true}, {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, {Address: "addr5", ShardId: 1, IsSynced: true}, + {Address: "addr6", ShardId: 1, IsSynced: true}, {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr9", ShardId: 1, IsSynced: true}, - }, convertSlice(bnp.syncedNodes)) + }, convertAndSortSlice(bnp.syncedNodes)) require.False(t, slicesHaveCommonObjects(bnp.syncedNodes, bnp.outOfSyncNodes)) } @@ -804,11 +907,3 @@ func slicesHaveCommonObjects(firstSlice []*data.NodeData, secondSlice []*data.No return false } - -func getNodesData(allNodesWithSyncState []*data.NodeData) []data.NodeData { - receivedNodes := make([]data.NodeData, len(allNodesWithSyncState)) - for idx, nodeWithState := range allNodesWithSyncState { - receivedNodes[idx] = *nodeWithState - } - return receivedNodes -} From 403a719c945cea413f38d5a316623361970c9153 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Mon, 19 Sep 2022 14:31:58 +0300 Subject: [PATCH 10/11] more fixes, covered case when backup node is deleted from list --- observer/baseNodeProvider.go | 20 ++- observer/baseNodeProvider_test.go | 283 +++++++++++++++++++++--------- 2 files changed, 217 insertions(+), 86 deletions(-) diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index f36ecabe..0c3ecdfd 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -156,14 +156,15 @@ func computeSyncedAndOutOfSyncNodes(nodes []*data.NodeData, shardIDs []uint32) ( } func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*data.NodeData, receivedSyncedFallbackNodes []*data.NodeData) { + syncedNodesPerShard := make(map[uint32][]string) for _, node := range receivedSyncedNodes { bnp.removeFromOutOfSyncIfNeededUnprotected(node) + syncedNodesPerShard[node.ShardId] = append(syncedNodesPerShard[node.ShardId], node.Address) if bnp.isReceivedSyncedNodeExistent(node) { continue } bnp.syncedNodes = append(bnp.syncedNodes, node) - bnp.lastSyncedNodes = make(map[uint32]*data.NodeData) } for _, node := range receivedSyncedFallbackNodes { @@ -174,6 +175,13 @@ func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*da bnp.syncedFallbackNodes = append(bnp.syncedFallbackNodes, node) } + + // if there are at least one synced node regular received, clean the backup list + for _, shardId := range bnp.shardIds { + if len(syncedNodesPerShard[shardId]) != 0 { + delete(bnp.lastSyncedNodes, shardId) + } + } } func (bnp *baseNodeProvider) removeFromOutOfSyncIfNeededUnprotected(node *data.NodeData) { @@ -266,8 +274,10 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( // if fallbacks are available, save this one as backup and use fallbacks // else, keep using this one // save this last regular observer as backup in case fallbacks go offline + // also, if this is the old fallback observer which didn't get synced, keep it in list wasSyncedAtPreviousStep := bnp.isReceivedSyncedNodeExistent(outOfSyncNode) - if !outOfSyncNode.IsFallback && wasSyncedAtPreviousStep { + isBackupObserver := bnp.lastSyncedNodes[outOfSyncNode.ShardId] == outOfSyncNode + if !outOfSyncNode.IsFallback && wasSyncedAtPreviousStep || isBackupObserver { log.Info("backup observer updated", "address", outOfSyncNode.Address, "is fallback", outOfSyncNode.IsFallback, @@ -283,7 +293,13 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( // safe to delete regular observer, as it is already in lastSyncedNodes map if !outOfSyncNode.IsFallback { bnp.removeNodeUnprotected(outOfSyncNode) + continue } + + // this is a fallback node, with no synced nodes. + // save it as backup and delete it from its list + bnp.lastSyncedNodes[outOfSyncNode.ShardId] = outOfSyncNode + bnp.removeNodeUnprotected(outOfSyncNode) } } diff --git a/observer/baseNodeProvider_test.go b/observer/baseNodeProvider_test.go index 698ea749..beaa00d4 100644 --- a/observer/baseNodeProvider_test.go +++ b/observer/baseNodeProvider_test.go @@ -338,36 +338,7 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter nodesCopy := copyNodes(allNodes) setFallbackNodes(nodesCopy, 0, 1, 5, 6) for i := 0; i < 3; i++ { - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr9", ShardId: 1, IsSynced: true}, - }, convertAndSortSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertAndSortSlice(bnp.syncedFallbackNodes)) - require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr9", ShardId: 1, IsSynced: true}, - }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) + syncAllNodesAndCheck(t, nodesCopy, bnp) } // unsync some nodes, call Update 3 times same state @@ -393,6 +364,7 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, 0, len(bnp.lastSyncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, @@ -412,36 +384,7 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter nodesCopy = prepareNodes(10) setFallbackNodes(nodesCopy, 0, 1, 5, 6) for i := 0; i < 3; i++ { - bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr9", ShardId: 1, IsSynced: true}, - }, convertAndSortSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncNodes)) - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - }, convertAndSortSlice(bnp.syncedFallbackNodes)) - require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) - - require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, - {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr7", ShardId: 1, IsSynced: true}, - {Address: "addr8", ShardId: 1, IsSynced: true}, - {Address: "addr9", ShardId: 1, IsSynced: true}, - }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) + syncAllNodesAndCheck(t, nodesCopy, bnp) } // unsync all regular observers, call update 3 times same state @@ -465,6 +408,8 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr8", ShardId: 1, IsSynced: false}, {Address: "addr9", ShardId: 1, IsSynced: false}, }, convertAndSortSlice(bnp.outOfSyncNodes)) + require.Equal(t, data.NodeData{Address: "addr4", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) + require.Equal(t, data.NodeData{Address: "addr9", ShardId: 1, IsSynced: false}, *bnp.lastSyncedNodes[1]) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, @@ -540,6 +485,7 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr5", ShardId: 1, IsSynced: false, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: false, IsFallback: true}, }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, 0, len(bnp.lastSyncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, @@ -558,6 +504,11 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter // sync all nodes nodesCopy = prepareNodes(10) setFallbackNodes(nodesCopy, 0, 1, 5, 6) + syncAllNodesAndCheck(t, nodesCopy, bnp) + + // unsync fallbacks from shard 0 + nodesCopy = copyNodes(nodesCopy) + setSyncedStateToNodes(nodesCopy, false, 0, 1) bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ {Address: "addr2", ShardId: 0, IsSynced: true}, @@ -569,16 +520,18 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter }, convertAndSortSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, }, convertAndSortSlice(bnp.syncedFallbackNodes)) - require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, 0, len(bnp.lastSyncedNodes)) require.Equal(t, []data.NodeData{ - {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, - {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr2", ShardId: 0, IsSynced: true}, {Address: "addr3", ShardId: 0, IsSynced: true}, {Address: "addr4", ShardId: 0, IsSynced: true}, @@ -589,34 +542,79 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr9", ShardId: 1, IsSynced: true}, }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) - // unsync fallbacks from shard 0 + // unsync all regular observers nodesCopy = copyNodes(nodesCopy) - setSyncedStateToNodes(nodesCopy, false, 0, 1) + setSyncedStateToNodes(nodesCopy, false, 2, 3, 4, 7, 8, 9) + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertAndSortSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, data.NodeData{Address: "addr4", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) + require.Equal(t, data.NodeData{Address: "addr9", ShardId: 1, IsSynced: false}, *bnp.lastSyncedNodes[1]) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr7", ShardId: 1, IsSynced: false}, + {Address: "addr8", ShardId: 1, IsSynced: false}, + {Address: "addr9", ShardId: 1, IsSynced: false}, + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) + + // sync all nodes + nodesCopy = prepareNodes(10) + setFallbackNodes(nodesCopy, 0, 1, 5, 6) + syncAllNodesAndCheck(t, nodesCopy, bnp) + + // unsync all observers from shard 0 + nodesCopy = copyNodes(nodesCopy) + setSyncedStateToNodes(nodesCopy, false, 0, 1, 2, 3, 4) bnp.UpdateNodesBasedOnSyncState(nodesCopy) require.Equal(t, []data.NodeData{ - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, {Address: "addr7", ShardId: 1, IsSynced: true}, {Address: "addr8", ShardId: 1, IsSynced: true}, {Address: "addr9", ShardId: 1, IsSynced: true}, }, convertAndSortSlice(bnp.syncedNodes)) - require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + }, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, data.NodeData{Address: "addr4", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) + require.Equal(t, 1, len(bnp.lastSyncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, - {Address: "addr2", ShardId: 0, IsSynced: true}, - {Address: "addr3", ShardId: 0, IsSynced: true}, - {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr7", ShardId: 1, IsSynced: true}, @@ -624,11 +622,97 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr9", ShardId: 1, IsSynced: true}, }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) - // unsync all regular observers + // sync all nodes + nodesCopy = prepareNodes(10) + setFallbackNodes(nodesCopy, 0, 1, 5, 6) + syncAllNodesAndCheck(t, nodesCopy, bnp) + + // unsync last regular observer from shard 0, call update 3 times nodesCopy = copyNodes(nodesCopy) - setSyncedStateToNodes(nodesCopy, false, 2, 3, 4, 7, 8, 9) + setSyncedStateToNodes(nodesCopy, false, 4) + for i := 0; i < 3; i++ { + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr4", ShardId: 0, IsSynced: false}, + }, convertAndSortSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, 0, len(bnp.lastSyncedNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) + } + + // unsync all regular observers from shard 0, call update 3 times + nodesCopy = copyNodes(nodesCopy) + setSyncedStateToNodes(nodesCopy, false, 2, 3) + for i := 0; i < 3; i++ { + bnp.UpdateNodesBasedOnSyncState(nodesCopy) + require.Equal(t, []data.NodeData{ + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + }, convertAndSortSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) + require.Equal(t, data.NodeData{Address: "addr3", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) // last synced + require.Equal(t, 1, len(bnp.lastSyncedNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr2", ShardId: 0, IsSynced: false}, + {Address: "addr3", ShardId: 0, IsSynced: false}, + {Address: "addr4", ShardId: 0, IsSynced: false}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) + } + + // unsync all fallback observers from shard 0 + nodesCopy = copyNodes(nodesCopy) + setSyncedStateToNodes(nodesCopy, false, 0, 1) bnp.UpdateNodesBasedOnSyncState(nodesCopy) - require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.syncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, @@ -637,15 +721,13 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr2", ShardId: 0, IsSynced: false}, {Address: "addr3", ShardId: 0, IsSynced: false}, {Address: "addr4", ShardId: 0, IsSynced: false}, - {Address: "addr7", ShardId: 1, IsSynced: false}, - {Address: "addr8", ShardId: 1, IsSynced: false}, - {Address: "addr9", ShardId: 1, IsSynced: false}, }, convertAndSortSlice(bnp.outOfSyncNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, {Address: "addr1", ShardId: 0, IsSynced: false, IsFallback: true}, }, convertAndSortSlice(bnp.outOfSyncFallbackNodes)) - require.Equal(t, data.NodeData{Address: "addr4", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) + require.Equal(t, data.NodeData{Address: "addr3", ShardId: 0, IsSynced: false}, *bnp.lastSyncedNodes[0]) + require.Equal(t, 1, len(bnp.lastSyncedNodes)) require.Equal(t, []data.NodeData{ {Address: "addr0", ShardId: 0, IsSynced: false, IsFallback: true}, @@ -655,9 +737,42 @@ func TestBaseNodeProvider_UpdateNodesBasedOnSyncStateShouldWorkAfterMultipleIter {Address: "addr4", ShardId: 0, IsSynced: false}, {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, - {Address: "addr7", ShardId: 1, IsSynced: false}, - {Address: "addr8", ShardId: 1, IsSynced: false}, - {Address: "addr9", ShardId: 1, IsSynced: false}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) +} + +func syncAllNodesAndCheck(t *testing.T, nodes []*data.NodeData, bnp *baseNodeProvider) { + bnp.UpdateNodesBasedOnSyncState(nodes) + require.Equal(t, []data.NodeData{ + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, + }, convertAndSortSlice(bnp.syncedNodes)) + require.Equal(t, []data.NodeData{}, convertAndSortSlice(bnp.outOfSyncNodes)) + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + }, convertAndSortSlice(bnp.syncedFallbackNodes)) + require.Equal(t, 0, len(bnp.lastSyncedNodes)) + + require.Equal(t, []data.NodeData{ + {Address: "addr0", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr1", ShardId: 0, IsSynced: true, IsFallback: true}, + {Address: "addr2", ShardId: 0, IsSynced: true}, + {Address: "addr3", ShardId: 0, IsSynced: true}, + {Address: "addr4", ShardId: 0, IsSynced: true}, + {Address: "addr5", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr6", ShardId: 1, IsSynced: true, IsFallback: true}, + {Address: "addr7", ShardId: 1, IsSynced: true}, + {Address: "addr8", ShardId: 1, IsSynced: true}, + {Address: "addr9", ShardId: 1, IsSynced: true}, }, convertAndSortSlice(bnp.GetAllNodesWithSyncState())) } From 35728b4e060b3cfb5a2cec5fa394bc55d16908b7 Mon Sep 17 00:00:00 2001 From: Sorin Stanculeanu Date: Wed, 21 Sep 2022 12:56:19 +0300 Subject: [PATCH 11/11] fixes after review --- observer/baseNodeProvider.go | 102 +++++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 27 deletions(-) diff --git a/observer/baseNodeProvider.go b/observer/baseNodeProvider.go index 0c3ecdfd..31e61e90 100644 --- a/observer/baseNodeProvider.go +++ b/observer/baseNodeProvider.go @@ -68,6 +68,9 @@ func (bnp *baseNodeProvider) GetAllNodesWithSyncState() []*data.NodeData { // UpdateNodesBasedOnSyncState will handle the nodes lists, by removing out of sync observers or by adding back observers // that were previously removed because they were out of sync. +// If all observers are removed, the last one synced will be saved and the fallbacks will be used. +// If even the fallbacks are out of sync, the last regular observer synced will be used, even though it is out of sync. +// When one or more regular observers are back in sync, the fallbacks will not be used anymore. func (bnp *baseNodeProvider) UpdateNodesBasedOnSyncState(nodesWithSyncStatus []*data.NodeData) { bnp.mutNodes.Lock() defer bnp.mutNodes.Unlock() @@ -176,7 +179,7 @@ func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*da bnp.syncedFallbackNodes = append(bnp.syncedFallbackNodes, node) } - // if there are at least one synced node regular received, clean the backup list + // if there is at least one synced node regular received, clean the backup list for _, shardId := range bnp.shardIds { if len(syncedNodesPerShard[shardId]) != 0 { delete(bnp.lastSyncedNodes, shardId) @@ -185,16 +188,12 @@ func (bnp *baseNodeProvider) addSyncedNodesUnprotected(receivedSyncedNodes []*da } func (bnp *baseNodeProvider) removeFromOutOfSyncIfNeededUnprotected(node *data.NodeData) { - source := &bnp.outOfSyncNodes if node.IsFallback { - source = &bnp.outOfSyncFallbackNodes - } - - if *source == nil { + bnp.removeFallbackFromOutOfSyncListUnprotected(node) return } - bnp.removeNodeFromListUnprotected(node, source) + bnp.removeRegularFromOutOfSyncListUnprotected(node) } func (bnp *baseNodeProvider) isReceivedSyncedNodeExistent(receivedNode *data.NodeData) bool { @@ -218,22 +217,32 @@ func (bnp *baseNodeProvider) isReceivedSyncedNodeExistentAsFallback(receivedNode } func (bnp *baseNodeProvider) addToOutOfSyncUnprotected(node *data.NodeData) { - source := &bnp.outOfSyncNodes if node.IsFallback { - source = &bnp.outOfSyncFallbackNodes + bnp.addFallbackToOutOfSyncUnprotected(node) + return } - if *source == nil { - *source = make([]*data.NodeData, 0) + bnp.addRegularToOutOfSyncUnprotected(node) +} + +func (bnp *baseNodeProvider) addRegularToOutOfSyncUnprotected(node *data.NodeData) { + for _, oosNode := range bnp.outOfSyncNodes { + if oosNode.Address == node.Address && oosNode.ShardId == node.ShardId { + return + } } - for _, oosNode := range *source { + bnp.outOfSyncNodes = append(bnp.outOfSyncNodes, node) +} + +func (bnp *baseNodeProvider) addFallbackToOutOfSyncUnprotected(node *data.NodeData) { + for _, oosNode := range bnp.outOfSyncFallbackNodes { if oosNode.Address == node.Address && oosNode.ShardId == node.ShardId { return } } - *source = append(*source, node) + bnp.outOfSyncFallbackNodes = append(bnp.outOfSyncFallbackNodes, node) } func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( @@ -277,7 +286,8 @@ func (bnp *baseNodeProvider) removeOutOfSyncNodesUnprotected( // also, if this is the old fallback observer which didn't get synced, keep it in list wasSyncedAtPreviousStep := bnp.isReceivedSyncedNodeExistent(outOfSyncNode) isBackupObserver := bnp.lastSyncedNodes[outOfSyncNode.ShardId] == outOfSyncNode - if !outOfSyncNode.IsFallback && wasSyncedAtPreviousStep || isBackupObserver { + isRegularSyncedBefore := !outOfSyncNode.IsFallback && wasSyncedAtPreviousStep + if isRegularSyncedBefore || isBackupObserver { log.Info("backup observer updated", "address", outOfSyncNode.Address, "is fallback", outOfSyncNode.IsFallback, @@ -309,30 +319,56 @@ func (bnp *baseNodeProvider) removeNodeUnprotected(node *data.NodeData) { } func (bnp *baseNodeProvider) removeNodeFromSyncedNodesUnprotected(nodeToRemove *data.NodeData) { - source := &bnp.syncedNodes if nodeToRemove.IsFallback { - source = &bnp.syncedFallbackNodes + bnp.removeFallbackFromSyncedListUnprotected(nodeToRemove) + return } - bnp.removeNodeFromListUnprotected(nodeToRemove, source) + bnp.removeRegularFromSyncedListUnprotected(nodeToRemove) } -func (bnp *baseNodeProvider) removeNodeFromListUnprotected(nodeToRemove *data.NodeData, source *[]*data.NodeData) { - nodeIndex := -1 - for idx, node := range *source { - if node.Address == nodeToRemove.Address && node.ShardId == nodeToRemove.ShardId { - nodeIndex = idx - break - } +func (bnp *baseNodeProvider) removeRegularFromSyncedListUnprotected(nodeToRemove *data.NodeData) { + nodeIndex := getIndexFromList(nodeToRemove, bnp.syncedNodes) + if nodeIndex == -1 { + return } + copy(bnp.syncedNodes[nodeIndex:], bnp.syncedNodes[nodeIndex+1:]) + bnp.syncedNodes[len(bnp.syncedNodes)-1] = nil + bnp.syncedNodes = bnp.syncedNodes[:len(bnp.syncedNodes)-1] +} + +func (bnp *baseNodeProvider) removeFallbackFromSyncedListUnprotected(nodeToRemove *data.NodeData) { + nodeIndex := getIndexFromList(nodeToRemove, bnp.syncedFallbackNodes) if nodeIndex == -1 { return } - copy((*source)[nodeIndex:], (*source)[nodeIndex+1:]) - (*source)[len(*source)-1] = nil - *source = (*source)[:len(*source)-1] + copy(bnp.syncedFallbackNodes[nodeIndex:], bnp.syncedFallbackNodes[nodeIndex+1:]) + bnp.syncedFallbackNodes[len(bnp.syncedFallbackNodes)-1] = nil + bnp.syncedFallbackNodes = bnp.syncedFallbackNodes[:len(bnp.syncedFallbackNodes)-1] +} + +func (bnp *baseNodeProvider) removeRegularFromOutOfSyncListUnprotected(nodeToRemove *data.NodeData) { + nodeIndex := getIndexFromList(nodeToRemove, bnp.outOfSyncNodes) + if nodeIndex == -1 { + return + } + + copy(bnp.outOfSyncNodes[nodeIndex:], bnp.outOfSyncNodes[nodeIndex+1:]) + bnp.outOfSyncNodes[len(bnp.outOfSyncNodes)-1] = nil + bnp.outOfSyncNodes = bnp.outOfSyncNodes[:len(bnp.outOfSyncNodes)-1] +} + +func (bnp *baseNodeProvider) removeFallbackFromOutOfSyncListUnprotected(nodeToRemove *data.NodeData) { + nodeIndex := getIndexFromList(nodeToRemove, bnp.outOfSyncFallbackNodes) + if nodeIndex == -1 { + return + } + + copy(bnp.outOfSyncFallbackNodes[nodeIndex:], bnp.outOfSyncFallbackNodes[nodeIndex+1:]) + bnp.outOfSyncFallbackNodes[len(bnp.outOfSyncFallbackNodes)-1] = nil + bnp.outOfSyncFallbackNodes = bnp.outOfSyncFallbackNodes[:len(bnp.outOfSyncFallbackNodes)-1] } // ReloadNodes will reload the observers or the full history observers @@ -505,3 +541,15 @@ func getSortedShardIDsSlice(nodesOnShards map[uint32][]*data.NodeData) []uint32 return shardIDs } + +func getIndexFromList(providedNode *data.NodeData, list []*data.NodeData) int { + nodeIndex := -1 + for idx, node := range list { + if node.Address == providedNode.Address && node.ShardId == providedNode.ShardId { + nodeIndex = idx + break + } + } + + return nodeIndex +}