ingest/ledgerbackend: Skip removing storage dir when Captive-Core in …

…front of requested ledger (#4605)

This commit improves `stellarCoreRunner` code to not remove storage dir if LCL
of Captive-Core is greater than or equal requested `from` ledger.

In e3d3abc we added a code that skips removing storage dir but it was not
removed if and only if `from` argument matched the LCL of Stellar-Core. The
problem is that in clusters with more than one ingesting instance it's possible
that other ingesting node would ingest one or more ledgers while Captive-Core
instance is being restarted. In such case we should not remove storage dir but
just skip the ledgers we don't need.

ingest/ledgerbackend/stellar_core_runner.go

@@ -373,7 +373,7 @@ func (r *stellarCoreRunner) runFrom(from uint32, hash string) error {
 		if err != nil {
 			r.log.Infof("Error running offline-info: %v, removing existing storage-dir contents", err)
 			removeStorageDir = true
-		} else if uint32(info.Info.Ledger.Num) != from {
+		} else if uint32(info.Info.Ledger.Num) > from {
 			r.log.Infof("Unexpected LCL in Stellar-Core DB: %d (want: %d), removing existing storage-dir contents", info.Info.Ledger.Num, from)
 			removeStorageDir = true
 		}

ingest/ledgerbackend/stellar_core_runner_test.go

@@ -203,7 +203,7 @@ func TestRunFromUseDBLedgersMatch(t *testing.T) {
 	assert.NoError(t, runner.close())
 }
 
-func TestRunFromUseDBLedgersNotMatch(t *testing.T) {
+func TestRunFromUseDBLedgersBehind(t *testing.T) {
 	captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{})
 	assert.NoError(t, err)
 
@@ -230,7 +230,67 @@ func TestRunFromUseDBLedgersNotMatch(t *testing.T) {
 
 	offlineInfoCmdMock := simpleCommandMock()
 	infoResponse := stellarcore.InfoResponse{}
-	infoResponse.Info.Ledger.Num = 101 // runner is one ledger behind
+	infoResponse.Info.Ledger.Num = 90 // runner is 10 ledgers behind
+	infoResponseBytes, err := json.Marshal(infoResponse)
+	assert.NoError(t, err)
+	offlineInfoCmdMock.On("Output").Return(infoResponseBytes, nil)
+	offlineInfoCmdMock.On("Wait").Return(nil)
+
+	// Replace system calls with a mock
+	scMock := &mockSystemCaller{}
+	defer scMock.AssertExpectations(t)
+	// Storage dir is not removed because ledgers do not match
+	scMock.On("stat", mock.Anything).Return(isDirImpl(true), nil)
+	scMock.On("writeFile", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	scMock.On("command",
+		"/usr/bin/stellar-core",
+		"--conf",
+		mock.Anything,
+		"offline-info",
+	).Return(offlineInfoCmdMock)
+	scMock.On("command",
+		"/usr/bin/stellar-core",
+		"--conf",
+		mock.Anything,
+		"--console",
+		"run",
+		"--metadata-output-stream",
+		"fd:3",
+	).Return(cmdMock)
+	runner.systemCaller = scMock
+
+	assert.NoError(t, runner.runFrom(100, "hash"))
+	assert.NoError(t, runner.close())
+}
+
+func TestRunFromUseDBLedgersInFront(t *testing.T) {
+	captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{})
+	assert.NoError(t, err)
+
+	captiveCoreToml.AddExamplePubnetValidators()
+
+	runner := newStellarCoreRunner(CaptiveCoreConfig{
+		BinaryPath:         "/usr/bin/stellar-core",
+		HistoryArchiveURLs: []string{"http://localhost"},
+		Log:                log.New(),
+		Context:            context.Background(),
+		Toml:               captiveCoreToml,
+		StoragePath:        "/tmp/captive-core",
+		UseDB:              true,
+	})
+
+	newDBCmdMock := simpleCommandMock()
+	newDBCmdMock.On("Run").Return(nil)
+
+	catchupCmdMock := simpleCommandMock()
+	catchupCmdMock.On("Run").Return(nil)
+
+	cmdMock := simpleCommandMock()
+	cmdMock.On("Wait").Return(nil)
+
+	offlineInfoCmdMock := simpleCommandMock()
+	infoResponse := stellarcore.InfoResponse{}
+	infoResponse.Info.Ledger.Num = 110 // runner is 10 ledgers in front
 	infoResponseBytes, err := json.Marshal(infoResponse)
 	assert.NoError(t, err)
 	offlineInfoCmdMock.On("Output").Return(infoResponseBytes, nil)