tests: Validate etcd linearizability

Signed-off-by: Marek Siarkowicz <[email protected]>

etcdutl/go.mod

@@ -56,6 +56,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
 	go.etcd.io/etcd/client/v2 v2.306.0-alpha.0 // indirect
+	go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.32.0 // indirect
 	go.opentelemetry.io/otel v1.7.0 // indirect
 	go.opentelemetry.io/otel/trace v1.7.0 // indirect

etcdutl/go.sum

@@ -261,6 +261,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 h1:TcXBU/YdVROXQ7FUowVK1ih9gu2yi3YMLE+tQb9q964=
+go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0/go.mod h1:bOzzUWJ5bNHifkNkoIN6Ydf/z/UPT0bYuPghFYVC8+4=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

go.mod

@@ -80,6 +80,7 @@ require (
 	github.com/stretchr/testify v1.7.2 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.32.0 // indirect
 	go.opentelemetry.io/otel v1.7.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect

go.sum

@@ -48,6 +48,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/anishathalye/porcupine v0.1.2/go.mod h1:/X9OQYnVb7DzfKCQVO4tI1Aq+o56UJW+RvN/5U4EuZA=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -327,6 +328,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 h1:TcXBU/YdVROXQ7FUowVK1ih9gu2yi3YMLE+tQb9q964=
+go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0/go.mod h1:bOzzUWJ5bNHifkNkoIN6Ydf/z/UPT0bYuPghFYVC8+4=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

pkg/expect/expect.go

@@ -36,9 +36,13 @@ const DEBUG_LINES_TAIL = 40
 type ExpectProcess struct {
 	cfg expectConfig
 
-	cmd  *exec.Cmd
-	fpty *os.File
-	wg   sync.WaitGroup
+	// StopSignal is the signal Stop sends to the process; defaults to SIGTERM.
+	StopSignal os.Signal
+
+	cmd           *exec.Cmd
+	fpty          *os.File
+	closech       chan struct{}
+	logsCollected sync.WaitGroup
 
 	mu    sync.Mutex // protects lines and err
 	lines []string
@@ -50,35 +54,38 @@ type ExpectProcess struct {
 // NewExpect creates a new process for expect testing.
 func NewExpect(name string, arg ...string) (ep *ExpectProcess, err error) {
 	// if env[] is nil, use current system env and the default command as name
-	return NewExpectWithEnv(name, arg, nil, name)
+	return NewExpectWithEnv(name, arg, nil, name, false)
 }
 
 // NewExpectWithEnv creates a new process with user defined env variables for expect testing.
-func NewExpectWithEnv(name string, args []string, env []string, serverProcessConfigName string) (ep *ExpectProcess, err error) {
+func NewExpectWithEnv(name string, args []string, env []string, serverProcessConfigName string, restart bool) (ep *ExpectProcess, err error) {
 	ep = &ExpectProcess{
 		cfg: expectConfig{
-			name: serverProcessConfigName,
-			cmd:  name,
-			args: args,
-			env:  env,
+			name:    serverProcessConfigName,
+			cmd:     name,
+			args:    args,
+			env:     env,
+			restart: restart,
 		},
+		closech: make(chan struct{}),
 	}
 	ep.cmd = commandFromConfig(ep.cfg)
 
 	if ep.fpty, err = pty.Start(ep.cmd); err != nil {
 		return nil, err
 	}
 
-	ep.wg.Add(1)
+	ep.logsCollected.Add(1)
 	go ep.read()
 	return ep, nil
 }
 
 type expectConfig struct {
-	name string
-	cmd  string
-	args []string
-	env  []string
+	name    string
+	cmd     string
+	args    []string
+	env     []string
+	restart bool
 }
 
 func commandFromConfig(config expectConfig) *exec.Cmd {
@@ -94,23 +101,52 @@ func (ep *ExpectProcess) Pid() int {
 }
 
 func (ep *ExpectProcess) read() {
-	defer ep.wg.Done()
+	defer ep.logsCollected.Done()
 	printDebugLines := os.Getenv("EXPECT_DEBUG") != ""
-	r := bufio.NewReader(ep.fpty)
 	for {
-		l, err := r.ReadString('\n')
 		ep.mu.Lock()
-		if l != "" {
-			if printDebugLines {
-				fmt.Printf("%s (%s) (%d): %s", ep.cmd.Path, ep.cfg.name, ep.cmd.Process.Pid, l)
+		cmd := ep.cmd
+		r := bufio.NewReader(ep.fpty)
+		ep.mu.Unlock()
+		if cmd == nil {
+			break
+		}
+		pid := cmd.Process.Pid
+		for {
+			l, err := r.ReadString('\n')
+			ep.mu.Lock()
+			if l != "" {
+				if printDebugLines {
+					fmt.Printf("%s (%s) (%d): %s", ep.cmd.Path, ep.cfg.name, pid, l)
+				}
+				ep.lines = append(ep.lines, l)
+				ep.count++
 			}
-			ep.lines = append(ep.lines, l)
-			ep.count++
+			if err != nil {
+				ep.err = err
+				ep.mu.Unlock()
+				break
+			}
+			ep.mu.Unlock()
 		}
-		if err != nil {
+		select {
+		case <-ep.closech:
+			return
+		default:
+		}
+		ep.mu.Lock()
+		cmd = ep.cmd
+		ep.mu.Unlock()
+		if cmd != nil {
+			cmd.Wait()
+		}
+		ep.mu.Lock()
+		var err error
+		ep.cmd = commandFromConfig(ep.cfg)
+		if ep.fpty, err = pty.Start(ep.cmd); err != nil {
+			fmt.Printf("Error %s\n", err)
 			ep.err = err
-			ep.mu.Unlock()
-			break
+			ep.cmd = nil
 		}
 		ep.mu.Unlock()
 	}
@@ -179,7 +215,10 @@ func (ep *ExpectProcess) Stop() error { return ep.close(true) }
 
 // Signal sends a signal to the expect process
 func (ep *ExpectProcess) Signal(sig os.Signal) error {
-	return ep.cmd.Process.Signal(sig)
+	ep.mu.Lock()
+	err := ep.cmd.Process.Signal(sig)
+	ep.mu.Unlock()
+	return err
 }
 
 // Close waits for the expect process to exit.
@@ -188,16 +227,22 @@ func (ep *ExpectProcess) Signal(sig os.Signal) error {
 func (ep *ExpectProcess) Close() error { return ep.close(false) }
 
 func (ep *ExpectProcess) close(kill bool) error {
-	if ep.cmd == nil {
+	ep.mu.Lock()
+	cmd := ep.cmd
+	ep.mu.Unlock()
+
+	if cmd == nil {
 		return ep.err
 	}
+	close(ep.closech)
+
 	if kill {
 		ep.Signal(syscall.SIGTERM)
 	}
 
-	err := ep.cmd.Wait()
+	err := cmd.Wait()
 	ep.fpty.Close()
-	ep.wg.Wait()
+	ep.logsCollected.Wait()
 
 	if err != nil {
 		if !kill && strings.Contains(err.Error(), "exit status") {
@@ -207,8 +252,9 @@ func (ep *ExpectProcess) close(kill bool) error {
 			err = nil
 		}
 	}
-
+	ep.mu.Lock()
 	ep.cmd = nil
+	ep.mu.Unlock()
 	return err
 }
 

server/etcdserver/raft.fail.go

@@ -0,0 +1,16 @@
+// GENERATED BY GOFAIL. DO NOT EDIT.
+
+package etcdserver
+
+import "go.etcd.io/gofail/runtime"
+
+var __fp_raftBeforeSaveWaitWalSync *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftBeforeSaveWaitWalSync", false)
+var __fp_raftBeforeLeaderSend *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftBeforeLeaderSend", false)
+var __fp_raftBeforeSaveSnap *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftBeforeSaveSnap", false)
+var __fp_raftAfterSaveSnap *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftAfterSaveSnap", false)
+var __fp_raftBeforeSave *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftBeforeSave", false)
+var __fp_raftAfterSave *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftAfterSave", false)
+var __fp_raftBeforeApplySnap *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftBeforeApplySnap", false)
+var __fp_raftAfterApplySnap *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftAfterApplySnap", false)
+var __fp_raftAfterWALRelease *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftAfterWALRelease", false)
+var __fp_raftBeforeFollowerSend *runtime.Failpoint = runtime.NewFailpoint("etcdserver", "raftBeforeFollowerSend", false)

server/etcdserver/raft.go

@@ -211,7 +211,7 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 
 				waitWALSync := shouldWaitWALSync(rd)
 				if waitWALSync {
-					// gofail: var raftBeforeSaveWaitWalSync struct{}
+					if vraftBeforeSaveWaitWalSync, __fpErr := __fp_raftBeforeSaveWaitWalSync.Acquire(); __fpErr == nil { defer __fp_raftBeforeSaveWaitWalSync.Release(); _, __fpTypeOK := vraftBeforeSaveWaitWalSync.(struct{}); if !__fpTypeOK { goto __badTyperaftBeforeSaveWaitWalSync} ; __badTyperaftBeforeSaveWaitWalSync: __fp_raftBeforeSaveWaitWalSync.BadType(vraftBeforeSaveWaitWalSync, "struct{}"); };
 					if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
 						r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err))
 					}
@@ -227,30 +227,30 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 				// writing to their disks.
 				// For more details, check raft thesis 10.2.1
 				if islead {
-					// gofail: var raftBeforeLeaderSend struct{}
+					if vraftBeforeLeaderSend, __fpErr := __fp_raftBeforeLeaderSend.Acquire(); __fpErr == nil { defer __fp_raftBeforeLeaderSend.Release(); _, __fpTypeOK := vraftBeforeLeaderSend.(struct{}); if !__fpTypeOK { goto __badTyperaftBeforeLeaderSend} ; __badTyperaftBeforeLeaderSend: __fp_raftBeforeLeaderSend.BadType(vraftBeforeLeaderSend, "struct{}"); };
 					r.transport.Send(r.processMessages(rd.Messages))
 				}
 
 				// Must save the snapshot file and WAL snapshot entry before saving any other entries or hardstate to
 				// ensure that recovery after a snapshot restore is possible.
 				if !raft.IsEmptySnap(rd.Snapshot) {
-					// gofail: var raftBeforeSaveSnap struct{}
+					if vraftBeforeSaveSnap, __fpErr := __fp_raftBeforeSaveSnap.Acquire(); __fpErr == nil { defer __fp_raftBeforeSaveSnap.Release(); _, __fpTypeOK := vraftBeforeSaveSnap.(struct{}); if !__fpTypeOK { goto __badTyperaftBeforeSaveSnap} ; __badTyperaftBeforeSaveSnap: __fp_raftBeforeSaveSnap.BadType(vraftBeforeSaveSnap, "struct{}"); };
 					if err := r.storage.SaveSnap(rd.Snapshot); err != nil {
 						r.lg.Fatal("failed to save Raft snapshot", zap.Error(err))
 					}
-					// gofail: var raftAfterSaveSnap struct{}
+					if vraftAfterSaveSnap, __fpErr := __fp_raftAfterSaveSnap.Acquire(); __fpErr == nil { defer __fp_raftAfterSaveSnap.Release(); _, __fpTypeOK := vraftAfterSaveSnap.(struct{}); if !__fpTypeOK { goto __badTyperaftAfterSaveSnap} ; __badTyperaftAfterSaveSnap: __fp_raftAfterSaveSnap.BadType(vraftAfterSaveSnap, "struct{}"); };
 				}
 
 				if !waitWALSync {
-					// gofail: var raftBeforeSave struct{}
+					if vraftBeforeSave, __fpErr := __fp_raftBeforeSave.Acquire(); __fpErr == nil { defer __fp_raftBeforeSave.Release(); _, __fpTypeOK := vraftBeforeSave.(struct{}); if !__fpTypeOK { goto __badTyperaftBeforeSave} ; __badTyperaftBeforeSave: __fp_raftBeforeSave.BadType(vraftBeforeSave, "struct{}"); };
 					if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
 						r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err))
 					}
 				}
 				if !raft.IsEmptyHardState(rd.HardState) {
 					proposalsCommitted.Set(float64(rd.HardState.Commit))
 				}
-				// gofail: var raftAfterSave struct{}
+				if vraftAfterSave, __fpErr := __fp_raftAfterSave.Acquire(); __fpErr == nil { defer __fp_raftAfterSave.Release(); _, __fpTypeOK := vraftAfterSave.(struct{}); if !__fpTypeOK { goto __badTyperaftAfterSave} ; __badTyperaftAfterSave: __fp_raftAfterSave.BadType(vraftAfterSave, "struct{}"); };
 
 				if !raft.IsEmptySnap(rd.Snapshot) {
 					// Force WAL to fsync its hard state before Release() releases
@@ -264,15 +264,15 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 					// etcdserver now claim the snapshot has been persisted onto the disk
 					notifyc <- struct{}{}
 
-					// gofail: var raftBeforeApplySnap struct{}
+					if vraftBeforeApplySnap, __fpErr := __fp_raftBeforeApplySnap.Acquire(); __fpErr == nil { defer __fp_raftBeforeApplySnap.Release(); _, __fpTypeOK := vraftBeforeApplySnap.(struct{}); if !__fpTypeOK { goto __badTyperaftBeforeApplySnap} ; __badTyperaftBeforeApplySnap: __fp_raftBeforeApplySnap.BadType(vraftBeforeApplySnap, "struct{}"); };
 					r.raftStorage.ApplySnapshot(rd.Snapshot)
 					r.lg.Info("applied incoming Raft snapshot", zap.Uint64("snapshot-index", rd.Snapshot.Metadata.Index))
-					// gofail: var raftAfterApplySnap struct{}
+					if vraftAfterApplySnap, __fpErr := __fp_raftAfterApplySnap.Acquire(); __fpErr == nil { defer __fp_raftAfterApplySnap.Release(); _, __fpTypeOK := vraftAfterApplySnap.(struct{}); if !__fpTypeOK { goto __badTyperaftAfterApplySnap} ; __badTyperaftAfterApplySnap: __fp_raftAfterApplySnap.BadType(vraftAfterApplySnap, "struct{}"); };
 
 					if err := r.storage.Release(rd.Snapshot); err != nil {
 						r.lg.Fatal("failed to release Raft wal", zap.Error(err))
 					}
-					// gofail: var raftAfterWALRelease struct{}
+					if vraftAfterWALRelease, __fpErr := __fp_raftAfterWALRelease.Acquire(); __fpErr == nil { defer __fp_raftAfterWALRelease.Release(); _, __fpTypeOK := vraftAfterWALRelease.(struct{}); if !__fpTypeOK { goto __badTyperaftAfterWALRelease} ; __badTyperaftAfterWALRelease: __fp_raftAfterWALRelease.BadType(vraftAfterWALRelease, "struct{}"); };
 				}
 
 				r.raftStorage.Append(rd.Entries)
@@ -309,7 +309,7 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 						}
 					}
 
-					// gofail: var raftBeforeFollowerSend struct{}
+					if vraftBeforeFollowerSend, __fpErr := __fp_raftBeforeFollowerSend.Acquire(); __fpErr == nil { defer __fp_raftBeforeFollowerSend.Release(); _, __fpTypeOK := vraftBeforeFollowerSend.(struct{}); if !__fpTypeOK { goto __badTyperaftBeforeFollowerSend} ; __badTyperaftBeforeFollowerSend: __fp_raftBeforeFollowerSend.BadType(vraftBeforeFollowerSend, "struct{}"); };
 					r.transport.Send(msgs)
 				} else {
 					// leader already processed 'MsgSnap' and signaled

server/go.mod

@@ -29,6 +29,7 @@ require (
 	go.etcd.io/etcd/client/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/raft/v3 v3.6.0-alpha.0
+	go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.32.0
 	go.opentelemetry.io/otel v1.7.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0

server/go.sum

@@ -286,6 +286,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 h1:TcXBU/YdVROXQ7FUowVK1ih9gu2yi3YMLE+tQb9q964=
+go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0/go.mod h1:bOzzUWJ5bNHifkNkoIN6Ydf/z/UPT0bYuPghFYVC8+4=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

server/storage/backend/backend.fail.go

@@ -0,0 +1,8 @@
+// GENERATED BY GOFAIL. DO NOT EDIT.
+
+package backend
+
+import "go.etcd.io/gofail/runtime"
+
+var __fp_defragBeforeCopy *runtime.Failpoint = runtime.NewFailpoint("backend", "defragBeforeCopy", false)
+var __fp_defragBeforeRename *runtime.Failpoint = runtime.NewFailpoint("backend", "defragBeforeRename", false)

server/storage/backend/backend.go

@@ -501,7 +501,7 @@ func (b *backend) defrag() error {
 			zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse1))),
 		)
 	}
-	// gofail: var defragBeforeCopy struct{}
+	if vdefragBeforeCopy, __fpErr := __fp_defragBeforeCopy.Acquire(); __fpErr == nil { defer __fp_defragBeforeCopy.Release(); _, __fpTypeOK := vdefragBeforeCopy.(struct{}); if !__fpTypeOK { goto __badTypedefragBeforeCopy} ; __badTypedefragBeforeCopy: __fp_defragBeforeCopy.BadType(vdefragBeforeCopy, "struct{}"); };
 	err = defragdb(b.db, tmpdb, defragLimit)
 	if err != nil {
 		tmpdb.Close()
@@ -519,7 +519,7 @@ func (b *backend) defrag() error {
 	if err != nil {
 		b.lg.Fatal("failed to close tmp database", zap.Error(err))
 	}
-	// gofail: var defragBeforeRename struct{}
+	if vdefragBeforeRename, __fpErr := __fp_defragBeforeRename.Acquire(); __fpErr == nil { defer __fp_defragBeforeRename.Release(); _, __fpTypeOK := vdefragBeforeRename.(struct{}); if !__fpTypeOK { goto __badTypedefragBeforeRename} ; __badTypedefragBeforeRename: __fp_defragBeforeRename.BadType(vdefragBeforeRename, "struct{}"); };
 	err = os.Rename(tdbp, dbp)
 	if err != nil {
 		b.lg.Fatal("failed to rename tmp database", zap.Error(err))

server/storage/backend/batch_tx.fail.go

@@ -0,0 +1,8 @@
+// GENERATED BY GOFAIL. DO NOT EDIT.
+
+package backend
+
+import "go.etcd.io/gofail/runtime"
+
+var __fp_beforeCommit *runtime.Failpoint = runtime.NewFailpoint("backend", "beforeCommit", false)
+var __fp_afterCommit *runtime.Failpoint = runtime.NewFailpoint("backend", "afterCommit", false)

server/storage/backend/batch_tx.go

@@ -267,9 +267,9 @@ func (t *batchTx) commit(stop bool) {
 
 		start := time.Now()
 
-		// gofail: var beforeCommit struct{}
+		if vbeforeCommit, __fpErr := __fp_beforeCommit.Acquire(); __fpErr == nil { defer __fp_beforeCommit.Release(); _, __fpTypeOK := vbeforeCommit.(struct{}); if !__fpTypeOK { goto __badTypebeforeCommit} ; __badTypebeforeCommit: __fp_beforeCommit.BadType(vbeforeCommit, "struct{}"); };
 		err := t.tx.Commit()
-		// gofail: var afterCommit struct{}
+		if vafterCommit, __fpErr := __fp_afterCommit.Acquire(); __fpErr == nil { defer __fp_afterCommit.Release(); _, __fpTypeOK := vafterCommit.(struct{}); if !__fpTypeOK { goto __badTypeafterCommit} ; __badTypeafterCommit: __fp_afterCommit.BadType(vafterCommit, "struct{}"); };
 
 		rebalanceSec.Observe(t.tx.Stats().RebalanceTime.Seconds())
 		spillSec.Observe(t.tx.Stats().SpillTime.Seconds())

tests/framework/config/cluster.go

@@ -33,4 +33,5 @@ type ClusterConfig struct {
 	QuotaBackendBytes          int64
 	DisableStrictReconfigCheck bool
 	SnapshotCount              int
+	Restart                    bool
 }