Merge pull request #9743 from gyuho/wal

wal: allow non-WAL files in wal directory

CHANGELOG-3.4.md

@@ -199,6 +199,8 @@ See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-g
   - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years).
   - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days!
 - Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/coreos/etcd/pull/9347).
+- Allow [non-WAL files in `--wal-dir` directory](https://github.com/coreos/etcd/pull/9743).
+  - Previously, existing files such as [`lost+found`](https://github.com/coreos/etcd/issues/7287) in WAL directory prevents etcd server boot.
 
 ### API
 

pkg/fileutil/fileutil.go

@@ -20,7 +20,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"sort"
 
 	"github.com/coreos/pkg/capnslog"
 )
@@ -32,9 +31,7 @@ const (
 	PrivateDirMode = 0700
 )
 
-var (
-	plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "pkg/fileutil")
-)
+var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "pkg/fileutil")
 
 // IsDirWriteable checks if dir is writable by writing and removing a file
 // to dir. It returns nil if dir is writable.
@@ -46,21 +43,6 @@ func IsDirWriteable(dir string) error {
 	return os.Remove(f)
 }
 
-// ReadDir returns the filenames in the given directory in sorted order.
-func ReadDir(dirpath string) ([]string, error) {
-	dir, err := os.Open(dirpath)
-	if err != nil {
-		return nil, err
-	}
-	defer dir.Close()
-	names, err := dir.Readdirnames(-1)
-	if err != nil {
-		return nil, err
-	}
-	sort.Strings(names)
-	return names, nil
-}
-
 // TouchDirAll is similar to os.MkdirAll. It creates directories with 0700 permission if any directory
 // does not exists. TouchDirAll also ensures the given directory is writable.
 func TouchDirAll(dir string) error {

pkg/fileutil/fileutil_test.go

@@ -22,7 +22,6 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
-	"reflect"
 	"runtime"
 	"strings"
 	"testing"
@@ -58,33 +57,6 @@ func TestIsDirWriteable(t *testing.T) {
 	}
 }
 
-func TestReadDir(t *testing.T) {
-	tmpdir, err := ioutil.TempDir("", "")
-	defer os.RemoveAll(tmpdir)
-	if err != nil {
-		t.Fatalf("unexpected ioutil.TempDir error: %v", err)
-	}
-	files := []string{"def", "abc", "xyz", "ghi"}
-	for _, f := range files {
-		var fh *os.File
-		fh, err = os.Create(filepath.Join(tmpdir, f))
-		if err != nil {
-			t.Fatalf("error creating file: %v", err)
-		}
-		if err = fh.Close(); err != nil {
-			t.Fatalf("error closing file: %v", err)
-		}
-	}
-	fs, err := ReadDir(tmpdir)
-	if err != nil {
-		t.Fatalf("error calling ReadDir: %v", err)
-	}
-	wfs := []string{"abc", "def", "ghi", "xyz"}
-	if !reflect.DeepEqual(fs, wfs) {
-		t.Fatalf("ReadDir: got %v, want %v", fs, wfs)
-	}
-}
-
 func TestCreateDirAll(t *testing.T) {
 	tmpdir, err := ioutil.TempDir(os.TempDir(), "foo")
 	if err != nil {

pkg/fileutil/read_dir.go

@@ -0,0 +1,70 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileutil
+
+import (
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+// ReadDirOp represents an read-directory operation.
+type ReadDirOp struct {
+	ext string
+}
+
+// ReadDirOption configures archiver operations.
+type ReadDirOption func(*ReadDirOp)
+
+// WithExt filters file names by their extensions.
+// (e.g. WithExt(".wal") to list only WAL files)
+func WithExt(ext string) ReadDirOption {
+	return func(op *ReadDirOp) { op.ext = ext }
+}
+
+func (op *ReadDirOp) applyOpts(opts []ReadDirOption) {
+	for _, opt := range opts {
+		opt(op)
+	}
+}
+
+// ReadDir returns the filenames in the given directory in sorted order.
+func ReadDir(d string, opts ...ReadDirOption) ([]string, error) {
+	op := &ReadDirOp{}
+	op.applyOpts(opts)
+
+	dir, err := os.Open(d)
+	if err != nil {
+		return nil, err
+	}
+	defer dir.Close()
+
+	names, err := dir.Readdirnames(-1)
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(names)
+
+	if op.ext != "" {
+		tss := make([]string, 0)
+		for _, v := range names {
+			if filepath.Ext(v) == op.ext {
+				tss = append(tss, v)
+			}
+		}
+		names = tss
+	}
+	return names, nil
+}

pkg/fileutil/read_dir_test.go

@@ -0,0 +1,67 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileutil
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+func TestReadDir(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "")
+	defer os.RemoveAll(tmpdir)
+	if err != nil {
+		t.Fatalf("unexpected ioutil.TempDir error: %v", err)
+	}
+
+	files := []string{"def", "abc", "xyz", "ghi"}
+	for _, f := range files {
+		writeFunc(t, filepath.Join(tmpdir, f))
+	}
+	fs, err := ReadDir(tmpdir)
+	if err != nil {
+		t.Fatalf("error calling ReadDir: %v", err)
+	}
+	wfs := []string{"abc", "def", "ghi", "xyz"}
+	if !reflect.DeepEqual(fs, wfs) {
+		t.Fatalf("ReadDir: got %v, want %v", fs, wfs)
+	}
+
+	files = []string{"def.wal", "abc.wal", "xyz.wal", "ghi.wal"}
+	for _, f := range files {
+		writeFunc(t, filepath.Join(tmpdir, f))
+	}
+	fs, err = ReadDir(tmpdir, WithExt(".wal"))
+	if err != nil {
+		t.Fatalf("error calling ReadDir: %v", err)
+	}
+	wfs = []string{"abc.wal", "def.wal", "ghi.wal", "xyz.wal"}
+	if !reflect.DeepEqual(fs, wfs) {
+		t.Fatalf("ReadDir: got %v, want %v", fs, wfs)
+	}
+}
+
+func writeFunc(t *testing.T, path string) {
+	fh, err := os.Create(path)
+	if err != nil {
+		t.Fatalf("error creating file: %v", err)
+	}
+	if err = fh.Close(); err != nil {
+		t.Fatalf("error closing file: %v", err)
+	}
+}

wal/util.go

@@ -28,7 +28,7 @@ var errBadWALName = errors.New("bad wal name")
 
 // Exist returns true if there are any files in a given directory.
 func Exist(dir string) bool {
-	names, err := fileutil.ReadDir(dir)
+	names, err := fileutil.ReadDir(dir, fileutil.WithExt(".wal"))
 	if err != nil {
 		return false
 	}