diff --git a/CHANGELOG-3.4.md b/CHANGELOG-3.4.md index 030d8d959d8..0d3ed553e82 100644 --- a/CHANGELOG-3.4.md +++ b/CHANGELOG-3.4.md @@ -199,6 +199,8 @@ See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-g - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years). - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days! - Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/coreos/etcd/pull/9347). +- Allow [non-WAL files in `--wal-dir` directory](https://github.com/coreos/etcd/pull/9743). + - Previously, existing files such as [`lost+found`](https://github.com/coreos/etcd/issues/7287) in WAL directory prevents etcd server boot. ### API diff --git a/pkg/fileutil/fileutil.go b/pkg/fileutil/fileutil.go index 468d78f65ae..82996b6fba5 100644 --- a/pkg/fileutil/fileutil.go +++ b/pkg/fileutil/fileutil.go @@ -20,7 +20,6 @@ import ( "io/ioutil" "os" "path/filepath" - "sort" "github.com/coreos/pkg/capnslog" ) @@ -32,9 +31,7 @@ const ( PrivateDirMode = 0700 ) -var ( - plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "pkg/fileutil") -) +var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "pkg/fileutil") // IsDirWriteable checks if dir is writable by writing and removing a file // to dir. It returns nil if dir is writable. @@ -46,21 +43,6 @@ func IsDirWriteable(dir string) error { return os.Remove(f) } -// ReadDir returns the filenames in the given directory in sorted order. -func ReadDir(dirpath string) ([]string, error) { - dir, err := os.Open(dirpath) - if err != nil { - return nil, err - } - defer dir.Close() - names, err := dir.Readdirnames(-1) - if err != nil { - return nil, err - } - sort.Strings(names) - return names, nil -} - // TouchDirAll is similar to os.MkdirAll. It creates directories with 0700 permission if any directory // does not exists. TouchDirAll also ensures the given directory is writable. func TouchDirAll(dir string) error { diff --git a/pkg/fileutil/fileutil_test.go b/pkg/fileutil/fileutil_test.go index e9497e9b1e0..a76d87ddc6c 100644 --- a/pkg/fileutil/fileutil_test.go +++ b/pkg/fileutil/fileutil_test.go @@ -22,7 +22,6 @@ import ( "os" "os/user" "path/filepath" - "reflect" "runtime" "strings" "testing" @@ -58,33 +57,6 @@ func TestIsDirWriteable(t *testing.T) { } } -func TestReadDir(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - defer os.RemoveAll(tmpdir) - if err != nil { - t.Fatalf("unexpected ioutil.TempDir error: %v", err) - } - files := []string{"def", "abc", "xyz", "ghi"} - for _, f := range files { - var fh *os.File - fh, err = os.Create(filepath.Join(tmpdir, f)) - if err != nil { - t.Fatalf("error creating file: %v", err) - } - if err = fh.Close(); err != nil { - t.Fatalf("error closing file: %v", err) - } - } - fs, err := ReadDir(tmpdir) - if err != nil { - t.Fatalf("error calling ReadDir: %v", err) - } - wfs := []string{"abc", "def", "ghi", "xyz"} - if !reflect.DeepEqual(fs, wfs) { - t.Fatalf("ReadDir: got %v, want %v", fs, wfs) - } -} - func TestCreateDirAll(t *testing.T) { tmpdir, err := ioutil.TempDir(os.TempDir(), "foo") if err != nil { diff --git a/pkg/fileutil/read_dir.go b/pkg/fileutil/read_dir.go new file mode 100644 index 00000000000..2eeaa89bc04 --- /dev/null +++ b/pkg/fileutil/read_dir.go @@ -0,0 +1,70 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "os" + "path/filepath" + "sort" +) + +// ReadDirOp represents an read-directory operation. +type ReadDirOp struct { + ext string +} + +// ReadDirOption configures archiver operations. +type ReadDirOption func(*ReadDirOp) + +// WithExt filters file names by their extensions. +// (e.g. WithExt(".wal") to list only WAL files) +func WithExt(ext string) ReadDirOption { + return func(op *ReadDirOp) { op.ext = ext } +} + +func (op *ReadDirOp) applyOpts(opts []ReadDirOption) { + for _, opt := range opts { + opt(op) + } +} + +// ReadDir returns the filenames in the given directory in sorted order. +func ReadDir(d string, opts ...ReadDirOption) ([]string, error) { + op := &ReadDirOp{} + op.applyOpts(opts) + + dir, err := os.Open(d) + if err != nil { + return nil, err + } + defer dir.Close() + + names, err := dir.Readdirnames(-1) + if err != nil { + return nil, err + } + sort.Strings(names) + + if op.ext != "" { + tss := make([]string, 0) + for _, v := range names { + if filepath.Ext(v) == op.ext { + tss = append(tss, v) + } + } + names = tss + } + return names, nil +} diff --git a/pkg/fileutil/read_dir_test.go b/pkg/fileutil/read_dir_test.go new file mode 100644 index 00000000000..6080ce750bd --- /dev/null +++ b/pkg/fileutil/read_dir_test.go @@ -0,0 +1,67 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestReadDir(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "") + defer os.RemoveAll(tmpdir) + if err != nil { + t.Fatalf("unexpected ioutil.TempDir error: %v", err) + } + + files := []string{"def", "abc", "xyz", "ghi"} + for _, f := range files { + writeFunc(t, filepath.Join(tmpdir, f)) + } + fs, err := ReadDir(tmpdir) + if err != nil { + t.Fatalf("error calling ReadDir: %v", err) + } + wfs := []string{"abc", "def", "ghi", "xyz"} + if !reflect.DeepEqual(fs, wfs) { + t.Fatalf("ReadDir: got %v, want %v", fs, wfs) + } + + files = []string{"def.wal", "abc.wal", "xyz.wal", "ghi.wal"} + for _, f := range files { + writeFunc(t, filepath.Join(tmpdir, f)) + } + fs, err = ReadDir(tmpdir, WithExt(".wal")) + if err != nil { + t.Fatalf("error calling ReadDir: %v", err) + } + wfs = []string{"abc.wal", "def.wal", "ghi.wal", "xyz.wal"} + if !reflect.DeepEqual(fs, wfs) { + t.Fatalf("ReadDir: got %v, want %v", fs, wfs) + } +} + +func writeFunc(t *testing.T, path string) { + fh, err := os.Create(path) + if err != nil { + t.Fatalf("error creating file: %v", err) + } + if err = fh.Close(); err != nil { + t.Fatalf("error closing file: %v", err) + } +} diff --git a/wal/util.go b/wal/util.go index 40065ec6496..04096a41cc5 100644 --- a/wal/util.go +++ b/wal/util.go @@ -28,7 +28,7 @@ var errBadWALName = errors.New("bad wal name") // Exist returns true if there are any files in a given directory. func Exist(dir string) bool { - names, err := fileutil.ReadDir(dir) + names, err := fileutil.ReadDir(dir, fileutil.WithExt(".wal")) if err != nil { return false }