From 553325721a19156c6f840bee7b1ef2d625c96ec4 Mon Sep 17 00:00:00 2001 From: Wenjia Zhang Date: Thu, 26 Apr 2018 18:59:51 -0700 Subject: [PATCH] etcd-dump-logs: add entry-type flag to list entries of specific types and add test --- tools/etcd-dump-logs/etcd-dump-log_test.go | 277 ++++++++++++++++++ .../expectedoutput/listAll.output | 44 +++ .../expectedoutput/listConfigChange.output | 14 + .../listConfigChangeIRRCompaction.output | 15 + .../listConfigChangeIRRLeaseRevoke.output | 11 + .../expectedoutput/listIRRCompaction.output | 11 + .../expectedoutput/listIRRDeleteRange.output | 11 + .../expectedoutput/listIRRLeaseGrant.output | 11 + .../expectedoutput/listIRRLeaseRevoke.output | 11 + .../expectedoutput/listIRRPut.output | 11 + .../expectedoutput/listIRRRange.output | 11 + .../expectedoutput/listIRRTxn.output | 11 + .../listInternalRaftRequest.output | 34 +++ .../expectedoutput/listNormal.output | 40 +++ .../expectedoutput/listRequest.output | 15 + tools/etcd-dump-logs/main.go | 221 +++++++++++--- 16 files changed, 707 insertions(+), 41 deletions(-) create mode 100644 tools/etcd-dump-logs/etcd-dump-log_test.go create mode 100644 tools/etcd-dump-logs/expectedoutput/listAll.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listConfigChange.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRCompaction.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRLeaseRevoke.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listIRRCompaction.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listIRRDeleteRange.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listIRRLeaseGrant.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listIRRLeaseRevoke.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listIRRPut.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listIRRRange.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listIRRTxn.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listInternalRaftRequest.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listNormal.output create mode 100644 tools/etcd-dump-logs/expectedoutput/listRequest.output diff --git a/tools/etcd-dump-logs/etcd-dump-log_test.go b/tools/etcd-dump-logs/etcd-dump-log_test.go new file mode 100644 index 00000000000..3947374b662 --- /dev/null +++ b/tools/etcd-dump-logs/etcd-dump-log_test.go @@ -0,0 +1,277 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/coreos/etcd/auth/authpb" + "github.com/coreos/etcd/etcdserver/etcdserverpb" + "github.com/coreos/etcd/pkg/fileutil" + "github.com/coreos/etcd/pkg/pbutil" + "github.com/coreos/etcd/raft/raftpb" + "github.com/coreos/etcd/wal" + "go.uber.org/zap" +) + +func TestEtcdDumpLogEntryType(t *testing.T) { + // directory where the command is + binDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + dumpLogsBinary := path.Join(binDir + "/etcd-dump-logs") + if !fileutil.Exist(dumpLogsBinary) { + t.Skipf("%q does not exist", dumpLogsBinary) + } + + p, err := ioutil.TempDir(os.TempDir(), "etcddumplogstest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(p) + + memberdir := filepath.Join(p, "member") + err = os.Mkdir(memberdir, 0744) + if err != nil { + t.Fatal(err) + } + waldir := walDir(p) + snapdir := snapDir(p) + + w, err := wal.Create(zap.NewExample(), waldir, nil) + if err != nil { + t.Fatal(err) + } + + err = os.Mkdir(snapdir, 0744) + if err != nil { + t.Fatal(err) + } + + ents := make([]raftpb.Entry, 0) + + // append entries into wal log + appendConfigChangeEnts(&ents) + appendNormalRequestEnts(&ents) + appendNormalIRREnts(&ents) + appendUnknownNormalEnts(&ents) + + // force commit newly appended entries + err = w.Save(raftpb.HardState{}, ents) + if err != nil { + t.Fatal(err) + } + w.Close() + + argtests := []struct { + name string + args []string + fileExpected string + }{ + {"no entry-type", []string{p}, "expectedoutput/listAll.output"}, + {"confchange entry-type", []string{"-entry-type", "ConfigChange", p}, "expectedoutput/listConfigChange.output"}, + {"normal entry-type", []string{"-entry-type", "Normal", p}, "expectedoutput/listNormal.output"}, + {"request entry-type", []string{"-entry-type", "Request", p}, "expectedoutput/listRequest.output"}, + {"internalRaftRequest entry-type", []string{"-entry-type", "InternalRaftRequest", p}, "expectedoutput/listInternalRaftRequest.output"}, + {"range entry-type", []string{"-entry-type", "IRRRange", p}, "expectedoutput/listIRRRange.output"}, + {"put entry-type", []string{"-entry-type", "IRRPut", p}, "expectedoutput/listIRRPut.output"}, + {"del entry-type", []string{"-entry-type", "IRRDeleteRange", p}, "expectedoutput/listIRRDeleteRange.output"}, + {"txn entry-type", []string{"-entry-type", "IRRTxn", p}, "expectedoutput/listIRRTxn.output"}, + {"compaction entry-type", []string{"-entry-type", "IRRCompaction", p}, "expectedoutput/listIRRCompaction.output"}, + {"lease grant entry-type", []string{"-entry-type", "IRRLeaseGrant", p}, "expectedoutput/listIRRLeaseGrant.output"}, + {"lease revoke entry-type", []string{"-entry-type", "IRRLeaseRevoke", p}, "expectedoutput/listIRRLeaseRevoke.output"}, + {"confchange and txn entry-type", []string{"-entry-type", "ConfigChange,IRRCompaction", p}, "expectedoutput/listConfigChangeIRRCompaction.output"}, + } + + for _, argtest := range argtests { + t.Run(argtest.name, func(t *testing.T) { + cmd := exec.Command(dumpLogsBinary, argtest.args...) + actual, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(err) + } + expected, err := ioutil.ReadFile(path.Join(binDir, argtest.fileExpected)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(actual, expected) { + t.Errorf(`Got input of length %d, wanted input of length %d +==== BEGIN RECIEVED FILE ==== +%s +==== END RECIEVED FILE ==== +==== BEGIN EXPECTED FILE ==== +%s +==== END EXPECTED FILE ==== +`, len(actual), len(expected), actual, expected) + } + }) + } + +} + +func appendConfigChangeEnts(ents *[]raftpb.Entry) { + configChangeData := []raftpb.ConfChange{ + {ID: 1, Type: raftpb.ConfChangeAddNode, NodeID: 2, Context: []byte("")}, + {ID: 2, Type: raftpb.ConfChangeRemoveNode, NodeID: 2, Context: []byte("")}, + {ID: 3, Type: raftpb.ConfChangeUpdateNode, NodeID: 2, Context: []byte("")}, + {ID: 4, Type: raftpb.ConfChangeAddLearnerNode, NodeID: 3, Context: []byte("")}, + } + configChangeEntries := []raftpb.Entry{ + {Term: 1, Index: 1, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[0])}, + {Term: 2, Index: 2, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[1])}, + {Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[2])}, + {Term: 2, Index: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[3])}, + } + *ents = append(*ents, configChangeEntries...) +} + +func appendNormalRequestEnts(ents *[]raftpb.Entry) { + a := true + b := false + + requests := []etcdserverpb.Request{ + {ID: 0, Method: "", Path: "/path0", Val: "{\"hey\":\"ho\",\"hi\":[\"yo\"]}", Dir: true, PrevValue: "", PrevIndex: 0, PrevExist: &b, Expiration: 9, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b}, + {ID: 1, Method: "QGET", Path: "/path1", Val: "{\"0\":\"1\",\"2\":[\"3\"]}", Dir: false, PrevValue: "", PrevIndex: 0, PrevExist: &b, Expiration: 9, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b}, + {ID: 2, Method: "SYNC", Path: "/path2", Val: "{\"0\":\"1\",\"2\":[\"3\"]}", Dir: false, PrevValue: "", PrevIndex: 0, PrevExist: &b, Expiration: 2, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b}, + {ID: 3, Method: "DELETE", Path: "/path3", Val: "{\"hey\":\"ho\",\"hi\":[\"yo\"]}", Dir: false, PrevValue: "", PrevIndex: 0, PrevExist: &a, Expiration: 2, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b}, + {ID: 4, Method: "RANDOM", Path: "/path4/superlong" + strings.Repeat("/path", 30), Val: "{\"hey\":\"ho\",\"hi\":[\"yo\"]}", Dir: false, PrevValue: "", PrevIndex: 0, PrevExist: &b, Expiration: 2, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b}, + } + + for i, request := range requests { + var currentry raftpb.Entry + currentry.Term = 3 + currentry.Index = uint64(i + 5) + currentry.Type = raftpb.EntryNormal + currentry.Data = pbutil.MustMarshal(&request) + *ents = append(*ents, currentry) + } +} + +func appendNormalIRREnts(ents *[]raftpb.Entry) { + irrrange := &etcdserverpb.RangeRequest{Key: []byte("1"), RangeEnd: []byte("hi"), Limit: 6, Revision: 1, SortOrder: 1, SortTarget: 0, Serializable: false, KeysOnly: false, CountOnly: false, MinModRevision: 0, MaxModRevision: 20000, MinCreateRevision: 0, MaxCreateRevision: 20000} + + irrput := &etcdserverpb.PutRequest{Key: []byte("foo1"), Value: []byte("bar1"), Lease: 1, PrevKv: false, IgnoreValue: false, IgnoreLease: true} + + irrdeleterange := &etcdserverpb.DeleteRangeRequest{Key: []byte("0"), RangeEnd: []byte("9"), PrevKv: true} + + delInRangeReq := &etcdserverpb.RequestOp{Request: &etcdserverpb.RequestOp_RequestDeleteRange{ + RequestDeleteRange: &etcdserverpb.DeleteRangeRequest{ + Key: []byte("a"), RangeEnd: []byte("b"), + }, + }, + } + + irrtxn := &etcdserverpb.TxnRequest{Success: []*etcdserverpb.RequestOp{delInRangeReq}, Failure: []*etcdserverpb.RequestOp{delInRangeReq}} + + irrcompaction := &etcdserverpb.CompactionRequest{Revision: 0, Physical: true} + + irrleasegrant := &etcdserverpb.LeaseGrantRequest{TTL: 1, ID: 1} + + irrleaserevoke := &etcdserverpb.LeaseRevokeRequest{ID: 2} + + irralarm := &etcdserverpb.AlarmRequest{Action: 3, MemberID: 4, Alarm: 5} + + irrauthenable := &etcdserverpb.AuthEnableRequest{} + + irrauthdisable := &etcdserverpb.AuthDisableRequest{} + + irrauthenticate := &etcdserverpb.InternalAuthenticateRequest{Name: "myname", Password: "password", SimpleToken: "token"} + + irrauthuseradd := &etcdserverpb.AuthUserAddRequest{Name: "name1", Password: "pass1"} + + irrauthuserdelete := &etcdserverpb.AuthUserDeleteRequest{Name: "name1"} + + irrauthuserget := &etcdserverpb.AuthUserGetRequest{Name: "name1"} + + irrauthuserchangepassword := &etcdserverpb.AuthUserChangePasswordRequest{Name: "name1", Password: "pass2"} + + irrauthusergrantrole := &etcdserverpb.AuthUserGrantRoleRequest{User: "user1", Role: "role1"} + + irrauthuserrevokerole := &etcdserverpb.AuthUserRevokeRoleRequest{Name: "user2", Role: "role2"} + + irrauthuserlist := &etcdserverpb.AuthUserListRequest{} + + irrauthrolelist := &etcdserverpb.AuthRoleListRequest{} + + irrauthroleadd := &etcdserverpb.AuthRoleAddRequest{Name: "role2"} + + irrauthroledelete := &etcdserverpb.AuthRoleDeleteRequest{Role: "role1"} + + irrauthroleget := &etcdserverpb.AuthRoleGetRequest{Role: "role3"} + + perm := &authpb.Permission{ + PermType: authpb.WRITE, + Key: []byte("Keys"), + RangeEnd: []byte("RangeEnd"), + } + + irrauthrolegrantpermission := &etcdserverpb.AuthRoleGrantPermissionRequest{Name: "role3", Perm: perm} + + irrauthrolerevokepermission := &etcdserverpb.AuthRoleRevokePermissionRequest{Role: "role3", Key: []byte("key"), RangeEnd: []byte("rangeend")} + + irrs := []etcdserverpb.InternalRaftRequest{ + {ID: 5, Range: irrrange}, + {ID: 6, Put: irrput}, + {ID: 7, DeleteRange: irrdeleterange}, + {ID: 8, Txn: irrtxn}, + {ID: 9, Compaction: irrcompaction}, + {ID: 10, LeaseGrant: irrleasegrant}, + {ID: 11, LeaseRevoke: irrleaserevoke}, + {ID: 12, Alarm: irralarm}, + {ID: 13, AuthEnable: irrauthenable}, + {ID: 14, AuthDisable: irrauthdisable}, + {ID: 15, Authenticate: irrauthenticate}, + {ID: 16, AuthUserAdd: irrauthuseradd}, + {ID: 17, AuthUserDelete: irrauthuserdelete}, + {ID: 18, AuthUserGet: irrauthuserget}, + {ID: 19, AuthUserChangePassword: irrauthuserchangepassword}, + {ID: 20, AuthUserGrantRole: irrauthusergrantrole}, + {ID: 21, AuthUserRevokeRole: irrauthuserrevokerole}, + {ID: 22, AuthUserList: irrauthuserlist}, + {ID: 23, AuthRoleList: irrauthrolelist}, + {ID: 24, AuthRoleAdd: irrauthroleadd}, + {ID: 25, AuthRoleDelete: irrauthroledelete}, + {ID: 26, AuthRoleGet: irrauthroleget}, + {ID: 27, AuthRoleGrantPermission: irrauthrolegrantpermission}, + {ID: 28, AuthRoleRevokePermission: irrauthrolerevokepermission}, + } + + for i, irr := range irrs { + var currentry raftpb.Entry + currentry.Term = uint64(i + 4) + currentry.Index = uint64(i + 10) + currentry.Type = raftpb.EntryNormal + currentry.Data = pbutil.MustMarshal(&irr) + *ents = append(*ents, currentry) + } +} + +func appendUnknownNormalEnts(ents *[]raftpb.Entry) { + var currentry raftpb.Entry + currentry.Term = 27 + currentry.Index = 34 + currentry.Type = raftpb.EntryNormal + currentry.Data = []byte("?") + *ents = append(*ents, currentry) +} diff --git a/tools/etcd-dump-logs/expectedoutput/listAll.output b/tools/etcd-dump-logs/expectedoutput/listAll.output new file mode 100644 index 00000000000..2b998c57409 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listAll.output @@ -0,0 +1,44 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 1 1 conf method=ConfChangeAddNode id=2 + 2 2 conf method=ConfChangeRemoveNode id=2 + 2 3 conf method=ConfChangeUpdateNode id=2 + 2 4 conf method=ConfChangeAddLearnerNode id=3 + 3 5 norm noop + 3 6 norm method=QGET path="/path1" + 3 7 norm method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST" + 3 8 norm method=DELETE path="/path3" + 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" + 4 10 norm ID:5 range: + 5 11 norm ID:6 put: + 6 12 norm ID:7 delete_range: + 7 13 norm ID:8 txn: > failure: > > + 8 14 norm ID:9 compaction: + 9 15 norm ID:10 lease_grant: + 10 16 norm ID:11 lease_revoke: + 11 17 norm ID:12 alarm: + 12 18 norm ID:13 auth_enable:<> + 13 19 norm ID:14 auth_disable:<> + 14 20 norm ID:15 authenticate: + 15 21 norm ID:16 auth_user_add: + 16 22 norm ID:17 auth_user_delete: + 17 23 norm ID:18 auth_user_get: + 18 24 norm ID:19 auth_user_change_password: + 19 25 norm ID:20 auth_user_grant_role: + 20 26 norm ID:21 auth_user_revoke_role: + 21 27 norm ID:22 auth_user_list:<> + 22 28 norm ID:23 auth_role_list:<> + 23 29 norm ID:24 auth_role_add: + 24 30 norm ID:25 auth_role_delete: + 25 31 norm ID:26 auth_role_get: + 26 32 norm ID:27 auth_role_grant_permission: > + 27 33 norm ID:28 auth_role_revoke_permission: + 27 34 norm ??? + +Entry types () count is : 34 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listConfigChange.output b/tools/etcd-dump-logs/expectedoutput/listConfigChange.output new file mode 100644 index 00000000000..c682b537273 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listConfigChange.output @@ -0,0 +1,14 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 1 1 conf method=ConfChangeAddNode id=2 + 2 2 conf method=ConfChangeRemoveNode id=2 + 2 3 conf method=ConfChangeUpdateNode id=2 + 2 4 conf method=ConfChangeAddLearnerNode id=3 + +Entry types (ConfigChange) count is : 4 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRCompaction.output b/tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRCompaction.output new file mode 100644 index 00000000000..663f2559fdf --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRCompaction.output @@ -0,0 +1,15 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 1 1 conf method=ConfChangeAddNode id=2 + 2 2 conf method=ConfChangeRemoveNode id=2 + 2 3 conf method=ConfChangeUpdateNode id=2 + 2 4 conf method=ConfChangeAddLearnerNode id=3 + 8 14 norm ID:9 compaction: + +Entry types (ConfigChange,IRRCompaction) count is : 5 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRLeaseRevoke.output b/tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRLeaseRevoke.output new file mode 100644 index 00000000000..0820848d378 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRLeaseRevoke.output @@ -0,0 +1,11 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 10 16 norm ID:11 lease_revoke: + +Entry types (IRRLeaseRevoke) count is : 1 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listIRRCompaction.output b/tools/etcd-dump-logs/expectedoutput/listIRRCompaction.output new file mode 100644 index 00000000000..ebd826bece7 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listIRRCompaction.output @@ -0,0 +1,11 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 8 14 norm ID:9 compaction: + +Entry types (IRRCompaction) count is : 1 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listIRRDeleteRange.output b/tools/etcd-dump-logs/expectedoutput/listIRRDeleteRange.output new file mode 100644 index 00000000000..b3dec283db6 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listIRRDeleteRange.output @@ -0,0 +1,11 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 6 12 norm ID:7 delete_range: + +Entry types (IRRDeleteRange) count is : 1 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listIRRLeaseGrant.output b/tools/etcd-dump-logs/expectedoutput/listIRRLeaseGrant.output new file mode 100644 index 00000000000..fdc042a0bd4 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listIRRLeaseGrant.output @@ -0,0 +1,11 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 9 15 norm ID:10 lease_grant: + +Entry types (IRRLeaseGrant) count is : 1 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listIRRLeaseRevoke.output b/tools/etcd-dump-logs/expectedoutput/listIRRLeaseRevoke.output new file mode 100644 index 00000000000..0820848d378 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listIRRLeaseRevoke.output @@ -0,0 +1,11 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 10 16 norm ID:11 lease_revoke: + +Entry types (IRRLeaseRevoke) count is : 1 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listIRRPut.output b/tools/etcd-dump-logs/expectedoutput/listIRRPut.output new file mode 100644 index 00000000000..bbda48f2342 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listIRRPut.output @@ -0,0 +1,11 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 5 11 norm ID:6 put: + +Entry types (IRRPut) count is : 1 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listIRRRange.output b/tools/etcd-dump-logs/expectedoutput/listIRRRange.output new file mode 100644 index 00000000000..2a90ab74401 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listIRRRange.output @@ -0,0 +1,11 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 4 10 norm ID:5 range: + +Entry types (IRRRange) count is : 1 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listIRRTxn.output b/tools/etcd-dump-logs/expectedoutput/listIRRTxn.output new file mode 100644 index 00000000000..3ad761027e7 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listIRRTxn.output @@ -0,0 +1,11 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 7 13 norm ID:8 txn: > failure: > > + +Entry types (IRRTxn) count is : 1 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listInternalRaftRequest.output b/tools/etcd-dump-logs/expectedoutput/listInternalRaftRequest.output new file mode 100644 index 00000000000..73728c508c3 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listInternalRaftRequest.output @@ -0,0 +1,34 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 4 10 norm ID:5 range: + 5 11 norm ID:6 put: + 6 12 norm ID:7 delete_range: + 7 13 norm ID:8 txn: > failure: > > + 8 14 norm ID:9 compaction: + 9 15 norm ID:10 lease_grant: + 10 16 norm ID:11 lease_revoke: + 11 17 norm ID:12 alarm: + 12 18 norm ID:13 auth_enable:<> + 13 19 norm ID:14 auth_disable:<> + 14 20 norm ID:15 authenticate: + 15 21 norm ID:16 auth_user_add: + 16 22 norm ID:17 auth_user_delete: + 17 23 norm ID:18 auth_user_get: + 18 24 norm ID:19 auth_user_change_password: + 19 25 norm ID:20 auth_user_grant_role: + 20 26 norm ID:21 auth_user_revoke_role: + 21 27 norm ID:22 auth_user_list:<> + 22 28 norm ID:23 auth_role_list:<> + 23 29 norm ID:24 auth_role_add: + 24 30 norm ID:25 auth_role_delete: + 25 31 norm ID:26 auth_role_get: + 26 32 norm ID:27 auth_role_grant_permission: > + 27 33 norm ID:28 auth_role_revoke_permission: + +Entry types (InternalRaftRequest) count is : 24 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listNormal.output b/tools/etcd-dump-logs/expectedoutput/listNormal.output new file mode 100644 index 00000000000..1afb8999437 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listNormal.output @@ -0,0 +1,40 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 3 5 norm noop + 3 6 norm method=QGET path="/path1" + 3 7 norm method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST" + 3 8 norm method=DELETE path="/path3" + 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" + 4 10 norm ID:5 range: + 5 11 norm ID:6 put: + 6 12 norm ID:7 delete_range: + 7 13 norm ID:8 txn: > failure: > > + 8 14 norm ID:9 compaction: + 9 15 norm ID:10 lease_grant: + 10 16 norm ID:11 lease_revoke: + 11 17 norm ID:12 alarm: + 12 18 norm ID:13 auth_enable:<> + 13 19 norm ID:14 auth_disable:<> + 14 20 norm ID:15 authenticate: + 15 21 norm ID:16 auth_user_add: + 16 22 norm ID:17 auth_user_delete: + 17 23 norm ID:18 auth_user_get: + 18 24 norm ID:19 auth_user_change_password: + 19 25 norm ID:20 auth_user_grant_role: + 20 26 norm ID:21 auth_user_revoke_role: + 21 27 norm ID:22 auth_user_list:<> + 22 28 norm ID:23 auth_role_list:<> + 23 29 norm ID:24 auth_role_add: + 24 30 norm ID:25 auth_role_delete: + 25 31 norm ID:26 auth_role_get: + 26 32 norm ID:27 auth_role_grant_permission: > + 27 33 norm ID:28 auth_role_revoke_permission: + 27 34 norm ??? + +Entry types (Normal) count is : 30 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listRequest.output b/tools/etcd-dump-logs/expectedoutput/listRequest.output new file mode 100644 index 00000000000..c64dc4d673d --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/listRequest.output @@ -0,0 +1,15 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 3 5 norm noop + 3 6 norm method=QGET path="/path1" + 3 7 norm method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST" + 3 8 norm method=DELETE path="/path3" + 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" + +Entry types (Request) count is : 5 \ No newline at end of file diff --git a/tools/etcd-dump-logs/main.go b/tools/etcd-dump-logs/main.go index 6ac060b2333..c741886bc56 100644 --- a/tools/etcd-dump-logs/main.go +++ b/tools/etcd-dump-logs/main.go @@ -1,4 +1,4 @@ -// Copyright 2015 The etcd Authors +// Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import ( "fmt" "log" "path/filepath" + "strings" "time" "github.com/coreos/etcd/etcdserver/etcdserverpb" @@ -35,6 +36,11 @@ import ( func main() { snapfile := flag.String("start-snap", "", "The base name of snapshot file to start dumping") index := flag.Uint64("start-index", 0, "The index to start dumping") + entrytype := flag.String("entry-type", "", `If set, filters output by entry type. Must be one or more than one of: + ConfigChange, Normal, Request, InternalRaftRequest, + IRRRange, IRRPut, IRRDeleteRange, IRRTxn, + IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke`) + flag.Parse() if len(flag.Args()) != 1 { @@ -96,46 +102,7 @@ func main() { fmt.Printf("WAL entries:\n") fmt.Printf("lastIndex=%d\n", ents[len(ents)-1].Index) fmt.Printf("%4s\t%10s\ttype\tdata\n", "term", "index") - for _, e := range ents { - msg := fmt.Sprintf("%4d\t%10d", e.Term, e.Index) - switch e.Type { - case raftpb.EntryNormal: - msg = fmt.Sprintf("%s\tnorm", msg) - - var rr etcdserverpb.InternalRaftRequest - if err := rr.Unmarshal(e.Data); err == nil { - msg = fmt.Sprintf("%s\t%s", msg, rr.String()) - break - } - - // TODO: remove sensitive information - // (https://github.com/coreos/etcd/issues/7620) - var r etcdserverpb.Request - if err := r.Unmarshal(e.Data); err == nil { - switch r.Method { - case "": - msg = fmt.Sprintf("%s\tnoop", msg) - case "SYNC": - msg = fmt.Sprintf("%s\tmethod=SYNC time=%q", msg, time.Unix(0, r.Time)) - case "QGET", "DELETE": - msg = fmt.Sprintf("%s\tmethod=%s path=%s", msg, r.Method, excerpt(r.Path, 64, 64)) - default: - msg = fmt.Sprintf("%s\tmethod=%s path=%s val=%s", msg, r.Method, excerpt(r.Path, 64, 64), excerpt(r.Val, 128, 0)) - } - break - } - msg = fmt.Sprintf("%s\t???", msg) - case raftpb.EntryConfChange: - msg = fmt.Sprintf("%s\tconf", msg) - var r raftpb.ConfChange - if err := r.Unmarshal(e.Data); err != nil { - msg = fmt.Sprintf("%s\t???", msg) - } else { - msg = fmt.Sprintf("%s\tmethod=%s id=%s", msg, r.Type, types.ID(r.NodeID)) - } - } - fmt.Println(msg) - } + listEntriesType(*entrytype, ents) } func walDir(dataDir string) string { return filepath.Join(dataDir, "member", "wal") } @@ -166,3 +133,175 @@ func excerpt(str string, pre, suf int) string { } return fmt.Sprintf("%q...%q", str[:pre], str[len(str)-suf:]) } + +type EntryFilter func(e raftpb.Entry) (bool, string) + +// The 9 pass functions below takes the raftpb.Entry and return if the entry should be printed and the type of entry, +// the type of the entry will used in the following print function +func passConfChange(entry raftpb.Entry) (bool, string) { + return entry.Type == raftpb.EntryConfChange, "ConfigChange" +} + +func passInternalRaftRequest(entry raftpb.Entry) (bool, string) { + var rr etcdserverpb.InternalRaftRequest + return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil, "InternalRaftRequest" +} + +func passUnknownNormal(entry raftpb.Entry) (bool, string) { + var rr1 etcdserverpb.Request + var rr2 etcdserverpb.InternalRaftRequest + return (entry.Type == raftpb.EntryNormal) && (rr1.Unmarshal(entry.Data) != nil) && (rr2.Unmarshal(entry.Data) != nil), "UnknownNormal" +} + +func passIRRRange(entry raftpb.Entry) (bool, string) { + var rr etcdserverpb.InternalRaftRequest + return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.Range != nil, "InternalRaftRequest" +} + +func passIRRPut(entry raftpb.Entry) (bool, string) { + var rr etcdserverpb.InternalRaftRequest + return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.Put != nil, "InternalRaftRequest" +} + +func passIRRDeleteRange(entry raftpb.Entry) (bool, string) { + var rr etcdserverpb.InternalRaftRequest + return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.DeleteRange != nil, "InternalRaftRequest" +} + +func passIRRTxn(entry raftpb.Entry) (bool, string) { + var rr etcdserverpb.InternalRaftRequest + return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.Txn != nil, "InternalRaftRequest" +} + +func passIRRCompaction(entry raftpb.Entry) (bool, string) { + var rr etcdserverpb.InternalRaftRequest + return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.Compaction != nil, "InternalRaftRequest" +} + +func passIRRLeaseGrant(entry raftpb.Entry) (bool, string) { + var rr etcdserverpb.InternalRaftRequest + return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.LeaseGrant != nil, "InternalRaftRequest" +} + +func passIRRLeaseRevoke(entry raftpb.Entry) (bool, string) { + var rr etcdserverpb.InternalRaftRequest + return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.LeaseRevoke != nil, "InternalRaftRequest" +} + +func passRequest(entry raftpb.Entry) (bool, string) { + var rr1 etcdserverpb.Request + var rr2 etcdserverpb.InternalRaftRequest + return entry.Type == raftpb.EntryNormal && rr1.Unmarshal(entry.Data) == nil && rr2.Unmarshal(entry.Data) != nil, "Request" +} + +type EntryPrinter func(e raftpb.Entry) + +// The 4 print functions below print the entry format based on there types + +// printInternalRaftRequest is used to print entry information for IRRRange, IRRPut, +// IRRDeleteRange and IRRTxn entries +func printInternalRaftRequest(entry raftpb.Entry) { + var rr etcdserverpb.InternalRaftRequest + if err := rr.Unmarshal(entry.Data); err == nil { + fmt.Printf("%4d\t%10d\tnorm\t%s\n", entry.Term, entry.Index, rr.String()) + } +} + +func printUnknownNormal(entry raftpb.Entry) { + fmt.Printf("%4d\t%10d\tnorm\t???\n", entry.Term, entry.Index) +} + +func printConfChange(entry raftpb.Entry) { + fmt.Printf("%4d\t%10d", entry.Term, entry.Index) + fmt.Printf("\tconf") + var r raftpb.ConfChange + if err := r.Unmarshal(entry.Data); err != nil { + fmt.Printf("\t???\n") + } else { + fmt.Printf("\tmethod=%s id=%s\n", r.Type, types.ID(r.NodeID)) + } +} + +func printRequest(entry raftpb.Entry) { + var r etcdserverpb.Request + if err := r.Unmarshal(entry.Data); err == nil { + fmt.Printf("%4d\t%10d\tnorm", entry.Term, entry.Index) + switch r.Method { + case "": + fmt.Printf("\tnoop\n") + case "SYNC": + fmt.Printf("\tmethod=SYNC time=%q\n", time.Unix(0, r.Time)) + case "QGET", "DELETE": + fmt.Printf("\tmethod=%s path=%s\n", r.Method, excerpt(r.Path, 64, 64)) + default: + fmt.Printf("\tmethod=%s path=%s val=%s\n", r.Method, excerpt(r.Path, 64, 64), excerpt(r.Val, 128, 0)) + } + } +} + +// evaluateEntrytypeFlag evaluates entry-type flag and choose proper filter/filters to filter entries +func evaluateEntrytypeFlag(entrytype string) []EntryFilter { + var entrytypelist []string + if entrytype != "" { + entrytypelist = strings.Split(entrytype, ",") + } + + validRequest := map[string][]EntryFilter{"ConfigChange": {passConfChange}, + "Normal": {passInternalRaftRequest, passRequest, passUnknownNormal}, + "Request": {passRequest}, + "InternalRaftRequest": {passInternalRaftRequest}, + "IRRRange": {passIRRRange}, + "IRRPut": {passIRRPut}, + "IRRDeleteRange": {passIRRDeleteRange}, + "IRRTxn": {passIRRTxn}, + "IRRCompaction": {passIRRCompaction}, + "IRRLeaseGrant": {passIRRLeaseGrant}, + "IRRLeaseRevoke": {passIRRLeaseRevoke}, + } + filters := make([]EntryFilter, 0) + if len(entrytypelist) == 0 { + filters = append(filters, passInternalRaftRequest) + filters = append(filters, passRequest) + filters = append(filters, passUnknownNormal) + filters = append(filters, passConfChange) + } + for _, et := range entrytypelist { + if f, ok := validRequest[et]; ok { + filters = append(filters, f...) + } else { + log.Printf(`[%+v] is not a valid entry-type, ignored. +Please set entry-type to one or more of the following: +ConfigChange, Normal, Request, InternalRaftRequest, +IRRRange, IRRPut, IRRDeleteRange, IRRTxn, +IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke`, et) + } + } + + return filters +} + +// listEntriesType filters and prints entries based on the entry-type flag, +func listEntriesType(entrytype string, ents []raftpb.Entry) { + entryFilters := evaluateEntrytypeFlag(entrytype) + printerMap := map[string]EntryPrinter{"InternalRaftRequest": printInternalRaftRequest, + "Request": printRequest, + "ConfigChange": printConfChange, + "UnknownNormal": printUnknownNormal} + cnt := 0 + for _, e := range ents { + passed := false + currtype := "" + for _, filter := range entryFilters { + passed, currtype = filter(e) + if passed { + cnt++ + break + } + } + if passed { + printer := printerMap[currtype] + printer(e) + } + } + fmt.Printf("\nEntry types (%s) count is : %d", entrytype, cnt) +}