From fd984d87a393f5c224be2fc8aa68cedc25eb16f1 Mon Sep 17 00:00:00 2001 From: millken Date: Fri, 9 Sep 2022 15:03:48 +0800 Subject: [PATCH 01/18] add collector package to monitor node cpu/loadavg/memory usage --- config/config.go | 1 + go.mod | 1 + go.sum | 2 + pkg/collector/collector.go | 87 +++++++++++++++++++++++++++++++++ pkg/collector/collector_test.go | 36 ++++++++++++++ pkg/collector/cpu.go | 36 ++++++++++++++ pkg/collector/cpu_test.go | 29 +++++++++++ pkg/collector/loadavg.go | 38 ++++++++++++++ pkg/collector/loadavg_test.go | 29 +++++++++++ pkg/collector/memory.go | 35 +++++++++++++ pkg/collector/memory_test.go | 29 +++++++++++ server/main.go | 18 +++++++ 12 files changed, 341 insertions(+) create mode 100644 pkg/collector/collector.go create mode 100644 pkg/collector/collector_test.go create mode 100644 pkg/collector/cpu.go create mode 100644 pkg/collector/cpu_test.go create mode 100644 pkg/collector/loadavg.go create mode 100644 pkg/collector/loadavg_test.go create mode 100644 pkg/collector/memory.go create mode 100644 pkg/collector/memory_test.go diff --git a/config/config.go b/config/config.go index d628bc2df1..ff48ce3594 100644 --- a/config/config.go +++ b/config/config.go @@ -241,6 +241,7 @@ type ( Log log.GlobalConfig `yaml:"log"` SubLogs map[string]log.GlobalConfig `yaml:"subLogs"` Genesis genesis.Genesis `yaml:"genesis"` + Collector []string `yaml:"collector"` } // Validate is the interface of validating the config diff --git a/go.mod b/go.mod index f249692541..08675295a9 100644 --- a/go.mod +++ b/go.mod @@ -55,6 +55,7 @@ require ( require ( github.com/golang/protobuf v1.5.2 + github.com/mackerelio/go-osstat v0.2.3 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/shirou/gopsutil/v3 v3.22.2 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0 diff --git a/go.sum b/go.sum index dc8724f64d..283ff31471 100644 --- a/go.sum +++ b/go.sum @@ -840,6 +840,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/mackerelio/go-osstat v0.2.3 h1:jAMXD5erlDE39kdX2CU7YwCGRcxIO33u/p8+Fhe5dJw= +github.com/mackerelio/go-osstat v0.2.3/go.mod h1:DQbPOnsss9JHIXgBStc/dnhhir3gbd3YH+Dbdi7ptMA= github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go new file mode 100644 index 0000000000..e98c4b0729 --- /dev/null +++ b/pkg/collector/collector.go @@ -0,0 +1,87 @@ +package collector + +import ( + "fmt" + "sync" + + "github.com/prometheus/client_golang/prometheus" +) + +// Namespace defines the common namespace to be used by all metrics. +const namespace = "node" + +var ( + factories = make(map[string]func() (Collector, error)) + initiatedCollectorsMtx = sync.Mutex{} + initiatedCollectors = make(map[string]Collector) +) + +func registerCollector(collector string, factory func() (Collector, error)) { + factories[collector] = factory +} + +// NodeCollector implements the prometheus.Collector interface. +type NodeCollector struct { + Collectors map[string]Collector +} + +// NewNodeCollector creates a new NodeCollector. +func NewNodeCollector(filters ...string) (*NodeCollector, error) { + f := make(map[string]bool) + for _, filter := range filters { + _, exist := factories[filter] + if !exist { + return nil, fmt.Errorf("missing collector: %s", filter) + } + f[filter] = true + } + collectors := make(map[string]Collector) + initiatedCollectorsMtx.Lock() + defer initiatedCollectorsMtx.Unlock() + for key := range f { + if collector, ok := initiatedCollectors[key]; ok { + collectors[key] = collector + } else { + collector, err := factories[key]() + if err != nil { + return nil, err + } + collectors[key] = collector + initiatedCollectors[key] = collector + } + } + return &NodeCollector{Collectors: collectors}, nil +} + +// Describe implements the prometheus.Collector interface. +func (n NodeCollector) Describe(ch chan<- *prometheus.Desc) { + +} + +// Collect implements the prometheus.Collector interface. +func (n NodeCollector) Collect(ch chan<- prometheus.Metric) { + wg := sync.WaitGroup{} + wg.Add(len(n.Collectors)) + for name, c := range n.Collectors { + go func(name string, c Collector) { + c.Update(ch) + wg.Done() + }(name, c) + } + wg.Wait() +} + +// Collector is the interface a collector has to implement. +type Collector interface { + // Get new metrics and expose them via prometheus registry. + Update(ch chan<- prometheus.Metric) error +} + +type typedDesc struct { + desc *prometheus.Desc + valueType prometheus.ValueType +} + +func (d *typedDesc) mustNewConstMetric(value float64, labels ...string) prometheus.Metric { + return prometheus.MustNewConstMetric(d.desc, d.valueType, value, labels...) +} diff --git a/pkg/collector/collector_test.go b/pkg/collector/collector_test.go new file mode 100644 index 0000000000..4f1cffefab --- /dev/null +++ b/pkg/collector/collector_test.go @@ -0,0 +1,36 @@ +package collector + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +func TestCollector(t *testing.T) { + require := require.New(t) + c, err := NewNodeCollector("test") + require.ErrorContains(err, "missing collector") + c, err = NewNodeCollector("cpu") + require.NoError(err) + ch := make(chan prometheus.Metric) + go func() { + c.Collect(ch) + close(ch) + }() + + ch2 := make(chan *prometheus.Desc) + go func() { + c.Describe(ch2) + close(ch2) + }() + n := 0 + for m := range ch { + n++ + pb := &dto.Metric{} + err := m.Write(pb) + require.NoError(err) + } + require.Equal(4, n) +} diff --git a/pkg/collector/cpu.go b/pkg/collector/cpu.go new file mode 100644 index 0000000000..63e73ba751 --- /dev/null +++ b/pkg/collector/cpu.go @@ -0,0 +1,36 @@ +package collector + +import ( + "fmt" + + "github.com/mackerelio/go-osstat/cpu" + "github.com/prometheus/client_golang/prometheus" +) + +type cpuCollector struct { + metric *prometheus.Desc +} + +func init() { + registerCollector("cpu", NewCPUCollector) +} + +// NewCPUCollector returns a new Collector exposing kernel/system statistics. +func NewCPUCollector() (Collector, error) { + return &cpuCollector{ + metric: prometheus.NewDesc(namespace+"_cpu_stats", "cpu statistics", []string{"mode"}, nil), + }, nil +} + +func (c *cpuCollector) Update(ch chan<- prometheus.Metric) error { + stats, err := cpu.Get() + if err != nil { + return fmt.Errorf("couldn't get cpu: %w", err) + } + ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(stats.User), "user") + ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(stats.System), "system") + ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(stats.Total), "total") + ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(stats.Idle), "idle") + + return err +} diff --git a/pkg/collector/cpu_test.go b/pkg/collector/cpu_test.go new file mode 100644 index 0000000000..6d9a146a83 --- /dev/null +++ b/pkg/collector/cpu_test.go @@ -0,0 +1,29 @@ +package collector + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +func TestCPU(t *testing.T) { + require := require.New(t) + c, err := NewCPUCollector() + require.NoError(err) + require.NotNil(c) + ch := make(chan prometheus.Metric) + go func() { + c.Update(ch) + close(ch) + }() + n := 0 + for m := range ch { + n++ + pb := &dto.Metric{} + err := m.Write(pb) + require.NoError(err) + } + require.Equal(4, n) +} diff --git a/pkg/collector/loadavg.go b/pkg/collector/loadavg.go new file mode 100644 index 0000000000..d5e356db18 --- /dev/null +++ b/pkg/collector/loadavg.go @@ -0,0 +1,38 @@ +package collector + +import ( + "fmt" + + "github.com/mackerelio/go-osstat/loadavg" + "github.com/prometheus/client_golang/prometheus" +) + +type loadavgCollector struct { + metric []typedDesc +} + +func init() { + registerCollector("loadavg", NewLoadavgCollector) +} + +// NewLoadavgCollector returns a new Collector exposing load average stats. +func NewLoadavgCollector() (Collector, error) { + return &loadavgCollector{ + metric: []typedDesc{ + {prometheus.NewDesc(namespace+"_load1", "1m load average.", nil, nil), prometheus.GaugeValue}, + {prometheus.NewDesc(namespace+"_load5", "5m load average.", nil, nil), prometheus.GaugeValue}, + {prometheus.NewDesc(namespace+"_load15", "15m load average.", nil, nil), prometheus.GaugeValue}, + }, + }, nil +} + +func (c *loadavgCollector) Update(ch chan<- prometheus.Metric) error { + loads, err := loadavg.Get() + if err != nil { + return fmt.Errorf("couldn't get load: %w", err) + } + ch <- c.metric[0].mustNewConstMetric(loads.Loadavg1) + ch <- c.metric[1].mustNewConstMetric(loads.Loadavg5) + ch <- c.metric[2].mustNewConstMetric(loads.Loadavg15) + return err +} diff --git a/pkg/collector/loadavg_test.go b/pkg/collector/loadavg_test.go new file mode 100644 index 0000000000..c79e011def --- /dev/null +++ b/pkg/collector/loadavg_test.go @@ -0,0 +1,29 @@ +package collector + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +func TestLoadavg(t *testing.T) { + require := require.New(t) + c, err := NewLoadavgCollector() + require.NoError(err) + require.NotNil(c) + ch := make(chan prometheus.Metric) + go func() { + c.Update(ch) + close(ch) + }() + n := 0 + for m := range ch { + n++ + pb := &dto.Metric{} + err := m.Write(pb) + require.NoError(err) + } + require.Equal(3, n) +} diff --git a/pkg/collector/memory.go b/pkg/collector/memory.go new file mode 100644 index 0000000000..14ef7b1aa3 --- /dev/null +++ b/pkg/collector/memory.go @@ -0,0 +1,35 @@ +package collector + +import ( + "fmt" + + "github.com/mackerelio/go-osstat/memory" + "github.com/prometheus/client_golang/prometheus" +) + +type meminfoCollector struct { + metric *prometheus.Desc +} + +func init() { + registerCollector("memory", NewMeminfoCollector) +} + +// NewMeminfoCollector returns a new Collector exposing memory stats. +func NewMeminfoCollector() (Collector, error) { + return &meminfoCollector{ + metric: prometheus.NewDesc(namespace+"_memory_stats", "memory stats", []string{"mode"}, nil), + }, nil +} + +// Update to get the platform specific memory stats. +func (c *meminfoCollector) Update(ch chan<- prometheus.Metric) error { + memInfo, err := memory.Get() + if err != nil { + return fmt.Errorf("couldn't get meminfo: %w", err) + } + ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(memInfo.Total), "total") + ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(memInfo.Free), "free") + ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(memInfo.Used), "used") + return nil +} diff --git a/pkg/collector/memory_test.go b/pkg/collector/memory_test.go new file mode 100644 index 0000000000..d2536efc8f --- /dev/null +++ b/pkg/collector/memory_test.go @@ -0,0 +1,29 @@ +package collector + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +func TestMemory(t *testing.T) { + require := require.New(t) + c, err := NewMeminfoCollector() + require.NoError(err) + require.NotNil(c) + ch := make(chan prometheus.Metric) + go func() { + c.Update(ch) + close(ch) + }() + n := 0 + for m := range ch { + n++ + pb := &dto.Metric{} + err := m.Write(pb) + require.NoError(err) + } + require.Equal(3, n) +} diff --git a/server/main.go b/server/main.go index 2bb316173c..6536b4680e 100644 --- a/server/main.go +++ b/server/main.go @@ -22,12 +22,14 @@ import ( "syscall" "github.com/iotexproject/go-pkgs/hash" + "github.com/prometheus/client_golang/prometheus" _ "go.uber.org/automaxprocs" "go.uber.org/zap" "github.com/iotexproject/iotex-core/blockchain/block" "github.com/iotexproject/iotex-core/blockchain/genesis" "github.com/iotexproject/iotex-core/config" + "github.com/iotexproject/iotex-core/pkg/collector" "github.com/iotexproject/iotex-core/pkg/log" "github.com/iotexproject/iotex-core/pkg/probe" "github.com/iotexproject/iotex-core/pkg/recovery" @@ -103,6 +105,10 @@ func main() { glog.Fatalln("Cannot config global logger, use default one: ", zap.Error(err)) } + if err = initCollector(cfg); err != nil { + glog.Fatalln("Cannot init collector: ", zap.Error(err)) + } + if err = recovery.SetCrashlogDir(cfg.System.SystemLogDBPath); err != nil { glog.Fatalln("Failed to set directory of crashlog: ", zap.Error(err)) } @@ -127,6 +133,7 @@ func main() { if err := probeSvr.Start(ctx); err != nil { log.L().Fatal("Failed to start probe server.", zap.Error(err)) } + go func() { <-stop // start stopping @@ -173,3 +180,14 @@ func initLogger(cfg config.Config) error { zap.String("ioAddr", addr.String()), )) } + +func initCollector(cfg config.Config) error { + nc, err := collector.NewNodeCollector(cfg.Collector...) + if err != nil { + return err + } + if err := prometheus.Register(nc); err != nil { + return err + } + return nil +} From dc8b7257ca88423940fe19fe8e8f41001b25d7c8 Mon Sep 17 00:00:00 2001 From: millken Date: Fri, 9 Sep 2022 15:19:53 +0800 Subject: [PATCH 02/18] fix test --- config/config.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index ff48ce3594..6f1ed6ba84 100644 --- a/config/config.go +++ b/config/config.go @@ -120,9 +120,10 @@ var ( StartSubChainInterval: 10 * time.Second, SystemLogDBPath: "/var/log", }, - DB: db.DefaultConfig, - Indexer: blockindex.DefaultConfig, - Genesis: genesis.Default, + DB: db.DefaultConfig, + Indexer: blockindex.DefaultConfig, + Genesis: genesis.Default, + Collector: []string{}, } // ErrInvalidCfg indicates the invalid config value From ee087b0eb523985a88b9ed97122a7f7509a769cb Mon Sep 17 00:00:00 2001 From: millken Date: Tue, 13 Sep 2022 11:53:52 +0800 Subject: [PATCH 03/18] Revert "fix test" This reverts commit dc8b7257ca88423940fe19fe8e8f41001b25d7c8. --- config/config.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index 6f1ed6ba84..ff48ce3594 100644 --- a/config/config.go +++ b/config/config.go @@ -120,10 +120,9 @@ var ( StartSubChainInterval: 10 * time.Second, SystemLogDBPath: "/var/log", }, - DB: db.DefaultConfig, - Indexer: blockindex.DefaultConfig, - Genesis: genesis.Default, - Collector: []string{}, + DB: db.DefaultConfig, + Indexer: blockindex.DefaultConfig, + Genesis: genesis.Default, } // ErrInvalidCfg indicates the invalid config value From fc7ddd6f638f4adaf306b4557d3a44784ed96451 Mon Sep 17 00:00:00 2001 From: millken Date: Tue, 13 Sep 2022 11:54:24 +0800 Subject: [PATCH 04/18] Revert "add collector package to monitor node cpu/loadavg/memory usage" This reverts commit fd984d87a393f5c224be2fc8aa68cedc25eb16f1. --- config/config.go | 1 - go.mod | 1 - go.sum | 2 - pkg/collector/collector.go | 87 --------------------------------- pkg/collector/collector_test.go | 36 -------------- pkg/collector/cpu.go | 36 -------------- pkg/collector/cpu_test.go | 29 ----------- pkg/collector/loadavg.go | 38 -------------- pkg/collector/loadavg_test.go | 29 ----------- pkg/collector/memory.go | 35 ------------- pkg/collector/memory_test.go | 29 ----------- server/main.go | 18 ------- 12 files changed, 341 deletions(-) delete mode 100644 pkg/collector/collector.go delete mode 100644 pkg/collector/collector_test.go delete mode 100644 pkg/collector/cpu.go delete mode 100644 pkg/collector/cpu_test.go delete mode 100644 pkg/collector/loadavg.go delete mode 100644 pkg/collector/loadavg_test.go delete mode 100644 pkg/collector/memory.go delete mode 100644 pkg/collector/memory_test.go diff --git a/config/config.go b/config/config.go index ff48ce3594..d628bc2df1 100644 --- a/config/config.go +++ b/config/config.go @@ -241,7 +241,6 @@ type ( Log log.GlobalConfig `yaml:"log"` SubLogs map[string]log.GlobalConfig `yaml:"subLogs"` Genesis genesis.Genesis `yaml:"genesis"` - Collector []string `yaml:"collector"` } // Validate is the interface of validating the config diff --git a/go.mod b/go.mod index 08675295a9..f249692541 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,6 @@ require ( require ( github.com/golang/protobuf v1.5.2 - github.com/mackerelio/go-osstat v0.2.3 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/shirou/gopsutil/v3 v3.22.2 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0 diff --git a/go.sum b/go.sum index 283ff31471..dc8724f64d 100644 --- a/go.sum +++ b/go.sum @@ -840,8 +840,6 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/mackerelio/go-osstat v0.2.3 h1:jAMXD5erlDE39kdX2CU7YwCGRcxIO33u/p8+Fhe5dJw= -github.com/mackerelio/go-osstat v0.2.3/go.mod h1:DQbPOnsss9JHIXgBStc/dnhhir3gbd3YH+Dbdi7ptMA= github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go deleted file mode 100644 index e98c4b0729..0000000000 --- a/pkg/collector/collector.go +++ /dev/null @@ -1,87 +0,0 @@ -package collector - -import ( - "fmt" - "sync" - - "github.com/prometheus/client_golang/prometheus" -) - -// Namespace defines the common namespace to be used by all metrics. -const namespace = "node" - -var ( - factories = make(map[string]func() (Collector, error)) - initiatedCollectorsMtx = sync.Mutex{} - initiatedCollectors = make(map[string]Collector) -) - -func registerCollector(collector string, factory func() (Collector, error)) { - factories[collector] = factory -} - -// NodeCollector implements the prometheus.Collector interface. -type NodeCollector struct { - Collectors map[string]Collector -} - -// NewNodeCollector creates a new NodeCollector. -func NewNodeCollector(filters ...string) (*NodeCollector, error) { - f := make(map[string]bool) - for _, filter := range filters { - _, exist := factories[filter] - if !exist { - return nil, fmt.Errorf("missing collector: %s", filter) - } - f[filter] = true - } - collectors := make(map[string]Collector) - initiatedCollectorsMtx.Lock() - defer initiatedCollectorsMtx.Unlock() - for key := range f { - if collector, ok := initiatedCollectors[key]; ok { - collectors[key] = collector - } else { - collector, err := factories[key]() - if err != nil { - return nil, err - } - collectors[key] = collector - initiatedCollectors[key] = collector - } - } - return &NodeCollector{Collectors: collectors}, nil -} - -// Describe implements the prometheus.Collector interface. -func (n NodeCollector) Describe(ch chan<- *prometheus.Desc) { - -} - -// Collect implements the prometheus.Collector interface. -func (n NodeCollector) Collect(ch chan<- prometheus.Metric) { - wg := sync.WaitGroup{} - wg.Add(len(n.Collectors)) - for name, c := range n.Collectors { - go func(name string, c Collector) { - c.Update(ch) - wg.Done() - }(name, c) - } - wg.Wait() -} - -// Collector is the interface a collector has to implement. -type Collector interface { - // Get new metrics and expose them via prometheus registry. - Update(ch chan<- prometheus.Metric) error -} - -type typedDesc struct { - desc *prometheus.Desc - valueType prometheus.ValueType -} - -func (d *typedDesc) mustNewConstMetric(value float64, labels ...string) prometheus.Metric { - return prometheus.MustNewConstMetric(d.desc, d.valueType, value, labels...) -} diff --git a/pkg/collector/collector_test.go b/pkg/collector/collector_test.go deleted file mode 100644 index 4f1cffefab..0000000000 --- a/pkg/collector/collector_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package collector - -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/stretchr/testify/require" -) - -func TestCollector(t *testing.T) { - require := require.New(t) - c, err := NewNodeCollector("test") - require.ErrorContains(err, "missing collector") - c, err = NewNodeCollector("cpu") - require.NoError(err) - ch := make(chan prometheus.Metric) - go func() { - c.Collect(ch) - close(ch) - }() - - ch2 := make(chan *prometheus.Desc) - go func() { - c.Describe(ch2) - close(ch2) - }() - n := 0 - for m := range ch { - n++ - pb := &dto.Metric{} - err := m.Write(pb) - require.NoError(err) - } - require.Equal(4, n) -} diff --git a/pkg/collector/cpu.go b/pkg/collector/cpu.go deleted file mode 100644 index 63e73ba751..0000000000 --- a/pkg/collector/cpu.go +++ /dev/null @@ -1,36 +0,0 @@ -package collector - -import ( - "fmt" - - "github.com/mackerelio/go-osstat/cpu" - "github.com/prometheus/client_golang/prometheus" -) - -type cpuCollector struct { - metric *prometheus.Desc -} - -func init() { - registerCollector("cpu", NewCPUCollector) -} - -// NewCPUCollector returns a new Collector exposing kernel/system statistics. -func NewCPUCollector() (Collector, error) { - return &cpuCollector{ - metric: prometheus.NewDesc(namespace+"_cpu_stats", "cpu statistics", []string{"mode"}, nil), - }, nil -} - -func (c *cpuCollector) Update(ch chan<- prometheus.Metric) error { - stats, err := cpu.Get() - if err != nil { - return fmt.Errorf("couldn't get cpu: %w", err) - } - ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(stats.User), "user") - ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(stats.System), "system") - ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(stats.Total), "total") - ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(stats.Idle), "idle") - - return err -} diff --git a/pkg/collector/cpu_test.go b/pkg/collector/cpu_test.go deleted file mode 100644 index 6d9a146a83..0000000000 --- a/pkg/collector/cpu_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package collector - -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/stretchr/testify/require" -) - -func TestCPU(t *testing.T) { - require := require.New(t) - c, err := NewCPUCollector() - require.NoError(err) - require.NotNil(c) - ch := make(chan prometheus.Metric) - go func() { - c.Update(ch) - close(ch) - }() - n := 0 - for m := range ch { - n++ - pb := &dto.Metric{} - err := m.Write(pb) - require.NoError(err) - } - require.Equal(4, n) -} diff --git a/pkg/collector/loadavg.go b/pkg/collector/loadavg.go deleted file mode 100644 index d5e356db18..0000000000 --- a/pkg/collector/loadavg.go +++ /dev/null @@ -1,38 +0,0 @@ -package collector - -import ( - "fmt" - - "github.com/mackerelio/go-osstat/loadavg" - "github.com/prometheus/client_golang/prometheus" -) - -type loadavgCollector struct { - metric []typedDesc -} - -func init() { - registerCollector("loadavg", NewLoadavgCollector) -} - -// NewLoadavgCollector returns a new Collector exposing load average stats. -func NewLoadavgCollector() (Collector, error) { - return &loadavgCollector{ - metric: []typedDesc{ - {prometheus.NewDesc(namespace+"_load1", "1m load average.", nil, nil), prometheus.GaugeValue}, - {prometheus.NewDesc(namespace+"_load5", "5m load average.", nil, nil), prometheus.GaugeValue}, - {prometheus.NewDesc(namespace+"_load15", "15m load average.", nil, nil), prometheus.GaugeValue}, - }, - }, nil -} - -func (c *loadavgCollector) Update(ch chan<- prometheus.Metric) error { - loads, err := loadavg.Get() - if err != nil { - return fmt.Errorf("couldn't get load: %w", err) - } - ch <- c.metric[0].mustNewConstMetric(loads.Loadavg1) - ch <- c.metric[1].mustNewConstMetric(loads.Loadavg5) - ch <- c.metric[2].mustNewConstMetric(loads.Loadavg15) - return err -} diff --git a/pkg/collector/loadavg_test.go b/pkg/collector/loadavg_test.go deleted file mode 100644 index c79e011def..0000000000 --- a/pkg/collector/loadavg_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package collector - -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/stretchr/testify/require" -) - -func TestLoadavg(t *testing.T) { - require := require.New(t) - c, err := NewLoadavgCollector() - require.NoError(err) - require.NotNil(c) - ch := make(chan prometheus.Metric) - go func() { - c.Update(ch) - close(ch) - }() - n := 0 - for m := range ch { - n++ - pb := &dto.Metric{} - err := m.Write(pb) - require.NoError(err) - } - require.Equal(3, n) -} diff --git a/pkg/collector/memory.go b/pkg/collector/memory.go deleted file mode 100644 index 14ef7b1aa3..0000000000 --- a/pkg/collector/memory.go +++ /dev/null @@ -1,35 +0,0 @@ -package collector - -import ( - "fmt" - - "github.com/mackerelio/go-osstat/memory" - "github.com/prometheus/client_golang/prometheus" -) - -type meminfoCollector struct { - metric *prometheus.Desc -} - -func init() { - registerCollector("memory", NewMeminfoCollector) -} - -// NewMeminfoCollector returns a new Collector exposing memory stats. -func NewMeminfoCollector() (Collector, error) { - return &meminfoCollector{ - metric: prometheus.NewDesc(namespace+"_memory_stats", "memory stats", []string{"mode"}, nil), - }, nil -} - -// Update to get the platform specific memory stats. -func (c *meminfoCollector) Update(ch chan<- prometheus.Metric) error { - memInfo, err := memory.Get() - if err != nil { - return fmt.Errorf("couldn't get meminfo: %w", err) - } - ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(memInfo.Total), "total") - ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(memInfo.Free), "free") - ch <- prometheus.MustNewConstMetric(c.metric, prometheus.GaugeValue, float64(memInfo.Used), "used") - return nil -} diff --git a/pkg/collector/memory_test.go b/pkg/collector/memory_test.go deleted file mode 100644 index d2536efc8f..0000000000 --- a/pkg/collector/memory_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package collector - -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/stretchr/testify/require" -) - -func TestMemory(t *testing.T) { - require := require.New(t) - c, err := NewMeminfoCollector() - require.NoError(err) - require.NotNil(c) - ch := make(chan prometheus.Metric) - go func() { - c.Update(ch) - close(ch) - }() - n := 0 - for m := range ch { - n++ - pb := &dto.Metric{} - err := m.Write(pb) - require.NoError(err) - } - require.Equal(3, n) -} diff --git a/server/main.go b/server/main.go index 6536b4680e..2bb316173c 100644 --- a/server/main.go +++ b/server/main.go @@ -22,14 +22,12 @@ import ( "syscall" "github.com/iotexproject/go-pkgs/hash" - "github.com/prometheus/client_golang/prometheus" _ "go.uber.org/automaxprocs" "go.uber.org/zap" "github.com/iotexproject/iotex-core/blockchain/block" "github.com/iotexproject/iotex-core/blockchain/genesis" "github.com/iotexproject/iotex-core/config" - "github.com/iotexproject/iotex-core/pkg/collector" "github.com/iotexproject/iotex-core/pkg/log" "github.com/iotexproject/iotex-core/pkg/probe" "github.com/iotexproject/iotex-core/pkg/recovery" @@ -105,10 +103,6 @@ func main() { glog.Fatalln("Cannot config global logger, use default one: ", zap.Error(err)) } - if err = initCollector(cfg); err != nil { - glog.Fatalln("Cannot init collector: ", zap.Error(err)) - } - if err = recovery.SetCrashlogDir(cfg.System.SystemLogDBPath); err != nil { glog.Fatalln("Failed to set directory of crashlog: ", zap.Error(err)) } @@ -133,7 +127,6 @@ func main() { if err := probeSvr.Start(ctx); err != nil { log.L().Fatal("Failed to start probe server.", zap.Error(err)) } - go func() { <-stop // start stopping @@ -180,14 +173,3 @@ func initLogger(cfg config.Config) error { zap.String("ioAddr", addr.String()), )) } - -func initCollector(cfg config.Config) error { - nc, err := collector.NewNodeCollector(cfg.Collector...) - if err != nil { - return err - } - if err := prometheus.Register(nc); err != nil { - return err - } - return nil -} From 7cd23c478788d54ab25f8c5a6e05283662abcb80 Mon Sep 17 00:00:00 2001 From: millken Date: Tue, 13 Sep 2022 16:34:39 +0800 Subject: [PATCH 05/18] add event for node --- db/trie/mptrie/branchnode.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db/trie/mptrie/branchnode.go b/db/trie/mptrie/branchnode.go index 4a1a1497f7..7098c68707 100644 --- a/db/trie/mptrie/branchnode.go +++ b/db/trie/mptrie/branchnode.go @@ -42,6 +42,7 @@ func newBranchNode( return bnode.store() } } + eventOnUpsert(bnode) return bnode, nil } @@ -88,6 +89,7 @@ func (b *branchNode) Children() []node { } func (b *branchNode) Delete(key keyType, offset uint8) (node, error) { + eventOnDelete(b) offsetKey := key[offset] child, err := b.child(offsetKey) if err != nil { @@ -141,6 +143,7 @@ func (b *branchNode) Delete(key keyType, offset uint8) (node, error) { } func (b *branchNode) Upsert(key keyType, offset uint8, value []byte) (node, error) { + eventOnUpsert(b) var newChild node offsetKey := key[offset] child, err := b.child(offsetKey) @@ -158,6 +161,7 @@ func (b *branchNode) Upsert(key keyType, offset uint8, value []byte) (node, erro } func (b *branchNode) Search(key keyType, offset uint8) (node, error) { + eventOnSearch(b) child, err := b.child(key[offset]) if err != nil { return nil, err From 0acee52d1a3a3e5580d44ef3c621b9cf8a8f2664 Mon Sep 17 00:00:00 2001 From: millken Date: Tue, 13 Sep 2022 16:34:57 +0800 Subject: [PATCH 06/18] add event for node --- db/trie/mptrie/eventmap.go | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 db/trie/mptrie/eventmap.go diff --git a/db/trie/mptrie/eventmap.go b/db/trie/mptrie/eventmap.go new file mode 100644 index 0000000000..b8afaa3222 --- /dev/null +++ b/db/trie/mptrie/eventmap.go @@ -0,0 +1,42 @@ +package mptrie + +import ( + "sync" + "time" +) + +var eventMap sync.Map + +type eventOp int + +const ( + eventUpsert eventOp = iota + eventSearch + eventDelete +) + +type event struct { + op eventOp + time time.Time +} + +func eventOnUpsert(n node) { + eventMap.Store(n, event{ + op: eventUpsert, + time: time.Now(), + }) +} + +func eventOnDelete(n node) { + eventMap.Store(n, event{ + op: eventDelete, + time: time.Now(), + }) +} + +func eventOnSearch(n node) { + eventMap.Store(n, event{ + op: eventDelete, + time: time.Now(), + }) +} From 549a8ecf28257053d0be414dbbde362815828590 Mon Sep 17 00:00:00 2001 From: millken Date: Tue, 13 Sep 2022 20:41:54 +0800 Subject: [PATCH 07/18] update event --- db/trie/mptrie/event.go | 78 ++++++++++++++++++++++++++++++++++++ db/trie/mptrie/event_test.go | 27 +++++++++++++ db/trie/mptrie/eventmap.go | 42 ------------------- 3 files changed, 105 insertions(+), 42 deletions(-) create mode 100644 db/trie/mptrie/event.go create mode 100644 db/trie/mptrie/event_test.go delete mode 100644 db/trie/mptrie/eventmap.go diff --git a/db/trie/mptrie/event.go b/db/trie/mptrie/event.go new file mode 100644 index 0000000000..2856b98318 --- /dev/null +++ b/db/trie/mptrie/event.go @@ -0,0 +1,78 @@ +package mptrie + +import ( + "sync" + "time" +) + +var eventMap sync.Map + +type eventOp int + +const ( + eventUpsert eventOp = iota + eventSearch + eventDelete +) + +type event struct { + op eventOp + time time.Time +} + +//Event is a struct that contains the node and the operation performed on it +type Event struct { + Node node + Op eventOp + LastVisit time.Time +} + +// Events is a slice of events +type Events []Event + +func (e Events) Len() int { + return len(e) +} + +func (e Events) Less(i, j int) bool { + return e[i].LastVisit.Before(e[j].LastVisit) +} + +func (e Events) Swap(i, j int) { + e[i], e[j] = e[j], e[i] +} + +func eventOnUpsert(n node) { + eventMap.Store(n, event{ + op: eventUpsert, + time: time.Now(), + }) +} + +func eventOnDelete(n node) { + eventMap.Store(n, event{ + op: eventDelete, + time: time.Now(), + }) +} + +func eventOnSearch(n node) { + eventMap.Store(n, event{ + op: eventDelete, + time: time.Now(), + }) +} + +// GetEvents returns a slice of events +func GetEvents() Events { + var evv Events + eventMap.Range(func(key, value interface{}) bool { + evv = append(evv, Event{ + Node: key.(node), + Op: value.(event).op, + LastVisit: value.(event).time, + }) + return true + }) + return evv +} diff --git a/db/trie/mptrie/event_test.go b/db/trie/mptrie/event_test.go new file mode 100644 index 0000000000..68ced3823d --- /dev/null +++ b/db/trie/mptrie/event_test.go @@ -0,0 +1,27 @@ +package mptrie + +import ( + "sort" + "testing" + "time" +) + +var fakeTime = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + +func TestEvents(t *testing.T) { + evv := Events{ + {Op: eventUpsert, LastVisit: fakeTime.Add(3 * time.Second)}, + {Op: eventSearch, LastVisit: fakeTime.Add(2 * time.Second)}, + {Op: eventDelete, LastVisit: fakeTime.Add(1 * time.Second)}, + } + sort.Sort(evv) + if evv[0].Op != eventDelete { + t.Errorf("expected eventDelete, got %v", evv[0].Op) + } + if evv[1].Op != eventSearch { + t.Errorf("expected eventSearch, got %v", evv[1].Op) + } + if evv[2].Op != eventUpsert { + t.Errorf("expected eventUpsert, got %v", evv[2].Op) + } +} diff --git a/db/trie/mptrie/eventmap.go b/db/trie/mptrie/eventmap.go deleted file mode 100644 index b8afaa3222..0000000000 --- a/db/trie/mptrie/eventmap.go +++ /dev/null @@ -1,42 +0,0 @@ -package mptrie - -import ( - "sync" - "time" -) - -var eventMap sync.Map - -type eventOp int - -const ( - eventUpsert eventOp = iota - eventSearch - eventDelete -) - -type event struct { - op eventOp - time time.Time -} - -func eventOnUpsert(n node) { - eventMap.Store(n, event{ - op: eventUpsert, - time: time.Now(), - }) -} - -func eventOnDelete(n node) { - eventMap.Store(n, event{ - op: eventDelete, - time: time.Now(), - }) -} - -func eventOnSearch(n node) { - eventMap.Store(n, event{ - op: eventDelete, - time: time.Now(), - }) -} From b54aaca70a9e07edb342b9e429d3085c0859c112 Mon Sep 17 00:00:00 2001 From: millken Date: Mon, 10 Oct 2022 09:58:04 +0800 Subject: [PATCH 08/18] merge master (#3) * check nil for response.EpochData (#3642) * remove unused db/sql (#3645) * [ioctl] Build Read unittest (#3635) * build Read unittest * fix code smells * remove redundant argument * fix error * remove redundant code * remove redundant space * delete useless code Co-authored-by: huof6890 <68298506@qq.com> * [pkg] rotate log file (#3638) * add lumberjack * add lograte conf in Dockerfile * del space line * [ioctl] Build did geturi command line into new ioctl (#3650) * [ioctl] build did geturi command line into new ioctl * add test data * move abi reader to init * build unittest to cover the modification * move global var _didABI to did.go * move abi reader to did * alter did string with did prefix * rename did string Co-authored-by: huof6890 <68298506@qq.com> * [ioctl] build did gethash command line into new ioctl (#3639) * [ioctl] build did gethash command line into new ioctl * remove redundant code * build unittest to cover the modification * remove redundant args * add command description * modify test value * remove func encodeGet * remove func encodeGet * build unittest to cover the modification * reset default gas limit * use fault address from identityset * remove redundant space * update command name * move abi reader to init * move abi reader to did * alter did string with did prefix * rename did string Co-authored-by: huofei <68298506@qq.com> * [factory] add error details (#3657) Co-authored-by: huofei <68298506@qq.com> Co-authored-by: Jeremy Chou Co-authored-by: CoderZhi --- Dockerfile | 10 +- db/sql/rds.go | 36 ---- db/sql/rds_test.go | 31 ---- db/sql/sqlite3.go | 23 --- db/sql/sqlite3_test.go | 45 ----- db/sql/storebase.go | 113 ------------ db/sql/storebase_tests.go | 233 ------------------------- db/sql/util.go | 39 ----- entrypoint.sh | 5 + go.mod | 5 +- go.sum | 6 - ioctl/cmd/node/nodedelegate.go | 3 + ioctl/cmd/node/nodeprobationlist.go | 3 + ioctl/newcmd/action/action.go | 1 - ioctl/newcmd/action/action_test.go | 68 +++++--- ioctl/newcmd/action/xrc20.go | 2 +- ioctl/newcmd/did/did.go | 13 ++ ioctl/newcmd/did/didgethash.go | 79 +++++++++ ioctl/newcmd/did/didgethash_test.go | 91 ++++++++++ ioctl/newcmd/did/didgeturi.go | 78 +++++++++ ioctl/newcmd/did/didgeturi_test.go | 95 ++++++++++ ioctl/newcmd/did/didregister.go | 9 +- ioctl/newcmd/node/node.go | 1 + ioctl/newcmd/node/nodedelegate.go | 3 + ioctl/newcmd/node/nodeprobationlist.go | 3 + ioctl/newcmd/root.go | 4 + logrotate.conf | 4 + state/factory/workingset.go | 7 +- 28 files changed, 435 insertions(+), 575 deletions(-) delete mode 100644 db/sql/rds.go delete mode 100644 db/sql/rds_test.go delete mode 100644 db/sql/sqlite3.go delete mode 100644 db/sql/sqlite3_test.go delete mode 100644 db/sql/storebase.go delete mode 100644 db/sql/storebase_tests.go delete mode 100644 db/sql/util.go create mode 100755 entrypoint.sh create mode 100644 ioctl/newcmd/did/didgethash.go create mode 100644 ioctl/newcmd/did/didgethash_test.go create mode 100644 ioctl/newcmd/did/didgeturi.go create mode 100644 ioctl/newcmd/did/didgeturi_test.go diff --git a/Dockerfile b/Dockerfile index a527584fb3..d03c08edf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,11 +22,15 @@ COPY --from=build /go/apps/iotex-core/bin/actioninjectorv2 /usr/local/bin/iotex- COPY --from=build /go/apps/iotex-core/bin/addrgen /usr/local/bin/iotex-addrgen COPY --from=build /go/apps/iotex-core/bin/ioctl /usr/local/bin/ioctl -CMD [ "iotex-server"] # logrotate log file daily -RUN apk add logrotate +RUN apk add --no-cache logrotate COPY logrotate.conf /etc/logrotate.d/iotex RUN mkdir -p /var/lib/ RUN touch /var/lib/logrotate.status -RUN logrotate /etc/logrotate.d/iotex +RUN echo -e "#!/bin/sh\n\n/usr/sbin/logrotate -f /etc/logrotate.d/iotex" > /etc/periodic/daily/logrotate + +COPY entrypoint.sh /usr/local/bin +RUN chmod +x /usr/local/bin/entrypoint.sh +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["iotex-server"] diff --git a/db/sql/rds.go b/db/sql/rds.go deleted file mode 100644 index d1ead20f41..0000000000 --- a/db/sql/rds.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2019 IoTeX Foundation -// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no -// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent -// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache -// License 2.0 that can be found in the LICENSE file. - -package sql - -import ( - "fmt" - - // we need mysql import because it's called in file, (but compile will complain because there is no display) - _ "github.com/go-sql-driver/mysql" -) - -// RDS is the cloud rds config -type RDS struct { - // AwsRDSEndpoint is the endpoint of aws rds - AwsRDSEndpoint string `yaml:"awsRDSEndpoint"` - // AwsRDSPort is the port of aws rds - AwsRDSPort uint64 `yaml:"awsRDSPort"` - // AwsRDSUser is the user to access aws rds - AwsRDSUser string `yaml:"awsRDSUser"` - // AwsPass is the pass to access aws rds - AwsPass string `yaml:"awsPass"` - // AwsDBName is the db name of aws rds - AwsDBName string `yaml:"awsDBName"` -} - -// NewAwsRDS instantiates an aws rds -func NewAwsRDS(cfg RDS) Store { - connectStr := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", - cfg.AwsRDSUser, cfg.AwsPass, cfg.AwsRDSEndpoint, cfg.AwsRDSPort, cfg.AwsDBName, - ) - return newStoreBase("mysql", connectStr) -} diff --git a/db/sql/rds_test.go b/db/sql/rds_test.go deleted file mode 100644 index 2d83c4954b..0000000000 --- a/db/sql/rds_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2019 IoTeX Foundation -// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no -// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent -// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache -// License 2.0 that can be found in the LICENSE file. - -package sql - -import ( - "testing" -) - -func TestRDSStorePutGet(t *testing.T) { - t.Skip("Skipping when RDS credentail not provided.") - testRDSStorePutGet := TestStorePutGet - - cfg := RDS{} - t.Run("RDS Store", func(t *testing.T) { - testRDSStorePutGet(NewAwsRDS(cfg), t) - }) -} - -func TestRDSStoreTransaction(t *testing.T) { - t.Skip("Skipping when RDS credentail not provided.") - testRDSStoreTransaction := TestStoreTransaction - - cfg := RDS{} - t.Run("RDS Store", func(t *testing.T) { - testRDSStoreTransaction(NewAwsRDS(cfg), t) - }) -} diff --git a/db/sql/sqlite3.go b/db/sql/sqlite3.go deleted file mode 100644 index 08cd2f4213..0000000000 --- a/db/sql/sqlite3.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2019 IoTeX Foundation -// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no -// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent -// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache -// License 2.0 that can be found in the LICENSE file. - -package sql - -import ( - // this is required for sqlite3 usage - _ "github.com/mattn/go-sqlite3" -) - -// CQLITE3 is the local sqlite3 config -type CQLITE3 struct { - // SQLite3File is the sqlite3 db file - SQLite3File string `yaml:"sqlite3File"` -} - -// NewSQLite3 instantiates an sqlite3 -func NewSQLite3(cfg CQLITE3) Store { - return newStoreBase("sqlite3", cfg.SQLite3File) -} diff --git a/db/sql/sqlite3_test.go b/db/sql/sqlite3_test.go deleted file mode 100644 index 78383ad525..0000000000 --- a/db/sql/sqlite3_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2019 IoTeX Foundation -// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no -// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent -// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache -// License 2.0 that can be found in the LICENSE file. - -package sql - -import ( - "testing" - - "github.com/iotexproject/iotex-core/testutil" - - "github.com/stretchr/testify/require" -) - -const ( - _path = "explorer.db" -) - -func TestSQLite3StorePutGet(t *testing.T) { - testRDSStorePutGet := TestStorePutGet - testPath, err := testutil.PathOfTempFile(_path) - require.NoError(t, err) - defer testutil.CleanupPath(testPath) - cfg := CQLITE3{ - SQLite3File: testPath, - } - t.Run("SQLite3 Store", func(t *testing.T) { - testRDSStorePutGet(NewSQLite3(cfg), t) - }) -} - -func TestSQLite3StoreTransaction(t *testing.T) { - testSQLite3StoreTransaction := TestStoreTransaction - testPath, err := testutil.PathOfTempFile(_path) - require.NoError(t, err) - defer testutil.CleanupPath(testPath) - cfg := CQLITE3{ - SQLite3File: testPath, - } - t.Run("SQLite3 Store", func(t *testing.T) { - testSQLite3StoreTransaction(NewSQLite3(cfg), t) - }) -} diff --git a/db/sql/storebase.go b/db/sql/storebase.go deleted file mode 100644 index aa76da72bb..0000000000 --- a/db/sql/storebase.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2019 IoTeX Foundation -// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no -// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent -// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache -// License 2.0 that can be found in the LICENSE file. - -package sql - -import ( - "context" - "database/sql" - "os" - "sync" - - "github.com/rs/zerolog" - - // this is required for sqlite3 usage - "github.com/iotexproject/iotex-core/pkg/lifecycle" -) - -// Store is the interface of KV store. -type Store interface { - lifecycle.StartStopper - - // Get DB instance - GetDB() *sql.DB - - // Transact wrap the transaction - Transact(txFunc func(*sql.Tx) error) (err error) -} - -// storebase is local sqlite3 -type storeBase struct { - mutex sync.RWMutex - db *sql.DB - connectStr string - driverName string -} - -// logger is initialized with default settings -var logger = zerolog.New(os.Stderr).Level(zerolog.InfoLevel).With().Timestamp().Logger() - -// NewStoreBase instantiates an store base -func newStoreBase(driverName string, connectStr string) Store { - return &storeBase{db: nil, connectStr: connectStr, driverName: driverName} -} - -// Start opens the SQL (creates new file if not existing yet) -func (s *storeBase) Start(_ context.Context) error { - s.mutex.Lock() - defer s.mutex.Unlock() - - if s.db != nil { - return nil - } - - // Use db to perform SQL operations on database - db, err := sql.Open(s.driverName, s.connectStr) - if err != nil { - return err - } - s.db = db - return nil -} - -// Stop closes the SQL -func (s *storeBase) Stop(_ context.Context) error { - s.mutex.Lock() - defer s.mutex.Unlock() - - if s.db != nil { - err := s.db.Close() - s.db = nil - return err - } - return nil -} - -// Stop closes the SQL -func (s *storeBase) GetDB() *sql.DB { - s.mutex.RLock() - defer s.mutex.RUnlock() - - return s.db -} - -// Transact wrap the transaction -func (s *storeBase) Transact(txFunc func(*sql.Tx) error) (err error) { - tx, err := s.db.Begin() - if err != nil { - return err - } - defer func() { - switch { - case recover() != nil: - if rollbackErr := tx.Rollback(); rollbackErr != nil { - logger.Error().Err(rollbackErr) // log err after Rollback - } - case err != nil: - // err is non-nil; don't change it - if rollbackErr := tx.Rollback(); rollbackErr != nil { - logger.Error().Err(rollbackErr) - } - default: - // err is nil; if Commit returns error update err - if commitErr := tx.Commit(); commitErr != nil { - logger.Error().Err(commitErr) - } - } - }() - err = txFunc(tx) - return err -} diff --git a/db/sql/storebase_tests.go b/db/sql/storebase_tests.go deleted file mode 100644 index c05ce6dde3..0000000000 --- a/db/sql/storebase_tests.go +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright (c) 2019 IoTeX Foundation -// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no -// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent -// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache -// License 2.0 that can be found in the LICENSE file. - -package sql - -import ( - "context" - "database/sql" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/require" - - "github.com/iotexproject/go-pkgs/hash" -) - -// ActionHistory define the schema for action history -type ActionHistory struct { - NodeAddress string - UserAddress string - ActionHash string -} - -// TestStorePutGet define the common test cases for put and get -func TestStorePutGet(sqlStore Store, t *testing.T) { - require := require.New(t) - ctx := context.Background() - - require.NoError(sqlStore.Start(ctx)) - defer func() { - require.NoError(sqlStore.Stop(ctx)) - }() - - dbinstance := sqlStore.GetDB() - - nodeAddress := "aaa" - userAddress := "bbb" - actionHash := hash.ZeroHash256 - - // create table - _, err := dbinstance.Exec("CREATE TABLE IF NOT EXISTS action_history ([node_address] TEXT NOT NULL, [user_address] " + - "TEXT NOT NULL, [action_hash] BLOB(32) NOT NULL)") - require.NoError(err) - - // insert - stmt, err := dbinstance.Prepare("INSERT INTO action_history (node_address,user_address,action_hash) VALUES (?, ?, ?)") - require.NoError(err) - - res, err := stmt.Exec(nodeAddress, userAddress, actionHash[:]) - require.NoError(err) - - affect, err := res.RowsAffected() - require.NoError(err) - require.Equal(int64(1), affect) - - // get - stmt, err = dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=?") - require.NoError(err) - - rows, err := stmt.Query(nodeAddress) - require.NoError(err) - - var actionHistory ActionHistory - parsedRows, err := ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(1, len(parsedRows)) - require.Equal(nodeAddress, parsedRows[0].(*ActionHistory).NodeAddress) - require.Equal(userAddress, parsedRows[0].(*ActionHistory).UserAddress) - require.Equal(string(actionHash[:]), parsedRows[0].(*ActionHistory).ActionHash) - - // delete - stmt, err = dbinstance.Prepare("DELETE FROM action_history WHERE node_address=? AND user_address=? AND action_hash=?") - require.NoError(err) - - res, err = stmt.Exec(nodeAddress, userAddress, actionHash[:]) - require.NoError(err) - - affect, err = res.RowsAffected() - require.NoError(err) - require.Equal(int64(1), affect) - - // get - stmt, err = dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=?") - require.NoError(err) - - rows, err = stmt.Query(nodeAddress) - require.NoError(err) - - parsedRows, err = ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(0, len(parsedRows)) -} - -// TestStoreTransaction define the common test cases for transaction -func TestStoreTransaction(sqlStore Store, t *testing.T) { - require := require.New(t) - ctx := context.Background() - - require.NoError(sqlStore.Start(ctx)) - defer func() { - require.NoError(sqlStore.Stop(ctx)) - }() - - dbinstance := sqlStore.GetDB() - - nodeAddress := "aaa" - userAddress1 := "bbb1" - userAddress2 := "bbb2" - actionHash := hash.ZeroHash256 - - // create table - _, err := dbinstance.Exec("CREATE TABLE IF NOT EXISTS action_history ([node_address] TEXT NOT NULL, [user_address] " + - "TEXT NOT NULL, [action_hash] BLOB(32) NOT NULL)") - require.NoError(err) - - // get - stmt, err := dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=? AND user_address=?") - require.NoError(err) - rows, err := stmt.Query(nodeAddress, userAddress1) - require.NoError(err) - var actionHistory ActionHistory - parsedRows, err := ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(0, len(parsedRows)) - - stmt, err = dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=? AND user_address=?") - require.NoError(err) - rows, err = stmt.Query(nodeAddress, userAddress2) - require.NoError(err) - parsedRows, err = ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(0, len(parsedRows)) - - // insert transaction with fail - err = sqlStore.Transact(func(tx *sql.Tx) error { - insertQuery := "INSERT INTO action_history (node_address,user_address,action_hash) VALUES (?, ?, ?)" - if _, err := tx.Exec(insertQuery, nodeAddress, userAddress1, actionHash[:]); err != nil { - return err - } - if _, err := tx.Exec(insertQuery, nodeAddress, userAddress1, actionHash[:]); err != nil { - return errors.New("create an error") - } - return errors.New("create an error") - }) - println(err) - require.Error(err) - - // get - stmt, err = dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=? AND user_address=?") - require.NoError(err) - rows, err = stmt.Query(nodeAddress, userAddress1) - require.NoError(err) - parsedRows, err = ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(0, len(parsedRows)) - - stmt, err = dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=? AND user_address=?") - require.NoError(err) - rows, err = stmt.Query(nodeAddress, userAddress2) - require.NoError(err) - parsedRows, err = ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(0, len(parsedRows)) - - // insert - err = sqlStore.Transact(func(tx *sql.Tx) error { - insertQuery := "INSERT INTO action_history (node_address,user_address,action_hash) VALUES (?, ?, ?)" - if _, err := tx.Exec(insertQuery, nodeAddress, userAddress1, actionHash[:]); err != nil { - return err - } - if _, err := tx.Exec(insertQuery, nodeAddress, userAddress2, actionHash[:]); err != nil { - return err - } - return nil - }) - require.NoError(err) - - // get - stmt, err = dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=? AND user_address=?") - require.NoError(err) - rows, err = stmt.Query(nodeAddress, userAddress1) - require.NoError(err) - parsedRows, err = ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(1, len(parsedRows)) - require.Equal(nodeAddress, parsedRows[0].(*ActionHistory).NodeAddress) - require.Equal(userAddress1, parsedRows[0].(*ActionHistory).UserAddress) - require.Equal(string(actionHash[:]), parsedRows[0].(*ActionHistory).ActionHash) - - stmt, err = dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=? AND user_address=?") - require.NoError(err) - rows, err = stmt.Query(nodeAddress, userAddress2) - require.NoError(err) - parsedRows, err = ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(1, len(parsedRows)) - require.Equal(nodeAddress, parsedRows[0].(*ActionHistory).NodeAddress) - require.Equal(userAddress2, parsedRows[0].(*ActionHistory).UserAddress) - require.Equal(string(actionHash[:]), parsedRows[0].(*ActionHistory).ActionHash) - - // delete - err = sqlStore.Transact(func(tx *sql.Tx) error { - deleteQuery := "DELETE FROM action_history WHERE node_address=? AND user_address=? AND action_hash=?" - if _, err := tx.Exec(deleteQuery, nodeAddress, userAddress1, actionHash[:]); err != nil { - return err - } - if _, err := tx.Exec(deleteQuery, nodeAddress, userAddress2, actionHash[:]); err != nil { - return err - } - return nil - }) - require.NoError(err) - - // get - stmt, err = dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=? AND user_address=?") - require.NoError(err) - rows, err = stmt.Query(nodeAddress, userAddress1) - require.NoError(err) - parsedRows, err = ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(0, len(parsedRows)) - - stmt, err = dbinstance.Prepare("SELECT * FROM action_history WHERE node_address=? AND user_address=?") - require.NoError(err) - rows, err = stmt.Query(nodeAddress, userAddress2) - require.NoError(err) - parsedRows, err = ParseSQLRows(rows, &actionHistory) - require.NoError(err) - require.Equal(0, len(parsedRows)) -} diff --git a/db/sql/util.go b/db/sql/util.go deleted file mode 100644 index 03ff6eb439..0000000000 --- a/db/sql/util.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2019 IoTeX Foundation -// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no -// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent -// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache -// License 2.0 that can be found in the LICENSE file. - -package sql - -import ( - "reflect" - - "database/sql" -) - -// ParseSQLRows will parse the row -func ParseSQLRows(rows *sql.Rows, schema interface{}) ([]interface{}, error) { - var parsedRows []interface{} - - // Fetch rows - for rows.Next() { - newSchema := reflect.New(reflect.ValueOf(schema).Elem().Type()).Interface() - - s := reflect.ValueOf(newSchema).Elem() - - var fields []interface{} - for i := 0; i < s.NumField(); i++ { - fields = append(fields, s.Field(i).Addr().Interface()) - } - - err := rows.Scan(fields...) - if err != nil { - return nil, err - } - - parsedRows = append(parsedRows, newSchema) - } - - return parsedRows, nil -} diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000000..e1626a775d --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +/usr/sbin/crond + +exec "$@" \ No newline at end of file diff --git a/go.mod b/go.mod index a6e6e418fb..4a6d0f71ba 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/ethereum/go-ethereum v1.10.21 github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a github.com/go-redis/redis/v8 v8.11.4 - github.com/go-sql-driver/mysql v1.4.1 github.com/golang/mock v1.4.4 github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.4 @@ -23,14 +22,13 @@ require ( github.com/iotexproject/iotex-election v0.3.5-0.20210611041425-20ddf674363d github.com/iotexproject/iotex-proto v0.5.10 github.com/libp2p/go-libp2p-core v0.8.5 - github.com/mattn/go-sqlite3 v1.14.8 + github.com/mattn/go-sqlite3 v1.14.8 // indirect github.com/miguelmota/go-ethereum-hdwallet v0.1.1 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/multiformats/go-multiaddr v0.3.3 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 github.com/rodaine/table v1.0.1 - github.com/rs/zerolog v1.18.0 github.com/schollz/progressbar/v2 v2.15.0 github.com/spf13/cobra v1.1.1 github.com/stretchr/testify v1.8.0 @@ -197,7 +195,6 @@ require ( golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - google.golang.org/appengine v1.6.5 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 062cf78a8c..395fc7b1a3 100644 --- a/go.sum +++ b/go.sum @@ -285,7 +285,6 @@ github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F4 github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -1132,9 +1131,6 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= -github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -1277,7 +1273,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.elastic.co/ecszap v1.0.0 h1:PdQkRUeraR3XHJ14T7JMa+ncU0XXrVrcEN/BoRa2nMI= go.elastic.co/ecszap v1.0.0/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1596,7 +1591,6 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/ioctl/cmd/node/nodedelegate.go b/ioctl/cmd/node/nodedelegate.go index c6ca3a3f03..e9119c5b18 100644 --- a/ioctl/cmd/node/nodedelegate.go +++ b/ioctl/cmd/node/nodedelegate.go @@ -137,6 +137,9 @@ func delegates() error { if err != nil { return output.NewError(0, "failed to get epoch meta", err) } + if response.EpochData == nil { + return output.NewError(0, "ROLLDPOS is not registered", nil) + } epochData := response.EpochData aliases := alias.GetAliasMap() message := delegatesMessage{ diff --git a/ioctl/cmd/node/nodeprobationlist.go b/ioctl/cmd/node/nodeprobationlist.go index 6f5ce0bc64..60847cce9d 100644 --- a/ioctl/cmd/node/nodeprobationlist.go +++ b/ioctl/cmd/node/nodeprobationlist.go @@ -79,6 +79,9 @@ func probationlist() error { if err != nil { return output.NewError(0, "failed to get epoch meta", err) } + if response.EpochData == nil { + return output.NewError(0, "ROLLDPOS is not registered", nil) + } probationlist, err := getProbationList(_epochNum, response.EpochData.Height) if err != nil { return output.NewError(0, "failed to get probation list", err) diff --git a/ioctl/newcmd/action/action.go b/ioctl/newcmd/action/action.go index 4a2085246d..00bd018792 100644 --- a/ioctl/newcmd/action/action.go +++ b/ioctl/newcmd/action/action.go @@ -438,7 +438,6 @@ func Execute(client ioctl.Client, // Read reads smart contract on IoTeX blockchain func Read(client ioctl.Client, - cmd *cobra.Command, contract address.Address, amount string, bytecode []byte, diff --git a/ioctl/newcmd/action/action_test.go b/ioctl/newcmd/action/action_test.go index 9e561337bf..8280d6f188 100644 --- a/ioctl/newcmd/action/action_test.go +++ b/ioctl/newcmd/action/action_test.go @@ -20,6 +20,7 @@ import ( "github.com/iotexproject/iotex-core/ioctl/config" "github.com/iotexproject/iotex-core/ioctl/util" + "github.com/iotexproject/iotex-core/test/identityset" "github.com/iotexproject/iotex-core/test/mock/mock_ioctlclient" ) @@ -28,8 +29,8 @@ func TestSigner(t *testing.T) { ctrl := gomock.NewController(t) client := mock_ioctlclient.NewMockClient(ctrl) client.EXPECT().SelectTranslation(gomock.Any()).Return("mockTranslationString", config.English).Times(2) - client.EXPECT().SetEndpointWithFlag(gomock.Any()).Do(func(_ func(*string, string, string, string)) {}) - client.EXPECT().SetInsecureWithFlag(gomock.Any()).Do(func(_ func(*bool, string, bool, string)) {}) + client.EXPECT().SetEndpointWithFlag(gomock.Any()) + client.EXPECT().SetInsecureWithFlag(gomock.Any()) t.Run("returns signer's address", func(t *testing.T) { client.EXPECT().AddressWithDefaultIfNotExist(gomock.Any()).Return("test", nil).AnyTimes() @@ -119,7 +120,6 @@ func TestSendRaw(t *testing.T) { t.Run("failed to invoke SendAction api", func(t *testing.T) { expectedErr := errors.New("failed to invoke SendAction api") - apiServiceClient.EXPECT().SendAction(gomock.Any(), gomock.Any()).Return(nil, expectedErr) cmd := NewActionCmd(client) @@ -155,8 +155,8 @@ func TestSendAction(t *testing.T) { }} client.EXPECT().SelectTranslation(gomock.Any()).Return("action", config.English).Times(64) - client.EXPECT().SetEndpointWithFlag(gomock.Any()).Do(func(_ func(*string, string, string, string)) {}).AnyTimes() - client.EXPECT().SetInsecureWithFlag(gomock.Any()).Do(func(_ func(*bool, string, bool, string)) {}).AnyTimes() + client.EXPECT().SetEndpointWithFlag(gomock.Any()).AnyTimes() + client.EXPECT().SetInsecureWithFlag(gomock.Any()).AnyTimes() client.EXPECT().IsCryptoSm2().Return(false).Times(15) client.EXPECT().NewKeyStore().Return(ks).Times(15) @@ -165,9 +165,6 @@ func TestSendAction(t *testing.T) { client.EXPECT().ReadSecret().Return("", expectedErr).Times(1) cmd := NewActionCmd(client) - RegisterWriteCommand(client, cmd) - _, err := util.ExecuteCmd(cmd, "--password", "") - require.NoError(err) err = SendAction(client, cmd, elp, accAddr.String(), "", 0, false) require.Contains(err.Error(), expectedErr.Error()) }) @@ -189,9 +186,6 @@ func TestSendAction(t *testing.T) { t.Run("sends signed action to blockchain", func(t *testing.T) { cmd := NewActionCmd(client) - RegisterWriteCommand(client, cmd) - _, err := util.ExecuteCmd(cmd, "--password", passwd) - require.NoError(err) err = SendAction(client, cmd, elp, accAddr.String(), passwd, 1, false) require.NoError(err) }) @@ -201,9 +195,6 @@ func TestSendAction(t *testing.T) { client.EXPECT().HdwalletMnemonic(gomock.Any()).Return(mnemonic, nil) cmd := NewActionCmd(client) - RegisterWriteCommand(client, cmd) - _, err := util.ExecuteCmd(cmd, "--password", passwd) - require.NoError(err) err = SendAction(client, cmd, elp, "hdw::1/2", passwd, 1, false) require.NoError(err) }) @@ -212,9 +203,6 @@ func TestSendAction(t *testing.T) { client.EXPECT().AskToConfirm(gomock.Any()).Return(false, nil) cmd := NewActionCmd(client) - RegisterWriteCommand(client, cmd) - _, err := util.ExecuteCmd(cmd, "--password", passwd) - require.NoError(err) err = SendAction(client, cmd, elp, accAddr.String(), passwd, 1, false) require.NoError(err) }) @@ -224,9 +212,6 @@ func TestSendAction(t *testing.T) { client.EXPECT().AskToConfirm(gomock.Any()).Return(false, expectedErr) cmd := NewActionCmd(client) - RegisterWriteCommand(client, cmd) - _, err := util.ExecuteCmd(cmd, "--password", passwd) - require.NoError(err) err = SendAction(client, cmd, elp, accAddr.String(), passwd, 1, false) require.Contains(err.Error(), expectedErr.Error()) }) @@ -236,9 +221,6 @@ func TestSendAction(t *testing.T) { apiServiceClient.EXPECT().GetAccount(gomock.Any(), gomock.Any()).Return(nil, expectedErr) cmd := NewActionCmd(client) - RegisterWriteCommand(client, cmd) - _, err := util.ExecuteCmd(cmd, "--password", passwd) - require.NoError(err) err = SendAction(client, cmd, elp, accAddr.String(), passwd, 1, false) require.Contains(err.Error(), expectedErr.Error()) }) @@ -250,9 +232,6 @@ func TestSendAction(t *testing.T) { apiServiceClient.EXPECT().GetAccount(gomock.Any(), gomock.Any()).Return(nil, expectedErr) cmd := NewActionCmd(client) - RegisterWriteCommand(client, cmd) - _, err := util.ExecuteCmd(cmd, "--password", passwd) - require.NoError(err) err = SendAction(client, cmd, elp, "hdw::1/2", passwd, 1, false) require.Contains(err.Error(), expectedErr.Error()) }) @@ -263,10 +242,41 @@ func TestSendAction(t *testing.T) { apiServiceClient.EXPECT().GetAccount(gomock.Any(), gomock.Any()).Return(accountResponse, nil) cmd := NewActionCmd(client) - RegisterWriteCommand(client, cmd) - _, err := util.ExecuteCmd(cmd, "--password", passwd) - require.NoError(err) err = SendAction(client, cmd, elp, accAddr.String(), passwd, 1, false) require.Contains(err.Error(), expectedErr.Error()) }) } + +func TestRead(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + client := mock_ioctlclient.NewMockClient(ctrl) + apiServiceClient := mock_iotexapi.NewMockAPIServiceClient(ctrl) + accAddr := identityset.Address(0) + bytecode := "608060405234801561001057600080fd5b506040516040806108018339810180604052810190808051906020019092919080519060200190929190505050816004819055508060058190555050506107a58061005c6000396000f300608060405260043610610078576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680631249c58b1461007d57806327e235e31461009457806353277879146100eb5780636941b84414610142578063810ad50514610199578063a9059cbb14610223575b600080fd5b34801561008957600080fd5b50610092610270565b005b3480156100a057600080fd5b506100d5600480360381019080803573ffffffffffffffffffffffffffffffffffffffff169060200190929190505050610475565b6040518082815260200191505060405180910390f35b3480156100f757600080fd5b5061012c600480360381019080803573ffffffffffffffffffffffffffffffffffffffff16906020019092919050505061048d565b6040518082815260200191505060405180910390f35b34801561014e57600080fd5b50610183600480360381019080803573ffffffffffffffffffffffffffffffffffffffff1690602001909291905050506104a5565b6040518082815260200191505060405180910390f35b3480156101a557600080fd5b506101da600480360381019080803573ffffffffffffffffffffffffffffffffffffffff1690602001909291905050506104bd565b604051808373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020018281526020019250505060405180910390f35b34801561022f57600080fd5b5061026e600480360381019080803573ffffffffffffffffffffffffffffffffffffffff16906020019092919080359060200190929190505050610501565b005b436004546000803373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002054011115151561032a576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260108152602001807f746f6f20736f6f6e20746f206d696e740000000000000000000000000000000081525060200191505060405180910390fd5b436000803373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002081905550600554600160003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282540192505081905550600260003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600081548092919060010191905055503373ffffffffffffffffffffffffffffffffffffffff16600073ffffffffffffffffffffffffffffffffffffffff167fec61728879a33aa50b55e1f4789dcfc1c680f30a24d7b8694a9f874e242a97b46005546040518082815260200191505060405180910390a3565b60016020528060005260406000206000915090505481565b60026020528060005260406000206000915090505481565b60006020528060005260406000206000915090505481565b60036020528060005260406000206000915090508060000160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff16908060010154905082565b80600160003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002054101515156105b8576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260148152602001807f696e73756666696369656e742062616c616e636500000000000000000000000081525060200191505060405180910390fd5b80600160003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828254039250508190555080600160008473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828254019250508190555060408051908101604052803373ffffffffffffffffffffffffffffffffffffffff16815260200182815250600360008473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008201518160000160006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550602082015181600101559050508173ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fec61728879a33aa50b55e1f4789dcfc1c680f30a24d7b8694a9f874e242a97b4836040518082815260200191505060405180910390a350505600a165627a7a7230582047e5e1380e66d6b109548617ae59ff7baf70ee2d4a6734559b8fc5cabca0870b0029000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000186a0" + chainMetaResponse := &iotexapi.GetChainMetaResponse{ChainMeta: &iotextypes.ChainMeta{}} + + client.EXPECT().SelectTranslation(gomock.Any()).Return("action", config.English).Times(4) + client.EXPECT().SetEndpointWithFlag(gomock.Any()).Times(2) + client.EXPECT().SetInsecureWithFlag(gomock.Any()).Times(2) + client.EXPECT().APIServiceClient().Return(apiServiceClient, nil).Times(2) + + t.Run("reads smart contract on IoTeX blockchain", func(t *testing.T) { + client.EXPECT().AddressWithDefaultIfNotExist(gomock.Any()).Return("test", nil) + apiServiceClient.EXPECT().GetChainMeta(gomock.Any(), gomock.Any()).Return(chainMetaResponse, nil) + apiServiceClient.EXPECT().ReadContract(gomock.Any(), gomock.Any()).Return(&iotexapi.ReadContractResponse{ + Data: "test", + }, nil) + + result, err := Read(client, accAddr, "100", []byte(bytecode), "test", 100) + require.NoError(err) + require.Equal("test", result) + }) + + t.Run("failed to get signer address", func(t *testing.T) { + expectErr := errors.New("failed to get signer address") + client.EXPECT().AddressWithDefaultIfNotExist(gomock.Any()).Return("", expectErr) + _, err := Read(client, accAddr, "100", []byte(bytecode), "test", 100) + require.Contains(err.Error(), expectErr.Error()) + }) +} diff --git a/ioctl/newcmd/action/xrc20.go b/ioctl/newcmd/action/xrc20.go index 89f055d5d6..cf8c729d30 100644 --- a/ioctl/newcmd/action/xrc20.go +++ b/ioctl/newcmd/action/xrc20.go @@ -86,7 +86,7 @@ func parseAmount(client ioctl.Client, cmd *cobra.Command, contract address.Addre if err != nil { return nil, errors.Wrap(err, "failed to get flag gas-limit") } - result, err := Read(client, cmd, contract, "0", decimalBytecode, signer, gasLimit) + result, err := Read(client, contract, "0", decimalBytecode, signer, gasLimit) if err != nil { return nil, errors.New("failed to read contract") } diff --git a/ioctl/newcmd/did/did.go b/ioctl/newcmd/did/did.go index 9a3f6885e7..683845e131 100644 --- a/ioctl/newcmd/did/did.go +++ b/ioctl/newcmd/did/did.go @@ -7,6 +7,9 @@ package did import ( + "strings" + + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/spf13/cobra" "github.com/iotexproject/iotex-core/ioctl" @@ -19,6 +22,9 @@ var ( config.English: "Manage Decentralized Identity of IoTeX blockchain", config.Chinese: "管理IoTeX区块链上的去中心化数字身份", } + // _didABI is the interface of the abi encoding of did + _didABI abi.ABI + err error ) const ( @@ -31,6 +37,13 @@ const ( DIDABI = `[{"constant": false,"inputs": [],"name": "deregisterDID","outputs": [],"payable": false,"stateMutability": "nonpayable","type": "function"},{"constant": true,"inputs": [{"internalType": "bytes","name": "did","type": "bytes"}],"name": "getHash","outputs": [{"internalType": "bytes32","name": "","type": "bytes32"}],"payable": false,"stateMutability": "view","type": "function"}, {"constant": true,"inputs": [{"internalType": "bytes","name": "did","type": "bytes"}],"name": "getURI","outputs": [{"internalType": "bytes","name": "","type": "bytes"}],"payable": false,"stateMutability": "view","type": "function"},{"constant": false,"inputs": [{"internalType": "bytes32","name": "h","type": "bytes32"},{"internalType": "bytes","name": "uri","type": "bytes"}],"name": "registerDID","outputs": [],"payable": false,"stateMutability": "nonpayable","type": "function"},{"constant": false,"inputs": [{"internalType": "bytes32","name": "h","type": "bytes32"},{"internalType": "bytes","name": "uri","type": "bytes"}],"name": "updateDID","outputs": [],"payable": false,"stateMutability": "nonpayable","type": "function"}]` ) +func init() { + _didABI, err = abi.JSON(strings.NewReader(DIDABI)) + if err != nil { + panic(err) + } +} + // NewDidCmd represents the did command func NewDidCmd(client ioctl.Client) *cobra.Command { short, _ := client.SelectTranslation(_dIDCmdShorts) diff --git a/ioctl/newcmd/did/didgethash.go b/ioctl/newcmd/did/didgethash.go new file mode 100644 index 0000000000..808b9f39c6 --- /dev/null +++ b/ioctl/newcmd/did/didgethash.go @@ -0,0 +1,79 @@ +// Copyright (c) 2022 IoTeX Foundation +// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no +// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent +// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache +// License 2.0 that can be found in the LICENSE file. + +package did + +import ( + "encoding/hex" + + "github.com/iotexproject/iotex-address/address" + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/iotexproject/iotex-core/ioctl" + "github.com/iotexproject/iotex-core/ioctl/config" + "github.com/iotexproject/iotex-core/ioctl/newcmd/action" + "github.com/iotexproject/iotex-core/ioctl/util" +) + +// Multi-language support +var ( + _getHashCmdUses = map[config.Language]string{ + config.English: "gethash (CONTRACT_ADDRESS|ALIAS) DID", + config.Chinese: "gethash (合约地址|别名) DID", + } + _getHashCmdShorts = map[config.Language]string{ + config.English: "Gethash get DID doc's hash on IoTeX blockchain", + config.Chinese: "Gethash 在IoTeX链上获取相应DID的doc hash", + } +) + +// NewDidGetHash represents the did get hash command +func NewDidGetHash(client ioctl.Client) *cobra.Command { + use, _ := client.SelectTranslation(_getHashCmdUses) + short, _ := client.SelectTranslation(_getHashCmdShorts) + + cmd := &cobra.Command{ + Use: use, + Short: short, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + contract, err := client.Address(args[0]) + if err != nil { + return errors.Wrap(err, "failed to get contract address") + } + addr, err := address.FromString(contract) + if err != nil { + return errors.Wrap(err, "invalid contract address") + } + bytecode, err := _didABI.Pack(_getHashName, []byte(args[1])) + if err != nil { + return errors.Wrap(err, "invalid bytecode") + } + + result, err := action.Read(client, addr, "0", bytecode, contract, 20000000) + if err != nil { + return errors.Wrap(err, "failed to read contract") + } + ret, err := hex.DecodeString(result) + if err != nil { + return errors.Wrap(err, "failed to decode contract") + } + res, err := _didABI.Unpack(_getHashName, ret) + if err != nil { + return errors.New("DID does not exist") + } + out, err := util.To32Bytes(res[0]) + if err != nil { + return errors.Wrap(err, "failed to convert hash to bytes") + } + cmd.Println(hex.EncodeToString(out[:])) + return nil + }, + } + return cmd +} diff --git a/ioctl/newcmd/did/didgethash_test.go b/ioctl/newcmd/did/didgethash_test.go new file mode 100644 index 0000000000..35163c92dc --- /dev/null +++ b/ioctl/newcmd/did/didgethash_test.go @@ -0,0 +1,91 @@ +// Copyright (c) 2022 IoTeX Foundation +// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no +// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent +// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache +// License 2.0 that can be found in the LICENSE file. + +package did + +import ( + "encoding/hex" + "testing" + + "github.com/golang/mock/gomock" + "github.com/iotexproject/iotex-proto/golang/iotexapi" + "github.com/iotexproject/iotex-proto/golang/iotexapi/mock_iotexapi" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + + "github.com/iotexproject/iotex-core/ioctl/config" + "github.com/iotexproject/iotex-core/ioctl/util" + "github.com/iotexproject/iotex-core/test/identityset" + "github.com/iotexproject/iotex-core/test/mock/mock_ioctlclient" +) + +func TestNewDidGetHashCmd(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + client := mock_ioctlclient.NewMockClient(ctrl) + apiServiceClient := mock_iotexapi.NewMockAPIServiceClient(ctrl) + accAddr := identityset.Address(0).String() + did := "did:io:0x11111111111111111" + + client.EXPECT().SelectTranslation(gomock.Any()).Return("did", config.English).Times(12) + client.EXPECT().Address(gomock.Any()).Return(accAddr, nil).Times(4) + client.EXPECT().AddressWithDefaultIfNotExist(gomock.Any()).Return(accAddr, nil).Times(4) + client.EXPECT().APIServiceClient().Return(apiServiceClient, nil).Times(4) + + t.Run("get did hash", func(t *testing.T) { + apiServiceClient.EXPECT().ReadContract(gomock.Any(), gomock.Any()).Return(&iotexapi.ReadContractResponse{ + Data: hex.EncodeToString([]byte("60fe47b100000000000000000000000000000000000000000000000000000000")), + }, nil) + cmd := NewDidGetHash(client) + _, err := util.ExecuteCmd(cmd, accAddr, did) + require.NoError(err) + }) + + t.Run("failed to decode contract", func(t *testing.T) { + expectedErr := errors.New("failed to decode contract") + apiServiceClient.EXPECT().ReadContract(gomock.Any(), gomock.Any()).Return(&iotexapi.ReadContractResponse{ + Data: "test", + }, nil) + cmd := NewDidGetHash(client) + _, err := util.ExecuteCmd(cmd, "test", did) + require.Contains(err.Error(), expectedErr.Error()) + }) + + t.Run("DID does not exist", func(t *testing.T) { + expectedErr := errors.New("DID does not exist") + apiServiceClient.EXPECT().ReadContract(gomock.Any(), gomock.Any()).Return(&iotexapi.ReadContractResponse{ + Data: hex.EncodeToString([]byte("test")), + }, nil) + cmd := NewDidGetHash(client) + _, err := util.ExecuteCmd(cmd, accAddr, did) + require.Contains(err.Error(), expectedErr.Error()) + }) + + t.Run("failed to read contract", func(t *testing.T) { + expectedErr := errors.New("failed to read contract") + apiServiceClient.EXPECT().ReadContract(gomock.Any(), gomock.Any()).Return(nil, expectedErr) + cmd := NewDidGetHash(client) + _, err := util.ExecuteCmd(cmd, accAddr, did) + require.Contains(err.Error(), expectedErr.Error()) + }) + + t.Run("invalid contract address", func(t *testing.T) { + expectedErr := errors.New("invalid contract address") + client.EXPECT().Address(gomock.Any()).Return("test", nil) + cmd := NewDidGetHash(client) + _, err := util.ExecuteCmd(cmd, "test", did) + require.Contains(err.Error(), expectedErr.Error()) + }) + + t.Run("failed to get contract address", func(t *testing.T) { + expectedErr := errors.New("failed to get contract address") + client.EXPECT().Address(gomock.Any()).Return("", expectedErr) + cmd := NewDidGetHash(client) + _, err := util.ExecuteCmd(cmd, "test", did) + require.Contains(err.Error(), expectedErr.Error()) + }) +} diff --git a/ioctl/newcmd/did/didgeturi.go b/ioctl/newcmd/did/didgeturi.go new file mode 100644 index 0000000000..73f5b3424b --- /dev/null +++ b/ioctl/newcmd/did/didgeturi.go @@ -0,0 +1,78 @@ +// Copyright (c) 2022 IoTeX Foundation +// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no +// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent +// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache +// License 2.0 that can be found in the LICENSE file. + +package did + +import ( + "encoding/hex" + + "github.com/iotexproject/iotex-address/address" + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/iotexproject/iotex-core/ioctl" + "github.com/iotexproject/iotex-core/ioctl/config" + "github.com/iotexproject/iotex-core/ioctl/newcmd/action" + "github.com/iotexproject/iotex-core/ioctl/util" +) + +// Multi-language support +var ( + _getURICmdUses = map[config.Language]string{ + config.English: "geturi (CONTRACT_ADDRESS|ALIAS) DID", + config.Chinese: "geturi (合约地址|别名) DID", + } + _getURICmdShorts = map[config.Language]string{ + config.English: "Geturi get DID URI on IoTeX blockchain", + config.Chinese: "Geturi 在IoTeX链上获取相应DID的uri", + } +) + +// NewDidGetURICmd represents the did get uri command +func NewDidGetURICmd(client ioctl.Client) *cobra.Command { + use, _ := client.SelectTranslation(_getURICmdUses) + short, _ := client.SelectTranslation(_getURICmdShorts) + + cmd := &cobra.Command{ + Use: use, + Short: short, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + contract, err := client.Address(args[0]) + if err != nil { + return errors.Wrap(err, "failed to get contract address") + } + addr, err := address.FromString(contract) + if err != nil { + return errors.Wrap(err, "invalid contract address") + } + bytecode, err := _didABI.Pack(_getURIName, []byte(args[1])) + if err != nil { + return errors.Wrap(err, "invalid bytecode") + } + result, err := action.Read(client, addr, "0", bytecode, contract, 20000000) + if err != nil { + return errors.Wrap(err, "failed to read contract") + } + ret, err := hex.DecodeString(result) + if err != nil { + return errors.Wrap(err, "failed to decode contract") + } + res, err := _didABI.Unpack(_getURIName, ret) + if err != nil { + return errors.Wrap(err, "DID does not exist") + } + out, err := util.ToByteSlice(res[0]) + if err != nil { + return errors.Wrap(err, "failed to convert hash to bytes") + } + cmd.Println(hex.EncodeToString(out[:])) + return nil + }, + } + return cmd +} diff --git a/ioctl/newcmd/did/didgeturi_test.go b/ioctl/newcmd/did/didgeturi_test.go new file mode 100644 index 0000000000..1847efe1df --- /dev/null +++ b/ioctl/newcmd/did/didgeturi_test.go @@ -0,0 +1,95 @@ +// Copyright (c) 2022 IoTeX Foundation +// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no +// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent +// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache +// License 2.0 that can be found in the LICENSE file. + +package did + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/iotexproject/iotex-proto/golang/iotexapi" + "github.com/iotexproject/iotex-proto/golang/iotexapi/mock_iotexapi" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + + "github.com/iotexproject/iotex-core/ioctl/config" + "github.com/iotexproject/iotex-core/ioctl/util" + "github.com/iotexproject/iotex-core/test/identityset" + "github.com/iotexproject/iotex-core/test/mock/mock_ioctlclient" +) + +func TestNewDidGetURICmd(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + client := mock_ioctlclient.NewMockClient(ctrl) + apiServiceClient := mock_iotexapi.NewMockAPIServiceClient(ctrl) + accAddr := identityset.Address(0).String() + payload := "0000000000000000000000000000000000000000000000000000000000000020" + + "0000000000000000000000000000000000000000000000000000000000000020" + + "0000000000000000000000000000000000000000000000000000000000000001" + + "0000000000000000000000000000000000000000000000000000000000000002" + did := "did:io:0x11111111111111111" + + client.EXPECT().SelectTranslation(gomock.Any()).Return("did", config.English).Times(12) + client.EXPECT().Address(gomock.Any()).Return(accAddr, nil).Times(4) + client.EXPECT().AddressWithDefaultIfNotExist(gomock.Any()).Return(accAddr, nil).Times(4) + client.EXPECT().APIServiceClient().Return(apiServiceClient, nil).Times(4) + + t.Run("get did uri", func(t *testing.T) { + apiServiceClient.EXPECT().ReadContract(gomock.Any(), gomock.Any()).Return(&iotexapi.ReadContractResponse{ + Data: payload, + }, nil) + cmd := NewDidGetURICmd(client) + result, err := util.ExecuteCmd(cmd, accAddr, "did:io:0x11111111111111111") + require.NoError(err) + require.Contains(result, "0000000000000000000000000000000000000000000000000000000000000001") + }) + + t.Run("failed to decode contract", func(t *testing.T) { + expectedErr := errors.New("failed to decode contract") + apiServiceClient.EXPECT().ReadContract(gomock.Any(), gomock.Any()).Return(&iotexapi.ReadContractResponse{ + Data: "test", + }, nil) + cmd := NewDidGetURICmd(client) + _, err := util.ExecuteCmd(cmd, "test", did) + require.Contains(err.Error(), expectedErr.Error()) + }) + + t.Run("DID does not exist", func(t *testing.T) { + expectedErr := errors.New("DID does not exist") + apiServiceClient.EXPECT().ReadContract(gomock.Any(), gomock.Any()).Return(&iotexapi.ReadContractResponse{ + Data: "0000000000000000000000000000000000000000000000000000000000000020", + }, nil) + cmd := NewDidGetURICmd(client) + _, err := util.ExecuteCmd(cmd, accAddr, did) + require.Contains(err.Error(), expectedErr.Error()) + }) + + t.Run("failed to read contract", func(t *testing.T) { + expectedErr := errors.New("failed to read contract") + apiServiceClient.EXPECT().ReadContract(gomock.Any(), gomock.Any()).Return(nil, expectedErr) + cmd := NewDidGetURICmd(client) + _, err := util.ExecuteCmd(cmd, accAddr, did) + require.Contains(err.Error(), expectedErr.Error()) + }) + + t.Run("invalid contract address", func(t *testing.T) { + expectedErr := errors.New("invalid contract address") + client.EXPECT().Address(gomock.Any()).Return("test", nil) + cmd := NewDidGetURICmd(client) + _, err := util.ExecuteCmd(cmd, "test", did) + require.Contains(err.Error(), expectedErr.Error()) + }) + + t.Run("failed to get contract address", func(t *testing.T) { + expectedErr := errors.New("failed to get contract address") + client.EXPECT().Address(gomock.Any()).Return("", expectedErr) + cmd := NewDidGetURICmd(client) + _, err := util.ExecuteCmd(cmd, "test", did) + require.Contains(err.Error(), expectedErr.Error()) + }) +} diff --git a/ioctl/newcmd/did/didregister.go b/ioctl/newcmd/did/didregister.go index ad4fd91e71..a8bbedd4c6 100644 --- a/ioctl/newcmd/did/didregister.go +++ b/ioctl/newcmd/did/didregister.go @@ -9,9 +9,7 @@ package did import ( "encoding/hex" "math/big" - "strings" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -47,18 +45,13 @@ func NewDidRegisterCmd(client ioctl.Client) *cobra.Command { if err != nil { return errors.Wrap(err, "failed to get contract address") } - hashSlice, err := hex.DecodeString(args[1]) if err != nil { return errors.Wrap(err, "failed to decode data") } var hashArray [32]byte copy(hashArray[:], hashSlice) - abi, err := abi.JSON(strings.NewReader(DIDABI)) - if err != nil { - return errors.Wrap(err, "falied to parse abi") - } - bytecode, err := abi.Pack(_registerDIDName, hashArray, []byte(args[2])) + bytecode, err := _didABI.Pack(_registerDIDName, hashArray, []byte(args[2])) if err != nil { return errors.Wrap(err, "invalid bytecode") } diff --git a/ioctl/newcmd/node/node.go b/ioctl/newcmd/node/node.go index 1a39ce01cb..484897e110 100644 --- a/ioctl/newcmd/node/node.go +++ b/ioctl/newcmd/node/node.go @@ -25,6 +25,7 @@ func NewNodeCmd(client ioctl.Client) *cobra.Command { } nc.AddCommand(NewNodeDelegateCmd(client)) nc.AddCommand(NewNodeRewardCmd(client)) + nc.AddCommand(NewNodeProbationlistCmd(client)) client.SetEndpointWithFlag(nc.PersistentFlags().StringVar) client.SetInsecureWithFlag(nc.PersistentFlags().BoolVar) diff --git a/ioctl/newcmd/node/nodedelegate.go b/ioctl/newcmd/node/nodedelegate.go index c10fc5a485..471e1fd2e6 100644 --- a/ioctl/newcmd/node/nodedelegate.go +++ b/ioctl/newcmd/node/nodedelegate.go @@ -182,6 +182,9 @@ func NewNodeDelegateCmd(client ioctl.Client) *cobra.Command { if err != nil { return errors.Wrap(err, "failed to get epoch meta") } + if response.EpochData == nil { + return errors.New("ROLLDPOS is not registered") + } epochData := response.EpochData aliases := client.AliasMap() message := delegatesMessage{ diff --git a/ioctl/newcmd/node/nodeprobationlist.go b/ioctl/newcmd/node/nodeprobationlist.go index d27e3804b0..3119a30a35 100644 --- a/ioctl/newcmd/node/nodeprobationlist.go +++ b/ioctl/newcmd/node/nodeprobationlist.go @@ -55,6 +55,9 @@ func NewNodeProbationlistCmd(client ioctl.Client) *cobra.Command { if err != nil { return errors.Wrap(err, "failed to get epoch meta") } + if response.EpochData == nil { + return errors.New("ROLLDPOS is not registered") + } probationlist, err := getProbationList(client, epochNum, response.EpochData.Height) if err != nil { return errors.Wrap(err, "failed to get probation list") diff --git a/ioctl/newcmd/root.go b/ioctl/newcmd/root.go index c60cf08a96..021080c2db 100644 --- a/ioctl/newcmd/root.go +++ b/ioctl/newcmd/root.go @@ -12,6 +12,8 @@ import ( "github.com/iotexproject/iotex-core/ioctl" "github.com/iotexproject/iotex-core/ioctl/config" "github.com/iotexproject/iotex-core/ioctl/newcmd/account" + "github.com/iotexproject/iotex-core/ioctl/newcmd/bc" + "github.com/iotexproject/iotex-core/ioctl/newcmd/node" ) // Multi-language support @@ -47,6 +49,8 @@ func NewIoctl(client ioctl.Client) *cobra.Command { rootCmd.AddCommand(config.ConfigCmd) rootCmd.AddCommand(account.NewAccountCmd(client)) + rootCmd.AddCommand(bc.NewBCCmd(client)) + rootCmd.AddCommand(node.NewNodeCmd(client)) return rootCmd } diff --git a/logrotate.conf b/logrotate.conf index 04bca29a6b..77af717932 100644 --- a/logrotate.conf +++ b/logrotate.conf @@ -4,4 +4,8 @@ nocompress missingok copytruncate + notifempty + dateext + dateformat -%Y-%m-%d + extension .log } diff --git a/state/factory/workingset.go b/state/factory/workingset.go index 3970669bab..048c84d532 100644 --- a/state/factory/workingset.go +++ b/state/factory/workingset.go @@ -530,10 +530,11 @@ func (ws *workingSet) ValidateBlock(ctx context.Context, blk *block.Block) error return err } if !blk.VerifyDeltaStateDigest(digest) { - return block.ErrDeltaStateMismatch + return errors.Wrapf(block.ErrDeltaStateMismatch, "digest in block '%x' vs digest in workingset '%x'", blk.DeltaStateDigest(), digest) } - if !blk.VerifyReceiptRoot(calculateReceiptRoot(ws.receipts)) { - return block.ErrReceiptRootMismatch + receiptRoot := calculateReceiptRoot(ws.receipts) + if !blk.VerifyReceiptRoot(receiptRoot) { + return errors.Wrapf(block.ErrReceiptRootMismatch, "receipt root in block '%x' vs receipt root in workingset '%x'", blk.ReceiptRoot(), receiptRoot) } return nil From af14413615242725e00256e3a20076309cb8431a Mon Sep 17 00:00:00 2001 From: millken Date: Tue, 11 Oct 2022 15:16:46 +0800 Subject: [PATCH 09/18] log mptrie to db --- config/config.go | 2 + db/sql/db.go | 35 +++++++++++++++ db/sql/log.go | 27 ++++++++++++ db/trie/mptrie/branchnode.go | 18 ++++++++ db/trie/mptrie/event.go | 78 --------------------------------- db/trie/mptrie/event_test.go | 27 ------------ db/trie/mptrie/extensionnode.go | 15 +++++++ db/trie/mptrie/leafnode.go | 15 +++++++ db/trie/mptrie/node.go | 28 ++++++++++++ go.mod | 1 + go.sum | 1 + server/main.go | 11 +++++ 12 files changed, 153 insertions(+), 105 deletions(-) create mode 100644 db/sql/db.go create mode 100644 db/sql/log.go delete mode 100644 db/trie/mptrie/event.go delete mode 100644 db/trie/mptrie/event_test.go diff --git a/config/config.go b/config/config.go index d628bc2df1..2c0d4d32e2 100644 --- a/config/config.go +++ b/config/config.go @@ -19,6 +19,7 @@ import ( "github.com/iotexproject/iotex-core/blockchain/genesis" "github.com/iotexproject/iotex-core/blockindex" "github.com/iotexproject/iotex-core/db" + "github.com/iotexproject/iotex-core/db/sql" "github.com/iotexproject/iotex-core/dispatcher" "github.com/iotexproject/iotex-core/p2p" "github.com/iotexproject/iotex-core/pkg/log" @@ -241,6 +242,7 @@ type ( Log log.GlobalConfig `yaml:"log"` SubLogs map[string]log.GlobalConfig `yaml:"subLogs"` Genesis genesis.Genesis `yaml:"genesis"` + Database sql.Database `yaml:"database"` } // Validate is the interface of validating the config diff --git a/db/sql/db.go b/db/sql/db.go new file mode 100644 index 0000000000..a40a724825 --- /dev/null +++ b/db/sql/db.go @@ -0,0 +1,35 @@ +package sql + +import ( + "database/sql" + "fmt" + + _ "github.com/lib/pq" +) + +var db *sql.DB +var dbOpened bool + +// Database is the configuration of database +type Database struct { + Host string `yaml:"host"` + Port string `yaml:"port"` + User string `yaml:"user"` + Password string `yaml:"password"` + Name string `yaml:"name"` +} + +// DSN returns the data source name +func (cfg *Database) DSN() string { + return fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=disable", cfg.Host, cfg.User, cfg.Password, cfg.Name, cfg.Port) +} + +// Open opens the database +func Open(cfg *Database) (*sql.DB, error) { + var err error + db, err = sql.Open("postgres", cfg.DSN()) + if err == nil { + dbOpened = true + } + return db, err +} diff --git a/db/sql/log.go b/db/sql/log.go new file mode 100644 index 0000000000..81c048b5e9 --- /dev/null +++ b/db/sql/log.go @@ -0,0 +1,27 @@ +package sql + +import ( + "database/sql" + "encoding/hex" + "sync" + "time" +) + +var _createMptrieTable sync.Once + +//InitTables init tables +func InitTables(db *sql.DB) error { + if _, err := db.Exec("CREATE TABLE IF NOT EXISTS log_mptrie(id serial8, node_type varchar(2), node_key varchar(64) NOT NULL DEFAULT '', node_path varchar(64) NOT NULL DEFAULT '', node_children varchar(64) NOT NULL DEFAULT '', last_visited timestamp, PRIMARY KEY (id), UNIQUE (node_type,node_key,node_path,node_children))"); err != nil { + return err + } + return nil +} + +// StoreMptrieNode store mptrie node to db +func StoreMptrieNode(nodeType byte, nodeKey []byte, nodePath []byte, nodeChildren []byte) error { + if !dbOpened { + return nil + } + _, err := db.Exec("INSERT INTO log_mptrie (node_type, node_key, node_path, node_children,last_visited) VALUES ($1, $2, $3, $4,$5) ON CONFLICT (node_type,node_key,node_path,node_children) DO UPDATE SET last_visited = excluded.last_visited", string(nodeType), hex.EncodeToString(nodeKey), hex.EncodeToString(nodePath), hex.EncodeToString(nodeChildren), time.Now()) + return err +} diff --git a/db/trie/mptrie/branchnode.go b/db/trie/mptrie/branchnode.go index 4460a25601..94c01f763a 100644 --- a/db/trie/mptrie/branchnode.go +++ b/db/trie/mptrie/branchnode.go @@ -47,6 +47,9 @@ func newBranchNode( } } } + if err := logNode(bnode); err != nil { + return nil, err + } return bnode, nil } @@ -70,6 +73,9 @@ func newRootBranchNode(cli client, children map[byte]node, indices *SortedList, } } } + if err := logNode(bnode); err != nil { + return nil, err + } return bnode, nil } @@ -86,6 +92,9 @@ func newBranchNodeFromProtoPb(pb *triepb.BranchPb, hashVal []byte) *branchNode { } bnode.indices = NewSortedList(bnode.children) bnode.cacheNode.serializable = bnode + if err := logNode(bnode); err != nil { + panic(err) + } return bnode } @@ -102,6 +111,9 @@ func (b *branchNode) Children() []node { } func (b *branchNode) Delete(cli client, key keyType, offset uint8) (node, error) { + if err := logNode(b); err != nil { + return nil, err + } offsetKey := key[offset] child, err := b.child(offsetKey) if err != nil { @@ -155,6 +167,9 @@ func (b *branchNode) Delete(cli client, key keyType, offset uint8) (node, error) } func (b *branchNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) { + if err := logNode(b); err != nil { + return nil, err + } var newChild node offsetKey := key[offset] child, err := b.child(offsetKey) @@ -172,6 +187,9 @@ func (b *branchNode) Upsert(cli client, key keyType, offset uint8, value []byte) } func (b *branchNode) Search(cli client, key keyType, offset uint8) (node, error) { + if err := logNode(b); err != nil { + return nil, err + } child, err := b.child(key[offset]) if err != nil { return nil, err diff --git a/db/trie/mptrie/event.go b/db/trie/mptrie/event.go deleted file mode 100644 index 2856b98318..0000000000 --- a/db/trie/mptrie/event.go +++ /dev/null @@ -1,78 +0,0 @@ -package mptrie - -import ( - "sync" - "time" -) - -var eventMap sync.Map - -type eventOp int - -const ( - eventUpsert eventOp = iota - eventSearch - eventDelete -) - -type event struct { - op eventOp - time time.Time -} - -//Event is a struct that contains the node and the operation performed on it -type Event struct { - Node node - Op eventOp - LastVisit time.Time -} - -// Events is a slice of events -type Events []Event - -func (e Events) Len() int { - return len(e) -} - -func (e Events) Less(i, j int) bool { - return e[i].LastVisit.Before(e[j].LastVisit) -} - -func (e Events) Swap(i, j int) { - e[i], e[j] = e[j], e[i] -} - -func eventOnUpsert(n node) { - eventMap.Store(n, event{ - op: eventUpsert, - time: time.Now(), - }) -} - -func eventOnDelete(n node) { - eventMap.Store(n, event{ - op: eventDelete, - time: time.Now(), - }) -} - -func eventOnSearch(n node) { - eventMap.Store(n, event{ - op: eventDelete, - time: time.Now(), - }) -} - -// GetEvents returns a slice of events -func GetEvents() Events { - var evv Events - eventMap.Range(func(key, value interface{}) bool { - evv = append(evv, Event{ - Node: key.(node), - Op: value.(event).op, - LastVisit: value.(event).time, - }) - return true - }) - return evv -} diff --git a/db/trie/mptrie/event_test.go b/db/trie/mptrie/event_test.go deleted file mode 100644 index 68ced3823d..0000000000 --- a/db/trie/mptrie/event_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package mptrie - -import ( - "sort" - "testing" - "time" -) - -var fakeTime = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) - -func TestEvents(t *testing.T) { - evv := Events{ - {Op: eventUpsert, LastVisit: fakeTime.Add(3 * time.Second)}, - {Op: eventSearch, LastVisit: fakeTime.Add(2 * time.Second)}, - {Op: eventDelete, LastVisit: fakeTime.Add(1 * time.Second)}, - } - sort.Sort(evv) - if evv[0].Op != eventDelete { - t.Errorf("expected eventDelete, got %v", evv[0].Op) - } - if evv[1].Op != eventSearch { - t.Errorf("expected eventSearch, got %v", evv[1].Op) - } - if evv[2].Op != eventUpsert { - t.Errorf("expected eventUpsert, got %v", evv[2].Op) - } -} diff --git a/db/trie/mptrie/extensionnode.go b/db/trie/mptrie/extensionnode.go index 9bf28d1eae..9295f8e622 100644 --- a/db/trie/mptrie/extensionnode.go +++ b/db/trie/mptrie/extensionnode.go @@ -39,6 +39,9 @@ func newExtensionNode( return nil, err } } + if err := logNode(e); err != nil { + return nil, err + } return e, nil } @@ -52,10 +55,16 @@ func newExtensionNodeFromProtoPb(pb *triepb.ExtendPb, hashVal []byte) *extension child: newHashNode(pb.Value), } e.cacheNode.serializable = e + if err := logNode(e); err != nil { + panic(err) + } return e } func (e *extensionNode) Delete(cli client, key keyType, offset uint8) (node, error) { + if err := logNode(e); err != nil { + return nil, err + } matched := e.commonPrefixLength(key[offset:]) if matched != uint8(len(e.path)) { return nil, trie.ErrNotExist @@ -86,6 +95,9 @@ func (e *extensionNode) Delete(cli client, key keyType, offset uint8) (node, err } func (e *extensionNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) { + if err := logNode(e); err != nil { + return nil, err + } matched := e.commonPrefixLength(key[offset:]) if matched == uint8(len(e.path)) { newChild, err := e.child.Upsert(cli, key, offset+matched, value) @@ -121,6 +133,9 @@ func (e *extensionNode) Upsert(cli client, key keyType, offset uint8, value []by } func (e *extensionNode) Search(cli client, key keyType, offset uint8) (node, error) { + if err := logNode(e); err != nil { + return nil, err + } matched := e.commonPrefixLength(key[offset:]) if matched != uint8(len(e.path)) { return nil, trie.ErrNotExist diff --git a/db/trie/mptrie/leafnode.go b/db/trie/mptrie/leafnode.go index 49647ff11f..a39342a38c 100644 --- a/db/trie/mptrie/leafnode.go +++ b/db/trie/mptrie/leafnode.go @@ -39,6 +39,9 @@ func newLeafNode( return nil, err } } + if err := logNode(l); err != nil { + return nil, err + } return l, nil } @@ -52,6 +55,9 @@ func newLeafNodeFromProtoPb(pb *triepb.LeafPb, hashVal []byte) *leafNode { value: pb.Value, } l.cacheNode.serializable = l + if err := logNode(l); err != nil { + panic(err) + } return l } @@ -64,6 +70,9 @@ func (l *leafNode) Value() []byte { } func (l *leafNode) Delete(cli client, key keyType, offset uint8) (node, error) { + if err := logNode(l); err != nil { + return nil, err + } if !bytes.Equal(l.key[offset:], key[offset:]) { return nil, trie.ErrNotExist } @@ -71,6 +80,9 @@ func (l *leafNode) Delete(cli client, key keyType, offset uint8) (node, error) { } func (l *leafNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) { + if err := logNode(l); err != nil { + return nil, err + } matched := commonPrefixLength(l.key[offset:], key[offset:]) if offset+matched == uint8(len(key)) { if err := l.delete(cli); err != nil { @@ -108,6 +120,9 @@ func (l *leafNode) Upsert(cli client, key keyType, offset uint8, value []byte) ( } func (l *leafNode) Search(_ client, key keyType, offset uint8) (node, error) { + if err := logNode(l); err != nil { + return nil, err + } if !bytes.Equal(l.key[offset:], key[offset:]) { return nil, trie.ErrNotExist } diff --git a/db/trie/mptrie/node.go b/db/trie/mptrie/node.go index b759982e25..c9e8219aaa 100644 --- a/db/trie/mptrie/node.go +++ b/db/trie/mptrie/node.go @@ -7,6 +7,7 @@ package mptrie import ( + "github.com/iotexproject/iotex-core/db/sql" "github.com/pkg/errors" "google.golang.org/protobuf/proto" ) @@ -72,3 +73,30 @@ func commonPrefixLength(key1, key2 []byte) uint8 { return match } + +func logNode(n node) error { + nodeType, nodeKey, nodePath, nodeChildren, err := parseNode(n) + if err != nil { + return err + } + return sql.StoreMptrieNode(nodeType, nodeKey, nodePath, nodeChildren) +} +func parseNode(n node) (nodeType byte, nodeKey []byte, nodePath []byte, nodeChildren []byte, err error) { + switch n := n.(type) { + case *hashNode: + nodeType = 'h' + nodeKey = n.hashVal + case *leafNode: + nodeType = 'l' + nodeKey = n.key + case *extensionNode: + nodeType = 'e' + nodePath = n.path + case *branchNode: + nodeType = 'b' + nodeChildren = n.indices.List() + default: + err = errors.Errorf("unknown node type %T", n) + } + return +} diff --git a/go.mod b/go.mod index 4a6d0f71ba..39869d7a9d 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( ) require ( + github.com/lib/pq v1.0.0 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/shirou/gopsutil/v3 v3.22.2 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0 diff --git a/go.sum b/go.sum index 395fc7b1a3..2b842c543e 100644 --- a/go.sum +++ b/go.sum @@ -621,6 +621,7 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU= diff --git a/server/main.go b/server/main.go index 2bb316173c..8e4dea3397 100644 --- a/server/main.go +++ b/server/main.go @@ -28,6 +28,7 @@ import ( "github.com/iotexproject/iotex-core/blockchain/block" "github.com/iotexproject/iotex-core/blockchain/genesis" "github.com/iotexproject/iotex-core/config" + "github.com/iotexproject/iotex-core/db/sql" "github.com/iotexproject/iotex-core/pkg/log" "github.com/iotexproject/iotex-core/pkg/probe" "github.com/iotexproject/iotex-core/pkg/recovery" @@ -140,6 +141,16 @@ func main() { livenessCancel() }() + if &cfg.Database != nil { + db, err := sql.Open(&cfg.Database) + if err != nil { + log.L().Fatal("Failed to connect to database.", zap.Error(err)) + } + if err := sql.InitTables(db); err != nil { + log.L().Fatal("Failed to initialize tables.", zap.Error(err)) + } + + } // create and start the node svr, err := itx.NewServer(cfg) if err != nil { From 12fd4cfc6fec771206ca1a80bdea576342f6169c Mon Sep 17 00:00:00 2001 From: millken Date: Tue, 11 Oct 2022 22:16:34 +0800 Subject: [PATCH 10/18] fix ci test --- db/sql/db.go | 1 + 1 file changed, 1 insertion(+) diff --git a/db/sql/db.go b/db/sql/db.go index a40a724825..e7329b0f86 100644 --- a/db/sql/db.go +++ b/db/sql/db.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" + //postgres driver _ "github.com/lib/pq" ) From c9b5792fb8e9a911f817fb0c56d699495083441d Mon Sep 17 00:00:00 2001 From: millken Date: Mon, 31 Oct 2022 12:48:28 +0800 Subject: [PATCH 11/18] rewrite log node event, change to append-only Log --- config/config.go | 4 +- db/sql/db.go | 36 --------- db/sql/log.go | 27 ------- db/trie/mptrie/lognode.go | 142 +++++++++++++++++++++++++++++++++ db/trie/mptrie/lognode_test.go | 78 ++++++++++++++++++ db/trie/mptrie/node.go | 28 ------- go.mod | 1 - go.sum | 1 - server/main.go | 18 ++--- 9 files changed, 231 insertions(+), 104 deletions(-) delete mode 100644 db/sql/db.go delete mode 100644 db/sql/log.go create mode 100644 db/trie/mptrie/lognode.go create mode 100644 db/trie/mptrie/lognode_test.go diff --git a/config/config.go b/config/config.go index 2c0d4d32e2..db519efec1 100644 --- a/config/config.go +++ b/config/config.go @@ -19,7 +19,6 @@ import ( "github.com/iotexproject/iotex-core/blockchain/genesis" "github.com/iotexproject/iotex-core/blockindex" "github.com/iotexproject/iotex-core/db" - "github.com/iotexproject/iotex-core/db/sql" "github.com/iotexproject/iotex-core/dispatcher" "github.com/iotexproject/iotex-core/p2p" "github.com/iotexproject/iotex-core/pkg/log" @@ -242,7 +241,8 @@ type ( Log log.GlobalConfig `yaml:"log"` SubLogs map[string]log.GlobalConfig `yaml:"subLogs"` Genesis genesis.Genesis `yaml:"genesis"` - Database sql.Database `yaml:"database"` + // MptrieLogPath is the path to store mptrie logs + MptrieLogPath string `yaml:"mptrieLogPath"` } // Validate is the interface of validating the config diff --git a/db/sql/db.go b/db/sql/db.go deleted file mode 100644 index e7329b0f86..0000000000 --- a/db/sql/db.go +++ /dev/null @@ -1,36 +0,0 @@ -package sql - -import ( - "database/sql" - "fmt" - - //postgres driver - _ "github.com/lib/pq" -) - -var db *sql.DB -var dbOpened bool - -// Database is the configuration of database -type Database struct { - Host string `yaml:"host"` - Port string `yaml:"port"` - User string `yaml:"user"` - Password string `yaml:"password"` - Name string `yaml:"name"` -} - -// DSN returns the data source name -func (cfg *Database) DSN() string { - return fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=disable", cfg.Host, cfg.User, cfg.Password, cfg.Name, cfg.Port) -} - -// Open opens the database -func Open(cfg *Database) (*sql.DB, error) { - var err error - db, err = sql.Open("postgres", cfg.DSN()) - if err == nil { - dbOpened = true - } - return db, err -} diff --git a/db/sql/log.go b/db/sql/log.go deleted file mode 100644 index 81c048b5e9..0000000000 --- a/db/sql/log.go +++ /dev/null @@ -1,27 +0,0 @@ -package sql - -import ( - "database/sql" - "encoding/hex" - "sync" - "time" -) - -var _createMptrieTable sync.Once - -//InitTables init tables -func InitTables(db *sql.DB) error { - if _, err := db.Exec("CREATE TABLE IF NOT EXISTS log_mptrie(id serial8, node_type varchar(2), node_key varchar(64) NOT NULL DEFAULT '', node_path varchar(64) NOT NULL DEFAULT '', node_children varchar(64) NOT NULL DEFAULT '', last_visited timestamp, PRIMARY KEY (id), UNIQUE (node_type,node_key,node_path,node_children))"); err != nil { - return err - } - return nil -} - -// StoreMptrieNode store mptrie node to db -func StoreMptrieNode(nodeType byte, nodeKey []byte, nodePath []byte, nodeChildren []byte) error { - if !dbOpened { - return nil - } - _, err := db.Exec("INSERT INTO log_mptrie (node_type, node_key, node_path, node_children,last_visited) VALUES ($1, $2, $3, $4,$5) ON CONFLICT (node_type,node_key,node_path,node_children) DO UPDATE SET last_visited = excluded.last_visited", string(nodeType), hex.EncodeToString(nodeKey), hex.EncodeToString(nodePath), hex.EncodeToString(nodeChildren), time.Now()) - return err -} diff --git a/db/trie/mptrie/lognode.go b/db/trie/mptrie/lognode.go new file mode 100644 index 0000000000..80634fcd96 --- /dev/null +++ b/db/trie/mptrie/lognode.go @@ -0,0 +1,142 @@ +package mptrie + +import ( + "bufio" + "os" + + "github.com/pkg/errors" +) + +var ( + enabledLogMptrie = false + logFile *os.File + logWriter *bufio.Writer +) + +// NodeEvent is the event of node +type NodeEvent struct { + Type byte + KeyLen uint8 + Key []byte + PathLen uint8 + Path []byte + ChildrenLen uint8 + Children []byte +} + +// Bytes returns the bytes of node event +func (e NodeEvent) Bytes() []byte { + b := make([]byte, 0, 1+1+e.KeyLen+1+e.PathLen+1+e.ChildrenLen) + b = append(b, e.Type) + b = append(b, e.KeyLen) + b = append(b, e.Key...) + b = append(b, e.PathLen) + b = append(b, e.Path...) + b = append(b, e.ChildrenLen) + b = append(b, e.Children...) + return b +} + +// ParseNodeEvent parse the node event +func ParseNodeEvent(b []byte) (NodeEvent, error) { + if len(b) < 1 { + return NodeEvent{}, errors.New("invalid node event") + } + event := NodeEvent{ + Type: b[0], + } + if len(b) < 2 { + return event, nil + } + event.KeyLen = b[1] + if len(b) < 2+int(event.KeyLen) { + return event, nil + } + event.Key = b[2 : 2+event.KeyLen] + if len(b) < 2+int(event.KeyLen)+1 { + return event, nil + } + event.PathLen = b[2+event.KeyLen] + if len(b) < 2+int(event.KeyLen)+1+int(event.PathLen) { + return event, nil + } + event.Path = b[2+event.KeyLen+1 : 2+event.KeyLen+1+event.PathLen] + if len(b) < 2+int(event.KeyLen)+1+int(event.PathLen)+1 { + return event, nil + } + event.ChildrenLen = b[2+event.KeyLen+1+event.PathLen] + if len(b) < 2+int(event.KeyLen)+1+int(event.PathLen)+1+int(event.ChildrenLen) { + return event, nil + } + event.Children = b[2+event.KeyLen+1+event.PathLen+1 : 2+event.KeyLen+1+event.PathLen+1+event.ChildrenLen] + return event, nil +} + +// OpenLogDB open the log DB file +func OpenLogDB(dbPath string) error { + var err error + logFile, err = os.OpenFile(dbPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return err + } + logWriter = bufio.NewWriter(logFile) + enabledLogMptrie = true + return nil +} + +// CloseLogDB close the log DB file +func CloseLogDB() error { + if !enabledLogMptrie { + return nil + } + if err := logWriter.Flush(); err != nil { + return err + } + return logFile.Close() +} + +func logNode(n node) error { + if !enabledLogMptrie { + return nil + } + nodeType, nodeKey, nodePath, nodeChildren, err := parseNode(n) + if err != nil { + return err + } + event := NodeEvent{ + Type: nodeType, + KeyLen: uint8(len(nodeKey)), + Key: nodeKey, + PathLen: uint8(len(nodePath)), + Path: nodePath, + ChildrenLen: uint8(len(nodeChildren)), + Children: nodeChildren, + } + // write events length + if err = logWriter.WriteByte(byte(len(event.Bytes()))); err != nil { + return err + } + // write events body + _, err = logWriter.Write(event.Bytes()) + return err +} + +func parseNode(n node) (nodeType byte, nodeKey []byte, nodePath []byte, nodeChildren []byte, err error) { + switch n := n.(type) { + case *hashNode: + nodeType = 'h' + nodeKey = n.hashVal + case *leafNode: + nodeType = 'l' + nodeKey = n.key + case *extensionNode: + nodeType = 'e' + nodePath = n.path + case *branchNode: + nodeType = 'b' + nodeChildren = n.indices.List() + default: + err = errors.Errorf("unknown node type %T", n) + } + return +} diff --git a/db/trie/mptrie/lognode_test.go b/db/trie/mptrie/lognode_test.go new file mode 100644 index 0000000000..b1b36779e2 --- /dev/null +++ b/db/trie/mptrie/lognode_test.go @@ -0,0 +1,78 @@ +package mptrie + +import ( + "os" + "testing" +) + +func TestNodeEvent(t *testing.T) { + tests := []struct { + event NodeEvent + }{ + { + event: NodeEvent{ + Type: 1, + KeyLen: 2, + Key: []byte{3, 4}, + PathLen: 5, + Path: []byte{6, 7, 8, 9, 10}, + ChildrenLen: 11, + Children: []byte{12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22}, + }, + }, + { + event: NodeEvent{ + Type: 6, + KeyLen: 6, + Key: []byte("123456"), + PathLen: 6, + Path: []byte("abcdef"), + ChildrenLen: 6, + Children: []byte("ABCDEF"), + }, + }, + } + + for _, test := range tests { + b := test.event.Bytes() + event, err := ParseNodeEvent(b) + if err != nil { + t.Errorf("unexpected error %v", err) + } + if event.Type != test.event.Type { + t.Errorf("unexpected type %v", event.Type) + } + if event.KeyLen != test.event.KeyLen { + t.Errorf("unexpected key length %v", event.KeyLen) + } + if event.Key != nil && string(event.Key) != string(test.event.Key) { + t.Errorf("unexpected key %v", event.Key) + } + if event.PathLen != test.event.PathLen { + t.Errorf("unexpected path length %v", event.PathLen) + } + if event.Path != nil && string(event.Path) != string(test.event.Path) { + t.Errorf("unexpected path %v", event.Path) + } + if event.ChildrenLen != test.event.ChildrenLen { + t.Errorf("unexpected children length %v", event.ChildrenLen) + } + if event.Children != nil && string(event.Children) != string(test.event.Children) { + t.Errorf("unexpected children %v", event.Children) + } + } +} + +func TestLogDB(t *testing.T) { + dbPath := "test_log_db" + defer func() { + _ = os.RemoveAll(dbPath) + }() + if err := OpenLogDB(dbPath); err != nil { + t.Errorf("unexpected open error %v", err) + } + if err := CloseLogDB(); err != nil { + t.Errorf("unexpected close error %v", err) + } + +} diff --git a/db/trie/mptrie/node.go b/db/trie/mptrie/node.go index c9e8219aaa..b759982e25 100644 --- a/db/trie/mptrie/node.go +++ b/db/trie/mptrie/node.go @@ -7,7 +7,6 @@ package mptrie import ( - "github.com/iotexproject/iotex-core/db/sql" "github.com/pkg/errors" "google.golang.org/protobuf/proto" ) @@ -73,30 +72,3 @@ func commonPrefixLength(key1, key2 []byte) uint8 { return match } - -func logNode(n node) error { - nodeType, nodeKey, nodePath, nodeChildren, err := parseNode(n) - if err != nil { - return err - } - return sql.StoreMptrieNode(nodeType, nodeKey, nodePath, nodeChildren) -} -func parseNode(n node) (nodeType byte, nodeKey []byte, nodePath []byte, nodeChildren []byte, err error) { - switch n := n.(type) { - case *hashNode: - nodeType = 'h' - nodeKey = n.hashVal - case *leafNode: - nodeType = 'l' - nodeKey = n.key - case *extensionNode: - nodeType = 'e' - nodePath = n.path - case *branchNode: - nodeType = 'b' - nodeChildren = n.indices.List() - default: - err = errors.Errorf("unknown node type %T", n) - } - return -} diff --git a/go.mod b/go.mod index 39869d7a9d..4a6d0f71ba 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,6 @@ require ( ) require ( - github.com/lib/pq v1.0.0 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/shirou/gopsutil/v3 v3.22.2 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.34.0 diff --git a/go.sum b/go.sum index 2b842c543e..395fc7b1a3 100644 --- a/go.sum +++ b/go.sum @@ -621,7 +621,6 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2 h1:7cWK5cdA5x72jX0g8iLrQWm5TRJZ6CzGdPEhWj7plWU= diff --git a/server/main.go b/server/main.go index 8e4dea3397..965a1aa48d 100644 --- a/server/main.go +++ b/server/main.go @@ -28,7 +28,7 @@ import ( "github.com/iotexproject/iotex-core/blockchain/block" "github.com/iotexproject/iotex-core/blockchain/genesis" "github.com/iotexproject/iotex-core/config" - "github.com/iotexproject/iotex-core/db/sql" + "github.com/iotexproject/iotex-core/db/trie/mptrie" "github.com/iotexproject/iotex-core/pkg/log" "github.com/iotexproject/iotex-core/pkg/probe" "github.com/iotexproject/iotex-core/pkg/recovery" @@ -141,15 +141,15 @@ func main() { livenessCancel() }() - if &cfg.Database != nil { - db, err := sql.Open(&cfg.Database) - if err != nil { - log.L().Fatal("Failed to connect to database.", zap.Error(err)) - } - if err := sql.InitTables(db); err != nil { - log.L().Fatal("Failed to initialize tables.", zap.Error(err)) + if cfg.MptrieLogPath != "" { + if err = mptrie.OpenLogDB(cfg.MptrieLogPath); err != nil { + log.L().Fatal("Failed to open mptrie log DB.", zap.Error(err)) } - + defer func() { + if err = mptrie.CloseLogDB(); err != nil { + log.L().Error("Failed to close mptrie log DB.", zap.Error(err)) + } + }() } // create and start the node svr, err := itx.NewServer(cfg) From 23622ed4587196db5dbbb0a71dc64f4250bb9003 Mon Sep 17 00:00:00 2001 From: millken Date: Mon, 31 Oct 2022 16:33:39 +0800 Subject: [PATCH 12/18] update test --- db/trie/mptrie/lognode_test.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/db/trie/mptrie/lognode_test.go b/db/trie/mptrie/lognode_test.go index b1b36779e2..ea35c44e92 100644 --- a/db/trie/mptrie/lognode_test.go +++ b/db/trie/mptrie/lognode_test.go @@ -1,8 +1,10 @@ package mptrie import ( - "os" "testing" + + "github.com/iotexproject/iotex-core/testutil" + "github.com/stretchr/testify/require" ) func TestNodeEvent(t *testing.T) { @@ -64,15 +66,12 @@ func TestNodeEvent(t *testing.T) { } func TestLogDB(t *testing.T) { - dbPath := "test_log_db" + require := require.New(t) + testPath, err := testutil.PathOfTempFile("test-log-db") + require.NoError(err) defer func() { - _ = os.RemoveAll(dbPath) + testutil.CleanupPath(testPath) }() - if err := OpenLogDB(dbPath); err != nil { - t.Errorf("unexpected open error %v", err) - } - if err := CloseLogDB(); err != nil { - t.Errorf("unexpected close error %v", err) - } - + require.NoError(OpenLogDB(testPath)) + require.NoError(CloseLogDB()) } From 8166a5d1a0f338996b7b8d1828a3c1e07a759e67 Mon Sep 17 00:00:00 2001 From: millken Date: Mon, 31 Oct 2022 16:34:57 +0800 Subject: [PATCH 13/18] update test --- db/trie/mptrie/lognode_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/db/trie/mptrie/lognode_test.go b/db/trie/mptrie/lognode_test.go index ea35c44e92..21ad46763f 100644 --- a/db/trie/mptrie/lognode_test.go +++ b/db/trie/mptrie/lognode_test.go @@ -74,4 +74,5 @@ func TestLogDB(t *testing.T) { }() require.NoError(OpenLogDB(testPath)) require.NoError(CloseLogDB()) + enabledLogMptrie = false } From a575201b850080f567b47b25c9179c3a2a91e363 Mon Sep 17 00:00:00 2001 From: millken Date: Thu, 1 Dec 2022 16:15:00 +0800 Subject: [PATCH 14/18] fix comments --- db/trie/mptrie/branchnode.go | 17 ++++--- db/trie/mptrie/extensionnode.go | 14 ++++-- db/trie/mptrie/leafnode.go | 16 ++++--- db/trie/mptrie/lognode.go | 84 +++++++++++++-------------------- db/trie/mptrie/lognode_test.go | 79 +++++++++++++++++++++++++++---- 5 files changed, 133 insertions(+), 77 deletions(-) diff --git a/db/trie/mptrie/branchnode.go b/db/trie/mptrie/branchnode.go index 94c01f763a..2deaf1d66f 100644 --- a/db/trie/mptrie/branchnode.go +++ b/db/trie/mptrie/branchnode.go @@ -47,7 +47,8 @@ func newBranchNode( } } } - if err := logNode(bnode); err != nil { + hashVal, _ := bnode.cacheNode.Hash(cli) + if err := logNode(_nodeTypeBranch, _actionTypeNew, hashVal, bnode); err != nil { return nil, err } return bnode, nil @@ -73,7 +74,8 @@ func newRootBranchNode(cli client, children map[byte]node, indices *SortedList, } } } - if err := logNode(bnode); err != nil { + hashVal, _ := bnode.cacheNode.Hash(cli) + if err := logNode(_nodeTypeBranch, _actionTypeNew, hashVal, bnode); err != nil { return nil, err } return bnode, nil @@ -92,7 +94,7 @@ func newBranchNodeFromProtoPb(pb *triepb.BranchPb, hashVal []byte) *branchNode { } bnode.indices = NewSortedList(bnode.children) bnode.cacheNode.serializable = bnode - if err := logNode(bnode); err != nil { + if err := logNode(_nodeTypeBranch, _actionTypeNew, hashVal, bnode); err != nil { panic(err) } return bnode @@ -111,7 +113,8 @@ func (b *branchNode) Children() []node { } func (b *branchNode) Delete(cli client, key keyType, offset uint8) (node, error) { - if err := logNode(b); err != nil { + hashVal, _ := b.cacheNode.Hash(cli) + if err := logNode(_nodeTypeBranch, _actionTypeDelete, hashVal, b); err != nil { return nil, err } offsetKey := key[offset] @@ -167,7 +170,8 @@ func (b *branchNode) Delete(cli client, key keyType, offset uint8) (node, error) } func (b *branchNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) { - if err := logNode(b); err != nil { + hashVal, _ := b.cacheNode.Hash(cli) + if err := logNode(_nodeTypeBranch, _actionTypeUpsert, hashVal, b); err != nil { return nil, err } var newChild node @@ -187,7 +191,8 @@ func (b *branchNode) Upsert(cli client, key keyType, offset uint8, value []byte) } func (b *branchNode) Search(cli client, key keyType, offset uint8) (node, error) { - if err := logNode(b); err != nil { + hashVal, _ := b.cacheNode.Hash(cli) + if err := logNode(_nodeTypeBranch, _actionTypeSearch, hashVal, b); err != nil { return nil, err } child, err := b.child(key[offset]) diff --git a/db/trie/mptrie/extensionnode.go b/db/trie/mptrie/extensionnode.go index 9295f8e622..4d16cff63b 100644 --- a/db/trie/mptrie/extensionnode.go +++ b/db/trie/mptrie/extensionnode.go @@ -39,7 +39,8 @@ func newExtensionNode( return nil, err } } - if err := logNode(e); err != nil { + hashVal, _ := e.cacheNode.Hash(cli) + if err := logNode(_nodeTypeExtension, _actionTypeNew, hashVal, e); err != nil { return nil, err } return e, nil @@ -55,14 +56,15 @@ func newExtensionNodeFromProtoPb(pb *triepb.ExtendPb, hashVal []byte) *extension child: newHashNode(pb.Value), } e.cacheNode.serializable = e - if err := logNode(e); err != nil { + if err := logNode(_nodeTypeExtension, _actionTypeNew, hashVal, e); err != nil { panic(err) } return e } func (e *extensionNode) Delete(cli client, key keyType, offset uint8) (node, error) { - if err := logNode(e); err != nil { + hashVal, _ := e.cacheNode.Hash(cli) + if err := logNode(_nodeTypeExtension, _actionTypeDelete, hashVal, e); err != nil { return nil, err } matched := e.commonPrefixLength(key[offset:]) @@ -95,7 +97,8 @@ func (e *extensionNode) Delete(cli client, key keyType, offset uint8) (node, err } func (e *extensionNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) { - if err := logNode(e); err != nil { + hashVal, _ := e.cacheNode.Hash(cli) + if err := logNode(_nodeTypeExtension, _actionTypeUpsert, hashVal, e); err != nil { return nil, err } matched := e.commonPrefixLength(key[offset:]) @@ -133,7 +136,8 @@ func (e *extensionNode) Upsert(cli client, key keyType, offset uint8, value []by } func (e *extensionNode) Search(cli client, key keyType, offset uint8) (node, error) { - if err := logNode(e); err != nil { + hashVal, _ := e.cacheNode.Hash(cli) + if err := logNode(_nodeTypeExtension, _actionTypeSearch, hashVal, e); err != nil { return nil, err } matched := e.commonPrefixLength(key[offset:]) diff --git a/db/trie/mptrie/leafnode.go b/db/trie/mptrie/leafnode.go index a39342a38c..15b1feffa5 100644 --- a/db/trie/mptrie/leafnode.go +++ b/db/trie/mptrie/leafnode.go @@ -39,7 +39,8 @@ func newLeafNode( return nil, err } } - if err := logNode(l); err != nil { + hashVal, _ := l.cacheNode.Hash(cli) + if err := logNode(_nodeTypeLeaf, _actionTypeNew, hashVal, l); err != nil { return nil, err } return l, nil @@ -55,7 +56,7 @@ func newLeafNodeFromProtoPb(pb *triepb.LeafPb, hashVal []byte) *leafNode { value: pb.Value, } l.cacheNode.serializable = l - if err := logNode(l); err != nil { + if err := logNode(_nodeTypeLeaf, _actionTypeNew, hashVal, l); err != nil { panic(err) } return l @@ -70,7 +71,8 @@ func (l *leafNode) Value() []byte { } func (l *leafNode) Delete(cli client, key keyType, offset uint8) (node, error) { - if err := logNode(l); err != nil { + hashVal, _ := l.cacheNode.Hash(cli) + if err := logNode(_nodeTypeLeaf, _actionTypeDelete, hashVal, l); err != nil { return nil, err } if !bytes.Equal(l.key[offset:], key[offset:]) { @@ -80,7 +82,8 @@ func (l *leafNode) Delete(cli client, key keyType, offset uint8) (node, error) { } func (l *leafNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) { - if err := logNode(l); err != nil { + hashVal, _ := l.cacheNode.Hash(cli) + if err := logNode(_nodeTypeLeaf, _actionTypeUpsert, hashVal, l); err != nil { return nil, err } matched := commonPrefixLength(l.key[offset:], key[offset:]) @@ -119,8 +122,9 @@ func (l *leafNode) Upsert(cli client, key keyType, offset uint8, value []byte) ( return newExtensionNode(cli, l.key[offset:offset+matched], bnode) } -func (l *leafNode) Search(_ client, key keyType, offset uint8) (node, error) { - if err := logNode(l); err != nil { +func (l *leafNode) Search(cli client, key keyType, offset uint8) (node, error) { + hashVal, _ := l.cacheNode.Hash(cli) + if err := logNode(_nodeTypeLeaf, _actionTypeSearch, hashVal, l); err != nil { return nil, err } if !bytes.Equal(l.key[offset:], key[offset:]) { diff --git a/db/trie/mptrie/lognode.go b/db/trie/mptrie/lognode.go index 80634fcd96..368dc8c7d9 100644 --- a/db/trie/mptrie/lognode.go +++ b/db/trie/mptrie/lognode.go @@ -13,65 +13,50 @@ var ( logWriter *bufio.Writer ) -// NodeEvent is the event of node -type NodeEvent struct { - Type byte +type nodeType byte +type actionType byte + +const ( + _nodeTypeLeaf nodeType = 'l' + _nodeTypeExtension nodeType = 'e' + _nodeTypeBranch nodeType = 'b' + + _actionTypeSearch actionType = 's' + _actionTypeUpsert actionType = 'u' + _actionTypeDelete actionType = 'd' + _actionTypeNew actionType = 'n' +) + +// nodeEvent is the event of node +type nodeEvent struct { + NodeType nodeType + ActionType actionType KeyLen uint8 Key []byte PathLen uint8 Path []byte ChildrenLen uint8 Children []byte + HashLen uint8 + HashVal []byte } // Bytes returns the bytes of node event -func (e NodeEvent) Bytes() []byte { - b := make([]byte, 0, 1+1+e.KeyLen+1+e.PathLen+1+e.ChildrenLen) - b = append(b, e.Type) +func (e nodeEvent) Bytes() []byte { + b := make([]byte, 0, 1+1+1+e.KeyLen+1+e.PathLen+1+e.ChildrenLen+1+e.HashLen) + b = append(b, byte(e.NodeType)) + b = append(b, byte(e.ActionType)) b = append(b, e.KeyLen) b = append(b, e.Key...) b = append(b, e.PathLen) b = append(b, e.Path...) b = append(b, e.ChildrenLen) b = append(b, e.Children...) + b = append(b, e.HashLen) + b = append(b, e.HashVal...) return b } -// ParseNodeEvent parse the node event -func ParseNodeEvent(b []byte) (NodeEvent, error) { - if len(b) < 1 { - return NodeEvent{}, errors.New("invalid node event") - } - event := NodeEvent{ - Type: b[0], - } - if len(b) < 2 { - return event, nil - } - event.KeyLen = b[1] - if len(b) < 2+int(event.KeyLen) { - return event, nil - } - event.Key = b[2 : 2+event.KeyLen] - if len(b) < 2+int(event.KeyLen)+1 { - return event, nil - } - event.PathLen = b[2+event.KeyLen] - if len(b) < 2+int(event.KeyLen)+1+int(event.PathLen) { - return event, nil - } - event.Path = b[2+event.KeyLen+1 : 2+event.KeyLen+1+event.PathLen] - if len(b) < 2+int(event.KeyLen)+1+int(event.PathLen)+1 { - return event, nil - } - event.ChildrenLen = b[2+event.KeyLen+1+event.PathLen] - if len(b) < 2+int(event.KeyLen)+1+int(event.PathLen)+1+int(event.ChildrenLen) { - return event, nil - } - event.Children = b[2+event.KeyLen+1+event.PathLen+1 : 2+event.KeyLen+1+event.PathLen+1+event.ChildrenLen] - return event, nil -} - // OpenLogDB open the log DB file func OpenLogDB(dbPath string) error { var err error @@ -95,16 +80,17 @@ func CloseLogDB() error { return logFile.Close() } -func logNode(n node) error { +func logNode(nt nodeType, at actionType, hashvalue []byte, n node) error { if !enabledLogMptrie { return nil } - nodeType, nodeKey, nodePath, nodeChildren, err := parseNode(n) + nodeKey, nodePath, nodeChildren, err := parseNode(n) if err != nil { return err } - event := NodeEvent{ - Type: nodeType, + event := nodeEvent{ + NodeType: nt, + ActionType: at, KeyLen: uint8(len(nodeKey)), Key: nodeKey, PathLen: uint8(len(nodePath)), @@ -121,19 +107,13 @@ func logNode(n node) error { return err } -func parseNode(n node) (nodeType byte, nodeKey []byte, nodePath []byte, nodeChildren []byte, err error) { +func parseNode(n node) (nodeKey []byte, nodePath []byte, nodeChildren []byte, err error) { switch n := n.(type) { - case *hashNode: - nodeType = 'h' - nodeKey = n.hashVal case *leafNode: - nodeType = 'l' nodeKey = n.key case *extensionNode: - nodeType = 'e' nodePath = n.path case *branchNode: - nodeType = 'b' nodeChildren = n.indices.List() default: err = errors.Errorf("unknown node type %T", n) diff --git a/db/trie/mptrie/lognode_test.go b/db/trie/mptrie/lognode_test.go index 21ad46763f..7ce9049ddb 100644 --- a/db/trie/mptrie/lognode_test.go +++ b/db/trie/mptrie/lognode_test.go @@ -4,45 +4,55 @@ import ( "testing" "github.com/iotexproject/iotex-core/testutil" + "github.com/pkg/errors" "github.com/stretchr/testify/require" ) func TestNodeEvent(t *testing.T) { tests := []struct { - event NodeEvent + event nodeEvent }{ { - event: NodeEvent{ - Type: 1, + event: nodeEvent{ + NodeType: _nodeTypeBranch, + ActionType: _actionTypeNew, KeyLen: 2, Key: []byte{3, 4}, PathLen: 5, Path: []byte{6, 7, 8, 9, 10}, ChildrenLen: 11, Children: []byte{12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22}, + HashLen: 23, + HashVal: []byte{24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46}, }, }, { - event: NodeEvent{ - Type: 6, + event: nodeEvent{ + NodeType: _nodeTypeLeaf, + ActionType: _actionTypeSearch, KeyLen: 6, Key: []byte("123456"), PathLen: 6, Path: []byte("abcdef"), ChildrenLen: 6, Children: []byte("ABCDEF"), + HashLen: 6, + HashVal: []byte("abcdef"), }, }, } for _, test := range tests { b := test.event.Bytes() - event, err := ParseNodeEvent(b) + event, err := parseNodeEvent(b) if err != nil { t.Errorf("unexpected error %v", err) } - if event.Type != test.event.Type { - t.Errorf("unexpected type %v", event.Type) + if event.NodeType != test.event.NodeType { + t.Errorf("unexpected NodeType %v", event.NodeType) + } + if event.ActionType != test.event.ActionType { + t.Errorf("unexpected ActionType %v", event.ActionType) } if event.KeyLen != test.event.KeyLen { t.Errorf("unexpected key length %v", event.KeyLen) @@ -62,7 +72,60 @@ func TestNodeEvent(t *testing.T) { if event.Children != nil && string(event.Children) != string(test.event.Children) { t.Errorf("unexpected children %v", event.Children) } + if event.HashLen != test.event.HashLen { + t.Errorf("unexpected hash length %v", event.HashLen) + } + if event.HashVal != nil && string(event.HashVal) != string(test.event.HashVal) { + t.Errorf("unexpected hash %v", event.HashVal) + } + } +} + +// parseNodeEvent parse the node event +func parseNodeEvent(b []byte) (nodeEvent, error) { + if len(b) < 1 { + return nodeEvent{}, errors.New("invalid node event") + } + event := nodeEvent{ + NodeType: nodeType(b[0]), + } + if len(b) < 2 { + return event, nil + } + event.ActionType = actionType(b[1]) + if len(b) < 3 { + return event, nil + } + event.KeyLen = uint8(b[2]) + if len(b) < 3+int(event.KeyLen) { + return event, nil + } + event.Key = b[3 : 3+event.KeyLen] + if len(b) < 3+int(event.KeyLen)+1 { + return event, nil + } + event.PathLen = b[3+event.KeyLen] + if len(b) < 3+int(event.KeyLen)+1+int(event.PathLen) { + return event, nil + } + event.Path = b[3+event.KeyLen+1 : 3+event.KeyLen+1+event.PathLen] + if len(b) < 3+int(event.KeyLen)+1+int(event.PathLen)+1 { + return event, nil + } + event.ChildrenLen = b[3+event.KeyLen+1+event.PathLen] + if len(b) < 3+int(event.KeyLen)+1+int(event.PathLen)+1+int(event.ChildrenLen) { + return event, nil + } + event.Children = b[3+event.KeyLen+1+event.PathLen+1 : 3+event.KeyLen+1+event.PathLen+1+event.ChildrenLen] + if len(b) < 3+int(event.KeyLen)+1+int(event.PathLen)+1+int(event.ChildrenLen)+1 { + return event, nil + } + event.HashLen = b[3+event.KeyLen+1+event.PathLen+1+event.ChildrenLen] + if len(b) < 3+int(event.KeyLen)+1+int(event.PathLen)+1+int(event.ChildrenLen)+1+int(event.HashLen) { + return event, nil } + event.HashVal = b[3+event.KeyLen+1+event.PathLen+1+event.ChildrenLen+1 : 3+event.KeyLen+1+event.PathLen+1+event.ChildrenLen+1+event.HashLen] + return event, nil } func TestLogDB(t *testing.T) { From 33ed1835fbd934c7ed4769462bafd4dfdb43149c Mon Sep 17 00:00:00 2001 From: millken Date: Thu, 1 Dec 2022 16:17:05 +0800 Subject: [PATCH 15/18] fix comments --- db/trie/mptrie/lognode.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/trie/mptrie/lognode.go b/db/trie/mptrie/lognode.go index 368dc8c7d9..6124ae276d 100644 --- a/db/trie/mptrie/lognode.go +++ b/db/trie/mptrie/lognode.go @@ -97,6 +97,8 @@ func logNode(nt nodeType, at actionType, hashvalue []byte, n node) error { Path: nodePath, ChildrenLen: uint8(len(nodeChildren)), Children: nodeChildren, + HashLen: uint8(len(hashvalue)), + HashVal: hashvalue, } // write events length if err = logWriter.WriteByte(byte(len(event.Bytes()))); err != nil { From 99efe1a270caa8c58abfd93c9f4c9264e623030f Mon Sep 17 00:00:00 2001 From: millken Date: Thu, 1 Dec 2022 17:45:34 +0800 Subject: [PATCH 16/18] fix tests --- db/trie/mptrie/branchnode_test.go | 9 +++++---- db/trie/mptrie/cachenode.go | 5 +++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/db/trie/mptrie/branchnode_test.go b/db/trie/mptrie/branchnode_test.go index 33ec6de58c..bd632f424e 100644 --- a/db/trie/mptrie/branchnode_test.go +++ b/db/trie/mptrie/branchnode_test.go @@ -49,10 +49,11 @@ func equals(bn *branchNode, clone *branchNode) bool { func TestBranchNodeClone(t *testing.T) { require := require.New(t) + cli := &merklePatriciaTrie{async: true, hashFunc: DefaultHashFunc} t.Run("dirty empty root", func(t *testing.T) { children := map[byte]node{} indices := NewSortedList(children) - node, err := newRootBranchNode(nil, children, indices, true) + node, err := newRootBranchNode(cli, children, indices, true) require.NoError(err) bn, ok := node.(*branchNode) require.True(ok) @@ -65,7 +66,7 @@ func TestBranchNodeClone(t *testing.T) { t.Run("clean empty root", func(t *testing.T) { children := map[byte]node{} indices := NewSortedList(children) - node, err := newRootBranchNode(nil, children, indices, false) + node, err := newRootBranchNode(cli, children, indices, false) require.NoError(err) bn, ok := node.(*branchNode) require.True(ok) @@ -82,7 +83,7 @@ func TestBranchNodeClone(t *testing.T) { children['c'] = &hashNode{hashVal: []byte("c")} children['d'] = &hashNode{hashVal: []byte("d")} indices := NewSortedList(children) - node, err := newBranchNode(&merklePatriciaTrie{async: true}, children, indices) + node, err := newBranchNode(cli, children, indices) require.NoError(err) bn, ok := node.(*branchNode) require.True(ok) @@ -106,7 +107,7 @@ func TestBranchNodeProto(t *testing.T) { children: children, indices: indices, } - cli := &merklePatriciaTrie{async: true} + cli := &merklePatriciaTrie{async: true, hashFunc: DefaultHashFunc} proto, err := bnode.proto(cli, true) require.NoError(err) nodepb, ok := proto.(*triepb.NodePb) diff --git a/db/trie/mptrie/cachenode.go b/db/trie/mptrie/cachenode.go index edf23fe4eb..f1448c191f 100644 --- a/db/trie/mptrie/cachenode.go +++ b/db/trie/mptrie/cachenode.go @@ -7,6 +7,8 @@ package mptrie import ( + "errors" + "google.golang.org/protobuf/proto" ) @@ -25,6 +27,9 @@ func (cn *cacheNode) hash(cli client, flush bool) ([]byte, error) { if len(cn.hashVal) != 0 { return cn.hashVal, nil } + if cli == nil { + return []byte{}, errors.New("client cannot be nil") + } pb, err := cn.proto(cli, flush) if err != nil { return nil, err From c5014168ea7580bc168852e78f5f531a1ff7e126 Mon Sep 17 00:00:00 2001 From: millken Date: Thu, 29 Dec 2022 11:34:56 +0800 Subject: [PATCH 17/18] create hashval inside logNode func --- db/trie/mptrie/branchnode.go | 17 ++++++----------- db/trie/mptrie/extensionnode.go | 14 +++++--------- db/trie/mptrie/leafnode.go | 14 +++++--------- db/trie/mptrie/lognode.go | 13 ++++++++----- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/db/trie/mptrie/branchnode.go b/db/trie/mptrie/branchnode.go index d38bc7d1ab..ccfef9908c 100644 --- a/db/trie/mptrie/branchnode.go +++ b/db/trie/mptrie/branchnode.go @@ -46,8 +46,7 @@ func newBranchNode( } } } - hashVal, _ := bnode.cacheNode.Hash(cli) - if err := logNode(_nodeTypeBranch, _actionTypeNew, hashVal, bnode); err != nil { + if err := logNode(_nodeTypeBranch, _actionTypeNew, bnode, cli); err != nil { return nil, err } return bnode, nil @@ -73,8 +72,7 @@ func newRootBranchNode(cli client, children map[byte]node, indices *SortedList, } } } - hashVal, _ := bnode.cacheNode.Hash(cli) - if err := logNode(_nodeTypeBranch, _actionTypeNew, hashVal, bnode); err != nil { + if err := logNode(_nodeTypeBranch, _actionTypeNew, bnode, cli); err != nil { return nil, err } return bnode, nil @@ -93,7 +91,7 @@ func newBranchNodeFromProtoPb(pb *triepb.BranchPb, hashVal []byte) *branchNode { } bnode.indices = NewSortedList(bnode.children) bnode.cacheNode.serializable = bnode - if err := logNode(_nodeTypeBranch, _actionTypeNew, hashVal, bnode); err != nil { + if err := logNode(_nodeTypeBranch, _actionTypeNew, bnode, nil); err != nil { panic(err) } return bnode @@ -112,8 +110,7 @@ func (b *branchNode) Children() []node { } func (b *branchNode) Delete(cli client, key keyType, offset uint8) (node, error) { - hashVal, _ := b.cacheNode.Hash(cli) - if err := logNode(_nodeTypeBranch, _actionTypeDelete, hashVal, b); err != nil { + if err := logNode(_nodeTypeBranch, _actionTypeDelete, b, cli); err != nil { return nil, err } offsetKey := key[offset] @@ -169,8 +166,7 @@ func (b *branchNode) Delete(cli client, key keyType, offset uint8) (node, error) } func (b *branchNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) { - hashVal, _ := b.cacheNode.Hash(cli) - if err := logNode(_nodeTypeBranch, _actionTypeUpsert, hashVal, b); err != nil { + if err := logNode(_nodeTypeBranch, _actionTypeUpsert, b, cli); err != nil { return nil, err } var newChild node @@ -190,8 +186,7 @@ func (b *branchNode) Upsert(cli client, key keyType, offset uint8, value []byte) } func (b *branchNode) Search(cli client, key keyType, offset uint8) (node, error) { - hashVal, _ := b.cacheNode.Hash(cli) - if err := logNode(_nodeTypeBranch, _actionTypeSearch, hashVal, b); err != nil { + if err := logNode(_nodeTypeBranch, _actionTypeSearch, b, cli); err != nil { return nil, err } child, err := b.child(key[offset]) diff --git a/db/trie/mptrie/extensionnode.go b/db/trie/mptrie/extensionnode.go index 57574a341e..9e8a5ab130 100644 --- a/db/trie/mptrie/extensionnode.go +++ b/db/trie/mptrie/extensionnode.go @@ -38,8 +38,7 @@ func newExtensionNode( return nil, err } } - hashVal, _ := e.cacheNode.Hash(cli) - if err := logNode(_nodeTypeExtension, _actionTypeNew, hashVal, e); err != nil { + if err := logNode(_nodeTypeExtension, _actionTypeNew, e, cli); err != nil { return nil, err } return e, nil @@ -55,15 +54,14 @@ func newExtensionNodeFromProtoPb(pb *triepb.ExtendPb, hashVal []byte) *extension child: newHashNode(pb.Value), } e.cacheNode.serializable = e - if err := logNode(_nodeTypeExtension, _actionTypeNew, hashVal, e); err != nil { + if err := logNode(_nodeTypeExtension, _actionTypeNew, e, nil); err != nil { panic(err) } return e } func (e *extensionNode) Delete(cli client, key keyType, offset uint8) (node, error) { - hashVal, _ := e.cacheNode.Hash(cli) - if err := logNode(_nodeTypeExtension, _actionTypeDelete, hashVal, e); err != nil { + if err := logNode(_nodeTypeExtension, _actionTypeDelete, e, cli); err != nil { return nil, err } matched := e.commonPrefixLength(key[offset:]) @@ -96,8 +94,7 @@ func (e *extensionNode) Delete(cli client, key keyType, offset uint8) (node, err } func (e *extensionNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) { - hashVal, _ := e.cacheNode.Hash(cli) - if err := logNode(_nodeTypeExtension, _actionTypeUpsert, hashVal, e); err != nil { + if err := logNode(_nodeTypeExtension, _actionTypeUpsert, e, cli); err != nil { return nil, err } matched := e.commonPrefixLength(key[offset:]) @@ -135,8 +132,7 @@ func (e *extensionNode) Upsert(cli client, key keyType, offset uint8, value []by } func (e *extensionNode) Search(cli client, key keyType, offset uint8) (node, error) { - hashVal, _ := e.cacheNode.Hash(cli) - if err := logNode(_nodeTypeExtension, _actionTypeSearch, hashVal, e); err != nil { + if err := logNode(_nodeTypeExtension, _actionTypeSearch, e, cli); err != nil { return nil, err } matched := e.commonPrefixLength(key[offset:]) diff --git a/db/trie/mptrie/leafnode.go b/db/trie/mptrie/leafnode.go index 5e6513fb0d..563e50be19 100644 --- a/db/trie/mptrie/leafnode.go +++ b/db/trie/mptrie/leafnode.go @@ -38,8 +38,7 @@ func newLeafNode( return nil, err } } - hashVal, _ := l.cacheNode.Hash(cli) - if err := logNode(_nodeTypeLeaf, _actionTypeNew, hashVal, l); err != nil { + if err := logNode(_nodeTypeLeaf, _actionTypeNew, l, cli); err != nil { return nil, err } return l, nil @@ -55,7 +54,7 @@ func newLeafNodeFromProtoPb(pb *triepb.LeafPb, hashVal []byte) *leafNode { value: pb.Value, } l.cacheNode.serializable = l - if err := logNode(_nodeTypeLeaf, _actionTypeNew, hashVal, l); err != nil { + if err := logNode(_nodeTypeLeaf, _actionTypeNew, l, nil); err != nil { panic(err) } return l @@ -70,8 +69,7 @@ func (l *leafNode) Value() []byte { } func (l *leafNode) Delete(cli client, key keyType, offset uint8) (node, error) { - hashVal, _ := l.cacheNode.Hash(cli) - if err := logNode(_nodeTypeLeaf, _actionTypeDelete, hashVal, l); err != nil { + if err := logNode(_nodeTypeLeaf, _actionTypeDelete, l, cli); err != nil { return nil, err } if !bytes.Equal(l.key[offset:], key[offset:]) { @@ -81,8 +79,7 @@ func (l *leafNode) Delete(cli client, key keyType, offset uint8) (node, error) { } func (l *leafNode) Upsert(cli client, key keyType, offset uint8, value []byte) (node, error) { - hashVal, _ := l.cacheNode.Hash(cli) - if err := logNode(_nodeTypeLeaf, _actionTypeUpsert, hashVal, l); err != nil { + if err := logNode(_nodeTypeLeaf, _actionTypeUpsert, l, cli); err != nil { return nil, err } matched := commonPrefixLength(l.key[offset:], key[offset:]) @@ -122,8 +119,7 @@ func (l *leafNode) Upsert(cli client, key keyType, offset uint8, value []byte) ( } func (l *leafNode) Search(cli client, key keyType, offset uint8) (node, error) { - hashVal, _ := l.cacheNode.Hash(cli) - if err := logNode(_nodeTypeLeaf, _actionTypeSearch, hashVal, l); err != nil { + if err := logNode(_nodeTypeLeaf, _actionTypeSearch, l, cli); err != nil { return nil, err } if !bytes.Equal(l.key[offset:], key[offset:]) { diff --git a/db/trie/mptrie/lognode.go b/db/trie/mptrie/lognode.go index 6124ae276d..53c9f75e48 100644 --- a/db/trie/mptrie/lognode.go +++ b/db/trie/mptrie/lognode.go @@ -80,11 +80,11 @@ func CloseLogDB() error { return logFile.Close() } -func logNode(nt nodeType, at actionType, hashvalue []byte, n node) error { +func logNode(nt nodeType, at actionType, n node, cli client) error { if !enabledLogMptrie { return nil } - nodeKey, nodePath, nodeChildren, err := parseNode(n) + nodeKey, nodePath, nodeChildren, hashvalue, err := parseNode(n, cli) if err != nil { return err } @@ -100,23 +100,26 @@ func logNode(nt nodeType, at actionType, hashvalue []byte, n node) error { HashLen: uint8(len(hashvalue)), HashVal: hashvalue, } - // write events length + // write event length if err = logWriter.WriteByte(byte(len(event.Bytes()))); err != nil { return err } - // write events body + // write event body _, err = logWriter.Write(event.Bytes()) return err } -func parseNode(n node) (nodeKey []byte, nodePath []byte, nodeChildren []byte, err error) { +func parseNode(n node, cli client) (nodeKey, nodePath, nodeChildren, hashvalue []byte, err error) { switch n := n.(type) { case *leafNode: nodeKey = n.key + hashvalue, err = n.cacheNode.Hash(cli) case *extensionNode: nodePath = n.path + hashvalue, err = n.cacheNode.Hash(cli) case *branchNode: nodeChildren = n.indices.List() + hashvalue, err = n.cacheNode.Hash(cli) default: err = errors.Errorf("unknown node type %T", n) } From 1b1944d3754715bfd1697ce81d2fd3ec7f513d69 Mon Sep 17 00:00:00 2001 From: millken Date: Fri, 13 Jan 2023 13:51:55 +0800 Subject: [PATCH 18/18] fix comments --- db/trie/mptrie/lognode_test.go | 45 +++++++++------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/db/trie/mptrie/lognode_test.go b/db/trie/mptrie/lognode_test.go index 7ce9049ddb..65dd7c10f9 100644 --- a/db/trie/mptrie/lognode_test.go +++ b/db/trie/mptrie/lognode_test.go @@ -9,6 +9,7 @@ import ( ) func TestNodeEvent(t *testing.T) { + require := require.New(t) tests := []struct { event nodeEvent }{ @@ -45,39 +46,17 @@ func TestNodeEvent(t *testing.T) { for _, test := range tests { b := test.event.Bytes() event, err := parseNodeEvent(b) - if err != nil { - t.Errorf("unexpected error %v", err) - } - if event.NodeType != test.event.NodeType { - t.Errorf("unexpected NodeType %v", event.NodeType) - } - if event.ActionType != test.event.ActionType { - t.Errorf("unexpected ActionType %v", event.ActionType) - } - if event.KeyLen != test.event.KeyLen { - t.Errorf("unexpected key length %v", event.KeyLen) - } - if event.Key != nil && string(event.Key) != string(test.event.Key) { - t.Errorf("unexpected key %v", event.Key) - } - if event.PathLen != test.event.PathLen { - t.Errorf("unexpected path length %v", event.PathLen) - } - if event.Path != nil && string(event.Path) != string(test.event.Path) { - t.Errorf("unexpected path %v", event.Path) - } - if event.ChildrenLen != test.event.ChildrenLen { - t.Errorf("unexpected children length %v", event.ChildrenLen) - } - if event.Children != nil && string(event.Children) != string(test.event.Children) { - t.Errorf("unexpected children %v", event.Children) - } - if event.HashLen != test.event.HashLen { - t.Errorf("unexpected hash length %v", event.HashLen) - } - if event.HashVal != nil && string(event.HashVal) != string(test.event.HashVal) { - t.Errorf("unexpected hash %v", event.HashVal) - } + require.NoError(err) + require.Equal(test.event.NodeType, event.NodeType) + require.Equal(test.event.ActionType, event.ActionType) + require.Equal(test.event.KeyLen, event.KeyLen) + require.Equal(test.event.Key, event.Key) + require.Equal(test.event.PathLen, event.PathLen) + require.Equal(test.event.Path, event.Path) + require.Equal(test.event.ChildrenLen, event.ChildrenLen) + require.Equal(test.event.Children, event.Children) + require.Equal(test.event.HashLen, event.HashLen) + require.Equal(test.event.HashVal, event.HashVal) } }