From 4e4b7307c9f2d3a6ddfc863bb37ad19b1bc3c50c Mon Sep 17 00:00:00 2001 From: Trino Date: Thu, 20 Jun 2024 00:00:56 +0800 Subject: [PATCH 01/15] =?UTF-8?q?feat(server):=20=E5=AE=8C=E6=88=90reactor?= =?UTF-8?q?=20server=E5=BC=80=E5=8F=91=EF=BC=8C=E4=BD=86=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E8=BF=87=E7=A8=8B=E4=B8=AD=E9=81=87=E5=88=B0=E4=B8=AA=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Trino --- go.mod | 4 +- go.sum | 8 +- storage/server/netpoll_server.go | 38 +++ storage/server/proactor_server.go | 18 + storage/server/pure_goroutine_server.go | 90 +++++ storage/server/reactor_server.go | 415 ++++++++++++++++++++++++ storage/server/server.go | 4 + storage/server/server_test.go | 202 ++++++++++++ 8 files changed, 776 insertions(+), 3 deletions(-) create mode 100644 storage/server/netpoll_server.go create mode 100644 storage/server/proactor_server.go create mode 100644 storage/server/pure_goroutine_server.go create mode 100644 storage/server/reactor_server.go create mode 100644 storage/server/server_test.go diff --git a/go.mod b/go.mod index 3ddc3a6..6a6d57e 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module github.com/Trinoooo/eggie_kv go 1.19 require ( - github.com/bytedance/gopkg v0.0.0-20231219111115-a5eedbe96960 + github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3 github.com/chzyer/readline v1.5.1 + github.com/cloudwego/netpoll v0.6.1 github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f github.com/mitchellh/go-homedir v1.1.0 + github.com/pkg/errors v0.9.1 github.com/spf13/viper v1.18.2 github.com/urfave/cli/v2 v2.26.0 go.uber.org/zap v1.26.0 diff --git a/go.sum b/go.sum index ad99235..d992b60 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,13 @@ -github.com/bytedance/gopkg v0.0.0-20231219111115-a5eedbe96960 h1:t2xAuIlnhWJDIpcHZEbpoVsQH1hOk9eGGaKU2dXl1PE= -github.com/bytedance/gopkg v0.0.0-20231219111115-a5eedbe96960/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ= +github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3 h1:ZKUHguI38SRQJkq7hhmwn8lAv3xM6B5qkj1IneS15YY= +github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/cloudwego/netpoll v0.6.1 h1:Cjftvi6bmumsOijmuUFy6HqAUXMxAT3fKK96wsrm3XA= +github.com/cloudwego/netpoll v0.6.1/go.mod h1:kaqvfZ70qd4T2WtIIpCOi5Cxyob8viEpzLhCrTrz3HM= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -29,6 +31,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= diff --git a/storage/server/netpoll_server.go b/storage/server/netpoll_server.go new file mode 100644 index 0000000..943d827 --- /dev/null +++ b/storage/server/netpoll_server.go @@ -0,0 +1,38 @@ +package server + +import ( + "context" + "github.com/cloudwego/netpoll" + "net" +) + +// NetpollEventLoopServer 字节的NIO eventLoop 网络库 +type NetpollEventLoopServer struct { + listener net.Listener + eventLoop netpoll.EventLoop +} + +func NewNetpollEventLoopServer(addr string, handler netpoll.OnRequest) (*NetpollEventLoopServer, error) { + var srv = &NetpollEventLoopServer{} + var err error + + srv.listener, err = netpoll.CreateListener("tcp", addr) + if err != nil { + return nil, err + } + + srv.eventLoop, err = netpoll.NewEventLoop(handler) + if err != nil { + return nil, err + } + + return srv, nil +} + +func (els *NetpollEventLoopServer) Serve() error { + return els.eventLoop.Serve(els.listener) +} + +func (els *NetpollEventLoopServer) Close() error { + return els.eventLoop.Shutdown(context.Background()) +} diff --git a/storage/server/proactor_server.go b/storage/server/proactor_server.go new file mode 100644 index 0000000..eb69b9d --- /dev/null +++ b/storage/server/proactor_server.go @@ -0,0 +1,18 @@ +package server + +// ProactorServer 自实现的proactor网络模式服务器 +// 或许后续可以实现个windows版本 +type ProactorServer struct { +} + +func NewProactorServer() *ProactorServer { + return &ProactorServer{} +} + +func (ps *ProactorServer) Serve() { + +} + +func (ps *ProactorServer) Close() error { + return nil +} diff --git a/storage/server/pure_goroutine_server.go b/storage/server/pure_goroutine_server.go new file mode 100644 index 0000000..7ce01d1 --- /dev/null +++ b/storage/server/pure_goroutine_server.go @@ -0,0 +1,90 @@ +package server + +import ( + "context" + "log" + "net" + "sync" +) + +// PureGoroutineServer 没用多路IO复用，纯goroutine并发 +type PureGoroutineServer struct { + mutex sync.Mutex + serverTransport net.Listener + handler simpleHandler + stop chan struct{} + done sync.WaitGroup +} + +func NewPureGoroutineServer(addr string, handler simpleHandler) (*PureGoroutineServer, error) { + var srv = &PureGoroutineServer{ + handler: handler, + stop: make(chan struct{}), + } + var err error + srv.serverTransport, err = net.Listen("tcp", addr) + if err != nil { + return nil, err + } + return srv, nil +} + +func (pgc *PureGoroutineServer) Serve() error { + for { + select { + case <-pgc.stop: + return nil + default: + } + + conn, err := pgc.serverTransport.Accept() + if err != nil { + pgc.mutex.Lock() + select { + case <-pgc.stop: + pgc.mutex.Unlock() + return nil + default: + log.Println(err) + close(pgc.stop) + pgc.mutex.Unlock() + return err + } + } + + pgc.done.Add(2) + ctx, cancel := context.WithCancel(context.Background()) + // bizHandler + go func() { + defer func() { + if err := conn.Close(); err != nil { + log.Println(err) + } + pgc.done.Done() + cancel() + }() + pgc.handler(ctx, conn) + }() + // notifier + go func() { + defer pgc.done.Done() + select { + case <-pgc.stop: + cancel() + case <-ctx.Done(): + } + }() + } +} + +func (pgc *PureGoroutineServer) Close() error { + pgc.mutex.Lock() + defer pgc.mutex.Unlock() + err := pgc.serverTransport.Close() + if err != nil { + return err + } + close(pgc.stop) + pgc.done.Wait() + return nil +} diff --git a/storage/server/reactor_server.go b/storage/server/reactor_server.go new file mode 100644 index 0000000..79426af --- /dev/null +++ b/storage/server/reactor_server.go @@ -0,0 +1,415 @@ +package server + +import ( + "context" + "github.com/bytedance/gopkg/util/gopool" + "github.com/pkg/errors" + "log" + "math/rand" + "net" + "sync" + "syscall" + "time" + "unsafe" +) + +type ReactorServer struct { + mutex sync.Mutex + serverTransport net.Listener + bizHandler simpleHandler + pool gopool.Pool + dp *dispatcher + reactors map[int64]*reactor + stop chan struct{} + done sync.WaitGroup +} + +type connWrapper struct { + tcpConn *net.TCPConn + fd *uintptr +} + +type reactor struct { + id int64 + connects chan *connWrapper + srv *ReactorServer + w *waiter + poller poller +} + +func newReactor(id int64, srv *ReactorServer, poller poller) *reactor { + r := &reactor{ + srv: srv, + id: id, + connects: make(chan *connWrapper), + poller: poller, + w: &waiter{ + events: make(chan pevent), + poller: poller, + }, + } + r.w.parent = r + return r +} + +func (r *reactor) run() { + defer r.srv.done.Done() + log.Printf("reactor #%d start", r.id) + + connects := r.connects + r.srv.done.Add(1) + r.srv.pool.Go(r.w.run) + for { + select { + case wrapper, ok := <-connects: + // output been close by dispatcher + if !ok { + log.Printf("reactor #%d ready to closes kqfd", r.id) + if e := r.poller.close(); e != nil { + log.Printf("reactor #%d close poller failed. err: %v", r.id, e) + } + connects = nil + log.Printf("reactor #%d output set to nil", r.id) + continue + } + log.Printf("reactor #%d receive output connection", r.id) + // get wrapper file descriptor + if wrapper.fd == nil { + file, err := wrapper.tcpConn.File() + if err != nil { + log.Println(err) + continue + } + fd := file.Fd() + wrapper.fd = &fd + } + + log.Printf("reactor #%d ready to register event, wrapper fd: %v", r.id, uint64(*wrapper.fd)) + + changes := []pevent{{ + connFd: uint64(*wrapper.fd), + operation: syscall.EVFILT_READ, + flag: syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT, // edge trigger mode + userData: (*byte)(unsafe.Pointer(wrapper)), + }} + // register event to kqueue + if err := r.poller.register(changes); err != nil { + e := wrapper.tcpConn.Close() + if e != nil { + err = errors.Wrap(err, e.Error()) + } + log.Println(err, changes, wrapper.tcpConn.RemoteAddr(), wrapper.tcpConn.LocalAddr()) + continue + } + + log.Printf("reactor #%d register event success", r.id) + case evt, ok := <-r.w.output(): + if !ok { + log.Printf("reactor #%d stop", r.id) + return + } + log.Printf("reactor #%d handle event", r.id) + // in this case, close is already called. + if r.srv.pool == nil { + continue + } + ctx, cancel := context.WithTimeout(context.Background(), processTimeout) + r.srv.done.Add(2) + r.srv.pool.Go(func() { + wrapper := (*connWrapper)(unsafe.Pointer(evt.userData)) + r.srv.handler(ctx, cancel, wrapper) + }) + r.srv.pool.Go(func() { r.srv.notifier(ctx, cancel) }) + } + } +} + +func (r *reactor) input() chan<- *connWrapper { + return r.connects +} + +type waiter struct { + events chan pevent + parent *reactor + poller poller +} + +func (w *waiter) run() { + log.Printf("waiter #%d start", w.parent.id) + defer w.parent.srv.done.Done() + // event buf + evts := make([]pevent, 10) + for { + log.Printf("waiter #%d ready to wait event trigger", w.parent.id) + + // wait for event to be trigger + n, err := w.poller.wait(evts) + if err != nil { + log.Printf("waiter #%d stop, err: %v", w.parent.id, err) + close(w.events) + return + } + + log.Printf("waiter #%d event trigger success, evts: %#v, n: %d", w.parent.id, evts, n) + + for i := 0; i < n; i++ { + evt := evts[i] + switch { + // we do not care about eof + case evt.flag&syscall.EV_EOF != 0: + log.Printf("waiter #%d meet eof, skip", w.parent.id) + default: + w.events <- evt + log.Printf("waiter #%d sent evt to reactor success", w.parent.id) + } + } + } +} + +func (w *waiter) output() <-chan pevent { + return w.events +} + +type pevent struct { + connFd uint64 + operation int64 + flag int64 + userData *byte +} + +type poller interface { + register(changes []pevent) error + wait(events []pevent) (int, error) + close() error +} + +type kqueuePoller struct { + kq *int +} + +func newKqueuePoller() (*kqueuePoller, error) { + kqFd, err := syscall.Kqueue() + if err != nil { + return nil, err + } + + return &kqueuePoller{ + kq: &kqFd, + }, nil +} + +func (kp *kqueuePoller) register(changes []pevent) error { + kchanges := kp.fromPevent(changes) + _, err := syscall.Kevent(*kp.kq, kchanges, nil, nil) + return err +} + +func (kp *kqueuePoller) wait(events []pevent) (int, error) { + kevents := kp.fromPevent(events) + n, err := syscall.Kevent(*kp.kq, nil, kevents, nil) + if err != nil { + return 0, err + } + kp.toPevent(kevents, events) + return n, nil +} + +func (kp *kqueuePoller) fromPevent(events []pevent) []syscall.Kevent_t { + kevents := make([]syscall.Kevent_t, 0, len(events)) + for _, pevt := range events { + kevents = append(kevents, syscall.Kevent_t{ + Ident: pevt.connFd, + Filter: int16(pevt.operation), + Flags: uint16(pevt.flag), + Udata: pevt.userData, + }) + } + return kevents +} + +func (kp *kqueuePoller) toPevent(kevents []syscall.Kevent_t, pevent []pevent) { + for idx, kevt := range kevents { + pevent[idx].connFd = kevt.Ident + pevent[idx].operation = int64(kevt.Filter) + pevent[idx].flag = int64(kevt.Flags) + pevent[idx].userData = kevt.Udata + } +} + +func (kq *kqueuePoller) close() error { + var err error + if kq.kq != nil { + err = syscall.Close(*kq.kq) + } + return err +} + +// todo: extract to config +const ( + numReactor = 3 + pollCapacity = 1000 + processTimeout = 1 * time.Second +) + +func NewReactorServer(addr string, handler simpleHandler) (*ReactorServer, error) { + var err error + srv := &ReactorServer{ + bizHandler: handler, + reactors: make(map[int64]*reactor), + pool: gopool.NewPool("handlers", pollCapacity, gopool.NewConfig()), + stop: make(chan struct{}), + } + + // init dispatcher + srv.dp = &dispatcher{ + connections: make(chan *connWrapper), + parent: srv, + } + + // init reactors + for i := 0; i < numReactor; i++ { + tmpIdx := int64(i) + kp, err := newKqueuePoller() + if err != nil { + return nil, err + } + srv.reactors[tmpIdx] = newReactor(tmpIdx, srv, kp) + } + + // init server transport + srv.serverTransport, err = net.Listen("tcp", addr) + if err != nil { + return nil, err + } + return srv, nil +} + +type dispatcher struct { + connections chan *connWrapper + parent *ReactorServer +} + +func (dp *dispatcher) run() { + defer dp.parent.done.Done() + log.Println("dispatcher start") + + for conn := range dp.connections { + log.Println("dispatcher receive connection") + // random load balance + id := rand.Int63n(numReactor) + if reactor, exist := dp.parent.reactors[id]; exist { + reactor.input() <- conn + log.Printf("dispatcher send connection to reactor #%v success", id) + } else { + log.Printf("dispatcher find reactor #%v not exist", id) + } + } + + for _, reactor := range dp.parent.reactors { + if reactor != nil { + close(reactor.input()) + } + } + + log.Println("dispatcher stop") +} + +func (dp *dispatcher) input() chan<- *connWrapper { + return dp.connections +} + +func (rs *ReactorServer) Serve() error { + rs.done.Add(numReactor + 1) + + // start connection dispatcher + rs.pool.Go(rs.dp.run) + for _, reactor := range rs.reactors { + rs.pool.Go(reactor.run) + } + + // mainReactor & acceptor + for { + conn, err := rs.serverTransport.Accept() + if err != nil { + rs.mutex.Lock() + close(rs.dp.input()) + e := rs.close() + if e != nil { + e = errors.Wrap(e, err.Error()) + } + rs.done.Wait() + rs.clearState() + rs.mutex.Unlock() + return e + } + rs.mutex.Lock() + rs.dp.input() <- &connWrapper{ + tcpConn: conn.(*net.TCPConn), + fd: nil, + } + rs.mutex.Unlock() + } +} + +func (rs *ReactorServer) handler(ctx context.Context, cancel context.CancelFunc, wrapper *connWrapper) { + defer func() { + rs.done.Done() + cancel() + }() + rs.bizHandler(ctx, wrapper.tcpConn) + rs.mutex.Lock() + select { + case <-rs.stop: + rs.mutex.Unlock() + log.Printf("bizHandler ready to close connFd %d", *wrapper.fd) + if e := wrapper.tcpConn.Close(); e != nil { + log.Printf("bizHandler close connection failed. err: %v", e) + return + } + default: + rs.dp.input() <- wrapper // reuse long connection + rs.mutex.Unlock() + } +} + +func (rs *ReactorServer) notifier(ctx context.Context, cancel context.CancelFunc) { + defer rs.done.Done() + select { + case <-rs.stop: + cancel() + case <-ctx.Done(): + // do nothing + } +} + +func (rs *ReactorServer) Close() error { + rs.mutex.Lock() + if err := rs.close(); err != nil { + return err + } + rs.mutex.Unlock() + rs.done.Wait() + rs.mutex.Lock() + rs.clearState() + rs.mutex.Unlock() + return nil +} + +func (rs *ReactorServer) close() error { + select { + case <-rs.stop: + return nil + // stop already closed + default: + close(rs.stop) + return rs.serverTransport.Close() + } +} + +func (rs *ReactorServer) clearState() { + rs.pool = nil + rs.serverTransport = nil + rs.reactors = nil + rs.bizHandler = nil + rs.reactors = nil +} diff --git a/storage/server/server.go b/storage/server/server.go index 0ec7697..f47b79c 100644 --- a/storage/server/server.go +++ b/storage/server/server.go @@ -1,6 +1,7 @@ package server import ( + "context" "encoding/json" "errors" "fmt" @@ -11,6 +12,7 @@ import ( "github.com/Trinoooo/eggie_kv/storage/logs" "github.com/spf13/viper" "io" + "net" "net/http" ) @@ -154,3 +156,5 @@ func newSuccessResp(data []byte) *consts.KvResponse { Data: data, } } + +type simpleHandler func(ctx context.Context, conn net.Conn) diff --git a/storage/server/server_test.go b/storage/server/server_test.go new file mode 100644 index 0000000..e486523 --- /dev/null +++ b/storage/server/server_test.go @@ -0,0 +1,202 @@ +package server + +import ( + "context" + "encoding/binary" + "github.com/cloudwego/netpoll" + "log" + "net" + "net/http" + _ "net/http/pprof" + "sync" + "testing" + "time" +) + +const ( + addr = "127.0.0.1:9999" + + fixSize = 8 + + concurrency = 18 +) + +func TestMain(m *testing.M) { + + log.SetFlags(log.Llongfile | log.LstdFlags) + m.Run() +} + +func mockClient(t *testing.T, closeServerCallback func() error) { + time.Sleep(500 * time.Millisecond) // wait for server start + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, 100) + inflight := sync.WaitGroup{} + // 短连接场景下可控制并发度 + for i := 0; i < concurrency; i++ { + inflight.Add(2) + go func() { + defer inflight.Done() + shortConnection(t, buf) + }() + go func() { + defer inflight.Done() + longConnection(t, buf) + }() + } + inflight.Wait() + if err := closeServerCallback(); err != nil { + t.Error(err) + } +} + +func shortConnection(t *testing.T, buf []byte) { + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Error(err) + return + } + _, e := conn.Write(buf) + if e != nil { + t.Error(e) + return + } + log.Println("[short] client send request successfully") + innerBuf := make([]byte, 8) + _, e = conn.Read(innerBuf) + if e != nil { + t.Error(e) + return + } + log.Println("[short] client recv response successfully") + e = conn.Close() + if e != nil { + t.Error(e) + return + } + log.Println("[short] client close connection") +} + +func longConnection(t *testing.T, buf []byte) { + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Error(err) + return + } + for i := 0; i < 10; i++ { + _, e := conn.Write(buf) + if e != nil { + t.Error(e) + return + } + log.Println("[long] client send request successfully") + innerBuf := make([]byte, 8) + _, e = conn.Read(innerBuf) + if e != nil { + t.Error(e) + return + } + log.Println("[long] client recv response successfully") + } + e := conn.Close() + if e != nil { + t.Error(e) + return + } + log.Println("[long] client close connection") +} + +func commonHandler(conn net.Conn, t *testing.T) { + buf := make([]byte, fixSize) + _, err := conn.Read(buf) + if err != nil { + t.Error(err) + } + v := binary.BigEndian.Uint64(buf) + v = bizLogic(v) + binary.BigEndian.PutUint64(buf, v) + _, err = conn.Write(buf) + if err != nil { + t.Error(err) + } +} + +func bizLogic(v uint64) uint64 { + // 模拟耗时cpu密集性计算 + for i := 0; i < 1000000; i++ { + v += uint64(i) + } + // 模拟耗时io密集性操作 + time.Sleep(1 * time.Second) + return v +} + +func TestPureGoroutineServer(t *testing.T) { + return + server, err := NewPureGoroutineServer(addr, func(ctx context.Context, conn net.Conn) { + commonHandler(conn, t) + }) + if err != nil { + t.Error(err) + return + } + go mockClient(t, server.Close) + err = server.Serve() + if err != nil { + t.Error(err) + } +} + +func TestNetpollEventLoopServer(t *testing.T) { + return + server, err := NewNetpollEventLoopServer(addr, func(ctx context.Context, connection netpoll.Connection) error { + commonHandler(connection, t) + return nil + }) + if err != nil { + t.Error(err) + return + } + go mockClient(t, server.Close) + err = server.Serve() + if err != nil { + t.Error(err) + } +} + +func TestReactorServer(t *testing.T) { + // 启动性能分析服务器 + go func() { + log.Println(http.ListenAndServe("localhost:6060", nil)) + }() + + server, err := NewReactorServer(addr, func(ctx context.Context, conn net.Conn) { + commonHandler(conn, t) + }) + if err != nil { + t.Error(err) + return + } + go mockClient(t, server.Close) + err = server.Serve() + if err != nil { + t.Error(err) + } + +} + +func TestProactorServer(t *testing.T) { + //todo +} + +func BenchmarkNewNetpollEventLoopServer(b *testing.B) { + +} + +func BenchmarkNewPureGoroutineServer(b *testing.B) { + +} + +func BenchmarkNewReactorServer(b *testing.B) { + +} From 2012223ffa3af926cc26a64791fc42f34bbe5e1b Mon Sep 17 00:00:00 2001 From: Trino Date: Fri, 21 Jun 2024 00:09:41 +0800 Subject: [PATCH 02/15] =?UTF-8?q?fix(server):=20=E5=B0=86net.Conn=E6=8D=A2?= =?UTF-8?q?=E6=88=90=E8=87=AA=E5=B7=B1=E6=89=8B=E6=90=93=E7=9A=84conn?= =?UTF-8?q?=E5=90=8E=EF=BC=8C=E9=97=AE=E9=A2=98=E8=A7=A3=E5=86=B3=E4=BA=86?= =?UTF-8?q?=EF=BC=8Cconcurrency=3D30?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Trino --- storage/server/connections.go | 66 +++++++++ storage/server/netpoll_server.go | 38 ----- storage/server/poller/iface.go | 14 ++ storage/server/poller/impl_bsd_arm64.go | 64 ++++++++ storage/server/proactor_server.go | 18 --- storage/server/pure_goroutine_server.go | 90 ----------- storage/server/reactor_server.go | 189 ++++++------------------ storage/server/server.go | 3 +- storage/server/server_test.go | 52 ++----- 9 files changed, 203 insertions(+), 331 deletions(-) create mode 100644 storage/server/connections.go delete mode 100644 storage/server/netpoll_server.go create mode 100644 storage/server/poller/iface.go create mode 100644 storage/server/poller/impl_bsd_arm64.go delete mode 100644 storage/server/proactor_server.go delete mode 100644 storage/server/pure_goroutine_server.go diff --git a/storage/server/connections.go b/storage/server/connections.go new file mode 100644 index 0000000..5ba4b84 --- /dev/null +++ b/storage/server/connections.go @@ -0,0 +1,66 @@ +package server + +import "syscall" + +type Conn struct { + fd int +} + +func (c *Conn) Read(buf []byte) (int, error) { + return syscall.Read(c.fd, buf) +} + +func (c *Conn) Write(buf []byte) (int, error) { + return syscall.Write(c.fd, buf) +} + +func (c *Conn) Close() error { + return syscall.Close(c.fd) +} + +type Listener struct { + conn *Conn +} + +func (l *Listener) Accept() (*Conn, error) { + socket, _, err := syscall.Accept(l.conn.fd) + if err != nil { + return nil, err + } + + if err = syscall.SetNonblock(socket, true); err != nil { + return nil, err + } + + return &Conn{ + fd: socket, + }, nil +} + +func (l *Listener) Close() error { + return syscall.Close(l.conn.fd) +} + +func Listen(addr [4]byte, port int) (*Listener, error) { + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != nil { + return nil, err + } + + if err = syscall.Bind(fd, &syscall.SockaddrInet4{ + Port: port, + Addr: addr, + }); err != nil { + return nil, err + } + + if err = syscall.Listen(fd, syscall.SOMAXCONN); err != nil { + return nil, err + } + + return &Listener{ + conn: &Conn{ + fd: fd, + }, + }, nil +} diff --git a/storage/server/netpoll_server.go b/storage/server/netpoll_server.go deleted file mode 100644 index 943d827..0000000 --- a/storage/server/netpoll_server.go +++ /dev/null @@ -1,38 +0,0 @@ -package server - -import ( - "context" - "github.com/cloudwego/netpoll" - "net" -) - -// NetpollEventLoopServer 字节的NIO eventLoop 网络库 -type NetpollEventLoopServer struct { - listener net.Listener - eventLoop netpoll.EventLoop -} - -func NewNetpollEventLoopServer(addr string, handler netpoll.OnRequest) (*NetpollEventLoopServer, error) { - var srv = &NetpollEventLoopServer{} - var err error - - srv.listener, err = netpoll.CreateListener("tcp", addr) - if err != nil { - return nil, err - } - - srv.eventLoop, err = netpoll.NewEventLoop(handler) - if err != nil { - return nil, err - } - - return srv, nil -} - -func (els *NetpollEventLoopServer) Serve() error { - return els.eventLoop.Serve(els.listener) -} - -func (els *NetpollEventLoopServer) Close() error { - return els.eventLoop.Shutdown(context.Background()) -} diff --git a/storage/server/poller/iface.go b/storage/server/poller/iface.go new file mode 100644 index 0000000..eaeebf8 --- /dev/null +++ b/storage/server/poller/iface.go @@ -0,0 +1,14 @@ +package poller + +type Pevent struct { + ConnFd uint64 + Operation int64 + Flag int64 + UserData *byte +} + +type Poller interface { + Register(changes []Pevent) error + Wait(events []Pevent) (int, error) + Close() error +} diff --git a/storage/server/poller/impl_bsd_arm64.go b/storage/server/poller/impl_bsd_arm64.go new file mode 100644 index 0000000..d85a1ee --- /dev/null +++ b/storage/server/poller/impl_bsd_arm64.go @@ -0,0 +1,64 @@ +package poller + +import "syscall" + +type KqueuePoller struct { + kq *int +} + +func NewKqueuePoller() (*KqueuePoller, error) { + kqFd, err := syscall.Kqueue() + if err != nil { + return nil, err + } + + return &KqueuePoller{ + kq: &kqFd, + }, nil +} + +func (kp *KqueuePoller) Register(changes []Pevent) error { + kchanges := kp.fromPevent(changes) + _, err := syscall.Kevent(*kp.kq, kchanges, nil, nil) + return err +} + +func (kp *KqueuePoller) Wait(events []Pevent) (int, error) { + kevents := kp.fromPevent(events) + n, err := syscall.Kevent(*kp.kq, nil, kevents, nil) + if err != nil { + return 0, err + } + kp.toPevent(kevents, events) + return n, nil +} + +func (kp *KqueuePoller) fromPevent(events []Pevent) []syscall.Kevent_t { + kevents := make([]syscall.Kevent_t, 0, len(events)) + for _, pevt := range events { + kevents = append(kevents, syscall.Kevent_t{ + Ident: pevt.ConnFd, + Filter: int16(pevt.Operation), + Flags: uint16(pevt.Flag), + Udata: pevt.UserData, + }) + } + return kevents +} + +func (kp *KqueuePoller) toPevent(kevents []syscall.Kevent_t, pevent []Pevent) { + for idx, kevt := range kevents { + pevent[idx].ConnFd = kevt.Ident + pevent[idx].Operation = int64(kevt.Filter) + pevent[idx].Flag = int64(kevt.Flags) + pevent[idx].UserData = kevt.Udata + } +} + +func (kq *KqueuePoller) Close() error { + var err error + if kq.kq != nil { + err = syscall.Close(*kq.kq) + } + return err +} diff --git a/storage/server/proactor_server.go b/storage/server/proactor_server.go deleted file mode 100644 index eb69b9d..0000000 --- a/storage/server/proactor_server.go +++ /dev/null @@ -1,18 +0,0 @@ -package server - -// ProactorServer 自实现的proactor网络模式服务器 -// 或许后续可以实现个windows版本 -type ProactorServer struct { -} - -func NewProactorServer() *ProactorServer { - return &ProactorServer{} -} - -func (ps *ProactorServer) Serve() { - -} - -func (ps *ProactorServer) Close() error { - return nil -} diff --git a/storage/server/pure_goroutine_server.go b/storage/server/pure_goroutine_server.go deleted file mode 100644 index 7ce01d1..0000000 --- a/storage/server/pure_goroutine_server.go +++ /dev/null @@ -1,90 +0,0 @@ -package server - -import ( - "context" - "log" - "net" - "sync" -) - -// PureGoroutineServer 没用多路IO复用，纯goroutine并发 -type PureGoroutineServer struct { - mutex sync.Mutex - serverTransport net.Listener - handler simpleHandler - stop chan struct{} - done sync.WaitGroup -} - -func NewPureGoroutineServer(addr string, handler simpleHandler) (*PureGoroutineServer, error) { - var srv = &PureGoroutineServer{ - handler: handler, - stop: make(chan struct{}), - } - var err error - srv.serverTransport, err = net.Listen("tcp", addr) - if err != nil { - return nil, err - } - return srv, nil -} - -func (pgc *PureGoroutineServer) Serve() error { - for { - select { - case <-pgc.stop: - return nil - default: - } - - conn, err := pgc.serverTransport.Accept() - if err != nil { - pgc.mutex.Lock() - select { - case <-pgc.stop: - pgc.mutex.Unlock() - return nil - default: - log.Println(err) - close(pgc.stop) - pgc.mutex.Unlock() - return err - } - } - - pgc.done.Add(2) - ctx, cancel := context.WithCancel(context.Background()) - // bizHandler - go func() { - defer func() { - if err := conn.Close(); err != nil { - log.Println(err) - } - pgc.done.Done() - cancel() - }() - pgc.handler(ctx, conn) - }() - // notifier - go func() { - defer pgc.done.Done() - select { - case <-pgc.stop: - cancel() - case <-ctx.Done(): - } - }() - } -} - -func (pgc *PureGoroutineServer) Close() error { - pgc.mutex.Lock() - defer pgc.mutex.Unlock() - err := pgc.serverTransport.Close() - if err != nil { - return err - } - close(pgc.stop) - pgc.done.Wait() - return nil -} diff --git a/storage/server/reactor_server.go b/storage/server/reactor_server.go index 79426af..4bb4662 100644 --- a/storage/server/reactor_server.go +++ b/storage/server/reactor_server.go @@ -2,11 +2,11 @@ package server import ( "context" + "github.com/Trinoooo/eggie_kv/storage/server/poller" "github.com/bytedance/gopkg/util/gopool" "github.com/pkg/errors" "log" "math/rand" - "net" "sync" "syscall" "time" @@ -15,7 +15,7 @@ import ( type ReactorServer struct { mutex sync.Mutex - serverTransport net.Listener + serverTransport *Listener bizHandler simpleHandler pool gopool.Pool dp *dispatcher @@ -24,28 +24,23 @@ type ReactorServer struct { done sync.WaitGroup } -type connWrapper struct { - tcpConn *net.TCPConn - fd *uintptr -} - type reactor struct { - id int64 - connects chan *connWrapper srv *ReactorServer + id int64 + connects chan *Conn w *waiter - poller poller + p poller.Poller } -func newReactor(id int64, srv *ReactorServer, poller poller) *reactor { +func newReactor(id int64, srv *ReactorServer, p poller.Poller) *reactor { r := &reactor{ srv: srv, id: id, - connects: make(chan *connWrapper), - poller: poller, + connects: make(chan *Conn), + p: p, w: &waiter{ - events: make(chan pevent), - poller: poller, + events: make(chan poller.Pevent), + p: p, }, } r.w.parent = r @@ -61,44 +56,35 @@ func (r *reactor) run() { r.srv.pool.Go(r.w.run) for { select { - case wrapper, ok := <-connects: + case conn, ok := <-connects: + log.Printf("reactor #%d receive output connection", r.id) // output been close by dispatcher if !ok { - log.Printf("reactor #%d ready to closes kqfd", r.id) - if e := r.poller.close(); e != nil { - log.Printf("reactor #%d close poller failed. err: %v", r.id, e) + log.Printf("reactor #%d ready to closes poller", r.id) + if e := r.p.Close(); e != nil { + log.Printf("reactor #%d close p failed. err: %v", r.id, e) } connects = nil log.Printf("reactor #%d output set to nil", r.id) continue } - log.Printf("reactor #%d receive output connection", r.id) - // get wrapper file descriptor - if wrapper.fd == nil { - file, err := wrapper.tcpConn.File() - if err != nil { - log.Println(err) - continue - } - fd := file.Fd() - wrapper.fd = &fd - } - log.Printf("reactor #%d ready to register event, wrapper fd: %v", r.id, uint64(*wrapper.fd)) + log.Printf("reactor #%d ready to register event, wrapper fd: %v", r.id, uint64(conn.fd)) - changes := []pevent{{ - connFd: uint64(*wrapper.fd), - operation: syscall.EVFILT_READ, - flag: syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT, // edge trigger mode - userData: (*byte)(unsafe.Pointer(wrapper)), + changes := []poller.Pevent{{ + ConnFd: uint64(conn.fd), + Operation: syscall.EVFILT_READ, + Flag: syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT, // edge trigger mode + UserData: (*byte)(unsafe.Pointer(conn)), }} - // register event to kqueue - if err := r.poller.register(changes); err != nil { - e := wrapper.tcpConn.Close() + // register event to poller + if err := r.p.Register(changes); err != nil { + e := conn.Close() if e != nil { err = errors.Wrap(err, e.Error()) } - log.Println(err, changes, wrapper.tcpConn.RemoteAddr(), wrapper.tcpConn.LocalAddr()) + // log.Println(err, changes, conn.RemoteAddr(), wrapper.tcpConn.LocalAddr()) + log.Println(err, changes) continue } @@ -116,34 +102,34 @@ func (r *reactor) run() { ctx, cancel := context.WithTimeout(context.Background(), processTimeout) r.srv.done.Add(2) r.srv.pool.Go(func() { - wrapper := (*connWrapper)(unsafe.Pointer(evt.userData)) - r.srv.handler(ctx, cancel, wrapper) + conn := (*Conn)(unsafe.Pointer(evt.UserData)) + r.srv.handler(ctx, cancel, conn) }) r.srv.pool.Go(func() { r.srv.notifier(ctx, cancel) }) } } } -func (r *reactor) input() chan<- *connWrapper { +func (r *reactor) input() chan<- *Conn { return r.connects } type waiter struct { - events chan pevent + events chan poller.Pevent parent *reactor - poller poller + p poller.Poller } func (w *waiter) run() { log.Printf("waiter #%d start", w.parent.id) defer w.parent.srv.done.Done() // event buf - evts := make([]pevent, 10) + evts := make([]poller.Pevent, 10) for { log.Printf("waiter #%d ready to wait event trigger", w.parent.id) // wait for event to be trigger - n, err := w.poller.wait(evts) + n, err := w.p.Wait(evts) if err != nil { log.Printf("waiter #%d stop, err: %v", w.parent.id, err) close(w.events) @@ -155,9 +141,9 @@ func (w *waiter) run() { for i := 0; i < n; i++ { evt := evts[i] switch { - // we do not care about eof - case evt.flag&syscall.EV_EOF != 0: - log.Printf("waiter #%d meet eof, skip", w.parent.id) + case evt.Flag&syscall.EV_EOF != 0: + conn := (*Conn)(unsafe.Pointer(evt.UserData)) + log.Printf("waiter #%d meet eof, server close connection, err: %v", w.parent.id, conn.Close()) default: w.events <- evt log.Printf("waiter #%d sent evt to reactor success", w.parent.id) @@ -166,84 +152,10 @@ func (w *waiter) run() { } } -func (w *waiter) output() <-chan pevent { +func (w *waiter) output() <-chan poller.Pevent { return w.events } -type pevent struct { - connFd uint64 - operation int64 - flag int64 - userData *byte -} - -type poller interface { - register(changes []pevent) error - wait(events []pevent) (int, error) - close() error -} - -type kqueuePoller struct { - kq *int -} - -func newKqueuePoller() (*kqueuePoller, error) { - kqFd, err := syscall.Kqueue() - if err != nil { - return nil, err - } - - return &kqueuePoller{ - kq: &kqFd, - }, nil -} - -func (kp *kqueuePoller) register(changes []pevent) error { - kchanges := kp.fromPevent(changes) - _, err := syscall.Kevent(*kp.kq, kchanges, nil, nil) - return err -} - -func (kp *kqueuePoller) wait(events []pevent) (int, error) { - kevents := kp.fromPevent(events) - n, err := syscall.Kevent(*kp.kq, nil, kevents, nil) - if err != nil { - return 0, err - } - kp.toPevent(kevents, events) - return n, nil -} - -func (kp *kqueuePoller) fromPevent(events []pevent) []syscall.Kevent_t { - kevents := make([]syscall.Kevent_t, 0, len(events)) - for _, pevt := range events { - kevents = append(kevents, syscall.Kevent_t{ - Ident: pevt.connFd, - Filter: int16(pevt.operation), - Flags: uint16(pevt.flag), - Udata: pevt.userData, - }) - } - return kevents -} - -func (kp *kqueuePoller) toPevent(kevents []syscall.Kevent_t, pevent []pevent) { - for idx, kevt := range kevents { - pevent[idx].connFd = kevt.Ident - pevent[idx].operation = int64(kevt.Filter) - pevent[idx].flag = int64(kevt.Flags) - pevent[idx].userData = kevt.Udata - } -} - -func (kq *kqueuePoller) close() error { - var err error - if kq.kq != nil { - err = syscall.Close(*kq.kq) - } - return err -} - // todo: extract to config const ( numReactor = 3 @@ -251,7 +163,7 @@ const ( processTimeout = 1 * time.Second ) -func NewReactorServer(addr string, handler simpleHandler) (*ReactorServer, error) { +func NewReactorServer(addr [4]byte, port int, handler simpleHandler) (*ReactorServer, error) { var err error srv := &ReactorServer{ bizHandler: handler, @@ -262,14 +174,14 @@ func NewReactorServer(addr string, handler simpleHandler) (*ReactorServer, error // init dispatcher srv.dp = &dispatcher{ - connections: make(chan *connWrapper), + connections: make(chan *Conn), parent: srv, } // init reactors for i := 0; i < numReactor; i++ { tmpIdx := int64(i) - kp, err := newKqueuePoller() + kp, err := poller.NewKqueuePoller() if err != nil { return nil, err } @@ -277,7 +189,7 @@ func NewReactorServer(addr string, handler simpleHandler) (*ReactorServer, error } // init server transport - srv.serverTransport, err = net.Listen("tcp", addr) + srv.serverTransport, err = Listen(addr, port) if err != nil { return nil, err } @@ -285,7 +197,7 @@ func NewReactorServer(addr string, handler simpleHandler) (*ReactorServer, error } type dispatcher struct { - connections chan *connWrapper + connections chan *Conn parent *ReactorServer } @@ -314,7 +226,7 @@ func (dp *dispatcher) run() { log.Println("dispatcher stop") } -func (dp *dispatcher) input() chan<- *connWrapper { +func (dp *dispatcher) input() chan<- *Conn { return dp.connections } @@ -343,31 +255,28 @@ func (rs *ReactorServer) Serve() error { return e } rs.mutex.Lock() - rs.dp.input() <- &connWrapper{ - tcpConn: conn.(*net.TCPConn), - fd: nil, - } + rs.dp.input() <- conn rs.mutex.Unlock() } } -func (rs *ReactorServer) handler(ctx context.Context, cancel context.CancelFunc, wrapper *connWrapper) { +func (rs *ReactorServer) handler(ctx context.Context, cancel context.CancelFunc, conn *Conn) { defer func() { rs.done.Done() cancel() }() - rs.bizHandler(ctx, wrapper.tcpConn) + rs.bizHandler(ctx, conn) rs.mutex.Lock() select { case <-rs.stop: rs.mutex.Unlock() - log.Printf("bizHandler ready to close connFd %d", *wrapper.fd) - if e := wrapper.tcpConn.Close(); e != nil { + log.Printf("bizHandler ready to close connFd %d", conn.fd) + if e := conn.Close(); e != nil { log.Printf("bizHandler close connection failed. err: %v", e) return } default: - rs.dp.input() <- wrapper // reuse long connection + rs.dp.input() <- conn // reuse long connection rs.mutex.Unlock() } } diff --git a/storage/server/server.go b/storage/server/server.go index f47b79c..3017b7c 100644 --- a/storage/server/server.go +++ b/storage/server/server.go @@ -12,7 +12,6 @@ import ( "github.com/Trinoooo/eggie_kv/storage/logs" "github.com/spf13/viper" "io" - "net" "net/http" ) @@ -157,4 +156,4 @@ func newSuccessResp(data []byte) *consts.KvResponse { } } -type simpleHandler func(ctx context.Context, conn net.Conn) +type simpleHandler func(ctx context.Context, conn *Conn) diff --git a/storage/server/server_test.go b/storage/server/server_test.go index e486523..251aad9 100644 --- a/storage/server/server_test.go +++ b/storage/server/server_test.go @@ -3,7 +3,6 @@ package server import ( "context" "encoding/binary" - "github.com/cloudwego/netpoll" "log" "net" "net/http" @@ -18,7 +17,7 @@ const ( fixSize = 8 - concurrency = 18 + concurrency = 30 ) func TestMain(m *testing.M) { @@ -61,20 +60,20 @@ func shortConnection(t *testing.T, buf []byte) { t.Error(e) return } - log.Println("[short] client send request successfully") + log.Println("[short] client send request successfully", conn.RemoteAddr(), conn.LocalAddr()) innerBuf := make([]byte, 8) _, e = conn.Read(innerBuf) if e != nil { t.Error(e) return } - log.Println("[short] client recv response successfully") + log.Println("[short] client recv response successfully", conn.RemoteAddr(), conn.LocalAddr()) e = conn.Close() if e != nil { t.Error(e) return } - log.Println("[short] client close connection") + log.Println("[short] client close connection", conn.RemoteAddr(), conn.LocalAddr()) } func longConnection(t *testing.T, buf []byte) { @@ -89,24 +88,24 @@ func longConnection(t *testing.T, buf []byte) { t.Error(e) return } - log.Println("[long] client send request successfully") + log.Println("[long] client send request successfully", conn.RemoteAddr(), conn.LocalAddr()) innerBuf := make([]byte, 8) _, e = conn.Read(innerBuf) if e != nil { t.Error(e) return } - log.Println("[long] client recv response successfully") + log.Println("[long] client recv response successfully", conn.RemoteAddr(), conn.LocalAddr()) } e := conn.Close() if e != nil { t.Error(e) return } - log.Println("[long] client close connection") + log.Println("[long] client close connection", conn.RemoteAddr(), conn.LocalAddr()) } -func commonHandler(conn net.Conn, t *testing.T) { +func commonHandler(conn *Conn, t *testing.T) { buf := make([]byte, fixSize) _, err := conn.Read(buf) if err != nil { @@ -131,46 +130,13 @@ func bizLogic(v uint64) uint64 { return v } -func TestPureGoroutineServer(t *testing.T) { - return - server, err := NewPureGoroutineServer(addr, func(ctx context.Context, conn net.Conn) { - commonHandler(conn, t) - }) - if err != nil { - t.Error(err) - return - } - go mockClient(t, server.Close) - err = server.Serve() - if err != nil { - t.Error(err) - } -} - -func TestNetpollEventLoopServer(t *testing.T) { - return - server, err := NewNetpollEventLoopServer(addr, func(ctx context.Context, connection netpoll.Connection) error { - commonHandler(connection, t) - return nil - }) - if err != nil { - t.Error(err) - return - } - go mockClient(t, server.Close) - err = server.Serve() - if err != nil { - t.Error(err) - } -} - func TestReactorServer(t *testing.T) { // 启动性能分析服务器 go func() { log.Println(http.ListenAndServe("localhost:6060", nil)) }() - server, err := NewReactorServer(addr, func(ctx context.Context, conn net.Conn) { + server, err := NewReactorServer([4]byte{127, 0, 0, 1}, 9999, func(ctx context.Context, conn *Conn) { commonHandler(conn, t) }) if err != nil { From 5f6367106438f0898c656586ab99205dae5d8911 Mon Sep 17 00:00:00 2001 From: Trino Date: Sat, 22 Jun 2024 01:26:34 +0800 Subject: [PATCH 03/15] =?UTF-8?q?fix=EF=BC=9A=E8=A7=A3=E4=BA=86=E4=BA=9Bbu?= =?UTF-8?q?g?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 增加prometheus打点监控，便于排查问题。没有使用在应用程序开个端口prometheus 服务器主动拉取这种模式，而是启动PushGateway作为sidecar中转，主要考虑是调试阶段 程序运行时间短，不好收集指标。 2. 修复runtime: marked free object in span 0x12a642b58, elemsize=32 freeindex=18 (bad use of unsafe.Pointer? try -d=checkptr)问题，原因是*conn通过unsafe.Pointer 转*byte会导致conn结构超过8字节的部分失去引用被gc（个人推测，还得深入研究runtime 包明确结论；至于为什么要用`udata = *(**byte)(unsafe.Pointer(&conn))`是因为 cloudwego/netpoll是这样用的） 3. syscall.INTR可能导致系统调用被中断，给waiter、reactor、acceptor这类遇到syscall 错误会崩溃的协程加特判 4. 给各环节间通信管道加buf，提高整体协作效率。但从accept qps看没有明显优化收益， 后续量化请求qps给个优化结论。 5. 由于accept性能瓶颈（1s获取10个链接）存在，高并发场景下会出现由于backlog满导致 大量请求被server端reset。后续动作考虑看下我的用法是否正确，以及池化链接对象那内存 分配回收性能收益。 后续动作：kqueue改成非阻塞水平触发，reactor负责协议解析，handler纯处理业务逻辑。 Signed-off-by: Trino --- go.mod | 9 ++++- go.sum | 16 ++++++++ storage/server/connections.go | 39 +++++++++++++++---- storage/server/metrics.go | 39 +++++++++++++++++++ storage/server/reactor_server.go | 65 ++++++++++++++++++++------------ storage/server/server_test.go | 21 ++--------- 6 files changed, 139 insertions(+), 50 deletions(-) create mode 100644 storage/server/metrics.go diff --git a/go.mod b/go.mod index 6a6d57e..50fd267 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,18 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -32,8 +38,9 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d992b60..7881d3c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3 h1:ZKUHguI38SRQJkq7hhmwn8lAv3xM6B5qkj1IneS15YY= github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= @@ -35,6 +39,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -80,11 +92,15 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= diff --git a/storage/server/connections.go b/storage/server/connections.go index 5ba4b84..f228446 100644 --- a/storage/server/connections.go +++ b/storage/server/connections.go @@ -1,9 +1,14 @@ package server -import "syscall" +import ( + "net" + "syscall" +) type Conn struct { - fd int + fd int + laddr *syscall.SockaddrInet4 + raddr *syscall.SockaddrInet4 } func (c *Conn) Read(buf []byte) (int, error) { @@ -18,12 +23,28 @@ func (c *Conn) Close() error { return syscall.Close(c.fd) } +func (c *Conn) RemoteAddr() net.Addr { + ipv4 := net.IPv4(c.raddr.Addr[0], c.raddr.Addr[1], c.raddr.Addr[2], c.raddr.Addr[3]) + return &net.TCPAddr{ + IP: ipv4, + Port: c.raddr.Port, + } +} + +func (c *Conn) LocalAddr() net.Addr { + ipv4 := net.IPv4(c.laddr.Addr[0], c.laddr.Addr[1], c.laddr.Addr[2], c.laddr.Addr[3]) + return &net.TCPAddr{ + IP: ipv4, + Port: c.laddr.Port, + } +} + type Listener struct { conn *Conn } func (l *Listener) Accept() (*Conn, error) { - socket, _, err := syscall.Accept(l.conn.fd) + socket, sa, err := syscall.Accept(l.conn.fd) if err != nil { return nil, err } @@ -33,7 +54,9 @@ func (l *Listener) Accept() (*Conn, error) { } return &Conn{ - fd: socket, + fd: socket, + laddr: l.conn.laddr, + raddr: sa.(*syscall.SockaddrInet4), }, nil } @@ -47,10 +70,11 @@ func Listen(addr [4]byte, port int) (*Listener, error) { return nil, err } - if err = syscall.Bind(fd, &syscall.SockaddrInet4{ + laddr := &syscall.SockaddrInet4{ Port: port, Addr: addr, - }); err != nil { + } + if err = syscall.Bind(fd, laddr); err != nil { return nil, err } @@ -60,7 +84,8 @@ func Listen(addr [4]byte, port int) (*Listener, error) { return &Listener{ conn: &Conn{ - fd: fd, + fd: fd, + laddr: laddr, }, }, nil } diff --git a/storage/server/metrics.go b/storage/server/metrics.go new file mode 100644 index 0000000..09ea3b6 --- /dev/null +++ b/storage/server/metrics.go @@ -0,0 +1,39 @@ +package server + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/push" + "log" + "time" +) + +type MetricsHelper struct { + ConnectionAcceptCounter prometheus.Counter // socket accept qps +} + +func NewMetricsHelper() *MetricsHelper { + connectionAcceptCounter := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "eggie_kv_connection_accept_counter", + }) + registry := prometheus.NewRegistry() + registry.MustRegister( + connectionAcceptCounter, + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + pusher := push.New("http://localhost:9091", "eggie_kv").Gatherer(registry) + go func() { + for { + if err := pusher.Push(); err != nil { + log.Printf("prometheus pusher push failed. err: %v", err) + } + // push every 5 ms + time.Sleep(5 * time.Millisecond) + } + }() + + return &MetricsHelper{ + ConnectionAcceptCounter: connectionAcceptCounter, + } +} diff --git a/storage/server/reactor_server.go b/storage/server/reactor_server.go index 4bb4662..e454477 100644 --- a/storage/server/reactor_server.go +++ b/storage/server/reactor_server.go @@ -22,6 +22,7 @@ type ReactorServer struct { reactors map[int64]*reactor stop chan struct{} done sync.WaitGroup + metricsHelper *MetricsHelper } type reactor struct { @@ -36,10 +37,10 @@ func newReactor(id int64, srv *ReactorServer, p poller.Poller) *reactor { r := &reactor{ srv: srv, id: id, - connects: make(chan *Conn), + connects: make(chan *Conn, reactorInputBufferSize), p: p, w: &waiter{ - events: make(chan poller.Pevent), + events: make(chan poller.Pevent, waiterOutputBufferSize), p: p, }, } @@ -57,7 +58,6 @@ func (r *reactor) run() { for { select { case conn, ok := <-connects: - log.Printf("reactor #%d receive output connection", r.id) // output been close by dispatcher if !ok { log.Printf("reactor #%d ready to closes poller", r.id) @@ -69,32 +69,31 @@ func (r *reactor) run() { continue } - log.Printf("reactor #%d ready to register event, wrapper fd: %v", r.id, uint64(conn.fd)) + log.Printf("reactor #%d ready to register event, remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) changes := []poller.Pevent{{ ConnFd: uint64(conn.fd), Operation: syscall.EVFILT_READ, Flag: syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT, // edge trigger mode - UserData: (*byte)(unsafe.Pointer(conn)), + UserData: *(**byte)(unsafe.Pointer(&conn)), // bugfix：conn和byte内存大小不同，直接指针转换会有gc marked free object in span 风险 }} // register event to poller - if err := r.p.Register(changes); err != nil { + if err := r.p.Register(changes); err != nil && err != syscall.EINTR { e := conn.Close() if e != nil { err = errors.Wrap(err, e.Error()) } - // log.Println(err, changes, conn.RemoteAddr(), wrapper.tcpConn.LocalAddr()) - log.Println(err, changes) + log.Println(err, changes, conn, conn.RemoteAddr(), conn.LocalAddr()) continue } - - log.Printf("reactor #%d register event success", r.id) + log.Printf("reactor #%d register event success, evt remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) case evt, ok := <-r.w.output(): if !ok { log.Printf("reactor #%d stop", r.id) return } - log.Printf("reactor #%d handle event", r.id) + conn := *(**Conn)(unsafe.Pointer(&evt.UserData)) + log.Printf("reactor #%d handle event, event remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) // in this case, close is already called. if r.srv.pool == nil { continue @@ -102,7 +101,6 @@ func (r *reactor) run() { ctx, cancel := context.WithTimeout(context.Background(), processTimeout) r.srv.done.Add(2) r.srv.pool.Go(func() { - conn := (*Conn)(unsafe.Pointer(evt.UserData)) r.srv.handler(ctx, cancel, conn) }) r.srv.pool.Go(func() { r.srv.notifier(ctx, cancel) }) @@ -130,7 +128,7 @@ func (w *waiter) run() { // wait for event to be trigger n, err := w.p.Wait(evts) - if err != nil { + if err != nil && err != syscall.EINTR { // bugfix: ignore EINTR log.Printf("waiter #%d stop, err: %v", w.parent.id, err) close(w.events) return @@ -140,13 +138,15 @@ func (w *waiter) run() { for i := 0; i < n; i++ { evt := evts[i] + conn := *(**Conn)(unsafe.Pointer(&evt.UserData)) switch { case evt.Flag&syscall.EV_EOF != 0: - conn := (*Conn)(unsafe.Pointer(evt.UserData)) - log.Printf("waiter #%d meet eof, server close connection, err: %v", w.parent.id, conn.Close()) + log.Printf("waiter #%d meet eof, remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + log.Printf("waiter #%d close server connection, err: %v", w.parent.id, conn.Close()) default: + log.Printf("waiter #%d ready to send evt to reactor, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) w.events <- evt - log.Printf("waiter #%d sent evt to reactor success", w.parent.id) + log.Printf("waiter #%d sent evt to reactor success, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) } } } @@ -161,20 +161,25 @@ const ( numReactor = 3 pollCapacity = 1000 processTimeout = 1 * time.Second + + dispatcherInputBufferSize = 10 + reactorInputBufferSize = 10 + waiterOutputBufferSize = 10 ) func NewReactorServer(addr [4]byte, port int, handler simpleHandler) (*ReactorServer, error) { var err error srv := &ReactorServer{ - bizHandler: handler, - reactors: make(map[int64]*reactor), - pool: gopool.NewPool("handlers", pollCapacity, gopool.NewConfig()), - stop: make(chan struct{}), + bizHandler: handler, + reactors: make(map[int64]*reactor), + pool: gopool.NewPool("handlers", pollCapacity, gopool.NewConfig()), + stop: make(chan struct{}), + metricsHelper: NewMetricsHelper(), } // init dispatcher srv.dp = &dispatcher{ - connections: make(chan *Conn), + connections: make(chan *Conn, dispatcherInputBufferSize), parent: srv, } @@ -206,12 +211,12 @@ func (dp *dispatcher) run() { log.Println("dispatcher start") for conn := range dp.connections { - log.Println("dispatcher receive connection") + log.Printf("dispatcher receive connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.fd) // random load balance id := rand.Int63n(numReactor) if reactor, exist := dp.parent.reactors[id]; exist { reactor.input() <- conn - log.Printf("dispatcher send connection to reactor #%v success", id) + log.Printf("dispatcher send connection to reactor #%v success, remote addr: %v, local addr: %v, fd: %v", id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) } else { log.Printf("dispatcher find reactor #%v not exist", id) } @@ -239,10 +244,12 @@ func (rs *ReactorServer) Serve() error { rs.pool.Go(reactor.run) } - // mainReactor & acceptor + // acceptor for { + log.Printf("acceptor ready to accept connections") conn, err := rs.serverTransport.Accept() - if err != nil { + if err != nil && err != syscall.EINTR { + log.Printf("error occur when accept connection, err: %v", err) rs.mutex.Lock() close(rs.dp.input()) e := rs.close() @@ -254,8 +261,12 @@ func (rs *ReactorServer) Serve() error { rs.mutex.Unlock() return e } + log.Printf("acceptor accept connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + rs.metricsHelper.ConnectionAcceptCounter.Inc() rs.mutex.Lock() + log.Printf("acceptor ready to send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.fd) rs.dp.input() <- conn + log.Printf("acceptor success send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.fd) rs.mutex.Unlock() } } @@ -265,7 +276,9 @@ func (rs *ReactorServer) handler(ctx context.Context, cancel context.CancelFunc, rs.done.Done() cancel() }() + log.Printf("bizHandler ready to process socket fd: %d, remote addr: %v, local addr: %v", conn.fd, conn.RemoteAddr(), conn.LocalAddr()) rs.bizHandler(ctx, conn) + log.Printf("bizHandler success process socket fd: %d, remote addr: %v, local addr: %v", conn.fd, conn.RemoteAddr(), conn.LocalAddr()) rs.mutex.Lock() select { case <-rs.stop: @@ -276,7 +289,9 @@ func (rs *ReactorServer) handler(ctx context.Context, cancel context.CancelFunc, return } default: + log.Printf("bizHandler ready to send back socket fd: %d, remote addr: %v, local addr: %v", conn.fd, conn.RemoteAddr(), conn.LocalAddr()) rs.dp.input() <- conn // reuse long connection + log.Printf("bizHandler success send back socket fd: %d, remote addr: %v, local addr: %v", conn.fd, conn.RemoteAddr(), conn.LocalAddr()) rs.mutex.Unlock() } } diff --git a/storage/server/server_test.go b/storage/server/server_test.go index 251aad9..2655b82 100644 --- a/storage/server/server_test.go +++ b/storage/server/server_test.go @@ -17,7 +17,7 @@ const ( fixSize = 8 - concurrency = 30 + concurrency = 100 ) func TestMain(m *testing.M) { @@ -60,6 +60,7 @@ func shortConnection(t *testing.T, buf []byte) { t.Error(e) return } + log.Println("[short] client send request successfully", conn.RemoteAddr(), conn.LocalAddr()) innerBuf := make([]byte, 8) _, e = conn.Read(innerBuf) @@ -108,12 +109,14 @@ func longConnection(t *testing.T, buf []byte) { func commonHandler(conn *Conn, t *testing.T) { buf := make([]byte, fixSize) _, err := conn.Read(buf) + log.Println("server recv", binary.BigEndian.Uint64(buf), conn.LocalAddr(), conn.RemoteAddr(), conn.fd) if err != nil { t.Error(err) } v := binary.BigEndian.Uint64(buf) v = bizLogic(v) binary.BigEndian.PutUint64(buf, v) + log.Println("server send", binary.BigEndian.Uint64(buf), conn.LocalAddr(), conn.RemoteAddr(), conn.fd) _, err = conn.Write(buf) if err != nil { t.Error(err) @@ -150,19 +153,3 @@ func TestReactorServer(t *testing.T) { } } - -func TestProactorServer(t *testing.T) { - //todo -} - -func BenchmarkNewNetpollEventLoopServer(b *testing.B) { - -} - -func BenchmarkNewPureGoroutineServer(b *testing.B) { - -} - -func BenchmarkNewReactorServer(b *testing.B) { - -} From 8aebae562436e5403c1e0f0292090991ef424128 Mon Sep 17 00:00:00 2001 From: Trino Date: Sun, 23 Jun 2024 18:30:44 +0800 Subject: [PATCH 04/15] =?UTF-8?q?feat:=20kqueue=E6=94=B9=E6=88=90=E6=B0=B4?= =?UTF-8?q?=E5=B9=B3=E8=A7=A6=E5=8F=91=E6=A8=A1=E5=BC=8F=EF=BC=9B=E5=B0=86?= =?UTF-8?q?=E5=8D=8F=E8=AE=AE=E7=BC=96=E8=A7=A3=E7=A0=81=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E5=A7=94=E6=89=98=20=E7=BB=99reactor=EF=BC=9B=E4=B8=8D?= =?UTF-8?q?=E4=BD=BF=E7=94=A8kqueue=E7=9A=84udata=E5=AD=98=E6=94=BEconn?= =?UTF-8?q?=EF=BC=8C=E5=9B=A0=E4=B8=BA=E4=BC=9A=E6=9C=89=E5=A5=87=E6=80=AA?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Trino --- .github/workflows/test.yaml | 2 +- errs/error.go | 10 + go.mod | 3 +- go.sum | 10 +- storage/server/connections.go | 91 ------ storage/server/connections/iface.go | 18 ++ storage/server/connections/impl_bsd_arm64.go | 97 +++++++ storage/server/handler.go | 37 ++- storage/server/metrics.go | 2 +- storage/server/mw.go | 43 --- storage/server/processor.go | 289 +++++++++++++++++++ storage/server/protocol/binary.go | 94 ++++++ storage/server/protocol/iface.go | 27 ++ storage/server/reactor_server.go | 190 ++++++------ storage/server/server.go | 159 ---------- storage/server/server_test.go | 55 ++-- 16 files changed, 678 insertions(+), 449 deletions(-) delete mode 100644 storage/server/connections.go create mode 100644 storage/server/connections/iface.go create mode 100644 storage/server/connections/impl_bsd_arm64.go delete mode 100644 storage/server/mw.go create mode 100644 storage/server/processor.go create mode 100644 storage/server/protocol/binary.go create mode 100644 storage/server/protocol/iface.go delete mode 100644 storage/server/server.go diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9c99703..833aab0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v2 - name: Run tests - run: make -f makefile test-with-cover TestPackage=./storage/core/ragdoll/wal + run: make -f makefile test-with-cover TestPackage=./storage - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.0.1 diff --git a/errs/error.go b/errs/error.go index 50c5a17..d5df2de 100644 --- a/errs/error.go +++ b/errs/error.go @@ -82,6 +82,8 @@ const ( CoreNotFoundErrCode = 100035 ReadSocketErrCode = 100036 WriteSocketErrCode = 100037 + UnexpectHandlerErrCode = 100038 + TaskNotFinishErrCode = 100039 ) func NewUnknownErr() *KvErr { @@ -239,3 +241,11 @@ func NewReadSocketErr() *KvErr { func NewWriteSocketErr() *KvErr { return &KvErr{msg: "write socket failed", code: WriteSocketErrCode} } + +func NewUnexpectHandler() *KvErr { + return &KvErr{msg: "unexpect handler key", code: UnexpectHandlerErrCode} +} + +func NewTaskNotFinishErr() *KvErr { + return &KvErr{msg: "task not finish yet, try again.", code: TaskNotFinishErrCode} +} diff --git a/go.mod b/go.mod index 50fd267..87af61e 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,10 @@ go 1.19 require ( github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3 github.com/chzyer/readline v1.5.1 - github.com/cloudwego/netpoll v0.6.1 github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.19.1 github.com/spf13/viper v1.18.2 github.com/urfave/cli/v2 v2.26.0 go.uber.org/zap v1.26.0 @@ -23,7 +23,6 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect diff --git a/go.sum b/go.sum index 7881d3c..ea50f1f 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,6 @@ github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= -github.com/cloudwego/netpoll v0.6.1 h1:Cjftvi6bmumsOijmuUFy6HqAUXMxAT3fKK96wsrm3XA= -github.com/cloudwego/netpoll v0.6.1/go.mod h1:kaqvfZ70qd4T2WtIIpCOi5Cxyob8viEpzLhCrTrz3HM= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -20,7 +18,7 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -47,7 +45,7 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -90,8 +88,6 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -102,7 +98,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/storage/server/connections.go b/storage/server/connections.go deleted file mode 100644 index f228446..0000000 --- a/storage/server/connections.go +++ /dev/null @@ -1,91 +0,0 @@ -package server - -import ( - "net" - "syscall" -) - -type Conn struct { - fd int - laddr *syscall.SockaddrInet4 - raddr *syscall.SockaddrInet4 -} - -func (c *Conn) Read(buf []byte) (int, error) { - return syscall.Read(c.fd, buf) -} - -func (c *Conn) Write(buf []byte) (int, error) { - return syscall.Write(c.fd, buf) -} - -func (c *Conn) Close() error { - return syscall.Close(c.fd) -} - -func (c *Conn) RemoteAddr() net.Addr { - ipv4 := net.IPv4(c.raddr.Addr[0], c.raddr.Addr[1], c.raddr.Addr[2], c.raddr.Addr[3]) - return &net.TCPAddr{ - IP: ipv4, - Port: c.raddr.Port, - } -} - -func (c *Conn) LocalAddr() net.Addr { - ipv4 := net.IPv4(c.laddr.Addr[0], c.laddr.Addr[1], c.laddr.Addr[2], c.laddr.Addr[3]) - return &net.TCPAddr{ - IP: ipv4, - Port: c.laddr.Port, - } -} - -type Listener struct { - conn *Conn -} - -func (l *Listener) Accept() (*Conn, error) { - socket, sa, err := syscall.Accept(l.conn.fd) - if err != nil { - return nil, err - } - - if err = syscall.SetNonblock(socket, true); err != nil { - return nil, err - } - - return &Conn{ - fd: socket, - laddr: l.conn.laddr, - raddr: sa.(*syscall.SockaddrInet4), - }, nil -} - -func (l *Listener) Close() error { - return syscall.Close(l.conn.fd) -} - -func Listen(addr [4]byte, port int) (*Listener, error) { - fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) - if err != nil { - return nil, err - } - - laddr := &syscall.SockaddrInet4{ - Port: port, - Addr: addr, - } - if err = syscall.Bind(fd, laddr); err != nil { - return nil, err - } - - if err = syscall.Listen(fd, syscall.SOMAXCONN); err != nil { - return nil, err - } - - return &Listener{ - conn: &Conn{ - fd: fd, - laddr: laddr, - }, - }, nil -} diff --git a/storage/server/connections/iface.go b/storage/server/connections/iface.go new file mode 100644 index 0000000..3e46aa4 --- /dev/null +++ b/storage/server/connections/iface.go @@ -0,0 +1,18 @@ +package connections + +import ( + "io" + "net" +) + +type IListener interface { + Accept() (IConnection, error) + io.Closer +} + +type IConnection interface { + io.ReadWriteCloser + RemoteAddr() net.Addr + LocalAddr() net.Addr + RawFd() int +} diff --git a/storage/server/connections/impl_bsd_arm64.go b/storage/server/connections/impl_bsd_arm64.go new file mode 100644 index 0000000..3b6f2d6 --- /dev/null +++ b/storage/server/connections/impl_bsd_arm64.go @@ -0,0 +1,97 @@ +package connections + +import ( + "net" + "syscall" +) + +const maxSoMaxConn = 500 + +type Connection struct { + fd int + localAddr *syscall.SockaddrInet4 + remoteAddr *syscall.SockaddrInet4 +} + +func (c *Connection) Read(buf []byte) (int, error) { + return syscall.Read(c.fd, buf) +} + +func (c *Connection) Write(buf []byte) (int, error) { + return syscall.Write(c.fd, buf) +} + +func (c *Connection) Close() error { + return syscall.Close(c.fd) +} + +func (c *Connection) RemoteAddr() net.Addr { + ipv4 := net.IPv4(c.remoteAddr.Addr[0], c.remoteAddr.Addr[1], c.remoteAddr.Addr[2], c.remoteAddr.Addr[3]) + return &net.TCPAddr{ + IP: ipv4, + Port: c.remoteAddr.Port, + } +} + +func (c *Connection) LocalAddr() net.Addr { + ipv4 := net.IPv4(c.localAddr.Addr[0], c.localAddr.Addr[1], c.localAddr.Addr[2], c.localAddr.Addr[3]) + return &net.TCPAddr{ + IP: ipv4, + Port: c.localAddr.Port, + } +} + +func (c *Connection) RawFd() int { + return c.fd +} + +type Listener struct { + conn *Connection +} + +func (l *Listener) Accept() (IConnection, error) { + socket, sa, err := syscall.Accept(l.conn.fd) + if err != nil { + return nil, err + } + + if err = syscall.SetNonblock(socket, true); err != nil { + return nil, err + } + + return &Connection{ + fd: socket, + localAddr: l.conn.localAddr, + remoteAddr: sa.(*syscall.SockaddrInet4), + }, nil +} + +func (l *Listener) Close() error { + return syscall.Close(l.conn.fd) +} + +func Listen(addr [4]byte, port int) (IListener, error) { + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != nil { + return nil, err + } + + laddr := &syscall.SockaddrInet4{ + Port: port, + Addr: addr, + } + if err = syscall.Bind(fd, laddr); err != nil { + return nil, err + } + + if err = syscall.Listen(fd, maxSoMaxConn); err != nil { + return nil, err + } + + return &Listener{ + conn: &Connection{ + fd: fd, + localAddr: laddr, + }, + }, nil +} diff --git a/storage/server/handler.go b/storage/server/handler.go index 5067fe9..a0522f0 100644 --- a/storage/server/handler.go +++ b/storage/server/handler.go @@ -1,21 +1,36 @@ package server import ( - "github.com/Trinoooo/eggie_kv/consts" + "encoding/binary" + "log" + "time" ) -func (srv *Server) HandleGet(req *consts.KvRequest) (*consts.KvResponse, error) { - value, err := srv.core.Get(string(req.Key)) - if err != nil { - return nil, err +func HandleGet(req *KvRequest) (*KvResponse, error) { + resp := &KvResponse{ + Data: make([]byte, 8), } - return newSuccessResp(value), nil + log.Printf("HandleGet kvRequest: %#v", req) + v := bizLogic(binary.BigEndian.Uint64(req.Value)) + binary.BigEndian.PutUint64(resp.Data, v) + log.Printf("HandleGet kvResponse: %#v", resp) + return resp, nil } -func (srv *Server) HandleSet(req *consts.KvRequest) (*consts.KvResponse, error) { - err := srv.core.Set(string(req.Key), req.Value) - if err != nil { - return nil, err +func HandleSet(req *KvRequest) (*KvResponse, error) { + resp := &KvResponse{} + log.Printf("HandleSet kvRequest: %#v", req) + bizLogic(binary.BigEndian.Uint64(req.Value)) + log.Printf("HandleSet kvResponse: %#v", resp) + return resp, nil +} + +func bizLogic(v uint64) uint64 { + // 模拟耗时cpu密集性计算 + for i := 0; i < 1000000; i++ { + v += uint64(i) } - return newSuccessResp(nil), nil + // 模拟耗时io密集性操作 + time.Sleep(1 * time.Second) + return v } diff --git a/storage/server/metrics.go b/storage/server/metrics.go index 09ea3b6..ce7d782 100644 --- a/storage/server/metrics.go +++ b/storage/server/metrics.go @@ -25,7 +25,7 @@ func NewMetricsHelper() *MetricsHelper { pusher := push.New("http://localhost:9091", "eggie_kv").Gatherer(registry) go func() { for { - if err := pusher.Push(); err != nil { + if err := pusher.Add(); err != nil { log.Printf("prometheus pusher push failed. err: %v", err) } // push every 5 ms diff --git a/storage/server/mw.go b/storage/server/mw.go deleted file mode 100644 index 451c768..0000000 --- a/storage/server/mw.go +++ /dev/null @@ -1,43 +0,0 @@ -package server - -import ( - "fmt" - "github.com/Trinoooo/eggie_kv/consts" - "github.com/Trinoooo/eggie_kv/errs" - "github.com/Trinoooo/eggie_kv/storage/core/ragdoll/logs" - "github.com/luci/go-render/render" - "go.uber.org/zap" -) - -type HandleFunc func(request *consts.KvRequest) (*consts.KvResponse, error) - -type MiddlewareFunc func(handleFn HandleFunc) HandleFunc - -func LogMw(handleFn HandleFunc) HandleFunc { - return func(req *consts.KvRequest) (*consts.KvResponse, error) { - logs.Info(fmt.Sprintf("req: %#v", render.Render(req))) - resp, err := handleFn(req) - logs.Info(fmt.Sprintf("resp: %#v, errs: %#v", render.Render(resp), err)) - return resp, err - } -} - -func ParamsValidateMw(handleFn HandleFunc) HandleFunc { - return func(req *consts.KvRequest) (*consts.KvResponse, error) { - reqKeyLength := len(req.Key) - if reqKeyLength <= 0 || reqKeyLength > consts.KB { - e := errs.NewInvalidParamErr() - logs.Error(e.Error(), zap.String(consts.LogFieldParams, "reqKeyLength"), zap.Int(consts.LogFieldValue, reqKeyLength)) - return newExceptionResp(e), e - } - - reqValueLength := len(req.Value) - if reqValueLength < 0 || reqValueLength > consts.MB { - e := errs.NewInvalidParamErr() - logs.Error(e.Error(), zap.String(consts.LogFieldParams, "reqValueLength"), zap.Int(consts.LogFieldValue, reqValueLength)) - return newExceptionResp(e), e - } - - return handleFn(req) - } -} diff --git a/storage/server/processor.go b/storage/server/processor.go new file mode 100644 index 0000000..f18c574 --- /dev/null +++ b/storage/server/processor.go @@ -0,0 +1,289 @@ +package server + +import ( + "errors" + "github.com/Trinoooo/eggie_kv/errs" + "github.com/Trinoooo/eggie_kv/storage/server/protocol" + "log" + "strings" + "sync" + "syscall" +) + +type OpType int64 + +const ( + OpTypeUnknown OpType = 0 + OpTypeGet OpType = 1 + OpTypeSet OpType = 2 +) + +type KvRequest struct { + OperationType OpType `json:"operation_type"` + Key []byte `json:"key"` + Value []byte `json:"value"` +} + +type KvResponse struct { + Data []byte `json:"data"` +} + +type KvException struct { + Code int64 `json:"code"` + Message string `json:"message"` +} + +type HandlerFunc func(req *KvRequest) (*KvResponse, error) + +type Processor struct { + srv *ReactorServer + inputProtocol protocol.IProtocol + outputProtocol protocol.IProtocol + handlers map[string]HandlerFunc + stepState map[string]bool + task *Task + handlerTriggerOnce sync.Once +} + +func NewProcessor(srv *ReactorServer, inputProtocol, outputProtocol protocol.IProtocol) *Processor { + return &Processor{ + srv: srv, + inputProtocol: inputProtocol, + outputProtocol: outputProtocol, + handlers: map[string]HandlerFunc{ + "HandleGet": HandleGet, + "HandleSet": HandleSet, + }, + stepState: map[string]bool{ + "START": true, + }, + task: &Task{ + req: &KvRequest{}, + ready: make(chan struct{}), + }, + } +} + +func (p *Processor) Process() error { + if err := p.decodeHandler(); err != nil { + return err + } + + if err := p.decodeRequest(); err != nil { + return err + } + + if err := p.triggerHandler(); err != nil { + return err + } + + if p.task.resp != nil { + if err := p.encodeResponse(); err != nil { + return err + } + } else if p.task.exception != nil { + if err := p.encodeException(); err != nil { + return err + } + } + + return p.resetState() // for long connection reuse +} + +func (p *Processor) GetInputProtocol() protocol.IProtocol { + return p.inputProtocol +} + +func (p *Processor) GetOutputProtocol() protocol.IProtocol { + return p.outputProtocol +} + +func (p *Processor) decodeHandler() error { + return p.checkOrSetState("DecodeRequestHandler", []string{"START"}, func() (bool, error) { + handlerKey, err := p.inputProtocol.ReadString() + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { + return false, errs.NewTaskNotFinishErr() + } else if err != nil { + return false, err + } + + handler, exist := p.handlers[handlerKey] + if !exist { + return false, errs.NewUnexpectHandler() + } + p.task.handler = handler + return true, nil + }) +} + +func (p *Processor) decodeRequest() error { + if err := p.checkOrSetState("DecodeRequestOpType", []string{"DecodeRequestHandler"}, func() (bool, error) { + opTypeI64, err := p.inputProtocol.ReadI64() + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { + return false, errs.NewTaskNotFinishErr() + } else if err != nil { + return false, err + } + p.task.req.OperationType = OpType(opTypeI64) + return true, nil + }); err != nil { + return err + } + + if err := p.checkOrSetState("DecodeRequestKey", []string{"DecodeRequestOpType"}, func() (bool, error) { + key, err := p.inputProtocol.ReadBytes() + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { + return false, errs.NewTaskNotFinishErr() + } else if err != nil { + return false, err + } + p.task.req.Key = key + return true, nil + }); err != nil { + return err + } + + if err := p.checkOrSetState("DecodeRequestValue", []string{"DecodeRequestKey"}, func() (bool, error) { + value, err := p.inputProtocol.ReadBytes() + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { + return false, errs.NewTaskNotFinishErr() + } else if err != nil { + return false, err + } + p.task.req.Value = value + return true, nil + }); err != nil { + return err + } + return nil +} + +func (p *Processor) triggerHandler() error { + return p.checkOrSetState("TriggerHandler", []string{"DecodeRequestValue"}, func() (bool, error) { + select { + case <-p.task.ready: + return true, nil + default: + p.handlerTriggerOnce.Do(func() { + p.srv.done.Add(1) + p.srv.pool.Go(func() { + defer p.srv.done.Done() + defer close(p.task.ready) + p.task.Execute() + }) + }) + return false, errs.NewTaskNotFinishErr() + } + }) +} + +func (p *Processor) encodeResponse() error { + return p.checkOrSetState("EncodeResponseData", []string{"TriggerHandler"}, func() (bool, error) { + err := p.outputProtocol.WriteBytes(p.task.resp.Data) + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { + return false, errs.NewTaskNotFinishErr() + } else if err != nil { + return false, err + } + return true, nil + }) +} + +func (p *Processor) encodeException() error { + if err := p.checkOrSetState("EncodeExceptionCode", []string{"TriggerHandler"}, func() (bool, error) { + err := p.outputProtocol.WriteI64(p.task.exception.Code) + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { + return false, errs.NewTaskNotFinishErr() + } else if err != nil { + return false, err + } + return true, nil + }); err != nil { + return err + } + + if err := p.checkOrSetState("EncodeExceptionMessage", []string{"EncodeExceptionCode"}, func() (bool, error) { + err := p.outputProtocol.WriteString(p.task.exception.Message) + if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { + return false, errs.NewTaskNotFinishErr() + } else if err != nil { + return false, err + } + return true, nil + }); err != nil { + return err + } + + return nil +} + +func (p *Processor) resetState() error { + p.stepState = map[string]bool{ + "START": true, + } + p.task = &Task{ + req: &KvRequest{}, + ready: make(chan struct{}), + } + p.handlerTriggerOnce = sync.Once{} + return nil +} + +func (p *Processor) checkOrSetState(currentStepKey string, previousStepKeys []string, fn func() (bool, error)) error { + // previous step not finish, we should tell caller to retry. + var previousStepFinish bool + for _, previousStepKey := range previousStepKeys { + if psf := p.stepState[previousStepKey]; psf { + previousStepFinish = true + break + } + } + if !previousStepFinish { + connection := p.inputProtocol.GetConnection() + log.Printf("previous step %v not finish, retry. remote addr: %v, local addr: %v, fd: %v", strings.Join(previousStepKeys, ","), connection.RemoteAddr(), connection.LocalAddr(), connection.RawFd()) + return errs.NewTaskNotFinishErr() + } + + // current step has already finished. we should skip to avoid execute `fn` more than once. + if currentStepFinish := p.stepState[currentStepKey]; currentStepFinish { + connection := p.inputProtocol.GetConnection() + log.Printf("current step %v alreay finished, skip. remote addr: %v, local addr: %v, fd: %v", currentStepKey, connection.RemoteAddr(), connection.LocalAddr(), connection.RawFd()) + return nil + } + + isFinish, err := fn() + if err != nil { + return err + } + + p.stepState[currentStepKey] = isFinish + return nil +} + +type Task struct { + handler HandlerFunc + req *KvRequest + resp *KvResponse + exception *KvException + ready chan struct{} +} + +func (t *Task) Execute() { + resp, err := t.handler(t.req) + if err != nil { + log.Printf("handler return error: %v", err) + var kvErr *errs.KvErr + if errors.As(err, &kvErr) { + t.exception = &KvException{ + Code: kvErr.Code(), + Message: kvErr.Error(), + } + } else { + t.exception = &KvException{ + Code: errs.UnknownErrCode, + Message: err.Error(), + } + } + } + t.resp = resp +} diff --git a/storage/server/protocol/binary.go b/storage/server/protocol/binary.go new file mode 100644 index 0000000..a2f5517 --- /dev/null +++ b/storage/server/protocol/binary.go @@ -0,0 +1,94 @@ +package protocol + +import ( + "encoding/binary" + "github.com/Trinoooo/eggie_kv/storage/server/connections" +) + +type BinaryProtocol struct { + transport connections.IConnection + buf [64]byte // inline array. for memory allocate optimization +} + +func NewBinaryProtocol(transport connections.IConnection) *BinaryProtocol { + return &BinaryProtocol{ + transport: transport, + buf: [64]byte{}, + } +} + +func (cbp *BinaryProtocol) GetConnection() connections.IConnection { + return cbp.transport +} + +func (cbp *BinaryProtocol) ReadStructBegin() error { + return nil +} + +func (cbp *BinaryProtocol) ReadStructEnd() error { + return nil +} + +func (cbp *BinaryProtocol) ReadI64() (int64, error) { + i64Bytes := cbp.buf[:8] + _, err := cbp.transport.Read(i64Bytes) + if err != nil { + return 0, err + } + return int64(binary.BigEndian.Uint64(i64Bytes)), nil +} + +func (cbp *BinaryProtocol) ReadString() (string, error) { + bytes, err := cbp.ReadBytes() + return string(bytes), err +} + +func (cbp *BinaryProtocol) ReadBytes() ([]byte, error) { + length, err := cbp.ReadI64() + if err != nil { + return nil, err + } + + var buf []byte + if length <= 64 { + buf = cbp.buf[:length] + } else { + buf = make([]byte, 0, length) + } + + _, err = cbp.transport.Read(buf) + if err != nil { + return nil, err + } + return buf, nil +} + +func (cbp *BinaryProtocol) WriteStructBegin() error { + return nil +} + +func (cbp *BinaryProtocol) WriteStructEnd() error { + return nil +} + +func (cbp *BinaryProtocol) WriteI64(v int64) error { + i64Bytes := cbp.buf[:8] + binary.BigEndian.PutUint64(i64Bytes, uint64(v)) + _, err := cbp.transport.Write(i64Bytes) + return err +} + +func (cbp *BinaryProtocol) WriteString(v string) error { + return cbp.WriteBytes([]byte(v)) +} + +func (cbp *BinaryProtocol) WriteBytes(v []byte) error { + length := int64(len(v)) + err := cbp.WriteI64(length) + if err != nil { + return err + } + + _, err = cbp.transport.Write(v) + return err +} diff --git a/storage/server/protocol/iface.go b/storage/server/protocol/iface.go new file mode 100644 index 0000000..703c95e --- /dev/null +++ b/storage/server/protocol/iface.go @@ -0,0 +1,27 @@ +package protocol + +import ( + "github.com/Trinoooo/eggie_kv/storage/server/connections" +) + +type IProtocol interface { + Encoder + Decoder + GetConnection() connections.IConnection +} + +type Encoder interface { + WriteStructBegin() error + WriteStructEnd() error + WriteI64(v int64) error + WriteString(v string) error + WriteBytes(v []byte) error +} + +type Decoder interface { + ReadStructBegin() error + ReadStructEnd() error + ReadI64() (int64, error) + ReadString() (string, error) + ReadBytes() ([]byte, error) +} diff --git a/storage/server/reactor_server.go b/storage/server/reactor_server.go index e454477..e35b489 100644 --- a/storage/server/reactor_server.go +++ b/storage/server/reactor_server.go @@ -1,8 +1,10 @@ package server import ( - "context" + "github.com/Trinoooo/eggie_kv/errs" + "github.com/Trinoooo/eggie_kv/storage/server/connections" "github.com/Trinoooo/eggie_kv/storage/server/poller" + "github.com/Trinoooo/eggie_kv/storage/server/protocol" "github.com/bytedance/gopkg/util/gopool" "github.com/pkg/errors" "log" @@ -10,13 +12,11 @@ import ( "sync" "syscall" "time" - "unsafe" ) type ReactorServer struct { mutex sync.Mutex - serverTransport *Listener - bizHandler simpleHandler + serverTransport connections.IListener pool gopool.Pool dp *dispatcher reactors map[int64]*reactor @@ -26,18 +26,19 @@ type ReactorServer struct { } type reactor struct { - srv *ReactorServer - id int64 - connects chan *Conn - w *waiter - p poller.Poller + srv *ReactorServer + id int64 + connects chan connections.IConnection + processors sync.Map // to avoid using Kevent.Udata + w *waiter + p poller.Poller } func newReactor(id int64, srv *ReactorServer, p poller.Poller) *reactor { r := &reactor{ srv: srv, id: id, - connects: make(chan *Conn, reactorInputBufferSize), + connects: make(chan connections.IConnection, reactorInputBufferSize), p: p, w: &waiter{ events: make(chan poller.Pevent, waiterOutputBufferSize), @@ -69,16 +70,20 @@ func (r *reactor) run() { continue } - log.Printf("reactor #%d ready to register event, remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + log.Printf("reactor #%d ready to register event, remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + processor := NewProcessor(r.srv, protocol.NewBinaryProtocol(conn), protocol.NewBinaryProtocol(conn)) changes := []poller.Pevent{{ - ConnFd: uint64(conn.fd), + ConnFd: uint64(conn.RawFd()), Operation: syscall.EVFILT_READ, - Flag: syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT, // edge trigger mode - UserData: *(**byte)(unsafe.Pointer(&conn)), // bugfix：conn和byte内存大小不同，直接指针转换会有gc marked free object in span 风险 + Flag: syscall.EV_ADD | syscall.EV_ENABLE, + }, { + ConnFd: uint64(conn.RawFd()), + Operation: syscall.EVFILT_WRITE, + Flag: syscall.EV_ADD | syscall.EV_ENABLE, }} // register event to poller - if err := r.p.Register(changes); err != nil && err != syscall.EINTR { + if err := r.p.Register(changes); err != nil && !errors.Is(err, syscall.EINTR) { e := conn.Close() if e != nil { err = errors.Wrap(err, e.Error()) @@ -86,29 +91,38 @@ func (r *reactor) run() { log.Println(err, changes, conn, conn.RemoteAddr(), conn.LocalAddr()) continue } - log.Printf("reactor #%d register event success, evt remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + r.processors.Store(conn.RawFd(), processor) + log.Printf("reactor #%d register event success, evt remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) case evt, ok := <-r.w.output(): if !ok { log.Printf("reactor #%d stop", r.id) return } - conn := *(**Conn)(unsafe.Pointer(&evt.UserData)) - log.Printf("reactor #%d handle event, event remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + p, _ := r.processors.Load(int(evt.ConnFd)) + processor := p.(*Processor) + conn := processor.GetInputProtocol().GetConnection() + log.Printf("reactor #%d handle event. remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) // in this case, close is already called. if r.srv.pool == nil { continue } - ctx, cancel := context.WithTimeout(context.Background(), processTimeout) - r.srv.done.Add(2) - r.srv.pool.Go(func() { - r.srv.handler(ctx, cancel, conn) - }) - r.srv.pool.Go(func() { r.srv.notifier(ctx, cancel) }) + + err := processor.Process() + if errs.GetCode(err) == errs.TaskNotFinishErrCode { + // do nothing, wait for next event trigger + } else if err != nil { + e := conn.Close() + if e != nil { + err = errors.Wrap(err, e.Error()) + } + log.Println(err, conn, conn.RemoteAddr(), conn.LocalAddr()) + continue + } } } } -func (r *reactor) input() chan<- *Conn { +func (r *reactor) input() chan<- connections.IConnection { return r.connects } @@ -122,13 +136,13 @@ func (w *waiter) run() { log.Printf("waiter #%d start", w.parent.id) defer w.parent.srv.done.Done() // event buf - evts := make([]poller.Pevent, 10) + evts := make([]poller.Pevent, 100) for { log.Printf("waiter #%d ready to wait event trigger", w.parent.id) // wait for event to be trigger n, err := w.p.Wait(evts) - if err != nil && err != syscall.EINTR { // bugfix: ignore EINTR + if err != nil && !errors.Is(err, syscall.EINTR) { // bugfix: ignore EINTR log.Printf("waiter #%d stop, err: %v", w.parent.id, err) close(w.events) return @@ -138,15 +152,17 @@ func (w *waiter) run() { for i := 0; i < n; i++ { evt := evts[i] - conn := *(**Conn)(unsafe.Pointer(&evt.UserData)) + p, _ := w.parent.processors.Load(int(evt.ConnFd)) + processor := p.(*Processor) + conn := processor.GetInputProtocol().GetConnection() switch { case evt.Flag&syscall.EV_EOF != 0: - log.Printf("waiter #%d meet eof, remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + log.Printf("waiter #%d meet eof, remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) log.Printf("waiter #%d close server connection, err: %v", w.parent.id, conn.Close()) default: - log.Printf("waiter #%d ready to send evt to reactor, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + log.Printf("waiter #%d ready to send evt to reactor, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) w.events <- evt - log.Printf("waiter #%d sent evt to reactor success, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + log.Printf("waiter #%d sent evt to reactor success, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) } } } @@ -162,15 +178,14 @@ const ( pollCapacity = 1000 processTimeout = 1 * time.Second - dispatcherInputBufferSize = 10 + dispatcherInputBufferSize = 1 << 10 reactorInputBufferSize = 10 waiterOutputBufferSize = 10 ) -func NewReactorServer(addr [4]byte, port int, handler simpleHandler) (*ReactorServer, error) { +func NewReactorServer(addr [4]byte, port int) (*ReactorServer, error) { var err error srv := &ReactorServer{ - bizHandler: handler, reactors: make(map[int64]*reactor), pool: gopool.NewPool("handlers", pollCapacity, gopool.NewConfig()), stop: make(chan struct{}), @@ -179,7 +194,7 @@ func NewReactorServer(addr [4]byte, port int, handler simpleHandler) (*ReactorSe // init dispatcher srv.dp = &dispatcher{ - connections: make(chan *Conn, dispatcherInputBufferSize), + connections: make(chan connections.IConnection, dispatcherInputBufferSize), parent: srv, } @@ -194,7 +209,7 @@ func NewReactorServer(addr [4]byte, port int, handler simpleHandler) (*ReactorSe } // init server transport - srv.serverTransport, err = Listen(addr, port) + srv.serverTransport, err = connections.Listen(addr, port) if err != nil { return nil, err } @@ -202,7 +217,7 @@ func NewReactorServer(addr [4]byte, port int, handler simpleHandler) (*ReactorSe } type dispatcher struct { - connections chan *Conn + connections chan connections.IConnection parent *ReactorServer } @@ -211,28 +226,35 @@ func (dp *dispatcher) run() { log.Println("dispatcher start") for conn := range dp.connections { - log.Printf("dispatcher receive connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + log.Printf("dispatcher receive connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) // random load balance id := rand.Int63n(numReactor) if reactor, exist := dp.parent.reactors[id]; exist { reactor.input() <- conn - log.Printf("dispatcher send connection to reactor #%v success, remote addr: %v, local addr: %v, fd: %v", id, conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + log.Printf("dispatcher send connection to reactor #%v success, remote addr: %v, local addr: %v, fd: %v", id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) } else { log.Printf("dispatcher find reactor #%v not exist", id) } } + log.Println("dispatcher stop") +} + +func (dp *dispatcher) input() chan<- connections.IConnection { + return dp.connections +} + +func (dp *dispatcher) close() { + // lock free + c := dp.connections + dp.connections = nil + close(c) + for _, reactor := range dp.parent.reactors { if reactor != nil { close(reactor.input()) } } - - log.Println("dispatcher stop") -} - -func (dp *dispatcher) input() chan<- *Conn { - return dp.connections } func (rs *ReactorServer) Serve() error { @@ -248,61 +270,34 @@ func (rs *ReactorServer) Serve() error { for { log.Printf("acceptor ready to accept connections") conn, err := rs.serverTransport.Accept() - if err != nil && err != syscall.EINTR { - log.Printf("error occur when accept connection, err: %v", err) - rs.mutex.Lock() - close(rs.dp.input()) - e := rs.close() - if e != nil { - e = errors.Wrap(e, err.Error()) + if err != nil { + if errors.Is(err, syscall.EINTR) { + // pass + continue + } else if errors.Is(err, syscall.ECONNABORTED) { + log.Printf("software caused connection abort, maybe a darwin/ios bug, ignore") + continue + } else if errors.Is(err, syscall.EBADF) { + log.Printf("Close called. exit gracefully") + return nil + } else { + log.Printf("error occur when accept connection, err: %v", err) + rs.mutex.Lock() + rs.dp.close() + if e := rs.close(); e != nil { + e = errors.Wrap(err, e.Error()) + } + rs.done.Wait() + rs.clearState() + rs.mutex.Unlock() + return err } - rs.done.Wait() - rs.clearState() - rs.mutex.Unlock() - return e } - log.Printf("acceptor accept connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + log.Printf("acceptor accept connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) rs.metricsHelper.ConnectionAcceptCounter.Inc() - rs.mutex.Lock() - log.Printf("acceptor ready to send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.fd) + log.Printf("acceptor ready to send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) rs.dp.input() <- conn - log.Printf("acceptor success send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.fd) - rs.mutex.Unlock() - } -} - -func (rs *ReactorServer) handler(ctx context.Context, cancel context.CancelFunc, conn *Conn) { - defer func() { - rs.done.Done() - cancel() - }() - log.Printf("bizHandler ready to process socket fd: %d, remote addr: %v, local addr: %v", conn.fd, conn.RemoteAddr(), conn.LocalAddr()) - rs.bizHandler(ctx, conn) - log.Printf("bizHandler success process socket fd: %d, remote addr: %v, local addr: %v", conn.fd, conn.RemoteAddr(), conn.LocalAddr()) - rs.mutex.Lock() - select { - case <-rs.stop: - rs.mutex.Unlock() - log.Printf("bizHandler ready to close connFd %d", conn.fd) - if e := conn.Close(); e != nil { - log.Printf("bizHandler close connection failed. err: %v", e) - return - } - default: - log.Printf("bizHandler ready to send back socket fd: %d, remote addr: %v, local addr: %v", conn.fd, conn.RemoteAddr(), conn.LocalAddr()) - rs.dp.input() <- conn // reuse long connection - log.Printf("bizHandler success send back socket fd: %d, remote addr: %v, local addr: %v", conn.fd, conn.RemoteAddr(), conn.LocalAddr()) - rs.mutex.Unlock() - } -} - -func (rs *ReactorServer) notifier(ctx context.Context, cancel context.CancelFunc) { - defer rs.done.Done() - select { - case <-rs.stop: - cancel() - case <-ctx.Done(): - // do nothing + log.Printf("acceptor success send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) } } @@ -334,6 +329,7 @@ func (rs *ReactorServer) clearState() { rs.pool = nil rs.serverTransport = nil rs.reactors = nil - rs.bizHandler = nil rs.reactors = nil + rs.dp = nil + rs.metricsHelper = nil } diff --git a/storage/server/server.go b/storage/server/server.go deleted file mode 100644 index 3017b7c..0000000 --- a/storage/server/server.go +++ /dev/null @@ -1,159 +0,0 @@ -package server - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "github.com/Trinoooo/eggie_kv/consts" - "github.com/Trinoooo/eggie_kv/errs" - "github.com/Trinoooo/eggie_kv/storage/core" - "github.com/Trinoooo/eggie_kv/storage/core/iface" - "github.com/Trinoooo/eggie_kv/storage/logs" - "github.com/spf13/viper" - "io" - "net/http" -) - -type Server struct { - operatorHandlers map[consts.OperatorType]HandleFunc - mws []MiddlewareFunc - core iface.ICore - config *viper.Viper -} - -func NewServer() (*Server, error) { - srv := &Server{ - operatorHandlers: map[consts.OperatorType]HandleFunc{}, - mws: make([]MiddlewareFunc, 0), - } - - err := srv.withConfig() - if err != nil { - return nil, err - } - - err = srv.withCore() - if err != nil { - return nil, err - } - - srv.withMiddleware( - ParamsValidateMw, - LogMw, - ) - srv.withHandler(consts.OperatorTypeGet, srv.HandleGet) - srv.withHandler(consts.OperatorTypeSet, srv.HandleSet) - return srv, nil -} - -func (srv *Server) Server(resp http.ResponseWriter, req *http.Request) { - kvReq, err := parseKvReq(req) - if err != nil { - logs.Error(fmt.Sprintf("parse kvReq errs: %#v", err)) - _, _ = resp.Write(mustMarshalKvResp(newExceptionResp(err))) - return - } - - handler, ok := srv.operatorHandlers[kvReq.OperationType] - if !ok { - logs.Warn(fmt.Sprintf("unsupported operation type: %#v", kvReq.OperationType)) - _, _ = resp.Write(mustMarshalKvResp(newExceptionResp(errs.NewUnsupportedOperatorTypeErr()))) - return - } - - wrappedHandler := handler - for _, mw := range srv.mws { - wrappedHandler = mw(wrappedHandler) - } - - kvResp, err := wrappedHandler(kvReq) - if err != nil { - logs.Error(fmt.Sprintf("execute handle failed: %#v", err)) - _, _ = resp.Write(mustMarshalKvResp(newExceptionResp(err))) - return - } - - _, _ = resp.Write(mustMarshalKvResp(kvResp)) -} - -func (srv *Server) withHandler(op consts.OperatorType, handler HandleFunc) { - srv.operatorHandlers[op] = handler -} - -func (srv *Server) withMiddleware(mw ...MiddlewareFunc) { - srv.mws = append(srv.mws, mw...) -} - -func (srv *Server) withCore() error { - coreBuilder, exist := core.BuilderMap[srv.config.GetString(consts.Core)] - if !exist { - e := errs.NewCoreNotFoundErr() - logs.Error(e.Error()) - return e - } - c, err := coreBuilder(srv.config) - if err != nil { - return err - } - srv.core = c - return nil -} - -func (srv *Server) withConfig() error { - srv.config = viper.New() - srv.config.AddConfigPath(consts.DefaultConfigPath) - srv.config.SetConfigName("config") - srv.config.SetConfigType("yaml") - err := srv.config.ReadInConfig() - if err != nil { - return err - } - return nil -} - -func parseKvReq(req *http.Request) (*consts.KvRequest, error) { - kvReq := &consts.KvRequest{} - - bodyBytes, err := io.ReadAll(req.Body) - if err != nil { - e := errs.NewReadSocketErr().WithErr(err) - logs.Error(e.Error()) - return nil, e - } - - if err = json.Unmarshal(bodyBytes, kvReq); err != nil { - e := errs.NewJsonUnmarshalErr().WithErr(err) - logs.Error(e.Error()) - return nil, e - } - - return kvReq, nil -} - -func mustMarshalKvResp(resp *consts.KvResponse) []byte { - respBytes, err := json.Marshal(resp) - if err != nil { - panic(err) - } - return respBytes -} - -func newExceptionResp(err error) *consts.KvResponse { - var kvErr = errs.NewUnknownErr() - errors.As(err, &kvErr) - return &consts.KvResponse{ - Code: kvErr.Code(), - Message: kvErr.Error(), - } -} - -func newSuccessResp(data []byte) *consts.KvResponse { - return &consts.KvResponse{ - Message: "success", - Code: 0, - Data: data, - } -} - -type simpleHandler func(ctx context.Context, conn *Conn) diff --git a/storage/server/server_test.go b/storage/server/server_test.go index 2655b82..f307900 100644 --- a/storage/server/server_test.go +++ b/storage/server/server_test.go @@ -1,7 +1,6 @@ package server import ( - "context" "encoding/binary" "log" "net" @@ -15,8 +14,6 @@ import ( const ( addr = "127.0.0.1:9999" - fixSize = 8 - concurrency = 100 ) @@ -26,10 +23,23 @@ func TestMain(m *testing.M) { m.Run() } +/* +type KvRequest struct { + OperationType OpType `json:"operation_type"` + Key []byte `json:"key"` + Value []byte `json:"value"` +} +*/ + func mockClient(t *testing.T, closeServerCallback func() error) { time.Sleep(500 * time.Millisecond) // wait for server start - buf := make([]byte, 8) - binary.BigEndian.PutUint64(buf, 100) + buf := make([]byte, 8+9+8+8+0+8+8) + binary.BigEndian.PutUint64(buf, 9) + copy(buf[8:], "HandleGet") + binary.BigEndian.PutUint64(buf[17:], 0) + binary.BigEndian.PutUint64(buf[25:], 0) + binary.BigEndian.PutUint64(buf[33:], 8) + binary.BigEndian.PutUint64(buf[41:], 100) inflight := sync.WaitGroup{} // 短连接场景下可控制并发度 for i := 0; i < concurrency; i++ { @@ -55,6 +65,7 @@ func shortConnection(t *testing.T, buf []byte) { t.Error(err) return } + _, e := conn.Write(buf) if e != nil { t.Error(e) @@ -106,42 +117,13 @@ func longConnection(t *testing.T, buf []byte) { log.Println("[long] client close connection", conn.RemoteAddr(), conn.LocalAddr()) } -func commonHandler(conn *Conn, t *testing.T) { - buf := make([]byte, fixSize) - _, err := conn.Read(buf) - log.Println("server recv", binary.BigEndian.Uint64(buf), conn.LocalAddr(), conn.RemoteAddr(), conn.fd) - if err != nil { - t.Error(err) - } - v := binary.BigEndian.Uint64(buf) - v = bizLogic(v) - binary.BigEndian.PutUint64(buf, v) - log.Println("server send", binary.BigEndian.Uint64(buf), conn.LocalAddr(), conn.RemoteAddr(), conn.fd) - _, err = conn.Write(buf) - if err != nil { - t.Error(err) - } -} - -func bizLogic(v uint64) uint64 { - // 模拟耗时cpu密集性计算 - for i := 0; i < 1000000; i++ { - v += uint64(i) - } - // 模拟耗时io密集性操作 - time.Sleep(1 * time.Second) - return v -} - func TestReactorServer(t *testing.T) { - // 启动性能分析服务器 + // 启动性能采集服务器 go func() { log.Println(http.ListenAndServe("localhost:6060", nil)) }() - server, err := NewReactorServer([4]byte{127, 0, 0, 1}, 9999, func(ctx context.Context, conn *Conn) { - commonHandler(conn, t) - }) + server, err := NewReactorServer([4]byte{127, 0, 0, 1}, 9999) if err != nil { t.Error(err) return @@ -151,5 +133,4 @@ func TestReactorServer(t *testing.T) { if err != nil { t.Error(err) } - } From 533cedb8c3530c44b0b7922642f4a1b75c8ff80a Mon Sep 17 00:00:00 2001 From: Trino Date: Sun, 23 Jun 2024 20:00:26 +0800 Subject: [PATCH 05/15] fix: test config & concurrency number Signed-off-by: Trino --- .github/workflows/test.yaml | 1 + storage/server/server_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 833aab0..6b19026 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,6 +2,7 @@ name: Go Test on: pull_request: + push: jobs: test: diff --git a/storage/server/server_test.go b/storage/server/server_test.go index f307900..41e2608 100644 --- a/storage/server/server_test.go +++ b/storage/server/server_test.go @@ -14,7 +14,7 @@ import ( const ( addr = "127.0.0.1:9999" - concurrency = 100 + concurrency = 250 ) func TestMain(m *testing.M) { From d1139e6757ea1fad7762d1d2ffd79ea61883994e Mon Sep 17 00:00:00 2001 From: Trino Date: Sun, 23 Jun 2024 20:07:52 +0800 Subject: [PATCH 06/15] =?UTF-8?q?chore(server):=20=E4=BF=AE=E6=94=B9curren?= =?UTF-8?q?cy=E5=88=B0=E4=B8=80=E4=B8=AA=E8=BE=83=E4=BD=8E=E5=80=BC=20&=20?= =?UTF-8?q?=E5=8D=95=E6=B5=8B=E8=BF=90=E8=A1=8C=E5=99=A8=E6=94=B9=E4=B8=BA?= =?UTF-8?q?macos-latest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Trino --- .github/workflows/test.yaml | 2 +- storage/server/server_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6b19026..058a2b9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,7 +7,7 @@ on: jobs: test: name: Run Tests - runs-on: ubuntu-latest + runs-on: macos-latest steps: - name: Set up Go diff --git a/storage/server/server_test.go b/storage/server/server_test.go index 41e2608..a433daa 100644 --- a/storage/server/server_test.go +++ b/storage/server/server_test.go @@ -14,7 +14,7 @@ import ( const ( addr = "127.0.0.1:9999" - concurrency = 250 + concurrency = 10 ) func TestMain(m *testing.M) { From 934d1d8fad54358973ab1c25c2011fcb5caf2b69 Mon Sep 17 00:00:00 2001 From: Trino Date: Sun, 23 Jun 2024 20:21:12 +0800 Subject: [PATCH 07/15] =?UTF-8?q?fix(server):=20cli=E5=AF=B9=E4=B8=8D?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=E7=9A=84=E5=BC=95=E7=94=A8=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Trino --- storage/cli/cli.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/storage/cli/cli.go b/storage/cli/cli.go index e10e8cb..d23b2a2 100644 --- a/storage/cli/cli.go +++ b/storage/cli/cli.go @@ -8,7 +8,6 @@ import ( "github.com/Trinoooo/eggie_kv/storage/server" "github.com/urfave/cli/v2" "go.uber.org/zap" - "net/http" "os" "os/signal" "syscall" @@ -118,25 +117,22 @@ func (wrapper *Wrapper) withFlags() { func (wrapper *Wrapper) withAction() { wrapper.app.Action = func(ctx *cli.Context) error { - srv, err := server.NewServer() + srv, err := server.NewReactorServer([4]byte{127, 0, 0, 1}, 9999) if err != nil { return err } - http.HandleFunc("/", srv.Server) go func() { - addr := fmt.Sprintf("%s:%d", ctx.String("host"), ctx.Int64("port")) - if err := http.ListenAndServe(addr, nil); err != nil { - // 父协程没recover也会一起panic，导致程序崩溃 - logs.Fatal(err.Error()) + // bugfix: 使用缓冲通道避免执行信号处理程序（下面的for）之前有信号到达会被丢弃 + sig := make(chan os.Signal, 5) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + for range sig { + logs.Info("shutdown...") + logs.Error(fmt.Sprintf("server shutdown, err: %v", srv.Close())) } }() - // bugfix: 使用缓冲通道避免执行信号处理程序（下面的for）之前有信号到达会被丢弃 - sig := make(chan os.Signal, 5) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - for range sig { - logs.Info("shutdown...") - } + err = srv.Serve() + return nil } } From c5b4d979c220a93495fd85cc9b655e4350fb190f Mon Sep 17 00:00:00 2001 From: Trino Date: Sun, 23 Jun 2024 20:46:42 +0800 Subject: [PATCH 08/15] chore(server): lint config & test config Signed-off-by: Trino --- .github/workflows/golangci-lint.yaml | 2 +- .github/workflows/test.yaml | 2 +- makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index f1334e9..29c185f 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -13,7 +13,7 @@ permissions: jobs: golangci: name: lint - runs-on: ubuntu-latest + runs-on: macos-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 058a2b9..9dcac48 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v2 - name: Run tests - run: make -f makefile test-with-cover TestPackage=./storage + run: make -f makefile test-with-cover TestPackage=./storage/server - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.0.1 diff --git a/makefile b/makefile index 25cb4e7..a8e638c 100644 --- a/makefile +++ b/makefile @@ -16,7 +16,7 @@ build-all: build-storage build-cli # 测试 TestPackage := $(test_package) test-with-cover: - EGGIE_KV_ENV='test' go test -timeout=1h $(TestPackage) -v -coverprofile=./c.out -count=1 + EGGIE_KV_ENV='test' go test -gcflags="-d=checkptr" -race -timeout=1h $(TestPackage) -v -coverprofile=./c.out -count=1 BenchmarkPackage := $(benchmark_package) BenchmarkTarget := $(benchmark_target) From 7a7c71ca22313b05fcc8d0c5fbe9f4df6088bc74 Mon Sep 17 00:00:00 2001 From: Trino Date: Sun, 23 Jun 2024 20:57:52 +0800 Subject: [PATCH 09/15] =?UTF-8?q?chore(server):=20ci=E4=B8=AD=E8=A1=A5?= =?UTF-8?q?=E5=85=85prometheus=20pushgateway=E7=8E=AF=E5=A2=83=E6=90=AD?= =?UTF-8?q?=E5=BB=BA=E6=AD=A5=E9=AA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Trino --- .github/workflows/test.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9dcac48..64c3f81 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,6 +10,13 @@ jobs: runs-on: macos-latest steps: + - name: Set up Prometheus Pushgateway + run: | + curl -LO https://github.com/prometheus/pushgateway/releases/download/v1.5.2/pushgateway-1.5.2.darwin-amd64.tar.gz + tar xvfz pushgateway-1.5.2.darwin-amd64.tar.gz + cd pushgateway-1.5.2.darwin-amd64 + ./pushgateway & + - name: Set up Go uses: actions/setup-go@v2 with: From 98c851e1a804939e106dfcb270551f8551b61bed Mon Sep 17 00:00:00 2001 From: Trino Date: Sun, 23 Jun 2024 21:07:57 +0800 Subject: [PATCH 10/15] =?UTF-8?q?chore(server):=20ci=E4=B8=AD=E8=A1=A5?= =?UTF-8?q?=E5=85=85prometheus=20pushgateway=E7=8E=AF=E5=A2=83=E6=90=AD?= =?UTF-8?q?=E5=BB=BA=E6=AD=A5=E9=AA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Trino --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 64c3f81..39e815a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -13,7 +13,7 @@ jobs: - name: Set up Prometheus Pushgateway run: | curl -LO https://github.com/prometheus/pushgateway/releases/download/v1.5.2/pushgateway-1.5.2.darwin-amd64.tar.gz - tar xvfz pushgateway-1.5.2.darwin-amd64.tar.gz + tar -xvfz pushgateway-1.5.2.darwin-amd64.tar.gz cd pushgateway-1.5.2.darwin-amd64 ./pushgateway & From 2616b424db052dc151067b6a4c17faa303b3eb49 Mon Sep 17 00:00:00 2001 From: Trino Date: Sun, 23 Jun 2024 21:11:54 +0800 Subject: [PATCH 11/15] =?UTF-8?q?chore(server):=20ci=E4=B8=AD=E8=A1=A5?= =?UTF-8?q?=E5=85=85prometheus=20pushgateway=E7=8E=AF=E5=A2=83=E6=90=AD?= =?UTF-8?q?=E5=BB=BA=E6=AD=A5=E9=AA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Trino --- .github/workflows/test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 39e815a..ef69bff 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,9 +12,9 @@ jobs: steps: - name: Set up Prometheus Pushgateway run: | - curl -LO https://github.com/prometheus/pushgateway/releases/download/v1.5.2/pushgateway-1.5.2.darwin-amd64.tar.gz - tar -xvfz pushgateway-1.5.2.darwin-amd64.tar.gz - cd pushgateway-1.5.2.darwin-amd64 + curl -LO https://github.com/prometheus/pushgateway/releases/download/v1.5.2/pushgateway-1.5.2.darwin-amd64.tar.gz && + tar -xvfz pushgateway-1.5.2.darwin-amd64.tar.gz && + cd pushgateway-1.5.2.darwin-amd64 && ./pushgateway & - name: Set up Go From 86636a0245ae881b2e4a5bb9b1142d9747f63575 Mon Sep 17 00:00:00 2001 From: Trino Date: Sat, 29 Jun 2024 00:20:50 +0800 Subject: [PATCH 12/15] =?UTF-8?q?feat(server):=20=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=A2=9C=E8=89=B2=EF=BC=9B=E6=B5=8B=E8=AF=95?= =?UTF-8?q?ci=E6=94=B9=E4=B8=BA=E5=8C=85=E5=90=AB=E6=89=80=E6=9C=89?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B=EF=BC=9B=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=B8=80=E4=BA=9B=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Trino --- .github/workflows/test.yaml | 7 --- errs/error.go | 4 +- go.mod | 7 ++- go.sum | 18 +++++++ makefile | 2 +- storage/cli/cli.go | 3 +- storage/server/handler.go | 9 ++-- storage/server/processor.go | 23 ++++----- storage/server/reactor_server.go | 80 +++++++++++++++++++------------- storage/server/server_test.go | 13 ++++-- utils/color.go | 20 ++++++++ 11 files changed, 120 insertions(+), 66 deletions(-) create mode 100644 utils/color.go diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ef69bff..9dcac48 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,13 +10,6 @@ jobs: runs-on: macos-latest steps: - - name: Set up Prometheus Pushgateway - run: | - curl -LO https://github.com/prometheus/pushgateway/releases/download/v1.5.2/pushgateway-1.5.2.darwin-amd64.tar.gz && - tar -xvfz pushgateway-1.5.2.darwin-amd64.tar.gz && - cd pushgateway-1.5.2.darwin-amd64 && - ./pushgateway & - - name: Set up Go uses: actions/setup-go@v2 with: diff --git a/errs/error.go b/errs/error.go index d5df2de..a87cc6d 100644 --- a/errs/error.go +++ b/errs/error.go @@ -246,6 +246,6 @@ func NewUnexpectHandler() *KvErr { return &KvErr{msg: "unexpect handler key", code: UnexpectHandlerErrCode} } -func NewTaskNotFinishErr() *KvErr { - return &KvErr{msg: "task not finish yet, try again.", code: TaskNotFinishErrCode} +func NewTaskNotFinishErr(taskKey string) *KvErr { + return &KvErr{msg: fmt.Sprintf("task %v not finish yet, try again.", taskKey), code: TaskNotFinishErrCode} } diff --git a/go.mod b/go.mod index 87af61e..1b21326 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.19 require ( github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3 github.com/chzyer/readline v1.5.1 - github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 @@ -16,10 +15,13 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/mockey v1.2.10 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect @@ -29,6 +31,8 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect + github.com/smartystreets/goconvey v1.6.4 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect @@ -36,6 +40,7 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index ea50f1f..b24e36c 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3 h1:ZKUHguI38SRQJkq7hhmwn8lAv3xM6B5qkj1IneS15YY= github.com/bytedance/gopkg v0.0.0-20240507064146-197ded923ae3/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ= +github.com/bytedance/mockey v1.2.10 h1:4JlMpkm7HMXmTUtItid+iCu2tm61wvq+ca1X2u7ymzE= +github.com/bytedance/mockey v1.2.10/go.mod h1:bNrUnI1u7+pAc0TYDgPATM+wF2yzHxmNH+iDXg4AOCU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= @@ -19,8 +21,12 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f h1:WVPqVsbUsrzAebTEgWRAZMdDOfkFx06iyhbIoyMgtkE= @@ -52,6 +58,10 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -81,20 +91,27 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff h1:XmKBi9R6duxOB3lfc72wyrwiOY7X2Jl1wuI+RFOyMDE= +golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -104,3 +121,4 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/makefile b/makefile index a8e638c..2f2b736 100644 --- a/makefile +++ b/makefile @@ -16,7 +16,7 @@ build-all: build-storage build-cli # 测试 TestPackage := $(test_package) test-with-cover: - EGGIE_KV_ENV='test' go test -gcflags="-d=checkptr" -race -timeout=1h $(TestPackage) -v -coverprofile=./c.out -count=1 + EGGIE_KV_ENV='test' go test -gcflags="all=-l -N -d=checkptr " -race -timeout=1h $(TestPackage) -v -coverprofile=./c.out -count=1 BenchmarkPackage := $(benchmark_package) BenchmarkTarget := $(benchmark_target) diff --git a/storage/cli/cli.go b/storage/cli/cli.go index d23b2a2..7a1483e 100644 --- a/storage/cli/cli.go +++ b/storage/cli/cli.go @@ -131,9 +131,8 @@ func (wrapper *Wrapper) withAction() { logs.Error(fmt.Sprintf("server shutdown, err: %v", srv.Close())) } }() - err = srv.Serve() - return nil + return srv.Serve() } } diff --git a/storage/server/handler.go b/storage/server/handler.go index a0522f0..996134a 100644 --- a/storage/server/handler.go +++ b/storage/server/handler.go @@ -2,6 +2,7 @@ package server import ( "encoding/binary" + "github.com/Trinoooo/eggie_kv/utils" "log" "time" ) @@ -10,18 +11,18 @@ func HandleGet(req *KvRequest) (*KvResponse, error) { resp := &KvResponse{ Data: make([]byte, 8), } - log.Printf("HandleGet kvRequest: %#v", req) + log.Printf(utils.WrapInfo("HandleGet kvRequest: %#v", req)) v := bizLogic(binary.BigEndian.Uint64(req.Value)) binary.BigEndian.PutUint64(resp.Data, v) - log.Printf("HandleGet kvResponse: %#v", resp) + log.Printf(utils.WrapInfo("HandleGet kvResponse: %#v", resp)) return resp, nil } func HandleSet(req *KvRequest) (*KvResponse, error) { resp := &KvResponse{} - log.Printf("HandleSet kvRequest: %#v", req) + log.Printf(utils.WrapInfo("HandleGet kvRequest: %#v", req)) bizLogic(binary.BigEndian.Uint64(req.Value)) - log.Printf("HandleSet kvResponse: %#v", resp) + log.Printf(utils.WrapInfo("HandleGet kvResponse: %#v", resp)) return resp, nil } diff --git a/storage/server/processor.go b/storage/server/processor.go index f18c574..7ebbda6 100644 --- a/storage/server/processor.go +++ b/storage/server/processor.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/Trinoooo/eggie_kv/errs" "github.com/Trinoooo/eggie_kv/storage/server/protocol" + "github.com/Trinoooo/eggie_kv/utils" "log" "strings" "sync" @@ -102,7 +103,7 @@ func (p *Processor) decodeHandler() error { return p.checkOrSetState("DecodeRequestHandler", []string{"START"}, func() (bool, error) { handlerKey, err := p.inputProtocol.ReadString() if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { - return false, errs.NewTaskNotFinishErr() + return false, errs.NewTaskNotFinishErr("DecodeRequestHandler") } else if err != nil { return false, err } @@ -120,7 +121,7 @@ func (p *Processor) decodeRequest() error { if err := p.checkOrSetState("DecodeRequestOpType", []string{"DecodeRequestHandler"}, func() (bool, error) { opTypeI64, err := p.inputProtocol.ReadI64() if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { - return false, errs.NewTaskNotFinishErr() + return false, errs.NewTaskNotFinishErr("DecodeRequestOpType") } else if err != nil { return false, err } @@ -133,7 +134,7 @@ func (p *Processor) decodeRequest() error { if err := p.checkOrSetState("DecodeRequestKey", []string{"DecodeRequestOpType"}, func() (bool, error) { key, err := p.inputProtocol.ReadBytes() if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { - return false, errs.NewTaskNotFinishErr() + return false, errs.NewTaskNotFinishErr("DecodeRequestKey") } else if err != nil { return false, err } @@ -146,7 +147,7 @@ func (p *Processor) decodeRequest() error { if err := p.checkOrSetState("DecodeRequestValue", []string{"DecodeRequestKey"}, func() (bool, error) { value, err := p.inputProtocol.ReadBytes() if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { - return false, errs.NewTaskNotFinishErr() + return false, errs.NewTaskNotFinishErr("DecodeRequestValue") } else if err != nil { return false, err } @@ -172,7 +173,7 @@ func (p *Processor) triggerHandler() error { p.task.Execute() }) }) - return false, errs.NewTaskNotFinishErr() + return false, errs.NewTaskNotFinishErr("TriggerHandler") } }) } @@ -181,7 +182,7 @@ func (p *Processor) encodeResponse() error { return p.checkOrSetState("EncodeResponseData", []string{"TriggerHandler"}, func() (bool, error) { err := p.outputProtocol.WriteBytes(p.task.resp.Data) if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { - return false, errs.NewTaskNotFinishErr() + return false, errs.NewTaskNotFinishErr("EncodeResponseData") } else if err != nil { return false, err } @@ -193,7 +194,7 @@ func (p *Processor) encodeException() error { if err := p.checkOrSetState("EncodeExceptionCode", []string{"TriggerHandler"}, func() (bool, error) { err := p.outputProtocol.WriteI64(p.task.exception.Code) if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { - return false, errs.NewTaskNotFinishErr() + return false, errs.NewTaskNotFinishErr("EncodeExceptionCode") } else if err != nil { return false, err } @@ -205,7 +206,7 @@ func (p *Processor) encodeException() error { if err := p.checkOrSetState("EncodeExceptionMessage", []string{"EncodeExceptionCode"}, func() (bool, error) { err := p.outputProtocol.WriteString(p.task.exception.Message) if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK) { - return false, errs.NewTaskNotFinishErr() + return false, errs.NewTaskNotFinishErr("EncodeExceptionMessage") } else if err != nil { return false, err } @@ -241,13 +242,13 @@ func (p *Processor) checkOrSetState(currentStepKey string, previousStepKeys []st if !previousStepFinish { connection := p.inputProtocol.GetConnection() log.Printf("previous step %v not finish, retry. remote addr: %v, local addr: %v, fd: %v", strings.Join(previousStepKeys, ","), connection.RemoteAddr(), connection.LocalAddr(), connection.RawFd()) - return errs.NewTaskNotFinishErr() + return nil } // current step has already finished. we should skip to avoid execute `fn` more than once. if currentStepFinish := p.stepState[currentStepKey]; currentStepFinish { connection := p.inputProtocol.GetConnection() - log.Printf("current step %v alreay finished, skip. remote addr: %v, local addr: %v, fd: %v", currentStepKey, connection.RemoteAddr(), connection.LocalAddr(), connection.RawFd()) + log.Printf(utils.WrapInfo("current step %v alreay finished, skip. remote addr: %v, local addr: %v, fd: %v", currentStepKey, connection.RemoteAddr(), connection.LocalAddr(), connection.RawFd())) return nil } @@ -271,7 +272,7 @@ type Task struct { func (t *Task) Execute() { resp, err := t.handler(t.req) if err != nil { - log.Printf("handler return error: %v", err) + log.Printf(utils.WrapError("handler return error: %v", err)) var kvErr *errs.KvErr if errors.As(err, &kvErr) { t.exception = &KvException{ diff --git a/storage/server/reactor_server.go b/storage/server/reactor_server.go index e35b489..e82079c 100644 --- a/storage/server/reactor_server.go +++ b/storage/server/reactor_server.go @@ -5,6 +5,7 @@ import ( "github.com/Trinoooo/eggie_kv/storage/server/connections" "github.com/Trinoooo/eggie_kv/storage/server/poller" "github.com/Trinoooo/eggie_kv/storage/server/protocol" + "github.com/Trinoooo/eggie_kv/utils" "github.com/bytedance/gopkg/util/gopool" "github.com/pkg/errors" "log" @@ -51,7 +52,7 @@ func newReactor(id int64, srv *ReactorServer, p poller.Poller) *reactor { func (r *reactor) run() { defer r.srv.done.Done() - log.Printf("reactor #%d start", r.id) + log.Printf(utils.WrapInfo("reactor #%d start", r.id)) connects := r.connects r.srv.done.Add(1) @@ -61,16 +62,16 @@ func (r *reactor) run() { case conn, ok := <-connects: // output been close by dispatcher if !ok { - log.Printf("reactor #%d ready to closes poller", r.id) + log.Printf(utils.WrapWarn("reactor #%d ready to closes poller", r.id)) if e := r.p.Close(); e != nil { - log.Printf("reactor #%d close p failed. err: %v", r.id, e) + log.Printf(utils.WrapError("reactor #%d close p failed. err: %v", r.id, e)) } connects = nil - log.Printf("reactor #%d output set to nil", r.id) + log.Printf(utils.WrapWarn("reactor #%d output set to nil", r.id)) continue } - log.Printf("reactor #%d ready to register event, remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + log.Printf(utils.WrapInfo("reactor #%d ready to register event, remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) processor := NewProcessor(r.srv, protocol.NewBinaryProtocol(conn), protocol.NewBinaryProtocol(conn)) changes := []poller.Pevent{{ @@ -92,16 +93,16 @@ func (r *reactor) run() { continue } r.processors.Store(conn.RawFd(), processor) - log.Printf("reactor #%d register event success, evt remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + log.Printf(utils.WrapInfo("reactor #%d register event success, evt remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) case evt, ok := <-r.w.output(): if !ok { - log.Printf("reactor #%d stop", r.id) + log.Printf(utils.WrapWarn("reactor #%d stop", r.id)) return } p, _ := r.processors.Load(int(evt.ConnFd)) processor := p.(*Processor) conn := processor.GetInputProtocol().GetConnection() - log.Printf("reactor #%d handle event. remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + log.Printf(utils.WrapInfo("reactor #%d handle event. remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) // in this case, close is already called. if r.srv.pool == nil { continue @@ -109,13 +110,17 @@ func (r *reactor) run() { err := processor.Process() if errs.GetCode(err) == errs.TaskNotFinishErrCode { + log.Println(utils.WrapWarn("%s %s %s %d", err.Error(), conn.LocalAddr(), conn.RemoteAddr(), conn.RawFd())) + // do nothing, wait for next event trigger + } else if errors.Is(err, syscall.EBADF) { + log.Println(utils.WrapWarn("%s %s %s %d", err.Error(), conn.LocalAddr(), conn.RemoteAddr(), conn.RawFd())) // do nothing, wait for next event trigger } else if err != nil { e := conn.Close() if e != nil { err = errors.Wrap(err, e.Error()) } - log.Println(err, conn, conn.RemoteAddr(), conn.LocalAddr()) + log.Printf(utils.WrapError("error occur when reactor #%v process request, err: %v, remoteAddr: %v, localAddr: %v", r.id, err, conn.RemoteAddr(), conn.LocalAddr())) continue } } @@ -133,22 +138,29 @@ type waiter struct { } func (w *waiter) run() { - log.Printf("waiter #%d start", w.parent.id) + log.Printf(utils.WrapInfo("waiter #%d start", w.parent.id)) defer w.parent.srv.done.Done() // event buf evts := make([]poller.Pevent, 100) for { - log.Printf("waiter #%d ready to wait event trigger", w.parent.id) + log.Printf(utils.WrapInfo("waiter #%d ready to wait event trigger", w.parent.id)) // wait for event to be trigger n, err := w.p.Wait(evts) - if err != nil && !errors.Is(err, syscall.EINTR) { // bugfix: ignore EINTR - log.Printf("waiter #%d stop, err: %v", w.parent.id, err) + if errors.Is(err, syscall.EINTR) { + // pass + continue + } else if errors.Is(err, syscall.EBADF) { + log.Printf(utils.WrapWarn("waiter #%d stop gracefully", w.parent.id)) + close(w.events) + return + } else if err != nil { + log.Printf(utils.WrapError("error occur inside waiter #%d, err: %v", w.parent.id, err)) close(w.events) return } - log.Printf("waiter #%d event trigger success, evts: %#v, n: %d", w.parent.id, evts, n) + log.Printf(utils.WrapInfo("waiter #%d event trigger success, evts: %#v, n: %d", w.parent.id, evts, n)) for i := 0; i < n; i++ { evt := evts[i] @@ -157,12 +169,14 @@ func (w *waiter) run() { conn := processor.GetInputProtocol().GetConnection() switch { case evt.Flag&syscall.EV_EOF != 0: - log.Printf("waiter #%d meet eof, remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) - log.Printf("waiter #%d close server connection, err: %v", w.parent.id, conn.Close()) + log.Printf(utils.WrapInfo("waiter #%d meet eof, remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Printf(utils.WrapInfo("waiter #%d close server connection, err: %v", w.parent.id, conn.Close())) default: - log.Printf("waiter #%d ready to send evt to reactor, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + triggerRead := evt.Operation == syscall.EVFILT_READ + triggerWrite := evt.Operation == syscall.EVFILT_WRITE + log.Printf(utils.WrapInfo("waiter #%d ready to send evt(trigger read: %v, trigger write: %v, operation: %v) to reactor, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, triggerRead, triggerWrite, evt.Operation, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) w.events <- evt - log.Printf("waiter #%d sent evt to reactor success, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + log.Printf(utils.WrapInfo("waiter #%d sent evt to reactor success, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) } } } @@ -223,21 +237,21 @@ type dispatcher struct { func (dp *dispatcher) run() { defer dp.parent.done.Done() - log.Println("dispatcher start") + log.Println(utils.WrapInfo("dispatcher start")) for conn := range dp.connections { - log.Printf("dispatcher receive connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + log.Printf(utils.WrapInfo("dispatcher receive connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) // random load balance id := rand.Int63n(numReactor) if reactor, exist := dp.parent.reactors[id]; exist { reactor.input() <- conn - log.Printf("dispatcher send connection to reactor #%v success, remote addr: %v, local addr: %v, fd: %v", id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + log.Printf(utils.WrapInfo("dispatcher send connection to reactor #%v success, remote addr: %v, local addr: %v, fd: %v", id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) } else { - log.Printf("dispatcher find reactor #%v not exist", id) + log.Printf(utils.WrapError("dispatcher find reactor #%v not exist", id)) } } - log.Println("dispatcher stop") + log.Println(utils.WrapWarn("dispatcher stop")) } func (dp *dispatcher) input() chan<- connections.IConnection { @@ -268,36 +282,36 @@ func (rs *ReactorServer) Serve() error { // acceptor for { - log.Printf("acceptor ready to accept connections") + log.Printf(utils.WrapInfo("acceptor ready to accept connections")) conn, err := rs.serverTransport.Accept() if err != nil { if errors.Is(err, syscall.EINTR) { // pass continue } else if errors.Is(err, syscall.ECONNABORTED) { - log.Printf("software caused connection abort, maybe a darwin/ios bug, ignore") + log.Printf(utils.WrapWarn("software caused connection abort, maybe a darwin/ios bug, ignore")) continue } else if errors.Is(err, syscall.EBADF) { - log.Printf("Close called. exit gracefully") - return nil - } else { - log.Printf("error occur when accept connection, err: %v", err) + log.Printf(utils.WrapWarn("Close called. exit gracefully")) rs.mutex.Lock() rs.dp.close() if e := rs.close(); e != nil { - e = errors.Wrap(err, e.Error()) + err = errors.Wrap(err, e.Error()) } rs.done.Wait() rs.clearState() rs.mutex.Unlock() + return nil + } else { + log.Printf(utils.WrapError("error occur when accept connection, err: %v", err)) return err } } - log.Printf("acceptor accept connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + log.Printf(utils.WrapInfo("acceptor accept connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) rs.metricsHelper.ConnectionAcceptCounter.Inc() - log.Printf("acceptor ready to send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + log.Printf(utils.WrapInfo("acceptor ready to send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) rs.dp.input() <- conn - log.Printf("acceptor success send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd()) + log.Printf(utils.WrapInfo("acceptor success send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) } } diff --git a/storage/server/server_test.go b/storage/server/server_test.go index a433daa..d183b64 100644 --- a/storage/server/server_test.go +++ b/storage/server/server_test.go @@ -2,9 +2,11 @@ package server import ( "encoding/binary" + "github.com/bytedance/mockey" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/push" "log" "net" - "net/http" _ "net/http/pprof" "sync" "testing" @@ -18,7 +20,6 @@ const ( ) func TestMain(m *testing.M) { - log.SetFlags(log.Llongfile | log.LstdFlags) m.Run() } @@ -119,9 +120,11 @@ func longConnection(t *testing.T, buf []byte) { func TestReactorServer(t *testing.T) { // 启动性能采集服务器 - go func() { + /*go func() { log.Println(http.ListenAndServe("localhost:6060", nil)) - }() + }()*/ + mockey.Mock((*push.Pusher).Add).Return(nil).Build() + mockey.Mock(prometheus.Counter.Inc).Return().Build() server, err := NewReactorServer([4]byte{127, 0, 0, 1}, 9999) if err != nil { @@ -131,6 +134,6 @@ func TestReactorServer(t *testing.T) { go mockClient(t, server.Close) err = server.Serve() if err != nil { - t.Error(err) + t.Error("server shutdown", err) } } diff --git a/utils/color.go b/utils/color.go new file mode 100644 index 0000000..d518f9a --- /dev/null +++ b/utils/color.go @@ -0,0 +1,20 @@ +package utils + +import "fmt" + +const ( + ERROR = "\033[1;31;40m[ERROR] %s\033[0m" + WARN = "\033[1;33;40m[WARN] %s\033[0m" + INFO = "\033[1;34;40m[INFO] %s\033[0m" +) + +func WrapError(format string, args ...any) string { + return fmt.Sprintf(ERROR, fmt.Sprintf(format, args...)) +} + +func WrapWarn(format string, args ...any) string { + return fmt.Sprintf(WARN, fmt.Sprintf(format, args...)) +} +func WrapInfo(format string, args ...any) string { + return fmt.Sprintf(INFO, fmt.Sprintf(format, args...)) +} From 8b7912db376f8af9dfc3490c3516b795cc3e073a Mon Sep 17 00:00:00 2001 From: Trino Date: Sat, 29 Jun 2024 00:26:10 +0800 Subject: [PATCH 13/15] chore(server): resolve golang-cilint Signed-off-by: Trino --- .github/workflows/test.yaml | 2 +- storage/server/handler.go | 8 ++--- storage/server/processor.go | 4 +-- storage/server/reactor_server.go | 57 ++++++++++++++++---------------- 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9dcac48..02dd578 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v2 - name: Run tests - run: make -f makefile test-with-cover TestPackage=./storage/server + run: make -f makefile test-with-cover TestPackage=./storage/... - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.0.1 diff --git a/storage/server/handler.go b/storage/server/handler.go index 996134a..cc9dbec 100644 --- a/storage/server/handler.go +++ b/storage/server/handler.go @@ -11,18 +11,18 @@ func HandleGet(req *KvRequest) (*KvResponse, error) { resp := &KvResponse{ Data: make([]byte, 8), } - log.Printf(utils.WrapInfo("HandleGet kvRequest: %#v", req)) + log.Print(utils.WrapInfo("HandleGet kvRequest: %#v", req)) v := bizLogic(binary.BigEndian.Uint64(req.Value)) binary.BigEndian.PutUint64(resp.Data, v) - log.Printf(utils.WrapInfo("HandleGet kvResponse: %#v", resp)) + log.Print(utils.WrapInfo("HandleGet kvResponse: %#v", resp)) return resp, nil } func HandleSet(req *KvRequest) (*KvResponse, error) { resp := &KvResponse{} - log.Printf(utils.WrapInfo("HandleGet kvRequest: %#v", req)) + log.Print(utils.WrapInfo("HandleGet kvRequest: %#v", req)) bizLogic(binary.BigEndian.Uint64(req.Value)) - log.Printf(utils.WrapInfo("HandleGet kvResponse: %#v", resp)) + log.Print(utils.WrapInfo("HandleGet kvResponse: %#v", resp)) return resp, nil } diff --git a/storage/server/processor.go b/storage/server/processor.go index 7ebbda6..3d30a96 100644 --- a/storage/server/processor.go +++ b/storage/server/processor.go @@ -248,7 +248,7 @@ func (p *Processor) checkOrSetState(currentStepKey string, previousStepKeys []st // current step has already finished. we should skip to avoid execute `fn` more than once. if currentStepFinish := p.stepState[currentStepKey]; currentStepFinish { connection := p.inputProtocol.GetConnection() - log.Printf(utils.WrapInfo("current step %v alreay finished, skip. remote addr: %v, local addr: %v, fd: %v", currentStepKey, connection.RemoteAddr(), connection.LocalAddr(), connection.RawFd())) + log.Print(utils.WrapInfo("current step %v alreay finished, skip. remote addr: %v, local addr: %v, fd: %v", currentStepKey, connection.RemoteAddr(), connection.LocalAddr(), connection.RawFd())) return nil } @@ -272,7 +272,7 @@ type Task struct { func (t *Task) Execute() { resp, err := t.handler(t.req) if err != nil { - log.Printf(utils.WrapError("handler return error: %v", err)) + log.Print(utils.WrapError("handler return error: %v", err)) var kvErr *errs.KvErr if errors.As(err, &kvErr) { t.exception = &KvException{ diff --git a/storage/server/reactor_server.go b/storage/server/reactor_server.go index e82079c..b995c47 100644 --- a/storage/server/reactor_server.go +++ b/storage/server/reactor_server.go @@ -52,7 +52,7 @@ func newReactor(id int64, srv *ReactorServer, p poller.Poller) *reactor { func (r *reactor) run() { defer r.srv.done.Done() - log.Printf(utils.WrapInfo("reactor #%d start", r.id)) + log.Print(utils.WrapInfo("reactor #%d start", r.id)) connects := r.connects r.srv.done.Add(1) @@ -62,16 +62,16 @@ func (r *reactor) run() { case conn, ok := <-connects: // output been close by dispatcher if !ok { - log.Printf(utils.WrapWarn("reactor #%d ready to closes poller", r.id)) + log.Print(utils.WrapWarn("reactor #%d ready to closes poller", r.id)) if e := r.p.Close(); e != nil { - log.Printf(utils.WrapError("reactor #%d close p failed. err: %v", r.id, e)) + log.Print(utils.WrapError("reactor #%d close p failed. err: %v", r.id, e)) } connects = nil - log.Printf(utils.WrapWarn("reactor #%d output set to nil", r.id)) + log.Print(utils.WrapWarn("reactor #%d output set to nil", r.id)) continue } - log.Printf(utils.WrapInfo("reactor #%d ready to register event, remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("reactor #%d ready to register event, remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) processor := NewProcessor(r.srv, protocol.NewBinaryProtocol(conn), protocol.NewBinaryProtocol(conn)) changes := []poller.Pevent{{ @@ -93,16 +93,16 @@ func (r *reactor) run() { continue } r.processors.Store(conn.RawFd(), processor) - log.Printf(utils.WrapInfo("reactor #%d register event success, evt remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("reactor #%d register event success, evt remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) case evt, ok := <-r.w.output(): if !ok { - log.Printf(utils.WrapWarn("reactor #%d stop", r.id)) + log.Print(utils.WrapWarn("reactor #%d stop", r.id)) return } p, _ := r.processors.Load(int(evt.ConnFd)) processor := p.(*Processor) conn := processor.GetInputProtocol().GetConnection() - log.Printf(utils.WrapInfo("reactor #%d handle event. remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("reactor #%d handle event. remote addr: %v, local addr: %v, fd: %v", r.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) // in this case, close is already called. if r.srv.pool == nil { continue @@ -120,7 +120,7 @@ func (r *reactor) run() { if e != nil { err = errors.Wrap(err, e.Error()) } - log.Printf(utils.WrapError("error occur when reactor #%v process request, err: %v, remoteAddr: %v, localAddr: %v", r.id, err, conn.RemoteAddr(), conn.LocalAddr())) + log.Print(utils.WrapError("error occur when reactor #%v process request, err: %v, remoteAddr: %v, localAddr: %v", r.id, err, conn.RemoteAddr(), conn.LocalAddr())) continue } } @@ -138,12 +138,12 @@ type waiter struct { } func (w *waiter) run() { - log.Printf(utils.WrapInfo("waiter #%d start", w.parent.id)) + log.Print(utils.WrapInfo("waiter #%d start", w.parent.id)) defer w.parent.srv.done.Done() // event buf evts := make([]poller.Pevent, 100) for { - log.Printf(utils.WrapInfo("waiter #%d ready to wait event trigger", w.parent.id)) + log.Print(utils.WrapInfo("waiter #%d ready to wait event trigger", w.parent.id)) // wait for event to be trigger n, err := w.p.Wait(evts) @@ -151,16 +151,16 @@ func (w *waiter) run() { // pass continue } else if errors.Is(err, syscall.EBADF) { - log.Printf(utils.WrapWarn("waiter #%d stop gracefully", w.parent.id)) + log.Print(utils.WrapWarn("waiter #%d stop gracefully", w.parent.id)) close(w.events) return } else if err != nil { - log.Printf(utils.WrapError("error occur inside waiter #%d, err: %v", w.parent.id, err)) + log.Print(utils.WrapError("error occur inside waiter #%d, err: %v", w.parent.id, err)) close(w.events) return } - log.Printf(utils.WrapInfo("waiter #%d event trigger success, evts: %#v, n: %d", w.parent.id, evts, n)) + log.Print(utils.WrapInfo("waiter #%d event trigger success, evts: %#v, n: %d", w.parent.id, evts, n)) for i := 0; i < n; i++ { evt := evts[i] @@ -169,14 +169,14 @@ func (w *waiter) run() { conn := processor.GetInputProtocol().GetConnection() switch { case evt.Flag&syscall.EV_EOF != 0: - log.Printf(utils.WrapInfo("waiter #%d meet eof, remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) - log.Printf(utils.WrapInfo("waiter #%d close server connection, err: %v", w.parent.id, conn.Close())) + log.Print(utils.WrapInfo("waiter #%d meet eof, remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("waiter #%d close server connection, err: %v", w.parent.id, conn.Close())) default: triggerRead := evt.Operation == syscall.EVFILT_READ triggerWrite := evt.Operation == syscall.EVFILT_WRITE - log.Printf(utils.WrapInfo("waiter #%d ready to send evt(trigger read: %v, trigger write: %v, operation: %v) to reactor, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, triggerRead, triggerWrite, evt.Operation, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("waiter #%d ready to send evt(trigger read: %v, trigger write: %v, operation: %v) to reactor, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, triggerRead, triggerWrite, evt.Operation, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) w.events <- evt - log.Printf(utils.WrapInfo("waiter #%d sent evt to reactor success, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("waiter #%d sent evt to reactor success, evt remote addr: %v, local addr: %v, fd: %v", w.parent.id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) } } } @@ -240,14 +240,14 @@ func (dp *dispatcher) run() { log.Println(utils.WrapInfo("dispatcher start")) for conn := range dp.connections { - log.Printf(utils.WrapInfo("dispatcher receive connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("dispatcher receive connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) // random load balance id := rand.Int63n(numReactor) if reactor, exist := dp.parent.reactors[id]; exist { reactor.input() <- conn - log.Printf(utils.WrapInfo("dispatcher send connection to reactor #%v success, remote addr: %v, local addr: %v, fd: %v", id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("dispatcher send connection to reactor #%v success, remote addr: %v, local addr: %v, fd: %v", id, conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) } else { - log.Printf(utils.WrapError("dispatcher find reactor #%v not exist", id)) + log.Print(utils.WrapError("dispatcher find reactor #%v not exist", id)) } } @@ -282,36 +282,37 @@ func (rs *ReactorServer) Serve() error { // acceptor for { - log.Printf(utils.WrapInfo("acceptor ready to accept connections")) + log.Print(utils.WrapInfo("acceptor ready to accept connections")) conn, err := rs.serverTransport.Accept() if err != nil { if errors.Is(err, syscall.EINTR) { // pass continue } else if errors.Is(err, syscall.ECONNABORTED) { - log.Printf(utils.WrapWarn("software caused connection abort, maybe a darwin/ios bug, ignore")) + log.Print(utils.WrapWarn("software caused connection abort, maybe a darwin/ios bug, ignore")) continue } else if errors.Is(err, syscall.EBADF) { - log.Printf(utils.WrapWarn("Close called. exit gracefully")) + log.Print(utils.WrapWarn("Close called. exit gracefully")) rs.mutex.Lock() rs.dp.close() if e := rs.close(); e != nil { err = errors.Wrap(err, e.Error()) } + log.Print(utils.WrapError("error occur when close reactor server, err: %v", err)) rs.done.Wait() rs.clearState() rs.mutex.Unlock() return nil } else { - log.Printf(utils.WrapError("error occur when accept connection, err: %v", err)) + log.Print(utils.WrapError("error occur when accept connection, err: %v", err)) return err } } - log.Printf(utils.WrapInfo("acceptor accept connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("acceptor accept connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) rs.metricsHelper.ConnectionAcceptCounter.Inc() - log.Printf(utils.WrapInfo("acceptor ready to send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("acceptor ready to send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) rs.dp.input() <- conn - log.Printf(utils.WrapInfo("acceptor success send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) + log.Print(utils.WrapInfo("acceptor success send connection, remote addr: %v, local addr: %v, fd: %v", conn.RemoteAddr(), conn.LocalAddr(), conn.RawFd())) } } From 0f67d63c7b316aa2626a23d6f4c8f44db7813a8e Mon Sep 17 00:00:00 2001 From: Trino Date: Sat, 29 Jun 2024 00:57:48 +0800 Subject: [PATCH 14/15] chore(server): change test package Signed-off-by: Trino --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 02dd578..9dcac48 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v2 - name: Run tests - run: make -f makefile test-with-cover TestPackage=./storage/... + run: make -f makefile test-with-cover TestPackage=./storage/server - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.0.1 From 6945c83737aaa54e3461c775a09ecb0b55b198c1 Mon Sep 17 00:00:00 2001 From: Trino Date: Sat, 29 Jun 2024 01:07:55 +0800 Subject: [PATCH 15/15] chore(server): wrap errors Signed-off-by: Trino --- storage/server/reactor_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/server/reactor_server.go b/storage/server/reactor_server.go index b995c47..e750e4b 100644 --- a/storage/server/reactor_server.go +++ b/storage/server/reactor_server.go @@ -89,7 +89,7 @@ func (r *reactor) run() { if e != nil { err = errors.Wrap(err, e.Error()) } - log.Println(err, changes, conn, conn.RemoteAddr(), conn.LocalAddr()) + log.Println(utils.WrapError("%s %v %v %v %v", err, changes, conn, conn.RemoteAddr(), conn.LocalAddr())) continue } r.processors.Store(conn.RawFd(), processor)