Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Encode documents using MessagePack #117

Merged
merged 6 commits into from
Jul 18, 2020
Merged

Encode documents using MessagePack #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0a19ad3
Add msgpack package
asdine Jul 10, 2020
db4f12e
Remove legacy document encoding
asdine Jul 12, 2020
1ac12d2
Use msgpack.EncodedDocument everywhere
asdine Jul 12, 2020
e36edac
Remove unused struct field
asdine Jul 12, 2020
5cfa278
Apply suggestions from code review
asdine Jul 16, 2020
e97f3a1
Add comment about duration types
asdine Jul 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions cmd/genji/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/genjidb/genji v0.6.0 h1:Ksz9f6FRciZfkocfPAbD8ZSh1GKkuQPLGeGEeH9l6Es=
github.com/genjidb/genji v0.6.0/go.mod h1:bwsAzPOtKRos+Vblm0pPswIYRmc8aFlCzQlG/n5Fxiw=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk=
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
Expand Down Expand Up @@ -86,13 +86,21 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1 h1:d71/KA0LhvkrJ/Ok+Wx9qK7bU8meKA1Hk0jpVI5kJjk=
github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI=
github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY=
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand All @@ -105,7 +113,12 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
Expand Down
12 changes: 6 additions & 6 deletions database/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"time"

"github.com/genjidb/genji/document"
"github.com/genjidb/genji/document/encoding"
"github.com/genjidb/genji/document/encoding/msgpack"
"github.com/genjidb/genji/engine"
"github.com/genjidb/genji/index"
)
Expand Down Expand Up @@ -173,7 +173,7 @@ func (t *tableInfoStore) Insert(tableName string, info *TableInfo) ([]byte, erro
}
info.storeID = id

v, err := encoding.EncodeDocument(info.ToDocument())
v, err := msgpack.EncodeDocument(info.ToDocument())
if err != nil {
return nil, err
}
Expand All @@ -196,7 +196,7 @@ func (t *tableInfoStore) Get(tableName string) (*TableInfo, error) {
}

var ti TableInfo
err = ti.ScanDocument(encoding.EncodedDocument(v))
err = ti.ScanDocument(msgpack.EncodedDocument(v))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -318,7 +318,7 @@ func (t *indexStore) Insert(cfg IndexConfig) error {
return err
}

v, err := encoding.EncodeDocument(cfg.ToDocument())
v, err := msgpack.EncodeDocument(cfg.ToDocument())
if err != nil {
return err
}
Expand All @@ -337,7 +337,7 @@ func (t *indexStore) Get(indexName string) (*IndexConfig, error) {
}

var idxopts IndexConfig
err = idxopts.ScanDocument(encoding.EncodedDocument(v))
err = idxopts.ScanDocument(msgpack.EncodedDocument(v))
if err != nil {
return nil, err
}
Expand All @@ -358,7 +358,7 @@ func (t *indexStore) ListAll() ([]*IndexConfig, error) {
var idxList []*IndexConfig
it := t.st.NewIterator(engine.IteratorConfig{})

var buf encoding.EncodedDocument
var buf msgpack.EncodedDocument
var err error
for it.Seek(nil); it.Valid(); it.Next() {
item := it.Item()
Expand Down
9 changes: 5 additions & 4 deletions database/table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/genjidb/genji/document"
"github.com/genjidb/genji/document/encoding"
"github.com/genjidb/genji/document/encoding/msgpack"
"github.com/genjidb/genji/engine"
"github.com/genjidb/genji/index"
)
Expand All @@ -32,7 +33,7 @@ func (t *Table) Info() (*TableInfo, error) {
}

type encodedDocumentWithKey struct {
encoding.EncodedDocument
msgpack.EncodedDocument

key []byte
}
Expand Down Expand Up @@ -85,7 +86,7 @@ func (t *Table) GetDocument(key []byte) (document.Document, error) {
}

var d encodedDocumentWithKey
d.EncodedDocument = encoding.EncodedDocument(v)
d.EncodedDocument = msgpack.EncodedDocument(v)
d.key = key
return &d, err
}
Expand Down Expand Up @@ -357,7 +358,7 @@ func (t *Table) Insert(d document.Document) ([]byte, error) {
return nil, ErrDuplicateDocument
}

v, err := encoding.EncodeDocument(d)
v, err := msgpack.EncodeDocument(d)
if err != nil {
return nil, fmt.Errorf("failed to encode document: %w", err)
}
Expand Down Expand Up @@ -457,7 +458,7 @@ func (t *Table) replace(indexes map[string]Index, key []byte, d document.Documen
}

// encode new document
v, err := encoding.EncodeDocument(d)
v, err := msgpack.EncodeDocument(d)
if err != nil {
return fmt.Errorf("failed to encode document: %w", err)
}
Expand Down
3 changes: 2 additions & 1 deletion database/table_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/genjidb/genji/database"
"github.com/genjidb/genji/document"
"github.com/genjidb/genji/document/encoding"
"github.com/genjidb/genji/document/encoding/msgpack"
"github.com/genjidb/genji/engine/memoryengine"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -197,7 +198,7 @@ func TestTableInsert(t *testing.T) {

manualInsert := func(id int64) {
docid := encoding.EncodeInt64(id)
v, err := encoding.EncodeDocument(newDocument())
v, err := msgpack.EncodeDocument(newDocument())
require.NoError(t, err)
err = tb.Store.Put(docid, v)
require.NoError(t, err)
Expand Down
4 changes: 2 additions & 2 deletions database/transaction.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"fmt"

"github.com/genjidb/genji/document"
"github.com/genjidb/genji/document/encoding"
"github.com/genjidb/genji/document/encoding/msgpack"
"github.com/genjidb/genji/engine"
"github.com/genjidb/genji/index"
)
Expand Down Expand Up @@ -107,7 +107,7 @@ func (tx Transaction) GetTable(name string) (*Table, error) {
func (tx Transaction) DropTable(name string) error {
it := tx.indexStore.st.NewIterator(engine.IteratorConfig{})

var buf encoding.EncodedDocument
var buf msgpack.EncodedDocument
var err error
for it.Seek(nil); it.Valid(); it.Next() {
item := it.Item()
Expand Down
6 changes: 3 additions & 3 deletions document/document_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"testing"

"github.com/genjidb/genji/document"
"github.com/genjidb/genji/document/encoding"
"github.com/genjidb/genji/document/encoding/msgpack"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -587,14 +587,14 @@ func BenchmarkDocumentIterate(b *testing.B) {

b.Run("Encoding/Implementation", func(b *testing.B) {
for i := 0; i < b.N; i++ {
encoding.EncodeDocument(&f)
msgpack.EncodeDocument(&f)
}
})

b.Run("Encoding/Reflection", func(b *testing.B) {
for i := 0; i < b.N; i++ {
refd, _ := document.NewFromStruct(&f)
encoding.EncodeDocument(refd)
msgpack.EncodeDocument(refd)
}
})

Expand Down
Loading