Skip to content

Commit

Permalink
refactor: Move public members out of core and base packages (sourcene…
Browse files Browse the repository at this point in the history 
…twork#295)

* Use exisiting pub func instead of private

* Rename maker file

* Move key utils from descriptions file to col_keys file

* Remove unused variables

* Remove unused byteToType var

* Remove comment out code

* Move CType into client

Is used by all the client packages

* Move store namespaces into datastore package

* Delete simple document

* Move document and descriptions into client package

* Move encoded doc into fetcher

And make private

* Remove unused value types

* Give type to Meta field

* Make key utils take value instead of pointer

PR request
  • Loading branch information
@AndrewSisley
AndrewSisley authored Mar 17, 2022
1 parent 4fbcc4b commit 2f72d88
Show file tree
Hide file tree
Showing 55 changed files with 868 additions and 1,111 deletions.
10 changes: 4 additions & 6 deletions bench/bench_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ import (
"github.com/sourcenetwork/defradb/bench/fixtures"
"github.com/sourcenetwork/defradb/client"
testutils "github.com/sourcenetwork/defradb/db/tests"
"github.com/sourcenetwork/defradb/document"
"github.com/sourcenetwork/defradb/document/key"
"github.com/sourcenetwork/defradb/logging"
)

Expand Down Expand Up @@ -128,15 +126,15 @@ func SetupDBAndCollections(b *testing.B, ctx context.Context, fixture fixtures.G
// Loads the given test database using the provided fixture context.
// It loads docCount number of documents asynchronously in batches of *up to*
// writeBatchGroup.
func BackfillBenchmarkDB(b *testing.B, ctx context.Context, cols []client.Collection, fixture fixtures.Generator, docCount, opCount int, doSync bool) ([][]key.DocKey, error) {
func BackfillBenchmarkDB(b *testing.B, ctx context.Context, cols []client.Collection, fixture fixtures.Generator, docCount, opCount int, doSync bool) ([][]client.DocKey, error) {
numTypes := len(fixture.Types())

// load fixtures
var wg sync.WaitGroup
wg.Add(docCount)
errCh := make(chan error)
waitCh := make(chan struct{})
dockeys := make([][]key.DocKey, docCount)
dockeys := make([][]client.DocKey, docCount)

go func() {
// Cut up the job from into writeBatchGroup size grouped jobs.
Expand All @@ -159,10 +157,10 @@ func BackfillBenchmarkDB(b *testing.B, ctx context.Context, cols []client.Collec
}

// create the documents
keys := make([]key.DocKey, numTypes)
keys := make([]client.DocKey, numTypes)
for j := 0; j < numTypes; j++ {

doc, err := document.NewFromJSON([]byte(docs[j]))
doc, err := client.NewDocFromJSON([]byte(docs[j]))
if err != nil {
errCh <- fmt.Errorf("Failed to create document from fixture: %w", err)
return
Expand Down
16 changes: 7 additions & 9 deletions bench/collection/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ import (
benchutils "github.com/sourcenetwork/defradb/bench"
"github.com/sourcenetwork/defradb/bench/fixtures"
"github.com/sourcenetwork/defradb/client"
"github.com/sourcenetwork/defradb/document"
"github.com/sourcenetwork/defradb/document/key"
)

const (
Expand Down Expand Up @@ -52,7 +50,7 @@ func runCollectionBenchGetSync(b *testing.B,
collections []client.Collection,
fixture fixtures.Generator,
docCount, opCount int,
dockeys [][]key.DocKey,
dockeys [][]client.DocKey,
) error {
numTypes := len(fixture.Types())
b.ResetTimer()
Expand All @@ -75,7 +73,7 @@ func runCollectionBenchGetAsync(b *testing.B,
collections []client.Collection,
fixture fixtures.Generator,
docCount, opCount int,
dockeys [][]key.DocKey,
dockeys [][]client.DocKey,
) error {
var wg sync.WaitGroup
numTypes := len(fixture.Types())
Expand All @@ -84,7 +82,7 @@ func runCollectionBenchGetAsync(b *testing.B,
for j := 0; j < opCount/numTypes; j++ { // number of Get operations we want to execute
for k := 0; k < numTypes; k++ { // apply op to all the related types
wg.Add(1)
go func(ctx context.Context, col client.Collection, dockey key.DocKey) {
go func(ctx context.Context, col client.Collection, dockey client.DocKey) {
col.Get(ctx, dockey) //nolint
wg.Done()
}(ctx, collections[k], dockeys[j][k])
Expand Down Expand Up @@ -142,10 +140,10 @@ func runCollectionBenchCreateMany(b *testing.B, ctx context.Context, fixture fix

b.ResetTimer()
for i := 0; i < b.N; i++ {
docs := make([]*document.Document, opCount)
docs := make([]*client.Document, opCount)
for j := 0; j < opCount; j++ {
d, _ := fixture.GenerateDocs()
docs[j], _ = document.NewFromJSON([]byte(d[0]))
docs[j], _ = client.NewDocFromJSON([]byte(d[0]))
}

collections[0].CreateMany(ctx, docs) //nolint
Expand All @@ -168,7 +166,7 @@ func runCollectionBenchCreateSync(b *testing.B,
for j := 0; j < runs; j++ {
docs, _ := fixture.GenerateDocs()
for k := 0; k < numTypes; k++ {
doc, _ := document.NewFromJSON([]byte(docs[k]))
doc, _ := client.NewDocFromJSON([]byte(docs[k]))
collections[k].Create(ctx, doc) //nolint
}
}
Expand Down Expand Up @@ -205,7 +203,7 @@ func runCollectionBenchCreateAsync(b *testing.B,
docs, _ := fixture.GenerateDocs()
// create the documents
for j := 0; j < numTypes; j++ {
doc, _ := document.NewFromJSON([]byte(docs[j]))
doc, _ := client.NewDocFromJSON([]byte(docs[j]))
collections[j].Create(ctx, doc) //nolint
}

Expand Down
5 changes: 2 additions & 3 deletions bench/query/simple/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
benchutils "github.com/sourcenetwork/defradb/bench"
"github.com/sourcenetwork/defradb/bench/fixtures"
"github.com/sourcenetwork/defradb/client"
"github.com/sourcenetwork/defradb/document/key"
)

var (
Expand All @@ -47,7 +46,7 @@ func runQueryBenchGetSync(
ctx context.Context,
db client.DB,
docCount int,
dockeys [][]key.DocKey,
dockeys [][]client.DocKey,
query string,
) error {
// run any preprocessing on the query before execution (mostly just dockey insertion if needed)
Expand All @@ -72,7 +71,7 @@ func runQueryBenchGetSync(
return nil
}

func formatQuery(b *testing.B, query string, dockeys [][]key.DocKey) string {
func formatQuery(b *testing.B, query string, dockeys [][]client.DocKey) string {
numPlaceholders := strings.Count(query, "{{dockey}}")
if numPlaceholders == 0 {
return query
Expand Down
41 changes: 19 additions & 22 deletions client/core.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,6 @@ import (
"context"

"github.com/sourcenetwork/defradb/datastore"
"github.com/sourcenetwork/defradb/db/base"
"github.com/sourcenetwork/defradb/document"
"github.com/sourcenetwork/defradb/document/key"

ds "github.com/ipfs/go-datastore"
blockstore "github.com/ipfs/go-ipfs-blockstore"
Expand All @@ -25,7 +22,7 @@ import (
type DB interface {
AddSchema(context.Context, string) error

CreateCollection(context.Context, base.CollectionDescription) (Collection, error)
CreateCollection(context.Context, CollectionDescription) (Collection, error)
GetCollectionByName(context.Context, string) (Collection, error)
GetCollectionBySchemaID(context.Context, string) (Collection, error)
GetAllCollections(ctx context.Context) ([]Collection, error)
Expand All @@ -43,43 +40,43 @@ type DB interface {
}

type Collection interface {
Description() base.CollectionDescription
Description() CollectionDescription
Name() string
Schema() base.SchemaDescription
Schema() SchemaDescription
ID() uint32
SchemaID() string

Indexes() []base.IndexDescription
PrimaryIndex() base.IndexDescription
Index(uint32) (base.IndexDescription, error)
CreateIndex(base.IndexDescription) error
Indexes() []IndexDescription
PrimaryIndex() IndexDescription
Index(uint32) (IndexDescription, error)
CreateIndex(IndexDescription) error

Create(context.Context, *document.Document) error
CreateMany(context.Context, []*document.Document) error
Update(context.Context, *document.Document) error
Save(context.Context, *document.Document) error
Delete(context.Context, key.DocKey) (bool, error)
Exists(context.Context, key.DocKey) (bool, error)
Create(context.Context, *Document) error
CreateMany(context.Context, []*Document) error
Update(context.Context, *Document) error
Save(context.Context, *Document) error
Delete(context.Context, DocKey) (bool, error)
Exists(context.Context, DocKey) (bool, error)

UpdateWith(context.Context, interface{}, interface{}, ...UpdateOpt) error
UpdateWithFilter(context.Context, interface{}, interface{}, ...UpdateOpt) (*UpdateResult, error)
UpdateWithKey(context.Context, key.DocKey, interface{}, ...UpdateOpt) (*UpdateResult, error)
UpdateWithKeys(context.Context, []key.DocKey, interface{}, ...UpdateOpt) (*UpdateResult, error)
UpdateWithKey(context.Context, DocKey, interface{}, ...UpdateOpt) (*UpdateResult, error)
UpdateWithKeys(context.Context, []DocKey, interface{}, ...UpdateOpt) (*UpdateResult, error)

DeleteWith(context.Context, interface{}, ...DeleteOpt) error
DeleteWithFilter(context.Context, interface{}, ...DeleteOpt) (*DeleteResult, error)
DeleteWithKey(context.Context, key.DocKey, ...DeleteOpt) (*DeleteResult, error)
DeleteWithKeys(context.Context, []key.DocKey, ...DeleteOpt) (*DeleteResult, error)
DeleteWithKey(context.Context, DocKey, ...DeleteOpt) (*DeleteResult, error)
DeleteWithKeys(context.Context, []DocKey, ...DeleteOpt) (*DeleteResult, error)

Get(context.Context, key.DocKey) (*document.Document, error)
Get(context.Context, DocKey) (*Document, error)

WithTxn(datastore.Txn) Collection

GetAllDocKeys(ctx context.Context) (<-chan DocKeysResult, error)
}

type DocKeysResult struct {
Key key.DocKey
Key DocKey
Err error
}

Expand Down
20 changes: 9 additions & 11 deletions db/base/keys.go → client/ctype.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,15 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.

package base
package client

import (
ds "github.com/ipfs/go-datastore"
)
// CType indicates CRDT type
type CType byte

var (
// Individual Store Keys
RootStoreKey = ds.NewKey("/db")
SystemStoreKey = RootStoreKey.ChildString("/system")
DataStoreKey = RootStoreKey.ChildString("/data")
HeadStoreKey = RootStoreKey.ChildString("/heads")
BlockStoreKey = RootStoreKey.ChildString("/blocks")
const (
//no lint
NONE_CRDT = CType(iota) // reserved none type
LWW_REGISTER
OBJECT
COMPOSITE
)
Loading

0 comments on commit 2f72d88

Please sign in to comment.