Skip to content

Commit

Permalink
paginated range query (#135)
Browse files Browse the repository at this point in the history
This commit adds a range query API to the data query executor.1

Signed-off-by: senthil <[email protected]>
  • Loading branch information
cendhu authored Jun 28, 2022
1 parent ebd3789 commit 31c863f
Show file tree
Hide file tree
Showing 13 changed files with 794 additions and 399 deletions.
18 changes: 9 additions & 9 deletions examples/api/json_query/json_query.go
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ func insertData(session bcdb.DBSession) error {
}

func validQueries(session bcdb.DBSession) error {
q, err := session.JSONQuery()
q, err := session.Query()
if err != nil {
fmt.Printf("Failed to return handler to access bcdb data through JSON query, reason: %s\n", err.Error())
return err
Expand All @@ -243,7 +243,7 @@ func validQueries(session bcdb.DBSession) error {
}
}
`
kvs, err := q.Execute("db", query1)
kvs, err := q.ExecuteJSONQuery("db", query1)
if err != nil {
fmt.Printf("Failed to execute JSON query, reason: %s\n", err.Error())
return err
Expand Down Expand Up @@ -274,7 +274,7 @@ func validQueries(session bcdb.DBSession) error {
}
}
`
kvs, err = q.Execute("db", query2)
kvs, err = q.ExecuteJSONQuery("db", query2)
if err != nil {
fmt.Printf("Failed to execute JSON query, reason: %s\n", err.Error())
return err
Expand Down Expand Up @@ -309,7 +309,7 @@ func validQueries(session bcdb.DBSession) error {
}
}
`
kvs, err = q.Execute("db", query3)
kvs, err = q.ExecuteJSONQuery("db", query3)
if err != nil {
fmt.Printf("Failed to execute JSON query, reason: %s\n", err.Error())
return err
Expand Down Expand Up @@ -339,7 +339,7 @@ func validQueries(session bcdb.DBSession) error {
}
}
`
kvs, err = q.Execute("db", query4)
kvs, err = q.ExecuteJSONQuery("db", query4)
if err != nil {
fmt.Printf("Failed to execute JSON query, reason: %s\n", err.Error())
return err
Expand Down Expand Up @@ -370,7 +370,7 @@ func validQueries(session bcdb.DBSession) error {
}
}
`
kvs, err = q.Execute("db", query5)
kvs, err = q.ExecuteJSONQuery("db", query5)
if err != nil {
fmt.Printf("Failed to execute JSON query, reason: %s\n", err.Error())
return err
Expand Down Expand Up @@ -402,7 +402,7 @@ func validQueries(session bcdb.DBSession) error {
}
}
`
kvs, err = q.Execute("db", query6)
kvs, err = q.ExecuteJSONQuery("db", query6)
if err != nil {
fmt.Printf("Failed to execute JSON query, reason: %s\n", err.Error())
return err
Expand All @@ -425,7 +425,7 @@ func validQueries(session bcdb.DBSession) error {
}

func invalidQuery(session bcdb.DBSession) error {
q, err := session.JSONQuery()
q, err := session.Query()
if err != nil {
fmt.Printf("Failed to return handler to access bcdb data through JSON query, reason: %s\n", err.Error())
return err
Expand All @@ -442,7 +442,7 @@ func invalidQuery(session bcdb.DBSession) error {
}
}
`
_, err = q.Execute("db", query)
_, err = q.ExecuteJSONQuery("db", query)
if err != nil {
fmt.Printf("As expected, failed to execute JSON query, reason: %s\n", err.Error())
} else {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/golang/protobuf v1.5.2
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/hyperledger-labs/orion-server v0.2.4-0.20220531080909-badd4d7bb1f5
github.com/hyperledger-labs/orion-server v0.2.4-0.20220621134147-6a9aeaf38f9a
github.com/kr/text v0.2.0 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/pkg/errors v0.9.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,8 @@ github.com/hyperledger-labs/orion-server v0.2.3 h1:vY9IQRO2sHdWUwLNKz6J7x+kZJXYL
github.com/hyperledger-labs/orion-server v0.2.3/go.mod h1:8kXVAU1wvFYGbFL1qmXwMi2i8gKV2smOdp1F1kq0HMk=
github.com/hyperledger-labs/orion-server v0.2.4-0.20220531080909-badd4d7bb1f5 h1:qy4M67EISCrlmCmH1ePY0PPVXDoUKHL9Ir9pGQzZULg=
github.com/hyperledger-labs/orion-server v0.2.4-0.20220531080909-badd4d7bb1f5/go.mod h1:8kXVAU1wvFYGbFL1qmXwMi2i8gKV2smOdp1F1kq0HMk=
github.com/hyperledger-labs/orion-server v0.2.4-0.20220621134147-6a9aeaf38f9a h1:2gWosqOFXOyB+Daox6chgWrExTEqJ4O0UwogZjkQ3Mo=
github.com/hyperledger-labs/orion-server v0.2.4-0.20220621134147-6a9aeaf38f9a/go.mod h1:8kXVAU1wvFYGbFL1qmXwMi2i8gKV2smOdp1F1kq0HMk=
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
Expand Down
130 changes: 130 additions & 0 deletions pkg/bcdb/data_query.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
// Copyright IBM Corp. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package bcdb

import (
"encoding/json"

"github.com/hyperledger-labs/orion-server/pkg/constants"
"github.com/hyperledger-labs/orion-server/pkg/types"
"github.com/pkg/errors"
)

type QueryExecutor struct {
*commonTxContext
}

func (q *QueryExecutor) ExecuteJSONQuery(dbName, query string) ([]*types.KVWithMetadata, error) {
marshaledJSONQuery, err := json.Marshal(query)
if err != nil {
return nil, errors.WithMessage(err, "check whether the query string passed is in JSON format")
}
path := constants.URLForJSONQuery(dbName)
resEnv := &types.DataQueryResponseEnvelope{}
if err = q.handleRequestWithPost(
path,
marshaledJSONQuery,
&types.DataJSONQuery{
UserId: q.userID,
DbName: dbName,
Query: query,
},
resEnv,
); err != nil {
q.logger.Errorf("failed to execute ledger block query %s, due to %s", path, err)
return nil, err
}

return resEnv.GetResponse().KVs, nil
}

func (q *QueryExecutor) GetDataByRange(dbName, startKey, endKey string, limit uint64) (Iterator, error) {
kvs, pendingResult, nextStartKey, err := q.getDataByRange(dbName, startKey, endKey, limit)
if err != nil {
return nil, err
}

return &RangeQueryIterator{
kvs: kvs,
currentLoc: 0,
pendingResult: pendingResult,
dbName: dbName,
nextStartKey: nextStartKey,
endKey: endKey,
limit: limit,
limitReached: false,
q: q,
}, nil
}

func (q *QueryExecutor) getDataByRange(dbName, startKey, endKey string, limit uint64) ([]*types.KVWithMetadata, bool, string, error) {
path := constants.URLForGetDataRange(dbName, startKey, endKey, limit)
resEnv := &types.GetDataRangeResponseEnvelope{}
if err := q.handleRequest(
path,
&types.GetDataRangeQuery{
UserId: q.userID,
DbName: dbName,
StartKey: startKey,
EndKey: endKey,
Limit: limit,
},
resEnv,
); err != nil {
q.logger.Errorf("failed to execute range query %s, due to %s", path, err)
return nil, false, "", err
}

return resEnv.GetResponse().KVs, resEnv.GetResponse().PendingResult, resEnv.GetResponse().NextStartKey, nil
}

type RangeQueryIterator struct {
kvs []*types.KVWithMetadata
currentLoc int
pendingResult bool
dbName string
nextStartKey string
endKey string
limit uint64
limitReached bool
q *QueryExecutor
}

func (i *RangeQueryIterator) Next() (*types.KVWithMetadata, bool, error) {
if i.currentLoc < len(i.kvs) {
return i.fetchNextAndAdjustReaminingResultCount()
}

if !i.pendingResult || i.limitReached {
return nil, false, nil
}

kvs, pending, next, err := i.q.getDataByRange(i.dbName, i.nextStartKey, i.endKey, i.limit)
if err != nil {
return nil, false, err
}
if len(kvs) == 0 {
return nil, false, nil
}

i.kvs = kvs
i.pendingResult = pending
i.nextStartKey = next
i.currentLoc = 0

return i.fetchNextAndAdjustReaminingResultCount()
}

func (i *RangeQueryIterator) fetchNextAndAdjustReaminingResultCount() (*types.KVWithMetadata, bool, error) {
kv := i.kvs[i.currentLoc]
i.currentLoc++
if i.limit > 0 {
i.limit--
if i.limit == 0 {
i.limitReached = true
}
}

return kv, true, nil
}
Loading

0 comments on commit 31c863f

Please sign in to comment.